truf-server/docker/verify_packaged_workers.py

#!/usr/bin/env python3
"""Exercise the real packaged Windows and Linux remote workers end to end.

Docker is intentionally reached only through the configured WSL distribution.
The verifier creates individually named and labelled resources, never invokes
Compose or a broad cleanup command, and retains all owned artifacts on failure.
"""

import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import socket
import ssl
import stat
import subprocess
import sys
import threading
import time


LINUX_IMAGE = 'truf-remote-worker:linux-x86_64'
TEST_IMAGE = 'truf-worker-test:test'
WINDOWS_ARTIFACT = 'dist/truf-worker-windows-x86_64'
TARGETS = (
    'https://gitlab.com/truf-e2e/a.git',
    'https://gitlab.com/truf-e2e/b.git',
)
DOCKER_TARGET = 'docker.io/truf-e2e/synthetic@sha256:' + ('7' * 64)
HUGGINGFACE_TARGET = 'truf-e2e/synthetic'
DIRECT_TARGETS = {
    'dockerhub': DOCKER_TARGET,
    'huggingface': HUGGINGFACE_TARGET,
}
ALL_TARGETS = (*TARGETS, DOCKER_TARGET, HUGGINGFACE_TARGET)
PARALLELISM = 2
MAX_OUTPUT = 2 * 1024 * 1024
MAX_DOCKER_RESOURCES = 1024
MAX_SNAPSHOT_BYTES = 8 * 1024 * 1024
MAX_SAFE_EVIDENCE = 64 * 1024
RUN_LABEL = 'com.truf.packaged-worker-e2e.run'
PHASE_LABEL = 'com.truf.packaged-worker-e2e.phase'
KIND_LABEL = 'com.truf.packaged-worker-e2e.kind'
RESOURCE_NAME = re.compile(r'truf-packaged-worker-e2e-[a-f0-9]{16}-(windows|linux)-[a-z-]+')
HEX_40 = re.compile(r'[a-f0-9]{40}')
HEX_64 = re.compile(r'[a-f0-9]{64}')

CONTAINER_METADATA_FORMAT = (
    '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Image}},'
    '"status":{{json .State.Status}},"running":{{json .State.Running}},'
    '"paused":{{json .State.Paused}},"restarting":{{json .State.Restarting}},'
    '"dead":{{json .State.Dead}},"mounts":{{json .Mounts}}}'
)
VOLUME_METADATA_FORMAT = (
    '{"name":{{json .Name}},"driver":{{json .Driver}},"scope":{{json .Scope}},'
    '"created":{{json .CreatedAt}},"mountpoint":{{json .Mountpoint}},'
    '"labels":{{json .Labels}},"options":{{json .Options}}}'
)


def isolated_test_subnet(run_id, scope):
    value = int.from_bytes(
        hashlib.sha256(f'{run_id}\0{scope}'.encode('ascii')).digest()[:2], 'big',
    ) & 0x1fff
    return f'198.{18 + (value >> 12)}.{(value >> 4) & 0xff}.{(value & 0xf) * 16}/28'


LOOPBACK_PROXY_SCRIPT = r'''import socket
import sys
import threading

listen_port = int(sys.argv[1])
upstream_host = sys.argv[2]
upstream_port = int(sys.argv[3])


def pump(source, destination):
    try:
        while True:
            block = source.recv(64 * 1024)
            if not block:
                break
            destination.sendall(block)
    except OSError:
        pass
    try:
        destination.shutdown(socket.SHUT_WR)
    except OSError:
        pass


def bridge(client):
    upstream = None
    try:
        upstream = socket.create_connection((upstream_host, upstream_port), timeout=5)
        print('proxy upstream connected', flush=True)
        client.settimeout(None)
        upstream.settimeout(None)
        outbound = threading.Thread(target=pump, args=(client, upstream), daemon=True)
        outbound.start()
        pump(upstream, client)
        outbound.join(5)
    except OSError as exc:
        print('proxy upstream failed: ' + type(exc).__name__, flush=True)
    finally:
        client.close()
        if upstream is not None:
            upstream.close()


listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(('127.0.0.1', listen_port))
listener.listen(32)
print('proxy ready', flush=True)
while True:
    connection, _ = listener.accept()
    threading.Thread(target=bridge, args=(connection,), daemon=True).start()
'''


SEED_SCRIPT = r'''import os
import stat

root = '/seed'
for current, directories, files in os.walk(root, topdown=False, followlinks=False):
    for name in directories + files:
        path = os.path.join(current, name)
        details = os.lstat(path)
        if stat.S_ISLNK(details.st_mode) or not (
            stat.S_ISDIR(details.st_mode) or stat.S_ISREG(details.st_mode)
        ):
            raise SystemExit('unsupported seed entry')
        os.chown(path, 10001, 10001, follow_symlinks=False)
        os.chmod(path, 0o700 if stat.S_ISDIR(details.st_mode) else 0o600)
os.chown(root, 10001, 10001, follow_symlinks=False)
os.chmod(root, 0o700)
os.unlink('/seed/_seed.py')
'''


INSPECT_WORKER_SCRIPT = r'''import hashlib
import json
import os
from pathlib import Path
import stat


def digest(path):
    value = hashlib.sha256()
    with open(path, 'rb', buffering=0) as handle:
        for block in iter(lambda: handle.read(1024 * 1024), b''):
            value.update(block)
    return value.hexdigest()


def regular(path):
    details = os.stat(path, follow_symlinks=False)
    if not stat.S_ISREG(details.st_mode) or os.path.islink(path):
        raise RuntimeError('non-regular worker artifact')
    return details


def tree_identity(root):
    identity = hashlib.sha256()
    count = 0
    active_entries = 0
    root = Path(root)
    if not root.exists():
        return {'count': 0, 'active_entries': 0, 'sha256': identity.hexdigest()}
    for current, directories, files in os.walk(root, followlinks=False):
        directories.sort()
        files.sort()
        for name in directories:
            path = Path(current) / name
            if path.is_symlink() or not path.is_dir():
                raise RuntimeError('unsupported worker directory')
            if path.relative_to(root).parts[0] != 'abandoned':
                active_entries += 1
        for name in files:
            path = Path(current) / name
            details = regular(path)
            if path.relative_to(root).parts[0] != 'abandoned':
                active_entries += 1
            relative = path.relative_to(root).as_posix().encode('utf-8')
            identity.update(relative + b'\0')
            identity.update(str(details.st_size).encode('ascii') + b'\0')
            identity.update(str(details.st_mtime_ns).encode('ascii') + b'\0')
            identity.update(digest(path).encode('ascii') + b'\n')
            count += 1
    return {
        'count': count, 'active_entries': active_entries,
        'sha256': identity.hexdigest(),
    }


states = {}
state_root = Path('/data/state-base/truf/remote-worker')
if state_root.exists():
    for path in sorted(state_root.glob('slot-*.json')):
        details = regular(path)
        with open(path, 'rb') as handle:
            value = json.load(handle)
        reservation = dict((value.get('assignment') or {}).get('reservation') or {})
        assignment = dict(value.get('assignment') or {})
        snapshot = dict(assignment.get('execution_snapshot') or {})
        planning = dict(snapshot.get('planning') or {})
        states[path.name] = {
            'phase': value.get('phase'),
            'reservation_id': reservation.get('reservation_id'),
            'bundle_id': reservation.get('bundle_id'),
            'scan_event_id': reservation.get('scan_event_id'),
            'source': reservation.get('source'),
            'platform': reservation.get('platform'),
            'target': reservation.get('target'),
            'planning_kind': planning.get('kind'),
            'sha256': digest(path),
            'size': details.st_size,
            'mtime_ns': details.st_mtime_ns,
        }

bundles = {}
bundle_root = Path('/data/client/truf/remote-worker/bundles')
if bundle_root.exists():
    for path in sorted(bundle_root.rglob('*.trb')):
        details = regular(path)
        bundles[path.relative_to(bundle_root).as_posix()] = {
            'sha256': digest(path),
            'size': details.st_size,
            'mtime_ns': details.st_mtime_ns,
        }

shutdown_receipt = None
receipt_path = state_root / 'control' / 'worker.exit.json'
if receipt_path.exists():
    regular(receipt_path)
    with open(receipt_path, 'rb') as handle:
        shutdown_receipt = json.load(handle)

print(json.dumps({
    'states': states,
    'bundles': bundles,
    'work': tree_identity('/data/client/truf/remote-worker/work'),
    'shutdown_receipt': shutdown_receipt,
}, ensure_ascii=True, sort_keys=True, separators=(',', ':')))
'''

PUBLISH_DIRECT_SCRIPT = r'''import json
import os
from datetime import datetime, timezone
from pathlib import Path
import stat
import sys


def fail(message):
    raise RuntimeError('direct bundle publisher: ' + message)


def require(condition, message):
    if not condition:
        fail(message)


require(len(sys.argv) == 4, 'arguments')
package_root = Path(sys.argv[1]).resolve(strict=True)
state_root = Path(sys.argv[2]).resolve(strict=True)
bundle_root = Path(sys.argv[3]).resolve(strict=True)
manifest_path = package_root / 'worker-package.json'
with open(manifest_path, 'rb') as handle:
    bootstrap_manifest = json.load(handle)
app_root = package_root / str(bootstrap_manifest.get('app_root') or '')
require(app_root.is_dir(), 'application root')
sys.path.insert(0, os.fspath(app_root / 'dependencies'))
sys.path.insert(0, os.fspath(app_root))

from scan_execution import (  # noqa: E402
    QueueDispositionPolicy,
    stage_scan_result_in_scope,
    validate_protocol2_remote_assignment,
)
from worker_package import (  # noqa: E402
    load_worker_package_manifest,
    worker_package_build_compatibility,
)

manifest = load_worker_package_manifest(manifest_path)
local_build = worker_package_build_compatibility(manifest)
capabilities = tuple(
    (item['source'], item['platform'], item['planning_kind'])
    for item in manifest['capabilities']
)
expected = {
    ('dockerhub', 'docker', 'docker_direct_v1'),
    ('huggingface', 'huggingface', 'huggingface_space_v1'),
}
state_paths = sorted(state_root.glob('slot-*.json'))
require({path.name for path in state_paths} == {'slot-0.json', 'slot-1.json'}, 'slot states')
require(not any(bundle_root.rglob('*.trb')), 'bundle storage is not empty')
seen = set()
for path in state_paths:
    details = path.stat(follow_symlinks=False)
    require(stat.S_ISREG(details.st_mode) and not path.is_symlink(), 'regular slot state')
    require(0 < details.st_size <= 64 * 1024 * 1024, 'slot state bound')
    with open(path, 'rb') as handle:
        state = json.load(handle)
    require(state.get('phase') == 'assigned', 'slot phase')
    assignment = dict(state.get('assignment') or {})
    validated = validate_protocol2_remote_assignment(
        assignment, local_build, capabilities,
    )
    reservation = validated['reservation']
    identity = (
        reservation.source, reservation.platform, validated['planning_kind'],
    )
    require(identity in expected and identity not in seen, 'direct capability')
    require(
        validated['snapshot']['credential_ref'] == {
            'source': reservation.source, 'auth_entry': '',
        },
        'direct credential reference',
    )
    now = datetime.now(timezone.utc).isoformat()
    result = {
        'target': reservation.target,
        'scan_type': reservation.platform,
        'scan_event_id': reservation.scan_event_id,
        'findings': [],
        'errors': [],
        'warnings': [],
        'scan_started_at': now,
        'timestamp': now,
        'duration_sec': 0.0,
        'scan_meta': {'synthetic_claim_to_ingestion': True},
    }
    limits = dict(assignment['limits'])
    staged = stage_scan_result_in_scope(
        result,
        reservation,
        os.fspath(bundle_root),
        assignment['event_scan_options'],
        QueueDispositionPolicy(**dict(assignment['queue_policy'])),
        attempts=int(assignment['reservation'].get('attempts') or 1),
        candidate_max_items=int(limits.get('candidate_max_items') or 2000),
        candidate_max_bytes=int(limits.get('candidate_max_bytes') or 2 * 1024 * 1024),
        require_s_drive=False,
    )
    require(
        staged.queue_status == 'done'
        and staged.finding_count == staged.error_count == staged.candidate_count == 0,
        'clean staged bundle',
    )
    seen.add(identity)
require(seen == expected, 'direct source coverage')
require(len(tuple(bundle_root.rglob('*.trb'))) == 2, 'published bundle count')
print('published direct bundles: 2')
'''


class Failure(RuntimeError):
    """A fixed diagnostic label that cannot contain credentials or findings."""


def require(condition, label):
    if not condition:
        raise Failure(label)


def sha256_bytes(value):
    return hashlib.sha256(value).hexdigest()


def sha256_file(path):
    digest = hashlib.sha256()
    with open(path, 'rb', buffering=0) as handle:
        for block in iter(lambda: handle.read(1024 * 1024), b''):
            digest.update(block)
    return digest.hexdigest()


def write_bytes(path, value):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, 'xb') as handle:
        handle.write(value)


def write_json(path, value):
    write_bytes(
        path,
        json.dumps(
            value, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
        ).encode('ascii') + b'\n',
    )


def load_json_bytes(value, label):
    require(len(value) <= MAX_OUTPUT, label + '_size')
    try:
        result = json.loads(value.decode('utf-8', errors='strict'))
    except (UnicodeError, ValueError):
        raise Failure(label + '_json') from None
    return result


def regular_file(path, label, maximum=MAX_OUTPUT):
    path = Path(path)
    try:
        details = path.stat(follow_symlinks=False)
    except OSError:
        raise Failure(label + '_missing') from None
    require(
        stat.S_ISREG(details.st_mode) and not path.is_symlink()
        and 0 < details.st_size <= maximum,
        label + '_regular',
    )
    return path


def synthetic_openai_token(label):
    """Return a fixed, deliberately fake, detector-shaped high-entropy token."""
    alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'

    def material(part):
        seed = hashlib.sha512(
            ('truf-packaged-worker-e2e:' + label + ':' + part).encode('ascii')
        ).hexdigest()
        return ''.join(
            alphabet[int(seed[index:index + 2], 16) % len(alphabet)]
            for index in range(0, len(seed), 2)
        )[:24]

    token = 'sk-proj-' + material('prefix') + 'T3BlbkFJ' + material('suffix')
    require(
        re.fullmatch(r'sk-proj-[A-Za-z0-9]{24}T3BlbkFJ[A-Za-z0-9]{24}', token),
        'synthetic_token_shape',
    )
    return token


class CommandRunner:
    def __init__(self, root, run_root, distro, deadline):
        self.root = Path(root)
        self.run_root = Path(run_root)
        self.deadline = deadline
        self.wsl = shutil.which('wsl.exe') or shutil.which('wsl')
        require(self.wsl, 'wsl_unavailable')
        self.docker_prefix = [
            self.wsl, '-d', distro, '--', 'sudo', '-n', 'docker',
        ]
        allowed = (
            'PATH', 'PATHEXT', 'SystemRoot', 'SYSTEMROOT', 'WINDIR', 'COMSPEC',
            'TEMP', 'TMP', 'USERPROFILE', 'WSLENV',
        )
        self.host_env = {
            name: os.environ[name] for name in allowed if name in os.environ
        }

    def execute(self, label, command, *, timeout=60, check=True, env=None, cwd=None):
        remaining = self.deadline - time.monotonic()
        require(remaining > 0, 'aggregate_timeout')
        timeout = max(0.1, min(float(timeout), remaining))
        process = None
        try:
            process = subprocess.Popen(
                command,
                cwd=os.fspath(cwd or self.root),
                env=env or self.host_env,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0),
            )
            try:
                stdout, stderr = process.communicate(timeout=timeout)
            except subprocess.TimeoutExpired:
                process.kill()
                process.communicate()
                raise Failure(label + '_timeout') from None
        except Failure:
            raise
        except OSError:
            raise Failure(label + '_unavailable') from None
        require(
            len(stdout) <= MAX_OUTPUT and len(stderr) <= MAX_OUTPUT,
            label + '_output_bound',
        )
        if check:
            if process.returncode != 0:
                error = Failure(label + '_failed')
                error.exit_code = process.returncode
                raise error
        return process.returncode, stdout, stderr

    def docker(self, label, arguments, *, timeout=60, check=True):
        return self.execute(
            label, [*self.docker_prefix, *arguments], timeout=timeout, check=check,
        )

    def docker_json(self, label, arguments, *, timeout=60):
        _, stdout, _ = self.docker(label, arguments, timeout=timeout)
        return load_json_bytes(stdout, label)

    def wsl_path(self, path):
        _, stdout, _ = self.execute(
            'wsl_path',
            [self.wsl, '-d', self.docker_prefix[2], '--', 'wslpath', '-a', '-u',
             os.fspath(path).replace('\\', '/')],
            timeout=30,
        )
        try:
            value = stdout.decode('utf-8', errors='strict').strip()
        except UnicodeError:
            raise Failure('wsl_path_encoding') from None
        require(value.startswith('/') and '\x00' not in value, 'wsl_path_invalid')
        return value


class ResourceTracker:
    def __init__(self, runner, run_id):
        self.runner = runner
        self.run_id = run_id
        self.resources = {'containers': {}, 'volumes': {}, 'networks': {}}
        self.foreign_baseline = None

    def labels(self, phase, kind):
        return [
            '--label', f'{RUN_LABEL}={self.run_id}',
            '--label', f'{PHASE_LABEL}={phase}',
            '--label', f'{KIND_LABEL}={kind}',
        ]

    @staticmethod
    def singular(kind):
        return {'containers': 'container', 'volumes': 'volume', 'networks': 'network'}[kind]

    def list_names(self, kind, *, name=None, owned=False):
        singular = self.singular(kind)
        arguments = [singular, 'ls']
        if kind == 'containers':
            arguments.append('--all')
        if name is not None:
            pattern = '^/' + name + '$' if kind == 'containers' else '^' + name + '$'
            arguments.extend(['--filter', 'name=' + pattern])
        if owned:
            arguments.extend(['--filter', f'label={RUN_LABEL}={self.run_id}'])
        field = 'Names' if kind == 'containers' else 'Name'
        arguments.append('--format={{.' + field + '}}')
        _, stdout, _ = self.runner.docker('list_' + kind, arguments)
        try:
            names = {line for line in stdout.decode('ascii').splitlines() if line}
        except UnicodeError:
            raise Failure('resource_name_encoding') from None
        require(len(names) <= 32, 'resource_inventory_bound')
        return names

    def guard_new(self, kind, name):
        require(RESOURCE_NAME.fullmatch(name), 'resource_name_guard')
        require(not self.list_names(kind, name=name), 'resource_name_collision')

    def inspect_owned(self, kind, name, expected=None):
        singular = self.singular(kind)
        if kind == 'containers':
            template = (
                '{"id":{{json .Id}},"name":{{json .Name}},'
                '"run":{{json (index .Config.Labels "' + RUN_LABEL + '")}},'
                '"phase":{{json (index .Config.Labels "' + PHASE_LABEL + '")}},'
                '"kind":{{json (index .Config.Labels "' + KIND_LABEL + '")}},'
                '"running":{{json .State.Running}},"paused":{{json .State.Paused}},'
                '"restarting":{{json .State.Restarting}}}'
            )
        else:
            template = (
                '{"id":{{json ' + ('.Id' if kind == 'networks' else '""') + '}},'
                '"name":{{json .Name}},"run":{{json (index .Labels "' + RUN_LABEL + '")}},'
                '"phase":{{json (index .Labels "' + PHASE_LABEL + '")}},'
                '"kind":{{json (index .Labels "' + KIND_LABEL + '")}}}'
            )
        value = self.runner.docker_json(
            'inspect_owned_' + singular, [singular, 'inspect', '--format', template, name],
        )
        require(
            value.get('name') == ('/' + name if kind == 'containers' else name)
            and value.get('run') == self.run_id
            and isinstance(value.get('phase'), str) and isinstance(value.get('kind'), str)
            and (kind == 'volumes' or HEX_64.fullmatch(str(value.get('id') or ''))),
            'owned_' + singular + '_identity_guard',
        )
        if expected is not None:
            require(
                (expected.get('id') is None or value.get('id') == expected['id'])
                and value['phase'] == expected['phase'] and value['kind'] == expected['kind'],
                'owned_' + singular + '_identity_changed',
            )
        return value

    def metadata_names(self, kind):
        arguments = ([kind, 'ls', '--all', '--no-trunc', '--format={{.ID}}']
                     if kind == 'container' else [kind, 'ls', '--format={{.Name}}'])
        _, stdout, _ = self.runner.docker('foreign_' + kind + '_list', arguments)
        try:
            values = {line for line in stdout.decode('ascii').splitlines() if line}
        except UnicodeError:
            raise Failure('foreign_' + kind + '_inventory_encoding') from None
        require(len(values) <= MAX_DOCKER_RESOURCES, 'foreign_' + kind + '_inventory_bound')
        return values

    def container_metadata(self, identifier):
        require(HEX_64.fullmatch(identifier), 'foreign_container_id_guard')
        value = self.runner.docker_json(
            'foreign_container_inspect',
            ['container', 'inspect', '--format', CONTAINER_METADATA_FORMAT, identifier],
        )
        require(
            value.get('id') == identifier and isinstance(value.get('name'), str)
            and HEX_64.fullmatch(str(value.get('image') or '').removeprefix('sha256:'))
            and value.get('status') in (
                'created', 'running', 'paused', 'restarting', 'removing', 'exited', 'dead',
            )
            and all(type(value.get(key)) is bool for key in (
                'running', 'paused', 'restarting', 'dead',
            ))
            and isinstance(value.get('mounts'), list) and len(value['mounts']) <= 128,
            'foreign_container_metadata_guard',
        )
        mounts = [{
            key: mount.get(key) for key in (
                'Type', 'Name', 'Source', 'Destination', 'Driver', 'Mode', 'RW', 'Propagation',
            )
        } for mount in value['mounts']]
        return {
            'id': value['id'], 'name': value['name'], 'image': value['image'],
            'status': value['status'], 'running': value['running'], 'paused': value['paused'],
            'restarting': value['restarting'], 'dead': value['dead'],
            'mounts_sha256': sha256_bytes(json.dumps(
                mounts, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
            ).encode('ascii')),
        }

    def volume_metadata(self, name):
        value = self.runner.docker_json(
            'foreign_volume_inspect',
            ['volume', 'inspect', '--format', VOLUME_METADATA_FORMAT, name],
        )
        require(
            value.get('name') == name and isinstance(value.get('driver'), str)
            and isinstance(value.get('scope'), str) and isinstance(value.get('mountpoint'), str)
            and (value.get('created') is None or isinstance(value['created'], str))
            and (value.get('labels') is None or isinstance(value['labels'], dict))
            and (value.get('options') is None or isinstance(value['options'], dict)),
            'foreign_volume_metadata_guard',
        )
        return {
            'name': name, 'driver': value['driver'], 'scope': value['scope'],
            'created': value['created'],
            'mountpoint_sha256': sha256_bytes(value['mountpoint'].encode('utf-8')),
            'labels_sha256': sha256_bytes(json.dumps(
                value['labels'], ensure_ascii=True, sort_keys=True, separators=(',', ':'),
            ).encode('ascii')),
            'options_sha256': sha256_bytes(json.dumps(
                value['options'], ensure_ascii=True, sort_keys=True, separators=(',', ':'),
            ).encode('ascii')),
        }

    def metadata_snapshot(self, *, exclude_owned=False):
        containers = {}
        owned_ids = {
            value['id']: (name, value)
            for name, value in self.resources['containers'].items() if value.get('id')
        }
        for identifier in sorted(self.metadata_names('container')):
            if exclude_owned and identifier in owned_ids:
                name, expected = owned_ids[identifier]
                self.inspect_owned('containers', name, expected)
                continue
            containers[identifier] = self.container_metadata(identifier)
        volumes = {}
        for name in sorted(self.metadata_names('volume')):
            value = self.volume_metadata(name)
            expected = self.resources['volumes'].get(name)
            if exclude_owned and expected is not None and expected.get('metadata') == value:
                continue
            volumes[name] = value
        snapshot = {'containers': containers, 'volumes': volumes}
        require(len(json.dumps(snapshot, ensure_ascii=True, sort_keys=True)) <= MAX_SNAPSHOT_BYTES,
                'foreign_metadata_snapshot_bound')
        return snapshot

    def snapshot_foreign(self):
        require(self.foreign_baseline is None, 'foreign_snapshot_already_taken')
        self.foreign_baseline = self.metadata_snapshot()

    def assert_foreign_unchanged(self):
        require(self.foreign_baseline is not None, 'foreign_snapshot_missing')
        require(self.metadata_snapshot(exclude_owned=True) == self.foreign_baseline,
                'foreign_docker_state_changed')

    def create_volume(self, name, phase, role):
        self.guard_new('volumes', name)
        self.resources['volumes'][name] = {'id': None, 'phase': phase, 'kind': role}
        _, stdout, _ = self.runner.docker(
            'create_volume',
            ['volume', 'create', *self.labels(phase, role), name],
        )
        require(stdout.decode('ascii', errors='ignore').strip() == name, 'volume_create_result')
        self.inspect_owned('volumes', name, self.resources['volumes'][name])
        self.resources['volumes'][name]['metadata'] = self.volume_metadata(name)

    def create_network(self, name, phase):
        self.guard_new('networks', name)
        self.resources['networks'][name] = {
            'id': None, 'phase': phase, 'kind': 'network',
        }
        _, stdout, _ = self.runner.docker(
            'create_network',
            ['network', 'create', '--driver', 'bridge', '--internal',
             '--subnet', isolated_test_subnet(self.run_id, phase),
             *self.labels(phase, 'network'), name],
        )
        identifier = stdout.decode('ascii', errors='strict').strip()
        require(HEX_64.fullmatch(identifier), 'network_create_result')
        self.resources['networks'][name]['id'] = identifier
        self.inspect_owned('networks', name, self.resources['networks'][name])

    def planned_container(self, name, phase, role):
        self.resources['containers'][name] = {'id': None, 'phase': phase, 'kind': role}

    def created_container(self, name, identifier):
        self.resources['containers'][name]['id'] = identifier
        self.inspect_owned('containers', name, self.resources['containers'][name])

    def inventory(self):
        inventory = {}
        for kind in self.resources:
            try:
                inventory[kind] = sorted(self.list_names(kind, owned=True))
            except Exception:
                inventory[kind] = sorted(self.resources[kind])
        return inventory

    def guarded_stop(self):
        stopped = 0
        for name, expected in reversed(self.resources['containers'].items()):
            try:
                value = self.inspect_owned('containers', name, expected)
                if expected['id'] is None:
                    expected['id'] = value['id']
                if value['running'] or value['paused'] or value['restarting']:
                    value = self.inspect_owned('containers', name, expected)
                    code, _, _ = self.runner.docker(
                        'guarded_stop_owned_container',
                        ['container', 'stop', '--time', '30', value['id']],
                        timeout=45, check=False,
                    )
                    stopped += int(code == 0)
            except (Exception, KeyboardInterrupt):
                continue
        return stopped

    def cleanup(self):
        inventory = {
            kind: self.list_names(kind, owned=True) for kind in self.resources
        }
        for kind, expected in self.resources.items():
            require(inventory[kind] == set(expected), 'cleanup_ownership_guard')
        for name, expected in reversed(self.resources['containers'].items()):
            value = self.inspect_owned('containers', name, expected)
            if value['running'] or value['paused'] or value['restarting']:
                value = self.inspect_owned('containers', name, expected)
                self.runner.docker(
                    'stop_owned_container', ['container', 'stop', '--time', '30', value['id']],
                    timeout=45,
                )
            value = self.inspect_owned('containers', name, expected)
            self.runner.docker(
                'remove_owned_container', ['container', 'rm', value['id']],
                timeout=45,
            )
        for name, expected in reversed(self.resources['volumes'].items()):
            self.inspect_owned('volumes', name, expected)
            self.runner.docker('remove_owned_volume', ['volume', 'rm', name])
        for name, expected in reversed(self.resources['networks'].items()):
            value = self.inspect_owned('networks', name, expected)
            self.runner.docker('remove_owned_network', ['network', 'rm', value['id']])
        require(
            all(not self.list_names(kind, owned=True) for kind in self.resources),
            'cleanup_incomplete',
        )
        self.assert_foreign_unchanged()


class WindowsClient:
    def __init__(self, command, environment, log_path):
        self.command = command
        self.environment = environment
        self.log_path = Path(log_path)
        self.process = None
        self.reader = None
        self.output = bytearray()
        self.output_lock = threading.Lock()
        self.output_exceeded = False

    def _read_output(self, process):
        for block in iter(lambda: process.stdout.read(64 * 1024), b''):
            with self.output_lock:
                if len(self.output) + len(block) <= MAX_OUTPUT:
                    self.output.extend(block)
                else:
                    self.output_exceeded = True

    def read_log(self):
        with self.output_lock:
            require(not self.output_exceeded, 'windows_log_bound')
            return bytes(self.output)

    def start(self):
        require(self.process is None, 'windows_client_already_started')
        try:
            self.process = subprocess.Popen(
                self.command,
                cwd=os.fspath(self.log_path.parent),
                env=self.environment,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                creationflags=(
                    getattr(subprocess, 'CREATE_NEW_PROCESS_GROUP', 0)
                    | getattr(subprocess, 'CREATE_NO_WINDOW', 0)
                ),
            )
        except OSError:
            raise Failure('windows_client_unavailable') from None
        self.reader = threading.Thread(target=self._read_output, args=(self.process,), daemon=True)
        self.reader.start()

    def alive(self):
        return self.process is not None and self.process.poll() is None

    def stop(self):
        if self.process is None:
            return
        if self.process.poll() is None:
            taskkill = shutil.which('taskkill.exe') or 'taskkill.exe'
            subprocess.run(
                [taskkill, '/PID', str(self.process.pid), '/T', '/F'],
                stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL, check=False,
                creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0),
            )
            try:
                self.process.wait(timeout=15)
            except subprocess.TimeoutExpired:
                self.process.kill()
                self.process.wait(timeout=5)
        if self.reader is not None:
            self.reader.join(timeout=5)
        if self.process.stdout is not None:
            self.process.stdout.close()
        if self.reader is not None:
            self.reader.join(timeout=1)
        self.process = None
        self.reader = None


class LoopbackProxy:
    def __init__(self, runner, upstream, log_path):
        self.runner = runner
        self.upstream = upstream
        self.log_path = Path(log_path)
        self.process = None
        self.port = None
        self.output = None

    def start(self):
        require(self.process is None, 'loopback_proxy_already_started')
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation:
            reservation.bind(('127.0.0.1', 0))
            self.port = int(reservation.getsockname()[1])
        try:
            self.output = self.log_path.open('wb')
            self.process = subprocess.Popen(
                [self.runner.wsl, '-d', self.runner.docker_prefix[2], '--',
                 'python3', '-u', '-c', LOOPBACK_PROXY_SCRIPT,
                 str(self.port), self.upstream, '8443'],
                cwd=os.fspath(self.runner.run_root), env=self.runner.host_env,
                stdin=subprocess.DEVNULL, stdout=self.output,
                stderr=subprocess.STDOUT,
                creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0),
            )
        except OSError:
            if self.output is not None:
                self.output.close()
                self.output = None
            raise Failure('loopback_proxy_unavailable') from None

        end = min(self.runner.deadline, time.monotonic() + 60)
        context = ssl.create_default_context(
            cafile=os.fspath(self.runner.root / 'tests' / 'fixtures' / 'worker_tls_cert.pem'),
        )
        while time.monotonic() < end:
            require(self.process.poll() is None, 'loopback_proxy_exited')
            try:
                with socket.create_connection(
                    ('127.0.0.1', self.port), timeout=0.5,
                ) as connection, context.wrap_socket(
                    connection, server_hostname='localhost',
                ):
                    return self.port
            except OSError:
                time.sleep(0.1)
        try:
            detail = self.log_path.read_text(encoding='utf-8', errors='replace')
        except OSError:
            detail = ''
        if 'ConnectionRefusedError' in detail:
            raise Failure('loopback_proxy_upstream_refused')
        if 'TimeoutError' in detail:
            raise Failure('loopback_proxy_upstream_timeout')
        if 'proxy upstream connected' in detail:
            raise Failure('loopback_proxy_tls_timeout')
        raise Failure('loopback_proxy_timeout')

    def stop(self):
        if self.process is not None and self.process.poll() is None:
            self.process.terminate()
            try:
                self.process.wait(timeout=10)
            except subprocess.TimeoutExpired:
                self.process.kill()
                self.process.wait(timeout=5)
        self.process = None
        if self.output is not None:
            self.output.close()
            self.output = None


class Verifier:
    def __init__(self, args):
        self.args = args
        self.script = Path(__file__).resolve(strict=True)
        self.root = self.script.parent.parent.resolve(strict=True)
        require(self.script == self.root / 'docker' / 'verify_packaged_workers.py', 'script_path')
        require('build/' in (self.root / '.gitignore').read_text(encoding='utf-8').splitlines(),
                'build_not_gitignored')
        self.run_id = secrets.token_hex(8)
        self.prefix = 'truf-packaged-worker-e2e-' + self.run_id
        build = self.root / 'build'
        build.mkdir(exist_ok=True)
        self.run_root = build / ('pwe-' + self.run_id)
        self.run_root.mkdir()
        self.deadline = time.monotonic() + args.timeout_seconds
        self.runner = CommandRunner(self.root, self.run_root, args.wsl_distro, self.deadline)
        self.tracker = ResourceTracker(self.runner, self.run_id)
        self.windows_artifact = Path(args.windows_artifact)
        if not self.windows_artifact.is_absolute():
            self.windows_artifact = self.root / self.windows_artifact
        self.windows_artifact = self.windows_artifact.resolve(strict=True)
        require(self.windows_artifact.is_relative_to(self.root), 'windows_artifact_outside_root')
        self.tokens = {
            TARGETS[0]: synthetic_openai_token('a'),
            TARGETS[1]: synthetic_openai_token('b'),
        }
        require(len(set(self.tokens.values())) == 2, 'synthetic_tokens_distinct')
        self.device_tokens = {}
        self.windows_client = None
        self.loopback_proxy = None
        self.image_details = {}
        self.repository_heads = {}
        self.phase_results = {}

    def image(self, reference, label, *, worker=False):
        value = self.runner.docker_json(label, ['image', 'inspect', reference])
        require(isinstance(value, list) and len(value) == 1, label + '_shape')
        details = value[0]
        require(
            details.get('Os') == 'linux' and details.get('Architecture') == 'amd64'
            and HEX_64.fullmatch(str(details.get('Id') or '').removeprefix('sha256:')),
            label + '_platform',
        )
        if worker:
            expected = [
                '/usr/bin/tini', '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B',
                '/opt/truf-worker/app/remote_worker_bootstrap.py', '--',
            ]
            require(details.get('Config', {}).get('Entrypoint') == expected,
                    'linux_worker_entrypoint')
            require(details.get('Config', {}).get('Cmd') == ['run'],
                    'linux_worker_default_command')
            require(details.get('Config', {}).get('User') == '10001:10001',
                    'linux_worker_user')
        self.image_details[label] = details['Id']

    def prepare_repositories(self):
        git = shutil.which('git.exe') or shutil.which('git')
        require(git, 'git_unavailable')
        repositories = self.run_root / 'repositories'
        repositories.mkdir()
        base_env = dict(self.runner.host_env)
        base_env.update({
            'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': 'NUL',
            'GIT_TERMINAL_PROMPT': '0', 'TZ': 'UTC',
            'GIT_AUTHOR_NAME': 'TRUF Packaged Worker Fixture',
            'GIT_AUTHOR_EMAIL': 'packaged-worker-fixture@invalid.example',
            'GIT_COMMITTER_NAME': 'TRUF Packaged Worker Fixture',
            'GIT_COMMITTER_EMAIL': 'packaged-worker-fixture@invalid.example',
            'GIT_AUTHOR_DATE': '2001-01-01T00:00:00+00:00',
            'GIT_COMMITTER_DATE': '2001-01-01T00:00:00+00:00',
        })
        for index, target in enumerate(TARGETS):
            name = chr(ord('a') + index)
            path = repositories / name
            path.mkdir()
            write_bytes(path / 'synthetic.env', ('OPENAI_API_KEY=' + self.tokens[target] + '\n').encode('ascii'))
            self.runner.execute(
                'git_init_' + name,
                [git, 'init', '--quiet', '--initial-branch=main', '--template=', os.fspath(path)],
                env=base_env,
            )
            self.runner.execute(
                'git_config_' + name,
                [git, '-C', os.fspath(path), 'config', 'core.autocrlf', 'false'],
                env=base_env,
            )
            self.runner.execute(
                'git_add_' + name, [git, '-C', os.fspath(path), 'add', '--', 'synthetic.env'],
                env=base_env,
            )
            self.runner.execute(
                'git_commit_' + name,
                [git, '-C', os.fspath(path), 'commit', '--quiet', '--no-gpg-sign',
                 '--message', 'Add deterministic synthetic detector fixture'],
                env=base_env,
            )
            _, stdout, _ = self.runner.execute(
                'git_head_' + name, [git, '-C', os.fspath(path), 'rev-parse', 'HEAD'],
                env=base_env,
            )
            head = stdout.decode('ascii', errors='strict').strip()
            require(HEX_40.fullmatch(head), 'fixture_commit_hash')
            _, status_output, _ = self.runner.execute(
                'git_status_' + name,
                [git, '-C', os.fspath(path), 'status', '--porcelain=v1'], env=base_env,
            )
            require(not status_output, 'fixture_repository_dirty')
            self.repository_heads[target] = head
        require(len(set(self.repository_heads.values())) == 2, 'fixture_commits_distinct')

    def prepare_fixtures(self):
        manifest_path = regular_file(
            self.windows_artifact / 'worker-package.json', 'windows_manifest',
        )
        for relative, label in (
            ('truf-worker.cmd', 'windows_cli_entrypoint'),
            ('run-worker.cmd', 'windows_entrypoint'),
            ('runtime/python/python.exe', 'windows_python'),
            ('bin/trufflehog.exe', 'windows_scanner'),
        ):
            regular_file(self.windows_artifact / Path(relative), label, maximum=256 * 1024 * 1024)
        require(
            b'remote_worker_bootstrap.py" -- %*' in
            (self.windows_artifact / 'truf-worker.cmd').read_bytes(),
            'windows_cli_entrypoint_contents',
        )
        require(
            b'remote_worker_bootstrap.py" -- run %*' in
            (self.windows_artifact / 'run-worker.cmd').read_bytes(),
            'windows_compat_entrypoint_contents',
        )
        windows_manifest = manifest_path.read_bytes()
        windows_value = load_json_bytes(windows_manifest, 'windows_manifest')
        expected_capabilities = {
            ('gitlab', 'gitlab', 'exact_git_v1'),
            ('dockerhub', 'docker', 'docker_direct_v1'),
            ('huggingface', 'huggingface', 'huggingface_space_v1'),
        }
        windows_capabilities = {
            (
                item.get('source'), item.get('platform'),
                item.get('planning_kind'),
            )
            for item in windows_value.get('capabilities', [])
            if isinstance(item, dict)
        }
        require(
            windows_value.get('schema') == 3
            and windows_value.get('protocol_version') == 2
            and windows_value.get('platform_tag') == 'windows-x86_64',
            'windows_manifest_identity',
        )
        require(
            windows_capabilities == expected_capabilities,
            'windows_manifest_capabilities',
        )
        helper_name = self.prefix + '-linux-manifest'
        self.create_container(
            helper_name, 'linux', 'manifest-reader',
            ['--network', 'none', '--entrypoint', '/bin/cat', self.args.linux_image,
             '/opt/truf-worker/worker-package.json'],
        )
        _, linux_manifest, _ = self.runner.docker(
            'linux_manifest',
            ['container', 'start', '--attach', helper_name],
        )
        _, exit_code, _ = self.runner.docker(
            'linux_manifest_exit',
            ['container', 'inspect', '--format={{.State.ExitCode}}', helper_name],
        )
        require(exit_code.strip() == b'0', 'linux_manifest_reader_exit')
        linux_value = load_json_bytes(linux_manifest, 'linux_manifest')
        linux_capabilities = {
            (
                item.get('source'), item.get('platform'),
                item.get('planning_kind'),
            )
            for item in linux_value.get('capabilities', [])
            if isinstance(item, dict)
        }
        require(
            linux_value.get('schema') == 3
            and linux_value.get('protocol_version') == 2
            and linux_value.get('platform_tag') == 'linux-x86_64',
            'linux_manifest_identity',
        )
        require(
            linux_capabilities == expected_capabilities
            and linux_capabilities == windows_capabilities,
            'linux_manifest_capabilities',
        )

        fixture = self.run_root / 'fixture'
        fixture.mkdir()
        write_json(fixture / 'windows-manifest.json', windows_value)
        write_json(fixture / 'linux-manifest.json', linux_value)
        write_json(fixture / 'fixture.json', {
            'schema': 1,
            'targets': list(TARGETS),
            'direct_targets': DIRECT_TARGETS,
            'repositories': {
                target: {'head_sha': self.repository_heads[target]} for target in TARGETS
            },
        })
        certificate = regular_file(
            self.root / 'tests' / 'fixtures' / 'worker_tls_cert.pem', 'tls_certificate',
        )
        private_key = regular_file(
            self.root / 'tests' / 'fixtures' / 'worker_tls_key.pem', 'tls_private_key',
        )
        decoded = ssl._ssl._test_decode_cert(os.fspath(certificate))
        require(('DNS', 'localhost') in decoded.get('subjectAltName', ()), 'tls_localhost_san')
        require(ssl.cert_time_to_seconds(decoded['notAfter']) > time.time(), 'tls_certificate_expired')
        write_bytes(fixture / 'worker_tls_cert.pem', certificate.read_bytes())
        write_bytes(fixture / 'worker_tls_key.pem', private_key.read_bytes())
        self.fixture = fixture
        self.manifest_hashes = {
            'windows': sha256_bytes(windows_manifest),
            'linux': sha256_bytes(linux_manifest),
        }

    def preflight(self):
        require(os.name == 'nt', 'windows_host_required')
        self.tracker.snapshot_foreign()
        self.image(self.args.linux_image, 'linux_image', worker=True)
        self.image(self.args.test_image, 'test_image')
        self.prepare_repositories()
        self.prepare_fixtures()

    def phase_names(self, phase):
        base = self.prefix + '-' + phase
        return {
            'network': base + '-network',
            'server_data': base + '-server-data',
            'server_seed': base + '-server-seed',
            'fixture': base + '-fixture-data',
            'worker_data': base + '-worker-data',
            'fixture_seed': base + '-fixture-seed',
            'worker_seed': base + '-worker-seed',
            'server': base + '-server',
            'worker': base + '-worker',
        }

    def create_container(self, name, phase, role, arguments):
        self.tracker.guard_new('containers', name)
        self.tracker.planned_container(name, phase, role)
        _, stdout, _ = self.runner.docker(
            'create_' + role,
            ['container', 'create', '--pull=never', '--name', name,
             *self.tracker.labels(phase, role), *arguments],
        )
        identifier = stdout.decode('ascii', errors='ignore').strip()
        require(HEX_64.fullmatch(identifier), role + '_container_id')
        self.tracker.created_container(name, identifier)

    def seed_volume(self, phase, role, name, volume, source):
        self.create_container(
            name, phase, role,
            ['--network', 'none', '--user', '0:0',
             '--mount', f'type=volume,source={volume},target=/seed',
             '--entrypoint', '/usr/local/bin/python3', self.args.test_image,
             '-u', '-I', '-S', '-B', '/seed/_seed.py'],
        )
        source_wsl = self.runner.wsl_path(Path(source).resolve(strict=True))
        self.runner.docker(
            'copy_' + role, ['container', 'cp', source_wsl + '/.', name + ':/seed/'],
            timeout=120,
        )
        self.runner.docker('start_' + role, ['container', 'start', '--attach', name], timeout=120)
        _, stdout, _ = self.runner.docker(
            'wait_' + role,
            ['container', 'inspect', '--format={{.State.ExitCode}}', name],
        )
        require(stdout.strip() == b'0', role + '_exit')

    def prepare_phase_resources(self, phase):
        names = self.phase_names(phase)
        phase_root = self.run_root / phase
        phase_root.mkdir()
        if phase == 'windows':
            write_bytes(
                phase_root / 'publish-direct.py', PUBLISH_DIRECT_SCRIPT.encode('ascii'),
            )
        fixture_seed = phase_root / 'fixture-seed'
        shutil.copytree(self.fixture, fixture_seed)
        write_bytes(fixture_seed / '_seed.py', SEED_SCRIPT.encode('ascii'))
        server_seed = phase_root / 'server-seed'
        server_seed.mkdir()
        write_bytes(server_seed / '_seed.py', SEED_SCRIPT.encode('ascii'))
        self.tracker.create_network(names['network'], phase)
        self.tracker.create_volume(names['server_data'], phase, 'server-data')
        self.seed_volume(
            phase, 'server-seed', names['server_seed'], names['server_data'], server_seed,
        )
        self.tracker.create_volume(names['fixture'], phase, 'fixture-data')
        self.seed_volume(
            phase, 'fixture-seed', names['fixture_seed'], names['fixture'], fixture_seed,
        )
        if phase == 'linux':
            worker_seed = phase_root / 'worker-seed'
            worker_seed.mkdir()
            shutil.copytree(self.run_root / 'repositories', worker_seed / 'repos')
            shutil.copyfile(
                self.fixture / 'worker_tls_cert.pem', worker_seed / 'worker_tls_cert.pem',
            )
            write_bytes(worker_seed / '_seed.py', SEED_SCRIPT.encode('ascii'))
            write_bytes(worker_seed / '_inspect.py', INSPECT_WORKER_SCRIPT.encode('ascii'))
            write_bytes(
                worker_seed / '_publish_direct.py', PUBLISH_DIRECT_SCRIPT.encode('ascii'),
            )
            self.tracker.create_volume(names['worker_data'], phase, 'worker-data')
            self.seed_volume(
                phase, 'worker-seed', names['worker_seed'], names['worker_data'], worker_seed,
            )
        self.device_tokens[phase] = secrets.token_hex(32)
        self.start_server(phase, names)
        self.wait_control(phase, names['server'], 'prepared.json', {
            'schema': 1, 'phase': phase, 'target_count': 2,
        }, 180)
        return names, phase_root

    def start_server(self, phase, names):
        self.create_container(
            names['server'], phase, 'server',
            ['--network', names['network'],
             '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
             '--pids-limit', '512', '--stop-timeout', '30',
             '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=128m,mode=1777',
             '--mount', f'type=volume,source={names["server_data"]},target=/data',
             '--mount', f'type=volume,source={names["fixture"]},target=/fixture,readonly',
             '--env', 'TRUF_WORKER_E2E_TOKEN=' + self.device_tokens[phase],
             '--env', 'TRUF_WORKER_E2E_PHASE=' + phase,
             '--entrypoint', '/usr/bin/tini', self.args.test_image,
             '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B',
             '/opt/truf/tests/packaged_worker_e2e_server.py'],
        )
        self.runner.docker('start_server', ['container', 'start', names['server']])
        require(self.container_running(names['server']), 'server_not_running')

    def container_running(self, name):
        code, stdout, _ = self.runner.docker(
            'container_running',
            ['container', 'inspect', '--format={{.State.Running}}', name],
            check=False,
        )
        return code == 0 and stdout.strip() == b'true'

    def read_control(self, server, name):
        code, stdout, _ = self.runner.docker(
            'read_control', ['container', 'exec', server, '/bin/cat', '/data/control/' + name],
            check=False,
        )
        if code:
            return None
        value = load_json_bytes(stdout, 'control_' + name.replace('.', '_'))
        require(isinstance(value, dict), 'control_object')
        return value

    def wait_until(self, label, function, seconds, *, alive=None):
        end = min(self.deadline, time.monotonic() + seconds)
        while time.monotonic() < end:
            value = function()
            if value is not None and value is not False:
                return value
            if alive is not None:
                require(alive(), label + '_process_exited')
            time.sleep(0.25)
        raise Failure(label + '_timeout')

    def wait_control(self, phase, server, name, expected, seconds, *, alive=None):
        def marker():
            value = self.read_control(server, name)
            if value is None:
                return None
            require(value == expected, phase + '_' + name.replace('.', '_') + '_content')
            return value

        return self.wait_until(
            phase + '_' + name.replace('.', '_'), marker, seconds,
            alive=lambda: self.container_running(server) and (alive is None or alive()),
        )

    @staticmethod
    def git_environment(local_repositories, certificate, *, linux=False):
        environment = {
            'SSL_CERT_FILE': os.fspath(certificate),
            'GIT_ALLOW_PROTOCOL': 'file',
            'GIT_TERMINAL_PROMPT': '0',
            'GIT_LFS_SKIP_SMUDGE': '1',
            'GIT_CONFIG_NOSYSTEM': '1',
            'GIT_CONFIG_GLOBAL': '/dev/null' if linux else 'NUL',
            'GIT_CONFIG_COUNT': '2',
        }
        for index, target in enumerate(TARGETS):
            local = local_repositories[index]
            uri = local if linux else Path(local).resolve(strict=True).as_uri()
            environment[f'GIT_CONFIG_KEY_{index}'] = f'url.{uri}.insteadOf'
            environment[f'GIT_CONFIG_VALUE_{index}'] = target
        return environment

    def windows_snapshot(self, phase_root):
        state_root = self.run_root / 'w' / 'TRUF' / 'RemoteWorker'
        bundle_root = state_root / 'bundles'
        work_root = state_root / 'work'
        states = {}
        if state_root.exists():
            for path in sorted(state_root.glob('slot-*.json')):
                regular_file(path, 'windows_state', maximum=64 * 1024 * 1024)
                value = load_json_bytes(path.read_bytes(), 'windows_state')
                assignment = dict(value.get('assignment') or {})
                reservation = dict(assignment.get('reservation') or {})
                snapshot = dict(assignment.get('execution_snapshot') or {})
                planning = dict(snapshot.get('planning') or {})
                details = path.stat(follow_symlinks=False)
                states[path.name] = {
                    'phase': value.get('phase'),
                    'reservation_id': reservation.get('reservation_id'),
                    'bundle_id': reservation.get('bundle_id'),
                    'scan_event_id': reservation.get('scan_event_id'),
                    'source': reservation.get('source'),
                    'platform': reservation.get('platform'),
                    'target': reservation.get('target'),
                    'planning_kind': planning.get('kind'),
                    'sha256': sha256_file(path), 'size': details.st_size,
                    'mtime_ns': details.st_mtime_ns,
                }
        bundles = {}
        if bundle_root.exists():
            for path in sorted(bundle_root.rglob('*.trb')):
                regular_file(path, 'windows_bundle', maximum=64 * 1024 * 1024)
                details = path.stat(follow_symlinks=False)
                bundles[path.relative_to(bundle_root).as_posix()] = {
                    'sha256': sha256_file(path), 'size': details.st_size,
                    'mtime_ns': details.st_mtime_ns,
                }
        snapshot = {'states': states, 'bundles': bundles}
        if not self.durable(snapshot):
            return snapshot
        snapshot['work'] = self.host_tree_identity(work_root)
        return snapshot

    @staticmethod
    def host_tree_identity(root):
        digest = hashlib.sha256()
        count = 0
        active_entries = 0
        root = Path(root)
        if not root.exists():
            return {'count': 0, 'active_entries': 0, 'sha256': digest.hexdigest()}
        for current, directories, files in os.walk(root, followlinks=False):
            directories.sort()
            files.sort()
            for name in directories:
                path = Path(current) / name
                require(not path.is_symlink() and path.is_dir(), 'windows_work_directory')
                if path.relative_to(root).parts[0] != 'abandoned':
                    active_entries += 1
            for name in files:
                path = regular_file(
                    Path(current) / name, 'windows_work_file', maximum=256 * 1024 * 1024,
                )
                if path.relative_to(root).parts[0] != 'abandoned':
                    active_entries += 1
                details = path.stat(follow_symlinks=False)
                digest.update(path.relative_to(root).as_posix().encode('utf-8') + b'\0')
                digest.update(str(details.st_size).encode('ascii') + b'\0')
                digest.update(str(details.st_mtime_ns).encode('ascii') + b'\0')
                digest.update(sha256_file(path).encode('ascii') + b'\n')
                count += 1
        return {
            'count': count, 'active_entries': active_entries,
            'sha256': digest.hexdigest(),
        }

    def linux_snapshot(self, worker):
        code, stdout, _ = self.runner.docker(
            'linux_worker_snapshot',
            ['container', 'exec', worker, '/usr/local/bin/python3',
             '-u', '-I', '-S', '-B', '/data/_inspect.py'],
            check=False,
        )
        if code:
            return None
        value = load_json_bytes(stdout, 'linux_worker_snapshot')
        require(isinstance(value, dict), 'linux_snapshot_object')
        return value

    @staticmethod
    def durable(snapshot):
        if not isinstance(snapshot, dict):
            return False
        states = snapshot.get('states')
        bundles = snapshot.get('bundles')
        if set(states or {}) != {'slot-0.json', 'slot-1.json'} or len(bundles or {}) != 2:
            return False
        bundle_ids = set()
        for state in states.values():
            if (
                state.get('phase') != 'bundle_ready'
                or not HEX_64.fullmatch(str(state.get('sha256') or ''))
                or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('bundle_id') or ''))
                or int(state.get('reservation_id') or 0) <= 0
            ):
                return False
            bundle_ids.add(state['bundle_id'])
        if len(bundle_ids) != 2:
            return False
        for path, entry in bundles.items():
            if (
                Path(path).name.removesuffix('.trb') not in bundle_ids
                or not HEX_64.fullmatch(str(entry.get('sha256') or ''))
                or int(entry.get('size') or 0) <= 0
            ):
                return False
        return True

    @staticmethod
    def direct_assigned(snapshot, bundle_count):
        if not isinstance(snapshot, dict):
            return False
        states = snapshot.get('states')
        bundles = snapshot.get('bundles')
        if (
            set(states or {}) != {'slot-0.json', 'slot-1.json'}
            or len(bundles or {}) != bundle_count
        ):
            return False
        expected = {
            ('dockerhub', 'docker', 'docker_direct_v1', DOCKER_TARGET),
            ('huggingface', 'huggingface', 'huggingface_space_v1', HUGGINGFACE_TARGET),
        }
        identities = set()
        bundle_ids = set()
        for state in states.values():
            if (
                state.get('phase') != 'assigned'
                or not HEX_64.fullmatch(str(state.get('sha256') or ''))
                or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('bundle_id') or ''))
                or int(state.get('reservation_id') or 0) <= 0
                or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('scan_event_id') or ''))
            ):
                return False
            identities.add((
                state.get('source'), state.get('platform'),
                state.get('planning_kind'), state.get('target'),
            ))
            bundle_ids.add(state['bundle_id'])
        if identities != expected or len(bundle_ids) != 2:
            return False
        for path, entry in bundles.items():
            if (
                Path(path).name.removesuffix('.trb') not in bundle_ids
                or not HEX_64.fullmatch(str(entry.get('sha256') or ''))
                or int(entry.get('size') or 0) <= 0
            ):
                return False
        work = snapshot.get('work')
        if work is not None and int(work.get('active_entries') or 0) != 0:
            return False
        return True

    @staticmethod
    def resolved_client_storage(snapshot):
        if not isinstance(snapshot, dict) or snapshot.get('bundles'):
            return False
        work = snapshot.get('work') or {}
        if int(work.get('active_entries') or 0) != 0:
            return False
        for state in (snapshot.get('states') or {}).values():
            if (
                state.get('phase') != 'claiming'
                or state.get('reservation_id') is not None
                or state.get('bundle_id') is not None
                or state.get('scan_event_id') is not None
            ):
                return False
        return True

    def assert_logs_safe(self, logs):
        forbidden = [*self.tokens.values(), *self.device_tokens.values()]
        for value in logs:
            require(len(value) <= MAX_OUTPUT, 'log_output_bound')
            for secret in forbidden:
                require(secret.encode('ascii') not in value, 'secret_present_in_log')

    def docker_logs(self, container):
        _, stdout, stderr = self.runner.docker(
            'docker_logs', ['container', 'logs', container], check=False,
        )
        return stdout + (b'\n' if stdout and stderr else b'') + stderr

    def assert_restart_stable(self, phase, snapshot, expected, alive):
        started = time.monotonic()
        end = min(self.deadline, time.monotonic() + 3)
        # The expected snapshot was captured before the real process restart.
        checks = 1
        current = None
        while time.monotonic() < end:
            require(alive(), phase + '_restart_process_exited')
            current = snapshot()
            require(
                isinstance(current, dict)
                and current.get('states') == expected.get('states')
                and current.get('bundles') == expected.get('bundles')
                and int((current.get('work') or {}).get('count') or 0)
                <= int((expected.get('work') or {}).get('count') or 0),
                phase + '_restart_rescanned_or_rewrote_bundle',
            )
            require(alive(), phase + '_restart_process_exited')
            checks += 1
            time.sleep(0.25)
        require(
            checks >= 2 and time.monotonic() - started >= 2.5,
            phase + '_restart_stability_window',
        )
        return current

    def restore_and_wait_direct(self, phase, names, snapshot, alive):
        self.runner.docker(
            phase + '_restore',
            ['container', 'exec', names['server'], '/usr/bin/touch', '/data/control/restore'],
        )
        return self.wait_until(
            phase + '_direct_claims',
            lambda: (
                value if self.direct_assigned(value := snapshot(), 0) else None
            ),
            240,
            alive=lambda: self.container_running(names['server']) and alive(),
        )

    def publish_windows_direct(self, phase_root):
        state_root = self.run_root / 'w' / 'TRUF' / 'RemoteWorker'
        environment = dict(self.runner.host_env)
        environment.update({
            'LOCALAPPDATA': os.fspath(phase_root / 'localappdata'),
            'TEMP': os.fspath(phase_root / 'temp'),
            'TMP': os.fspath(phase_root / 'temp'),
        })
        _, stdout, stderr = self.runner.execute(
            'windows_publish_direct',
            [
                os.fspath(self.windows_artifact / 'runtime' / 'python' / 'python.exe'),
                '-I', '-S', '-B', os.fspath(phase_root / 'publish-direct.py'),
                os.fspath(self.windows_artifact), os.fspath(state_root),
                os.fspath(state_root / 'bundles'),
            ],
            timeout=120,
            env=environment,
        )
        require(
            stdout.strip() == b'published direct bundles: 2' and not stderr,
            'windows_direct_publish_output',
        )

    def publish_linux_direct(self, worker):
        _, stdout, stderr = self.runner.docker(
            'linux_publish_direct',
            [
                'container', 'exec', worker, '/usr/local/bin/python3',
                '-u', '-I', '-S', '-B', '/data/_publish_direct.py',
                '/opt/truf-worker', '/data/state-base/truf/remote-worker',
                '/data/client/truf/remote-worker/bundles',
            ],
            timeout=120,
        )
        require(
            stdout.strip() == b'published direct bundles: 2' and not stderr,
            'linux_direct_publish_output',
        )

    def start_windows_client(self, phase_root, port):
        localappdata = self.run_root / 'w'
        temporary = phase_root / 'temp'
        localappdata.mkdir()
        temporary.mkdir()
        system_keys = (
            'PATH', 'PATHEXT', 'SystemRoot', 'SYSTEMROOT', 'WINDIR', 'COMSPEC',
            'NUMBER_OF_PROCESSORS', 'PROCESSOR_ARCHITECTURE',
        )
        environment = {
            key: os.environ[key] for key in system_keys if key in os.environ
        }
        environment.update({
            'LOCALAPPDATA': os.fspath(localappdata),
            'TEMP': os.fspath(temporary), 'TMP': os.fspath(temporary),
            **self.git_environment(
                [self.run_root / 'repositories' / 'a', self.run_root / 'repositories' / 'b'],
                self.fixture / 'worker_tls_cert.pem',
            ),
        })
        command = [
            environment.get('COMSPEC') or 'cmd.exe', '/d', '/c',
            os.fspath(self.windows_artifact / 'run-worker.cmd'),
            '--server', f'https://localhost:{port}',
            '--token', self.device_tokens['windows'],
            '--parallelism', str(PARALLELISM),
        ]
        self.windows_client = WindowsClient(command, environment, phase_root / 'client.log')
        self.windows_client.start()

    def windows_phase(self):
        names, phase_root = self.prepare_phase_resources('windows')
        _, stdout, _ = self.runner.docker(
            'windows_server_address',
            ['container', 'inspect',
             '--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
             names['server']],
        )
        address = stdout.decode('ascii', errors='strict').strip()
        require(re.fullmatch(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', address),
                'windows_server_address')
        self.loopback_proxy = LoopbackProxy(
            self.runner, address, phase_root / 'loopback-proxy.log',
        )
        port = self.loopback_proxy.start()
        self.start_windows_client(phase_root, port)
        try:
            self.wait_control('windows', names['server'], 'outage.json', {
                'schema': 1, 'claim_count': 2, 'phase': 'windows',
            }, 240, alive=self.windows_client.alive)
            before = self.wait_until(
                'windows_durable_bundle',
                lambda: (snapshot if self.durable(snapshot := self.windows_snapshot(phase_root)) else None),
                240, alive=self.windows_client.alive,
            )
            self.windows_client.stop()
            self.windows_client.start()
            self.assert_restart_stable(
                'windows', lambda: self.windows_snapshot(phase_root), before,
                self.windows_client.alive,
            )
            snapshot = lambda: self.windows_snapshot(phase_root)
            self.restore_and_wait_direct(
                'windows', names, snapshot, self.windows_client.alive,
            )
            self.publish_windows_direct(phase_root)
            direct = self.wait_until(
                'windows_direct_bundles',
                lambda: (
                    value if self.direct_assigned(value := snapshot(), 2) else None
                ),
                120,
                alive=self.windows_client.alive,
            )
            self.windows_client.stop()
            self.windows_client.start()
            self.assert_restart_stable(
                'windows', snapshot, direct, self.windows_client.alive,
            )
            self.complete_after_direct_ready('windows', names, snapshot)
            return self.finish_phase('windows', names, phase_root, (before, direct))
        finally:
            if self.windows_client is not None:
                self.windows_client.stop()
            if self.loopback_proxy is not None:
                self.loopback_proxy.stop()

    def create_linux_worker(self, names):
        environment = self.git_environment(
            ['file:///data/repos/a', 'file:///data/repos/b'],
            '/data/worker_tls_cert.pem', linux=True,
        )
        arguments = [
            '--network', 'container:' + names['server'],
            '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
            '--pids-limit', '256', '--stop-timeout', '45',
            '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=128m,mode=1777',
            '--mount', f'type=volume,source={names["worker_data"]},target=/data',
        ]
        for key, value in environment.items():
            arguments.extend(['--env', key + '=' + value])
        arguments.extend([
            '--env', 'XDG_DATA_HOME=/data/client', '--env', 'XDG_STATE_HOME=/data/state-base',
            self.args.linux_image,
            '--server', 'https://localhost:8443', '--token', self.device_tokens['linux'],
            '--parallelism', str(PARALLELISM),
        ])
        self.create_container(names['worker'], 'linux', 'worker', arguments)
        self.runner.docker('start_linux_worker', ['container', 'start', names['worker']])
        require(self.container_running(names['worker']), 'linux_worker_not_running')

    def linux_phase(self):
        names, phase_root = self.prepare_phase_resources('linux')
        self.create_linux_worker(names)
        self.wait_control('linux', names['server'], 'outage.json', {
            'schema': 1, 'claim_count': 2, 'phase': 'linux',
        }, 240, alive=lambda: self.container_running(names['worker']))
        before = self.wait_until(
            'linux_durable_bundle',
            lambda: (snapshot if self.durable(snapshot := self.linux_snapshot(names['worker'])) else None),
            240, alive=lambda: self.container_running(names['worker']),
        )
        worker = self.tracker.inspect_owned(
            'containers', names['worker'], self.tracker.resources['containers'][names['worker']],
        )
        self.runner.docker(
            'stop_linux_worker', ['container', 'stop', '--time', '45', worker['id']],
            timeout=60,
        )
        require(not self.container_running(names['worker']), 'linux_worker_stop')
        self.runner.docker('restart_linux_worker', ['container', 'start', names['worker']])
        self.assert_restart_stable(
            'linux', lambda: self.linux_snapshot(names['worker']), before,
            lambda: self.container_running(names['worker']),
        )
        snapshot = lambda: self.linux_snapshot(names['worker'])
        alive = lambda: self.container_running(names['worker'])
        self.restore_and_wait_direct('linux', names, snapshot, alive)
        self.publish_linux_direct(names['worker'])
        direct = self.wait_until(
            'linux_direct_bundles',
            lambda: (
                value if self.direct_assigned(value := snapshot(), 2) else None
            ),
            120,
            alive=alive,
        )
        worker = self.tracker.inspect_owned(
            'containers', names['worker'], self.tracker.resources['containers'][names['worker']],
        )
        self.runner.docker(
            'stop_linux_direct_worker',
            ['container', 'stop', '--time', '45', worker['id']],
            timeout=60,
        )
        require(not self.container_running(names['worker']), 'linux_direct_worker_stop')
        self.runner.docker('restart_linux_direct_worker', ['container', 'start', names['worker']])
        self.assert_restart_stable('linux_direct', snapshot, direct, alive)
        self.complete_after_direct_ready('linux', names, snapshot)
        return self.finish_phase('linux', names, phase_root, (before, direct))

    def complete_after_direct_ready(self, phase, names, snapshot):
        self.runner.docker(
            phase + '_direct_ready',
            ['container', 'exec', names['server'], '/usr/bin/touch',
             '/data/control/direct-ready'],
        )

        def completed():
            return self.read_control(names['server'], 'completed.json')

        try:
            evidence = self.wait_until(
                phase + '_completed', completed, 240,
                alive=lambda: self.container_running(names['server']),
            )
        except Failure:
            waiting = self.read_control(names['server'], 'completion-wait.json')
            if waiting is not None:
                require(
                    set(waiting) == {'schema', 'phase', 'reason'}
                    and waiting['schema'] == 1 and waiting['phase'] == phase
                    and isinstance(waiting['reason'], str)
                    and re.fullmatch(r'[A-Za-z0-9 _-]{1,80}', waiting['reason']),
                    phase + '_completion_wait_shape',
                )
                print(phase + ' completion wait: ' + waiting['reason'], flush=True)
            raise

        def clean():
            value = snapshot()
            if self.resolved_client_storage(value):
                return value
            return None

        self.wait_until(phase + '_client_cleanup', clean, 120)
        self.phase_results[phase] = evidence

    def finish_phase(self, phase, names, phase_root, restart_snapshots):
        if phase == 'linux':
            worker = self.tracker.inspect_owned(
                'containers', names['worker'], self.tracker.resources['containers'][names['worker']],
            )
            self.runner.docker(
                'final_stop_linux_worker',
                ['container', 'stop', '--time', '45', worker['id']], timeout=60,
            )
            final, stopped_reader = self.linux_snapshot_stopped(
                names['worker'], names['worker_data'], phase,
            )
        else:
            self.windows_client.stop()
            final = self.windows_snapshot(phase_root)
        require(self.resolved_client_storage(final), phase + '_final_cleanup')
        if phase == 'linux':
            receipt = final.get('shutdown_receipt')
            require(
                isinstance(receipt, dict)
                and set(receipt) == {
                    'schema', 'instance_id', 'completed_at', 'exit_code',
                    'drained', 'last_sequence',
                }
                and receipt.get('schema') == 1
                and receipt.get('drained') is True
                and receipt.get('exit_code') == 0,
                'linux_clean_shutdown_receipt',
            )
        logs = []
        for role in ('server_seed', 'fixture_seed', 'server'):
            logs.append(self.docker_logs(names[role]))
        if phase == 'linux':
            logs.extend([
                self.docker_logs(names['worker_seed']), self.docker_logs(names['worker']),
                self.docker_logs(stopped_reader),
            ])
        else:
            logs.append(self.windows_client.read_log())
        self.assert_logs_safe(logs)
        return {
            'bundle_sha256': sorted(
                entry['sha256']
                for snapshot in restart_snapshots
                for entry in snapshot['bundles'].values()
            ),
            'evidence': self.normalize_evidence(phase, self.phase_results[phase]),
        }

    def linux_snapshot_stopped(self, worker, volume, phase):
        reader = self.prefix + '-' + phase + '-stopped-reader'
        self.create_container(
            reader, phase, 'stopped-reader',
            ['--network', 'none', '--read-only',
             '--mount', f'type=volume,source={volume},target=/data',
             '--entrypoint', '/usr/local/bin/python3', self.args.linux_image,
             '-u', '-I', '-S', '-B', '/data/_inspect.py'],
        )
        _, stdout, _ = self.runner.docker(
            'run_stopped_reader', ['container', 'start', '--attach', reader], timeout=60,
        )
        value = load_json_bytes(stdout, 'stopped_reader')
        require(isinstance(value, dict), 'stopped_reader_object')
        return value, reader

    def normalize_evidence(self, phase, value):
        expected_keys = {
            'schema', 'phase', 'counts', 'claim_count', 'detectors',
            'candidate_services', 'secret_hashes', 'commit_hashes',
            'receipt_count', 'source_counts', 'planning_counts', 'capacity',
        }
        require(isinstance(value, dict) and set(value) == expected_keys, phase + '_evidence_shape')
        require(
            value['schema'] == 1 and value['phase'] == phase
            and value['claim_count'] == value['receipt_count'] == 4
            and value['detectors'] == ['OpenAI']
            and value['candidate_services'] == ['openai']
            and value['source_counts'] == {
                'gitlab': 2, 'dockerhub': 1, 'huggingface': 1,
            }
            and value['planning_counts'] == {
                'exact_git_v1': 2,
                'docker_direct_v1': 1,
                'huggingface_space_v1': 1,
            },
            phase + '_evidence_identity',
        )
        expected_secret_hashes = sorted(
            sha256_bytes(token.encode('ascii')) for token in self.tokens.values()
        )
        require(value['secret_hashes'] == expected_secret_hashes, phase + '_secret_hashes')
        require(
            value['commit_hashes'] == sorted(self.repository_heads.values()),
            phase + '_commit_hashes',
        )
        normalized = dict(value)
        normalized.pop('phase')
        return normalized

    def clear_run_artifacts(self):
        require(
            self.run_root.parent == self.root / 'build'
            and re.fullmatch(r'pwe-[a-f0-9]{16}', self.run_root.name)
            and self.run_root.is_dir() and not self.run_root.is_symlink(),
            'run_root_guard',
        )

        def retry_writable(function, name, _error):
            details = os.lstat(name)
            if stat.S_ISLNK(details.st_mode):
                function(name)
                return
            mode = stat.S_IRUSR | stat.S_IWUSR
            if stat.S_ISDIR(details.st_mode):
                mode |= stat.S_IXUSR
            os.chmod(name, mode)
            function(name)

        for path in self.run_root.iterdir():
            if path.is_symlink() or path.is_file():
                try:
                    path.unlink()
                except PermissionError:
                    require(not path.is_symlink(), 'run_artifact_symlink_permission')
                    os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
                    path.unlink()
            elif path.is_dir():
                shutil.rmtree(path, onerror=retry_writable)
            else:
                raise Failure('run_artifact_type')
        require(not any(self.run_root.iterdir()), 'run_artifact_cleanup')

    def safe_evidence(self, value):
        content = json.dumps(
            value, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
        ).encode('ascii') + b'\n'
        require(len(content) <= MAX_SAFE_EVIDENCE, 'safe_evidence_bound')
        for secret in (*self.tokens.values(), *self.device_tokens.values()):
            require(secret.encode('ascii') not in content, 'secret_present_in_evidence')
        return content

    def retain_safe_evidence(self, name, value):
        content = self.safe_evidence(value)
        self.clear_run_artifacts()
        write_bytes(self.run_root / name, content)

    def write_summary(self, windows, linux, cleanup):
        summary = {
            'schema': 1, 'status': 'passed', 'run_id': self.run_id,
            'parallelism': PARALLELISM, 'targets': list(ALL_TARGETS),
            'repository_heads': self.repository_heads,
            'manifest_sha256': self.manifest_hashes,
            'image_ids': self.image_details,
            'bundle_sha256': {
                'windows': windows['bundle_sha256'], 'linux': linux['bundle_sha256'],
            },
            'normalized_evidence_sha256': sha256_bytes(json.dumps(
                windows['evidence'], ensure_ascii=True, sort_keys=True,
                separators=(',', ':'),
            ).encode('ascii')),
            'foreign_docker_state': 'unchanged',
            'cleanup': cleanup,
        }
        self.retain_safe_evidence('summary.json', summary)
        return summary

    def run(self):
        self.preflight()
        windows = self.windows_phase()
        linux = self.linux_phase()
        require(windows['evidence'] == linux['evidence'], 'cross_platform_evidence_mismatch')
        cleanup = 'retained' if self.args.keep else 'complete'
        if not self.args.keep:
            self.tracker.cleanup()
        else:
            self.tracker.assert_foreign_unchanged()
        return self.write_summary(windows, linux, cleanup)


def parse_args(argv=None):
    parser = argparse.ArgumentParser(
        description='Verify real packaged Windows and Linux workers across an API outage.',
    )
    parser.add_argument('--windows-artifact', default=WINDOWS_ARTIFACT,
                        help='portable Windows worker directory (default: %(default)s)')
    parser.add_argument('--linux-image', default=LINUX_IMAGE,
                        help='already-built Linux worker image (default: %(default)s)')
    parser.add_argument('--test-image', default=TEST_IMAGE,
                        help='already-built server/test image (default: %(default)s)')
    parser.add_argument('--wsl-distro', default='Ubuntu-24.04',
                        help='WSL distribution that owns the Docker socket (default: %(default)s)')
    parser.add_argument('--timeout-seconds', type=int, default=1800,
                        help='aggregate subprocess and test timeout (default: %(default)s)')
    parser.add_argument('--keep', action='store_true',
                        help='retain labelled Docker resources after a successful run')
    args = parser.parse_args(argv)
    if not 300 <= args.timeout_seconds <= 7200:
        parser.error('--timeout-seconds must be between 300 and 7200')
    for name in ('linux_image', 'test_image', 'wsl_distro'):
        if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,255}', getattr(args, name)):
            parser.error('--' + name.replace('_', '-') + ' contains unsupported characters')
    return args


def main(argv=None):
    verifier = None
    try:
        verifier = Verifier(parse_args(argv))
        summary = verifier.run()
    except KeyboardInterrupt as exc:
        label = 'interrupted'
        failure_class = type(exc).__name__
        exit_code = None
    except Failure as exc:
        label = str(exc)
        failure_class = type(exc).__name__
        exit_code = getattr(exc, 'exit_code', None)
    except Exception as exc:
        label = 'unexpected_exception'
        failure_class = type(exc).__name__
        exit_code = None
    else:
        print(json.dumps({
            'status': summary['status'],
            'artifacts': os.fspath(verifier.run_root),
            'cleanup': summary['cleanup'],
        }, ensure_ascii=True, sort_keys=True))
        return 0
    finally:
        if verifier is not None and verifier.windows_client is not None:
            verifier.windows_client.stop()
        if verifier is not None and verifier.loopback_proxy is not None:
            verifier.loopback_proxy.stop()
    if verifier is not None:
        verifier.runner.deadline = time.monotonic() + 720
        verifier.tracker.guarded_stop()
    if not re.fullmatch(r'[a-z0-9_-]{1,160}', label):
        label = 'verifier_failure'
    if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,79}', failure_class):
        failure_class = 'Exception'
    if type(exit_code) is not int or not -(2 ** 31) <= exit_code < 2 ** 31:
        exit_code = None
    failure = {
        'stage': label, 'class': failure_class, 'exit_code': exit_code,
    }
    if verifier is not None:
        try:
            if verifier.args.keep:
                write_bytes(
                    verifier.run_root / 'failure.json',
                    verifier.safe_evidence(failure),
                )
            else:
                verifier.retain_safe_evidence('failure.json', failure)
        except Exception:
            pass
    print(json.dumps(failure, ensure_ascii=True, sort_keys=True), file=sys.stderr)
    return 1


if __name__ == '__main__':
    raise SystemExit(main())