sashatrask
910 lines
47 KiB
Python
910 lines
47 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the offline worker E2E against dedicated, already-built local images.
|
|
|
|
Run from Linux/WSL: python3 docker/verify.py [--keep]
|
|
Requires Docker with Compose v2 (directly or via sudo -n docker), Linux named
|
|
volumes, and Git. docker/test-results/latest.json must already be gitignored;
|
|
this script never changes ignore files. No image builds, pulls, host data
|
|
mounts, application edits, production Compose files, or broad cleanup.
|
|
|
|
Checks have a 3600-second aggregate budget; failure shutdown has a separate
|
|
720-second budget. Each health wait is at most 240 seconds, and each stop uses
|
|
600 seconds of grace. A forced/nonzero/OOM exit never counts as success.
|
|
Failures retain owned Docker artifacts after a guarded stop attempt. --keep
|
|
also retains them on success. Evidence contains only counts, hashes, image
|
|
IDs, fixed statuses, and durations; command logs are never printed or saved.
|
|
|
|
Contract limits: prepare is not repaired or rerun after recreation. A stopped
|
|
container loses its /run/truf tmpfs, so its shutdown receipt cannot be read
|
|
afterward. Receipt-write success is inferred ONLY from the healthy foreground
|
|
runtime's zero-exit contract; private PostgreSQL PID-file absence is separately
|
|
checked using the same runtime image and a read-only data-volume mount.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import selectors
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
PYTHON = ['/usr/local/bin/python3', '-I', '-S', '-B']
|
|
APP = '/opt/truf/app/container_runtime.py'
|
|
DRIVER = '/opt/truf/tests/container_e2e.py'
|
|
CONFIG = '/data/config/e2e.yaml'
|
|
ENTRYPOINT = ['/usr/bin/tini', '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B', APP]
|
|
HEALTH = ['CMD', *PYTHON, APP, 'health', '--config', CONFIG]
|
|
SERVICES = {'tools', 'provision', 'prepare', 'runtime', 'stopped'}
|
|
IMAGE_REFERENCES = {
|
|
'runtime': 'truf-worker-test:runtime',
|
|
'test': 'truf-worker-test:test',
|
|
}
|
|
TMPFS = {
|
|
'/run/truf': 'rw,nosuid,nodev,noexec,size=64m,mode=0700,uid=10001,gid=10001',
|
|
'/tmp': 'rw,nosuid,nodev,noexec,size=128m,mode=1777',
|
|
}
|
|
PROXY_ENV = {
|
|
name: '*' if name.lower() == 'no_proxy' else ''
|
|
for stem in ('http_proxy', 'https_proxy', 'ftp_proxy', 'all_proxy', 'no_proxy')
|
|
for name in (stem, stem.upper())
|
|
}
|
|
MOUNTS = {
|
|
'tools': {'/opt/truf/tests': ('tools', False)},
|
|
'provision': {'/data': ('data', False)},
|
|
'prepare': {'/data': ('data', False), '/opt/truf/tests': ('tools', True)},
|
|
'runtime': {'/data': ('data', False), '/opt/truf/tests': ('tools', True)},
|
|
'stopped': {'/data': ('data', True)},
|
|
}
|
|
LABEL = 'com.docker.compose.'
|
|
MAX_OUTPUT = 128 * 1024
|
|
MAX_DOCKER_RESOURCES = 1024
|
|
MAX_SNAPSHOT_BYTES = 4 * 1024 * 1024
|
|
HEX = re.compile(r'[a-f0-9]{64}')
|
|
|
|
CONTAINER_METADATA_FORMAT = (
|
|
'{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Image}},'
|
|
'"status":{{json .State.Status}},"running":{{json .State.Running}},'
|
|
'"paused":{{json .State.Paused}},"restarting":{{json .State.Restarting}},'
|
|
'"dead":{{json .State.Dead}},"mounts":{{json .Mounts}}}'
|
|
)
|
|
VOLUME_METADATA_FORMAT = (
|
|
'{"name":{{json .Name}},"driver":{{json .Driver}},"scope":{{json .Scope}},'
|
|
'"created":{{json .CreatedAt}},"mountpoint":{{json .Mountpoint}},'
|
|
'"labels":{{json .Labels}},"options":{{json .Options}}}'
|
|
)
|
|
|
|
INSPECT_FIELDS = {
|
|
'id': '.Id', 'name': '.Name', 'image': '.Image', 'status': '.State.Status',
|
|
'running': '.State.Running', 'pid': '.State.Pid',
|
|
'exit_code': '.State.ExitCode', 'oom_killed': '.State.OOMKilled',
|
|
'restarts': '.RestartCount', 'user': '.Config.User',
|
|
'entrypoint': '.Config.Entrypoint', 'command': '.Config.Cmd',
|
|
'stop_timeout': '.Config.StopTimeout',
|
|
'stop_signal': '.Config.StopSignal', 'mounts': '.Mounts',
|
|
'readonly': '.HostConfig.ReadonlyRootfs', 'network': '.HostConfig.NetworkMode',
|
|
'cap_drop': '.HostConfig.CapDrop', 'cap_add': '.HostConfig.CapAdd',
|
|
'security_opt': '.HostConfig.SecurityOpt', 'init': '.HostConfig.Init',
|
|
'privileged': '.HostConfig.Privileged', 'pid_mode': '.HostConfig.PidMode',
|
|
'ports': '.HostConfig.PortBindings', 'tmpfs': '.HostConfig.Tmpfs',
|
|
'cpus': '.HostConfig.NanoCpus', 'memory': '.HostConfig.Memory',
|
|
'pids_limit': '.HostConfig.PidsLimit', 'restart_policy': '.HostConfig.RestartPolicy',
|
|
**{key: '(index .Config.Labels "' + LABEL + suffix + '")' for key, suffix in (
|
|
('project', 'project'), ('service', 'service'), ('oneoff', 'oneoff'),
|
|
('config_files', 'project.config_files'), ('working_dir', 'project.working_dir'),
|
|
)},
|
|
}
|
|
# Do not inspect .State or .Config wholesale: health logs and environments can
|
|
# contain credentials. Only these selected fields enter the host process.
|
|
INSPECT_FORMAT = '{' + ','.join(
|
|
json.dumps(key) + ':{{json ' + value + '}}' for key, value in INSPECT_FIELDS.items()
|
|
) + (',"health_test":{{with index .Config "Healthcheck"}}{{json .Test}}{{else}}null{{end}}'
|
|
',"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}}}')
|
|
|
|
|
|
class Failure(Exception):
|
|
"""Only fixed check labels, never subprocess output or exception messages."""
|
|
|
|
|
|
def require(condition, label):
|
|
if not condition:
|
|
raise Failure(label)
|
|
|
|
|
|
class Verifier:
|
|
def __init__(self):
|
|
self.script = Path(__file__).absolute()
|
|
self.root = self.script.parent.parent.resolve(strict=True)
|
|
self.file = self.root / 'compose.e2e.yaml'
|
|
self.project = 'truf-worker-test-' + secrets.token_hex(16)
|
|
self.started = time.monotonic()
|
|
self.deadline = self.started + 3600
|
|
self.docker = []
|
|
self.compose = []
|
|
self.images = {}
|
|
self.owned_containers = {}
|
|
self.owned_volumes = {}
|
|
self.foreign_baseline = None
|
|
self.mutated = False
|
|
self.evidence_ready = False
|
|
self.file_hashes = {}
|
|
self.report = {
|
|
'status': {'result': 'running', 'cleanup': 'not_started'},
|
|
'counts': {'schema': 1},
|
|
'hashes': {'project_sha256': hashlib.sha256(self.project.encode('ascii')).hexdigest()},
|
|
'image_ids': self.images, 'durations': {},
|
|
}
|
|
allowed_env = (
|
|
'PATH', 'HOME', 'XDG_CONFIG_HOME', 'XDG_RUNTIME_DIR', 'SSH_AUTH_SOCK',
|
|
'DOCKER_HOST', 'DOCKER_CONTEXT', 'DOCKER_CONFIG', 'DOCKER_TLS_VERIFY',
|
|
'DOCKER_CERT_PATH',
|
|
)
|
|
self.env = {name: os.environ[name] for name in allowed_env if name in os.environ}
|
|
self.env['COMPOSE_DISABLE_ENV_FILE'] = '1'
|
|
|
|
def execute(self, label, args, *, timeout=30, capture=False, check=True):
|
|
started = time.monotonic()
|
|
end = min(self.deadline, started + timeout)
|
|
require(end > started, 'aggregate_timeout')
|
|
process = None
|
|
output = bytearray()
|
|
selector = selectors.DefaultSelector()
|
|
counts = self.report['counts'].setdefault(label, {})
|
|
counts['cli_calls'] = counts.get('cli_calls', 0) + 1
|
|
self.report['status'][label] = 'running'
|
|
try:
|
|
process = subprocess.Popen(
|
|
args, cwd=self.root, env=self.env, stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL, start_new_session=True,
|
|
)
|
|
if capture:
|
|
selector.register(process.stdout, selectors.EVENT_READ)
|
|
while process.poll() is None or selector.get_map():
|
|
remaining = end - time.monotonic()
|
|
require(remaining > 0, label + '_timeout')
|
|
if selector.get_map():
|
|
for key, _ in selector.select(min(0.1, remaining)):
|
|
chunk = os.read(key.fd, 65536)
|
|
if not chunk:
|
|
selector.unregister(key.fileobj)
|
|
else:
|
|
output.extend(chunk)
|
|
require(len(output) <= MAX_OUTPUT, label + '_output_limit')
|
|
else:
|
|
time.sleep(min(0.05, remaining))
|
|
code = process.returncode
|
|
counts['exit_code' if code >= 0 else 'signal'] = abs(code)
|
|
self.report['status'][label] = 'passed' if code == 0 else 'failed'
|
|
if check and code != 0:
|
|
error = Failure(label + '_command_failed')
|
|
error.exit_code = code
|
|
raise error
|
|
return code, bytes(output)
|
|
except OSError:
|
|
self.report['status'][label] = 'failed'
|
|
raise Failure(label + '_unavailable') from None
|
|
except BaseException:
|
|
self.report['status'][label] = 'failed'
|
|
raise
|
|
finally:
|
|
selector.close()
|
|
if process is not None:
|
|
if process.poll() is None:
|
|
# Terminate only the local CLI process group, not containers.
|
|
# An interrupted Docker API operation can outlive its client;
|
|
# failure handling discovers and stops owned containers.
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except (ProcessLookupError, PermissionError):
|
|
pass
|
|
try:
|
|
process.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
if process.stdout is not None:
|
|
process.stdout.close()
|
|
durations = self.report['durations']
|
|
durations[label] = round(durations.get(label, 0) + time.monotonic() - started, 3)
|
|
|
|
def json_command(self, label, args, timeout=30):
|
|
_, output = self.execute(label, args, timeout=timeout, capture=True)
|
|
try:
|
|
return json.loads(output)
|
|
except (ValueError, UnicodeError):
|
|
raise Failure(label + '_invalid_json') from None
|
|
|
|
def words(self, label, args):
|
|
_, output = self.execute(label, [*self.docker, *args], capture=True)
|
|
try:
|
|
words = output.decode('ascii').split()
|
|
except UnicodeError:
|
|
raise Failure(label + '_invalid_inventory') from None
|
|
require(len(words) <= 16, label + '_inventory_limit')
|
|
return set(words)
|
|
|
|
def metadata_names(self, kind):
|
|
options = (['--all', '--no-trunc', '--format', '{{.ID}}']
|
|
if kind == 'container' else ['--format', '{{.Name}}'])
|
|
_, output = self.execute(
|
|
'foreign_' + kind + '_list', [*self.docker, kind, 'ls', *options], capture=True,
|
|
)
|
|
try:
|
|
values = {line for line in output.decode('ascii').splitlines() if line}
|
|
except UnicodeError:
|
|
raise Failure('foreign_' + kind + '_inventory_encoding') from None
|
|
require(len(values) <= MAX_DOCKER_RESOURCES, 'foreign_' + kind + '_inventory_bound')
|
|
return values
|
|
|
|
def container_metadata(self, identifier):
|
|
require(HEX.fullmatch(identifier), 'foreign_container_id_guard')
|
|
value = self.json_command('foreign_container_inspect', [
|
|
*self.docker, 'container', 'inspect', '--format', CONTAINER_METADATA_FORMAT,
|
|
identifier,
|
|
])
|
|
require(
|
|
value.get('id') == identifier and isinstance(value.get('name'), str)
|
|
and HEX.fullmatch(str(value.get('image') or '').removeprefix('sha256:'))
|
|
and value.get('status') in (
|
|
'created', 'running', 'paused', 'restarting', 'removing', 'exited', 'dead',
|
|
)
|
|
and all(type(value.get(key)) is bool for key in (
|
|
'running', 'paused', 'restarting', 'dead',
|
|
))
|
|
and isinstance(value.get('mounts'), list) and len(value['mounts']) <= 128,
|
|
'foreign_container_metadata_guard',
|
|
)
|
|
mounts = [{
|
|
key: mount.get(key) for key in (
|
|
'Type', 'Name', 'Source', 'Destination', 'Driver', 'Mode', 'RW', 'Propagation',
|
|
)
|
|
} for mount in value['mounts']]
|
|
return {
|
|
'id': value['id'], 'name': value['name'], 'image': value['image'],
|
|
'status': value['status'], 'running': value['running'], 'paused': value['paused'],
|
|
'restarting': value['restarting'], 'dead': value['dead'],
|
|
'mounts_sha256': hashlib.sha256(json.dumps(
|
|
mounts, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest(),
|
|
}
|
|
|
|
def volume_metadata(self, name):
|
|
value = self.json_command('foreign_volume_inspect', [
|
|
*self.docker, 'volume', 'inspect', '--format', VOLUME_METADATA_FORMAT, name,
|
|
])
|
|
require(
|
|
value.get('name') == name and isinstance(value.get('driver'), str)
|
|
and isinstance(value.get('scope'), str) and isinstance(value.get('mountpoint'), str)
|
|
and (value.get('created') is None or isinstance(value['created'], str))
|
|
and (value.get('labels') is None or isinstance(value['labels'], dict))
|
|
and (value.get('options') is None or isinstance(value['options'], dict)),
|
|
'foreign_volume_metadata_guard',
|
|
)
|
|
return {
|
|
'name': name, 'driver': value['driver'], 'scope': value['scope'],
|
|
'created': value['created'],
|
|
'mountpoint_sha256': hashlib.sha256(value['mountpoint'].encode('utf-8')).hexdigest(),
|
|
'labels_sha256': hashlib.sha256(json.dumps(
|
|
value['labels'], ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest(),
|
|
'options_sha256': hashlib.sha256(json.dumps(
|
|
value['options'], ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest(),
|
|
}
|
|
|
|
def metadata_snapshot(self, *, exclude_owned=False):
|
|
containers = {}
|
|
for identifier in sorted(self.metadata_names('container')):
|
|
if exclude_owned and identifier in self.owned_containers:
|
|
self.inspect(identifier, 'foreign_owned_exclusion_guard')
|
|
continue
|
|
containers[identifier] = self.container_metadata(identifier)
|
|
volumes = {}
|
|
for name in sorted(self.metadata_names('volume')):
|
|
value = self.volume_metadata(name)
|
|
if exclude_owned and self.owned_volumes.get(name) == value:
|
|
continue
|
|
volumes[name] = value
|
|
snapshot = {'containers': containers, 'volumes': volumes}
|
|
require(len(json.dumps(snapshot, ensure_ascii=True, sort_keys=True)) <= MAX_SNAPSHOT_BYTES,
|
|
'foreign_metadata_snapshot_bound')
|
|
return snapshot
|
|
|
|
def snapshot_foreign(self):
|
|
require(self.foreign_baseline is None, 'foreign_snapshot_already_taken')
|
|
self.foreign_baseline = self.metadata_snapshot()
|
|
|
|
def assert_foreign_unchanged(self):
|
|
require(self.foreign_baseline is not None, 'foreign_snapshot_missing')
|
|
require(self.metadata_snapshot(exclude_owned=True) == self.foreign_baseline,
|
|
'foreign_docker_state_changed')
|
|
self.report['status']['foreign_docker_state'] = 'unchanged'
|
|
|
|
def guard_files(self):
|
|
require(re.fullmatch(r'truf-worker-test-[a-f0-9]{32}', self.project),
|
|
'worker_test_project_guard')
|
|
require(self.script == self.root / 'docker' / 'verify.py', 'verifier_path_guard')
|
|
require(self.file == self.root / 'compose.e2e.yaml', 'compose_path_guard')
|
|
for path in (self.script, self.file):
|
|
require(path.resolve(strict=True) == path and path.is_file() and not path.is_symlink(),
|
|
'canonical_file_required')
|
|
require(path.stat().st_size <= 256 * 1024, 'source_file_size_bound')
|
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
require(path not in self.file_hashes or self.file_hashes[path] == digest,
|
|
'source_changed_during_verification')
|
|
self.file_hashes[path] = digest
|
|
|
|
def compose_command(self, label, args, **kwargs):
|
|
self.guard_files()
|
|
return self.execute(label, [*self.compose, *args], **kwargs)
|
|
|
|
def inspect(self, container, label='inspect', timeout=30):
|
|
require(HEX.fullmatch(container), 'container_id_guard')
|
|
value = self.json_command(label, [
|
|
*self.docker, 'container', 'inspect', '--format', INSPECT_FORMAT, container,
|
|
], timeout=timeout)
|
|
require(value['id'] == container and value['project'] == self.project
|
|
and isinstance(value['name'], str)
|
|
and value['name'].startswith('/' + self.project + '-')
|
|
and value['service'] in SERVICES
|
|
and value['config_files'] == str(self.file)
|
|
and value['working_dir'] == str(self.root), 'container_ownership_guard')
|
|
service = value['service']
|
|
require(value['oneoff'] == ('False' if service == 'runtime' else 'True'),
|
|
'container_role_guard')
|
|
require(value['image'] == self.images['test' if service == 'tools' else 'runtime'],
|
|
'container_image_guard')
|
|
require(value['status'] in ('created', 'running', 'paused', 'restarting', 'removing', 'exited', 'dead'),
|
|
'container_state_guard')
|
|
require(all(type(value[key]) is int and value[key] >= 0 for key in ('exit_code', 'restarts', 'pid'))
|
|
and all(type(value[key]) is bool for key in ('oom_killed', 'running')),
|
|
'container_state_types_guard')
|
|
identity = {key: value[key] for key in (
|
|
'id', 'name', 'image', 'project', 'service', 'config_files', 'working_dir',
|
|
)}
|
|
require(self.owned_containers.setdefault(container, identity) == identity,
|
|
'container_identity_changed')
|
|
self.report['status']['container_' + service] = value['status']
|
|
self.report['counts']['container_' + service] = {
|
|
'exit_code': value['exit_code'], 'oom_killed': int(value['oom_killed']),
|
|
'restarts': value['restarts'], 'running': int(value['running']),
|
|
}
|
|
return value
|
|
|
|
def inventory(self, *, complete=False):
|
|
self.guard_files()
|
|
found = {}
|
|
for kind in ('container', 'volume', 'network'):
|
|
options = ['--all', '--quiet', '--no-trunc'] if kind == 'container' else ['--format', '{{.Name}}']
|
|
named = self.words('inventory_' + kind, [
|
|
kind, 'ls', *options, '--filter', 'name=' + self.project,
|
|
])
|
|
labeled = self.words('ownership_' + kind, [
|
|
kind, 'ls', *options, '--filter', 'label=' + LABEL + 'project=' + self.project,
|
|
])
|
|
require(named == labeled, 'resource_ownership_guard')
|
|
found[kind] = named
|
|
require(not found['network'], 'unexpected_project_network')
|
|
expected = {self.project + '_data', self.project + '_tools'}
|
|
require(found['volume'] <= expected, 'volume_name_guard')
|
|
if complete:
|
|
require(found['volume'] == expected, 'required_volumes_missing')
|
|
for volume in sorted(found['volume']):
|
|
value = self.json_command('inspect_volume', [
|
|
*self.docker, 'volume', 'inspect', '--format',
|
|
'{"name":{{json .Name}},"driver":{{json .Driver}},"options":{{json .Options}},'
|
|
'"scope":{{json .Scope}},"project":{{json (index .Labels "com.docker.compose.project")}},'
|
|
'"volume":{{json (index .Labels "com.docker.compose.volume")}}}', volume,
|
|
])
|
|
require(value['name'] == volume and value['project'] == self.project
|
|
and value['volume'] in ('data', 'tools')
|
|
and volume == self.project + '_' + value['volume']
|
|
and value['driver'] == 'local' and not value['options']
|
|
and value['scope'] == 'local', 'native_named_volume_guard')
|
|
metadata = self.volume_metadata(volume)
|
|
require(self.owned_volumes.setdefault(volume, metadata) == metadata,
|
|
'volume_identity_changed')
|
|
users = self.words('volume_users', [
|
|
'container', 'ls', '--all', '--quiet', '--no-trunc', '--filter', 'volume=' + volume,
|
|
])
|
|
require(users <= found['container'], 'foreign_volume_user_guard')
|
|
self.report['counts']['owned_resources'] = {
|
|
kind + 's': len(names) for kind, names in found.items()
|
|
}
|
|
return [self.inspect(container) for container in sorted(found['container'])]
|
|
|
|
def validate_compose(self, value):
|
|
require(value.get('name') == self.project, 'worker_test_project_guard')
|
|
require(set(value['services']) == SERVICES and set(value['volumes']) == {'data', 'tools'}
|
|
and not any(value.get(key) for key in ('networks', 'secrets', 'configs')),
|
|
'compose_project_contract')
|
|
for name, volume in value['volumes'].items():
|
|
require(volume.get('name') == self.project + '_' + name,
|
|
'production_volume_forbidden')
|
|
require(
|
|
volume.get('name') not in {'truf-docker_data', 'truf-docker_tools'}
|
|
and volume.get('driver') == 'local'
|
|
and set(volume) <= {'name', 'driver'}, 'compose_volume_contract')
|
|
allowed = {
|
|
'image', 'pull_policy', 'read_only', 'user', 'cap_drop', 'cap_add', 'security_opt',
|
|
'network_mode', 'volumes', 'tmpfs', 'cpus', 'mem_limit', 'pids_limit', 'shm_size',
|
|
'stop_signal', 'stop_grace_period', 'logging', 'restart', 'entrypoint', 'command',
|
|
'healthcheck', 'environment', 'init', 'ports',
|
|
}
|
|
for name, service in value['services'].items():
|
|
require(set(service) <= allowed, 'compose_service_options_guard')
|
|
require(service['image'] == self.images['test' if name == 'tools' else 'runtime'],
|
|
'worker_test_image_reference_guard')
|
|
require(not service.get('ports'), 'published_port_contract')
|
|
require(
|
|
service.get('network_mode') == 'none', 'internal_network_contract')
|
|
require(
|
|
service['image'] == self.images['test' if name == 'tools' else 'runtime']
|
|
and service.get('pull_policy') == 'never'
|
|
and service.get('read_only') is True
|
|
and service.get('environment') == PROXY_ENV
|
|
and service.get('init') is False
|
|
and service.get('user') == ('0:0' if name == 'provision' else '10001:10001')
|
|
and set(service.get('cap_drop', ())) == {'ALL'}
|
|
and set(service.get('cap_add', ())) == (
|
|
{'CHOWN', 'DAC_OVERRIDE', 'FOWNER'} if name == 'provision' else set())
|
|
and service.get('security_opt') == ['no-new-privileges:true']
|
|
and service.get('restart') == 'no'
|
|
and float(service['cpus']) == 2
|
|
and int(service['mem_limit']) == 6 * 1024 ** 3
|
|
and int(service['pids_limit']) == 512
|
|
and int(service['shm_size']) == 256 * 1024 ** 2
|
|
and service['stop_signal'] == 'SIGTERM'
|
|
and service['logging'] == {
|
|
'driver': 'json-file', 'options': {'max-size': '16m', 'max-file': '4'},
|
|
}
|
|
and set(service['tmpfs']) == {key + ':' + val for key, val in TMPFS.items()},
|
|
'compose_isolation_contract')
|
|
mounts = {}
|
|
for mount in service['volumes']:
|
|
require(mount.get('type') == 'volume', 'bind_mount_forbidden')
|
|
require(set(mount) <= {'type', 'source', 'target', 'read_only', 'volume'}
|
|
and set(mount.get('volume', {})) <= {'nocopy'}, 'compose_mount_guard')
|
|
require(mount['target'] not in mounts, 'duplicate_mount_guard')
|
|
mounts[mount['target']] = (mount['source'], mount.get('read_only', False))
|
|
if mount['source'] == 'tools':
|
|
require(mount.get('volume', {}).get('nocopy', False) is (name != 'tools'),
|
|
'tools_copy_up_contract')
|
|
require(mounts == MOUNTS[name], 'compose_mount_contract')
|
|
runtime = value['services']['runtime']
|
|
require(runtime.get('entrypoint') is None and runtime['command'] == ['run', '--config', CONFIG]
|
|
and runtime['healthcheck']['test'] == HEALTH, 'production_entrypoint_contract')
|
|
require(value['services']['provision'].get('entrypoint') is None
|
|
and value['services']['provision']['command'] == ['provision']
|
|
and value['services']['prepare']['entrypoint'] == [*PYTHON, DRIVER]
|
|
and value['services']['prepare']['command'] == ['prepare', '--config', CONFIG],
|
|
'prepare_command_contract')
|
|
|
|
def preflight(self, env_file):
|
|
self.guard_files()
|
|
for path, digest in self.file_hashes.items():
|
|
self.report['hashes']['compose_sha256' if path == self.file else 'verifier_sha256'] = digest
|
|
docker = shutil.which('docker')
|
|
require(docker, 'docker_cli_required')
|
|
candidates = [[docker]]
|
|
sudo = shutil.which('sudo')
|
|
if sudo:
|
|
candidates.append([sudo, '-n', docker])
|
|
for index, candidate in enumerate(candidates):
|
|
try:
|
|
code, output = self.execute('docker_probe_' + str(index), [
|
|
*candidate, 'info', '--format', '{{json .OSType}}',
|
|
], timeout=15, capture=True, check=False)
|
|
except Failure:
|
|
continue
|
|
if code == 0:
|
|
require(json.loads(output) == 'linux', 'linux_docker_daemon_required')
|
|
self.docker = candidate
|
|
break
|
|
require(self.docker, 'docker_or_passwordless_sudo_required')
|
|
self.execute('compose_available', [*self.docker, 'compose', 'version', '--short'])
|
|
self.snapshot_foreign()
|
|
for name in ('runtime', 'test'):
|
|
value = self.json_command('image_' + name, [
|
|
*self.docker, 'image', 'inspect', '--format',
|
|
'{"id":{{json .Id}},"os":{{json .Os}},"user":{{json .Config.User}},'
|
|
'"entrypoint":{{json .Config.Entrypoint}},"volumes":{{json (index .Config "Volumes")}}}',
|
|
IMAGE_REFERENCES[name],
|
|
])
|
|
require(re.fullmatch(r'sha256:[a-f0-9]{64}', value['id'])
|
|
and value['os'] == 'linux' and value['user'] == '10001:10001'
|
|
and not value['volumes'], 'prebuilt_image_contract')
|
|
if name == 'runtime':
|
|
require(value['entrypoint'] == ENTRYPOINT, 'runtime_image_entrypoint_contract')
|
|
self.images[name] = value['id']
|
|
require(self.images['runtime'] != self.images['test'], 'distinct_image_targets_required')
|
|
env_file.write('TRUF_WORKER_TEST_PROJECT=' + self.project + '\n'
|
|
'TRUF_WORKER_TEST_RUNTIME_IMAGE=' + self.images['runtime'] + '\n'
|
|
'TRUF_WORKER_TEST_TEST_IMAGE=' + self.images['test'] + '\n')
|
|
env_file.flush()
|
|
# An explicit env file works with sudo's environment reset, too. Never
|
|
# load the checkout's .env or accept COMPOSE_FILE/PROJECT_NAME overrides.
|
|
self.compose = [
|
|
*self.docker, 'compose', '--ansi', 'never', '--project-name', self.project,
|
|
'--project-directory', str(self.root), '--env-file', env_file.name,
|
|
'--file', str(self.file),
|
|
]
|
|
self.validate_compose(self.json_command('compose_contract', [
|
|
*self.compose, 'config', '--format', 'json',
|
|
]))
|
|
git = shutil.which('git')
|
|
require(git, 'git_required_for_evidence_ignore_guard')
|
|
code, _ = self.execute('evidence_ignore_guard', [
|
|
git, 'check-ignore', '--quiet', '--no-index', '--', 'docker/test-results/latest.json',
|
|
], check=False)
|
|
require(code == 0, 'evidence_path_must_be_gitignored')
|
|
require(not self.inventory(), 'fresh_project_required')
|
|
require(not self.words('fresh_volumes', [
|
|
'volume', 'ls', '--format', '{{.Name}}', '--filter', 'name=' + self.project,
|
|
]), 'fresh_volumes_required')
|
|
directory = self.root / 'docker' / 'test-results'
|
|
require(not directory.is_symlink(), 'evidence_directory_guard')
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
require(directory.resolve(strict=True) == directory and directory.is_dir(),
|
|
'evidence_directory_guard')
|
|
self.evidence_ready = True
|
|
|
|
def summary(self, label, args, timeout):
|
|
print(label + ': running', flush=True)
|
|
code, output = self.execute(label, args, timeout=timeout, capture=True, check=False)
|
|
try:
|
|
value = json.loads(output)
|
|
except (ValueError, UnicodeError):
|
|
raise Failure(label + '_invalid_summary') from None
|
|
require(isinstance(value, dict) and set(value) == {'counts', 'hashes'}
|
|
and all(isinstance(value[key], dict) and len(value[key]) <= 128 for key in value),
|
|
label + '_invalid_summary')
|
|
require(all(re.fullmatch(r'[a-z][a-z0-9_:]{0,120}', key)
|
|
and type(count) is int and 0 <= count <= 2 ** 63 - 1
|
|
for key, count in value['counts'].items())
|
|
and all(re.fullmatch(r'[a-z][a-z0-9_:]{0,120}_sha256', key)
|
|
and isinstance(digest, str) and HEX.fullmatch(digest)
|
|
for key, digest in value['hashes'].items()), label + '_invalid_summary')
|
|
self.report['counts'][label].update(value['counts'])
|
|
self.report['hashes'][label] = value['hashes']
|
|
require(code == 0 and value['counts'].get('ok') == 1, label + '_check_failed')
|
|
return value
|
|
|
|
def oneoff(self, service, label=None, timeout=180):
|
|
self.guard_files()
|
|
args = [*self.compose, 'run', '--rm', '--no-deps', '--pull', 'never', '-T', service]
|
|
if service == 'provision':
|
|
print('provision: running', flush=True)
|
|
self.execute('provision', args, timeout=timeout)
|
|
return None
|
|
return self.summary(label or service, args, timeout)
|
|
|
|
def start(self, label, previous=None):
|
|
if previous:
|
|
value = self.inspect(previous, label + '_previous_ownership')
|
|
require(value['status'] == 'exited' and not value['running'],
|
|
label + '_previous_not_stopped')
|
|
self.inspect(previous, label + '_previous_remove_guard')
|
|
self.execute(label + '_remove_previous', [
|
|
*self.docker, 'container', 'rm', previous,
|
|
])
|
|
self.compose_command(label, [
|
|
'up', '--detach', '--no-deps', '--no-build', '--pull', 'never',
|
|
'runtime',
|
|
], timeout=120)
|
|
containers = self.inventory(complete=True)
|
|
runtimes = [value for value in containers if value['service'] == 'runtime']
|
|
require(len(runtimes) == 1, 'single_runtime_required')
|
|
value = runtimes[0]
|
|
container = value['id']
|
|
require(container != previous, 'new_runtime_container_required')
|
|
require(value['entrypoint'] == ENTRYPOINT and value['command'] == ['run', '--config', CONFIG]
|
|
and value['health_test'] == HEALTH and value['user'] == '10001:10001'
|
|
and value['readonly'] is True and value['network'] == 'none'
|
|
and set(value['cap_drop'] or ()) == {'ALL'} and not value['cap_add']
|
|
and value['security_opt'] == ['no-new-privileges:true']
|
|
and not value['init'] and not value['privileged'] and not value['pid_mode']
|
|
and not value['ports'] and value['tmpfs'] == TMPFS
|
|
and value['cpus'] == 2_000_000_000 and value['memory'] == 6 * 1024 ** 3
|
|
and value['pids_limit'] == 512 and value['stop_timeout'] == 600
|
|
and value['stop_signal'] == 'SIGTERM'
|
|
and value['restart_policy'] == {'Name': 'no', 'MaximumRetryCount': 0},
|
|
'actual_runtime_isolation_contract')
|
|
mounts = {}
|
|
for mount in value['mounts']:
|
|
if mount['Type'] == 'tmpfs':
|
|
require(mount['Destination'] in TMPFS, 'unexpected_tmpfs')
|
|
continue
|
|
require(mount['Type'] == 'volume' and mount['Destination'] not in mounts,
|
|
'actual_named_mount_required')
|
|
mounts[mount['Destination']] = (mount['Name'], not mount['RW'])
|
|
require(mounts == {target: (self.project + '_' + name, ro)
|
|
for target, (name, ro) in MOUNTS['runtime'].items()},
|
|
'actual_runtime_mount_contract')
|
|
self.report['hashes'][label + '_container_sha256'] = hashlib.sha256(container.encode('ascii')).hexdigest()
|
|
health_started = time.monotonic()
|
|
health_end = min(self.deadline, health_started + 240)
|
|
while time.monotonic() < health_end:
|
|
value = self.inspect(container, label + '_health', timeout=min(10, health_end - time.monotonic()))
|
|
require(value['running'] and value['status'] == 'running'
|
|
and not value['oom_killed'] and value['restarts'] == 0,
|
|
label + '_runtime_exited_or_restarted')
|
|
if value['health'] == 'healthy':
|
|
self.report['durations'][label + '_ready'] = round(time.monotonic() - health_started, 3)
|
|
return container
|
|
time.sleep(min(2, max(0, health_end - time.monotonic())))
|
|
raise Failure(label + '_health_timeout')
|
|
|
|
def driver(self, container, mode, label, timeout=240):
|
|
self.inspect(container)
|
|
return self.summary(label, [
|
|
*self.docker, 'exec', '--user', '10001:10001', container, *PYTHON,
|
|
DRIVER, mode, '--config', CONFIG, '--timeout', '180',
|
|
], timeout)
|
|
|
|
def health(self, container, label):
|
|
self.inspect(container)
|
|
value = self.json_command(label, [
|
|
*self.docker, 'exec', '--user', '10001:10001', container, *PYTHON,
|
|
APP, 'health', '--config', CONFIG,
|
|
], timeout=30)
|
|
require(value == {
|
|
'healthy': True, 'activation_state': 'ACTIVE', 'postgres': 'READY',
|
|
'workers': ['gitlab', 'janitor', 'jsonl-projector', 'result-ingester'],
|
|
}, label + '_authenticated_health_failed')
|
|
self.report['counts'][label]['authenticated_health'] = 1
|
|
|
|
def stop(self, container, label):
|
|
containers = self.inventory(complete=True)
|
|
require(any(value['id'] == container and value['running'] for value in containers),
|
|
label + '_runtime_not_running')
|
|
print(label + ': stopping (600-second grace)', flush=True)
|
|
self.inspect(container, label + '_ownership_guard')
|
|
self.execute(label, [
|
|
*self.docker, 'container', 'stop', '--time', '600', container,
|
|
], timeout=660)
|
|
value = self.inspect(container, label + '_inspect')
|
|
self.report['counts'][label].update({
|
|
'container_exit_code': value['exit_code'], 'oom_killed': int(value['oom_killed']),
|
|
'restarts': value['restarts'],
|
|
})
|
|
require(value['status'] == 'exited' and not value['running'] and value['pid'] == 0
|
|
and value['exit_code'] == 0 and value['oom_killed'] is False
|
|
and value['restarts'] == 0, label + '_unclean_exit')
|
|
self.oneoff('stopped', label + '_data')
|
|
# Foreground supervisor.main returns zero only after receipt publication.
|
|
# Do not claim to have read that receipt from a destroyed tmpfs.
|
|
self.report['status'][label + '_receipt'] = 'exit_contract_only_tmpfs_removed'
|
|
|
|
def cleanup(self, keep):
|
|
containers = self.inventory(complete=True)
|
|
require(len(containers) == 1 and containers[0]['service'] == 'runtime'
|
|
and containers[0]['status'] == 'exited' and containers[0]['exit_code'] == 0
|
|
and not containers[0]['oom_killed'], 'cleanup_stopped_runtime_guard')
|
|
if keep:
|
|
self.assert_foreign_unchanged()
|
|
self.report['status']['cleanup'] = 'kept'
|
|
return
|
|
self.report['status']['cleanup'] = 'running'
|
|
container = containers[0]['id']
|
|
self.inspect(container, 'cleanup_container_remove_guard')
|
|
self.execute('remove_owned_container', [*self.docker, 'container', 'rm', container])
|
|
for volume in sorted(self.owned_volumes):
|
|
require(self.volume_metadata(volume) == self.owned_volumes[volume],
|
|
'cleanup_volume_identity_changed')
|
|
self.execute('remove_owned_volume', [*self.docker, 'volume', 'rm', volume])
|
|
require(not self.inventory(), 'cleanup_containers_remaining')
|
|
require(not self.words('cleanup_volumes', [
|
|
'volume', 'ls', '--format', '{{.Name}}', '--filter', 'name=' + self.project,
|
|
]), 'cleanup_volumes_remaining')
|
|
self.assert_foreign_unchanged()
|
|
self.report['status']['cleanup'] = 'removed'
|
|
|
|
def failure_stop(self):
|
|
self.deadline = time.monotonic() + 720
|
|
self.report['status']['cleanup'] = 'retained_after_failure'
|
|
try:
|
|
containers = self.inventory()
|
|
active = [value for value in containers
|
|
if value['running'] or value['status'] in ('restarting', 'paused')]
|
|
if active:
|
|
print('failure_stop: stopping owned containers (600-second grace)', flush=True)
|
|
for value in active:
|
|
container = value['id']
|
|
self.inspect(container, 'failure_stop_ownership_guard')
|
|
self.execute('failure_stop', [
|
|
*self.docker, 'container', 'stop', '--time', '600', container,
|
|
], timeout=660)
|
|
remaining = self.inventory()
|
|
self.report['counts']['failure_retained'] = {
|
|
'containers': len(remaining), 'running': sum(int(value['running']) for value in remaining),
|
|
}
|
|
self.report['status']['failure_stop'] = (
|
|
'incomplete' if any(value['running'] for value in remaining) else 'observed_stopped'
|
|
)
|
|
except (Exception, KeyboardInterrupt):
|
|
# Loss of the daemon, changed files, or unproven ownership must never
|
|
# lead to an unguarded down/prune/kill attempt or a false success.
|
|
self.report['status']['failure_stop'] = 'failed_or_ownership_unproven'
|
|
|
|
def write_evidence(self):
|
|
self.report['durations']['total'] = round(time.monotonic() - self.started, 3)
|
|
directory = self.root / 'docker' / 'test-results'
|
|
require(directory.resolve(strict=True) == directory, 'evidence_directory_guard')
|
|
descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
temporary = '.' + self.project + '.json'
|
|
try:
|
|
try:
|
|
details = os.stat('latest.json', dir_fd=descriptor, follow_symlinks=False)
|
|
require(stat.S_ISREG(details.st_mode), 'evidence_file_guard')
|
|
except FileNotFoundError:
|
|
pass
|
|
output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
0o600, dir_fd=descriptor)
|
|
with os.fdopen(output, 'w', encoding='ascii') as handle:
|
|
json.dump(self.report, handle, sort_keys=True, indent=2, allow_nan=False)
|
|
handle.write('\n')
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, 'latest.json', src_dir_fd=descriptor, dst_dir_fd=descriptor)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
try:
|
|
os.unlink(temporary, dir_fd=descriptor)
|
|
except FileNotFoundError:
|
|
pass
|
|
os.close(descriptor)
|
|
|
|
|
|
def main(argv=None):
|
|
class Parser(argparse.ArgumentParser):
|
|
def error(self, message):
|
|
raise Failure('invalid_arguments')
|
|
|
|
parser = Parser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False)
|
|
parser.add_argument('--keep', action='store_true', help='retain stopped, owned Docker artifacts on success too')
|
|
args = parser.parse_args(argv)
|
|
require(sys.platform == 'linux', 'linux_host_required_use_wsl_python3')
|
|
verifier = Verifier()
|
|
print('E2E project: ' + verifier.project, flush=True)
|
|
|
|
def interrupted(_signum, _frame):
|
|
raise KeyboardInterrupt
|
|
|
|
previous = signal.signal(signal.SIGTERM, interrupted)
|
|
# This private file contains only the project nonce and image IDs.
|
|
with tempfile.NamedTemporaryFile(mode='w', encoding='ascii', prefix=verifier.project + '-', suffix='.env') as env_file:
|
|
try:
|
|
verifier.preflight(env_file)
|
|
verifier.write_evidence()
|
|
verifier.mutated = True
|
|
verifier.oneoff('tools')
|
|
verifier.oneoff('provision')
|
|
verifier.oneoff('prepare')
|
|
first = verifier.start('first_start')
|
|
verifier.driver(first, 'run-local-pipeline', 'local_pipeline', timeout=240)
|
|
verifier.driver(first, 'assert-pipeline', 'pipeline')
|
|
baseline = verifier.driver(first, 'keycheck-fixture', 'first_keycheck', timeout=510)
|
|
require(baseline['counts'].pop('http_requests', None) == 1, 'first_provider_request_count')
|
|
verifier.health(first, 'first_authenticated_health')
|
|
verifier.stop(first, 'first_stop')
|
|
second = verifier.start('second_start', previous=first)
|
|
persisted = verifier.driver(second, 'assert-persisted', 'persisted')
|
|
require(persisted == baseline, 'persisted_public_summary_changed')
|
|
checked = verifier.driver(second, 'keycheck-fixture', 'second_keycheck', timeout=510)
|
|
require(checked['counts'].pop('http_requests', None) == 0, 'repeated_provider_made_http_requests')
|
|
require(checked == baseline, 'repeated_provider_changed_public_summary')
|
|
verifier.health(second, 'second_authenticated_health')
|
|
verifier.driver(second, 'assert-remote-recovery', 'remote_recovery')
|
|
verifier.driver(second, 'prepare-remote-transport', 'remote_transport')
|
|
dockerhub_canary_prepare = verifier.driver(
|
|
second, 'prepare-dockerhub-canary',
|
|
'dockerhub_canary_prepare', timeout=510,
|
|
)
|
|
require(
|
|
dockerhub_canary_prepare['counts'].get('search_pages') == 1
|
|
and dockerhub_canary_prepare['counts'].get('cohort_targets') == 4
|
|
and dockerhub_canary_prepare['counts'].get('digest_resolutions') == 4
|
|
and dockerhub_canary_prepare['counts'].get('worker_provider_failures') == 2
|
|
and dockerhub_canary_prepare['counts'].get('accepted_uploads') == 3
|
|
and dockerhub_canary_prepare['counts'].get('expired_assignments') == 1
|
|
and dockerhub_canary_prepare['counts'].get('drain_cycles') == 2
|
|
and dockerhub_canary_prepare['counts'].get('pending_targets') == 1,
|
|
'dockerhub_canary_prepare_evidence',
|
|
)
|
|
remote_full_prepare = verifier.driver(
|
|
second, 'prepare-remote-full-race', 'remote_full_prepare', timeout=510,
|
|
)
|
|
require(remote_full_prepare['counts'].get('real_claims') == 2
|
|
and remote_full_prepare['counts'].get('native_scans') == 3
|
|
and remote_full_prepare['counts'].get('exact_expiries') == 1
|
|
and remote_full_prepare['counts'].get('pending_restart') == 1,
|
|
'remote_full_prepare_evidence')
|
|
verifier.stop(second, 'second_stop')
|
|
third = verifier.start('third_start', previous=second)
|
|
remote_full_finish = verifier.driver(
|
|
third, 'finish-remote-full-race', 'remote_full_finish', timeout=510,
|
|
)
|
|
require(remote_full_finish['counts'].get('concurrent_uploads') == 2
|
|
and remote_full_finish['counts'].get('authoritative_receipts') == 1
|
|
and remote_full_finish['counts'].get('authoritative_scans') == 1
|
|
and remote_full_finish['counts'].get('expired_losers') == 1
|
|
and remote_full_finish['counts'].get('completed_winners') == 1
|
|
and remote_full_finish['counts'].get('cached_keychecks') == 1
|
|
and remote_full_finish['counts'].get('provider_http_requests') == 0
|
|
and remote_full_finish['counts'].get('projection_streams') == 3,
|
|
'remote_full_finish_evidence')
|
|
verifier.driver(
|
|
third, 'assert-remote-transport-replay', 'remote_transport_replay',
|
|
)
|
|
dockerhub_canary_finish = verifier.driver(
|
|
third, 'finish-dockerhub-canary',
|
|
'dockerhub_canary_finish', timeout=510,
|
|
)
|
|
require(
|
|
dockerhub_canary_finish['counts'].get('runtime_restarts') == 1
|
|
and dockerhub_canary_finish['counts'].get('lost_claim_receipts') == 1
|
|
and dockerhub_canary_finish['counts'].get('receipt_replays') == 1
|
|
and dockerhub_canary_finish['counts'].get('conflicts_rejected') == 1
|
|
and dockerhub_canary_finish['counts'].get('exactly_once') == 1
|
|
and dockerhub_canary_finish['counts'].get('former_expiry_replays') == 1,
|
|
'dockerhub_canary_finish_evidence',
|
|
)
|
|
verifier.health(third, 'third_authenticated_health')
|
|
verifier.stop(third, 'third_stop')
|
|
verifier.report['status']['checks'] = 'passed'
|
|
# Persist the check results before deleting their Docker artifacts.
|
|
verifier.write_evidence()
|
|
verifier.cleanup(args.keep)
|
|
verifier.report['status']['result'] = 'passed'
|
|
except (Exception, KeyboardInterrupt) as exc:
|
|
verifier.report['status']['result'] = 'failed'
|
|
label = str(exc) if isinstance(exc, Failure) else (
|
|
'interrupted' if isinstance(exc, KeyboardInterrupt) else 'verifier_exception'
|
|
)
|
|
failure_class = type(exc).__name__
|
|
if not re.fullmatch(r'[a-z0-9_]{1,160}', label):
|
|
label = 'verifier_failure'
|
|
if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,79}', failure_class):
|
|
failure_class = 'Exception'
|
|
exit_code = getattr(exc, 'exit_code', None)
|
|
if type(exit_code) is not int or not -(2 ** 31) <= exit_code < 2 ** 31:
|
|
exit_code = None
|
|
verifier.report['failure'] = {
|
|
'stage': label, 'class': failure_class,
|
|
'exit_code': exit_code,
|
|
}
|
|
print('E2E failed: ' + label, flush=True)
|
|
if verifier.mutated:
|
|
verifier.failure_stop()
|
|
finally:
|
|
signal.signal(signal.SIGTERM, previous)
|
|
if verifier.evidence_ready:
|
|
try:
|
|
verifier.write_evidence()
|
|
except (Exception, KeyboardInterrupt):
|
|
verifier.report['status']['result'] = 'failed'
|
|
print('E2E failed: evidence_write_failed', flush=True)
|
|
else:
|
|
print('Evidence: docker/test-results/latest.json', flush=True)
|
|
print('E2E ' + verifier.report['status']['result'] + '; project ' + verifier.project
|
|
+ '; artifacts ' + verifier.report['status']['cleanup'], flush=True)
|
|
return 0 if verifier.report['status']['result'] == 'passed' else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except (Exception, KeyboardInterrupt) as exc:
|
|
print('E2E failed: ' + (str(exc) if isinstance(exc, Failure) else 'verifier_exception'), flush=True)
|
|
raise SystemExit(1) from None
|