sashatrask
1478 lines
57 KiB
Python
1478 lines
57 KiB
Python
"""Fixed privileged lifecycle for the managed runtime deployment."""
|
|
|
|
from dataclasses import dataclass
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
from host_agent_apply import _new_stopped_runtime_proof
|
|
from host_agent_state import HostOperationState, HostStateError
|
|
|
|
|
|
DOCKER = '/usr/bin/docker'
|
|
PROJECT = 'truf-docker'
|
|
PROJECT_DIRECTORY = '/opt/truf'
|
|
COMPOSE_FILES = ('/opt/truf/compose.yaml', '/opt/truf/compose.edge.yaml')
|
|
DEPLOYMENT_PROFILE_FILE = '/etc/truf/deployment-profile'
|
|
STANDALONE_PROFILE_NAME = 'standalone-edge-v1'
|
|
SHARED_HOST_PROFILE_NAME = 'shared-host-edge-v1'
|
|
EDGE_ENV_FILE = '/etc/truf-edge/edge.env'
|
|
RUNTIME_IMAGE = 'truf-local:runtime'
|
|
EDGE_IMAGE = 'truf-local:edge'
|
|
RUNTIME_SERVICE = 'runtime'
|
|
EDGE_SERVICE = 'edge'
|
|
|
|
COMMAND_OUTPUT_LIMIT = 16 * 1024
|
|
INSPECT_TIMEOUT = 15.0
|
|
EDGE_STOP_TIMEOUT = 45.0
|
|
RUNTIME_STOP_TIMEOUT = 660.0
|
|
REMOVE_TIMEOUT = 45.0
|
|
RECREATE_TIMEOUT = 120.0
|
|
RUNTIME_HEALTH_TIMEOUT = 240.0
|
|
HEALTH_COMMAND_TIMEOUT = 30.0
|
|
EDGE_START_TIMEOUT = 45.0
|
|
EDGE_STABILITY_SECONDS = 10.0
|
|
EDGE_VERIFY_TIMEOUT = 100.0
|
|
POLL_SECONDS = 2.0
|
|
|
|
_HEX_ID = re.compile(r'[0-9a-f]{64}')
|
|
_IMAGE_ID = re.compile(r'sha256:[0-9a-f]{64}')
|
|
_RUNTIME_ENTRYPOINT = (
|
|
'/usr/bin/tini', '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B',
|
|
'/opt/truf/app/container_runtime.py',
|
|
)
|
|
_RUNTIME_HEALTH_TEST = (
|
|
'CMD', '/usr/local/bin/python3', '-I', '-S', '-B',
|
|
'/opt/truf/app/container_runtime.py', 'health', '--config',
|
|
'/data/config/config.yaml',
|
|
)
|
|
_RUNTIME_TMPFS = {
|
|
'/run/truf': 'rw,nosuid,nodev,noexec,size=64m,mode=0700,uid=10001,gid=10001',
|
|
'/tmp': 'rw,nosuid,nodev,noexec,size=128m,mode=1777',
|
|
}
|
|
_EDGE_TMPFS = {
|
|
'/tmp': 'rw,nosuid,nodev,noexec,size=16m,mode=1777',
|
|
'/run': 'rw,nosuid,nodev,noexec,size=4m,mode=0700,uid=10001,gid=10001',
|
|
}
|
|
_INSPECT_FORMAT = (
|
|
'{"id":{{json .Id}},"image":{{json .Image}},'
|
|
'"status":{{json .State.Status}},"running":{{json .State.Running}},'
|
|
'"paused":{{json .State.Paused}},'
|
|
'"restarting":{{json .State.Restarting}},"dead":{{json .State.Dead}},'
|
|
'"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},'
|
|
'"oom_killed":{{json .State.OOMKilled}},'
|
|
'"restarts":{{json .RestartCount}},"user":{{json .Config.User}},'
|
|
'"entrypoint":{{json .Config.Entrypoint}},'
|
|
'"command":{{json .Config.Cmd}},'
|
|
'"stop_timeout":{{json .Config.StopTimeout}},'
|
|
'"stop_signal":{{with index .Config "StopSignal"}}{{json .}}'
|
|
'{{else}}""{{end}},'
|
|
'"mounts":"{{range .Mounts}}{{.Type}}|{{if eq .Type "volume"}}'
|
|
'{{.Name}}{{end}}|{{.Source}}|'
|
|
'{{.Destination}}|{{.RW}};{{end}}",'
|
|
'"readonly":{{json .HostConfig.ReadonlyRootfs}},'
|
|
'"privileged":{{json .HostConfig.Privileged}},'
|
|
'"network":{{json .HostConfig.NetworkMode}},'
|
|
'"pid_mode":{{json .HostConfig.PidMode}},'
|
|
'"ipc_mode":{{json .HostConfig.IpcMode}},'
|
|
'"userns_mode":{{json .HostConfig.UsernsMode}},'
|
|
'"cgroupns_mode":{{json .HostConfig.CgroupnsMode}},'
|
|
'"uts_mode":{{json .HostConfig.UTSMode}},'
|
|
'"group_add":{{json .HostConfig.GroupAdd}},'
|
|
'"oci_runtime":{{json .HostConfig.Runtime}},'
|
|
'"devices":{{if .HostConfig.Devices}}{{len .HostConfig.Devices}}'
|
|
'{{else}}0{{end}},'
|
|
'"device_requests":{{if .HostConfig.DeviceRequests}}'
|
|
'{{len .HostConfig.DeviceRequests}}{{else}}0{{end}},'
|
|
'"device_cgroup_rules":{{if .HostConfig.DeviceCgroupRules}}'
|
|
'{{len .HostConfig.DeviceCgroupRules}}{{else}}0{{end}},'
|
|
'"ports":{{json .HostConfig.PortBindings}},'
|
|
'"tmpfs":{{json .HostConfig.Tmpfs}},'
|
|
'"cpus":{{json .HostConfig.NanoCpus}},'
|
|
'"memory":{{json .HostConfig.Memory}},'
|
|
'"pids_limit":{{json .HostConfig.PidsLimit}},'
|
|
'"shm_size":{{json .HostConfig.ShmSize}},'
|
|
'"log_config":{{json .HostConfig.LogConfig}},'
|
|
'"cap_drop":{{json .HostConfig.CapDrop}},'
|
|
'"cap_add":{{json .HostConfig.CapAdd}},'
|
|
'"security_opt":{{json .HostConfig.SecurityOpt}},'
|
|
'"restart_policy":{{json .HostConfig.RestartPolicy}},'
|
|
'"project":{{json (index .Config.Labels "com.docker.compose.project")}},'
|
|
'"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
|
'"oneoff":{{json (index .Config.Labels "com.docker.compose.oneoff")}},'
|
|
'"config_hash":{{json (index .Config.Labels '
|
|
'"com.docker.compose.config-hash")}},'
|
|
'"config_files":{{json (index .Config.Labels '
|
|
'"com.docker.compose.project.config_files")}},'
|
|
'"working_dir":{{json (index .Config.Labels '
|
|
'"com.docker.compose.project.working_dir")}},'
|
|
'"health_test":{{with index .Config "Healthcheck"}}{{json .Test}}'
|
|
'{{else}}null{{end}},'
|
|
'"health":{{with index .State "Health"}}{{json .Status}}'
|
|
'{{else}}null{{end}}}'
|
|
)
|
|
_INSPECT_KEYS = {
|
|
'id', 'image', 'status', 'running', 'paused', 'restarting', 'dead',
|
|
'pid', 'exit_code', 'oom_killed', 'restarts', 'user', 'entrypoint',
|
|
'command', 'stop_timeout', 'stop_signal', 'mounts', 'readonly',
|
|
'privileged', 'network', 'pid_mode', 'ipc_mode', 'userns_mode',
|
|
'cgroupns_mode', 'uts_mode', 'group_add', 'oci_runtime', 'devices',
|
|
'device_requests', 'device_cgroup_rules', 'ports', 'tmpfs', 'cpus',
|
|
'memory', 'pids_limit', 'shm_size', 'log_config',
|
|
'cap_drop', 'cap_add', 'security_opt', 'restart_policy', 'project',
|
|
'service', 'oneoff', 'config_hash', 'config_files', 'working_dir',
|
|
'health_test', 'health',
|
|
}
|
|
|
|
|
|
class HostLifecycleError(RuntimeError):
|
|
def __init__(self, category):
|
|
super().__init__('host runtime lifecycle failed')
|
|
self.category = str(category)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _DeploymentProfile:
|
|
name: str
|
|
compose_files: tuple
|
|
runtime_network: str
|
|
runtime_ports: dict
|
|
runtime_data_volume: str
|
|
runtime_cpus: int
|
|
runtime_memory: int
|
|
edge_cap_add: tuple
|
|
edge_caddyfile: str
|
|
|
|
|
|
STANDALONE_PROFILE = _DeploymentProfile(
|
|
name=STANDALONE_PROFILE_NAME,
|
|
compose_files=COMPOSE_FILES,
|
|
runtime_network=f'{PROJECT}_default',
|
|
runtime_ports={'443/tcp': [{'HostIp': '', 'HostPort': '443'}]},
|
|
runtime_data_volume='truf-docker_data',
|
|
runtime_cpus=2_000_000_000,
|
|
runtime_memory=6 * 1024 ** 3,
|
|
edge_cap_add=('NET_BIND_SERVICE',),
|
|
edge_caddyfile='/etc/caddy/Caddyfile',
|
|
)
|
|
SHARED_HOST_PROFILE = _DeploymentProfile(
|
|
name=SHARED_HOST_PROFILE_NAME,
|
|
compose_files=(
|
|
'/opt/truf/compose.yaml', '/opt/truf/compose.shared-host.yaml',
|
|
),
|
|
runtime_network='host',
|
|
runtime_ports={},
|
|
runtime_data_volume='truf-remote-server-data',
|
|
runtime_cpus=900_000_000,
|
|
runtime_memory=720 * 1024 ** 2,
|
|
edge_cap_add=(),
|
|
edge_caddyfile='/etc/caddy/Caddyfile.shared-host',
|
|
)
|
|
|
|
|
|
def _compose_command(profile):
|
|
return (
|
|
DOCKER, 'compose', '--ansi', 'never', '--project-name', PROJECT,
|
|
'--env-file', EDGE_ENV_FILE, '--project-directory', PROJECT_DIRECTORY,
|
|
*(item for path in profile.compose_files for item in ('--file', path)),
|
|
)
|
|
|
|
|
|
_COMPOSE = _compose_command(STANDALONE_PROFILE)
|
|
_CONFIG_FILES_LABEL = ','.join(COMPOSE_FILES)
|
|
|
|
|
|
def _deployment_profile(path=DEPLOYMENT_PROFILE_FILE):
|
|
descriptor = None
|
|
try:
|
|
descriptor = os.open(
|
|
path, os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0)
|
|
| getattr(os, 'O_NOFOLLOW', 0),
|
|
)
|
|
before = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(before.st_mode) or before.st_uid != 0
|
|
or before.st_gid != 0 or stat.S_IMODE(before.st_mode) != 0o444
|
|
or before.st_nlink != 1 or before.st_size > 64
|
|
):
|
|
raise HostLifecycleError('profile')
|
|
payload = os.read(descriptor, 65)
|
|
after = os.fstat(descriptor)
|
|
if (
|
|
len(payload) > 64 or before.st_dev != after.st_dev
|
|
or before.st_ino != after.st_ino or before.st_mode != after.st_mode
|
|
or before.st_uid != after.st_uid or before.st_gid != after.st_gid
|
|
or before.st_nlink != after.st_nlink or before.st_size != after.st_size
|
|
or before.st_mtime_ns != after.st_mtime_ns
|
|
):
|
|
raise HostLifecycleError('profile')
|
|
except FileNotFoundError:
|
|
return STANDALONE_PROFILE
|
|
except HostLifecycleError:
|
|
raise
|
|
except OSError:
|
|
raise HostLifecycleError('profile') from None
|
|
finally:
|
|
if descriptor is not None:
|
|
os.close(descriptor)
|
|
try:
|
|
name = payload.decode('ascii').strip()
|
|
except UnicodeDecodeError:
|
|
raise HostLifecycleError('profile') from None
|
|
profiles = {
|
|
STANDALONE_PROFILE_NAME: STANDALONE_PROFILE,
|
|
SHARED_HOST_PROFILE_NAME: SHARED_HOST_PROFILE,
|
|
}
|
|
if name not in profiles:
|
|
raise HostLifecycleError('profile')
|
|
return profiles[name]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _ContainerState:
|
|
container_id: str
|
|
image_id: str
|
|
status: str
|
|
running: bool
|
|
paused: bool
|
|
restarting: bool
|
|
dead: bool
|
|
pid: int
|
|
exit_code: int
|
|
oom_killed: bool
|
|
restarts: int
|
|
user: str
|
|
entrypoint: object
|
|
command: object
|
|
stop_timeout: int
|
|
stop_signal: str
|
|
mounts: tuple
|
|
readonly: bool
|
|
privileged: bool
|
|
network: str
|
|
pid_mode: str
|
|
ipc_mode: str
|
|
userns_mode: str
|
|
cgroupns_mode: str
|
|
uts_mode: str
|
|
group_add: tuple
|
|
oci_runtime: str
|
|
devices: int
|
|
device_requests: int
|
|
device_cgroup_rules: int
|
|
ports: dict
|
|
tmpfs: dict
|
|
cpus: int
|
|
memory: int
|
|
pids_limit: int
|
|
shm_size: int
|
|
log_config: dict
|
|
cap_drop: tuple
|
|
cap_add: tuple
|
|
security_opt: tuple
|
|
restart_policy: dict
|
|
project: str
|
|
service: str
|
|
oneoff: str
|
|
config_hash: str
|
|
config_files: str
|
|
working_dir: str
|
|
health_test: object
|
|
health: object
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class DeploymentSnapshot:
|
|
runtime: _ContainerState
|
|
edge: _ContainerState
|
|
runtime_image_id: str
|
|
edge_image_id: str
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _DeploymentAttempt:
|
|
snapshot: DeploymentSnapshot
|
|
token: object
|
|
phase: str = 'preflight'
|
|
mutation_started: bool = False
|
|
edge_stop_issued: bool = False
|
|
runtime_stop_issued: bool = False
|
|
forward_runtime_id: str = None
|
|
forward_edge_id: str = None
|
|
forward_edge_config_hash: str = None
|
|
|
|
|
|
def _subprocess_runner(command, timeout):
|
|
environment = {
|
|
'HOME': '/root',
|
|
'LANG': 'C.UTF-8',
|
|
'LC_ALL': 'C.UTF-8',
|
|
'PATH': '/usr/bin:/bin',
|
|
}
|
|
process = None
|
|
reader = None
|
|
output_fd = None
|
|
output_lock = threading.Lock()
|
|
stop_output = threading.Event()
|
|
captured = bytearray()
|
|
overflow = threading.Event()
|
|
read_failed = threading.Event()
|
|
|
|
def close_output():
|
|
nonlocal output_fd
|
|
with output_lock:
|
|
descriptor = output_fd
|
|
output_fd = None
|
|
if descriptor is not None:
|
|
try:
|
|
os.close(descriptor)
|
|
except OSError:
|
|
pass
|
|
|
|
def terminate():
|
|
if process is None:
|
|
return
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except OSError:
|
|
if process.poll() is None:
|
|
try:
|
|
process.kill()
|
|
except OSError:
|
|
pass
|
|
if process.poll() is None:
|
|
try:
|
|
process.wait(timeout=1)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
|
|
def read_output():
|
|
try:
|
|
while not stop_output.is_set():
|
|
chunk = os.read(output_fd, 4096)
|
|
if not chunk:
|
|
return
|
|
if len(captured) <= COMMAND_OUTPUT_LIMIT:
|
|
captured.extend(chunk[:COMMAND_OUTPUT_LIMIT + 1 - len(captured)])
|
|
if len(captured) > COMMAND_OUTPUT_LIMIT:
|
|
overflow.set()
|
|
terminate()
|
|
except Exception:
|
|
if not stop_output.is_set():
|
|
read_failed.set()
|
|
terminate()
|
|
finally:
|
|
close_output()
|
|
|
|
try:
|
|
process = subprocess.Popen(
|
|
command,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
cwd='/',
|
|
env=environment,
|
|
close_fds=True,
|
|
start_new_session=True,
|
|
)
|
|
output_fd = os.dup(process.stdout.fileno())
|
|
process.stdout.close()
|
|
reader = threading.Thread(target=read_output, daemon=True)
|
|
reader.start()
|
|
try:
|
|
return_code = process.wait(timeout=max(0.001, float(timeout)))
|
|
except subprocess.TimeoutExpired:
|
|
terminate()
|
|
raise HostLifecycleError('timeout') from None
|
|
reader.join(timeout=1)
|
|
if (
|
|
reader.is_alive()
|
|
or read_failed.is_set()
|
|
or overflow.is_set()
|
|
or return_code != 0
|
|
):
|
|
terminate()
|
|
raise HostLifecycleError('command')
|
|
return bytes(captured)
|
|
except HostLifecycleError:
|
|
raise
|
|
except Exception:
|
|
terminate()
|
|
raise HostLifecycleError('command') from None
|
|
finally:
|
|
if process is not None and (
|
|
process.poll() is None or (reader is not None and reader.is_alive())
|
|
):
|
|
terminate()
|
|
if reader is not None and reader.is_alive():
|
|
stop_output.set()
|
|
close_output()
|
|
reader.join(timeout=1)
|
|
close_output()
|
|
if (
|
|
process is not None and process.stdout is not None
|
|
and not process.stdout.closed
|
|
):
|
|
process.stdout.close()
|
|
|
|
|
|
def _json_payload(payload):
|
|
try:
|
|
if not payload or len(payload) > COMMAND_OUTPUT_LIMIT:
|
|
raise ValueError('bounded JSON required')
|
|
return json.loads(payload.decode('utf-8'))
|
|
except (UnicodeDecodeError, ValueError, TypeError):
|
|
raise HostLifecycleError('evidence') from None
|
|
|
|
|
|
def _string_tuple(value):
|
|
if value is None:
|
|
return ()
|
|
if not isinstance(value, list) or any(type(item) is not str for item in value):
|
|
raise HostLifecycleError('evidence')
|
|
return tuple(value)
|
|
|
|
|
|
def _optional_string_tuple(value):
|
|
return None if value is None else _string_tuple(value)
|
|
|
|
|
|
def _mount_tuple(value):
|
|
if type(value) is not str or len(value) > 4096 or not value.endswith(';'):
|
|
raise HostLifecycleError('evidence')
|
|
mounts = []
|
|
entries = value.split(';')
|
|
if entries[-1] != '' or len(entries) > 17:
|
|
raise HostLifecycleError('evidence')
|
|
for entry in entries[:-1]:
|
|
fields = entry.split('|')
|
|
if len(fields) != 5 or fields[4] not in ('true', 'false'):
|
|
raise HostLifecycleError('evidence')
|
|
mount_type, name, source, destination, writable = fields
|
|
identity = name if mount_type == 'volume' else source
|
|
if not mount_type or not identity or not destination or any(
|
|
character in mount_type + identity + destination for character in '\r\n"'
|
|
):
|
|
raise HostLifecycleError('evidence')
|
|
mounts.append((mount_type, identity, destination, writable == 'true'))
|
|
return tuple(sorted(mounts))
|
|
|
|
|
|
def _string_map(value):
|
|
if value is None:
|
|
return {}
|
|
if (
|
|
not isinstance(value, dict)
|
|
or len(value) > 16
|
|
or any(type(key) is not str or type(item) is not str
|
|
for key, item in value.items())
|
|
):
|
|
raise HostLifecycleError('evidence')
|
|
return dict(value)
|
|
|
|
|
|
def _port_map(value):
|
|
if value is None:
|
|
return {}
|
|
if not isinstance(value, dict) or len(value) > 8:
|
|
raise HostLifecycleError('evidence')
|
|
result = {}
|
|
for port, bindings in value.items():
|
|
if type(port) is not str or not isinstance(bindings, list) or len(bindings) > 8:
|
|
raise HostLifecycleError('evidence')
|
|
normalized = []
|
|
for binding in bindings:
|
|
if (
|
|
not isinstance(binding, dict)
|
|
or set(binding) != {'HostIp', 'HostPort'}
|
|
or type(binding['HostIp']) is not str
|
|
or type(binding['HostPort']) is not str
|
|
):
|
|
raise HostLifecycleError('evidence')
|
|
normalized.append(dict(binding))
|
|
result[port] = normalized
|
|
return result
|
|
|
|
|
|
def _container_state(payload):
|
|
value = _json_payload(payload)
|
|
try:
|
|
if not isinstance(value, dict) or set(value) != _INSPECT_KEYS:
|
|
raise ValueError('inspect shape')
|
|
for name in ('running', 'paused', 'restarting', 'dead', 'oom_killed',
|
|
'readonly', 'privileged'):
|
|
if type(value[name]) is not bool:
|
|
raise ValueError('inspect boolean')
|
|
for name in (
|
|
'pid', 'exit_code', 'restarts', 'stop_timeout', 'devices',
|
|
'device_requests', 'device_cgroup_rules', 'cpus', 'memory',
|
|
'pids_limit', 'shm_size',
|
|
):
|
|
if type(value[name]) is not int or value[name] < 0:
|
|
raise ValueError('inspect integer')
|
|
for name in (
|
|
'id', 'image', 'status', 'user', 'stop_signal', 'network',
|
|
'pid_mode', 'ipc_mode', 'userns_mode', 'cgroupns_mode', 'uts_mode',
|
|
'oci_runtime', 'project', 'service', 'oneoff', 'config_hash',
|
|
'config_files', 'working_dir',
|
|
):
|
|
if type(value[name]) is not str:
|
|
raise ValueError('inspect string')
|
|
if value['health'] is not None and type(value['health']) is not str:
|
|
raise ValueError('inspect health')
|
|
if not isinstance(value['restart_policy'], dict):
|
|
raise ValueError('inspect restart policy')
|
|
if not isinstance(value['log_config'], dict):
|
|
raise ValueError('inspect log config')
|
|
return _ContainerState(
|
|
container_id=value['id'], image_id=value['image'],
|
|
status=value['status'], running=value['running'],
|
|
paused=value['paused'], restarting=value['restarting'],
|
|
dead=value['dead'], pid=value['pid'],
|
|
exit_code=value['exit_code'], oom_killed=value['oom_killed'],
|
|
restarts=value['restarts'], user=value['user'],
|
|
entrypoint=_optional_string_tuple(value['entrypoint']),
|
|
command=_optional_string_tuple(value['command']),
|
|
stop_timeout=value['stop_timeout'], stop_signal=value['stop_signal'],
|
|
mounts=_mount_tuple(value['mounts']),
|
|
readonly=value['readonly'], privileged=value['privileged'],
|
|
network=value['network'], pid_mode=value['pid_mode'],
|
|
ipc_mode=value['ipc_mode'], userns_mode=value['userns_mode'],
|
|
cgroupns_mode=value['cgroupns_mode'], uts_mode=value['uts_mode'],
|
|
group_add=_string_tuple(value['group_add']),
|
|
oci_runtime=value['oci_runtime'], devices=value['devices'],
|
|
device_requests=value['device_requests'],
|
|
device_cgroup_rules=value['device_cgroup_rules'],
|
|
ports=_port_map(value['ports']),
|
|
tmpfs=_string_map(value['tmpfs']), cpus=value['cpus'],
|
|
memory=value['memory'], pids_limit=value['pids_limit'],
|
|
shm_size=value['shm_size'], log_config=dict(value['log_config']),
|
|
cap_drop=_string_tuple(value['cap_drop']),
|
|
cap_add=_string_tuple(value['cap_add']),
|
|
security_opt=_string_tuple(value['security_opt']),
|
|
restart_policy=dict(value['restart_policy']),
|
|
project=value['project'], service=value['service'],
|
|
oneoff=value['oneoff'], config_hash=value['config_hash'],
|
|
config_files=value['config_files'], working_dir=value['working_dir'],
|
|
health_test=_optional_string_tuple(value['health_test']),
|
|
health=value['health'],
|
|
)
|
|
except (KeyError, TypeError, ValueError):
|
|
raise HostLifecycleError('evidence') from None
|
|
|
|
|
|
class FixedDeploymentLifecycle:
|
|
def __init__(self, *, _runner=None, _clock=None, _sleep=None, _profile=None):
|
|
self._runner = _runner or _subprocess_runner
|
|
self._clock = _clock or time.monotonic
|
|
self._sleep = _sleep or time.sleep
|
|
self._profile = _profile or _deployment_profile()
|
|
self._compose_command = _compose_command(self._profile)
|
|
self._token = object()
|
|
|
|
def _run(self, command, timeout):
|
|
try:
|
|
payload = self._runner(tuple(command), float(timeout))
|
|
if not isinstance(payload, bytes) or len(payload) > COMMAND_OUTPUT_LIMIT:
|
|
raise HostLifecycleError('command')
|
|
return payload
|
|
except HostLifecycleError:
|
|
raise
|
|
except Exception:
|
|
raise HostLifecycleError('command') from None
|
|
|
|
def _compose(self, *arguments, timeout=INSPECT_TIMEOUT):
|
|
return self._run((*self._compose_command, *arguments), timeout)
|
|
|
|
def _image_id(self, image):
|
|
payload = self._run(
|
|
(DOCKER, 'image', 'inspect', '--format', '{{json .Id}}', image),
|
|
INSPECT_TIMEOUT,
|
|
)
|
|
value = _json_payload(payload)
|
|
if type(value) is not str or _IMAGE_ID.fullmatch(value) is None:
|
|
raise HostLifecycleError('evidence')
|
|
return value
|
|
|
|
def _resolve(self, service, timeout=INSPECT_TIMEOUT):
|
|
payload = self._compose(
|
|
'ps', '--all', '--quiet', service, timeout=timeout,
|
|
)
|
|
try:
|
|
lines = payload.decode('ascii').splitlines()
|
|
except UnicodeDecodeError:
|
|
raise HostLifecycleError('evidence') from None
|
|
if len(lines) != 1 or _HEX_ID.fullmatch(lines[0]) is None:
|
|
raise HostLifecycleError('evidence')
|
|
return lines[0]
|
|
|
|
def _resolve_optional(self, service, timeout=INSPECT_TIMEOUT):
|
|
payload = self._compose(
|
|
'ps', '--all', '--quiet', service, timeout=timeout,
|
|
)
|
|
try:
|
|
lines = payload.decode('ascii').splitlines()
|
|
except UnicodeDecodeError:
|
|
raise HostLifecycleError('evidence') from None
|
|
if not lines:
|
|
return None
|
|
if len(lines) != 1 or _HEX_ID.fullmatch(lines[0]) is None:
|
|
raise HostLifecycleError('evidence')
|
|
return lines[0]
|
|
|
|
def _inspect(self, container_id, timeout=INSPECT_TIMEOUT):
|
|
if _HEX_ID.fullmatch(container_id) is None:
|
|
raise HostLifecycleError('evidence')
|
|
state = _container_state(self._run(
|
|
(DOCKER, 'container', 'inspect', '--format', _INSPECT_FORMAT,
|
|
container_id),
|
|
timeout,
|
|
))
|
|
if state.container_id != container_id:
|
|
raise HostLifecycleError('identity')
|
|
return state
|
|
|
|
def _require_metadata(
|
|
self, state, service, image_id, runtime_id, *, config_hash=None,
|
|
):
|
|
expected_policy = (
|
|
{'Name': 'on-failure', 'MaximumRetryCount': 3}
|
|
if service == RUNTIME_SERVICE
|
|
else {'Name': 'unless-stopped', 'MaximumRetryCount': 0}
|
|
)
|
|
expected_cap_add = (
|
|
() if service == RUNTIME_SERVICE else self._profile.edge_cap_add
|
|
)
|
|
expected_network = (
|
|
self._profile.runtime_network
|
|
if service == RUNTIME_SERVICE else f'container:{runtime_id}'
|
|
)
|
|
runtime = service == RUNTIME_SERVICE
|
|
expected_mounts = (
|
|
{
|
|
('volume', self._profile.runtime_data_volume, '/data', True),
|
|
('bind', '/etc/truf/runtime', '/data/config', False),
|
|
('bind', '/etc/truf/worker-packages', '/data/worker-packages', False),
|
|
('bind', '/var/lib/truf/runtime-document-candidates', '/data/runtime-document-candidates', True),
|
|
('bind', '/run/truf/host-agent.sock', '/run/truf/host-agent.sock', False),
|
|
('bind', '/var/lib/truf/host-agent/results', '/data/host-agent-results', False),
|
|
('bind', '/run/truf-postgres', '/run/truf-postgres', True),
|
|
} if runtime else {
|
|
('volume', '/data', True), ('volume', '/config', True),
|
|
('bind', '/var/log/caddy', True),
|
|
('bind', '/etc/caddy/denylist', False),
|
|
}
|
|
)
|
|
actual_mounts = (
|
|
set(state.mounts)
|
|
if runtime else {
|
|
(mount_type, destination, writable)
|
|
for mount_type, _identity, destination, writable in state.mounts
|
|
}
|
|
)
|
|
expected_ports = self._profile.runtime_ports if runtime else {}
|
|
expected_log = {
|
|
'Type': 'json-file',
|
|
'Config': {'max-size': '16m' if runtime else '8m', 'max-file': '4'},
|
|
}
|
|
if (
|
|
_HEX_ID.fullmatch(state.container_id) is None
|
|
or state.image_id != image_id
|
|
or _IMAGE_ID.fullmatch(state.image_id) is None
|
|
or state.user != '10001:10001'
|
|
or state.readonly is not True
|
|
or state.privileged is not False
|
|
or state.project != PROJECT
|
|
or state.service != service
|
|
or state.oneoff != 'False'
|
|
or _HEX_ID.fullmatch(state.config_hash) is None
|
|
or (config_hash is not None and state.config_hash != config_hash)
|
|
or state.config_files != ','.join(self._profile.compose_files)
|
|
or state.working_dir != PROJECT_DIRECTORY
|
|
or state.network != expected_network
|
|
or set(state.cap_drop) != {'ALL'}
|
|
or set(state.cap_add) != set(expected_cap_add)
|
|
or set(state.security_opt) != {'no-new-privileges:true'}
|
|
or state.restart_policy != expected_policy
|
|
or state.entrypoint != (
|
|
_RUNTIME_ENTRYPOINT if runtime
|
|
else ('/usr/local/bin/truf-edge-entrypoint',)
|
|
)
|
|
or state.command != (
|
|
('run', '--config', '/data/config/config.yaml') if runtime else None
|
|
)
|
|
or state.health_test != (_RUNTIME_HEALTH_TEST if runtime else None)
|
|
or state.stop_timeout != (600 if runtime else 30)
|
|
or state.stop_signal != ('SIGTERM' if runtime else '')
|
|
or len(state.mounts) != len(expected_mounts)
|
|
or actual_mounts != expected_mounts
|
|
or state.pid_mode != '' or state.ipc_mode != 'private'
|
|
or state.userns_mode != '' or state.cgroupns_mode != 'private'
|
|
or state.uts_mode != '' or state.group_add
|
|
or state.oci_runtime != 'runc' or state.devices != 0
|
|
or state.device_requests != 0 or state.device_cgroup_rules != 0
|
|
or state.ports != expected_ports
|
|
or state.tmpfs != (_RUNTIME_TMPFS if runtime else _EDGE_TMPFS)
|
|
or state.cpus != (
|
|
self._profile.runtime_cpus if runtime else 1_000_000_000
|
|
)
|
|
or state.memory != (
|
|
self._profile.runtime_memory if runtime else 256 * 1024 ** 2
|
|
)
|
|
or state.pids_limit != (512 if runtime else 128)
|
|
or state.shm_size != (256 * 1024 ** 2 if runtime else 64 * 1024 ** 2)
|
|
or state.log_config != expected_log
|
|
):
|
|
raise HostLifecycleError('identity')
|
|
|
|
def _require_running(
|
|
self, state, service, image_id, runtime_id, *, fresh, config_hash=None,
|
|
):
|
|
self._require_metadata(
|
|
state, service, image_id, runtime_id, config_hash=config_hash,
|
|
)
|
|
if (
|
|
state.status != 'running'
|
|
or state.running is not True
|
|
or state.paused or state.restarting or state.dead or state.oom_killed
|
|
or state.pid <= 0
|
|
or (fresh and state.restarts != 0)
|
|
):
|
|
raise HostLifecycleError('health')
|
|
|
|
def _require_stopped(self, current, previous, service, runtime_id):
|
|
self._require_metadata(
|
|
current, service, previous.image_id, runtime_id,
|
|
config_hash=previous.config_hash,
|
|
)
|
|
if (
|
|
current.status != 'exited'
|
|
or current.running or current.paused or current.restarting
|
|
or current.dead or current.oom_killed
|
|
or current.pid != 0 or current.exit_code != 0
|
|
or current.restarts != previous.restarts
|
|
):
|
|
raise HostLifecycleError('stop')
|
|
|
|
def preflight(self):
|
|
self._compose('config', '--quiet')
|
|
runtime_image = self._image_id(RUNTIME_IMAGE)
|
|
edge_image = self._image_id(EDGE_IMAGE)
|
|
runtime_id = self._resolve(RUNTIME_SERVICE)
|
|
edge_id = self._resolve(EDGE_SERVICE)
|
|
runtime = self._inspect(runtime_id)
|
|
edge = self._inspect(edge_id)
|
|
self._require_running(
|
|
runtime, RUNTIME_SERVICE, runtime_image, runtime_id, fresh=False,
|
|
)
|
|
self._require_running(
|
|
edge, EDGE_SERVICE, edge_image, runtime_id, fresh=False,
|
|
)
|
|
return DeploymentSnapshot(runtime, edge, runtime_image, edge_image)
|
|
|
|
def begin_attempt(self, snapshot):
|
|
if not isinstance(snapshot, DeploymentSnapshot):
|
|
raise HostLifecycleError('state')
|
|
return _DeploymentAttempt(snapshot, self._token)
|
|
|
|
def _require_attempt(self, attempt):
|
|
if (
|
|
not isinstance(attempt, _DeploymentAttempt)
|
|
or attempt.token is not self._token
|
|
):
|
|
raise HostLifecycleError('state')
|
|
return attempt
|
|
|
|
def stop_cleanly(self, attempt):
|
|
attempt = self._require_attempt(attempt)
|
|
snapshot = attempt.snapshot
|
|
attempt.mutation_started = True
|
|
attempt.edge_stop_issued = True
|
|
attempt.phase = 'stopping-edge'
|
|
self._compose(
|
|
'stop', '--timeout', '30', EDGE_SERVICE,
|
|
timeout=EDGE_STOP_TIMEOUT,
|
|
)
|
|
stopped_edge = self._inspect(snapshot.edge.container_id)
|
|
self._require_stopped(
|
|
stopped_edge, snapshot.edge, EDGE_SERVICE,
|
|
snapshot.runtime.container_id,
|
|
)
|
|
attempt.runtime_stop_issued = True
|
|
attempt.phase = 'stopping-runtime'
|
|
self._compose(
|
|
'stop', '--timeout', '600', RUNTIME_SERVICE,
|
|
timeout=RUNTIME_STOP_TIMEOUT,
|
|
)
|
|
stopped_runtime = self._inspect(snapshot.runtime.container_id)
|
|
self._require_stopped(
|
|
stopped_runtime, snapshot.runtime, RUNTIME_SERVICE,
|
|
snapshot.runtime.container_id,
|
|
)
|
|
attempt.phase = 'stopped'
|
|
return attempt
|
|
|
|
def authorize_replacement(self, attempt, operation_id):
|
|
attempt = self._require_attempt(attempt)
|
|
if attempt.phase != 'stopped':
|
|
raise HostLifecycleError('state')
|
|
snapshot = attempt.snapshot
|
|
runtime_id = snapshot.runtime.container_id
|
|
if (
|
|
self._resolve(RUNTIME_SERVICE) != runtime_id
|
|
or self._resolve(EDGE_SERVICE) != snapshot.edge.container_id
|
|
):
|
|
raise HostLifecycleError('identity')
|
|
self._require_stopped(
|
|
self._inspect(snapshot.edge.container_id), snapshot.edge,
|
|
EDGE_SERVICE, runtime_id,
|
|
)
|
|
self._require_stopped(
|
|
self._inspect(runtime_id), snapshot.runtime,
|
|
RUNTIME_SERVICE, runtime_id,
|
|
)
|
|
if (
|
|
self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id
|
|
or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id
|
|
):
|
|
raise HostLifecycleError('identity')
|
|
return _new_stopped_runtime_proof(operation_id, purpose='forward')
|
|
|
|
def _remaining(self, deadline, maximum):
|
|
remaining = deadline - self._clock()
|
|
if remaining <= 0:
|
|
raise HostLifecycleError('health')
|
|
return min(float(maximum), remaining)
|
|
|
|
def _strict_runtime_health(self, timeout):
|
|
try:
|
|
payload = self._compose(
|
|
'exec', '-T', '--user', '10001:10001', RUNTIME_SERVICE,
|
|
'/usr/local/bin/python3', '-I', '-S', '-B',
|
|
'/opt/truf/app/container_runtime.py', 'health',
|
|
'--config', '/data/config/config.yaml', '--require-worker-api',
|
|
'--require-discovery-producers',
|
|
timeout=min(HEALTH_COMMAND_TIMEOUT, timeout),
|
|
)
|
|
value = _json_payload(payload)
|
|
if (
|
|
not isinstance(value, dict)
|
|
or set(value) != {
|
|
'healthy', 'activation_state', 'postgres', 'workers',
|
|
}
|
|
or value.get('healthy') is not True
|
|
or value.get('activation_state') != 'ACTIVE'
|
|
or value.get('postgres') != 'READY'
|
|
or not isinstance(value.get('workers'), list)
|
|
or any(type(name) is not str for name in value['workers'])
|
|
or not {
|
|
'worker-api', 'result-ingester', 'jsonl-projector',
|
|
}.issubset(value['workers'])
|
|
):
|
|
raise HostLifecycleError('health')
|
|
except HostLifecycleError as error:
|
|
if error.category == 'health':
|
|
raise
|
|
raise HostLifecycleError('health') from None
|
|
|
|
def _wait_runtime(self, runtime_id, image_id, config_hash):
|
|
deadline = self._clock() + RUNTIME_HEALTH_TIMEOUT
|
|
while True:
|
|
if self._resolve(
|
|
RUNTIME_SERVICE, self._remaining(deadline, INSPECT_TIMEOUT),
|
|
) != runtime_id:
|
|
raise HostLifecycleError('identity')
|
|
current = self._inspect(
|
|
runtime_id, self._remaining(deadline, INSPECT_TIMEOUT),
|
|
)
|
|
self._require_running(
|
|
current, RUNTIME_SERVICE, image_id, runtime_id, fresh=True,
|
|
config_hash=config_hash,
|
|
)
|
|
if current.health == 'healthy':
|
|
try:
|
|
self._strict_runtime_health(
|
|
self._remaining(deadline, HEALTH_COMMAND_TIMEOUT),
|
|
)
|
|
except HostLifecycleError as error:
|
|
if error.category != 'health':
|
|
raise
|
|
self._sleep(min(
|
|
POLL_SECONDS,
|
|
self._remaining(deadline, POLL_SECONDS),
|
|
))
|
|
continue
|
|
else:
|
|
return
|
|
if current.health not in ('starting', 'unhealthy'):
|
|
raise HostLifecycleError('health')
|
|
self._sleep(min(POLL_SECONDS, self._remaining(deadline, POLL_SECONDS)))
|
|
|
|
def _wait_edge(self, edge_id, edge_image_id, runtime_id, config_hash):
|
|
deadline = self._clock() + EDGE_VERIFY_TIMEOUT
|
|
running_deadline = self._clock() + EDGE_START_TIMEOUT
|
|
while True:
|
|
if self._resolve(
|
|
EDGE_SERVICE, self._remaining(running_deadline, INSPECT_TIMEOUT),
|
|
) != edge_id:
|
|
raise HostLifecycleError('identity')
|
|
current = self._inspect(
|
|
edge_id, self._remaining(running_deadline, INSPECT_TIMEOUT),
|
|
)
|
|
self._require_metadata(
|
|
current, EDGE_SERVICE, edge_image_id, runtime_id,
|
|
config_hash=config_hash,
|
|
)
|
|
if (
|
|
current.status == 'running'
|
|
and current.running is True
|
|
and current.paused is False
|
|
and current.restarting is False
|
|
and current.dead is False
|
|
and current.oom_killed is False
|
|
and current.pid > 0
|
|
and current.restarts == 0
|
|
):
|
|
break
|
|
if (
|
|
current.restarting or current.dead or current.oom_killed
|
|
or current.status == 'exited'
|
|
):
|
|
raise HostLifecycleError('health')
|
|
self._sleep(min(
|
|
POLL_SECONDS, self._remaining(running_deadline, POLL_SECONDS),
|
|
))
|
|
self._compose(
|
|
'exec', '-T', '--user', '10001:10001', EDGE_SERVICE,
|
|
'caddy', 'validate', '--config', self._profile.edge_caddyfile,
|
|
'--adapter', 'caddyfile',
|
|
timeout=self._remaining(deadline, EDGE_START_TIMEOUT),
|
|
)
|
|
stable_until = self._clock() + EDGE_STABILITY_SECONDS
|
|
if stable_until > deadline:
|
|
raise HostLifecycleError('health')
|
|
while self._clock() < stable_until:
|
|
self._sleep(min(POLL_SECONDS, stable_until - self._clock()))
|
|
if self._resolve(
|
|
EDGE_SERVICE, self._remaining(deadline, INSPECT_TIMEOUT),
|
|
) != edge_id:
|
|
raise HostLifecycleError('identity')
|
|
current = self._inspect(
|
|
edge_id, self._remaining(deadline, INSPECT_TIMEOUT),
|
|
)
|
|
self._require_running(
|
|
current, EDGE_SERVICE, edge_image_id, runtime_id, fresh=True,
|
|
config_hash=config_hash,
|
|
)
|
|
|
|
def recreate_and_verify(self, attempt):
|
|
attempt = self._require_attempt(attempt)
|
|
snapshot = attempt.snapshot
|
|
runtime_id = snapshot.runtime.container_id
|
|
self._require_stopped(
|
|
self._inspect(snapshot.edge.container_id), snapshot.edge,
|
|
EDGE_SERVICE, runtime_id,
|
|
)
|
|
self._require_stopped(
|
|
self._inspect(runtime_id), snapshot.runtime,
|
|
RUNTIME_SERVICE, runtime_id,
|
|
)
|
|
if (
|
|
self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id
|
|
or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id
|
|
):
|
|
raise HostLifecycleError('identity')
|
|
self._compose('rm', '--force', EDGE_SERVICE, timeout=REMOVE_TIMEOUT)
|
|
self._compose('rm', '--force', RUNTIME_SERVICE, timeout=REMOVE_TIMEOUT)
|
|
attempt.phase = 'forward-removed'
|
|
if self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id:
|
|
raise HostLifecycleError('identity')
|
|
self._compose(
|
|
'up', '--detach', '--no-deps', '--no-build', '--pull', 'never',
|
|
'--force-recreate', RUNTIME_SERVICE, timeout=RECREATE_TIMEOUT,
|
|
)
|
|
new_runtime_id = self._resolve(RUNTIME_SERVICE)
|
|
attempt.forward_runtime_id = new_runtime_id
|
|
if new_runtime_id == runtime_id:
|
|
raise HostLifecycleError('identity')
|
|
new_runtime = self._inspect(new_runtime_id)
|
|
self._require_running(
|
|
new_runtime, RUNTIME_SERVICE, snapshot.runtime_image_id,
|
|
new_runtime_id, fresh=True, config_hash=snapshot.runtime.config_hash,
|
|
)
|
|
self._wait_runtime(
|
|
new_runtime_id, snapshot.runtime_image_id,
|
|
snapshot.runtime.config_hash,
|
|
)
|
|
if self._image_id(EDGE_IMAGE) != snapshot.edge_image_id:
|
|
raise HostLifecycleError('identity')
|
|
self._compose(
|
|
'up', '--detach', '--no-deps', '--no-build', '--pull', 'never',
|
|
'--force-recreate', EDGE_SERVICE, timeout=RECREATE_TIMEOUT,
|
|
)
|
|
new_edge_id = self._resolve(EDGE_SERVICE)
|
|
attempt.forward_edge_id = new_edge_id
|
|
if new_edge_id == snapshot.edge.container_id:
|
|
raise HostLifecycleError('identity')
|
|
new_edge = self._inspect(new_edge_id)
|
|
self._require_metadata(
|
|
new_edge, EDGE_SERVICE, snapshot.edge_image_id, new_runtime_id,
|
|
)
|
|
attempt.forward_edge_config_hash = new_edge.config_hash
|
|
self._wait_edge(
|
|
new_edge_id, snapshot.edge_image_id, new_runtime_id,
|
|
attempt.forward_edge_config_hash,
|
|
)
|
|
attempt.phase = 'forward-healthy'
|
|
|
|
@staticmethod
|
|
def _attempt_edge_config_hash(attempt, edge_id):
|
|
if edge_id == attempt.snapshot.edge.container_id:
|
|
return attempt.snapshot.edge.config_hash
|
|
if (
|
|
edge_id == attempt.forward_edge_id
|
|
and _HEX_ID.fullmatch(attempt.forward_edge_config_hash or '')
|
|
):
|
|
return attempt.forward_edge_config_hash
|
|
raise HostLifecycleError('rollback')
|
|
|
|
def _require_quiescent(self, current, service, image_id, runtime_id, config_hash):
|
|
self._require_metadata(
|
|
current, service, image_id, runtime_id, config_hash=config_hash,
|
|
)
|
|
if (
|
|
current.status != 'exited' or current.running or current.paused
|
|
or current.restarting or current.dead or current.pid != 0
|
|
):
|
|
raise HostLifecycleError('rollback')
|
|
|
|
def _quiesce_service(
|
|
self, service, current_id, image_id, runtime_id, config_hash, timeout, grace,
|
|
):
|
|
if current_id is None:
|
|
return
|
|
current = self._inspect(current_id)
|
|
self._require_metadata(
|
|
current, service, image_id, runtime_id, config_hash=config_hash,
|
|
)
|
|
if current.running:
|
|
try:
|
|
self._compose(
|
|
'stop', '--timeout', str(grace), service, timeout=timeout,
|
|
)
|
|
except HostLifecycleError:
|
|
pass
|
|
if self._resolve_optional(service) != current_id:
|
|
raise HostLifecycleError('rollback')
|
|
current = self._inspect(current_id)
|
|
self._require_quiescent(
|
|
current, service, image_id, runtime_id, config_hash,
|
|
)
|
|
|
|
def quiesce_for_rollback(self, attempt, operation_id):
|
|
attempt = self._require_attempt(attempt)
|
|
snapshot = attempt.snapshot
|
|
runtime_id = self._resolve_optional(RUNTIME_SERVICE)
|
|
edge_id = self._resolve_optional(EDGE_SERVICE)
|
|
if edge_id is not None and runtime_id is None:
|
|
raise HostLifecycleError('rollback')
|
|
edge_config_hash = (
|
|
self._attempt_edge_config_hash(attempt, edge_id)
|
|
if edge_id is not None else None
|
|
)
|
|
self._quiesce_service(
|
|
EDGE_SERVICE, edge_id, snapshot.edge_image_id, runtime_id,
|
|
edge_config_hash, EDGE_STOP_TIMEOUT, 30,
|
|
)
|
|
self._quiesce_service(
|
|
RUNTIME_SERVICE, runtime_id, snapshot.runtime_image_id, runtime_id,
|
|
snapshot.runtime.config_hash, RUNTIME_STOP_TIMEOUT, 600,
|
|
)
|
|
if (
|
|
self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id
|
|
or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id
|
|
):
|
|
raise HostLifecycleError('rollback')
|
|
attempt.phase = 'rollback-quiescent'
|
|
return _new_stopped_runtime_proof(operation_id, purpose='rollback')
|
|
|
|
def recreate_restored_and_verify(self, attempt):
|
|
attempt = self._require_attempt(attempt)
|
|
if attempt.phase != 'rollback-quiescent':
|
|
raise HostLifecycleError('state')
|
|
snapshot = attempt.snapshot
|
|
edge_id = self._resolve_optional(EDGE_SERVICE)
|
|
runtime_id = self._resolve_optional(RUNTIME_SERVICE)
|
|
if edge_id is not None:
|
|
self._require_quiescent(
|
|
self._inspect(edge_id), EDGE_SERVICE, snapshot.edge_image_id,
|
|
runtime_id, self._attempt_edge_config_hash(attempt, edge_id),
|
|
)
|
|
self._compose('rm', '--force', EDGE_SERVICE, timeout=REMOVE_TIMEOUT)
|
|
if runtime_id is not None:
|
|
self._require_quiescent(
|
|
self._inspect(runtime_id), RUNTIME_SERVICE,
|
|
snapshot.runtime_image_id, runtime_id,
|
|
snapshot.runtime.config_hash,
|
|
)
|
|
self._compose('rm', '--force', RUNTIME_SERVICE, timeout=REMOVE_TIMEOUT)
|
|
if self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id:
|
|
raise HostLifecycleError('rollback')
|
|
self._compose(
|
|
'up', '--detach', '--no-deps', '--no-build', '--pull', 'never',
|
|
'--force-recreate', RUNTIME_SERVICE, timeout=RECREATE_TIMEOUT,
|
|
)
|
|
restored_runtime_id = self._resolve(RUNTIME_SERVICE)
|
|
if restored_runtime_id in {
|
|
snapshot.runtime.container_id, attempt.forward_runtime_id,
|
|
}:
|
|
raise HostLifecycleError('rollback')
|
|
restored_runtime = self._inspect(restored_runtime_id)
|
|
self._require_running(
|
|
restored_runtime, RUNTIME_SERVICE, snapshot.runtime_image_id,
|
|
restored_runtime_id, fresh=True,
|
|
config_hash=snapshot.runtime.config_hash,
|
|
)
|
|
self._wait_runtime(
|
|
restored_runtime_id, snapshot.runtime_image_id,
|
|
snapshot.runtime.config_hash,
|
|
)
|
|
if self._image_id(EDGE_IMAGE) != snapshot.edge_image_id:
|
|
raise HostLifecycleError('rollback')
|
|
self._compose(
|
|
'up', '--detach', '--no-deps', '--no-build', '--pull', 'never',
|
|
'--force-recreate', EDGE_SERVICE, timeout=RECREATE_TIMEOUT,
|
|
)
|
|
restored_edge_id = self._resolve(EDGE_SERVICE)
|
|
if restored_edge_id in {snapshot.edge.container_id, attempt.forward_edge_id}:
|
|
raise HostLifecycleError('rollback')
|
|
restored_edge = self._inspect(restored_edge_id)
|
|
self._require_metadata(
|
|
restored_edge, EDGE_SERVICE, snapshot.edge_image_id,
|
|
restored_runtime_id,
|
|
)
|
|
self._wait_edge(
|
|
restored_edge_id, snapshot.edge_image_id, restored_runtime_id,
|
|
restored_edge.config_hash,
|
|
)
|
|
attempt.phase = 'rollback-healthy'
|
|
|
|
def contain_for_failed_hold(self, attempt, operation_id):
|
|
try:
|
|
self.quiesce_for_rollback(attempt, operation_id)
|
|
attempt.phase = 'failed-hold-contained'
|
|
return True
|
|
except Exception:
|
|
attempt.phase = 'failed-hold-uncontained'
|
|
return False
|
|
|
|
|
|
def execute_fixed_forward(session, lifecycle=None):
|
|
if getattr(session, '_entered', False) is not True:
|
|
raise HostLifecycleError('state')
|
|
lifecycle = lifecycle or FixedDeploymentLifecycle()
|
|
session.backup()
|
|
snapshot = lifecycle.preflight()
|
|
attempt = lifecycle.begin_attempt(snapshot)
|
|
session.revalidate_for_stop()
|
|
lifecycle.stop_cleanly(attempt)
|
|
proof = lifecycle.authorize_replacement(
|
|
attempt, session.request.operation_id,
|
|
)
|
|
resulting_identity = session.replace(proof)
|
|
lifecycle.recreate_and_verify(attempt)
|
|
return resulting_identity
|
|
|
|
|
|
def _forward_failure(request, error):
|
|
category = getattr(error, 'category', '')
|
|
if category == 'health':
|
|
return 'health_check_failed', 'health_check_failed'
|
|
if request.action.value == 'restart':
|
|
return 'restart_failed', 'restart_failed'
|
|
return 'apply_failed', 'apply_failed'
|
|
|
|
|
|
def _preflight_failure(error):
|
|
category = getattr(error, 'category', '')
|
|
if category == 'validation':
|
|
return 'validation_failed', 'validation_failed'
|
|
if category in ('identity', 'partial'):
|
|
return 'operation_conflict', 'bounded_result'
|
|
return 'apply_failed', 'apply_failed'
|
|
|
|
|
|
def _terminal_result_for_phase(session, phase):
|
|
name = phase['phase']
|
|
if name == 'succeeded':
|
|
identity = {
|
|
'active_config_sha256': (
|
|
session.request.candidate_config_sha256
|
|
or session.request.active_config_sha256
|
|
),
|
|
'active_secrets_sha256': (
|
|
session.request.candidate_secrets_sha256
|
|
or session.request.active_secrets_sha256
|
|
),
|
|
}
|
|
return name, None, None, identity
|
|
if name in {'failed', 'rolled_back'}:
|
|
category = phase.get('forward_category') or 'apply_failed'
|
|
detail = phase.get('safe_detail') or category
|
|
return name, category, detail, session.original_identity()
|
|
if name == 'failed_hold':
|
|
return name, 'rollback_failed', 'rollback_failed', None
|
|
raise HostLifecycleError('state')
|
|
|
|
|
|
def execute_fixed_operation(session, lifecycle=None, state=None):
|
|
"""Run one forward attempt and at most one fixed rollback attempt."""
|
|
if getattr(session, '_entered', False) is not True:
|
|
raise HostLifecycleError('state')
|
|
lifecycle = lifecycle or FixedDeploymentLifecycle()
|
|
state = state or HostOperationState(session.request)
|
|
try:
|
|
terminal = state.terminal_result()
|
|
if terminal is not None:
|
|
return terminal
|
|
phase = state.initialize(session.publication_state)
|
|
except HostStateError:
|
|
raise HostLifecycleError('state') from None
|
|
if getattr(session, '_failed_hold_replay', False):
|
|
if phase['phase'] == 'forward_started':
|
|
try:
|
|
phase = state.advance(
|
|
'forward_started', 'rollback_started',
|
|
phase['publication_state'],
|
|
forward_category=phase.get('forward_category'),
|
|
safe_detail=phase.get('safe_detail'),
|
|
)
|
|
except HostStateError:
|
|
raise HostLifecycleError('state') from None
|
|
if phase['phase'] == 'rollback_started':
|
|
try:
|
|
phase = state.advance(
|
|
'rollback_started', 'failed_hold',
|
|
phase['publication_state'],
|
|
forward_category=phase.get('forward_category'),
|
|
safe_detail='rollback_failed',
|
|
containment_confirmed=False,
|
|
)
|
|
except HostStateError:
|
|
raise HostLifecycleError('state') from None
|
|
elif phase['phase'] != 'failed_hold':
|
|
raise HostLifecycleError('state')
|
|
if phase['phase'] in {'succeeded', 'failed', 'rolled_back', 'failed_hold'}:
|
|
result, category, detail, identity = _terminal_result_for_phase(
|
|
session, phase,
|
|
)
|
|
try:
|
|
return state.publish_result(
|
|
result, safe_category=category, safe_detail=detail,
|
|
resulting_identity=identity,
|
|
)
|
|
except HostStateError:
|
|
raise HostLifecycleError('state') from None
|
|
|
|
current_phase = phase['phase']
|
|
attempt = None
|
|
cancellation = None
|
|
rollback_required = current_phase in {'forward_started', 'rollback_started'}
|
|
rollback_required = rollback_required or session.publication_state != 'original'
|
|
forward_category = phase.get('forward_category')
|
|
if rollback_required and forward_category is None:
|
|
forward_category = (
|
|
'restart_failed'
|
|
if session.request.action.value == 'restart' else 'apply_failed'
|
|
)
|
|
|
|
if not rollback_required:
|
|
try:
|
|
session.backup()
|
|
snapshot = lifecycle.preflight()
|
|
attempt = lifecycle.begin_attempt(snapshot)
|
|
session.revalidate_for_stop()
|
|
state.advance(
|
|
'prepared', 'forward_started', session.publication_state,
|
|
)
|
|
current_phase = 'forward_started'
|
|
lifecycle.stop_cleanly(attempt)
|
|
proof = lifecycle.authorize_replacement(
|
|
attempt, session.request.operation_id,
|
|
)
|
|
resulting_identity = session.replace(proof)
|
|
lifecycle.recreate_and_verify(attempt)
|
|
state.advance(
|
|
'forward_started', 'succeeded', session.publication_state,
|
|
)
|
|
current_phase = 'succeeded'
|
|
result = state.publish_result(
|
|
'succeeded', safe_category=None, safe_detail=None,
|
|
resulting_identity=resulting_identity,
|
|
)
|
|
return result
|
|
except BaseException as error:
|
|
is_cancellation = not isinstance(error, Exception)
|
|
if current_phase == 'succeeded':
|
|
if is_cancellation:
|
|
raise
|
|
raise HostLifecycleError('state') from None
|
|
if (
|
|
isinstance(error, HostStateError)
|
|
and error.category == 'uncertain'
|
|
):
|
|
if error.cancellation is not None:
|
|
raise error.cancellation
|
|
raise HostLifecycleError('state') from None
|
|
if attempt is None or not attempt.mutation_started:
|
|
if is_cancellation:
|
|
raise
|
|
category, detail = _preflight_failure(error)
|
|
try:
|
|
state.advance(
|
|
'prepared', 'failed', session.publication_state,
|
|
forward_category=category, safe_detail=detail,
|
|
)
|
|
current_phase = 'failed'
|
|
result = state.publish_result(
|
|
'failed', safe_category=category, safe_detail=detail,
|
|
resulting_identity=session.original_identity(),
|
|
)
|
|
return result
|
|
except HostStateError:
|
|
raise HostLifecycleError('state') from None
|
|
rollback_required = True
|
|
if is_cancellation:
|
|
cancellation = error
|
|
forward_category, _ = _forward_failure(session.request, error)
|
|
|
|
if attempt is None:
|
|
try:
|
|
session.backup()
|
|
snapshot = lifecycle.preflight()
|
|
attempt = lifecycle.begin_attempt(snapshot)
|
|
except BaseException as error:
|
|
if not isinstance(error, Exception) and not rollback_required:
|
|
raise
|
|
category, detail = _preflight_failure(error)
|
|
try:
|
|
if current_phase != 'rollback_started':
|
|
state.advance(
|
|
current_phase, 'rollback_started',
|
|
session.publication_state,
|
|
forward_category=forward_category or category,
|
|
safe_detail=detail,
|
|
)
|
|
current_phase = 'rollback_started'
|
|
state.publish_failed_hold(
|
|
forward_category=forward_category or category,
|
|
publication_state=session.publication_state,
|
|
containment_confirmed=False,
|
|
)
|
|
state.advance(
|
|
'rollback_started', 'failed_hold',
|
|
session.publication_state,
|
|
forward_category=forward_category or category,
|
|
safe_detail='rollback_failed',
|
|
containment_confirmed=False,
|
|
)
|
|
current_phase = 'failed_hold'
|
|
result = state.publish_result(
|
|
'failed_hold', safe_category='rollback_failed',
|
|
safe_detail='rollback_failed', resulting_identity=None,
|
|
)
|
|
if not isinstance(error, Exception):
|
|
raise error
|
|
return result
|
|
except HostStateError:
|
|
if not isinstance(error, Exception):
|
|
raise error
|
|
raise HostLifecycleError('state') from None
|
|
|
|
rollback_terminal_transition = False
|
|
try:
|
|
if current_phase != 'rollback_started':
|
|
state.advance(
|
|
current_phase, 'rollback_started', session.publication_state,
|
|
forward_category=forward_category, safe_detail=forward_category,
|
|
)
|
|
current_phase = 'rollback_started'
|
|
proof = lifecycle.quiesce_for_rollback(
|
|
attempt, session.request.operation_id,
|
|
)
|
|
original_identity = session.restore_backups(proof)
|
|
lifecycle.recreate_restored_and_verify(attempt)
|
|
detail = forward_category
|
|
rollback_terminal_transition = True
|
|
state.advance(
|
|
'rollback_started', 'rolled_back', session.publication_state,
|
|
forward_category=forward_category, safe_detail=detail,
|
|
)
|
|
current_phase = 'rolled_back'
|
|
result = state.publish_result(
|
|
'rolled_back', safe_category=forward_category, safe_detail=detail,
|
|
resulting_identity=original_identity,
|
|
)
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
return result
|
|
except BaseException as error:
|
|
if current_phase == 'rolled_back':
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
if not isinstance(error, Exception):
|
|
raise
|
|
raise HostLifecycleError('state') from None
|
|
if (
|
|
isinstance(error, HostStateError)
|
|
and error.category == 'uncertain'
|
|
and rollback_terminal_transition
|
|
):
|
|
if error.cancellation is not None:
|
|
raise error.cancellation
|
|
raise HostLifecycleError('state') from None
|
|
if (
|
|
isinstance(error, HostStateError)
|
|
and error.category == 'uncertain'
|
|
and error.cancellation is not None
|
|
):
|
|
cancellation = error.cancellation
|
|
if not isinstance(error, Exception):
|
|
cancellation = error
|
|
try:
|
|
state.publish_failed_hold(
|
|
forward_category=forward_category,
|
|
publication_state=session.publication_state,
|
|
containment_confirmed=False,
|
|
)
|
|
except HostStateError:
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
raise HostLifecycleError('state') from None
|
|
contained = lifecycle.contain_for_failed_hold(
|
|
attempt, session.request.operation_id,
|
|
)
|
|
try:
|
|
state.advance(
|
|
'rollback_started', 'failed_hold', session.publication_state,
|
|
forward_category=forward_category, safe_detail='rollback_failed',
|
|
containment_confirmed=contained,
|
|
)
|
|
current_phase = 'failed_hold'
|
|
result = state.publish_result(
|
|
'failed_hold', safe_category='rollback_failed',
|
|
safe_detail='rollback_failed', resulting_identity=None,
|
|
)
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
return result
|
|
except HostStateError:
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
raise HostLifecycleError('state') from None
|