sashatrask
669 lines
36 KiB
Python
669 lines
36 KiB
Python
"""Real PG16 regression; run only in the trusted, offline disposable test image.
|
|
|
|
Main provisions a fresh truf-projection-loss-test-<32 hex> volume separately.
|
|
Run python -u -I -S -B /opt/truf/tests/container_projection_recovery_e2e.py
|
|
--budget-seconds 300. The budget is cooperative: it NEVER kills PostgreSQL or
|
|
releases uncertain lifecycle authority. A FAILED_HOLD may outlive that budget.
|
|
|
|
The actual schema/migration helpers, queries, transactions and locks are used.
|
|
Only the recovery module's manifest pin is substituted in memory for this fresh
|
|
fixture. Fault adapters raise only AFTER real SQL execution or real commit.
|
|
No accepted journal is deleted, no old output is rebuilt, and no application
|
|
initialized marker, supervisor, scanner, ingester or provider is started.
|
|
"""
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import runpy
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
|
|
OUTPUT = sys.stdout
|
|
STOPPED = False
|
|
IMAGE_PATH = Path('/opt/truf/tests/container_projection_recovery_e2e.py')
|
|
STAMP = '2026-09-16T00:00:00+00:00'
|
|
APPENDS = 38024
|
|
OLD_OFFSET = 97783145
|
|
|
|
|
|
def require(value, check):
|
|
if not value:
|
|
raise AssertionError(check)
|
|
|
|
|
|
def emit(**values):
|
|
try:
|
|
print(json.dumps(values, sort_keys=True), file=OUTPUT, flush=True)
|
|
except BaseException:
|
|
pass
|
|
|
|
|
|
def safe_error(error):
|
|
try:
|
|
line, state, current = 0, None, error
|
|
for _ in range(8):
|
|
tb = current.__traceback__
|
|
while tb is not None:
|
|
if tb.tb_frame.f_code.co_filename == str(IMAGE_PATH):
|
|
line = tb.tb_lineno
|
|
tb = tb.tb_next
|
|
candidate = getattr(current, 'sqlstate', None)
|
|
if state is None and isinstance(candidate, str) and re.fullmatch(r'[A-Z0-9]{5}', candidate):
|
|
state = candidate
|
|
current = current.__cause__ or current.__context__
|
|
if current is None:
|
|
break
|
|
name = type(error).__name__
|
|
if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,63}', name or ''):
|
|
name = 'other'
|
|
return {'error_type': next((i for i, kind in enumerate(
|
|
(TimeoutError, OSError, ValueError, RuntimeError, KeyboardInterrupt, AssertionError,
|
|
TypeError, KeyError, AttributeError, ImportError, LookupError), 1)
|
|
if isinstance(error, kind)), 0), 'line': line, 'sqlstate': state, 'error_class': name}
|
|
except BaseException:
|
|
return {'error_type': 0, 'line': 0, 'sqlstate': None, 'error_class': 'other'}
|
|
|
|
|
|
def hold_pause():
|
|
try:
|
|
time.sleep(2)
|
|
except BaseException:
|
|
pass
|
|
|
|
|
|
def stop_confirmed(backend, pg):
|
|
"""Retain this exact backend and the caller's two locks until positive stop."""
|
|
global STOPPED
|
|
attempts = 0
|
|
while True:
|
|
try:
|
|
attempts += 1
|
|
result = backend.stop()
|
|
require(result.completed is True and result.stopped is True, 'stop_result')
|
|
require(backend.probe().kind == pg.ProbeKind.STOPPED, 'stopped_probe')
|
|
backend.close()
|
|
STOPPED = True
|
|
emit(stage='stop', stopped=True, attempts=attempts)
|
|
return
|
|
except BaseException as error:
|
|
emit(stage='stop', failed_hold=True, attempts=attempts, **safe_error(error))
|
|
hold_pause()
|
|
|
|
|
|
def initialize_empty(runtime, config_path):
|
|
"""The real lifecycle child owns initialization compensation; never kill it."""
|
|
try:
|
|
child = subprocess.Popen(runtime._bootstrap_command(
|
|
'postgres-runtime', 'initialize-empty', '--config', str(config_path)),
|
|
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
except BaseException as error:
|
|
emit(stage='initialize_empty', failed_hold=True, **safe_error(error))
|
|
return None
|
|
attempts = 0
|
|
while True:
|
|
try:
|
|
code = child.wait(timeout=10)
|
|
emit(stage='initialize_empty', completed=code == 0, exit_code=code)
|
|
return code
|
|
except BaseException as error:
|
|
attempts += 1
|
|
emit(stage='initialize_empty', failed_hold=True, attempts=attempts, **safe_error(error))
|
|
hold_pause()
|
|
|
|
|
|
def checkpoint(runtime, deadline):
|
|
require(runtime._shutdown_requested is False, 'shutdown_requested')
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError('driver_budget')
|
|
|
|
|
|
def identifier(name):
|
|
require(isinstance(name, str) and re.fullmatch(r'[a-z_][a-z0-9_]*', name), 'fixture_identifier')
|
|
return '"' + name + '"'
|
|
|
|
|
|
def digest(payload):
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def metadata(connection):
|
|
row = connection.execute("""SELECT pg_catalog.row_to_json(s) AS stream,
|
|
pg_catalog.row_to_json(c) AS cursor FROM public.projection_streams s
|
|
JOIN public.projection_cursors c USING (stream_name)
|
|
WHERE s.stream_name = 'found_secrets'""").fetchone()
|
|
parsed = {}
|
|
for key in ('stream', 'cursor'):
|
|
value = row[key]
|
|
if isinstance(value, str):
|
|
value = json.loads(value)
|
|
require(isinstance(value, dict), 'metadata_object')
|
|
parsed[key] = value
|
|
return parsed
|
|
|
|
|
|
def proof(connection, check):
|
|
"""Independent, bounded-fixture whole-row digests; never return raw rows."""
|
|
tables = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c
|
|
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
WHERE n.nspname = 'public' AND c.relkind = 'r' ORDER BY c.relname""").fetchall()
|
|
result = {'tables': {}, 'sequences': {'public': {}}}
|
|
for table in tables:
|
|
check()
|
|
name = table['name']
|
|
where = " WHERE t.stream_name <> 'found_secrets'" if name in ('projection_streams', 'projection_cursors') else ''
|
|
row = connection.execute("""SELECT count(*) AS rows, pg_catalog.encode(pg_catalog.sha256(
|
|
pg_catalog.convert_to(COALESCE(string_agg(h, '' ORDER BY h), ''), 'UTF8')), 'hex') AS sha256
|
|
FROM (SELECT pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(
|
|
pg_catalog.row_to_json(t)::text, 'UTF8')), 'hex') COLLATE "C" AS h FROM public."""
|
|
+ identifier(name) + ' AS t' + where + ') AS hashes').fetchone()
|
|
require(0 <= row['rows'] <= 100000, 'bounded_fixture')
|
|
result['tables'][name] = row
|
|
sequences = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c
|
|
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
WHERE n.nspname = 'public' AND c.relkind = 'S' ORDER BY c.relname""").fetchall()
|
|
for row in sequences:
|
|
result['sequences']['public'][row['name']] = connection.execute(
|
|
'SELECT last_value, is_called FROM public.' + identifier(row['name'])).fetchone()
|
|
return result
|
|
|
|
|
|
def journal_snapshot(runtime, recovery):
|
|
from container_import import _input, _json
|
|
|
|
path = runtime.DATA / 'config' / recovery.JOURNAL_NAME
|
|
with _input(path, runtime) as (handle, before):
|
|
require(0 < before[4] <= recovery.MAX_JOURNAL, 'journal_bound')
|
|
raw = handle.read(recovery.MAX_JOURNAL + 1)
|
|
value = _json(raw)
|
|
require(raw == recovery._encoded(value) and value['record']['state'] == 'PREPARED'
|
|
and value['sha256'] == digest(recovery._encoded(value['record'])), 'durable_journal')
|
|
return raw, before, value
|
|
|
|
|
|
class ObservedConnection:
|
|
"""Delegate everything to psycopg; inject only after a real operation."""
|
|
def __init__(self, connection, runtime, recovery, check, *, after_update=False, lost_ack=False):
|
|
self.connection, self.runtime, self.recovery, self.check = connection, runtime, recovery, check
|
|
self.after_update, self.lost_ack = after_update, lost_ack
|
|
self.updates, self.commits, self.injected = 0, 0, False
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.connection, name)
|
|
|
|
def execute(self, query, *args, **kwargs):
|
|
result = self.connection.execute(query, *args, **kwargs)
|
|
sql = ' '.join(query.split()).upper() if isinstance(query, str) else ''
|
|
if sql.startswith('UPDATE '):
|
|
self.updates += 1
|
|
if self.after_update and sql.startswith('UPDATE PUBLIC.PROJECTION_STREAMS '):
|
|
require(result.rowcount == 1, 'real_first_update')
|
|
journal_snapshot(self.runtime, self.recovery)
|
|
self.after_update, self.injected = False, True
|
|
raise OSError('synthetic_transport_after_real_update')
|
|
self.check()
|
|
return result
|
|
|
|
@contextlib.contextmanager
|
|
def transaction(self, *args, **kwargs):
|
|
with self.connection.transaction(*args, **kwargs) as transaction:
|
|
yield transaction
|
|
self.commits += 1
|
|
if self.lost_ack:
|
|
self.lost_ack, self.injected = False, True
|
|
raise OSError('synthetic_ack_loss_after_real_commit')
|
|
|
|
|
|
def seed_history(connection, runtime, importer):
|
|
"""Seed real production tables; historical byte proofs need no old files."""
|
|
with connection.transaction():
|
|
connection.execute("""INSERT INTO public.target_scans(
|
|
id, scan_event_id, scan_event_hash, source, target, normalized_target, scan_type,
|
|
status, ended_at, findings_count, raw_result_storage, created_at)
|
|
SELECT n, lpad(to_hex(n), 32, '0'), encode(sha256(convert_to('event-' || n, 'UTF8')), 'hex'),
|
|
'fixture', 'fixture:' || n, 'fixture:' || n, 'fixture', 'found', %s, 1, 'normalized_v2', %s
|
|
FROM generate_series(1, 38024) AS g(n)""", (STAMP, STAMP))
|
|
connection.execute("""INSERT INTO public.findings(
|
|
id, target_scan_id, source, target, detector_name, detector_type, verified,
|
|
raw_secret, redacted_secret, secret_hash, finding_uid, created_at)
|
|
SELECT id, id, 'fixture', target, 'Fixture', 'fixture', 0,
|
|
'synthetic-only-' || id, 'fixture', encode(sha256(convert_to('synthetic-only-' || id, 'UTF8')), 'hex'),
|
|
encode(sha256(convert_to('finding-' || id, 'UTF8')), 'hex'), %s FROM public.target_scans""", (STAMP,))
|
|
connection.execute("""INSERT INTO public.scan_result_compat(
|
|
target_scan_id, schema_version, metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at)
|
|
SELECT id, 2, '{}', encode(sha256(convert_to('{}', 'UTF8')), 'hex'), 2, 'exact', %s
|
|
FROM public.target_scans""", (STAMP,))
|
|
connection.execute("""INSERT INTO public.finding_compat_payloads(
|
|
finding_id, raw_value, extension_json, payload_sha256, payload_bytes, payload_omitted, created_at)
|
|
SELECT id, raw_secret, '{}', encode(sha256(convert_to(raw_secret, 'UTF8')), 'hex'),
|
|
octet_length(raw_secret), 0, %s FROM public.findings""", (STAMP,))
|
|
connection.execute("""INSERT INTO public.projection_jobs(
|
|
id, job_kind, event_id, event_hash, target_scan_id, status, required_stream_mask,
|
|
capacity_items, capacity_bytes, capacity_released, attempts, created_at, updated_at, completed_at)
|
|
SELECT id, 'scan_event', scan_event_id, scan_event_hash, id, 'completed', 2,
|
|
1, 65536, 1, 1, %s, %s, %s FROM public.target_scans""", (STAMP, STAMP, STAMP))
|
|
connection.execute("""WITH positions AS (
|
|
SELECT id, event_id, event_hash, (id - 1) / 2716 AS generation,
|
|
CASE WHEN id > 35308 THEN 97783145::bigint ELSE 134217728::bigint END AS bytes,
|
|
(id - 1) %% 2716 AS ordinal FROM public.projection_jobs)
|
|
INSERT INTO public.projection_appends(id, job_id, stream_name, event_id, event_hash,
|
|
generation, byte_offset, byte_length, payload_sha256, record_count, state, prepared_at, appended_at)
|
|
SELECT id, id, 'found_secrets', event_id, event_hash, generation,
|
|
bytes * ordinal / 2716, bytes * (ordinal + 1) / 2716 - bytes * ordinal / 2716,
|
|
encode(sha256(convert_to('historical-output-' || id, 'UTF8')), 'hex'),
|
|
1, 'appended', %s, %s FROM positions""", (STAMP, STAMP))
|
|
connection.execute("""INSERT INTO public.projection_rotations(
|
|
id, stream_name, from_generation, to_generation, source_bytes, segment_relative_path, state, created_at, completed_at)
|
|
SELECT n + 1, 'found_secrets', n, n + 1, 134217728,
|
|
'found_secrets.g' || lpad(n::text, 6, '0') || '.jsonl', 'completed', %s, %s
|
|
FROM generate_series(0, 12) AS g(n)""", (STAMP, STAMP))
|
|
for i in range(18):
|
|
provider = f'p{i:02d}'
|
|
for suffix, filename in (('results', 'Results.jsonl'), ('status', 'Checked.txt')):
|
|
if suffix == 'status' and i >= 13:
|
|
continue
|
|
name = f'keycheck:{provider}:{suffix}'
|
|
connection.execute("""INSERT INTO public.projection_streams(
|
|
stream_name, base_relative_path, current_generation, rotation_bytes, max_generations, created_at, updated_at)
|
|
VALUES (%s, %s, 0, 33554432, 16, %s, %s)""", (name, f'{provider}/{provider}{filename}', STAMP, STAMP))
|
|
connection.execute("""INSERT INTO public.projection_cursors(stream_name, generation, committed_offset, updated_at)
|
|
VALUES (%s, 0, %s, %s)""", (name, 1000 + i if suffix == 'status' else 0, STAMP))
|
|
connection.execute("UPDATE public.projection_streams SET current_generation = 13 WHERE stream_name = 'found_secrets'")
|
|
connection.execute("""UPDATE public.projection_cursors SET generation = 13, committed_offset = 97783145,
|
|
last_append_id = 38024, last_job_id = 38024,
|
|
last_event_id = (SELECT event_id FROM public.projection_jobs WHERE id = 38024),
|
|
last_event_hash = (SELECT event_hash FROM public.projection_jobs WHERE id = 38024)
|
|
WHERE stream_name = 'found_secrets'""")
|
|
for worker in ('result_ingester', 'jsonl_projector'):
|
|
connection.execute("""INSERT INTO public.pipeline_leases(worker_name, generation, lease_token,
|
|
supervisor_instance_id, owner_pid, owner_creation_time, owner_executable, state,
|
|
acquired_at, heartbeat_at, lease_expires_at, updated_at)
|
|
VALUES (%s, 7, 'synthetic-expired', 'synthetic-expired', 999999, 'synthetic',
|
|
'/synthetic/expired', 'ready', %s, %s, %s, %s)""", (worker, STAMP, STAMP, STAMP, STAMP))
|
|
for table in ('target_scans', 'findings', 'projection_jobs', 'projection_appends', 'projection_rotations'):
|
|
connection.execute('SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence(%s, %s), '
|
|
+ '(SELECT max(id) FROM public.' + identifier(table) + '), true)', ('public.' + table, 'id'))
|
|
status_files = []
|
|
for i in range(13):
|
|
provider = f'p{i:02d}'
|
|
directory = runtime.DATA / 'runtime-linux/keychecks' / provider
|
|
directory.mkdir(mode=0o700)
|
|
path = directory / f'{provider}Checked.txt'
|
|
importer._write(runtime, path, b'synthetic status snapshot\n')
|
|
status_files.append(path)
|
|
return status_files
|
|
|
|
|
|
def fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, system_identifier):
|
|
# Required source labels are inert format metadata, never opened as paths.
|
|
paths = list(status_files)
|
|
for name in sorted(importer.REQUIRED_FILES):
|
|
path = runtime.DATA / name
|
|
if not os.path.lexists(path):
|
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
importer._write(runtime, path, b'')
|
|
paths.append(path)
|
|
files = []
|
|
for path in paths:
|
|
with importer._input(path, runtime) as (handle, before):
|
|
raw = handle.read(4096)
|
|
require(len(raw) == before[4], 'status_fixture_bound')
|
|
files.append({'path': path.relative_to(runtime.DATA).as_posix(), 'size': len(raw), 'sha256': digest(raw)})
|
|
# Historical bytes are intentionally not materialized; this is not a restore test.
|
|
files.append({'path': 'runtime-linux/results/found_secrets.jsonl', 'size': OLD_OFFSET,
|
|
'sha256': digest(b'intentionally-unavailable-synthetic-output')})
|
|
value = {'format': 'truf-windows-snapshot-v1',
|
|
'source': {'root': r'D:\truf', 'postgres_data_dir': r'S:\postgres-data',
|
|
'supervisor_stopped': True, 'postgres_stopped': True},
|
|
'database': {'version_num': connection.execute("SELECT current_setting('server_version_num')::int AS version").fetchone()['version'],
|
|
'system_identifier': '1' if system_identifier != '1' else '2',
|
|
'database_name': 'synthetic_source', 'user_name': 'synthetic_source', 'port': 15432,
|
|
'data_directory': r'S:\postgres-data', 'bytes': 6, 'sha256': digest(b'PGDMPx'),
|
|
'table_counts': {name: item['rows'] + int(name in ('projection_streams', 'projection_cursors'))
|
|
for name, item in baseline['tables'].items()},
|
|
'sequence_states': baseline['sequences'], 'sequence_count': len(baseline['sequences']['public'])},
|
|
'archive': {'bytes': sum(item['size'] for item in files) + 10240,
|
|
'sha256': digest(b'synthetic archive identity')}, 'files': files}
|
|
raw = importer._encoded(value)
|
|
pin = digest(raw)
|
|
importer._manifest(raw, pin)
|
|
importer._write(runtime, runtime.DATA / 'config/windows-import-manifest.json', raw)
|
|
recovery.APPROVED_MANIFEST_SHA256 = pin
|
|
return pin
|
|
|
|
|
|
def exercise_projector(connection, db, runtime, config, check):
|
|
from jsonl_projector import JsonlProjector
|
|
from migrate_runtime_safety import initialize_projection_cursors_from_existing_files, require_legacy_cutover_clear
|
|
|
|
# This is a real offline cutover after recovery, not a mocked readiness gate.
|
|
db.require_runtime_safety_schema()
|
|
cursors = initialize_projection_cursors_from_existing_files(db, config)
|
|
legacy = require_legacy_cutover_clear(db, config)
|
|
migrations = db.conn.execute('SELECT version, code_sha256 FROM runtime_schema_migrations ORDER BY version').fetchall()
|
|
db.conn.commit()
|
|
db.record_final_cutover({'legacy': legacy, 'projection_cursors': cursors,
|
|
'schema_migrations': [dict(row) for row in migrations]})
|
|
db.require_final_cutover()
|
|
check()
|
|
event_id, event_hash, finding_uid = 'f' * 32, digest(b'future-event'), digest(b'future-finding')
|
|
with connection.transaction():
|
|
scan_id = connection.execute("""INSERT INTO public.target_scans(
|
|
scan_event_id, scan_event_hash, target, normalized_target, scan_type, status, ended_at,
|
|
findings_count, raw_result_storage, created_at)
|
|
VALUES (%s, %s, 'fixture:future', 'fixture:future', 'fixture', 'found', %s, 1, 'normalized_v2', %s)
|
|
RETURNING id""", (event_id, event_hash, STAMP, STAMP)).fetchone()['id']
|
|
connection.execute("""INSERT INTO public.scan_result_compat(target_scan_id, schema_version,
|
|
metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at)
|
|
VALUES (%s, 2, '{}', %s, 2, 'exact', %s)""", (scan_id, digest(b'{}'), STAMP))
|
|
finding_id = connection.execute("""INSERT INTO public.findings(target_scan_id, detector_name,
|
|
detector_type, verified, raw_secret, redacted_secret, finding_uid, created_at)
|
|
VALUES (%s, 'Fixture', 'fixture', 0, 'synthetic-future', 'fixture', %s, %s) RETURNING id""",
|
|
(scan_id, finding_uid, STAMP)).fetchone()['id']
|
|
connection.execute("""INSERT INTO public.finding_compat_payloads(finding_id, raw_value,
|
|
extension_json, payload_sha256, payload_bytes, payload_omitted, created_at)
|
|
VALUES (%s, 'synthetic-future', '{}', %s, 16, 0, %s)""", (finding_id, digest(b'synthetic-future'), STAMP))
|
|
job_id = connection.execute("""INSERT INTO public.projection_jobs(job_kind, event_id, event_hash,
|
|
target_scan_id, status, required_stream_mask, capacity_items, capacity_bytes, created_at, updated_at)
|
|
VALUES ('scan_event', %s, %s, %s, 'pending', 2, 1, 65536, %s, %s) RETURNING id""",
|
|
(event_id, event_hash, scan_id, STAMP, STAMP)).fetchone()['id']
|
|
connection.execute("""UPDATE public.pipeline_capacity SET projection_items = projection_items + 1,
|
|
projection_bytes = projection_bytes + 65536 WHERE id = 1""")
|
|
worker = JsonlProjector(db, str(runtime.DATA / 'runtime-linux/results'), 'projection-loss-e2e',
|
|
keycheck_dir=str(runtime.DATA / 'runtime-linux/keychecks'))
|
|
try:
|
|
worker.start()
|
|
require(worker.process_one() is True, 'projector_processed')
|
|
require(worker.process_one() is False, 'projector_idle')
|
|
row = connection.execute("""SELECT a.generation, a.byte_offset, a.byte_length,
|
|
a.payload_sha256, a.state, j.status, j.capacity_released
|
|
FROM public.projection_appends a JOIN public.projection_jobs j ON j.id = a.job_id
|
|
WHERE a.job_id = %s AND a.stream_name = 'found_secrets'""", (job_id,)).fetchone()
|
|
path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
|
runtime.private_path(path)
|
|
require(path.stat().st_size < 65536, 'bounded_new_output')
|
|
raw = path.read_bytes()
|
|
require(row and row['generation'] == 14 and row['byte_offset'] == 0
|
|
and row['byte_length'] == len(raw) and row['payload_sha256'] == digest(raw)
|
|
and row['state'] == 'appended' and row['status'] == 'completed' and row['capacity_released'] == 1,
|
|
'projector_fenced_append')
|
|
require(json.loads(raw)['finding_uid'] == finding_uid, 'projector_real_payload')
|
|
cursor = metadata(connection)['cursor']
|
|
require(cursor['generation'] == 14 and cursor['committed_offset'] == len(raw)
|
|
and cursor['last_job_id'] == job_id, 'projector_cursor')
|
|
capacity = connection.execute('SELECT projection_items, projection_bytes FROM public.pipeline_capacity WHERE id = 1').fetchone()
|
|
require(capacity == {'projection_items': 0, 'projection_bytes': 0}, 'projector_capacity')
|
|
emit(stage='projector', passed=True, generation=14, records=1, bytes=len(raw))
|
|
finally:
|
|
worker.stop()
|
|
|
|
|
|
def run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check):
|
|
import container_import as importer
|
|
import container_projection_recovery as recovery
|
|
from runtime_security import PrivateFileLock
|
|
from scanner_db import ScannerDB, migrate_runtime_safety_schema
|
|
|
|
schema_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False)
|
|
resources.append(schema_db)
|
|
require(schema_db.enabled and schema_db.conn.is_postgres, 'real_schema_connection')
|
|
try:
|
|
migrate_runtime_safety_schema(schema_db, initialize_base=True)
|
|
schema_db.require_runtime_safety_schema()
|
|
finally:
|
|
schema_db.close()
|
|
emit(stage='schema', passed=True)
|
|
check()
|
|
status_files = seed_history(connection, runtime, importer)
|
|
emit(stage='seed', passed=True)
|
|
baseline = proof(connection, check)
|
|
emit(stage='proof', passed=True, tables=len(baseline['tables']), sequences=len(baseline['sequences']['public']))
|
|
before = metadata(connection)
|
|
require(before['stream']['current_generation'] == before['cursor']['generation'] == 13
|
|
and before['cursor']['committed_offset'] == OLD_OFFSET, 'reviewed_before')
|
|
require(baseline['tables']['projection_appends']['rows'] == APPENDS
|
|
and baseline['tables']['projection_rotations']['rows'] == 13
|
|
and baseline['tables']['projection_append_audit']['rows'] == 0, 'reviewed_history')
|
|
original_pin = recovery.APPROVED_MANIFEST_SHA256
|
|
pin = fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, identity['system_identifier'])
|
|
journal_path = runtime.DATA / 'config' / recovery.JOURNAL_NAME
|
|
status_before = {path: (path.stat().st_ino, path.read_bytes()) for path in status_files}
|
|
emit(stage='fixture', passed=True, streams=34, appends=APPENDS, rotations=13,
|
|
tables=len(baseline['tables']), sequences=len(baseline['sequences']['public']))
|
|
|
|
def recover(observed):
|
|
return recovery.recover_found_secrets_projection(runtime, observed,
|
|
system_identifier=identity['system_identifier'], manifest_sha256=pin,
|
|
initialize_lock=initialize_lock, authority_lock=authority_lock)
|
|
|
|
def refused(observed):
|
|
try:
|
|
recover(observed)
|
|
except recovery.ProjectionRecoveryError as error:
|
|
if not observed.injected and (observed.after_update or observed.lost_ack):
|
|
emit(stage='fault_not_reached', passed=False, **safe_error(error))
|
|
return
|
|
raise AssertionError('expected_recovery_refusal')
|
|
|
|
def alone():
|
|
while True:
|
|
check()
|
|
connection.execute('SELECT pg_catalog.pg_stat_clear_snapshot()')
|
|
count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_stat_activity
|
|
WHERE backend_type = 'client backend' AND pid <> pg_backend_pid()""").fetchone()['count']
|
|
if count == 0:
|
|
return
|
|
time.sleep(0.05)
|
|
|
|
try:
|
|
alone()
|
|
contender = connect()
|
|
resources.append(contender)
|
|
try:
|
|
observed = ObservedConnection(connection, runtime, recovery, check)
|
|
refused(observed)
|
|
require(observed.updates == 0, 'other_client_no_updates')
|
|
with contender.transaction():
|
|
contender.execute('LOCK TABLE public.projection_streams IN ROW EXCLUSIVE MODE')
|
|
observed = ObservedConnection(connection, runtime, recovery, check)
|
|
refused(observed)
|
|
require(observed.updates == 0, 'writer_contention_no_updates')
|
|
finally:
|
|
contender.close()
|
|
alone()
|
|
with PrivateFileLock(str(runtime.DATA / 'runtime-linux/results/.jsonl-projector.lock')):
|
|
observed = ObservedConnection(connection, runtime, recovery, check)
|
|
refused(observed)
|
|
require(observed.updates == 0, 'projector_file_lock_no_updates')
|
|
require(metadata(connection) == before and proof(connection, check) == baseline
|
|
and not os.path.lexists(journal_path), 'contention_unchanged')
|
|
emit(stage='contention', passed=True, cases=3, updates=0)
|
|
|
|
observed = ObservedConnection(connection, runtime, recovery, check, after_update=True)
|
|
refused(observed)
|
|
require(observed.injected and observed.updates == 1 and observed.commits == 0, 'rollback_fault_window')
|
|
require(int(connection.info.transaction_status) == 0 and metadata(connection) == before
|
|
and proof(connection, check) == baseline, 'both_rows_rolled_back')
|
|
journal = journal_snapshot(runtime, recovery)
|
|
require(journal[2]['record']['before'] == before, 'journal_before')
|
|
emit(stage='rollback', passed=True, updates=1, commits=0, journal_retained=True)
|
|
|
|
observed = ObservedConnection(connection, runtime, recovery, check, lost_ack=True)
|
|
refused(observed)
|
|
require(observed.injected and observed.updates == 2 and observed.commits == 1, 'real_commit_before_ack_loss')
|
|
after = metadata(connection)
|
|
require(after == journal[2]['record']['after'] and proof(connection, check) == baseline
|
|
and journal_snapshot(runtime, recovery) == journal, 'committed_despite_ack_loss')
|
|
emit(stage='lost_ack', passed=True, updates=2, commits=1, history_unchanged=True)
|
|
|
|
observed = ObservedConnection(connection, runtime, recovery, check)
|
|
result = recover(observed)
|
|
require(result['status'] == 'already-committed' and observed.updates == 0
|
|
and result['journal_sha256'] == digest(journal[0]) and metadata(connection) == after
|
|
and proof(connection, check) == baseline and journal_snapshot(runtime, recovery) == journal,
|
|
'idempotent_readonly_retry')
|
|
require({path: (path.stat().st_ino, path.read_bytes()) for path in status_files} == status_before,
|
|
'other_output_unchanged')
|
|
emit(stage='retry', passed=True, updates=0, journal_unchanged=True, history_unchanged=True, sequences_unchanged=True)
|
|
|
|
writer_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False)
|
|
resources.append(writer_db)
|
|
require(writer_db.enabled and writer_db.conn.is_postgres, 'real_writer_connection')
|
|
try:
|
|
exercise_projector(connection, writer_db, runtime, config, check)
|
|
finally:
|
|
writer_db.close()
|
|
alone()
|
|
advanced, advanced_proof = metadata(connection), proof(connection, check)
|
|
future_path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
|
future_bytes, future_inode = future_path.read_bytes(), future_path.stat().st_ino
|
|
observed = ObservedConnection(connection, runtime, recovery, check)
|
|
refused(observed)
|
|
require(observed.updates == 0 and advanced['cursor']['committed_offset'] > 0
|
|
and metadata(connection) == advanced and proof(connection, check) == advanced_proof
|
|
and (future_path.read_bytes(), future_path.stat().st_ino) == (future_bytes, future_inode)
|
|
and journal_snapshot(runtime, recovery) == journal, 'future_output_not_rewound')
|
|
emit(stage='future_output', passed=True, updates=0, advanced_cursor_retained=True, journal_unchanged=True)
|
|
finally:
|
|
recovery.APPROVED_MANIFEST_SHA256 = original_pin
|
|
|
|
|
|
def main():
|
|
global OUTPUT
|
|
# Native pg_ctl also inherits fd 1/2: Python stream redirection alone is insufficient.
|
|
OUTPUT = os.fdopen(os.dup(1), 'w', encoding='utf-8', buffering=1)
|
|
sink = os.open(os.devnull, os.O_WRONLY)
|
|
try:
|
|
os.dup2(sink, 1)
|
|
os.dup2(sink, 2)
|
|
finally:
|
|
os.close(sink)
|
|
parser = argparse.ArgumentParser(allow_abbrev=False)
|
|
parser.add_argument('--budget-seconds', type=int, default=300)
|
|
args = parser.parse_args()
|
|
require(30 <= args.budget_seconds <= 3600, 'budget')
|
|
started = time.monotonic()
|
|
deadline = started + args.budget_seconds
|
|
require(sys.platform == 'linux' and os.geteuid() == os.getuid() == 10001
|
|
and os.getegid() == os.getgid() == 10001, 'test_identity')
|
|
require(Path(__file__) == IMAGE_PATH and sys.flags.isolated and sys.flags.no_site
|
|
and sys.flags.dont_write_bytecode, 'test_image')
|
|
require({name for _, name in socket.if_nameindex()} == {'lo'}, 'offline_test')
|
|
raw_mounts = Path('/proc/self/mountinfo').read_bytes()
|
|
require(len(raw_mounts) <= 1024 * 1024, 'mount_bound')
|
|
mounts = [line.split() for line in raw_mounts.decode('utf-8', errors='strict').splitlines()]
|
|
data = [row for row in mounts if len(row) > 6 and (row[4] == '/data' or row[4].startswith('/data/'))]
|
|
require(len(data) == 1 and data[0][4] == '/data' and '-' in data[0]
|
|
and re.fullmatch(r'/var/lib/docker/volumes/truf-projection-loss-test-[a-f0-9]{32}/_data', data[0][3])
|
|
and data[0][data[0].index('-') + 1] == 'ext4', 'fresh_test_volume')
|
|
runtime = SimpleNamespace(**runpy.run_path('/opt/truf/app/container_runtime.py'))
|
|
runtime.require_container()
|
|
runtime.private_path(IMAGE_PATH.parent, directory=True)
|
|
runtime.private_path(IMAGE_PATH)
|
|
runtime._shutdown_requested = False
|
|
for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
|
|
signal.signal(sig, lambda *_: setattr(runtime, '_shutdown_requested', True))
|
|
|
|
def fresh():
|
|
runtime.private_path(runtime.DATA / 'postgres-linux', directory=True)
|
|
require(not any((runtime.DATA / 'postgres-linux').iterdir()), 'empty_pgdata')
|
|
require(not any((runtime.DATA / 'runtime-linux/results').iterdir()), 'empty_results')
|
|
for name in ('runtime-linux/postgres/cluster_identity.json', 'initialized.json',
|
|
'config/windows-import-manifest.json', 'config/found-secrets-loss-g13-g14.prepared.json'):
|
|
require(not os.path.lexists(runtime.DATA / name), 'fresh_fixture')
|
|
|
|
fresh()
|
|
config_path = runtime.DEFAULT_CONFIG
|
|
config = runtime.prepare_environment(config_path)
|
|
os.environ.update(TRUF_DB_STATEMENT_TIMEOUT_MS='120000', TRUF_DB_LOCK_TIMEOUT_MS='5000',
|
|
TRUF_DB_IDLE_TRANSACTION_TIMEOUT_MS='300000')
|
|
import postgres_runtime as pg
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from runtime_security import ClusterAuthorityLock, PrivateFileLock
|
|
|
|
def connect():
|
|
return psycopg.connect(os.environ['SCANNER_DB_URL'], autocommit=True, row_factory=dict_row,
|
|
connect_timeout=5, application_name='truf-projection-loss-e2e', tcp_user_timeout=30000,
|
|
options='-c search_path=public -c statement_timeout=120000 -c lock_timeout=5000 '
|
|
'-c idle_in_transaction_session_timeout=300000 -c row_security=off '
|
|
'-c log_min_error_statement=panic -c log_min_messages=panic '
|
|
'-c log_statement=none -c log_min_duration_statement=-1')
|
|
|
|
resources = []
|
|
check = lambda: checkpoint(runtime, deadline)
|
|
with PrivateFileLock(str(runtime.INITIALIZE_LOCK)) as initialize_lock:
|
|
fresh()
|
|
authority_lock = ClusterAuthorityLock(config, endpoint_dsn=os.environ['SCANNER_DB_URL'])
|
|
code = initialize_empty(runtime, config_path)
|
|
while True:
|
|
try:
|
|
authority_lock.acquire()
|
|
break
|
|
except BaseException as error:
|
|
emit(stage='authority', failed_hold=True, **safe_error(error))
|
|
hold_pause()
|
|
try:
|
|
backend = None
|
|
while backend is None:
|
|
try:
|
|
backend = pg.PostgresBackend(config, stop_timeout_sec=60)
|
|
except BaseException as error:
|
|
emit(stage='backend', failed_hold=True, **safe_error(error))
|
|
hold_pause()
|
|
try:
|
|
require(code == 0, 'initialize_empty_exit')
|
|
check()
|
|
require(backend.probe().kind == pg.ProbeKind.STOPPED, 'initial_stopped_probe')
|
|
identity = pg.verify_cluster_identity(config)
|
|
require(identity['pg_major'] == 16 and identity['data_directory'] == '/data/postgres-linux', 'bound_fixture')
|
|
require(backend.start().accepted is True, 'direct_backend_start')
|
|
while True:
|
|
check()
|
|
probe = backend.probe()
|
|
if probe.kind == pg.ProbeKind.READY:
|
|
break
|
|
require(probe.kind == pg.ProbeKind.RECOVERING, 'authenticated_start')
|
|
time.sleep(0.1)
|
|
emit(stage='start', ready=True)
|
|
connection = connect()
|
|
resources.append(connection)
|
|
count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_class c
|
|
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
WHERE n.nspname = 'public' AND c.relkind IN ('r','p','f')""").fetchone()['count']
|
|
require(count == 0, 'virgin_schema')
|
|
run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check)
|
|
require(not os.path.lexists(runtime.INITIALIZED), 'no_application_marker')
|
|
finally:
|
|
try:
|
|
for resource in reversed(resources):
|
|
try:
|
|
resource.close()
|
|
except BaseException as error:
|
|
emit(stage='client_close', failed_hold=True, **safe_error(error))
|
|
finally:
|
|
stop_confirmed(backend, pg)
|
|
finally:
|
|
authority_lock.release()
|
|
emit(stage='finished', passed=True, stopped=STOPPED, application_initialized=False,
|
|
elapsed_ms=round((time.monotonic() - started) * 1000))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
result = main()
|
|
except BaseException as error:
|
|
emit(stage='finished', passed=False, stopped=STOPPED, **safe_error(error))
|
|
result = 1
|
|
raise SystemExit(result)
|