from concurrent.futures import Future import os from pathlib import Path import shutil import subprocess import sys import tempfile import unittest from unittest import mock ROOT = Path(__file__).resolve().parents[1] APP_DIR = ROOT / 'app' sys.path.insert(0, str(APP_DIR)) import lifecycle_authority import postgres_runtime import supervisor import supervisor_instance def canonical(path): return os.path.normcase(os.path.realpath(os.path.abspath(os.fspath(path)))) class ImmediateExecutor: def submit(self, function): future = Future() try: future.set_result(function()) except BaseException as exc: future.set_exception(exc) return future class ControlledStartExecutor: def __init__(self): self.submissions = [] def submit(self, function): future = Future() self.submissions.append((function.__name__, future)) if function.__name__ == 'stop': future.set_result(function()) return future class LifecycleBackend: def __init__(self, probes=(), starts=()): self.probes = list(probes) self.starts = list(starts) self.start_calls = 0 self.stop_calls = 0 self.closed = False def probe(self): return self.probes.pop(0) def start(self): self.start_calls += 1 return self.starts.pop(0) def stop(self): self.stop_calls += 1 return postgres_runtime.StopResult(True, True, 'stopped') def close(self): self.closed = True class PostgresLifecycleTests(unittest.TestCase): def test_close_of_unowned_ready_controller_never_stops_cluster(self): backend = LifecycleBackend(probes=[ postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'preexisting ready', 'epoch'), ]) controller = postgres_runtime.PostgresController( backend, executor=ImmediateExecutor(), stable_ready_interval_sec=0, ) controller.tick(0) controller.tick(0) self.assertEqual(controller.state, postgres_runtime.PostgresState.READY) self.assertFalse(controller._owned_start) self.assertTrue(controller.close(timeout_sec=0.1)) self.assertEqual(backend.stop_calls, 0) self.assertTrue(backend.closed) self.assertTrue(controller.terminal) self.assertTrue(controller.authority_release_safe) self.assertIn('did not start', controller.detail) def test_close_of_inflight_accepted_start_still_compensates(self): backend = LifecycleBackend() executor = ControlledStartExecutor() controller = postgres_runtime.PostgresController(backend, executor=executor) controller.tick(0) executor.submissions[0][1].set_result( postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline') ) controller.tick(0) self.assertTrue(controller.has_inflight_start) executor.submissions[1][1].set_result( postgres_runtime.StartResult(True, 'accepted before close') ) self.assertTrue(controller.close(timeout_sec=0.2)) self.assertEqual([name for name, _ in executor.submissions], ['probe', 'start', 'stop']) self.assertEqual(backend.stop_calls, 1) self.assertTrue(backend.closed) def test_owned_ready_waits_for_child_then_stops_exactly_once(self): backend = LifecycleBackend( probes=[ postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline'), postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'owned ready', 'epoch'), ], starts=[postgres_runtime.StartResult(True, 'accepted')], ) controller = postgres_runtime.PostgresController( backend, executor=ImmediateExecutor(), stable_ready_interval_sec=0, ) for _ in range(4): controller.tick(0) self.assertEqual(controller.state, postgres_runtime.PostgresState.READY) self.assertTrue(controller._owned_start) self.assertFalse(controller.has_inflight_start) source = mock.Mock() source.source = 'child' source.stop.side_effect = [False, True] source.is_running.side_effect = [True, False] context = { 'lifecycle_phase': lifecycle_authority.PHASE_ACTIVE, 'activation_state': lifecycle_authority.PHASE_ACTIVE, 'start_gate_open': True, 'postgres_controller': controller, 'supervisor_config': {}, } with mock.patch.object(controller, 'close', wraps=controller.close) as close: self.assertFalse(supervisor.coordinated_shutdown([source], context)) close.assert_not_called() self.assertEqual(backend.stop_calls, 0) self.assertEqual(context['lifecycle_phase'], lifecycle_authority.PHASE_FAILED_HOLD) self.assertTrue(supervisor.coordinated_shutdown([source], context)) close.assert_called_once() self.assertEqual(backend.stop_calls, 1) class RetainedProcess: def __init__(self, command): self.command = list(command) self.closed = False def command_line(self): return list(self.command) def close(self): self.closed = True class SupervisorCommandVerificationTests(unittest.TestCase): def setUp(self): self.supervisor_path = canonical(APP_DIR / 'supervisor.py') self.config_path = canonical(APP_DIR / 'config.yaml') self.bootstrap_path = canonical(APP_DIR / 'runtime_bootstrap.py') def command(self, lifecycle_mode='foreground'): command = [ sys.executable, '-I', '-S', '-B', self.bootstrap_path, 'supervisor', '--', f'--runtime-bootstrap-entrypoint={self.supervisor_path}', f'--config={self.config_path}', '--with-postgres', ] if lifecycle_mode == 'background': command.append('--background-child') return command def lifecycle_metadata(self): return { 'pid': 123, 'process_creation_time': 'fixture', 'executable': canonical(sys.executable), 'supervisor_path': self.supervisor_path, 'config_path': self.config_path, } def instance_metadata(self, lifecycle_mode='foreground'): return { **self.lifecycle_metadata(), 'lifecycle_mode': lifecycle_mode, 'code_manifest': {}, 'code_manifest_sha256': '1' * 64, 'supervisor_sha256': '2' * 64, 'config_sha256': '3' * 64, } def verify_instance(self, command, lifecycle_mode='foreground', shutdown=False): retained = RetainedProcess(command) metadata = self.instance_metadata(lifecycle_mode) def digest(path): return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64 with mock.patch.object( supervisor_instance, 'validate_instance_metadata', return_value=metadata, ), mock.patch.object( supervisor_instance, 'verify_code_manifest', ), mock.patch.object( supervisor_instance, 'sha256_file', side_effect=digest, ), mock.patch.object( supervisor_instance, 'verify_retained_process', return_value=retained, ): result = supervisor_instance.verify_instance_process( metadata, allow_config_drift=shutdown, allow_code_drift=shutdown, ) return result def assert_lifecycle_rejected(self, command, pattern): retained = RetainedProcess(command) with mock.patch.object( lifecycle_authority, 'verify_retained_process', return_value=retained, ): with self.assertRaisesRegex(lifecycle_authority.LifecycleAuthorityError, pattern): lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata()) self.assertTrue(retained.closed) def assert_instance_rejected(self, command, pattern): retained = RetainedProcess(command) metadata = self.instance_metadata() def digest(path): return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64 with mock.patch.object( supervisor_instance, 'validate_instance_metadata', return_value=metadata, ), mock.patch.object( supervisor_instance, 'verify_code_manifest', ), mock.patch.object( supervisor_instance, 'sha256_file', side_effect=digest, ), mock.patch.object( supervisor_instance, 'verify_retained_process', return_value=retained, ): with self.assertRaisesRegex(supervisor_instance.InstanceMetadataError, pattern): supervisor_instance.verify_instance_process(metadata) self.assertTrue(retained.closed) def test_equals_form_foreground_child_and_shutdown_verification(self): command = self.command() retained = RetainedProcess(command) with mock.patch.object( lifecycle_authority, 'verify_retained_process', return_value=retained, ): lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata()) self.assertTrue(retained.closed) foreground = self.verify_instance(command) self.assertFalse(foreground.closed) foreground.close() background = self.verify_instance( self.command('background'), lifecycle_mode='background', shutdown=True, ) self.assertFalse(background.closed) background.close() def test_canonical_direct_script_argument_remains_allowed(self): command = [sys.executable, self.supervisor_path, f'--config={self.config_path}'] retained = RetainedProcess(command) with mock.patch.object( lifecycle_authority, 'verify_retained_process', return_value=retained, ): lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata()) self.assertTrue(retained.closed) instance = self.verify_instance(command) instance.close() def test_conflicting_or_duplicate_bindings_are_rejected(self): cases = ( ( [*self.command(), '--runtime-bootstrap-entrypoint', self.supervisor_path], 'exactly one bound supervisor script', ), ( [*self.command(), '--config', canonical(APP_DIR / 'other.yaml')], 'exactly one bound config argument', ), ) for command, pattern in cases: with self.subTest(pattern=pattern): self.assert_lifecycle_rejected(command, pattern) self.assert_instance_rejected(command, pattern) class DirectReadOnlyBytecodeTests(unittest.TestCase): def test_background_status_in_clean_copy_creates_no_application_bytecode(self): with tempfile.TemporaryDirectory() as temp_dir: app_dir = Path(temp_dir) / 'app' shutil.copytree( APP_DIR, app_dir, ignore=shutil.ignore_patterns('__pycache__', '*.pyc'), ) environment = os.environ.copy() environment.pop('PYTHONDONTWRITEBYTECODE', None) environment.pop('PYTHONPYCACHEPREFIX', None) completed = subprocess.run( [ sys.executable, str(app_dir / 'supervisor.py'), '--config', str(Path(temp_dir) / 'missing-config.yaml'), '--background-status', ], cwd=temp_dir, env=environment, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=30, check=False, ) self.assertNotEqual(completed.returncode, 0) self.assertEqual(list(app_dir.rglob('*.pyc')), []) self.assertEqual( [path for path in app_dir.rglob('__pycache__') if path.is_dir()], [], ) if __name__ == '__main__': unittest.main()