import os from types import SimpleNamespace from unittest import mock import pytest from test_supervisor_foreground_shutdown import ( isolated_runtime, lifecycle_context, runtime, ) import process_identity import supervisor ManagedSource = supervisor.ManagedSource BoundedRotatingLogPump = supervisor.BoundedRotatingLogPump @pytest.fixture(autouse=True) def no_startup_io(monkeypatch, isolated_runtime): blocked = mock.Mock(side_effect=AssertionError('unexpected startup I/O')) for name in ( 'exact_process_identity_state', 'process_running_status', 'read_private_json', 'durable_unlink', 'verify_instance_process', 'send_control_request', ): monkeypatch.setattr(supervisor, name, blocked) for name in ('require_private_directory', 'append_bounded_log_record', 'finalize_source_runs'): monkeypatch.setattr(supervisor, name, mock.Mock()) monkeypatch.setattr(supervisor, 'BoundedRotatingLogPump', blocked) monkeypatch.setattr(process_identity, 'open_process', blocked) @pytest.fixture def launch(monkeypatch, tmp_path): source = ManagedSource( 'fixture', str(tmp_path / 'config.yaml'), str(tmp_path), str(tmp_path), {'log_dir': str(tmp_path / 'logs'), 'state_dir': str(tmp_path / 'state')}, {'enabled': True}, ) source.build_command = mock.Mock(return_value=['mock-child']) source.build_env = mock.Mock(return_value={}) process = mock.Mock(pid=41001, stdout=mock.sentinel.stream) process.poll.return_value = None process.wait.side_effect = supervisor.subprocess.TimeoutExpired('mock-child', 5) pump = mock.Mock(thread=None, handle=mock.sentinel.log_handle, error='') pump.attach.side_effect = OSError('log attachment failed') factory = mock.Mock(return_value=process) monkeypatch.setattr(supervisor, 'OwnedProcess', factory) monkeypatch.setattr(supervisor, 'BoundedRotatingLogPump', mock.Mock(return_value=pump)) retained = mock.MagicMock() retained.__enter__.return_value.identity = mock.sentinel.payload_identity monkeypatch.setattr(supervisor, 'open_process', mock.Mock(return_value=retained)) def confirm_exit(*args, **kwargs): process.poll.return_value = 0 return 0 return SimpleNamespace(source=source, process=process, pump=pump, factory=factory, confirm_exit=confirm_exit) @pytest.mark.parametrize('failure', ['terminate', 'timeout', 'unconfirmed_wait', 'poll']) def test_unconfirmed_launch_rollback_retains_owner_until_stop_retry(launch, failure): h = launch if failure == 'terminate': h.process.terminate.side_effect = OSError('stop unavailable') elif failure == 'unconfirmed_wait': h.process.wait.side_effect = None h.process.wait.return_value = 0 elif failure == 'poll': h.process.poll.side_effect = OSError('exit observation unavailable') assert not h.source.start(force=True) assert h.source.process is h.process assert h.source.payload_identity is mock.sentinel.payload_identity assert h.source.startup_cleanup_pending is True assert h.source.log_pump is h.pump assert h.source.started_at is None assert h.source.status == 'failed' assert 'unconfirmed' in h.source.last_action_error h.pump.join.assert_not_called() h.process.kill.assert_not_called() supervisor.finalize_source_runs.assert_not_called() h.process.terminate.side_effect = None h.process.poll.side_effect = None h.process.wait.side_effect = h.confirm_exit assert h.source.stop(final=True) assert h.source.process is None assert h.source.payload_identity is None assert h.source.startup_cleanup_pending is False assert h.source.log_pump is None assert not h.source.is_running() h.factory.assert_called_once() @pytest.mark.parametrize('exit_observation', [None, 0, OSError('observer unavailable')]) def test_retained_rollback_refuses_start_schedule_and_poll(launch, exit_observation): h = launch assert not h.source.start(force=True) h.process.poll.reset_mock() if isinstance(exit_observation, BaseException): h.process.poll.side_effect = exit_observation else: h.process.poll.return_value = exit_observation for force in (False, True): assert not h.source.start(force=force) assert not h.source.resume() h.source.schedule_start() h.source.poll() assert h.source.status == 'failed' assert h.source.process is h.process assert h.source.payload_identity is mock.sentinel.payload_identity assert h.source.startup_cleanup_pending is True h.process.poll.assert_not_called() h.factory.assert_called_once() supervisor.BoundedRotatingLogPump.assert_called_once() if exit_observation == 0: assert h.source.stop(final=True) assert h.source.process is None assert h.source.payload_identity is None assert h.source.startup_cleanup_pending is False @pytest.mark.parametrize('already_exited', [False, True]) def test_confirmed_launch_rollback_clears_owner_and_allows_explicit_retry(launch, already_exited): h = launch if already_exited: h.process.poll.return_value = 0 else: h.process.wait.side_effect = h.confirm_exit assert not h.source.start(force=True) assert h.source.process is None assert h.source.payload_identity is None assert h.source.startup_cleanup_pending is False assert h.source.log_pump is None assert h.source.status == 'failed' if already_exited: h.process.terminate.assert_not_called() h.process.poll.return_value = None h.pump.attach.side_effect = None assert h.source.start(force=True) assert h.factory.call_count == 2 def test_constructor_failure_without_returned_owner_still_closes_log(launch): launch.factory.side_effect = RuntimeError('construction failed') assert not launch.source.start(force=True) assert launch.source.process is None assert launch.source.payload_identity is None assert launch.source.startup_cleanup_pending is False assert launch.source.log_pump is None launch.pump.join.assert_called_once_with(timeout=1) launch.process.terminate.assert_not_called() @pytest.mark.parametrize('stage', ['attach', 'terminate', 'wait', 'poll']) @pytest.mark.parametrize('error', [KeyboardInterrupt(), SystemExit(7)]) def test_rollback_interruption_propagates_without_losing_owner(launch, stage, error): h = launch target = h.pump.attach if stage == 'attach' else getattr(h.process, stage) target.side_effect = error with pytest.raises(type(error)): h.source.start(force=True) assert h.source.process is h.process assert h.source.payload_identity is mock.sentinel.payload_identity assert h.source.startup_cleanup_pending is True assert h.source.status == 'failed' assert h.source.log_pump is h.pump h.process.terminate.side_effect = None h.process.poll.side_effect = None h.process.wait.side_effect = h.confirm_exit assert h.source.stop(final=True) assert h.source.startup_cleanup_pending is False @pytest.mark.parametrize('error', [OSError('log cleanup failed'), KeyboardInterrupt(), SystemExit(7)]) def test_log_cleanup_error_does_not_undo_confirmed_exit(launch, error): h = launch h.process.wait.side_effect = h.confirm_exit h.pump.join.side_effect = error if isinstance(error, Exception): assert not h.source.start(force=True) else: with pytest.raises(type(error)): h.source.start(force=True) assert h.source.process is None assert h.source.payload_identity is None assert h.source.startup_cleanup_pending is False h.pump.join.side_effect = None assert h.source.stop(final=True) def test_log_pump_with_failed_thread_start_can_close_on_rollback_retry(): pump = object.__new__(BoundedRotatingLogPump) pump.thread = mock.Mock(ident=None) pump.thread.join.side_effect = RuntimeError('cannot join thread before it is started') pump.close = mock.Mock() pump.join(timeout=1) pump.thread.join.assert_not_called() pump.close.assert_called_once() @pytest.mark.parametrize('publication_error', [None, OSError('private metadata unavailable'), KeyboardInterrupt()]) def test_rollback_pending_closes_admission_before_failed_hold_publication(launch, publication_error): h = launch context = lifecycle_context( managed_sources=[h.source], postgres_controller=mock.Mock(), instance_file='mock-instance', instance_id='exact-instance', ) h.source.start_gate = lambda: supervisor.lifecycle_start_allowed(context) assert not h.source.start(force=True) assert not supervisor.lifecycle_start_allowed(context) supervisor.update_instance_activation.side_effect = publication_error with context['control_lock']: if publication_error is None: assert supervisor.shutdown_checkpoint(context) else: with pytest.raises(type(publication_error)): supervisor.shutdown_checkpoint(context) assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD assert context['runtime_failed'] is True assert context['authority_release_safe'] is False assert not context['start_gate_open'] assert context['shutdown_event'].is_set() context['postgres_controller'].request_stop.assert_not_called() context['postgres_controller'].close.assert_not_called() @pytest.mark.parametrize('stop_error', [None, OSError('owned child uncertain'), KeyboardInterrupt()]) def test_owned_child_uncertain_defers_postgres_until_retained_retry_confirms_exit(launch, stop_error): h = launch assert not h.source.start(force=True) controller = mock.Mock( lifecycle_action_required=True, terminal=True, stop_succeeded=True, authority_release_safe=True, detail='', ) controller.close.return_value = True context = lifecycle_context(managed_sources=[h.source], postgres_controller=controller) if stop_error is not None: h.process.terminate.side_effect = stop_error assert not supervisor.retain_unsafe_authority([h.source], context, max_attempts=1) assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD assert context['authority_release_safe'] is False assert h.source.process is h.process assert h.source.payload_identity is mock.sentinel.payload_identity controller.request_stop.assert_not_called() controller.close.assert_not_called() supervisor.finalize_source_runs.assert_not_called() h.process.terminate.side_effect = None h.process.wait.side_effect = h.confirm_exit assert supervisor.retain_unsafe_authority([h.source], context, max_attempts=1) assert context['authority_release_safe'] is True assert context['runtime_failed'] is True assert h.source.process is None assert h.source.payload_identity is None assert not supervisor.lifecycle_start_allowed(context) controller.request_stop.assert_called_once() controller.close.assert_called_once() @pytest.mark.parametrize('background', [False, True]) @pytest.mark.parametrize('interrupted', [False, True]) def test_main_retains_locks_and_control_until_startup_rollback_is_confirmed(runtime, launch, monkeypatch, background, interrupted): h, child = runtime, launch h.args.background_child = background peer = mock.Mock(source='later', enabled=True, manual_only=False, status='stopped') peer.stop.return_value = True peer.is_running.return_value = False supervisor.load_supervisor_runtime.return_value[-1].append('later') def construct(name, *args): if name == 'later': return peer child.source.authority_check = args[7] child.source.start_gate = args[8] return child.source monkeypatch.setattr(supervisor, 'ManagedSource', mock.Mock(side_effect=construct)) if interrupted: child.pump.attach.side_effect = KeyboardInterrupt() real_retain = h.retain._mock_wraps def retry(sources, context): assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD assert not supervisor.lifecycle_start_allowed(context) assert context['runtime_failed'] is True assert child.source.process is child.process assert child.source.payload_identity is mock.sentinel.payload_identity h.instance_lock.release.assert_not_called() h.cluster_lock.release.assert_not_called() h.server.shutdown.assert_not_called() supervisor.write_shutdown_receipt.assert_not_called() h.controller.request_stop.assert_not_called() h.controller.close.assert_not_called() peer.start.assert_not_called() child.process.wait.side_effect = child.confirm_exit return real_retain(sources, context, max_attempts=1) h.retain.side_effect = retry assert supervisor.main() != 0 h.retain.assert_called_once() peer.start.assert_not_called() child.factory.assert_called_once() assert child.source.process is None assert child.source.payload_identity is None assert supervisor.write_shutdown_receipt.call_args.args[-1] != 0 h.instance_lock.release.assert_called_once() h.cluster_lock.release.assert_called_once() @pytest.fixture def stale_instance(runtime, monkeypatch): h = runtime h.stale = { 'schema': 2, 'instance_id': 'previous-instance', 'instance_file': h.instance_file, 'pid': 123, 'process_creation_time': 'exact-old-creation', 'executable': os.path.abspath('mock-python.exe'), 'startup_time': '2999-01-01T00:00:00Z', } h.legacy = False def load_metadata(path): if not h.metadata and h.legacy: raise ValueError('legacy manifest') return dict(h.metadata or h.stale) supervisor.os.path.exists.return_value = True supervisor.load_instance_metadata.side_effect = load_metadata monkeypatch.setattr(supervisor, 'read_private_json', mock.Mock(side_effect=lambda path: dict(h.stale))) monkeypatch.setattr(supervisor, 'durable_unlink', mock.Mock()) h.identity_state = 'reused' def inspect(pid, creation_time, executable): supervisor.SupervisorInstanceLock.return_value.acquire.assert_called() supervisor.ClusterAuthorityLock.return_value.acquire.assert_called() h.instance_lock.release.assert_not_called() h.cluster_lock.release.assert_not_called() supervisor.controller_from_config.assert_not_called() assert (pid, creation_time, executable) == ( h.stale['pid'], h.stale['process_creation_time'], h.stale['executable'], ) return h.identity_state monkeypatch.setattr(supervisor, 'exact_process_identity_state', mock.Mock(side_effect=inspect)) return h @pytest.mark.parametrize('legacy', [False, True]) @pytest.mark.parametrize('identity_state', ['dead', 'reused']) def test_locked_stale_metadata_recovery_uses_exact_identity_not_pid_or_timestamp(stale_instance, legacy, identity_state): h = stale_instance h.identity_state = identity_state h.legacy = legacy assert supervisor.main() == 0 supervisor.exact_process_identity_state.assert_called_once_with( h.stale['pid'], h.stale['process_creation_time'], h.stale['executable'], ) supervisor.process_running_status.assert_not_called() supervisor.verify_instance_process.assert_not_called() supervisor.send_control_request.assert_not_called() if legacy: supervisor.durable_unlink.assert_called_once_with(h.instance_file) supervisor.remove_instance_if_matches.assert_not_called() else: supervisor.remove_instance_if_matches.assert_called_once_with( h.instance_file, h.stale['instance_id'], instance_lock=h.instance_lock, ) supervisor.durable_unlink.assert_not_called() supervisor.remove_shutdown_receipt.assert_called_once_with(h.instance_file, h.stale['instance_id']) @pytest.mark.parametrize('observed', ['dead', 'reused', 'alive', 'unknown']) def test_foreground_reconciliation_uses_existing_exact_identity_api(stale_instance, monkeypatch, observed): h = stale_instance process = mock.Mock(spec=process_identity.RetainedProcess) process.identity = SimpleNamespace( creation_time='exact-new-creation' if observed == 'reused' else h.stale['process_creation_time'], executable=h.stale['executable'], ) process.is_running.return_value = observed != 'dead' opened = mock.Mock(return_value=process) if observed == 'unknown': opened.side_effect = process_identity.ProcessIdentityError('process inspection denied') monkeypatch.setattr(process_identity, 'open_process', opened) monkeypatch.setattr(supervisor, 'exact_process_identity_state', mock.Mock( wraps=process_identity.exact_process_identity_state, )) recoverable = observed in ('dead', 'reused') assert supervisor.main() == (0 if recoverable else 1) opened.assert_called_once_with(h.stale['pid']) assert supervisor.remove_instance_if_matches.called is recoverable assert supervisor.remove_shutdown_receipt.called is recoverable assert process.close.call_count == int(observed != 'unknown') process.wait.assert_not_called() supervisor.process_running_status.assert_not_called() supervisor.send_control_request.assert_not_called() @pytest.mark.parametrize('identity_state', ['alive', 'unknown', 'mismatched']) @pytest.mark.parametrize('legacy', [False, True]) def test_live_or_uncertain_owner_metadata_is_never_replaced(stale_instance, identity_state, legacy): h = stale_instance h.identity_state = identity_state h.stale['startup_time'] = '1900-01-01T00:00:00Z' h.legacy = legacy assert supervisor.main() != 0 supervisor.remove_instance_if_matches.assert_not_called() supervisor.durable_unlink.assert_not_called() supervisor.remove_shutdown_receipt.assert_not_called() supervisor.write_instance_metadata.assert_not_called() supervisor.controller_from_config.assert_not_called() h.instance_lock.release.assert_called_once() h.cluster_lock.release.assert_called_once() @pytest.mark.parametrize('legacy', [False, True]) def test_metadata_bound_to_another_instance_path_is_not_removed(stale_instance, legacy): h = stale_instance h.stale['instance_file'] += '.other' h.legacy = legacy assert supervisor.main() != 0 supervisor.remove_instance_if_matches.assert_not_called() supervisor.durable_unlink.assert_not_called() supervisor.remove_shutdown_receipt.assert_not_called() supervisor.controller_from_config.assert_not_called() @pytest.mark.parametrize('operation', ['identity', 'remove']) @pytest.mark.parametrize('error,expected', [(OSError('unavailable'), 1), (KeyboardInterrupt(), 0), (SystemExit(7), 7)]) def test_reconciliation_error_or_interruption_preserves_metadata_for_retry(stale_instance, operation, error, expected): h = stale_instance target = supervisor.exact_process_identity_state if operation == 'identity' else supervisor.remove_instance_if_matches previous = target.side_effect target.side_effect = error assert supervisor.main() == expected supervisor.durable_unlink.assert_not_called() supervisor.remove_shutdown_receipt.assert_not_called() supervisor.write_instance_metadata.assert_not_called() supervisor.controller_from_config.assert_not_called() target.side_effect = previous h.instance_lock.release.reset_mock() h.cluster_lock.release.reset_mock() assert supervisor.main() == 0 supervisor.remove_shutdown_receipt.assert_called_once_with(h.instance_file, h.stale['instance_id']) def test_changed_instance_during_matching_removal_fails_closed(stale_instance): supervisor.remove_instance_if_matches.return_value = False assert supervisor.main() != 0 supervisor.remove_instance_if_matches.assert_called_once_with( stale_instance.instance_file, stale_instance.stale['instance_id'], instance_lock=stale_instance.instance_lock, ) supervisor.remove_shutdown_receipt.assert_not_called() supervisor.durable_unlink.assert_not_called() supervisor.controller_from_config.assert_not_called()