sashatrask
371 lines
12 KiB
Python
371 lines
12 KiB
Python
from concurrent.futures import Future
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
APP_DIR = ROOT / 'app'
|
|
sys.path.insert(0, str(APP_DIR))
|
|
|
|
import lifecycle_authority
|
|
import postgres_runtime
|
|
import supervisor
|
|
import supervisor_instance
|
|
|
|
|
|
def canonical(path):
|
|
return os.path.normcase(os.path.realpath(os.path.abspath(os.fspath(path))))
|
|
|
|
|
|
class ImmediateExecutor:
|
|
def submit(self, function):
|
|
future = Future()
|
|
try:
|
|
future.set_result(function())
|
|
except BaseException as exc:
|
|
future.set_exception(exc)
|
|
return future
|
|
|
|
|
|
class ControlledStartExecutor:
|
|
def __init__(self):
|
|
self.submissions = []
|
|
|
|
def submit(self, function):
|
|
future = Future()
|
|
self.submissions.append((function.__name__, future))
|
|
if function.__name__ == 'stop':
|
|
future.set_result(function())
|
|
return future
|
|
|
|
|
|
class LifecycleBackend:
|
|
def __init__(self, probes=(), starts=()):
|
|
self.probes = list(probes)
|
|
self.starts = list(starts)
|
|
self.start_calls = 0
|
|
self.stop_calls = 0
|
|
self.closed = False
|
|
|
|
def probe(self):
|
|
return self.probes.pop(0)
|
|
|
|
def start(self):
|
|
self.start_calls += 1
|
|
return self.starts.pop(0)
|
|
|
|
def stop(self):
|
|
self.stop_calls += 1
|
|
return postgres_runtime.StopResult(True, True, 'stopped')
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class PostgresLifecycleTests(unittest.TestCase):
|
|
def test_close_of_unowned_ready_controller_never_stops_cluster(self):
|
|
backend = LifecycleBackend(probes=[
|
|
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'preexisting ready', 'epoch'),
|
|
])
|
|
controller = postgres_runtime.PostgresController(
|
|
backend,
|
|
executor=ImmediateExecutor(),
|
|
stable_ready_interval_sec=0,
|
|
)
|
|
controller.tick(0)
|
|
controller.tick(0)
|
|
self.assertEqual(controller.state, postgres_runtime.PostgresState.READY)
|
|
self.assertFalse(controller._owned_start)
|
|
|
|
self.assertTrue(controller.close(timeout_sec=0.1))
|
|
|
|
self.assertEqual(backend.stop_calls, 0)
|
|
self.assertTrue(backend.closed)
|
|
self.assertTrue(controller.terminal)
|
|
self.assertTrue(controller.authority_release_safe)
|
|
self.assertIn('did not start', controller.detail)
|
|
|
|
def test_close_of_inflight_accepted_start_still_compensates(self):
|
|
backend = LifecycleBackend()
|
|
executor = ControlledStartExecutor()
|
|
controller = postgres_runtime.PostgresController(backend, executor=executor)
|
|
controller.tick(0)
|
|
executor.submissions[0][1].set_result(
|
|
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline')
|
|
)
|
|
controller.tick(0)
|
|
self.assertTrue(controller.has_inflight_start)
|
|
executor.submissions[1][1].set_result(
|
|
postgres_runtime.StartResult(True, 'accepted before close')
|
|
)
|
|
|
|
self.assertTrue(controller.close(timeout_sec=0.2))
|
|
|
|
self.assertEqual([name for name, _ in executor.submissions], ['probe', 'start', 'stop'])
|
|
self.assertEqual(backend.stop_calls, 1)
|
|
self.assertTrue(backend.closed)
|
|
|
|
def test_owned_ready_waits_for_child_then_stops_exactly_once(self):
|
|
backend = LifecycleBackend(
|
|
probes=[
|
|
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline'),
|
|
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'owned ready', 'epoch'),
|
|
],
|
|
starts=[postgres_runtime.StartResult(True, 'accepted')],
|
|
)
|
|
controller = postgres_runtime.PostgresController(
|
|
backend,
|
|
executor=ImmediateExecutor(),
|
|
stable_ready_interval_sec=0,
|
|
)
|
|
for _ in range(4):
|
|
controller.tick(0)
|
|
self.assertEqual(controller.state, postgres_runtime.PostgresState.READY)
|
|
self.assertTrue(controller._owned_start)
|
|
self.assertFalse(controller.has_inflight_start)
|
|
|
|
source = mock.Mock()
|
|
source.source = 'child'
|
|
source.stop.side_effect = [False, True]
|
|
source.is_running.side_effect = [True, False]
|
|
context = {
|
|
'lifecycle_phase': lifecycle_authority.PHASE_ACTIVE,
|
|
'activation_state': lifecycle_authority.PHASE_ACTIVE,
|
|
'start_gate_open': True,
|
|
'postgres_controller': controller,
|
|
'supervisor_config': {},
|
|
}
|
|
with mock.patch.object(controller, 'close', wraps=controller.close) as close:
|
|
self.assertFalse(supervisor.coordinated_shutdown([source], context))
|
|
close.assert_not_called()
|
|
self.assertEqual(backend.stop_calls, 0)
|
|
self.assertEqual(context['lifecycle_phase'], lifecycle_authority.PHASE_FAILED_HOLD)
|
|
|
|
self.assertTrue(supervisor.coordinated_shutdown([source], context))
|
|
close.assert_called_once()
|
|
|
|
self.assertEqual(backend.stop_calls, 1)
|
|
|
|
|
|
class RetainedProcess:
|
|
def __init__(self, command):
|
|
self.command = list(command)
|
|
self.closed = False
|
|
|
|
def command_line(self):
|
|
return list(self.command)
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class SupervisorCommandVerificationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.supervisor_path = canonical(APP_DIR / 'supervisor.py')
|
|
self.config_path = canonical(APP_DIR / 'config.yaml')
|
|
self.bootstrap_path = canonical(APP_DIR / 'runtime_bootstrap.py')
|
|
|
|
def command(self, lifecycle_mode='foreground'):
|
|
command = [
|
|
sys.executable,
|
|
'-I',
|
|
'-S',
|
|
'-B',
|
|
self.bootstrap_path,
|
|
'supervisor',
|
|
'--',
|
|
f'--runtime-bootstrap-entrypoint={self.supervisor_path}',
|
|
f'--config={self.config_path}',
|
|
'--with-postgres',
|
|
]
|
|
if lifecycle_mode == 'background':
|
|
command.append('--background-child')
|
|
return command
|
|
|
|
def lifecycle_metadata(self):
|
|
return {
|
|
'pid': 123,
|
|
'process_creation_time': 'fixture',
|
|
'executable': canonical(sys.executable),
|
|
'supervisor_path': self.supervisor_path,
|
|
'config_path': self.config_path,
|
|
}
|
|
|
|
def instance_metadata(self, lifecycle_mode='foreground'):
|
|
return {
|
|
**self.lifecycle_metadata(),
|
|
'lifecycle_mode': lifecycle_mode,
|
|
'code_manifest': {},
|
|
'code_manifest_sha256': '1' * 64,
|
|
'supervisor_sha256': '2' * 64,
|
|
'config_sha256': '3' * 64,
|
|
}
|
|
|
|
def verify_instance(self, command, lifecycle_mode='foreground', shutdown=False):
|
|
retained = RetainedProcess(command)
|
|
metadata = self.instance_metadata(lifecycle_mode)
|
|
|
|
def digest(path):
|
|
return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64
|
|
|
|
with mock.patch.object(
|
|
supervisor_instance,
|
|
'validate_instance_metadata',
|
|
return_value=metadata,
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'verify_code_manifest',
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'sha256_file',
|
|
side_effect=digest,
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'verify_retained_process',
|
|
return_value=retained,
|
|
):
|
|
result = supervisor_instance.verify_instance_process(
|
|
metadata,
|
|
allow_config_drift=shutdown,
|
|
allow_code_drift=shutdown,
|
|
)
|
|
return result
|
|
|
|
def assert_lifecycle_rejected(self, command, pattern):
|
|
retained = RetainedProcess(command)
|
|
with mock.patch.object(
|
|
lifecycle_authority,
|
|
'verify_retained_process',
|
|
return_value=retained,
|
|
):
|
|
with self.assertRaisesRegex(lifecycle_authority.LifecycleAuthorityError, pattern):
|
|
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
|
self.assertTrue(retained.closed)
|
|
|
|
def assert_instance_rejected(self, command, pattern):
|
|
retained = RetainedProcess(command)
|
|
metadata = self.instance_metadata()
|
|
|
|
def digest(path):
|
|
return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64
|
|
|
|
with mock.patch.object(
|
|
supervisor_instance,
|
|
'validate_instance_metadata',
|
|
return_value=metadata,
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'verify_code_manifest',
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'sha256_file',
|
|
side_effect=digest,
|
|
), mock.patch.object(
|
|
supervisor_instance,
|
|
'verify_retained_process',
|
|
return_value=retained,
|
|
):
|
|
with self.assertRaisesRegex(supervisor_instance.InstanceMetadataError, pattern):
|
|
supervisor_instance.verify_instance_process(metadata)
|
|
self.assertTrue(retained.closed)
|
|
|
|
def test_equals_form_foreground_child_and_shutdown_verification(self):
|
|
command = self.command()
|
|
retained = RetainedProcess(command)
|
|
with mock.patch.object(
|
|
lifecycle_authority,
|
|
'verify_retained_process',
|
|
return_value=retained,
|
|
):
|
|
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
|
self.assertTrue(retained.closed)
|
|
|
|
foreground = self.verify_instance(command)
|
|
self.assertFalse(foreground.closed)
|
|
foreground.close()
|
|
|
|
background = self.verify_instance(
|
|
self.command('background'),
|
|
lifecycle_mode='background',
|
|
shutdown=True,
|
|
)
|
|
self.assertFalse(background.closed)
|
|
background.close()
|
|
|
|
def test_canonical_direct_script_argument_remains_allowed(self):
|
|
command = [sys.executable, self.supervisor_path, f'--config={self.config_path}']
|
|
retained = RetainedProcess(command)
|
|
with mock.patch.object(
|
|
lifecycle_authority,
|
|
'verify_retained_process',
|
|
return_value=retained,
|
|
):
|
|
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
|
self.assertTrue(retained.closed)
|
|
|
|
instance = self.verify_instance(command)
|
|
instance.close()
|
|
|
|
def test_conflicting_or_duplicate_bindings_are_rejected(self):
|
|
cases = (
|
|
(
|
|
[*self.command(), '--runtime-bootstrap-entrypoint', self.supervisor_path],
|
|
'exactly one bound supervisor script',
|
|
),
|
|
(
|
|
[*self.command(), '--config', canonical(APP_DIR / 'other.yaml')],
|
|
'exactly one bound config argument',
|
|
),
|
|
)
|
|
for command, pattern in cases:
|
|
with self.subTest(pattern=pattern):
|
|
self.assert_lifecycle_rejected(command, pattern)
|
|
self.assert_instance_rejected(command, pattern)
|
|
|
|
|
|
class DirectReadOnlyBytecodeTests(unittest.TestCase):
|
|
def test_background_status_in_clean_copy_creates_no_application_bytecode(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
app_dir = Path(temp_dir) / 'app'
|
|
shutil.copytree(
|
|
APP_DIR,
|
|
app_dir,
|
|
ignore=shutil.ignore_patterns('__pycache__', '*.pyc'),
|
|
)
|
|
environment = os.environ.copy()
|
|
environment.pop('PYTHONDONTWRITEBYTECODE', None)
|
|
environment.pop('PYTHONPYCACHEPREFIX', None)
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(app_dir / 'supervisor.py'),
|
|
'--config',
|
|
str(Path(temp_dir) / 'missing-config.yaml'),
|
|
'--background-status',
|
|
],
|
|
cwd=temp_dir,
|
|
env=environment,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
self.assertNotEqual(completed.returncode, 0)
|
|
self.assertEqual(list(app_dir.rglob('*.pyc')), [])
|
|
self.assertEqual(
|
|
[path for path in app_dir.rglob('__pycache__') if path.is_dir()],
|
|
[],
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|