Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,370 @@
|
||||
from concurrent.futures import Future
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP_DIR = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP_DIR))
|
||||
|
||||
import lifecycle_authority
|
||||
import postgres_runtime
|
||||
import supervisor
|
||||
import supervisor_instance
|
||||
|
||||
|
||||
def canonical(path):
|
||||
return os.path.normcase(os.path.realpath(os.path.abspath(os.fspath(path))))
|
||||
|
||||
|
||||
class ImmediateExecutor:
|
||||
def submit(self, function):
|
||||
future = Future()
|
||||
try:
|
||||
future.set_result(function())
|
||||
except BaseException as exc:
|
||||
future.set_exception(exc)
|
||||
return future
|
||||
|
||||
|
||||
class ControlledStartExecutor:
|
||||
def __init__(self):
|
||||
self.submissions = []
|
||||
|
||||
def submit(self, function):
|
||||
future = Future()
|
||||
self.submissions.append((function.__name__, future))
|
||||
if function.__name__ == 'stop':
|
||||
future.set_result(function())
|
||||
return future
|
||||
|
||||
|
||||
class LifecycleBackend:
|
||||
def __init__(self, probes=(), starts=()):
|
||||
self.probes = list(probes)
|
||||
self.starts = list(starts)
|
||||
self.start_calls = 0
|
||||
self.stop_calls = 0
|
||||
self.closed = False
|
||||
|
||||
def probe(self):
|
||||
return self.probes.pop(0)
|
||||
|
||||
def start(self):
|
||||
self.start_calls += 1
|
||||
return self.starts.pop(0)
|
||||
|
||||
def stop(self):
|
||||
self.stop_calls += 1
|
||||
return postgres_runtime.StopResult(True, True, 'stopped')
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class PostgresLifecycleTests(unittest.TestCase):
|
||||
def test_close_of_unowned_ready_controller_never_stops_cluster(self):
|
||||
backend = LifecycleBackend(probes=[
|
||||
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'preexisting ready', 'epoch'),
|
||||
])
|
||||
controller = postgres_runtime.PostgresController(
|
||||
backend,
|
||||
executor=ImmediateExecutor(),
|
||||
stable_ready_interval_sec=0,
|
||||
)
|
||||
controller.tick(0)
|
||||
controller.tick(0)
|
||||
self.assertEqual(controller.state, postgres_runtime.PostgresState.READY)
|
||||
self.assertFalse(controller._owned_start)
|
||||
|
||||
self.assertTrue(controller.close(timeout_sec=0.1))
|
||||
|
||||
self.assertEqual(backend.stop_calls, 0)
|
||||
self.assertTrue(backend.closed)
|
||||
self.assertTrue(controller.terminal)
|
||||
self.assertTrue(controller.authority_release_safe)
|
||||
self.assertIn('did not start', controller.detail)
|
||||
|
||||
def test_close_of_inflight_accepted_start_still_compensates(self):
|
||||
backend = LifecycleBackend()
|
||||
executor = ControlledStartExecutor()
|
||||
controller = postgres_runtime.PostgresController(backend, executor=executor)
|
||||
controller.tick(0)
|
||||
executor.submissions[0][1].set_result(
|
||||
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline')
|
||||
)
|
||||
controller.tick(0)
|
||||
self.assertTrue(controller.has_inflight_start)
|
||||
executor.submissions[1][1].set_result(
|
||||
postgres_runtime.StartResult(True, 'accepted before close')
|
||||
)
|
||||
|
||||
self.assertTrue(controller.close(timeout_sec=0.2))
|
||||
|
||||
self.assertEqual([name for name, _ in executor.submissions], ['probe', 'start', 'stop'])
|
||||
self.assertEqual(backend.stop_calls, 1)
|
||||
self.assertTrue(backend.closed)
|
||||
|
||||
def test_owned_ready_waits_for_child_then_stops_exactly_once(self):
|
||||
backend = LifecycleBackend(
|
||||
probes=[
|
||||
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.STOPPED, 'offline'),
|
||||
postgres_runtime.ProbeResult(postgres_runtime.ProbeKind.READY, 'owned ready', 'epoch'),
|
||||
],
|
||||
starts=[postgres_runtime.StartResult(True, 'accepted')],
|
||||
)
|
||||
controller = postgres_runtime.PostgresController(
|
||||
backend,
|
||||
executor=ImmediateExecutor(),
|
||||
stable_ready_interval_sec=0,
|
||||
)
|
||||
for _ in range(4):
|
||||
controller.tick(0)
|
||||
self.assertEqual(controller.state, postgres_runtime.PostgresState.READY)
|
||||
self.assertTrue(controller._owned_start)
|
||||
self.assertFalse(controller.has_inflight_start)
|
||||
|
||||
source = mock.Mock()
|
||||
source.source = 'child'
|
||||
source.stop.side_effect = [False, True]
|
||||
source.is_running.side_effect = [True, False]
|
||||
context = {
|
||||
'lifecycle_phase': lifecycle_authority.PHASE_ACTIVE,
|
||||
'activation_state': lifecycle_authority.PHASE_ACTIVE,
|
||||
'start_gate_open': True,
|
||||
'postgres_controller': controller,
|
||||
'supervisor_config': {},
|
||||
}
|
||||
with mock.patch.object(controller, 'close', wraps=controller.close) as close:
|
||||
self.assertFalse(supervisor.coordinated_shutdown([source], context))
|
||||
close.assert_not_called()
|
||||
self.assertEqual(backend.stop_calls, 0)
|
||||
self.assertEqual(context['lifecycle_phase'], lifecycle_authority.PHASE_FAILED_HOLD)
|
||||
|
||||
self.assertTrue(supervisor.coordinated_shutdown([source], context))
|
||||
close.assert_called_once()
|
||||
|
||||
self.assertEqual(backend.stop_calls, 1)
|
||||
|
||||
|
||||
class RetainedProcess:
|
||||
def __init__(self, command):
|
||||
self.command = list(command)
|
||||
self.closed = False
|
||||
|
||||
def command_line(self):
|
||||
return list(self.command)
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class SupervisorCommandVerificationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.supervisor_path = canonical(APP_DIR / 'supervisor.py')
|
||||
self.config_path = canonical(APP_DIR / 'config.yaml')
|
||||
self.bootstrap_path = canonical(APP_DIR / 'runtime_bootstrap.py')
|
||||
|
||||
def command(self, lifecycle_mode='foreground'):
|
||||
command = [
|
||||
sys.executable,
|
||||
'-I',
|
||||
'-S',
|
||||
'-B',
|
||||
self.bootstrap_path,
|
||||
'supervisor',
|
||||
'--',
|
||||
f'--runtime-bootstrap-entrypoint={self.supervisor_path}',
|
||||
f'--config={self.config_path}',
|
||||
'--with-postgres',
|
||||
]
|
||||
if lifecycle_mode == 'background':
|
||||
command.append('--background-child')
|
||||
return command
|
||||
|
||||
def lifecycle_metadata(self):
|
||||
return {
|
||||
'pid': 123,
|
||||
'process_creation_time': 'fixture',
|
||||
'executable': canonical(sys.executable),
|
||||
'supervisor_path': self.supervisor_path,
|
||||
'config_path': self.config_path,
|
||||
}
|
||||
|
||||
def instance_metadata(self, lifecycle_mode='foreground'):
|
||||
return {
|
||||
**self.lifecycle_metadata(),
|
||||
'lifecycle_mode': lifecycle_mode,
|
||||
'code_manifest': {},
|
||||
'code_manifest_sha256': '1' * 64,
|
||||
'supervisor_sha256': '2' * 64,
|
||||
'config_sha256': '3' * 64,
|
||||
}
|
||||
|
||||
def verify_instance(self, command, lifecycle_mode='foreground', shutdown=False):
|
||||
retained = RetainedProcess(command)
|
||||
metadata = self.instance_metadata(lifecycle_mode)
|
||||
|
||||
def digest(path):
|
||||
return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64
|
||||
|
||||
with mock.patch.object(
|
||||
supervisor_instance,
|
||||
'validate_instance_metadata',
|
||||
return_value=metadata,
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'verify_code_manifest',
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'sha256_file',
|
||||
side_effect=digest,
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'verify_retained_process',
|
||||
return_value=retained,
|
||||
):
|
||||
result = supervisor_instance.verify_instance_process(
|
||||
metadata,
|
||||
allow_config_drift=shutdown,
|
||||
allow_code_drift=shutdown,
|
||||
)
|
||||
return result
|
||||
|
||||
def assert_lifecycle_rejected(self, command, pattern):
|
||||
retained = RetainedProcess(command)
|
||||
with mock.patch.object(
|
||||
lifecycle_authority,
|
||||
'verify_retained_process',
|
||||
return_value=retained,
|
||||
):
|
||||
with self.assertRaisesRegex(lifecycle_authority.LifecycleAuthorityError, pattern):
|
||||
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
||||
self.assertTrue(retained.closed)
|
||||
|
||||
def assert_instance_rejected(self, command, pattern):
|
||||
retained = RetainedProcess(command)
|
||||
metadata = self.instance_metadata()
|
||||
|
||||
def digest(path):
|
||||
return '2' * 64 if canonical(path) == self.supervisor_path else '3' * 64
|
||||
|
||||
with mock.patch.object(
|
||||
supervisor_instance,
|
||||
'validate_instance_metadata',
|
||||
return_value=metadata,
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'verify_code_manifest',
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'sha256_file',
|
||||
side_effect=digest,
|
||||
), mock.patch.object(
|
||||
supervisor_instance,
|
||||
'verify_retained_process',
|
||||
return_value=retained,
|
||||
):
|
||||
with self.assertRaisesRegex(supervisor_instance.InstanceMetadataError, pattern):
|
||||
supervisor_instance.verify_instance_process(metadata)
|
||||
self.assertTrue(retained.closed)
|
||||
|
||||
def test_equals_form_foreground_child_and_shutdown_verification(self):
|
||||
command = self.command()
|
||||
retained = RetainedProcess(command)
|
||||
with mock.patch.object(
|
||||
lifecycle_authority,
|
||||
'verify_retained_process',
|
||||
return_value=retained,
|
||||
):
|
||||
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
||||
self.assertTrue(retained.closed)
|
||||
|
||||
foreground = self.verify_instance(command)
|
||||
self.assertFalse(foreground.closed)
|
||||
foreground.close()
|
||||
|
||||
background = self.verify_instance(
|
||||
self.command('background'),
|
||||
lifecycle_mode='background',
|
||||
shutdown=True,
|
||||
)
|
||||
self.assertFalse(background.closed)
|
||||
background.close()
|
||||
|
||||
def test_canonical_direct_script_argument_remains_allowed(self):
|
||||
command = [sys.executable, self.supervisor_path, f'--config={self.config_path}']
|
||||
retained = RetainedProcess(command)
|
||||
with mock.patch.object(
|
||||
lifecycle_authority,
|
||||
'verify_retained_process',
|
||||
return_value=retained,
|
||||
):
|
||||
lifecycle_authority._verify_supervisor_process(self.lifecycle_metadata())
|
||||
self.assertTrue(retained.closed)
|
||||
|
||||
instance = self.verify_instance(command)
|
||||
instance.close()
|
||||
|
||||
def test_conflicting_or_duplicate_bindings_are_rejected(self):
|
||||
cases = (
|
||||
(
|
||||
[*self.command(), '--runtime-bootstrap-entrypoint', self.supervisor_path],
|
||||
'exactly one bound supervisor script',
|
||||
),
|
||||
(
|
||||
[*self.command(), '--config', canonical(APP_DIR / 'other.yaml')],
|
||||
'exactly one bound config argument',
|
||||
),
|
||||
)
|
||||
for command, pattern in cases:
|
||||
with self.subTest(pattern=pattern):
|
||||
self.assert_lifecycle_rejected(command, pattern)
|
||||
self.assert_instance_rejected(command, pattern)
|
||||
|
||||
|
||||
class DirectReadOnlyBytecodeTests(unittest.TestCase):
|
||||
def test_background_status_in_clean_copy_creates_no_application_bytecode(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
app_dir = Path(temp_dir) / 'app'
|
||||
shutil.copytree(
|
||||
APP_DIR,
|
||||
app_dir,
|
||||
ignore=shutil.ignore_patterns('__pycache__', '*.pyc'),
|
||||
)
|
||||
environment = os.environ.copy()
|
||||
environment.pop('PYTHONDONTWRITEBYTECODE', None)
|
||||
environment.pop('PYTHONPYCACHEPREFIX', None)
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(app_dir / 'supervisor.py'),
|
||||
'--config',
|
||||
str(Path(temp_dir) / 'missing-config.yaml'),
|
||||
'--background-status',
|
||||
],
|
||||
cwd=temp_dir,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(completed.returncode, 0)
|
||||
self.assertEqual(list(app_dir.rglob('*.pyc')), [])
|
||||
self.assertEqual(
|
||||
[path for path in app_dir.rglob('__pycache__') if path.is_dir()],
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user