Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,371 @@
|
||||
import contextlib
|
||||
import hashlib
|
||||
import ntpath
|
||||
import os
|
||||
from pathlib import Path
|
||||
import posixpath
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'app'))
|
||||
|
||||
import lifecycle_authority
|
||||
import migrate_runtime_safety as migration
|
||||
|
||||
|
||||
class LegacySpoolPathTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.path = SimpleNamespace(**vars(posixpath))
|
||||
self.path.lexists = mock.Mock(return_value=False)
|
||||
self.os = SimpleNamespace(name='posix', path=self.path, sep='/')
|
||||
self.enterContext(mock.patch.object(migration, 'os', self.os))
|
||||
self.private = self.enterContext(mock.patch.object(migration, 'require_private_directory', side_effect=lambda path, **_: path))
|
||||
self.reject = self.enterContext(mock.patch.object(migration, 'reject_reparse_components', side_effect=lambda path: path))
|
||||
self.canonical = self.enterContext(mock.patch.object(migration, 'canonical_path', side_effect=posixpath.normpath))
|
||||
self.harden = self.enterContext(mock.patch.object(migration, 'harden_private_directory'))
|
||||
self.ensure = self.enterContext(mock.patch.object(migration, 'ensure_private_directory'))
|
||||
self.config = {'global': {
|
||||
'runtime_dir': '/data/runtime-linux',
|
||||
'legacy_result_spool_dir': '/data/runtime-linux/result_spool',
|
||||
}}
|
||||
|
||||
def test_prepared_exact_spool_is_accepted_without_repair(self):
|
||||
self.assertEqual(migration.reviewed_legacy_result_spool(self.config), '/data/runtime-linux/result_spool')
|
||||
self.assertEqual(self.private.call_args_list, [
|
||||
mock.call('/data/runtime-linux', create=False),
|
||||
mock.call('/data/runtime-linux/result_spool', create=False),
|
||||
])
|
||||
self.harden.assert_not_called()
|
||||
self.ensure.assert_not_called()
|
||||
|
||||
def test_result_spool_fallback_uses_same_exact_authority(self):
|
||||
self.config['global']['result_spool_dir'] = self.config['global'].pop('legacy_result_spool_dir')
|
||||
self.assertEqual(migration.reviewed_legacy_result_spool(self.config), '/data/runtime-linux/result_spool')
|
||||
|
||||
def test_unresolved_relative_noncanonical_or_different_paths_are_rejected(self):
|
||||
cases = (
|
||||
('runtime_dir', None), ('runtime_dir', 'runtime'),
|
||||
('runtime_dir', '/data/{runtime}'), ('runtime_dir', '/data/runtime/../runtime-linux'),
|
||||
('legacy_result_spool_dir', None), ('legacy_result_spool_dir', 'result_spool'),
|
||||
('legacy_result_spool_dir', '{runtime_dir}/result_spool'),
|
||||
('legacy_result_spool_dir', '/data/runtime-linux/not_result_spool'),
|
||||
('legacy_result_spool_dir', '/elsewhere/result_spool'),
|
||||
('legacy_result_spool_dir', '/data/runtime-linux/alias/../result_spool'),
|
||||
('legacy_result_spool_dir', r'D:\truf\runtime\result_spool'),
|
||||
)
|
||||
for key, value in cases:
|
||||
config = {'global': dict(self.config['global'], **{key: value})}
|
||||
with self.subTest(key=key, value=value), self.assertRaises(RuntimeError):
|
||||
migration.reviewed_legacy_result_spool(config)
|
||||
self.ensure.assert_not_called()
|
||||
self.harden.assert_not_called()
|
||||
|
||||
def test_runtime_or_spool_link_is_rejected_before_canonicalization(self):
|
||||
for linked in ('/data/runtime-linux', '/data/runtime-linux/result_spool'):
|
||||
def reject(path):
|
||||
if path == linked:
|
||||
raise OSError('link or reparse component')
|
||||
return path
|
||||
|
||||
with self.subTest(linked=linked):
|
||||
self.reject.side_effect = reject
|
||||
with self.assertRaisesRegex(OSError, 'link or reparse'):
|
||||
migration.reviewed_legacy_result_spool(self.config)
|
||||
self.canonical.assert_not_called()
|
||||
|
||||
def test_missing_or_unsafe_prepared_directories_are_not_created_or_repaired(self):
|
||||
for bad in ('/data/runtime-linux', '/data/runtime-linux/result_spool'):
|
||||
def require(path, **kwargs):
|
||||
self.assertEqual(kwargs, {'create': False})
|
||||
if path == bad:
|
||||
raise OSError('private directory absent or not ready')
|
||||
return path
|
||||
|
||||
with self.subTest(bad=bad):
|
||||
self.private.side_effect = require
|
||||
with self.assertRaisesRegex(OSError, 'absent or not ready'):
|
||||
migration.reviewed_legacy_result_spool(self.config)
|
||||
self.ensure.assert_not_called()
|
||||
self.harden.assert_not_called()
|
||||
|
||||
def test_existing_reservation_and_quarantine_directories_must_also_be_private(self):
|
||||
self.path.lexists.return_value = True
|
||||
migration.reviewed_legacy_result_spool(self.config)
|
||||
self.private.assert_any_call('/data/runtime-linux/result_spool/reservations', create=False)
|
||||
self.private.assert_any_call('/data/runtime-linux/result_spool/quarantine', create=False)
|
||||
self.private.side_effect = OSError('unsafe shard')
|
||||
with self.assertRaises(OSError):
|
||||
migration.reviewed_legacy_result_spool(self.config)
|
||||
self.ensure.assert_not_called()
|
||||
|
||||
def test_canonical_escape_is_rejected(self):
|
||||
self.canonical.side_effect = lambda path: '/elsewhere' if path.endswith('/result_spool') else path
|
||||
with self.assertRaisesRegex(RuntimeError, 'outside its runtime authority'):
|
||||
migration.reviewed_legacy_result_spool(self.config)
|
||||
|
||||
def test_windows_reviewed_constant_and_no_new_directory_policy_are_preserved(self):
|
||||
self.os.name = 'nt'
|
||||
self.os.path = ntpath
|
||||
reviewed = ntpath.normcase(r'D:\truf\runtime\result_spool')
|
||||
with mock.patch.object(migration, 'REVIEWED_LEGACY_RESULT_SPOOL', reviewed):
|
||||
config = {'global': {'legacy_result_spool_dir': r'D:\truf\runtime\result_spool'}}
|
||||
self.assertEqual(migration.reviewed_legacy_result_spool(config), r'D:\truf\runtime\result_spool')
|
||||
config['global']['legacy_result_spool_dir'] = r'E:\fixture\result_spool'
|
||||
with self.assertRaisesRegex(RuntimeError, 'does not match reviewed'):
|
||||
migration.reviewed_legacy_result_spool(config)
|
||||
self.private.assert_not_called()
|
||||
self.canonical.assert_not_called()
|
||||
|
||||
|
||||
class ImmutableNativeHardeningTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.config_path = '/opt/truf/app/config.linux.yaml'
|
||||
self.env_path = '/run/secrets/postgres.env'
|
||||
self.config = {'global': {
|
||||
'root_dir': '/opt/truf', 'project_dir': '/opt/truf/app',
|
||||
'runtime_dir': '/data/runtime-linux', 'results_dir': '/data/runtime-linux/results',
|
||||
'result_spool_dir': '/data/runtime-linux/result_spool',
|
||||
'postgres_data_dir': '/data/postgres-linux',
|
||||
'work_dir': '/data/scanner-work',
|
||||
'trufflehog_path': '/usr/local/bin/trufflehog',
|
||||
'trufflehog_config': '/opt/truf/app/policy.yaml',
|
||||
'secrets_file': '/run/secrets/truf-secrets.yaml',
|
||||
}}
|
||||
self.code_files = ['/opt/truf/app/paths.py', '/opt/truf/app/provider.py']
|
||||
self.native = ['/usr/local/bin/trufflehog', '/usr/bin/git']
|
||||
self.pgpaths = {
|
||||
'postgres_dir': '/data/runtime-linux/postgres', 'data_dir': '/data/postgres-linux',
|
||||
}
|
||||
for key in ('postgres', 'pg_ctl', 'pg_isready', 'pg_controldata', 'initdb', 'psql'):
|
||||
self.pgpaths[key] = '/usr/lib/postgresql/16/bin/' + key
|
||||
self.native.append(self.pgpaths[key])
|
||||
self.files = set(self.code_files + self.native + [
|
||||
self.config_path, self.env_path,
|
||||
self.config['global']['trufflehog_config'], self.config['global']['secrets_file'],
|
||||
])
|
||||
self.path = SimpleNamespace(**vars(posixpath))
|
||||
self.path.lexists = mock.Mock(side_effect=lambda path: path in self.files)
|
||||
self.path.isfile = mock.Mock(side_effect=lambda path: path in self.files)
|
||||
self.os = SimpleNamespace(name='posix', path=self.path, sep='/', makedirs=mock.Mock())
|
||||
self.enterContext(mock.patch.object(migration, 'os', self.os))
|
||||
self.enterContext(mock.patch.object(migration, 'postgres_runtime_paths', return_value=self.pgpaths))
|
||||
self.enterContext(mock.patch.object(migration, 'canonical_cluster_data_directory', return_value='/data/postgres-linux'))
|
||||
self.enterContext(mock.patch.object(migration, 'reject_reparse_components', side_effect=lambda path: path))
|
||||
self.enterContext(mock.patch.object(migration, 'private_directory_ready', return_value=True))
|
||||
self.enterContext(mock.patch.object(migration, 'private_file_ready', return_value=True))
|
||||
self.harden_dir = self.enterContext(mock.patch.object(migration, 'harden_private_directory'))
|
||||
self.harden_file = self.enterContext(mock.patch.object(migration, 'harden_private_file'))
|
||||
self.harden_tree = self.enterContext(mock.patch.object(migration, 'harden_private_tree', return_value=1))
|
||||
self.trust = self.enterContext(mock.patch.object(migration, 'require_trusted_native_executable', side_effect=lambda path: path))
|
||||
self.manifest = self.enterContext(mock.patch.object(lifecycle_authority, 'manifest_authority_paths', return_value=self.code_files))
|
||||
self.enterContext(mock.patch.object(
|
||||
lifecycle_authority, 'resolve_manifest_executable',
|
||||
side_effect=lambda value, **kwargs: '/usr/bin/git' if kwargs['name'] == 'git' else '/usr/local/bin/trufflehog',
|
||||
))
|
||||
|
||||
def harden(self):
|
||||
return migration.harden_runtime_paths(self.config, env_path=self.env_path, config_path=self.config_path)
|
||||
|
||||
def test_native_files_and_parents_are_verified_but_never_hardened(self):
|
||||
self.assertGreater(self.harden(), 0)
|
||||
self.assertFalse(self.manifest.call_args.kwargs['include_executables'])
|
||||
self.assertEqual({call.args[0] for call in self.trust.call_args_list}, set(self.native))
|
||||
self.assertEqual(len(self.trust.call_args_list), len(self.native) * 2)
|
||||
hardened_files = {call.args[0] for call in self.harden_file.call_args_list}
|
||||
self.assertTrue(set(self.code_files).issubset(hardened_files))
|
||||
self.assertTrue({self.config_path, self.env_path}.issubset(hardened_files))
|
||||
self.assertFalse(hardened_files & set(self.native))
|
||||
native_parents = {posixpath.dirname(path) for path in self.native}
|
||||
self.assertFalse(native_parents & {call.args[0] for call in self.harden_dir.call_args_list})
|
||||
self.assertTrue(any(call.args[0] == '/data/runtime-linux' for call in self.harden_tree.call_args_list))
|
||||
self.assertTrue(any(call.args[0] == '/data/postgres-linux' for call in self.harden_tree.call_args_list))
|
||||
for call in self.harden_tree.call_args_list:
|
||||
for native in self.native:
|
||||
self.assertNotEqual(posixpath.commonpath((call.args[0], native)), call.args[0])
|
||||
|
||||
def test_failed_native_verification_precedes_every_filesystem_mutation(self):
|
||||
self.trust.side_effect = OSError('untrusted native executable')
|
||||
with self.assertRaisesRegex(OSError, 'untrusted'):
|
||||
self.harden()
|
||||
self.os.makedirs.assert_not_called()
|
||||
self.harden_dir.assert_not_called()
|
||||
self.harden_file.assert_not_called()
|
||||
self.harden_tree.assert_not_called()
|
||||
|
||||
def test_private_native_directory_conflicts_fail_without_chmod(self):
|
||||
for key, value in (
|
||||
('runtime_dir', '/usr/bin'), ('results_dir', '/usr'),
|
||||
('secrets_file', '/usr/local/bin/trufflehog'),
|
||||
):
|
||||
with self.subTest(key=key):
|
||||
original = self.config['global'][key]
|
||||
self.config['global'][key] = value
|
||||
with self.assertRaisesRegex(OSError, 'overlaps a native executable'):
|
||||
self.harden()
|
||||
self.config['global'][key] = original
|
||||
self.os.makedirs.assert_not_called()
|
||||
self.harden_dir.assert_not_called()
|
||||
self.harden_file.assert_not_called()
|
||||
self.harden_tree.assert_not_called()
|
||||
|
||||
def test_windows_keeps_private_executable_and_parent_hardening(self):
|
||||
self.os.name = 'nt'
|
||||
self.manifest.return_value = self.code_files + self.native[:2]
|
||||
self.harden()
|
||||
self.assertTrue(self.manifest.call_args.kwargs['include_executables'])
|
||||
self.trust.assert_not_called()
|
||||
for path in self.native[:2]:
|
||||
self.harden_file.assert_any_call(path)
|
||||
self.harden_dir.assert_any_call(posixpath.dirname(path))
|
||||
|
||||
|
||||
class LegacyCutoverChecksTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = Path(self.enterContext(tempfile.TemporaryDirectory()))
|
||||
self.spool = self.temp / 'result_spool'
|
||||
self.results = self.temp / 'results'
|
||||
for directory in (self.spool, self.results, self.spool / 'reservations', self.spool / 'quarantine'):
|
||||
directory.mkdir(mode=0o700)
|
||||
self.config = {'global': {'result_spool_dir': str(self.spool), 'results_dir': str(self.results)}}
|
||||
self.enterContext(mock.patch.object(migration, 'reviewed_legacy_result_spool', return_value=str(self.spool)))
|
||||
self.counts = {'scan_publication_outbox': 0, 'target_scans': 0}
|
||||
self.conn = mock.Mock(is_postgres=True)
|
||||
self.conn.table_exists.side_effect = lambda name: name in self.counts
|
||||
self.conn.execute.side_effect = lambda sql: mock.Mock(fetchone=lambda: {
|
||||
'count': self.counts['scan_publication_outbox' if 'scan_publication_outbox' in sql else 'target_scans'],
|
||||
})
|
||||
self.db = SimpleNamespace(conn=self.conn)
|
||||
|
||||
def test_empty_legacy_state_produces_evidence_not_a_bypass(self):
|
||||
evidence = migration.require_legacy_cutover_clear(self.db, self.config)
|
||||
self.assertEqual(evidence['legacy_outbox_rows'], 0)
|
||||
self.assertEqual(evidence['legacy_raw_result_rows'], 0)
|
||||
self.assertTrue(evidence['legacy_spool_present'])
|
||||
self.assertEqual(evidence['legacy_prepared_appends'], 0)
|
||||
self.assertEqual(self.conn.execute.call_count, 2)
|
||||
self.conn.commit.assert_called_once()
|
||||
|
||||
def test_outbox_or_legacy_raw_rows_still_block_cutover(self):
|
||||
for table in self.counts:
|
||||
with self.subTest(table=table):
|
||||
self.counts[table] = 1
|
||||
with self.assertRaisesRegex(RuntimeError, 'final cutover refused'):
|
||||
migration.require_legacy_cutover_clear(self.db, self.config)
|
||||
self.counts[table] = 0
|
||||
self.conn.commit.assert_not_called()
|
||||
|
||||
def test_json_in_spool_reservations_or_quarantine_still_blocks_cutover(self):
|
||||
for parent in (self.spool, self.spool / 'reservations', self.spool / 'quarantine'):
|
||||
path = parent / 'unresolved.json'
|
||||
with self.subTest(parent=parent):
|
||||
path.write_text('{}', encoding='ascii')
|
||||
with self.assertRaisesRegex(RuntimeError, 'unresolved objects'):
|
||||
migration.require_legacy_cutover_clear(self.db, self.config)
|
||||
path.unlink()
|
||||
self.conn.commit.assert_not_called()
|
||||
|
||||
def test_prepared_legacy_ledger_append_still_blocks_cutover(self):
|
||||
path = self.results / 'scan_results.publication-ledger.sqlite3'
|
||||
connection = sqlite3.connect(path)
|
||||
try:
|
||||
connection.execute('CREATE TABLE publication_identity(state TEXT)')
|
||||
connection.execute("INSERT INTO publication_identity VALUES ('prepared')")
|
||||
connection.commit()
|
||||
finally:
|
||||
connection.close()
|
||||
with self.assertRaisesRegex(RuntimeError, 'prepared append'):
|
||||
migration.require_legacy_cutover_clear(self.db, self.config)
|
||||
self.conn.commit.assert_not_called()
|
||||
|
||||
def test_spool_entry_bound_still_fails_closed(self):
|
||||
with self.assertRaisesRegex(RuntimeError, 'entry bound'):
|
||||
migration.require_legacy_cutover_clear(self.db, self.config, max_entries=1)
|
||||
|
||||
|
||||
class NormalMigrationContractTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.argv = ['migrate_runtime_safety.py', '--config', 'fixture.yaml', '--initialize-base', '--apply', '--sources-stopped']
|
||||
self.enterContext(mock.patch.object(sys, 'argv', self.argv))
|
||||
self.config = {'global': {}}
|
||||
self.dsn = 'postgresql://fixture:fixture@127.0.0.1:55432/fixture'
|
||||
self.enterContext(mock.patch.object(migration, 'load_config', return_value=self.config))
|
||||
self.enterContext(mock.patch.object(migration, 'load_postgres_environment'))
|
||||
self.enterContext(mock.patch.object(migration, 'database_url_from_env', return_value=self.dsn))
|
||||
self.enterContext(mock.patch.object(migration, 'preflight_lifecycle_paths'))
|
||||
self.lock = self.enterContext(mock.patch.object(migration, 'ClusterAuthorityLock', return_value=contextlib.nullcontext()))
|
||||
self.stopped = self.enterContext(mock.patch.object(migration, 'require_local_sources_stopped'))
|
||||
self.enterContext(mock.patch.object(migration, 'verify_cluster_identity', return_value={
|
||||
'database': 'fixture', 'user': 'fixture', 'port': 55432,
|
||||
}))
|
||||
self.enterContext(mock.patch.object(migration, '_online_postgres_identity'))
|
||||
self.guard = self.enterContext(mock.patch.object(migration, 'postgres_migration_guard', return_value=contextlib.nullcontext()))
|
||||
self.events = []
|
||||
self.db = mock.Mock(enabled=True)
|
||||
self.db.conn.is_postgres = True
|
||||
self.db.conn.table_exists.return_value = True
|
||||
self.migrations = [
|
||||
{'version': version, 'code_sha256': hashlib.sha256(version.encode('ascii')).hexdigest()}
|
||||
for version in migration.PIPELINE_MIGRATION_VERSIONS
|
||||
]
|
||||
self.db.conn.execute.return_value.fetchall.return_value = self.migrations
|
||||
self.db.revoke_final_cutover.side_effect = lambda: self.events.append('revoke')
|
||||
self.db.require_runtime_safety_schema.side_effect = lambda: self.events.append('verify-schema')
|
||||
self.db.record_final_cutover.side_effect = lambda evidence: self.events.append('record') or {'evidence_sha256': 'fixture-evidence'}
|
||||
self.db.require_final_cutover.side_effect = lambda: self.events.append('verify-cutover')
|
||||
self.construct = self.enterContext(mock.patch.object(migration, 'ScannerDB', return_value=self.db))
|
||||
self.schema = self.enterContext(mock.patch.object(migration, 'migrate_runtime_safety_schema', side_effect=lambda *_, **__: self.events.append('migrate')))
|
||||
self.cursors = self.enterContext(mock.patch.object(migration, 'initialize_projection_cursors_from_existing_files', side_effect=lambda *args: self.events.append('cursors') or {'cursor-evidence': True}))
|
||||
self.legacy = self.enterContext(mock.patch.object(migration, 'require_legacy_cutover_clear', side_effect=lambda *args: self.events.append('legacy') or {'legacy-evidence': True}))
|
||||
self.enterContext(mock.patch('builtins.print'))
|
||||
|
||||
def test_normal_initialize_base_runs_every_gate_and_records_all_30_migration_rows(self):
|
||||
self.assertEqual(migration.main(), 0)
|
||||
self.schema.assert_called_once_with(self.db, initialize_base=True)
|
||||
self.assertEqual(self.events, ['revoke', 'migrate', 'cursors', 'legacy', 'verify-schema', 'record', 'verify-cutover'])
|
||||
self.assertEqual(len(self.migrations), 32)
|
||||
self.db.record_final_cutover.assert_called_once_with({
|
||||
'legacy': {'legacy-evidence': True},
|
||||
'projection_cursors': {'cursor-evidence': True},
|
||||
'schema_migrations': self.migrations,
|
||||
})
|
||||
self.assertEqual(self.stopped.call_count, 2)
|
||||
self.guard.assert_called_once_with(self.db)
|
||||
self.lock.assert_called_once_with(self.config, endpoint_dsn=self.dsn)
|
||||
self.db.close.assert_called_once()
|
||||
|
||||
def test_both_explicit_apply_and_sources_stopped_are_required(self):
|
||||
for flag in ('--apply', '--sources-stopped'):
|
||||
with self.subTest(flag=flag), mock.patch.object(sys, 'argv', [arg for arg in self.argv if arg != flag]):
|
||||
with self.assertRaisesRegex(SystemExit, 'both --apply and --sources-stopped'):
|
||||
migration.main()
|
||||
self.construct.assert_not_called()
|
||||
self.lock.assert_not_called()
|
||||
|
||||
def test_legacy_failure_cannot_record_or_validate_cutover(self):
|
||||
self.legacy.side_effect = RuntimeError('legacy state unresolved')
|
||||
with self.assertRaisesRegex(RuntimeError, 'legacy state unresolved'):
|
||||
migration.main()
|
||||
self.db.revoke_final_cutover.assert_called_once()
|
||||
self.schema.assert_called_once_with(self.db, initialize_base=True)
|
||||
self.db.record_final_cutover.assert_not_called()
|
||||
self.db.require_final_cutover.assert_not_called()
|
||||
self.db.close.assert_called_once()
|
||||
|
||||
def test_schema_failure_does_not_skip_ahead_to_cursors_or_marker(self):
|
||||
self.schema.side_effect = RuntimeError('migration failed')
|
||||
with self.assertRaisesRegex(RuntimeError, 'migration failed'):
|
||||
migration.main()
|
||||
self.cursors.assert_not_called()
|
||||
self.legacy.assert_not_called()
|
||||
self.db.record_final_cutover.assert_not_called()
|
||||
self.db.close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user