Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,307 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'app'))
|
||||
import janitor
|
||||
import scanner
|
||||
from test_docker_staging_bounds import command_harness
|
||||
|
||||
|
||||
REAL_SHARED_OWNERS = scanner._shared_staging_owners
|
||||
REAL_WRITE_OWNER = scanner.write_temp_owner
|
||||
REAL_CLEANUP = scanner.cleanup_command_work_dir
|
||||
REAL_ATOMIC_WRITE = scanner.atomic_write_private_json
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owned_runner(command_harness, monkeypatch):
|
||||
state = command_harness
|
||||
state.work_root = state.command_dir.parent
|
||||
state.command_dir = state.work_root / 'trufflehog-run-command'
|
||||
state.blobs = state.work_root / 'docker-layer-shared'
|
||||
for path in (state.work_root, state.blobs):
|
||||
scanner.ensure_private_directory(str(path))
|
||||
monkeypatch.setattr(scanner, '_runtime_initialized', True)
|
||||
monkeypatch.setattr(scanner.scan_config, 'work_dir', str(state.work_root))
|
||||
monkeypatch.setattr(scanner, '_shared_staging_owners', REAL_SHARED_OWNERS)
|
||||
monkeypatch.setattr(scanner, 'write_temp_owner', REAL_WRITE_OWNER)
|
||||
monkeypatch.setattr(scanner, 'require_git_clone_launch_authority', lambda cmd: None)
|
||||
state.parent = scanner.serialize_process_identity(scanner.current_process_identity())
|
||||
REAL_WRITE_OWNER(str(state.blobs), required=True)
|
||||
state.cleanups = mock.Mock(wraps=REAL_CLEANUP)
|
||||
monkeypatch.setattr(scanner, 'cleanup_command_work_dir', state.cleanups)
|
||||
|
||||
def create_command_dir():
|
||||
scanner.ensure_private_directory(str(state.command_dir))
|
||||
REAL_WRITE_OWNER(str(state.command_dir), required=True)
|
||||
return str(state.command_dir)
|
||||
|
||||
monkeypatch.setattr(scanner, 'create_command_work_dir', create_command_dir)
|
||||
original_launch = scanner.OwnedProcess
|
||||
state.identities = []
|
||||
state.pending_at_launch = []
|
||||
|
||||
def launch(command, **options):
|
||||
state.pending_at_launch.append(all(
|
||||
scanner.read_private_json(str(root / scanner.TEMP_OWNER_FILE)).get('child_pid', 'absent') is None
|
||||
for root in (state.command_dir, state.blobs)
|
||||
))
|
||||
process = original_launch(command, **options)
|
||||
identity = {'pid': 41001 + len(state.identities), 'creation_time': f'fixture-{len(state.identities)}',
|
||||
'executable': state.parent['executable']}
|
||||
state.identities.append(identity)
|
||||
process.pid = identity['pid']
|
||||
process.payload_identity = identity
|
||||
process.returncode = None
|
||||
state.children = {process.pid}
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(scanner, 'OwnedProcess', launch)
|
||||
|
||||
def identity_state(pid, created, executable):
|
||||
if pid is None or not created or not executable:
|
||||
return 'unknown'
|
||||
return 'alive' if pid in state.children or pid == state.parent['pid'] else 'dead'
|
||||
|
||||
monkeypatch.setattr(scanner, 'exact_process_identity_state', identity_state)
|
||||
state.marker = lambda root=state.blobs: scanner.read_private_json(str(root / scanner.TEMP_OWNER_FILE))
|
||||
state.command = lambda: ['fixture', 'clone', '--no-checkout', '--no-recurse-submodules', '--',
|
||||
'https://fixture.invalid/repo.git', str(state.blobs / 'repo')]
|
||||
return state
|
||||
|
||||
|
||||
@pytest.mark.parametrize('native', [False, True])
|
||||
def test_retained_roots_cannot_be_reaped_with_dead_scanner_and_live_child(owned_runner, monkeypatch, native):
|
||||
state = owned_runner
|
||||
state.unkillable = True
|
||||
checks = ['', 'TruffleHog staging limit exceeded'] if native else ['TruffleHog staging limit exceeded']
|
||||
monkeypatch.setattr(scanner, '_check_command_staging', mock.Mock(side_effect=checks))
|
||||
with pytest.raises(scanner.ScanSlotFatalError):
|
||||
with scanner.run_command_streamed(state.command(), 30, staging_roots=(str(state.blobs),), native_git_clone=native):
|
||||
pass
|
||||
state.cleanups.assert_not_called()
|
||||
state.slot.release.assert_not_called()
|
||||
assert state.pending_at_launch == [True]
|
||||
marker = state.marker()
|
||||
assert marker['owner_pid'] == marker['parent_pid'] == state.parent['pid']
|
||||
assert marker['child_pid'] == state.identities[0]['pid']
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state',
|
||||
lambda pid, *_: 'alive' if pid in state.children else 'dead')
|
||||
valid, reason = janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0)
|
||||
assert not valid and reason == 'child_live_or_unknown'
|
||||
report = janitor.run_janitor_pass(str(state.work_root), [state.parent['executable']], minimum_age_sec=0)
|
||||
assert report['considered'] >= 2
|
||||
assert report['removed'] == 0
|
||||
assert state.blobs.is_dir() and state.command_dir.is_dir()
|
||||
# Direct shared-root callers (including Docker cleanup) cannot remove it either.
|
||||
REAL_CLEANUP(str(state.blobs))
|
||||
assert state.blobs.is_dir() and (state.blobs / scanner.TEMP_OWNER_FILE).is_file()
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda *_: 'dead')
|
||||
assert janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0) == (True, 'eligible')
|
||||
|
||||
|
||||
def test_sequential_children_refresh_identity_and_restore_parent_lifetime(owned_runner, monkeypatch):
|
||||
state = owned_runner
|
||||
active = []
|
||||
|
||||
def while_active():
|
||||
marker = state.marker()
|
||||
active.append((marker['child_pid'], marker['child_creation_time']))
|
||||
with mock.patch.object(janitor, 'exact_process_identity_state',
|
||||
side_effect=lambda pid, *_: 'alive' if pid in state.children else 'dead'):
|
||||
assert not janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0)[0]
|
||||
state.completed = True
|
||||
|
||||
state.on_wait = while_active
|
||||
for _ in range(2):
|
||||
state.completed = False
|
||||
with scanner.run_command_streamed(['fixture'], 30, staging_roots=(str(state.blobs),)):
|
||||
pass
|
||||
assert state.blobs.is_dir()
|
||||
marker = state.marker()
|
||||
assert marker['owner_pid'] == state.parent['pid']
|
||||
assert 'child_pid' not in marker
|
||||
with mock.patch.object(janitor, 'exact_process_identity_state', return_value='alive'):
|
||||
assert not janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0)[0]
|
||||
assert len(set(active)) == 2
|
||||
assert state.pending_at_launch == [True, True]
|
||||
REAL_CLEANUP(str(state.blobs))
|
||||
assert not state.blobs.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('publication', ['command', 'shared'])
|
||||
def test_failed_child_publication_leaves_pending_marker_when_termination_is_unknown(owned_runner, monkeypatch, publication):
|
||||
state = owned_runner
|
||||
state.unkillable = True
|
||||
|
||||
def write(path, value):
|
||||
command_failure = publication == 'command' and value.get('owner_pid') in state.children
|
||||
shared_failure = publication == 'shared' and value.get('child_pid') in state.children
|
||||
if command_failure or shared_failure:
|
||||
raise OSError('synthetic publication failure')
|
||||
return REAL_ATOMIC_WRITE(path, value)
|
||||
|
||||
monkeypatch.setattr(scanner, 'atomic_write_private_json', write)
|
||||
with pytest.raises(scanner.ScanSlotFatalError):
|
||||
with scanner.run_command_streamed(['fixture'], 30, staging_roots=(str(state.blobs),)):
|
||||
pass
|
||||
state.cleanups.assert_not_called()
|
||||
marker = state.marker()
|
||||
assert 'child_pid' in marker and marker['child_pid'] is None
|
||||
with mock.patch.object(janitor, 'exact_process_identity_state', return_value='dead'):
|
||||
assert not janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0)[0]
|
||||
REAL_CLEANUP(str(state.blobs))
|
||||
assert state.blobs.is_dir()
|
||||
|
||||
|
||||
def test_restore_failure_does_not_leak_slot_or_delete_unconfirmed_marker(owned_runner, monkeypatch):
|
||||
state = owned_runner
|
||||
state.completed = True
|
||||
|
||||
def write(path, value):
|
||||
if (state.identities and scanner.canonical_path(path) == scanner.canonical_path(state.blobs / scanner.TEMP_OWNER_FILE)
|
||||
and 'child_pid' not in value):
|
||||
raise OSError('synthetic restore failure')
|
||||
return REAL_ATOMIC_WRITE(path, value)
|
||||
|
||||
monkeypatch.setattr(scanner, 'atomic_write_private_json', write)
|
||||
with pytest.raises(RuntimeError, match='restore shared staging ownership'):
|
||||
with scanner.run_command_streamed(['fixture'], 30, staging_roots=(str(state.blobs),)):
|
||||
pass
|
||||
state.slot.release.assert_called_once()
|
||||
assert state.marker()['child_pid'] == state.identities[0]['pid']
|
||||
assert state.blobs.is_dir()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('state_value', ['alive', 'unknown', 'reused'])
|
||||
def test_janitor_never_accepts_unconfirmed_recorded_child(owned_runner, monkeypatch, state_value):
|
||||
state = owned_runner
|
||||
marker = dict(state.marker(), child_pid=41001, child_creation_time='child', child_executable=state.parent['executable'])
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state',
|
||||
lambda pid, *_: state_value if pid == 41001 else 'dead')
|
||||
assert janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0) == (False, 'child_live_or_unknown')
|
||||
|
||||
|
||||
def test_child_executable_must_remain_authorized(owned_runner, monkeypatch):
|
||||
state = owned_runner
|
||||
marker = dict(state.marker(), child_pid=41001, child_creation_time='child',
|
||||
child_executable=str(state.work_root / 'unapproved.exe'))
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda *_: 'dead')
|
||||
assert janitor.validate_marker(str(state.work_root), str(state.blobs), marker,
|
||||
[state.parent['executable']], 0) == (False, 'child_executable_unapproved')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change', [
|
||||
{'schema': 99}, {'relative_path': 'other'}, {'parent_pid': 123}, {'owner_pid': 123},
|
||||
{'child_pid': None}, {'child_pid': 41001, 'child_creation_time': 'child', 'child_executable': sys.executable},
|
||||
])
|
||||
def test_shared_root_validation_rejects_untrusted_or_busy_markers(owned_runner, monkeypatch, change):
|
||||
state = owned_runner
|
||||
marker_path = str(state.blobs / scanner.TEMP_OWNER_FILE)
|
||||
REAL_ATOMIC_WRITE(marker_path, dict(state.marker(), **change))
|
||||
state.children = {41001}
|
||||
write = mock.Mock(side_effect=AssertionError('validation must not write'))
|
||||
monkeypatch.setattr(scanner, 'atomic_write_private_json', write)
|
||||
with pytest.raises(RuntimeError):
|
||||
REAL_SHARED_OWNERS((str(state.blobs),))
|
||||
write.assert_not_called()
|
||||
|
||||
|
||||
def test_outside_and_unmarked_roots_are_not_adopted(owned_runner):
|
||||
state = owned_runner
|
||||
with pytest.raises(RuntimeError, match='escapes'):
|
||||
REAL_SHARED_OWNERS((str(state.work_root),))
|
||||
unmarked = state.work_root / 'unmarked'
|
||||
scanner.ensure_private_directory(str(unmarked))
|
||||
with pytest.raises(RuntimeError, match='no authenticated owner'):
|
||||
REAL_SHARED_OWNERS((str(unmarked),))
|
||||
with tempfile.TemporaryDirectory(dir=state.work_root.parent) as outside:
|
||||
scanner.harden_private_directory(outside)
|
||||
with pytest.raises(RuntimeError, match='escapes'):
|
||||
REAL_SHARED_OWNERS((outside,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('guard', ['require_private_directory', 'require_private_file', 'read_private_json'])
|
||||
def test_private_directory_marker_and_json_checks_fail_closed(owned_runner, monkeypatch, guard):
|
||||
state = owned_runner
|
||||
monkeypatch.setattr(scanner, guard, mock.Mock(side_effect=OSError('synthetic private-file rejection')))
|
||||
writer = mock.Mock(side_effect=AssertionError('untrusted root must not be marked'))
|
||||
monkeypatch.setattr(scanner, 'atomic_write_private_json', writer)
|
||||
with pytest.raises((OSError, RuntimeError)):
|
||||
REAL_SHARED_OWNERS((str(state.blobs),))
|
||||
writer.assert_not_called()
|
||||
|
||||
|
||||
def test_nested_staging_resolves_only_its_existing_owned_parent(owned_runner):
|
||||
state = owned_runner
|
||||
nested = state.blobs / 'nested'
|
||||
scanner.ensure_private_directory(str(nested))
|
||||
owners = REAL_SHARED_OWNERS((str(nested), str(state.blobs)))
|
||||
assert len(owners) == 1
|
||||
assert owners[0][0] == scanner.canonical_path(state.blobs)
|
||||
assert not (nested / scanner.TEMP_OWNER_FILE).exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('relative_parent', ['', 'tmp'])
|
||||
def test_actual_docker_layer_prefix_is_enumerated_but_reaped_only_after_death_and_age(owned_runner, monkeypatch, relative_parent):
|
||||
state = owned_runner
|
||||
parent = state.work_root / relative_parent
|
||||
scanner.ensure_private_directory(str(parent))
|
||||
candidate = parent / 'docker-layer-aged'
|
||||
scanner.ensure_private_directory(str(candidate))
|
||||
REAL_WRITE_OWNER(str(candidate), required=True)
|
||||
marker_path = str(candidate / scanner.TEMP_OWNER_FILE)
|
||||
marker = dict(scanner.read_private_json(marker_path), child_pid=41001,
|
||||
child_creation_time='synthetic-child', child_executable=state.parent['executable'])
|
||||
REAL_ATOMIC_WRITE(marker_path, marker)
|
||||
payload = candidate / 'blob-0000'
|
||||
payload.write_bytes(b'synthetic retained payload')
|
||||
scanner.harden_private_file(str(payload))
|
||||
cursor = janitor._MemoryCursorStore()
|
||||
try:
|
||||
candidates = list(janitor.iter_candidates(str(state.work_root), janitor.JanitorBudget(), cursor))
|
||||
finally:
|
||||
cursor.close()
|
||||
assert scanner.canonical_path(candidate) in {scanner.canonical_path(row[3]) for row in candidates}
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda *_: 'dead')
|
||||
assert janitor.validate_marker(str(state.work_root), str(candidate), marker,
|
||||
[state.parent['executable']], 3600) == (False, 'too_young')
|
||||
young = janitor.run_janitor_pass(str(state.work_root), [state.parent['executable']], minimum_age_sec=3600)
|
||||
assert young['considered'] >= 1 and young['removed'] == 0 and payload.is_file()
|
||||
marker['created_at'] = '2020-01-01T00:00:00+00:00'
|
||||
REAL_ATOMIC_WRITE(marker_path, marker)
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda pid, *_: 'alive' if pid == 41001 else 'dead')
|
||||
live = janitor.run_janitor_pass(str(state.work_root), [state.parent['executable']], minimum_age_sec=3600)
|
||||
assert live['considered'] >= 1 and live['removed'] == 0 and payload.is_file()
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda *_: 'dead')
|
||||
assert janitor.validate_marker(str(state.work_root), str(candidate), marker,
|
||||
[state.parent['executable']], 3600) == (True, 'eligible')
|
||||
dead = janitor.run_janitor_pass(str(state.work_root), [state.parent['executable']], minimum_age_sec=3600)
|
||||
assert dead['removed'] == 1 and not candidate.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('relative', ['docker-layerish-unapproved', 'hg/docker-layer-wrong-layout'])
|
||||
def test_docker_prefix_expansion_does_not_admit_other_names_or_layouts(owned_runner, monkeypatch, relative):
|
||||
state = owned_runner
|
||||
candidate = state.work_root / relative
|
||||
scanner.ensure_private_directory(str(candidate.parent))
|
||||
scanner.ensure_private_directory(str(candidate))
|
||||
REAL_WRITE_OWNER(str(candidate), required=True)
|
||||
marker_path = str(candidate / scanner.TEMP_OWNER_FILE)
|
||||
marker = scanner.read_private_json(marker_path)
|
||||
marker['created_at'] = '2020-01-01T00:00:00+00:00'
|
||||
REAL_ATOMIC_WRITE(marker_path, marker)
|
||||
monkeypatch.setattr(janitor, 'exact_process_identity_state', lambda *_: 'dead')
|
||||
report = janitor.run_janitor_pass(str(state.work_root), [state.parent['executable']], minimum_age_sec=3600)
|
||||
assert report['removed'] == 0 and candidate.is_dir()
|
||||
Reference in New Issue
Block a user