sashatrask
456 lines
18 KiB
Python
456 lines
18 KiB
Python
import sys
|
|
|
|
sys.dont_write_bytecode = True
|
|
if not sys.dont_write_bytecode:
|
|
raise RuntimeError('janitor could not disable bytecode writes')
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
|
|
from lifecycle_authority import require_active_supervisor_child
|
|
from paths import apply_path_config
|
|
from process_identity import exact_process_identity_state
|
|
from runtime_security import (
|
|
atomic_write_private_json,
|
|
canonical_path,
|
|
fsync_directory,
|
|
is_reparse_point,
|
|
private_directory_ready,
|
|
private_file_ready,
|
|
read_private_json,
|
|
reject_reparse_components,
|
|
require_private_directory,
|
|
)
|
|
|
|
|
|
MARKER_NAME = '.scanner-owner.json'
|
|
MARKER_SCHEMA = 2
|
|
APPROVED_LAYOUTS = (
|
|
('work', '', ('trufflehog-', 'trufflehog-run-', 'trufflehog-probe-', 'hg-run-', 'docker-config-', 'docker-layer-', 'tmp-', 'worker-assignment-')),
|
|
('work', 'docker-config', ('docker-config-',)),
|
|
('work', 'hg', ('hg-run-',)),
|
|
('work', 'tmp', ('trufflehog-', 'trufflehog-run-', 'hg-run-', 'docker-config-', 'docker-layer-', 'tmp-')),
|
|
('work', os.path.join('tmp', 'docker-config'), ('docker-config-',)),
|
|
('work', 'abandoned', ('worker-assignment-',)),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class JanitorBudget:
|
|
max_candidates: int = 50
|
|
max_entries: int = 10000
|
|
max_bytes: int = 1024 * 1024 * 1024
|
|
max_seconds: float = 30.0
|
|
max_depth: int = 64
|
|
max_enumerated: int = 1000
|
|
candidates: int = 0
|
|
entries: int = 0
|
|
bytes: int = 0
|
|
started_at: float = 0.0
|
|
exhausted: bool = False
|
|
enumerated: int = 0
|
|
|
|
def __post_init__(self):
|
|
self.started_at = self.started_at or time.monotonic()
|
|
|
|
def consume(self, size=0, candidate=False, depth=0, allow_oversized=False):
|
|
if candidate:
|
|
self.candidates += 1
|
|
else:
|
|
self.entries += 1
|
|
self.bytes += max(0, int(size or 0))
|
|
candidate_limit = self.candidates > self.max_candidates
|
|
entry_limit = self.entries > self.max_entries
|
|
byte_limit = self.bytes > self.max_bytes
|
|
depth_limit = depth > self.max_depth
|
|
time_limit = time.monotonic() - self.started_at >= self.max_seconds
|
|
self.exhausted = bool(
|
|
candidate_limit or entry_limit or byte_limit or depth_limit or time_limit
|
|
)
|
|
# Unlinking one regular file is bounded metadata work regardless of its
|
|
# payload size. Directory traversal remains bounded by the other limits.
|
|
oversized_progress = bool(
|
|
allow_oversized and byte_limit
|
|
and not (candidate_limit or entry_limit or depth_limit or time_limit)
|
|
)
|
|
return not self.exhausted or oversized_progress
|
|
|
|
def consume_enumerated(self):
|
|
self.enumerated += 1
|
|
self.exhausted = bool(
|
|
self.enumerated > self.max_enumerated
|
|
or time.monotonic() - self.started_at >= self.max_seconds
|
|
)
|
|
return not self.exhausted
|
|
|
|
|
|
def _marker_relative_path(root, path):
|
|
relative = os.path.relpath(path, root)
|
|
if relative == '.' or relative.startswith('..' + os.sep) or os.path.isabs(relative):
|
|
raise ValueError('candidate escapes the approved janitor root')
|
|
return relative.replace(os.sep, '/')
|
|
|
|
|
|
def validate_marker(root, path, marker, allowed_executables, minimum_age_sec, now=None):
|
|
if marker.get('schema') != MARKER_SCHEMA or marker.get('root_kind') != 'work':
|
|
return False, 'unsupported_marker'
|
|
try:
|
|
if marker.get('relative_path') != _marker_relative_path(root, path):
|
|
return False, 'path_mismatch'
|
|
created = datetime.fromisoformat(str(marker.get('created_at') or '').replace('Z', '+00:00'))
|
|
if created.tzinfo is None:
|
|
created = created.replace(tzinfo=timezone.utc)
|
|
now_value = now or datetime.now(timezone.utc)
|
|
if (now_value - created).total_seconds() < max(0, float(minimum_age_sec)):
|
|
return False, 'too_young'
|
|
except (TypeError, ValueError):
|
|
return False, 'invalid_time_or_path'
|
|
|
|
allowed = {canonical_path(value) for value in allowed_executables if value}
|
|
identities = {}
|
|
states = {}
|
|
prefixes = ('owner', 'parent')
|
|
if any(f'child_{field}' in marker for field in ('pid', 'creation_time', 'executable')):
|
|
prefixes += ('child',)
|
|
for prefix in prefixes:
|
|
if prefix == 'child' and any(not marker.get(f'child_{field}') for field in ('pid', 'creation_time', 'executable')):
|
|
return False, 'child_identity_invalid'
|
|
identity = {
|
|
'pid': marker.get(f'{prefix}_pid'),
|
|
'creation_time': marker.get(f'{prefix}_creation_time'),
|
|
'executable': marker.get(f'{prefix}_executable'),
|
|
}
|
|
try:
|
|
executable = canonical_path(identity['executable'])
|
|
except (OSError, TypeError, ValueError):
|
|
return False, f'{prefix}_identity_invalid'
|
|
if executable not in allowed:
|
|
return False, f'{prefix}_executable_unapproved'
|
|
identity['executable'] = executable
|
|
identities[prefix] = identity
|
|
states[prefix] = exact_process_identity_state(
|
|
identity['pid'], identity['creation_time'], identity['executable'],
|
|
)
|
|
if 'child' in states and states['child'] != 'dead':
|
|
return False, 'child_live_or_unknown'
|
|
if states['owner'] != 'dead':
|
|
return False, 'owner_live_or_unknown'
|
|
same_identity = all(
|
|
identities['owner'].get(field) == identities['parent'].get(field)
|
|
for field in ('pid', 'creation_time', 'executable')
|
|
)
|
|
if same_identity and states['parent'] != 'dead':
|
|
return False, 'parent_owned_live_or_unknown'
|
|
return True, 'eligible'
|
|
|
|
|
|
def bounded_remove_tree(path, budget, marker_name=MARKER_NAME):
|
|
"""Delete without recursion or reparse traversal; leave the root marker last."""
|
|
path = os.path.abspath(path)
|
|
reject_reparse_components(path)
|
|
if is_reparse_point(path) or not private_directory_ready(path):
|
|
raise OSError(f'janitor candidate is not an exact private directory: {path}')
|
|
marker_path = os.path.join(path, marker_name)
|
|
stack = []
|
|
root_iterator = os.scandir(path)
|
|
stack.append((path, root_iterator, 0))
|
|
try:
|
|
while stack:
|
|
if time.monotonic() - budget.started_at >= budget.max_seconds:
|
|
budget.exhausted = True
|
|
return False
|
|
directory, iterator, depth = stack[-1]
|
|
try:
|
|
entry = next(iterator)
|
|
except StopIteration:
|
|
iterator.close()
|
|
stack.pop()
|
|
if directory == path:
|
|
if os.path.lexists(marker_path):
|
|
details = os.stat(marker_path, follow_symlinks=False)
|
|
# The verified owner marker is removed only after every payload
|
|
# entry is gone, so finishing the empty root must make progress
|
|
# even when one oversized payload exhausted this pass's budget.
|
|
budget.consume(details.st_size, depth=depth + 1)
|
|
if not private_file_ready(marker_path):
|
|
raise OSError('janitor owner marker lost its private identity')
|
|
os.remove(marker_path)
|
|
os.rmdir(directory)
|
|
fsync_directory(os.path.dirname(directory))
|
|
return True
|
|
os.rmdir(directory)
|
|
continue
|
|
if entry.path == marker_path:
|
|
continue
|
|
details = entry.stat(follow_symlinks=False)
|
|
is_regular = stat.S_ISREG(details.st_mode)
|
|
if not budget.consume(
|
|
details.st_size, depth=depth + 1, allow_oversized=is_regular,
|
|
):
|
|
return False
|
|
if entry.is_symlink() or is_reparse_point(entry.path):
|
|
raise OSError(f'janitor candidate contains a link or reparse point: {entry.path}')
|
|
if stat.S_ISDIR(details.st_mode):
|
|
child_iterator = os.scandir(entry.path)
|
|
stack.append((entry.path, child_iterator, depth + 1))
|
|
elif is_regular:
|
|
os.chmod(entry.path, stat.S_IWRITE | stat.S_IREAD)
|
|
os.remove(entry.path)
|
|
else:
|
|
raise OSError(f'janitor candidate contains an unsupported entry: {entry.path}')
|
|
finally:
|
|
for _, iterator, _ in stack:
|
|
iterator.close()
|
|
return False
|
|
|
|
|
|
def _layout_cursor_name(root, relative_parent, prefixes):
|
|
identity = '|'.join((
|
|
canonical_path(root), str(relative_parent).replace(os.sep, '/'), ','.join(prefixes),
|
|
))
|
|
return 'layout:' + hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
|
|
|
|
|
class JanitorCursorStore:
|
|
SCHEMA = 1
|
|
|
|
def __init__(self, path, root):
|
|
self.path = os.path.abspath(path)
|
|
self.root_hash = hashlib.sha256(canonical_path(root).encode('utf-8')).hexdigest()
|
|
self.dirty = False
|
|
self.state = {
|
|
'schema': self.SCHEMA,
|
|
'root_sha256': self.root_hash,
|
|
'next_layout': 0,
|
|
'layouts': {},
|
|
}
|
|
self.iterators = {}
|
|
self.seeking = {}
|
|
if os.path.lexists(self.path):
|
|
loaded = read_private_json(self.path, max_bytes=256 * 1024)
|
|
if (
|
|
not isinstance(loaded, dict)
|
|
or loaded.get('schema') != self.SCHEMA
|
|
or loaded.get('root_sha256') != self.root_hash
|
|
or not isinstance(loaded.get('layouts'), dict)
|
|
):
|
|
raise RuntimeError('janitor cursor authority is invalid')
|
|
self.state = loaded
|
|
else:
|
|
require_private_directory(os.path.dirname(self.path), create=True)
|
|
atomic_write_private_json(self.path, self.state)
|
|
|
|
def _save(self):
|
|
atomic_write_private_json(self.path, self.state)
|
|
self.dirty = False
|
|
|
|
def _mark_dirty(self):
|
|
self.dirty = True
|
|
|
|
def flush(self):
|
|
if self.dirty:
|
|
self._save()
|
|
|
|
def next_layout(self, count):
|
|
index = int(self.state.get('next_layout') or 0) % max(1, int(count))
|
|
self.state['next_layout'] = (index + 1) % max(1, int(count))
|
|
self._mark_dirty()
|
|
return index
|
|
|
|
def next_entry(self, layout_name, parent):
|
|
iterator = self.iterators.get(layout_name)
|
|
if iterator is None:
|
|
iterator = os.scandir(parent)
|
|
self.iterators[layout_name] = iterator
|
|
last_name = str((self.state['layouts'].get(layout_name) or {}).get('last_name') or '')
|
|
self.seeking[layout_name] = bool(last_name)
|
|
try:
|
|
entry = next(iterator)
|
|
except StopIteration:
|
|
iterator.close()
|
|
self.iterators.pop(layout_name, None)
|
|
self.seeking.pop(layout_name, None)
|
|
current = self.state['layouts'].setdefault(layout_name, {})
|
|
current['last_name'] = ''
|
|
current['wrap_count'] = int(current.get('wrap_count') or 0) + 1
|
|
self._mark_dirty()
|
|
return None, False
|
|
|
|
current = self.state['layouts'].setdefault(layout_name, {'last_name': '', 'wrap_count': 0})
|
|
target = str(current.get('last_name') or '')
|
|
if self.seeking.get(layout_name):
|
|
if entry.name == target:
|
|
self.seeking[layout_name] = False
|
|
return entry, True
|
|
current['last_name'] = entry.name
|
|
self._mark_dirty()
|
|
return entry, False
|
|
|
|
def close(self):
|
|
for iterator in self.iterators.values():
|
|
iterator.close()
|
|
self.iterators.clear()
|
|
|
|
|
|
class _MemoryCursorStore(JanitorCursorStore):
|
|
def __init__(self):
|
|
self.path = ''
|
|
self.root_hash = ''
|
|
self.dirty = False
|
|
self.state = {'schema': 1, 'root_sha256': '', 'next_layout': 0, 'layouts': {}}
|
|
self.iterators = {}
|
|
self.seeking = {}
|
|
|
|
def _save(self):
|
|
self.dirty = False
|
|
return None
|
|
|
|
|
|
def iter_candidates(root, budget, cursor_store):
|
|
layouts = list(APPROVED_LAYOUTS)
|
|
completed_layouts = set()
|
|
while not budget.exhausted and len(completed_layouts) < len(layouts):
|
|
if (
|
|
budget.enumerated >= budget.max_enumerated
|
|
or time.monotonic() - budget.started_at >= budget.max_seconds
|
|
):
|
|
budget.exhausted = True
|
|
return
|
|
index = cursor_store.next_layout(len(layouts))
|
|
root_kind, relative_parent, prefixes = layouts[index]
|
|
layout_name = _layout_cursor_name(root, relative_parent, prefixes)
|
|
if layout_name in completed_layouts:
|
|
continue
|
|
parent = os.path.join(root, relative_parent) if relative_parent else root
|
|
try:
|
|
if not os.path.isdir(parent) or is_reparse_point(parent):
|
|
completed_layouts.add(layout_name)
|
|
continue
|
|
entry, seeking = cursor_store.next_entry(layout_name, parent)
|
|
except OSError:
|
|
completed_layouts.add(layout_name)
|
|
continue
|
|
if entry is None:
|
|
completed_layouts.add(layout_name)
|
|
continue
|
|
if not budget.consume_enumerated():
|
|
return
|
|
if seeking:
|
|
continue
|
|
if not entry.name.startswith(prefixes) or not entry.is_dir(follow_symlinks=False):
|
|
continue
|
|
if not budget.consume(candidate=True):
|
|
return
|
|
yield root_kind, layout_name, entry.name, entry.path
|
|
|
|
|
|
def run_janitor_pass(
|
|
root, allowed_executables, minimum_age_sec=7200, budget=None,
|
|
cursor_store=None, excluded_relative_paths=(),
|
|
):
|
|
budget = budget or JanitorBudget()
|
|
root = require_private_directory(root, create=False)
|
|
cursor_store = cursor_store or _MemoryCursorStore()
|
|
excluded = set()
|
|
for value in excluded_relative_paths:
|
|
relative = str(value or '').replace('\\', '/')
|
|
if (
|
|
not relative or relative.startswith('/') or relative.endswith('/')
|
|
or any(part in ('', '.', '..') for part in relative.split('/'))
|
|
):
|
|
raise ValueError('janitor exclusion path is invalid')
|
|
excluded.add(relative)
|
|
if len(excluded) > 4096:
|
|
raise ValueError('janitor exclusion set exceeds its bound')
|
|
report = {'considered': 0, 'removed': 0, 'retained': 0, 'errors': 0, 'exhausted': False}
|
|
try:
|
|
for _, _, _, path in iter_candidates(root, budget, cursor_store):
|
|
report['considered'] += 1
|
|
if _marker_relative_path(root, path) in excluded:
|
|
report['retained'] += 1
|
|
continue
|
|
marker_path = os.path.join(path, MARKER_NAME)
|
|
try:
|
|
if not private_file_ready(marker_path):
|
|
report['retained'] += 1
|
|
continue
|
|
marker = read_private_json(marker_path)
|
|
eligible, _ = validate_marker(
|
|
root, path, marker, allowed_executables, minimum_age_sec,
|
|
)
|
|
if not eligible:
|
|
report['retained'] += 1
|
|
continue
|
|
if bounded_remove_tree(path, budget):
|
|
report['removed'] += 1
|
|
else:
|
|
report['retained'] += 1
|
|
except (OSError, ValueError):
|
|
report['errors'] += 1
|
|
if budget.exhausted:
|
|
break
|
|
finally:
|
|
cursor_store.flush()
|
|
report['exhausted'] = budget.exhausted
|
|
report['entries'] = budget.entries
|
|
report['bytes'] = budget.bytes
|
|
report['enumerated'] = budget.enumerated
|
|
return report
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description='Bounded scanner work-directory janitor')
|
|
parser.add_argument('--config', required=True)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
metadata = require_active_supervisor_child(child_kind='janitor', require_dsn=False)
|
|
args = parse_args()
|
|
import yaml
|
|
|
|
with open(args.config, 'r', encoding='utf-8') as handle:
|
|
config = apply_path_config(yaml.safe_load(handle) or {}, args.config)
|
|
global_config = config.get('global') or {}
|
|
janitor_config = ((config.get('supervisor') or {}).get('janitor') or {})
|
|
manifest = metadata.get('code_manifest') or {}
|
|
allowed = [sys.executable]
|
|
allowed.extend(
|
|
item.get('path') for item in (manifest.get('executables') or {}).values()
|
|
if isinstance(item, dict) and item.get('path')
|
|
)
|
|
interval = max(5.0, float(janitor_config.get('interval_sec', 60) or 60))
|
|
cursor_store = JanitorCursorStore(
|
|
os.path.join(global_config['state_dir'], 'janitor.cursor.json'),
|
|
global_config['work_dir'],
|
|
)
|
|
try:
|
|
while True:
|
|
budget = JanitorBudget(
|
|
max_candidates=max(1, int(janitor_config.get('max_candidates', 50) or 50)),
|
|
max_entries=max(1, int(janitor_config.get('max_entries', 10000) or 10000)),
|
|
max_bytes=max(1, int(janitor_config.get('max_bytes', 1024 * 1024 * 1024) or 1)),
|
|
max_seconds=max(0.1, float(janitor_config.get('max_seconds', 30) or 30)),
|
|
max_depth=max(1, int(janitor_config.get('max_depth', 64) or 64)),
|
|
max_enumerated=max(1, int(janitor_config.get('max_enumerated', 1000) or 1000)),
|
|
)
|
|
report = run_janitor_pass(
|
|
global_config['work_dir'], allowed,
|
|
minimum_age_sec=max(0, int(janitor_config.get('minimum_age_sec', 7200) or 0)),
|
|
budget=budget, cursor_store=cursor_store,
|
|
)
|
|
print(json.dumps(report, ensure_ascii=True, sort_keys=True), flush=True)
|
|
time.sleep(interval)
|
|
finally:
|
|
cursor_store.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|