Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,904 @@
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP))
|
||||
|
||||
import scan_execution
|
||||
import scanner
|
||||
import scanner_db
|
||||
from lifecycle_authority import LifecycleAuthorityError
|
||||
from parity_helpers import (
|
||||
configured_trufflehog, native_streamed_command, normalized_bundle_evidence,
|
||||
)
|
||||
from result_bundle import BundleReservation, FORMAT_VERSION, ResultBundleReader
|
||||
from worker_contracts import WorkerPhase, validate_phase_transition
|
||||
|
||||
|
||||
def reservation(platform='github'):
|
||||
return BundleReservation(
|
||||
reservation_id=7, reservation_token='request-token', bundle_id='a' * 32,
|
||||
scan_event_id='b' * 32, queue_id=11, claim_lease_token='lease-token',
|
||||
declared_bytes=1024, ready_path='ready/aa/' + 'a' * 32 + '.trb',
|
||||
source=platform, platform=platform, query='fixture',
|
||||
target='https://example.invalid/repo',
|
||||
normalized_target='https://example.invalid/repo',
|
||||
)
|
||||
|
||||
|
||||
def scan_policy():
|
||||
return {
|
||||
'drop_detectors': ['generic'],
|
||||
'strict_git_provider_token_filter': True,
|
||||
'trufflehog_stdout_max_mb': 2,
|
||||
'trufflehog_stderr_max_mb': 1,
|
||||
'result_bundle_max_event_bytes': 64 * 1024 * 1024,
|
||||
'trufflehog_max_findings_per_target': 20000,
|
||||
'trufflehog_job_memory_limit_bytes': 0,
|
||||
'trufflehog_windows_job_cpu_weight': 0,
|
||||
'trufflehog_windows_memory_priority': 0,
|
||||
'trufflehog_diagnostic_max_lines': 100,
|
||||
'trufflehog_diagnostic_max_line_chars': 1000,
|
||||
'trufflehog_diagnostic_max_line_bytes': 1000,
|
||||
'trufflehog_diagnostic_max_errors': 10,
|
||||
'trufflehog_diagnostic_max_warnings': 10,
|
||||
'trufflehog_diagnostic_max_unclassified': 5,
|
||||
}
|
||||
|
||||
|
||||
def source_reservation(platform, target, bundle_id='a' * 32, reservation_id=7):
|
||||
return BundleReservation(
|
||||
reservation_id=reservation_id, reservation_token='request-token',
|
||||
bundle_id=bundle_id, scan_event_id='b' * 32, queue_id=11,
|
||||
claim_lease_token='lease-token', declared_bytes=1024 * 1024,
|
||||
ready_path=f'ready/{bundle_id[:2]}/{bundle_id}.trb',
|
||||
source=platform, platform=platform, query='fixture', target=target,
|
||||
normalized_target=scanner.normalize_target(target, platform),
|
||||
)
|
||||
|
||||
|
||||
def noop_git_plan():
|
||||
head = 'a' * 40
|
||||
return {
|
||||
'version': 1, 'provider': 'github',
|
||||
'repo_url': 'https://github.com/Owner/Repo.git',
|
||||
'repo_path': 'Owner/Repo', 'branch': 'Feature/Main',
|
||||
'ref': 'refs/heads/Feature/Main', 'head_sha': head,
|
||||
'ref_source': 'explicit', 'base_sha': head, 'mode': 'noop',
|
||||
'baseline_depth': 100,
|
||||
}
|
||||
|
||||
|
||||
class ScanExecutionTests(unittest.TestCase):
|
||||
def test_compatibility_requires_exact_policy_and_platform(self):
|
||||
value = scan_execution.ScanCompatibility(
|
||||
scan_execution.PROTOCOL_VERSION, FORMAT_VERSION, 'windows-x86_64',
|
||||
'a' * 64, 'b' * 64, 'c' * 64,
|
||||
)
|
||||
self.assertIs(scan_execution.validate_scan_compatibility(value, value), value)
|
||||
for field in ('platform_tag', 'code_manifest_sha256', 'effective_config_sha256',
|
||||
'detector_policy_sha256'):
|
||||
changed = dict(value.as_dict())
|
||||
changed[field] = 'linux-x86_64' if field == 'platform_tag' else 'd' * 64
|
||||
with self.subTest(field=field), self.assertRaises(scan_execution.ScanExecutionError):
|
||||
scan_execution.validate_scan_compatibility(value, changed)
|
||||
|
||||
def test_scan_kwargs_reject_unknown_commands_and_unbounded_timeout(self):
|
||||
self.assertEqual(
|
||||
scan_execution.validate_scan_kwargs('github', {'timeout_sec': 60, 'git_plan': {}}),
|
||||
{'timeout_sec': 60.0, 'git_plan': {}},
|
||||
)
|
||||
for value in ({'timeout_sec': 60, 'command': ['calc']}, {'timeout_sec': 0}):
|
||||
with self.assertRaises(scan_execution.ScanExecutionError):
|
||||
scan_execution.validate_scan_kwargs('github', value)
|
||||
|
||||
def test_remote_assignment_deadlines_are_exact_and_immutable(self):
|
||||
reservation_value = {
|
||||
'remote_issued_at': '2026-09-17T00:00:00+00:00',
|
||||
'remote_expires_at': '2026-09-18T00:00:00+00:00',
|
||||
}
|
||||
deadlines = {
|
||||
'target_scan_timeout_seconds': 60,
|
||||
'result_upload_body_timeout_seconds': 1800,
|
||||
'assignment_ttl_seconds': 86400,
|
||||
'assignment_issued_at': reservation_value['remote_issued_at'],
|
||||
'assignment_deadline_at': reservation_value['remote_expires_at'],
|
||||
}
|
||||
self.assertEqual(
|
||||
scan_execution._normalize_remote_assignment_deadlines(
|
||||
deadlines, reservation_value, {'timeout_sec': 60.0},
|
||||
),
|
||||
deadlines,
|
||||
)
|
||||
|
||||
invalid = []
|
||||
extra = copy.deepcopy(deadlines)
|
||||
extra['unknown'] = 1
|
||||
invalid.append((extra, reservation_value, {'timeout_sec': 60.0}))
|
||||
boolean = copy.deepcopy(deadlines)
|
||||
boolean['assignment_ttl_seconds'] = True
|
||||
invalid.append((boolean, reservation_value, {'timeout_sec': 60.0}))
|
||||
wrong_scan = copy.deepcopy(deadlines)
|
||||
wrong_scan['target_scan_timeout_seconds'] = 61
|
||||
invalid.append((wrong_scan, reservation_value, {'timeout_sec': 60.0}))
|
||||
wrong_interval = copy.deepcopy(deadlines)
|
||||
wrong_interval['assignment_ttl_seconds'] = 86399
|
||||
invalid.append((wrong_interval, reservation_value, {'timeout_sec': 60.0}))
|
||||
changed_reservation = copy.deepcopy(reservation_value)
|
||||
changed_reservation['remote_expires_at'] = '2026-09-18T00:00:01+00:00'
|
||||
invalid.append((deadlines, changed_reservation, {'timeout_sec': 60.0}))
|
||||
noncanonical = copy.deepcopy(deadlines)
|
||||
noncanonical['assignment_issued_at'] = '2026-09-17T00:00:00Z'
|
||||
noncanonical_reservation = copy.deepcopy(reservation_value)
|
||||
noncanonical_reservation['remote_issued_at'] = noncanonical[
|
||||
'assignment_issued_at'
|
||||
]
|
||||
invalid.append((noncanonical, noncanonical_reservation, {'timeout_sec': 60.0}))
|
||||
for value, reserved, scan_kwargs in invalid:
|
||||
with self.subTest(value=value), self.assertRaises(
|
||||
scan_execution.ScanExecutionError,
|
||||
):
|
||||
scan_execution._normalize_remote_assignment_deadlines(
|
||||
value, reserved, scan_kwargs,
|
||||
)
|
||||
|
||||
def test_every_remote_scan_policy_field_changes_effective_identity(self):
|
||||
kwargs = {'timeout_sec': 60, 'trufflehog_config': '@package/detector_policy'}
|
||||
event = {'timeout_sec': 60.0, 'trufflehog_config': '@package/detector_policy'}
|
||||
limits = {'candidate_max_items': 20, 'candidate_max_bytes': 4096}
|
||||
policy = scan_policy()
|
||||
original, _ = scan_execution.remote_execution_identity(
|
||||
'github', kwargs, event, {}, limits, policy,
|
||||
)
|
||||
for name, value in policy.items():
|
||||
changed = dict(policy)
|
||||
if name == 'drop_detectors':
|
||||
changed[name] = ['other']
|
||||
elif isinstance(value, bool):
|
||||
changed[name] = not value
|
||||
else:
|
||||
changed[name] = value + 1
|
||||
with self.subTest(name=name):
|
||||
updated, _ = scan_execution.remote_execution_identity(
|
||||
'github', kwargs, event, {}, limits, changed,
|
||||
)
|
||||
self.assertNotEqual(updated, original)
|
||||
|
||||
def test_remote_scan_policy_overrides_inherited_process_settings(self):
|
||||
findings = [
|
||||
{'DetectorName': 'Generic', 'Raw': 'fixture'},
|
||||
{'DetectorName': 'GitHub', 'Raw': 'not-a-token', 'Verified': False},
|
||||
]
|
||||
with mock.patch.dict(os.environ, {
|
||||
'TRUFFLEHOG_STDOUT_MAX_MB': '999',
|
||||
'TRUFFLEHOG_STDERR_MAX_MB': '999',
|
||||
'TRUFFLEHOG_MAX_FINDINGS_PER_TARGET': '999999',
|
||||
}):
|
||||
with scanner.client_scan_execution_policy(scan_policy()):
|
||||
kept, dropped, _ = scanner.filter_dropped_detectors(findings)
|
||||
self.assertEqual(dropped, 1)
|
||||
kept, noisy = scanner.filter_noisy_findings(kept)
|
||||
self.assertEqual((kept, noisy), ([], 1))
|
||||
self.assertEqual(
|
||||
scanner.command_output_limits(), (2 * 1024 * 1024, 1024 * 1024),
|
||||
)
|
||||
self.assertEqual(scanner._trufflehog_diagnostic_limits(), {
|
||||
'lines': 100, 'line_chars': 1000, 'line_bytes': 1000,
|
||||
'errors': 10, 'warnings': 10, 'unclassified': 5,
|
||||
})
|
||||
|
||||
def test_queue_disposition_preserves_existing_retry_classes(self):
|
||||
policy = scan_execution.QueueDispositionPolicy(
|
||||
target_retry_max_attempts=3, target_retry_base_delay_sec=10,
|
||||
target_retry_max_delay_sec=100, target_timeout_retry_delay_sec=60,
|
||||
soft_skip_reasons=('no_ci_runs',),
|
||||
)
|
||||
cases = (
|
||||
({'errors': []}, 1, 'done', False),
|
||||
({'errors': ['bad'], 'retryable': False}, 1, 'failed', False),
|
||||
({'errors': ['timeout'], 'error_class': 'timeout'}, 1, 'deferred', False),
|
||||
({'errors': ['source'], 'source_failure': True, 'retryable': True}, 3,
|
||||
'deferred', True),
|
||||
({'errors': [], 'skipped': 'no_ci_runs'}, 1, 'deferred', True),
|
||||
)
|
||||
for result, attempts, status, reset in cases:
|
||||
with self.subTest(status=status, result=result):
|
||||
disposition = scan_execution.queue_disposition_for_result(
|
||||
result, 'github_actions', attempts, policy,
|
||||
)
|
||||
self.assertEqual(disposition['queue_status'], status)
|
||||
self.assertEqual(disposition['reset_attempts'], reset)
|
||||
|
||||
def test_planned_execution_reuses_scanner_and_canonical_bundle_staging(self):
|
||||
result = {'findings': [], 'errors': [], 'target': 'wrong', 'scan_type': 'wrong'}
|
||||
staged = object()
|
||||
with mock.patch.object(scan_execution, 'scan_target_result', return_value=result) as scan, \
|
||||
mock.patch.object(scan_execution, 'stage_result_bundle', return_value=staged) as stage:
|
||||
actual = scan_execution.execute_planned_result_in_scope(
|
||||
reservation(), '/private/bundles', {'timeout_sec': 60}, {'detectors': 'OpenAI'},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
scan_meta_defaults={'assignment': {'mode': 'remote'}},
|
||||
)
|
||||
self.assertIs(actual, staged)
|
||||
scan.assert_called_once_with(
|
||||
'https://example.invalid/repo', 'github', 'b' * 32, {'timeout_sec': 60.0},
|
||||
)
|
||||
args = stage.call_args.args
|
||||
self.assertEqual(args[0]['target'], reservation().target)
|
||||
self.assertEqual(args[0]['scan_type'], 'github')
|
||||
self.assertEqual(args[0]['scan_meta']['assignment']['mode'], 'remote')
|
||||
self.assertEqual(args[4]['queue_status'], 'done')
|
||||
|
||||
def test_runner_phase_callback_tracks_real_execution_boundaries_and_cleanup_counts(self):
|
||||
phases = []
|
||||
|
||||
def scan(*_args, **_kwargs):
|
||||
scanner.emit_client_scan_phase('scanning', {'records_seen': 3})
|
||||
scanner.emit_client_scan_phase('filtering', {'records_seen': 3})
|
||||
return {'findings': [], 'errors': []}
|
||||
|
||||
with mock.patch.object(
|
||||
scan_execution, 'scan_slot_scope', return_value=mock.MagicMock(),
|
||||
), mock.patch.object(
|
||||
scan_execution, 'scan_target_result', side_effect=scan,
|
||||
), mock.patch.object(
|
||||
scan_execution, 'cleanup_assignment_work_dir',
|
||||
return_value={'enumerated': 2, 'removed': 2, 'retained': 0},
|
||||
), mock.patch.object(
|
||||
scan_execution, 'stage_result_bundle', return_value=object(),
|
||||
):
|
||||
scan_execution.execute_planned_claim(
|
||||
reservation(), '/private/bundles', {'timeout_sec': 60}, {},
|
||||
scan_execution.QueueDispositionPolicy(), scan_policy(), attempts=1,
|
||||
phase_callback=lambda phase, progress=None: phases.append(
|
||||
(phase, dict(progress or {})),
|
||||
),
|
||||
)
|
||||
self.assertEqual([item[0] for item in phases], [
|
||||
'waiting_permit', 'scanning', 'filtering',
|
||||
'cleaning', 'cleaning', 'bundling',
|
||||
])
|
||||
self.assertEqual(phases[-2][1]['removed'], 2)
|
||||
self.assertEqual(phases[1][1], {'records_seen': 3})
|
||||
self.assertEqual(phases[0][1], {'boundary': 'scan_slot_scope'})
|
||||
|
||||
def test_provider_callbacks_expose_only_observable_or_integrated_boundaries(self):
|
||||
def streamed_output():
|
||||
output = mock.MagicMock()
|
||||
output.returncode = 0
|
||||
output.stderr_lines.return_value = iter(())
|
||||
output.stdout_lines.return_value = iter(())
|
||||
context = mock.MagicMock()
|
||||
context.__enter__.return_value = output
|
||||
context.__exit__.return_value = False
|
||||
return context
|
||||
|
||||
docker_target = 'docker.io/library/alpine@sha256:' + ('a' * 64)
|
||||
docker_phases = []
|
||||
manifest = scanner._client_scan_manifest.set({'executables': {}})
|
||||
direct = scanner._client_remote_execution_kind.set('docker_direct_v1')
|
||||
try:
|
||||
with scanner.client_scan_phase_events(
|
||||
lambda phase, progress=None: docker_phases.append(
|
||||
(phase, dict(progress or {})),
|
||||
)
|
||||
), mock.patch.object(
|
||||
scanner, 'get_trufflehog_cmd', return_value='trufflehog',
|
||||
), mock.patch.object(
|
||||
scanner, 'run_command_streamed', return_value=streamed_output(),
|
||||
):
|
||||
scanner.scan_docker_image(docker_target, timeout_sec=60)
|
||||
finally:
|
||||
scanner._client_remote_execution_kind.reset(direct)
|
||||
scanner._client_scan_manifest.reset(manifest)
|
||||
self.assertEqual(docker_phases[0], (
|
||||
'scanning', {'integrated_operation': 'docker_pull_and_scan'},
|
||||
))
|
||||
self.assertNotIn('downloading', [item[0] for item in docker_phases])
|
||||
|
||||
hf_phases = []
|
||||
direct = scanner._client_remote_execution_kind.set('huggingface_space_v1')
|
||||
try:
|
||||
with scanner.client_scan_phase_events(
|
||||
lambda phase, progress=None: hf_phases.append(
|
||||
(phase, dict(progress or {})),
|
||||
)
|
||||
), mock.patch.object(
|
||||
scanner, 'get_trufflehog_cmd', return_value='trufflehog',
|
||||
), mock.patch.object(
|
||||
scanner, 'run_command_streamed', return_value=streamed_output(),
|
||||
):
|
||||
scanner.scan_huggingface_space('Example/Public-Space', timeout_sec=60)
|
||||
finally:
|
||||
scanner._client_remote_execution_kind.reset(direct)
|
||||
self.assertEqual(hf_phases[0], (
|
||||
'scanning', {'integrated_operation': 'huggingface_clone_and_scan'},
|
||||
))
|
||||
self.assertNotIn('cloning', [item[0] for item in hf_phases])
|
||||
|
||||
resolving = []
|
||||
with scanner.client_scan_phase_events(
|
||||
lambda phase, progress=None: resolving.append((phase, dict(progress or {}))),
|
||||
), mock.patch.object(
|
||||
scanner, 'recent_commit_boundary',
|
||||
return_value={'skip': True, 'reason': 'fixture'},
|
||||
):
|
||||
scanner.scan_git_repo('https://example.invalid/repo', timeout_sec=60)
|
||||
self.assertEqual(resolving[0][0], 'resolving')
|
||||
self.assertEqual(resolving[0][1]['operation'], 'recent_commit_boundary')
|
||||
|
||||
docker_resolution = []
|
||||
with scanner.client_scan_phase_events(
|
||||
lambda phase, progress=None: docker_resolution.append(
|
||||
(phase, dict(progress or {}), time.monotonic()),
|
||||
)
|
||||
), mock.patch.object(
|
||||
scanner, 'resolve_docker_content_manifest',
|
||||
side_effect=scanner.DockerContentScanError(
|
||||
'fixture_resolution', 'fixture resolution stopped',
|
||||
),
|
||||
):
|
||||
scanner._recover_docker_image_contents(
|
||||
docker_target, time.monotonic() + 60, None, None, 0,
|
||||
None, None, False, None, anonymous_public_client=True,
|
||||
)
|
||||
self.assertEqual(docker_resolution[0][0], 'resolving')
|
||||
self.assertEqual(
|
||||
docker_resolution[0][1]['operation'],
|
||||
'docker_manifest_resolution_recovery',
|
||||
)
|
||||
self.assertEqual(
|
||||
validate_phase_transition(
|
||||
WorkerPhase.SCANNING, WorkerPhase(docker_resolution[0][0]),
|
||||
),
|
||||
WorkerPhase.RESOLVING,
|
||||
)
|
||||
|
||||
def test_native_git_checkout_recovery_emits_real_cloning_boundary(self):
|
||||
plan = {**noop_git_plan(), 'mode': 'baseline'}
|
||||
phases = []
|
||||
outputs = []
|
||||
for returncode in (1, 0, 0):
|
||||
output = mock.MagicMock()
|
||||
output.returncode = returncode
|
||||
output.stderr_lines.return_value = iter(())
|
||||
output.stdout_lines.return_value = iter(())
|
||||
context = mock.MagicMock()
|
||||
context.__enter__.return_value = output
|
||||
context.__exit__.return_value = False
|
||||
outputs.append(context)
|
||||
diagnostics = [0]
|
||||
|
||||
def apply(result, *_args, **_kwargs):
|
||||
diagnostics[0] += 1
|
||||
if diagnostics[0] == 1:
|
||||
result['errors'] = ['checkout failed']
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary, \
|
||||
scanner.client_scan_phase_events(
|
||||
lambda phase, progress=None: phases.append(
|
||||
(phase, dict(progress or {}), time.monotonic()),
|
||||
),
|
||||
), mock.patch.object(
|
||||
scanner, 'get_trufflehog_cmd', return_value='trufflehog',
|
||||
), mock.patch.object(
|
||||
scanner, 'get_git_cmd', return_value='git',
|
||||
), mock.patch.object(
|
||||
scanner, 'run_command_streamed', side_effect=outputs,
|
||||
), mock.patch.object(
|
||||
scanner, 'apply_trufflehog_diagnostics', side_effect=apply,
|
||||
), mock.patch.object(
|
||||
scanner, 'append_trufflehog_findings',
|
||||
), mock.patch.object(
|
||||
scanner, 'git_checkout_recovery_allowed', return_value=True,
|
||||
), mock.patch.object(
|
||||
scanner, 'create_command_work_dir', return_value=temporary,
|
||||
), mock.patch.object(
|
||||
scanner, 'cleanup_command_work_dir',
|
||||
):
|
||||
scanner.scan_exact_git_plan(
|
||||
plan['repo_url'], plan, 'f' * 64, 60,
|
||||
None, None, False, None, None, False,
|
||||
)
|
||||
names = [item[0] for item in phases]
|
||||
self.assertEqual(names[:3], ['scanning', 'cloning', 'scanning'])
|
||||
for previous, current in zip(names, names[1:]):
|
||||
validate_phase_transition(WorkerPhase(previous), WorkerPhase(current))
|
||||
measured = [
|
||||
later[2] - earlier[2]
|
||||
for earlier, later in zip(phases, phases[1:])
|
||||
]
|
||||
self.assertTrue(measured)
|
||||
self.assertTrue(all(duration >= 0 for duration in measured))
|
||||
|
||||
def test_docker_direct_claim_executes_bound_target_and_stages_bundle(self):
|
||||
target = 'docker.io/library/alpine@sha256:' + ('a' * 64)
|
||||
claim = BundleReservation(
|
||||
reservation_id=17, reservation_token='docker-request-token',
|
||||
bundle_id='d' * 32, scan_event_id='e' * 32, queue_id=23,
|
||||
claim_lease_token='docker-lease-token', declared_bytes=1024 * 1024,
|
||||
ready_path='ready/dd/' + ('d' * 32) + '.trb',
|
||||
source='dockerhub', platform='docker', query='fixture',
|
||||
target=target, normalized_target=scanner.normalize_target(target, 'docker'),
|
||||
)
|
||||
validated = {
|
||||
'reservation': claim,
|
||||
'planning_kind': 'docker_direct_v1',
|
||||
'execution_target': target,
|
||||
}
|
||||
options = {'timeout_sec': 60.0}
|
||||
manifest_token = scanner._client_scan_manifest.set({'executables': {}})
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
bundle_root = os.path.join(temp_dir, 'bundles')
|
||||
scanner.ensure_private_directory(bundle_root, reject_reparse=True)
|
||||
with mock.patch.object(
|
||||
scan_execution, 'scan_slot_scope', return_value=mock.MagicMock(),
|
||||
), mock.patch.object(
|
||||
scanner, 'scan_docker_image',
|
||||
return_value={'findings': [], 'errors': [], 'scan_meta': {}},
|
||||
) as docker_scan, mock.patch.object(
|
||||
scanner.docker_token_manager, 'get_next_config',
|
||||
side_effect=AssertionError('direct Docker cannot select server credentials'),
|
||||
):
|
||||
staged = scan_execution.execute_protocol2_remote_claim(
|
||||
validated, bundle_root, options, options,
|
||||
scan_execution.QueueDispositionPolicy(), scan_policy(), attempts=1,
|
||||
)
|
||||
|
||||
docker_scan.assert_called_once()
|
||||
self.assertEqual(docker_scan.call_args.args[0], target)
|
||||
self.assertIsNone(docker_scan.call_args.kwargs['config_dir'])
|
||||
reader = ResultBundleReader(
|
||||
os.path.join(bundle_root, staged.relative_path),
|
||||
)
|
||||
metadata = reader.metadata()
|
||||
self.assertEqual(metadata['target'], target)
|
||||
self.assertEqual(metadata['scan_type'], 'docker')
|
||||
encoded = json.dumps(metadata, sort_keys=True)
|
||||
for forbidden in (
|
||||
'docker_layer_work', 'docker_registry_auth',
|
||||
'git_scan_plan', 'git_scan_execution',
|
||||
):
|
||||
self.assertNotIn(forbidden, encoded)
|
||||
finally:
|
||||
scanner._client_scan_manifest.reset(manifest_token)
|
||||
self.assertIsNone(scanner._client_remote_execution_kind.get())
|
||||
|
||||
def test_huggingface_direct_claim_is_tokenless_and_stages_bundle(self):
|
||||
target = 'ExampleOrg/Public-Space'
|
||||
claim = BundleReservation(
|
||||
reservation_id=18, reservation_token='hf-request-token',
|
||||
bundle_id='f' * 32, scan_event_id='1' * 32, queue_id=24,
|
||||
claim_lease_token='hf-lease-token', declared_bytes=1024 * 1024,
|
||||
ready_path='ready/ff/' + ('f' * 32) + '.trb',
|
||||
source='huggingface', platform='huggingface', query='fixture',
|
||||
target=target,
|
||||
normalized_target=scanner.normalize_target(target, 'huggingface'),
|
||||
)
|
||||
validated = {
|
||||
'reservation': claim,
|
||||
'planning_kind': 'huggingface_space_v1',
|
||||
'execution_target': target,
|
||||
}
|
||||
options = {'timeout_sec': 60.0}
|
||||
manifest_token = scanner._client_scan_manifest.set({'executables': {}})
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
bundle_root = os.path.join(temp_dir, 'bundles')
|
||||
scanner.ensure_private_directory(bundle_root, reject_reparse=True)
|
||||
with mock.patch.object(
|
||||
scan_execution, 'scan_slot_scope', return_value=mock.MagicMock(),
|
||||
), mock.patch.object(
|
||||
scanner, 'scan_huggingface_space',
|
||||
return_value={'findings': [], 'errors': [], 'scan_meta': {}},
|
||||
) as hf_scan:
|
||||
staged = scan_execution.execute_protocol2_remote_claim(
|
||||
validated, bundle_root, options, options,
|
||||
scan_execution.QueueDispositionPolicy(), scan_policy(), attempts=1,
|
||||
)
|
||||
|
||||
hf_scan.assert_called_once()
|
||||
self.assertEqual(hf_scan.call_args.args[0], target)
|
||||
self.assertIsNone(hf_scan.call_args.kwargs.get('token'))
|
||||
reader = ResultBundleReader(
|
||||
os.path.join(bundle_root, staged.relative_path),
|
||||
)
|
||||
metadata = reader.metadata()
|
||||
self.assertEqual(metadata['target'], target)
|
||||
self.assertEqual(metadata['scan_type'], 'huggingface')
|
||||
encoded = json.dumps(metadata, sort_keys=True)
|
||||
for forbidden in (
|
||||
'huggingface_token', 'hf_token', 'authorization',
|
||||
'docker_layer_work', 'docker_registry_auth',
|
||||
'git_scan_plan', 'git_scan_execution',
|
||||
):
|
||||
self.assertNotIn(forbidden, encoded.lower())
|
||||
finally:
|
||||
scanner._client_scan_manifest.reset(manifest_token)
|
||||
self.assertIsNone(scanner._client_remote_execution_kind.get())
|
||||
|
||||
def test_exact_git_noop_local_and_db_free_bundles_have_identical_coverage(self):
|
||||
plan = noop_git_plan()
|
||||
claim = source_reservation('github', plan['repo_url'])
|
||||
kwargs = {'timeout_sec': 60, 'git_plan': plan}
|
||||
queue_policy = scan_execution.QueueDispositionPolicy()
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
local_root = os.path.join(temp_dir, 'local')
|
||||
remote_root = os.path.join(temp_dir, 'remote')
|
||||
scanner.ensure_private_directory(local_root, reject_reparse=True)
|
||||
scanner.ensure_private_directory(remote_root, reject_reparse=True)
|
||||
with mock.patch.object(
|
||||
scanner, 'run_command_streamed',
|
||||
side_effect=AssertionError('noop Git plan must not launch a scanner'),
|
||||
):
|
||||
local_result = scanner.scan_target_result(
|
||||
claim.target, claim.platform, claim.scan_event_id, kwargs,
|
||||
)
|
||||
local = scan_execution.stage_scan_result_in_scope(
|
||||
local_result, claim, local_root, {}, queue_policy, attempts=1,
|
||||
)
|
||||
remote = scan_execution.execute_planned_result_in_scope(
|
||||
claim, remote_root, kwargs, {}, queue_policy, attempts=1,
|
||||
)
|
||||
|
||||
local_metadata = ResultBundleReader(
|
||||
os.path.join(local_root, local.relative_path),
|
||||
).metadata()
|
||||
remote_metadata = ResultBundleReader(
|
||||
os.path.join(remote_root, remote.relative_path),
|
||||
).metadata()
|
||||
|
||||
self.assertEqual(local.queue_status, remote.queue_status)
|
||||
for name in ('git_scan_plan', 'git_scan_execution'):
|
||||
self.assertEqual(local_metadata[name], remote_metadata[name])
|
||||
self.assertEqual(
|
||||
local_metadata['scan_meta']['exact_git_scope'],
|
||||
remote_metadata['scan_meta']['exact_git_scope'],
|
||||
)
|
||||
encoded_plan = scanner_db.canonical_git_scan_plan_bytes(plan)
|
||||
stored = {
|
||||
'git_scan_plan_json': encoded_plan.decode('ascii'),
|
||||
'git_scan_plan_sha256': hashlib.sha256(encoded_plan).hexdigest(),
|
||||
}
|
||||
for metadata in (local_metadata, remote_metadata):
|
||||
self.assertEqual(
|
||||
scanner_db.matching_git_coverage(
|
||||
stored, metadata, metadata['queue_status'], metadata['error_count'],
|
||||
),
|
||||
(True, plan['ref'], plan['head_sha']),
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
configured_trufflehog() and shutil.which('git'),
|
||||
'configured TruffleHog and Git executables are required',
|
||||
)
|
||||
def test_native_exact_git_local_and_db_free_bundles_are_equivalent(self):
|
||||
executable = configured_trufflehog()
|
||||
git_executable = shutil.which('git')
|
||||
xai_before = 'xai-' + hashlib.sha512(b'xai-before-parity').hexdigest()[:48]
|
||||
xai_after = 'xai-' + hashlib.sha512(b'xai-after-parity').hexdigest()[:48]
|
||||
zai_before = 'zai-' + base64.urlsafe_b64encode(
|
||||
hashlib.sha512(b'zai-before-parity').digest()
|
||||
).decode('ascii')[:48]
|
||||
zai_after = 'zai-' + base64.urlsafe_b64encode(
|
||||
hashlib.sha512(b'zai-after-parity').digest()
|
||||
).decode('ascii')[:48]
|
||||
fixtures = (
|
||||
('xai-before.env', f'XAI_API_KEY={xai_before}\n'),
|
||||
('xai-after.env', f'{xai_after} api.x.ai\n'),
|
||||
('zai-before.env', f'ZAI_API_KEY={zai_before}\n'),
|
||||
('zai-after.env', f'{zai_after} api.z.ai\n'),
|
||||
)
|
||||
policy = str(APP / 'trufflehog-custom-detectors.yaml')
|
||||
repo_url = 'https://gitlab.com/Fixture/Parity.git'
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
source = Path(temp_dir) / 'source'
|
||||
source.mkdir()
|
||||
git_env = os.environ.copy()
|
||||
git_env.update({
|
||||
'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': os.devnull,
|
||||
'GIT_TERMINAL_PROMPT': '0', 'GIT_ALLOW_PROTOCOL': 'file',
|
||||
})
|
||||
|
||||
def git(*args):
|
||||
completed = subprocess.run(
|
||||
[git_executable, '-c', 'user.name=Parity Fixture', '-c',
|
||||
'user.email=parity@example.invalid', '-c', 'commit.gpgsign=false',
|
||||
*args],
|
||||
cwd=source, env=git_env, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
check=False, timeout=30,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode, 0,
|
||||
completed.stderr.decode('utf-8', errors='replace'),
|
||||
)
|
||||
return completed.stdout.decode('ascii').strip()
|
||||
|
||||
git('init', '--quiet', '--initial-branch=main', '--template=')
|
||||
for name, content in fixtures:
|
||||
(source / name).write_text(content, encoding='ascii', newline='\n')
|
||||
git('add', '--', name)
|
||||
git('commit', '--quiet', '-m', name)
|
||||
head = git('rev-parse', 'HEAD')
|
||||
plan = {
|
||||
'version': 1, 'provider': 'gitlab', 'repo_url': repo_url,
|
||||
'repo_path': 'Fixture/Parity', 'branch': 'main',
|
||||
'ref': 'refs/heads/main', 'head_sha': head, 'base_sha': None,
|
||||
'mode': 'baseline', 'baseline_depth': len(fixtures),
|
||||
'ref_source': 'explicit',
|
||||
}
|
||||
claim = source_reservation('gitlab', repo_url)
|
||||
kwargs = {
|
||||
'timeout_sec': 60, 'git_plan': plan, 'no_verification': True,
|
||||
'trufflehog_config': policy,
|
||||
'external_trufflehog_lifecycle': True,
|
||||
}
|
||||
local_root = os.path.join(temp_dir, 'local')
|
||||
remote_root = os.path.join(temp_dir, 'remote')
|
||||
scanner.ensure_private_directory(local_root, reject_reparse=True)
|
||||
scanner.ensure_private_directory(remote_root, reject_reparse=True)
|
||||
execution_env = dict(git_env)
|
||||
execution_env.update({
|
||||
'GIT_CONFIG_COUNT': '1',
|
||||
'GIT_CONFIG_KEY_0': f'url.{source.as_uri()}.insteadOf',
|
||||
'GIT_CONFIG_VALUE_0': repo_url,
|
||||
})
|
||||
with mock.patch.dict(os.environ, execution_env, clear=True), \
|
||||
mock.patch.object(
|
||||
scanner, 'get_trufflehog_cmd', return_value=str(executable),
|
||||
), mock.patch.object(
|
||||
scanner, 'run_command_streamed',
|
||||
side_effect=native_streamed_command,
|
||||
):
|
||||
local_result = scanner.scan_target_result(
|
||||
claim.target, claim.platform, claim.scan_event_id, kwargs,
|
||||
)
|
||||
local = scan_execution.stage_scan_result_in_scope(
|
||||
local_result, claim, local_root, {},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
)
|
||||
remote = scan_execution.execute_planned_result_in_scope(
|
||||
claim, remote_root, kwargs, {},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
)
|
||||
|
||||
local_evidence = normalized_bundle_evidence(
|
||||
os.path.join(local_root, local.relative_path),
|
||||
)
|
||||
remote_evidence = normalized_bundle_evidence(
|
||||
os.path.join(remote_root, remote.relative_path),
|
||||
)
|
||||
|
||||
self.assertEqual(local_evidence, remote_evidence)
|
||||
self.assertEqual(local.queue_status, 'done')
|
||||
self.assertEqual(local.queue_status, remote.queue_status)
|
||||
findings = local_evidence['findings']
|
||||
self.assertEqual(
|
||||
{item.get('DetectorName') for item in findings}, {'Xai', 'ZaiGLM'},
|
||||
)
|
||||
self.assertEqual(
|
||||
{item.get('ExtraData', {}).get('name') for item in findings},
|
||||
{'Xai', 'XaiContextAfter', 'ZaiGLM', 'ZaiGLMContextAfter'},
|
||||
)
|
||||
self.assertEqual(
|
||||
{item.get('service') for item in local_evidence['candidates']},
|
||||
{'xai', 'zai'},
|
||||
)
|
||||
encoded_plan = scanner_db.canonical_git_scan_plan_bytes(plan)
|
||||
stored = {
|
||||
'git_scan_plan_json': encoded_plan.decode('ascii'),
|
||||
'git_scan_plan_sha256': hashlib.sha256(encoded_plan).hexdigest(),
|
||||
}
|
||||
metadata = local_evidence['metadata']
|
||||
self.assertEqual(
|
||||
scanner_db.matching_git_coverage(
|
||||
stored, metadata, metadata['queue_status'], metadata['error_count'],
|
||||
),
|
||||
(True, plan['ref'], plan['head_sha']),
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
configured_trufflehog(), 'configured TruffleHog executable is required',
|
||||
)
|
||||
def test_native_postman_local_and_db_free_bundles_are_equivalent(self):
|
||||
executable = configured_trufflehog()
|
||||
gemini_key = 'AIza' + ('A' * 35)
|
||||
azure_key = 'a' * 32
|
||||
endpoint = 'fixture.openai.azure.com'
|
||||
content = json.dumps({
|
||||
'values': [
|
||||
{'key': 'GEMINI_API_KEY', 'value': gemini_key},
|
||||
{'key': 'AZURE_OPENAI_KEY', 'value': azure_key},
|
||||
{'url': f'https://{endpoint}/openai/deployments/demo'},
|
||||
],
|
||||
}, sort_keys=True).encode('utf-8')
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
cache_root = os.path.join(temp_dir, 'cache')
|
||||
work_root = os.path.join(temp_dir, 'work')
|
||||
local_root = os.path.join(temp_dir, 'local')
|
||||
remote_root = os.path.join(temp_dir, 'remote')
|
||||
for path in (cache_root, work_root, local_root, remote_root):
|
||||
scanner.ensure_private_directory(path, reject_reparse=True)
|
||||
with mock.patch.multiple(
|
||||
scanner.scan_config, postman_cache_dir=cache_root, runtime_dir=temp_dir,
|
||||
postman_cache_min_free_bytes=0, min_free_gb=0,
|
||||
):
|
||||
cache_path, digest, size = scanner.write_postman_cache(
|
||||
content, kind='collection', cache_dir=cache_root,
|
||||
)
|
||||
postman = {
|
||||
'source': 'fixture', 'repo': 'Owner/Repo',
|
||||
'path': 'collection.json', 'kind': 'collection',
|
||||
'cache_path': cache_path, 'sha256': digest, 'size': size,
|
||||
}
|
||||
target = json.dumps(postman, sort_keys=True)
|
||||
claim = source_reservation('postman', target)
|
||||
kwargs = {
|
||||
'timeout_sec': 60, 'max_artifact_size_mb': 1,
|
||||
'no_verification': True,
|
||||
}
|
||||
with mock.patch.object(
|
||||
scanner, 'get_work_dir', return_value=work_root,
|
||||
), mock.patch.object(
|
||||
scanner, 'require_scanner_runtime_initialized', return_value=None,
|
||||
), mock.patch.object(
|
||||
scanner, 'get_trufflehog_cmd', return_value=str(executable),
|
||||
), mock.patch.object(
|
||||
scanner, 'run_command_streamed', side_effect=native_streamed_command,
|
||||
):
|
||||
result = scanner.scan_target_result(
|
||||
target, 'postman', claim.scan_event_id, kwargs,
|
||||
)
|
||||
self.assertTrue(
|
||||
result.get('structured_keycheck_pending'),
|
||||
result.get('warnings') or result.get('errors'),
|
||||
)
|
||||
local = scan_execution.stage_scan_result_in_scope(
|
||||
result, claim, local_root, {},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
)
|
||||
remote = scan_execution.execute_planned_result_in_scope(
|
||||
claim, remote_root, kwargs, {},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
)
|
||||
|
||||
local_evidence = normalized_bundle_evidence(
|
||||
os.path.join(local_root, local.relative_path),
|
||||
)
|
||||
remote_evidence = normalized_bundle_evidence(
|
||||
os.path.join(remote_root, remote.relative_path),
|
||||
)
|
||||
|
||||
self.assertEqual(local_evidence, remote_evidence)
|
||||
self.assertEqual(local.queue_status, remote.queue_status)
|
||||
self.assertEqual(local.queue_status, 'done')
|
||||
candidates = local_evidence['candidates']
|
||||
|
||||
self.assertEqual({item['service'] for item in candidates}, {'gemini', 'azure'})
|
||||
self.assertTrue(all(
|
||||
item['candidate_kind'] == 'structured_postman' for item in candidates
|
||||
))
|
||||
origin = f'fixture:Owner/Repo:collection.json:{digest}'
|
||||
expected = {
|
||||
'gemini': (gemini_key, gemini_key, 'GoogleAIStudio'),
|
||||
'azure': (
|
||||
f'{endpoint}:{azure_key}', f'{azure_key}:{endpoint}', 'AzureOpenAI',
|
||||
),
|
||||
}
|
||||
for item in candidates:
|
||||
probe, raw, detector = expected[item['service']]
|
||||
self.assertEqual(item['metadata']['origin'], origin)
|
||||
self.assertEqual(item['metadata']['detector_name'], detector)
|
||||
self.assertTrue(item['metadata']['structured_origin'].startswith(f'{origin}:$'))
|
||||
self.assertEqual(item['attribution']['origin'], item['metadata']['structured_origin'])
|
||||
self.assertEqual(
|
||||
item['provider_key_hash'], hashlib.sha256(probe.encode('utf-8')).hexdigest(),
|
||||
)
|
||||
self.assertEqual(
|
||||
item['secret_hash'], hashlib.sha256(raw.encode('utf-8')).hexdigest(),
|
||||
)
|
||||
self.assertEqual(item['credential_hash'], hashlib.sha256(
|
||||
f"truf-credential-v2|{item['service']}|{probe}".encode('utf-8')
|
||||
).hexdigest())
|
||||
encoded = json.dumps(item, sort_keys=True)
|
||||
for forbidden in ('status', 'status_group', 'checked_at', 'result_source'):
|
||||
self.assertNotIn(f'"{forbidden}"', encoded)
|
||||
self.assertEqual(len(candidates), 2)
|
||||
|
||||
def test_structured_postman_candidate_staging_rejects_size_or_hash_drift(self):
|
||||
content = json.dumps({'token': 'AIza' + ('A' * 35)}).encode('utf-8')
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
cache_root = os.path.join(temp_dir, 'cache')
|
||||
scanner.ensure_private_directory(cache_root, reject_reparse=True)
|
||||
with mock.patch.multiple(
|
||||
scanner.scan_config, postman_cache_dir=cache_root, runtime_dir=temp_dir,
|
||||
postman_cache_min_free_bytes=0,
|
||||
):
|
||||
cache_path, digest, size = scanner.write_postman_cache(
|
||||
content, cache_dir=cache_root,
|
||||
)
|
||||
for index, drift in enumerate(('size', 'hash'), start=1):
|
||||
with self.subTest(drift=drift):
|
||||
bundle_root = os.path.join(temp_dir, f'bundles-{index}')
|
||||
scanner.ensure_private_directory(bundle_root, reject_reparse=True)
|
||||
declared_digest = 'c' * 64 if drift == 'hash' else digest
|
||||
target = f'postman:sha256:{declared_digest}'
|
||||
claim = source_reservation(
|
||||
'postman', target, bundle_id=str(index) * 32,
|
||||
reservation_id=index,
|
||||
)
|
||||
result = {
|
||||
'scan_event_id': claim.scan_event_id, 'target': target,
|
||||
'scan_type': 'postman', 'findings': [], 'errors': [],
|
||||
'structured_keycheck_pending': True,
|
||||
'postman': {
|
||||
'source': 'fixture', 'cache_path': cache_path,
|
||||
'sha256': declared_digest, 'size': size,
|
||||
},
|
||||
'bytes': size + 1 if drift == 'size' else size,
|
||||
'postman_max_artifact_size_mb': 1,
|
||||
}
|
||||
staged = scan_execution.stage_scan_result_in_scope(
|
||||
result, claim, bundle_root, {},
|
||||
scan_execution.QueueDispositionPolicy(), attempts=1,
|
||||
)
|
||||
reader = ResultBundleReader(
|
||||
os.path.join(bundle_root, staged.relative_path),
|
||||
)
|
||||
self.assertEqual(list(reader.iter_candidates()), [])
|
||||
metadata = reader.metadata()
|
||||
self.assertTrue(metadata['degraded'])
|
||||
self.assertTrue(any(
|
||||
'structured keycheck extraction failed' in warning.lower()
|
||||
for warning in metadata['warnings']
|
||||
))
|
||||
|
||||
def test_client_manifest_authorizes_only_manifested_tools_and_policy(self):
|
||||
manifest = {
|
||||
'executables': {
|
||||
'trufflehog': {'path': os.path.abspath('trufflehog'), 'sha256': 'a' * 64},
|
||||
'git': {'path': os.path.abspath('git'), 'sha256': 'b' * 64},
|
||||
},
|
||||
'assets': {},
|
||||
}
|
||||
with mock.patch.object(scanner, 'verify_code_manifest', return_value=manifest), \
|
||||
mock.patch.object(scanner, 'resolve_manifest_executable',
|
||||
return_value=manifest['executables']['trufflehog']['path']):
|
||||
with scanner.client_scan_launch_authority({}, expected_sha256='c' * 64):
|
||||
self.assertEqual(scanner.get_git_cmd(), manifest['executables']['git']['path'])
|
||||
metadata = scanner.require_trufflehog_launch_authority(
|
||||
[manifest['executables']['trufflehog']['path'], 'filesystem', '/fixture'],
|
||||
)
|
||||
self.assertEqual(metadata['authority'], 'remote-worker')
|
||||
with mock.patch.object(scanner.os.path, 'isabs', return_value=True):
|
||||
metadata = scanner.require_git_clone_launch_authority([
|
||||
manifest['executables']['git']['path'], 'clone', '--no-checkout',
|
||||
'--no-recurse-submodules', '--', 'https://example.invalid/repo',
|
||||
os.path.abspath('checkout'),
|
||||
])
|
||||
self.assertEqual(metadata['authority'], 'remote-worker')
|
||||
with self.assertRaises(LifecycleAuthorityError):
|
||||
scanner.require_trufflehog_launch_authority(
|
||||
[manifest['executables']['trufflehog']['path'], 'filesystem', '/fixture'],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user