sashatrask/truf-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
170dd941b95098efd6d3c06452f866c0e2214833
truf-server/tests/test_validated_high_scanner_fixes.py
T
sashatrask 170dd941b9 Initial server source import
2026-09-30 20:30:56 +03:00

3007 lines
144 KiB
Python
Raw Blame History

import copy
from datetime import datetime, timedelta, timezone
import json
import os
from pathlib import Path
import re
import sqlite3
import subprocess
import sys
import tempfile
import threading
import time
from types import SimpleNamespace
import unittest
import uuid
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
APP_DIR = ROOT / 'app'
sys.path.insert(0, str(APP_DIR))
import scanner
import console_runner
import keycheck_runner
from keycheckers import keycheck_common
from keycheckers.anthropic import anthropicKeycheck as anthropic
from keycheckers.deepseek import deepseekKeycheck as deepseek
from keycheckers.qwen import qwenKeycheck as qwen
from result_spool import ResultSpool
from scanner_db import ScannerDB, normalize_target
from target_identity import parse_docker_target, serialize_docker_tag_target
def clear_scan_fatal():
with scanner._scan_slot_fatal_lock:
scanner._scan_slot_fatal_detail = None
scanner._scan_slot_fatal_event.clear()
class DeepSeekRecheckFlagTests(unittest.TestCase):
def test_retry_valid_is_supported_end_to_end(self):
self.assertIn('retry_valid', keycheck_runner.SERVICE_CAPABILITIES['deepseek']['flags'])
with mock.patch.object(sys, 'argv', ['deepseekKeycheck.py', '--retry-valid']):
args = deepseek.parse_args()
self.assertTrue(args.retry_valid)
class QwenRoutingPersistenceTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
def setUp(self):
clear_scan_fatal()
self.routing_db_temp = tempfile.TemporaryDirectory()
self.routing_db_path = os.path.join(self.routing_db_temp.name, 'scanner.db')
self.database_environment = mock.patch.dict(os.environ, {
'SCANNER_DB_URL': '', 'DATABASE_URL': '', 'TRUF_MANAGED_POSTGRES_DSN': '',
'KEYCHECK_DB_URL': '', 'KEYCHECK_DB_PATH': self.routing_db_path,
})
self.database_environment.start()
ScannerDB(db_path=self.routing_db_path, db_url='').close()
keycheck_common.clear_provider_routing_evidence_cache()
def tearDown(self):
keycheck_common.clear_provider_routing_evidence_cache()
clear_scan_fatal()
self.database_environment.stop()
self.routing_db_temp.cleanup()
def persist_finding_event(self, finding, target, clear_cache=True):
result = {'findings': [finding], 'errors': [], 'scan_type': 'fixture'}
db = ScannerDB(db_path=self.routing_db_path, initialize=False, db_url='')
try:
run_id = db.start_run('test', ['test'])
cycle_id = db.start_source_cycle(
run_id, 'fixture', 'fixture', 'search', target, 1, 1, None, {}, {},
)
db.record_target_result(run_id, cycle_id, 'fixture', target, target, result)
finally:
db.close()
if clear_cache:
keycheck_common.clear_provider_routing_evidence_cache()
return finding
def persist_provider_event(self, key, provider, target, clear_cache=True):
detector = {
'qwen': 'QwenDashScope',
'deepseek': 'DeepSeekApiKey',
'kimi': 'KimiMoonshot',
}[provider]
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': detector},
'Raw': key,
}
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
return self.persist_finding_event(finding, target, clear_cache=clear_cache)
def assert_provider_request_counts(self, qwen_findings, deepseek_findings, qwen_count, deepseek_count):
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': {'message': 'unauthorized'}}
qwen_items = [
{'raw': finding.get('Raw', ''), 'source': 'roundtrip', 'finding': finding}
for finding in qwen_findings
]
deepseek_items = [
{'raw': finding.get('Raw', ''), 'source': 'roundtrip', 'finding': finding}
for finding in deepseek_findings
]
with mock.patch.object(qwen, 'iter_findings', return_value=qwen_items), \
mock.patch.object(qwen.requests, 'get', return_value=response) as qwen_request, \
mock.patch.object(qwen.requests, 'post') as qwen_post:
candidates = list(qwen.iter_candidate_keys('unused.jsonl', []))
for candidate, _, _ in candidates:
qwen.check_key(candidate, [qwen.DEFAULT_BASE_URLS[0]], False, None, 1)
self.assertEqual(qwen_request.call_count, qwen_count)
qwen_post.assert_not_called()
with mock.patch.object(deepseek, 'iter_findings', return_value=deepseek_items), \
mock.patch.object(deepseek.requests, 'get', return_value=response) as deepseek_request:
candidates = list(deepseek.extract_candidates('unused.jsonl', []))
for candidate, _, _ in candidates:
deepseek.check_key(candidate, None, 1)
self.assertEqual(deepseek_request.call_count, deepseek_count)
def test_separate_events_without_context_route_weak_detector_to_resolver(self):
key = 'sk-' + ('1' * 32)
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
deepseek_finding = {'DetectorName': 'DeepSeek', 'Raw': key}
scanner.strip_nearby_context_for_persistence({'findings': [qwen_finding]})
scanner.strip_nearby_context_for_persistence({'findings': [qwen_finding]})
scanner.strip_nearby_context_for_persistence({'findings': [deepseek_finding]})
scanner.strip_nearby_context_for_persistence({'findings': [deepseek_finding]})
self.assertEqual(qwen_finding['ScannerContext']['provider_hint'], 'qwen')
self.assertEqual(
deepseek_finding['ScannerContext']['provider_hint'],
'ambiguous_generic_sk',
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 1, 0)
def test_explicit_qwen_context_routes_only_to_qwen(self):
key = 'sk-' + ('2' * 32)
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'ScannerContext': {'nearby': 'DASHSCOPE_API_KEY is configured here'},
}
deepseek_finding = {
'DetectorName': 'DeepSeek',
'Raw': key,
'ScannerContext': {'nearby': 'DASHSCOPE_API_KEY is configured here'},
}
for finding in (qwen_finding, deepseek_finding):
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
self.assertEqual(finding['ScannerContext']['provider_hint'], 'qwen')
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 1, 0)
def test_explicit_deepseek_metadata_routes_only_to_deepseek(self):
key = 'sk-' + ('3' * 32)
metadata = {'Data': {'Filesystem': {'file': 'config/deepseek.env'}}}
qwen_finding = {
'DetectorName': 'Qwen',
'Raw': key,
'SourceMetadata': copy.deepcopy(metadata),
}
deepseek_finding = {
'DetectorName': 'DeepSeek',
'Raw': key,
'SourceMetadata': copy.deepcopy(metadata),
}
for finding in (qwen_finding, deepseek_finding):
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
self.assertEqual(finding['ScannerContext']['provider_hint'], 'deepseek')
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 1)
def test_explicit_qwen_assignment_ignores_deepseek_filename_metadata(self):
key = 'sk-' + ('h' * 32)
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'SourceMetadata': {'Data': {'Filesystem': {'file': 'config/deepseek.env'}}},
}
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
once = copy.deepcopy(finding)
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
self.assertEqual(finding, once)
self.assertEqual(finding['ScannerContext'], {
'provider_hint': 'qwen',
'provider_hint_source': 'explicit_assignment',
})
self.assert_provider_request_counts([finding], [finding], 1, 0)
def test_explicit_deepseek_assignment_ignores_qwen_filename_metadata(self):
key = 'sk-' + ('j' * 32)
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
'SourceMetadata': {'Data': {'Filesystem': {'file': 'config/qwen.env'}}},
}
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
once = copy.deepcopy(finding)
scanner.strip_nearby_context_for_persistence({'findings': [finding]})
self.assertEqual(finding, once)
self.assertEqual(finding['ScannerContext'], {
'provider_hint': 'deepseek',
'provider_hint_source': 'explicit_assignment',
})
self.assert_provider_request_counts([finding], [finding], 0, 1)
def test_explicit_qwen_detector_beats_duplicate_broad_deepseek_sibling(self):
key = 'sk-' + ('g' * 32)
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
deepseek_finding = {'DetectorName': 'DeepSeek', 'Raw': key}
result = {'findings': [qwen_finding, deepseek_finding]}
scanner.strip_nearby_context_for_persistence(result)
once = copy.deepcopy(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(result, once)
self.assertEqual(
[finding['ScannerContext']['provider_hint'] for finding in result['findings']],
['qwen', 'qwen'],
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 1, 0)
def test_explicit_deepseek_detector_beats_duplicate_generic_qwen_sibling(self):
key = 'sk-' + ('z' * 32)
qwen_finding = {'DetectorName': 'Qwen', 'Raw': key}
deepseek_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
}
result = {'findings': [qwen_finding, deepseek_finding]}
scanner.strip_nearby_context_for_persistence(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(
[finding['ScannerContext']['provider_hint'] for finding in result['findings']],
['deepseek', 'deepseek'],
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 1)
def test_independent_custom_detector_evidence_is_ambiguous(self):
key = 'sk-' + ('m' * 32)
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
deepseek_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
}
result = {'findings': [qwen_finding, deepseek_finding]}
scanner.strip_nearby_context_for_persistence(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(
[finding['ScannerContext']['provider_hint'] for finding in result['findings']],
['ambiguous_qwen_deepseek', 'ambiguous_qwen_deepseek'],
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 0)
def test_generic_overlapping_detectors_without_context_route_to_resolver(self):
key = 'sk-' + ('n' * 32)
qwen_finding = {'DetectorName': 'Qwen', 'Raw': key}
deepseek_finding = {'DetectorName': 'DeepSeek', 'Raw': key}
result = {'findings': [qwen_finding, deepseek_finding]}
scanner.strip_nearby_context_for_persistence(result)
scanner.strip_nearby_context_for_persistence(result)
for finding in result['findings']:
self.assertEqual(
finding['ScannerContext']['provider_hint'],
'ambiguous_generic_sk',
)
self.assertEqual(
finding['ScannerContext']['provider_candidates'],
['deepseek', 'zai', 'qwen', 'kimi'],
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 0)
def test_qwen_rejects_anthropic_prefix_even_with_qwen_hint(self):
key = 'sk-ant-api03-' + ('A' * 93) + 'AA'
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'ScannerContext': {'provider_hint': 'qwen'},
}
with mock.patch.object(qwen, 'iter_findings', return_value=[{
'raw': key,
'source': 'fixture',
'finding': finding,
}]):
self.assertEqual(list(qwen.iter_candidate_keys('unused.jsonl', [])), [])
def test_mixed_provider_context_skips_both_providers(self):
key = 'sk-' + ('4' * 32)
context = {'nearby': 'QWEN_API_KEY is used beside api.deepseek.com'}
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'ScannerContext': copy.deepcopy(context),
}
deepseek_finding = {
'DetectorName': 'DeepSeek',
'Raw': key,
'ScannerContext': copy.deepcopy(context),
}
scanner.strip_nearby_context_for_persistence({'findings': [qwen_finding]})
scanner.strip_nearby_context_for_persistence({'findings': [deepseek_finding]})
self.assertEqual(qwen_finding['ScannerContext']['provider_hint'], 'qwen')
self.assertEqual(
deepseek_finding['ScannerContext']['provider_hint'],
'ambiguous_qwen_deepseek',
)
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 1, 0)
def test_distinct_explicit_assignments_route_independently_in_mixed_env(self):
qwen_key = 'sk-' + ('u' * 32)
deepseek_key = 'sk-' + ('v' * 32)
nearby = 'QWEN_API_KEY=' + qwen_key + '\nDEEPSEEK_API_KEY=' + deepseek_key
qwen_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': qwen_key,
'ScannerContext': {'nearby': nearby, 'file': 'mixed.env'},
}
deepseek_finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': deepseek_key,
'ScannerContext': {'nearby': nearby, 'file': 'mixed.env'},
}
result = {'findings': [qwen_finding, deepseek_finding]}
scanner.strip_nearby_context_for_persistence(result)
once = copy.deepcopy(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(result, once)
self.assertEqual(qwen_finding['ScannerContext'], {
'file': 'mixed.env', 'provider_hint': 'qwen',
'provider_hint_source': 'explicit_assignment',
})
self.assertEqual(deepseek_finding['ScannerContext'], {
'file': 'mixed.env', 'provider_hint': 'deepseek',
'provider_hint_source': 'explicit_assignment',
})
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 1, 1)
def test_non_overlapping_qwen_format_does_not_require_provider_hint(self):
key = 'sk-sp-' + ('A' * 24)
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
with mock.patch.object(qwen, 'iter_findings', return_value=[{
'raw': key,
'source': 'fixture',
'finding': finding,
}]):
self.assertEqual(
[candidate for candidate, _, _ in qwen.iter_candidate_keys('unused.jsonl', [])],
[key],
)
def test_same_secret_is_ambiguous_across_repeated_strip_spool_db_and_jsonl(self):
key = 'sk-' + ('a' * 32)
event_id = str(uuid.uuid4())
result = {
'scan_event_id': event_id,
'target': 'fixture-target',
'scan_type': 'filesystem',
'timestamp': '2026-07-20T00:00:00+00:00',
'findings': [
{
'DetectorName': 'DeepSeekApiKey',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
'ScannerContext': {
'nearby': 'DEEPSEEK_API_KEY and QWEN_API_KEY are both configured here',
},
},
{
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'ScannerContext': {
'nearby': 'dashscope client beside api.deepseek.com',
},
},
],
'errors': [],
}
scanner.strip_nearby_context_for_persistence(result)
once_stripped = copy.deepcopy(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(result, once_stripped)
for persisted in result['findings']:
self.assertEqual(
persisted['ScannerContext']['provider_hint'],
'ambiguous_qwen_deepseek',
)
self.assertNotIn('nearby', persisted['ScannerContext'])
self.assertNotIn(key, json.dumps(persisted['ScannerContext']))
roundtrips = [copy.deepcopy(result['findings'])]
with tempfile.TemporaryDirectory() as temp_dir:
spool = ResultSpool(os.path.join(temp_dir, 'spool'), min_free_bytes=0)
record = spool.write_event({
'version': 1,
'scan_event_id': event_id,
'result': result,
})
roundtrips.append(record.envelope['result']['findings'])
db = ScannerDB(db_path=os.path.join(temp_dir, 'scanner.db'))
try:
run_id = db.start_run('test', ['test'])
cycle_id = db.start_source_cycle(
run_id, 'fixture', 'filesystem', 'search', 'q', 1, 1, None, {}, {},
)
db.record_target_result(run_id, cycle_id, 'fixture', 'q', 'fixture-target', result)
rows = db.conn.execute(
'SELECT raw_finding_json FROM findings ORDER BY id'
).fetchall()
roundtrips.append([json.loads(row['raw_finding_json']) for row in rows])
finally:
db.close()
results_dir = os.path.join(temp_dir, 'results')
scanner.ensure_private_directory(results_dir, reject_reparse=True)
with mock.patch.object(scanner.scan_config, 'results_dir', results_dir), \
mock.patch.object(scanner.scan_config, 'jsonl_rotation_enabled', False), \
mock.patch.object(scanner, 'write_foundry_keycheck_candidates_from_findings', return_value=0):
self.assertTrue(scanner.save_scan_result(copy.deepcopy(result)))
jsonl_findings = [
json.loads(line) for line in
Path(os.path.join(results_dir, 'found_secrets.jsonl')).read_text(encoding='utf-8').splitlines()
if line.strip()
]
roundtrips.append(jsonl_findings)
for findings in roundtrips:
deepseek_finding = next(
finding for finding in findings if finding.get('DetectorName') == 'DeepSeekApiKey'
)
qwen_finding = next(
finding for finding in findings if finding.get('DetectorName') == 'CustomRegex'
)
self.assertEqual(
deepseek_finding['ScannerContext']['provider_hint'],
'ambiguous_qwen_deepseek',
)
self.assertEqual(
qwen_finding['ScannerContext']['provider_hint'],
'ambiguous_qwen_deepseek',
)
with mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key,
'source': 'roundtrip',
'finding': deepseek_finding,
}]), mock.patch.object(deepseek.requests, 'get') as request:
candidates = list(deepseek.extract_candidates('unused.jsonl', []))
for candidate, _, _ in candidates:
deepseek.check_key(candidate, None, 1)
self.assertEqual(candidates, [])
request.assert_not_called()
with mock.patch.object(qwen, 'iter_findings', return_value=[{
'raw': key,
'source': 'roundtrip',
'finding': qwen_finding,
}]), mock.patch.object(qwen.requests, 'get') as get_request, \
mock.patch.object(qwen.requests, 'post') as post_request:
candidates = list(qwen.iter_candidate_keys('unused.jsonl', []))
for candidate, _, _ in candidates:
qwen.check_key(candidate, qwen.DEFAULT_BASE_URLS, False, None, 1)
self.assertEqual(candidates, [])
get_request.assert_not_called()
post_request.assert_not_called()
def test_deepseek_routing_decision_does_not_change_after_nearby_removal(self):
deepseek_key = 'sk-' + ('d' * 32)
result = {'findings': [
{
'DetectorName': 'DeepSeek',
'Raw': deepseek_key,
'ScannerContext': {'nearby': 'DEEPSEEK_API_KEY is configured here'},
},
{
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': 'sk-' + ('e' * 32),
},
]}
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(result['findings'][0]['ScannerContext'], {'provider_hint': 'deepseek'})
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(result['findings'][0]['ScannerContext'], {'provider_hint': 'deepseek'})
def test_legitimate_deepseek_candidate_still_makes_request(self):
key = 'sk-' + ('b' * 32)
finding = {
'DetectorName': 'DeepSeek',
'Raw': key,
'ScannerContext': {'nearby': 'DEEPSEEK_API_KEY is configured here'},
}
scanner.strip_nearby_context_for_persistence({'findings': [
finding,
]})
self.assertNotEqual(finding.get('ScannerContext', {}).get('provider_hint'), 'qwen')
self.assertFalse(deepseek.finding_looks_like_qwen_context(finding))
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': 'unauthorized'}
with mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key,
'source': 'roundtrip',
'finding': finding,
}]), mock.patch.object(deepseek.requests, 'get', return_value=response) as request:
candidates = list(deepseek.extract_candidates('unused.jsonl', []))
self.assertEqual(len(candidates), 1)
deepseek.check_key(candidates[0][0], None, 1)
request.assert_called_once()
def test_postgres_deepseek_route_rejection_completes_as_no_context(self):
key = 'sk-' + ('f' * 32)
finding = {
'DetectorName': 'DeepSeek',
'Raw': key,
'ScannerContext': {'provider_hint': 'ambiguous_qwen_deepseek'},
}
with mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key,
'source': 'postgres-fixture',
'finding': finding,
}]):
decisions = list(deepseek.iter_candidate_decisions('unused.jsonl', []))
self.assertEqual(len(decisions), 1)
self.assertEqual(decisions[0][3], 'ambiguous_qwen_deepseek')
result = deepseek.route_rejection_result(decisions[0][3])
self.assertEqual(result['status'], 'NO_CONTEXT')
self.assertEqual(keycheck_common.keycheck_status_group(result['status']), 'no_context')
self.assertEqual(
keycheck_runner.postgres_status_projection_targets('deepseek', 'NO_CONTEXT'),
['deepseekNoContext.txt'],
)
def test_legitimate_qwen_candidate_still_makes_request(self):
key = 'sk-' + ('c' * 32)
finding = {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
'ScannerContext': {'nearby': 'QWEN_API_KEY is configured here'},
}
result = {'findings': [finding]}
scanner.strip_nearby_context_for_persistence(result)
scanner.strip_nearby_context_for_persistence(result)
self.assertEqual(finding['ScannerContext']['provider_hint'], 'qwen')
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': {'message': 'unauthorized'}}
with mock.patch.object(qwen, 'iter_findings', return_value=[{
'raw': key,
'source': 'roundtrip',
'finding': finding,
}]), mock.patch.object(qwen.requests, 'get', return_value=response) as request, \
mock.patch.object(qwen.requests, 'post') as post_request:
candidates = list(qwen.iter_candidate_keys('unused.jsonl', []))
self.assertEqual(len(candidates), 1)
qwen.check_key(candidates[0][0], [qwen.DEFAULT_BASE_URLS[0]], False, None, 1)
request.assert_called_once()
post_request.assert_not_called()
def test_separate_database_events_with_conflicting_hints_route_neither(self):
key = 'sk-' + ('p' * 32)
qwen_finding = self.persist_provider_event(key, 'qwen', 'target-qwen')
deepseek_finding = self.persist_provider_event(key, 'deepseek', 'target-deepseek')
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 0)
def test_legacy_explicit_detector_identities_are_database_evidence(self):
fixtures = (
('qwen', 'a', {
'DetectorName': 'Qwen_DashScope',
'ScannerContext': {'provider_hint': 'qwen'},
}),
('qwen', 'b', {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'qwendashscope'},
}),
('deepseek', 'c', {
'DetectorName': 'DeepSeekApiKey',
'ScannerContext': {'provider_hint': 'deepseek'},
}),
('deepseek', 'd', {
'DetectorName': 'CustomRegex',
'ExtraData': {'name': 'DeepSeek_API_Key'},
}),
)
for expected, character, finding in fixtures:
with self.subTest(detector=finding):
key = 'sk-' + (character * 32)
finding['Raw'] = key
self.persist_finding_event(finding, f'legacy-{character}')
self.assertEqual(keycheck_common.global_provider_routing_evidence(key), {expected})
def test_empty_legacy_finding_json_uses_canonical_detector_identity(self):
key = 'sk-' + ('0' * 40)
rows = [
{'detector_name': 'DeepSeek', 'raw_finding_json': '', 'truncated': 0},
{'detector_name': 'QwenDashScope', 'raw_finding_json': '', 'truncated': 0},
]
with mock.patch.object(
ScannerDB, 'provider_routing_finding_rows', return_value=rows,
):
keycheck_common.clear_provider_routing_evidence_cache()
self.assertEqual(
keycheck_common.global_provider_routing_evidence(key),
{'qwen'},
)
def test_empty_legacy_finding_json_with_unknown_detector_fails_closed(self):
key = 'sk-' + ('9' * 40)
rows = [
{'detector_name': 'CustomRegex', 'raw_finding_json': '', 'truncated': 0},
]
with mock.patch.object(
ScannerDB, 'provider_routing_finding_rows', return_value=rows,
):
keycheck_common.clear_provider_routing_evidence_cache()
self.assertEqual(
keycheck_common.global_provider_routing_evidence(key),
{'qwen', 'deepseek', 'kimi', 'zai'},
)
def test_legacy_broad_deepseek_detector_and_its_extra_name_are_not_explicit_evidence(self):
key = 'sk-' + ('e' * 32)
self.persist_finding_event({
'DetectorName': 'DeepSeek',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
'ScannerContext': {'provider_hint': 'deepseek'},
}, 'legacy-broad-deepseek')
self.assertEqual(keycheck_common.global_provider_routing_evidence(key), set())
def test_legacy_qwen_and_current_deepseek_database_events_are_ambiguous(self):
key = 'sk-' + ('f' * 32)
self.persist_finding_event({
'DetectorName': 'QwenDashScope',
'Raw': key,
'ScannerContext': {'provider_hint': 'qwen'},
}, 'legacy-qwen')
self.persist_provider_event(
key, 'deepseek', 'current-deepseek', clear_cache=False,
)
self.assertEqual(
keycheck_common.global_provider_routing_evidence(key),
{'qwen', 'deepseek'},
)
def test_kimi_detector_is_persisted_generic_sk_routing_evidence(self):
key = 'sk-' + ('w' * 40)
self.persist_provider_event(key, 'kimi', 'current-kimi')
self.assertEqual(keycheck_common.global_provider_routing_evidence(key), {'kimi'})
def test_postgres_routing_merges_local_and_historical_provider_evidence(self):
key = 'sk-' + ('x' * 40)
self.persist_provider_event(key, 'kimi', 'current-kimi')
with mock.patch.dict(os.environ, {
'KEYCHECK_INPUT_MODE': 'postgres', 'KEYCHECK_SERVICE': 'qwen',
}, clear=False):
keycheck_common.clear_provider_routing_evidence_cache()
self.assertEqual(
keycheck_common.combined_provider_routing_hint(key, 'qwen'),
'ambiguous_generic_sk',
)
def test_empty_evidence_is_not_stale_after_concurrent_conflict_insert(self):
key = 'sk-' + ('i' * 32)
real_lookup = keycheck_common._load_provider_routing_evidence
with mock.patch.object(
keycheck_common, '_load_provider_routing_evidence', wraps=real_lookup,
) as lookup:
self.assertEqual(keycheck_common.combined_provider_routing_hint(key, 'qwen'), 'qwen')
self.persist_provider_event(
key, 'deepseek', 'concurrent-deepseek', clear_cache=False,
)
self.assertEqual(
keycheck_common.combined_provider_routing_hint(key, 'qwen'),
'ambiguous_qwen_deepseek',
)
self.assertEqual(lookup.call_count, 2)
def test_single_provider_evidence_is_not_stale_after_conflicting_insert(self):
key = 'sk-' + ('k' * 32)
self.persist_provider_event(key, 'qwen', 'first-qwen')
real_lookup = keycheck_common._load_provider_routing_evidence
with mock.patch.object(
keycheck_common, '_load_provider_routing_evidence', wraps=real_lookup,
) as lookup:
self.assertEqual(keycheck_common.global_provider_routing_evidence(key), {'qwen'})
self.persist_provider_event(
key, 'deepseek', 'concurrent-deepseek', clear_cache=False,
)
self.assertEqual(
keycheck_common.global_provider_routing_evidence(key),
{'qwen', 'deepseek'},
)
self.assertEqual(lookup.call_count, 2)
def test_qwen_deduplicates_routing_lookup_per_key_within_one_run(self):
key = 'sk-' + ('r' * 32)
finding = {
'DetectorName': 'QwenDashScope',
'Raw': key,
'ScannerContext': {
'provider_hint': 'qwen', 'provider_hint_source': 'explicit_assignment',
},
}
items = [
{'raw': key, 'source': f'fixture:{index}', 'finding': copy.deepcopy(finding)}
for index in range(100)
]
routing_common = sys.modules[qwen.combined_provider_routing_hint.__module__]
routing_common.clear_provider_routing_evidence_cache()
real_lookup = routing_common._load_provider_routing_evidence
with mock.patch.object(qwen, 'iter_findings', return_value=items), mock.patch.object(
routing_common, '_load_provider_routing_evidence', wraps=real_lookup,
) as lookup:
candidates = list(qwen.iter_candidate_keys('unused.jsonl', []))
self.assertEqual([candidate for candidate, _, _ in candidates], [key] * len(items))
lookup.assert_called_once()
def test_qwen_recheck_uses_authenticated_candidate_route(self):
key = 'sk-' + ('v' * 32)
finding = {'DetectorName': 'DeepSeek', 'Raw': key}
item = {
'raw': key,
'source': 'postgres-fixture',
'finding': finding,
'candidate_metadata': {'provider_hint': 'qwen'},
}
args = SimpleNamespace(
input='unused.jsonl', plain=[], proxy_file='', timeout=1, max_keys=1,
base_url=[], no_default_base_urls=False, retry_network=False,
retry_limited=False, retry_unknown=False, retry_restricted=False,
retry_no_balance=False, retry_valid=True, recheck_all=False,
debug=False,
)
result = {'status': 'DEAD', 'message': 'fixture invalid key', 'models': []}
with mock.patch.dict(os.environ, {
'KEYCHECK_INPUT_MODE': 'postgres', 'KEYCHECK_SERVICE': 'qwen',
}, clear=False), mock.patch.object(
qwen, 'require_provider_authority',
), mock.patch.object(
qwen, 'parse_args', return_value=args,
), mock.patch.object(
qwen, 'ensure_files',
), mock.patch.object(
qwen, 'load_proxies', return_value=None,
), mock.patch.object(
qwen, 'load_checked_statuses', return_value={},
), mock.patch.object(
qwen, 'load_known_keys', return_value=set(),
), mock.patch.object(
qwen, 'iter_findings', return_value=iter([item]),
), mock.patch.object(
qwen, 'should_skip_key', return_value=False,
), mock.patch.object(
qwen, 'base_urls_from_args', return_value=(['https://fixture.invalid/v1'], False),
), mock.patch.object(
qwen, 'combined_provider_routing_hint', return_value='deepseek',
) as routing, mock.patch.object(
qwen, 'check_key', return_value=result,
) as check_key, mock.patch.object(
qwen, 'resolve_provider_key', return_value=result,
) as resolve, mock.patch.object(
qwen, 'write_result',
) as write_result:
qwen.main()
routing.assert_not_called()
check_key.assert_called_once()
resolve.assert_not_called()
self.assertNotIn(
qwen.CANDIDATE_PROVIDER_ROUTE_FIELD,
write_result.call_args.args[3],
)
def test_single_persisted_provider_hint_routes_only_that_provider(self):
for provider, character, expected in (
('qwen', 's', (1, 0)),
('deepseek', 'y', (0, 1)),
):
with self.subTest(provider=provider):
key = 'sk-' + (character * 32)
self.persist_provider_event(key, provider, f'target-{provider}')
qwen_finding = {'DetectorName': 'Qwen', 'Raw': key}
deepseek_finding = {'DetectorName': 'DeepSeek', 'Raw': key}
self.assert_provider_request_counts(
[qwen_finding], [deepseek_finding], *expected,
)
def test_database_query_failure_aborts_qwen_candidate_iteration(self):
key = 'sk-' + ('t' * 32)
qwen_finding = {
'DetectorName': 'QwenDashScope', 'Raw': key,
'ScannerContext': {
'provider_hint': 'qwen', 'provider_hint_source': 'explicit_assignment',
},
}
keycheck_common.clear_provider_routing_evidence_cache()
with mock.patch.object(
ScannerDB,
'provider_routing_finding_rows',
side_effect=RuntimeError('fixture query failure'),
) as lookup:
with mock.patch.object(qwen, 'iter_findings', return_value=[{
'raw': key, 'source': 'roundtrip', 'finding': qwen_finding,
}]), self.assertRaisesRegex(RuntimeError, 'routing evidence lookup failed'):
list(qwen.iter_candidate_keys('unused.jsonl', []))
lookup.assert_called_once()
def test_database_query_failure_aborts_deepseek_candidate_iteration(self):
key = 'sk-' + ('u' * 32)
deepseek_finding = {
'DetectorName': 'DeepSeekApiKey', 'Raw': key,
'ScannerContext': {
'provider_hint': 'deepseek', 'provider_hint_source': 'explicit_assignment',
},
}
keycheck_common.clear_provider_routing_evidence_cache()
with mock.patch.object(
ScannerDB,
'provider_routing_finding_rows',
side_effect=RuntimeError('fixture query failure'),
) as lookup, mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key, 'source': 'roundtrip', 'finding': deepseek_finding,
}]), mock.patch.object(deepseek.requests, 'get') as request:
with self.assertRaisesRegex(RuntimeError, 'routing evidence lookup failed closed'):
list(deepseek.extract_candidates('unused.jsonl', []))
lookup.assert_called_once()
request.assert_not_called()
def test_database_evidence_row_overflow_fails_closed(self):
key = 'sk-' + ('o' * 32)
self.persist_provider_event(key, 'qwen', 'target-one')
self.persist_provider_event(key, 'qwen', 'target-two')
qwen_finding = {'DetectorName': 'Qwen', 'Raw': key}
deepseek_finding = {'DetectorName': 'DeepSeek', 'Raw': key}
with mock.patch.dict(
os.environ, {'KEYCHECK_PROVIDER_EVIDENCE_MAX_ROWS': '1'}, clear=False,
):
keycheck_common.clear_provider_routing_evidence_cache()
self.assert_provider_request_counts([qwen_finding], [deepseek_finding], 0, 0)
class QwenCandidateBoundTests(unittest.TestCase):
@staticmethod
def args(input_file):
return SimpleNamespace(
input=input_file, plain=[], proxy_file='unused-proxy.txt', timeout=1,
max_keys=0, base_url=['https://fixture.invalid/v1'],
no_default_base_urls=True, retry_network=False, retry_limited=False,
retry_unknown=False, retry_restricted=False, retry_no_balance=False,
retry_valid=False, recheck_all=False, debug=False,
)
def test_concatenated_keys_are_not_qwen_candidates(self):
deepseek_key = 'sk-' + ('a1' * 16)
concatenated = deepseek_key + deepseek_key
self.assertEqual(qwen.key_from_text(concatenated), concatenated)
self.assertFalse(qwen.is_qwen_key(concatenated))
self.assertIn('multiple concatenated', qwen.qwen_key_rejection_reason(concatenated))
def test_openai_legacy_marker_makes_no_request_and_later_qwen_candidate_runs(self):
openai_key = 'sk-' + ('A' * 12) + qwen.OPENAI_LEGACY_KEY_MARKER + ('B' * 24)
qwen_key = 'sk-sp-' + ('Q' * 24)
findings = (
{'DetectorName': 'QwenDashScope', 'Raw': f'QWEN_API_KEY={openai_key}'},
{'DetectorName': 'QwenDashScope', 'Raw': f'QWEN_API_KEY={qwen_key}'},
)
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': {'message': 'unauthorized'}}
self.assertIn('OpenAI legacy', qwen.qwen_key_rejection_reason(openai_key))
self.assertEqual(qwen.key_from_text(f'QWEN_API_KEY={openai_key}'), '')
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(
''.join(json.dumps(finding) + '\n' for finding in findings),
encoding='utf-8',
)
scanner.harden_private_file(input_file)
checked_file = os.path.join(temp_dir, 'qwenChecked.txt')
results_file = os.path.join(temp_dir, 'qwenResults.jsonl')
status_files = {
status: os.path.join(temp_dir, f'qwen{status.title().replace("_", "")}.txt')
for status in qwen.STATUS_FILES
}
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_INLINE': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
'SCANNER_DB_URL': '',
'DATABASE_URL': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.multiple(
qwen,
CHECKED_FILE=checked_file,
RESULTS_FILE=results_file,
STATUS_FILES=status_files,
), mock.patch.object(qwen, 'require_provider_authority'), mock.patch.object(
qwen, 'parse_args', return_value=self.args(input_file),
), mock.patch.object(qwen, 'load_proxies', return_value=None), mock.patch.object(
qwen.requests, 'get', return_value=response,
) as provider_request, mock.patch.object(qwen.requests, 'post') as provider_post:
qwen.main()
provider_request.assert_called_once()
self.assertEqual(
provider_request.call_args.kwargs['headers']['Authorization'],
f'Bearer {qwen_key}',
)
provider_post.assert_not_called()
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
self.assertEqual(state['processed'], 2)
def test_8193_byte_explicit_key_is_skipped_later_key_runs_and_checkpoint_advances(self):
oversized = 'sk-' + ('a' * (8193 - 3))
valid = 'sk-sp-' + ('A' * 24)
findings = (
{'DetectorName': 'QwenDashScope', 'Raw': oversized},
{'DetectorName': 'QwenDashScope', 'Raw': valid},
)
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': {'message': 'unauthorized'}}
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(
''.join(json.dumps(finding) + '\n' for finding in findings),
encoding='utf-8',
)
scanner.harden_private_file(input_file)
checked_file = os.path.join(temp_dir, 'qwenChecked.txt')
results_file = os.path.join(temp_dir, 'qwenResults.jsonl')
status_files = {
status: os.path.join(temp_dir, f'qwen{status.title().replace("_", "")}.txt')
for status in qwen.STATUS_FILES
}
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_INLINE': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
'SCANNER_DB_URL': '',
'DATABASE_URL': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.multiple(
qwen,
CHECKED_FILE=checked_file,
RESULTS_FILE=results_file,
STATUS_FILES=status_files,
), mock.patch.object(qwen, 'require_provider_authority'), mock.patch.object(
qwen, 'parse_args', return_value=self.args(input_file),
), mock.patch.object(qwen, 'load_proxies', return_value=None), mock.patch.object(
qwen.requests, 'get', return_value=response,
) as provider_request, mock.patch.object(qwen.requests, 'post') as provider_post:
qwen.main()
self.assertIsNone(qwen.QWEN_KEY_REGEX.fullmatch(oversized))
provider_request.assert_called_once()
self.assertEqual(
provider_request.call_args.kwargs['headers']['Authorization'],
f'Bearer {valid}',
)
provider_post.assert_not_called()
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
self.assertEqual(state['processed'], 2)
self.assertFalse(os.path.exists(
keycheck_common.status_transaction_journal_path(checked_file),
))
def test_routing_lookup_failure_preserves_checkpoint_and_replays_candidate(self):
key = 'sk-' + ('m' * 32)
finding = {
'DetectorName': 'QwenDashScope',
'Raw': key,
'ScannerContext': {
'provider_hint': 'qwen',
'provider_hint_source': 'explicit_assignment',
},
}
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(json.dumps(finding) + '\n', encoding='utf-8')
scanner.harden_private_file(input_file)
db_path = os.path.join(temp_dir, 'scanner.db')
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.object(
qwen, 'require_provider_authority',
), mock.patch.object(
qwen, 'parse_args', side_effect=lambda: self.args(input_file),
), mock.patch.object(qwen, 'ensure_files'), mock.patch.object(
qwen, 'load_proxies', return_value=None,
), mock.patch.object(qwen, 'load_checked_statuses', return_value={}), mock.patch.object(
qwen, 'load_known_keys', return_value=set(),
), mock.patch.object(
qwen, 'check_key', return_value={'status': 'DEAD', 'message': 'fixture'},
) as check_key, mock.patch.object(qwen, 'write_result') as write_result:
keycheck_common.clear_provider_routing_evidence_cache()
with self.assertRaisesRegex(RuntimeError, 'routing evidence lookup failed'):
qwen.main()
self.assertFalse(os.path.exists(Path(temp_dir, 'input_state.json')))
check_key.assert_not_called()
write_result.assert_not_called()
ScannerDB(db_path=db_path, db_url='').close()
keycheck_common.clear_provider_routing_evidence_cache()
with mock.patch.dict(os.environ, {'KEYCHECK_DB_PATH': db_path}, clear=False):
qwen.main()
check_key.assert_called_once_with(
key, ['https://fixture.invalid/v1'], True, None, 1, False,
)
write_result.assert_called_once()
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
def test_should_skip_failure_preserves_checkpoint_and_replays_candidate(self):
key = 'sk-sp-' + ('S' * 24)
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(json.dumps({
'DetectorName': 'QwenDashScope', 'Raw': key,
}) + '\n', encoding='utf-8')
scanner.harden_private_file(input_file)
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.object(
qwen, 'require_provider_authority',
), mock.patch.object(
qwen, 'parse_args', side_effect=lambda: self.args(input_file),
), mock.patch.object(qwen, 'ensure_files'), mock.patch.object(
qwen, 'load_proxies', return_value=None,
), mock.patch.object(qwen, 'load_checked_statuses', return_value={}), mock.patch.object(
qwen, 'load_known_keys', return_value=set(),
), mock.patch.object(
qwen, 'should_skip_key', side_effect=[RuntimeError('fixture preparation failure'), False],
), mock.patch.object(
qwen, 'check_key', return_value={'status': 'DEAD', 'message': 'fixture'},
) as check_key, mock.patch.object(qwen, 'write_result'):
with self.assertRaisesRegex(RuntimeError, 'fixture preparation failure'):
qwen.main()
self.assertFalse(os.path.exists(Path(temp_dir, 'input_state.json')))
qwen.main()
check_key.assert_called_once()
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
def test_candidate_publication_exception_preserves_checkpoint_and_retries_candidate(self):
keys = ('sk-sp-' + ('B' * 24), 'sk-sp-' + ('C' * 24))
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(''.join(
json.dumps({'DetectorName': 'QwenDashScope', 'Raw': key}) + '\n'
for key in keys
), encoding='utf-8')
scanner.harden_private_file(input_file)
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.object(
qwen, 'require_provider_authority',
), mock.patch.object(
qwen, 'parse_args', side_effect=lambda: self.args(input_file),
), mock.patch.object(qwen, 'ensure_files'), mock.patch.object(
qwen, 'load_proxies', return_value=None,
), mock.patch.object(qwen, 'load_checked_statuses', return_value={}), mock.patch.object(
qwen, 'load_known_keys', return_value=set(),
), mock.patch.object(
qwen, 'check_key', return_value={'status': 'DEAD', 'message': 'fixture'},
) as check_key, mock.patch.object(
qwen, 'write_result', side_effect=[ValueError('fixture journal failure'), None, None],
) as write_result:
with self.assertRaisesRegex(ValueError, 'fixture journal failure'):
qwen.main()
self.assertFalse(os.path.exists(Path(temp_dir, 'input_state.json')))
qwen.main()
self.assertEqual([call.args[0] for call in check_key.call_args_list], [keys[0], *keys])
self.assertEqual([call.args[0] for call in write_result.call_args_list], [keys[0], *keys])
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
def test_result_segment_bound_failure_preserves_checkpoint_and_replays_candidate(self):
keys = ('sk-sp-' + ('D' * 24), 'sk-sp-' + ('E' * 24))
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(''.join(
json.dumps({'DetectorName': 'QwenDashScope', 'Raw': key}) + '\n'
for key in keys
), encoding='utf-8')
scanner.harden_private_file(input_file)
checked_file = os.path.join(temp_dir, 'qwenChecked.txt')
results_file = os.path.join(temp_dir, 'qwenResults.jsonl')
segment_file = os.path.join(temp_dir, 'qwenResults.000001.jsonl')
Path(results_file).write_text('{}\n' * 350000, encoding='utf-8')
Path(segment_file).write_text('{"segment": true}\n', encoding='utf-8')
scanner.harden_private_file(results_file)
scanner.harden_private_file(segment_file)
status_files = {
status: os.path.join(temp_dir, f'qwen{status.title().replace("_", "")}.txt')
for status in qwen.STATUS_FILES
}
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_RESULTS_MAX_MB': '1',
'KEYCHECK_RESULTS_MAX_SEGMENTS': '1',
'KEYCHECK_DB_INLINE': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.multiple(
qwen,
CHECKED_FILE=checked_file,
RESULTS_FILE=results_file,
STATUS_FILES=status_files,
), mock.patch.object(qwen, 'require_provider_authority'), mock.patch.object(
qwen, 'parse_args', side_effect=lambda: self.args(input_file),
), mock.patch.object(qwen, 'load_proxies', return_value=None), mock.patch.object(
qwen, 'check_key', return_value={'status': 'DEAD', 'message': 'fixture'},
) as check_key:
with self.assertRaisesRegex(RuntimeError, 'segment bound reached'):
qwen.main()
self.assertFalse(os.path.exists(Path(temp_dir, 'input_state.json')))
os.remove(segment_file)
qwen.main()
self.assertEqual([call.args[0] for call in check_key.call_args_list], [keys[0], *keys])
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
def test_network_results_checkpoint_and_replay_in_retry_mode(self):
keys = ('sk-sp-' + ('N' * 24), 'sk-sp-' + ('R' * 24))
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
Path(input_file).write_text(''.join(
json.dumps({'DetectorName': 'QwenDashScope', 'Raw': key}) + '\n'
for key in keys
), encoding='utf-8')
scanner.harden_private_file(input_file)
checked_file = os.path.join(temp_dir, 'qwenChecked.txt')
results_file = os.path.join(temp_dir, 'qwenResults.jsonl')
status_files = {
status: os.path.join(temp_dir, f'qwen{status.title().replace("_", "")}.txt')
for status in qwen.STATUS_FILES
}
first_args = self.args(input_file)
retry_args = self.args(input_file)
retry_args.retry_network = True
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_INLINE': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': '',
}
with mock.patch.dict(os.environ, environment, clear=False), mock.patch.multiple(
qwen,
CHECKED_FILE=checked_file,
RESULTS_FILE=results_file,
STATUS_FILES=status_files,
), mock.patch.object(qwen, 'require_provider_authority'), mock.patch.object(
qwen, 'parse_args', side_effect=[first_args, retry_args],
), mock.patch.object(qwen, 'load_proxies', return_value=None), mock.patch.object(
qwen.requests, 'get', side_effect=qwen.requests.ConnectionError('fixture transport failure'),
) as provider_request, mock.patch.object(qwen.requests, 'post') as provider_post:
qwen.main()
first_state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(first_state['offset'], os.path.getsize(input_file))
with mock.patch.dict(
os.environ, {'KEYCHECK_DISABLE_HIGH_WATERMARK': '1'}, clear=False,
):
qwen.main()
self.assertEqual(
[call.kwargs['headers']['Authorization'] for call in provider_request.call_args_list],
[f'Bearer {key}' for key in (*keys, *keys)],
)
provider_post.assert_not_called()
result_rows = [
json.loads(line) for line in Path(results_file).read_text(encoding='utf-8').splitlines()
]
self.assertEqual([row['status'] for row in result_rows], ['NETWORK'] * 4)
network_keys = [
keycheck_common.normalize_status_key(line)
for line in Path(status_files['NETWORK']).read_text(encoding='utf-8').splitlines()
]
self.assertEqual(network_keys, list(keys))
state = json.loads(Path(temp_dir, 'input_state.json').read_text(encoding='utf-8'))
self.assertEqual(state['offset'], os.path.getsize(input_file))
class QwenRegionalStatusTests(unittest.TestCase):
def test_status_output_is_utf8_safe_on_cp1251(self):
qwen_path = APP_DIR / 'keycheckers' / 'qwen' / 'qwenKeycheck.py'
script = "\n".join((
'import io, runpy, sys',
"sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='cp1251', errors='strict')",
f'runpy.run_path({str(qwen_path)!r}, run_name="qwen_encoding_fixture")',
"print(' STATUS: UNKNOWN | ' + ('x' * 200) + '\\uff08')",
))
completed = subprocess.run(
[sys.executable, '-B', '-c', script],
cwd=str(APP_DIR),
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
timeout=30,
)
self.assertEqual(completed.returncode, 0, completed.stderr.decode('utf-8', 'replace'))
self.assertIn('\uff08', completed.stdout.decode('utf-8', 'strict'))
def test_choose_final_status_has_deterministic_pairwise_priority(self):
priority = (
'VALID', 'NO_BALANCE', 'LIMITED', 'RESTRICTED',
'UNKNOWN', 'NO_CONTEXT', 'NETWORK', 'DEAD',
)
rank = {status: index for index, status in enumerate(priority)}
for left in priority:
for right in priority:
expected = left if rank[left] < rank[right] else right
with self.subTest(left=left, right=right):
self.assertEqual(
qwen.choose_final_status(
'sk-' + ('5' * 32),
[{'status': left}, {'status': right}],
False,
),
expected,
)
def test_network_and_dead_is_retryable_network_in_either_order(self):
for statuses in (('NETWORK', 'DEAD'), ('DEAD', 'NETWORK')):
with self.subTest(statuses=statuses):
self.assertEqual(
qwen.choose_final_status(
'sk-' + ('6' * 32),
[{'status': status} for status in statuses],
False,
),
'NETWORK',
)
def test_coding_plan_endpoint_is_enabled_by_default(self):
key = 'sk-sp-' + ('B' * 24)
self.assertEqual(qwen.DEFAULT_BASE_URLS[0], 'https://coding-intl.dashscope.aliyuncs.com/v1')
self.assertEqual(qwen.choose_final_status(key, [], False), 'DEAD')
self.assertEqual(qwen.choose_final_status(key, [{'status': 'DEAD'}], False), 'DEAD')
self.assertEqual(qwen.choose_final_status(key, [{'status': 'DEAD'}], True), 'DEAD')
def test_models_authentication_remains_valid_without_chat_model(self):
key = 'sk-' + ('M' * 40)
response = mock.Mock(status_code=200, text='{}')
response.json.return_value = {'data': [{'id': 'text-embedding-v3'}]}
with mock.patch.object(qwen.requests, 'get', return_value=response), \
mock.patch.object(qwen.requests, 'post') as post_request:
result = qwen.check_base_url(
key, 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1',
None, 1,
)
self.assertEqual(result['status'], 'VALID')
self.assertTrue(result['authenticated'])
self.assertEqual(result['llm_probe_status'], 'NO_CONTEXT')
post_request.assert_not_called()
def test_models_authentication_does_not_override_generation_failure(self):
key = 'sk-' + ('F' * 40)
models_response = mock.Mock(status_code=200, text='{}')
models_response.json.return_value = {'data': [{'id': 'qwen-plus'}]}
cases = (
(401, 'invalid_api_key', 'Invalid access token or token expired', 'DEAD'),
(400, 'Arrearage', 'The account is in arrears', 'NO_BALANCE'),
(429, 'Throttling.RateQuota', 'Request rate quota exceeded', 'LIMITED'),
(503, 'InternalError', 'Service unavailable', 'NETWORK'),
(400, 'InvalidParameter', 'Unsupported request', 'UNKNOWN'),
)
for http_status, code, message, expected in cases:
with self.subTest(expected=expected):
probe_response = mock.Mock(status_code=http_status, text=message)
probe_response.json.return_value = {
'error': {'code': code, 'message': message},
}
with mock.patch.object(qwen.requests, 'get', return_value=models_response), \
mock.patch.object(qwen.requests, 'post', return_value=probe_response):
result = qwen.check_base_url(
key, 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1',
None, 1,
)
self.assertEqual(result['status'], expected)
self.assertEqual(result['llm_probe_status'], expected)
def test_valid_result_preserves_generation_probe_metadata(self):
key = 'sk-' + ('P' * 40)
attempt = {
'status': 'VALID', 'authenticated': True, 'region': 'fixture',
'base_url': 'https://fixture.invalid/v1', 'model_count': 2,
'models': ['qwen-plus'], 'llm_probe_status': 'GENERATION_OK',
'llm_probe_model': 'qwen-plus', 'message': 'ok',
}
with mock.patch.object(qwen, 'check_base_url', return_value=attempt):
result = qwen.check_key(
key, ['https://fixture.invalid/v1'], True, None, 1,
)
self.assertEqual(result['llm_probe_status'], 'GENERATION_OK')
self.assertEqual(result['llm_probe_model'], 'qwen-plus')
self.assertTrue(result['authenticated'])
def test_retry_network_status_prevents_should_skip(self):
key = 'sk-' + ('7' * 32)
args = SimpleNamespace(
recheck_all=False,
retry_network=True,
retry_limited=False,
retry_unknown=False,
retry_restricted=False,
retry_no_balance=False,
retry_valid=False,
)
retry_statuses = qwen.retry_statuses_from_args(args)
self.assertEqual(retry_statuses, {'NETWORK'})
self.assertFalse(qwen.should_skip_key(
key,
{key: 'NETWORK'},
{key},
args,
retry_statuses,
))
self.assertTrue(qwen.should_skip_key(
key,
{key: 'DEAD'},
{key},
args,
retry_statuses,
))
def test_forced_network_replay_attempts_each_key_once_and_max_counts_unique_keys(self):
first = 'sk-' + ('7' * 32)
second = 'sk-' + ('8' * 32)
third = 'sk-' + ('9' * 32)
args = SimpleNamespace(
input='unused.jsonl', plain=[], proxy_file='unused-proxy.txt', timeout=1,
max_keys=2, base_url=[], no_default_base_urls=False,
retry_network=True, retry_limited=False, retry_unknown=False,
retry_restricted=False, retry_no_balance=False, retry_valid=False,
recheck_all=False, debug=False,
)
candidates = [
(key, f'finding:{index}', {'DetectorName': 'QwenDashScope', 'Raw': key})
for key in (first, first, first, second, second, third)
for index in range(20)
]
checked = {first: 'NETWORK', second: 'NETWORK', third: 'NETWORK'}
with mock.patch.object(qwen, 'require_provider_authority'), \
mock.patch.object(qwen, 'parse_args', return_value=args), \
mock.patch.object(qwen, 'ensure_output_files'), \
mock.patch.object(qwen, 'load_proxies', return_value=None), \
mock.patch.object(qwen, 'load_checked_statuses', return_value=checked), \
mock.patch.object(qwen, 'load_known_keys', return_value=set(checked)), \
mock.patch.object(qwen, 'iter_candidate_keys', return_value=iter(candidates)), \
mock.patch.object(qwen, 'check_key', return_value={
'status': 'NETWORK', 'message': 'fixture network failure',
}) as check_key, \
mock.patch.object(qwen, 'write_result'), \
mock.patch.object(
sys.modules[qwen.should_skip_key.__module__],
'record_cached_keycheck_occurrence',
) as cached:
qwen.main()
self.assertEqual([call.args[0] for call in check_key.call_args_list], [first, second])
self.assertGreater(cached.call_count, 1)
class QwenDurableRetryInputTests(unittest.TestCase):
@staticmethod
def args(input_file):
return SimpleNamespace(
input=input_file, plain=[], proxy_file='unused-proxy.txt', timeout=1,
max_keys=0, base_url=[], no_default_base_urls=False,
retry_network=True, retry_limited=False, retry_unknown=False,
retry_restricted=False, retry_no_balance=False, retry_valid=False,
recheck_all=False, debug=False,
)
def test_provider_network_backlog_retries_without_retained_finding_and_success_clears_it(self):
key = 'sk-' + ('k' * 32)
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'found_secrets.jsonl')
Path(input_file).write_text('', encoding='utf-8')
scanner.harden_private_file(input_file)
status_files = {
status: os.path.join(temp_dir, f'qwen{status.title().replace("_", "")}.txt')
for status in qwen.STATUS_FILES
}
for path in status_files.values():
Path(path).write_text('', encoding='utf-8')
scanner.harden_private_file(path)
Path(status_files['NETWORK']).write_text(f'{key}\tNETWORK\n', encoding='utf-8')
scanner.harden_private_file(status_files['NETWORK'])
checked_file = os.path.join(temp_dir, 'qwenChecked.txt')
results_file = os.path.join(temp_dir, 'qwenResults.jsonl')
db_path = os.path.join(temp_dir, 'scanner.db')
ScannerDB(db_path=db_path, db_url='').close()
environment = {
'KEYCHECK_OUTPUT_DIR': temp_dir,
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': db_path,
}
with mock.patch.dict(os.environ, environment, clear=False), \
mock.patch.multiple(
qwen, STATUS_FILES=status_files, CHECKED_FILE=checked_file,
RESULTS_FILE=results_file,
), mock.patch.object(qwen, 'require_provider_authority'), \
mock.patch.object(qwen, 'parse_args', side_effect=lambda: self.args(input_file)), \
mock.patch.object(qwen, 'load_proxies', return_value=None), \
mock.patch.object(qwen, 'check_key', return_value={
'status': 'VALID', 'message': 'fixture success', 'models': [],
}) as check_key, mock.patch.object(qwen, 'write_keycheck_event'), \
mock.patch.object(qwen, 'record_validation_result', return_value=True):
qwen.main()
qwen.main()
check_key.assert_called_once()
self.assertNotIn(key, Path(status_files['NETWORK']).read_text(encoding='utf-8'))
self.assertIn(key, Path(status_files['VALID']).read_text(encoding='utf-8'))
def test_only_provider_owned_retry_file_bypasses_overlap_context_gate(self):
key = 'sk-' + ('l' * 32)
with tempfile.TemporaryDirectory() as temp_dir:
input_file = os.path.join(temp_dir, 'found_secrets.jsonl')
plain_file = os.path.join(temp_dir, 'attacker-plain.txt')
network_file = os.path.join(temp_dir, 'qwenNetwork.txt')
for path, text in (
(input_file, ''), (plain_file, key + '\n'), (network_file, key + '\tNETWORK\n'),
):
Path(path).write_text(text, encoding='utf-8')
status_files = dict(qwen.STATUS_FILES, NETWORK=network_file)
db_path = os.path.join(temp_dir, 'scanner.db')
ScannerDB(db_path=db_path, db_url='').close()
with mock.patch.dict(os.environ, {
'KEYCHECK_DB_URL': '', 'KEYCHECK_DB_PATH': db_path,
}, clear=False), mock.patch.object(
qwen, 'STATUS_FILES', status_files,
), mock.patch.object(qwen, 'iter_findings', return_value=[]):
candidates = list(qwen.iter_candidate_keys(
input_file, [plain_file], [network_file],
))
self.assertEqual([(candidate, source) for candidate, source, _ in candidates], [(key, network_file)])
class ProviderDetectorPolicyTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
import yaml
policy = yaml.safe_load((APP_DIR / 'trufflehog-custom-detectors.yaml').read_text(encoding='utf-8'))
cls.detectors = {item['name']: item for item in policy['detectors']}
def test_qwen_detector_requires_explicit_assignment(self):
detector = self.detectors['QwenDashScope']
self.assertEqual(set(detector['keywords']), {'QWEN_API_KEY', 'DASHSCOPE_API_KEY'})
self.assertEqual(len(detector['regex']), 1)
pattern = re.compile(next(iter(detector['regex'].values())))
key = 'sk-' + ('q' * 32)
for text in (f'QWEN_API_KEY={key}', f'"DASHSCOPE_API_KEY": "{key}"'):
with self.subTest(text=text):
self.assertEqual(pattern.search(text).group(1), key)
self.assertIsNone(pattern.search(f'ANTHROPIC_API_KEY=sk-ant-example; model=qwen; key={key}'))
self.assertIsNone(pattern.search(f'qwen_model = "qwen-plus"\nOPENAI_API_KEY={key}'))
longest = 'sk-sp-' + ('a' * (qwen.QWEN_KEY_MAX_BYTES - len('sk-sp-')))
oversized = longest + 'a'
self.assertEqual(len(longest.encode('ascii')), qwen.QWEN_KEY_MAX_BYTES)
self.assertEqual(pattern.search(f'QWEN_API_KEY={longest}').group(1), longest)
self.assertIsNone(pattern.search(f'QWEN_API_KEY={oversized}'))
self.assertIsNone(pattern.search('QWEN_API_KEY=' + ('sk-' + ('a' * 8190))))
hyphenated_oversized = 'sk-' + ('a' * 506) + '-' + ('a' * 100)
self.assertIsNone(qwen.QWEN_KEY_REGEX.search(hyphenated_oversized))
self.assertTrue(qwen.is_qwen_key(longest))
self.assertFalse(qwen.is_qwen_key(oversized))
def test_deepseek_detector_requires_canonical_assignment(self):
detector = self.detectors['DeepSeekApiKey']
self.assertEqual(detector['keywords'], ['DEEPSEEK_API_KEY'])
pattern = re.compile(next(iter(detector['regex'].values())))
key = 'sk-' + ('z9' * 16)
self.assertEqual(pattern.search(f'DEEPSEEK_API_KEY="{key}"').group(1), key)
self.assertEqual(pattern.search(f'"DEEPSEEK_API_KEY": "{key}"').group(1), key)
self.assertIsNone(pattern.search('DEEPSEEK_API_KEY=sk-' + ('A' * 32)))
self.assertIsNone(pattern.search('DEEPSEEK_MODEL=deepseek-chat'))
class ProviderFormatTests(unittest.TestCase):
def test_deepseek_accepts_lowercase_alphanumeric_with_explicit_evidence(self):
key = 'sk-' + ('z9' * 16)
finding = {
'DetectorName': 'DeepSeekApiKey',
'ExtraData': {'name': 'DeepSeekApiKey'},
'Raw': key,
}
with mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key,
'source': 'fixture',
'finding': finding,
}]), mock.patch.object(
deepseek, 'combined_provider_routing_hint', side_effect=lambda _, hint: hint,
):
self.assertEqual(
[candidate for candidate, _, _ in deepseek.extract_candidates('unused.jsonl', [])],
[key],
)
def test_deepseek_builtin_shape_without_evidence_is_rejected(self):
key = 'sk-' + ('z9' * 16)
finding = {'DetectorName': 'DeepSeek', 'Raw': key}
with mock.patch.object(deepseek, 'iter_findings', return_value=[{
'raw': key,
'source': 'fixture',
'finding': finding,
}]), mock.patch.object(
deepseek, 'combined_provider_routing_hint', return_value='',
), mock.patch.object(deepseek, 'provider_routing_database_failed', return_value=False):
self.assertEqual(list(deepseek.extract_candidates('unused.jsonl', [])), [])
def test_anthropic_extracts_api03_admin01_and_legacy_formats(self):
keys = (
'sk-ant-api03-' + ('A' * 93) + 'AA',
'sk-ant-admin01-' + ('b' * 93) + 'AA',
'sk-ant-' + ('C' * 86),
)
items = [
{'raw': key, 'source': 'fixture', 'finding': {'DetectorName': 'Anthropic', 'Raw': key}}
for key in keys
]
with mock.patch.object(anthropic, 'iter_findings', return_value=items):
self.assertEqual(
[candidate for candidate, _, _ in anthropic.extract_candidates('unused.jsonl', [])],
list(keys),
)
self.assertIsNone(anthropic.ANTHROPIC_REGEX.fullmatch('sk-ant-admin01-' + ('b' * 92) + 'AA'))
class AnthropicAdminValidationTests(unittest.TestCase):
def setUp(self):
self.key = 'sk-ant-admin01-' + ('b' * 93) + 'AA'
self.proxy = {'https': 'http://proxy.invalid'}
@staticmethod
def response(status_code):
response = mock.Mock(status_code=status_code, text='fixture error')
response.json.return_value = {'error': {'message': 'fixture error'}}
return response
def test_admin_key_uses_only_read_only_admin_list_endpoint(self):
with mock.patch.object(
anthropic.requests, 'get', return_value=self.response(200),
) as get_request, mock.patch.object(anthropic.requests, 'post') as post_request:
result = anthropic.check_key(
self.key, self.proxy, 7, model='must-not-be-used', include_models=True,
)
self.assertEqual(result['status'], 'VALID')
self.assertTrue(result['admin'])
get_request.assert_called_once_with(
'https://api.anthropic.com/v1/organizations/api_keys',
headers={
'anthropic-version': '2023-06-01',
'x-api-key': self.key,
},
params={'limit': 1},
proxies=self.proxy,
timeout=7,
)
post_request.assert_not_called()
def test_admin_http_statuses_are_classified_without_messages_call(self):
expected = {
401: 'DEAD',
403: 'RESTRICTED',
429: 'LIMITED',
500: 'UNKNOWN',
}
for status_code, status in expected.items():
with self.subTest(status_code=status_code), mock.patch.object(
anthropic.requests, 'get', return_value=self.response(status_code),
), mock.patch.object(anthropic.requests, 'post') as post_request:
result = anthropic.check_key(self.key, None, 3)
self.assertEqual(result['status'], status)
self.assertEqual(result['http_status'], status_code)
post_request.assert_not_called()
def test_admin_transport_failure_is_network(self):
with mock.patch.object(
anthropic.requests, 'get',
side_effect=anthropic.requests.ConnectionError('fixture transport failure'),
), mock.patch.object(anthropic.requests, 'post') as post_request:
result = anthropic.check_key(self.key, None, 3)
self.assertEqual(result['status'], 'NETWORK')
self.assertTrue(result['admin'])
post_request.assert_not_called()
def test_standard_api_key_keeps_messages_validation(self):
key = 'sk-ant-api03-' + ('A' * 93) + 'AA'
with mock.patch.object(anthropic.requests, 'get') as get_request, mock.patch.object(
anthropic.requests, 'post', return_value=self.response(401),
) as post_request:
result = anthropic.check_key(key, None, 5, model='claude-fixture')
self.assertEqual(result['status'], 'DEAD')
post_request.assert_called_once()
self.assertEqual(post_request.call_args.args[0], 'https://api.anthropic.com/v1/messages')
self.assertEqual(post_request.call_args.kwargs['headers'], {
'content-type': 'application/json',
'anthropic-version': '2023-06-01',
'x-api-key': key,
})
get_request.assert_not_called()
class KeycheckRetryReplayTests(unittest.TestCase):
@staticmethod
def args(**overrides):
values = {
'config': 'config.yaml', 'input': None, 'input_mode': 'jsonl',
'proxy_file': None, 'max_keys': 0,
'retry_network': False, 'retry_limited': False, 'retry_unknown': False,
'retry_restricted': False, 'retry_no_balance': False, 'retry_valid': False,
'no_resource_probe': False, 'recheck_all': False,
}
values.update(overrides)
return SimpleNamespace(**values)
@staticmethod
def write_private(path, text):
Path(path).write_text(text, encoding='utf-8')
scanner.harden_private_file(path)
def captured_runner_env(self, temp_dir, args, status_text=None):
results = os.path.join(temp_dir, 'results')
keychecks = os.path.join(temp_dir, 'keychecks')
output = os.path.join(keychecks, 'qwen')
db_path = os.path.join(temp_dir, 'scanner.db')
for path in (temp_dir, results, keychecks, output):
scanner.ensure_private_directory(path, reject_reparse=True)
if not os.path.exists(db_path):
ScannerDB(db_path=db_path, db_url='').close()
if status_text is not None:
self.write_private(os.path.join(output, 'qwenNetwork.txt'), status_text)
layout = {
'results_dir': results, 'keycheck_dir': keychecks,
'proxy_file': os.path.join(temp_dir, 'proxy.txt'), 'database_path': db_path,
}
class CapturedProcess:
def wait(self):
return 0
with mock.patch.object(keycheck_runner, 'require_active_supervisor_child', return_value={}), \
mock.patch.object(keycheck_runner, 'supervised_child_environment', return_value={}), \
mock.patch.object(keycheck_runner, 'OwnedProcess', return_value=CapturedProcess()) as owned, \
mock.patch.dict(os.environ, {
'KEYCHECK_DISABLE_HIGH_WATERMARK': '1',
'TRUF_MANAGED_POSTGRES_DSN': '', 'SCANNER_DB_URL': '', 'DATABASE_URL': '',
'KEYCHECK_DB_URL': '',
}, clear=False):
self.assertEqual(keycheck_runner.run_service(
'qwen', keycheck_runner.SERVICES['qwen'], args, layout,
), 0)
return owned.call_args.kwargs['env']
def test_network_backlog_enables_replay_for_provider_invocation(self):
with tempfile.TemporaryDirectory() as temp_dir:
env = self.captured_runner_env(temp_dir, self.args(retry_network=True), 'candidate\tNETWORK\n')
self.assertEqual(env.get('KEYCHECK_DISABLE_HIGH_WATERMARK'), '1')
def test_no_network_backlog_preserves_incremental_high_watermark(self):
with tempfile.TemporaryDirectory() as temp_dir:
env = self.captured_runner_env(temp_dir, self.args(retry_network=True), '')
self.assertNotIn('KEYCHECK_DISABLE_HIGH_WATERMARK', env)
def test_recheck_all_always_enables_replay(self):
with tempfile.TemporaryDirectory() as temp_dir:
env = self.captured_runner_env(temp_dir, self.args(recheck_all=True))
self.assertEqual(env.get('KEYCHECK_DISABLE_HIGH_WATERMARK'), '1')
def test_network_at_eof_is_visible_on_retry_replay(self):
key = 'sk-' + ('r' * 32)
finding = {
'DetectorName': 'QwenDashScope',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
input_file = os.path.join(temp_dir, 'findings.jsonl')
self.write_private(input_file, json.dumps(finding) + '\n')
status_file = os.path.join(temp_dir, 'qwenNetwork.txt')
self.write_private(status_file, '')
db_path = os.path.join(temp_dir, 'scanner.db')
ScannerDB(db_path=db_path, db_url='').close()
environment = {
'KEYCHECK_STATE_DIR': temp_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': db_path,
}
attempts = []
def consume():
for candidate, _, _ in qwen.iter_candidate_keys(input_file, []):
attempts.append(candidate)
with mock.patch.dict(os.environ, environment, clear=False):
consume()
consume()
self.assertEqual(attempts, [key])
self.write_private(status_file, f'{key}\tNETWORK\n')
self.assertTrue(keycheck_runner.provider_replay_required(
'qwen', self.args(retry_network=True), temp_dir,
))
with mock.patch.dict(os.environ, {'KEYCHECK_DISABLE_HIGH_WATERMARK': '1'}, clear=False):
consume()
self.assertEqual(attempts, [key, key])
def test_segmented_force_replay_ignores_every_eof_checkpoint(self):
with tempfile.TemporaryDirectory() as temp_dir:
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
state_dir = os.path.join(temp_dir, 'state')
scanner.ensure_private_directory(state_dir, reject_reparse=True)
current = os.path.join(temp_dir, 'findings.jsonl')
segments = [
os.path.join(temp_dir, 'findings.000001.jsonl'),
os.path.join(temp_dir, 'findings.000002.jsonl'),
]
for path, row_id in zip(segments + [current], ('one', 'two', 'three')):
self.write_private(path, json.dumps({'id': row_id}) + '\n')
keycheck_common.write_jsonl_manifest(current, {
'current_path': os.path.abspath(current),
'segments': [
{'name': os.path.basename(path), 'path': path, 'sequence': index}
for index, path in enumerate(segments, 1)
],
})
environment = {
'KEYCHECK_STATE_DIR': state_dir,
'KEYCHECK_OUTPUT_DIR': state_dir,
'KEYCHECK_SERVICE': 'fixture',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
}
def consume():
return [item['data']['id'] for item in keycheck_common.iter_jsonl_input(current)]
with mock.patch.dict(os.environ, environment, clear=False):
self.assertEqual(consume(), ['one', 'two', 'three'])
self.assertEqual(consume(), [])
with mock.patch.dict(
os.environ, {'KEYCHECK_DISABLE_HIGH_WATERMARK': '1'}, clear=False,
):
self.assertEqual(consume(), ['one', 'two', 'three'])
state = json.loads(Path(state_dir, 'input_state.json').read_text(encoding='utf-8'))
for path in segments + [current]:
self.assertEqual(state['files'][os.path.abspath(path)]['offset'], os.path.getsize(path))
def test_runner_retry_and_recheck_replay_segmented_candidates_to_provider(self):
replay_modes = (
('retry-network', {'retry_network': True}, True),
('recheck-all', {'recheck_all': True}, False),
)
for label, overrides, needs_backlog in replay_modes:
with self.subTest(mode=label), tempfile.TemporaryDirectory() as temp_dir:
current = os.path.join(temp_dir, 'findings.jsonl')
segment = os.path.join(temp_dir, 'findings.000001.jsonl')
keys = ('sk-' + ('w' * 32), 'sk-' + ('x' * 32))
findings = [
{
'DetectorName': 'QwenDashScope',
'ExtraData': {'name': 'QwenDashScope'},
'Raw': key,
}
for key in keys
]
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
self.write_private(segment, json.dumps(findings[0]) + '\n')
self.write_private(current, json.dumps(findings[1]) + '\n')
keycheck_common.write_jsonl_manifest(current, {
'current_path': os.path.abspath(current),
'segments': [{
'name': os.path.basename(segment), 'path': segment, 'sequence': 1,
}],
})
output_dir = os.path.join(temp_dir, 'keychecks', 'qwen')
scanner.ensure_private_directory(os.path.join(temp_dir, 'keychecks'), reject_reparse=True)
scanner.ensure_private_directory(output_dir, reject_reparse=True)
db_path = os.path.join(temp_dir, 'scanner.db')
ScannerDB(db_path=db_path, db_url='').close()
reader_environment = {
'KEYCHECK_STATE_DIR': output_dir,
'KEYCHECK_OUTPUT_DIR': output_dir,
'KEYCHECK_SERVICE': 'qwen',
'KEYCHECK_INPUT_TAIL_BYTES': '0',
'KEYCHECK_DISABLE_HIGH_WATERMARK': '0',
'KEYCHECK_DB_URL': '',
'KEYCHECK_DB_PATH': db_path,
}
with mock.patch.dict(os.environ, reader_environment, clear=False):
self.assertEqual(
[candidate for candidate, _, _ in qwen.iter_candidate_keys(current, [])],
list(keys),
)
self.assertEqual(list(qwen.iter_candidate_keys(current, [])), [])
args = self.args(input=current, **overrides)
status_text = f'{keys[0]}\tNETWORK\n' if needs_backlog else None
runner_env = self.captured_runner_env(temp_dir, args, status_text)
self.assertEqual(runner_env.get('KEYCHECK_DISABLE_HIGH_WATERMARK'), '1')
response = mock.Mock(status_code=401, text='unauthorized')
response.json.return_value = {'error': {'message': 'unauthorized'}}
with mock.patch.dict(os.environ, runner_env, clear=True), mock.patch.object(
qwen.requests, 'get', return_value=response,
) as provider_request, mock.patch.object(qwen.requests, 'post') as provider_post:
candidates = list(qwen.iter_candidate_keys(current, []))
for candidate, _, _ in candidates:
qwen.check_key(candidate, [qwen.DEFAULT_BASE_URLS[0]], False, None, 1)
self.assertEqual([candidate for candidate, _, _ in candidates], list(keys))
self.assertEqual(provider_request.call_count, 2)
provider_post.assert_not_called()
class SlowOwnedProcess:
def __init__(self, started, killed, reaped):
self.pid = 987654
self.job_membership_verified = True
self.payload_identity = {'pid': self.pid}
self.returncode = None
self._stopped = False
self._started = started
self._killed = killed
self._reaped = reaped
self._started.set()
def poll(self):
return -9 if self._stopped else None
def kill(self):
self._stopped = True
self.returncode = -9
self._killed.set()
def wait(self, timeout=None):
self._stopped = True
self.returncode = -9
self._reaped.set()
return -9
class FatalScanCancellationTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
def setUp(self):
clear_scan_fatal()
self.database_environment = mock.patch.dict(os.environ, {
'SCANNER_DB_URL': '', 'DATABASE_URL': '', 'TRUF_MANAGED_POSTGRES_DSN': '',
})
self.database_environment.start()
def tearDown(self):
clear_scan_fatal()
self.database_environment.stop()
if hasattr(scanner._scan_slot_scope_local, 'scope'):
del scanner._scan_slot_scope_local.scope
def test_fatal_before_command_workdir_prevents_launch(self):
scanner._set_scan_slot_fatal('fixture fatal')
with mock.patch.object(scanner, 'create_command_work_dir') as workdir, \
mock.patch.object(scanner, 'OwnedProcess') as process:
with self.assertRaisesRegex(scanner.ScanSlotFatalError, 'fixture fatal'):
scanner.run_command(['fixture'], 5)
workdir.assert_not_called()
process.assert_not_called()
def test_download_chunks_stop_on_peer_fatal(self):
class Response:
def __enter__(self):
return self
def __exit__(self, *args):
return False
def raise_for_status(self):
return None
def iter_content(self, chunk_size):
yield b'first'
scanner._set_scan_slot_fatal('download peer fatal')
yield b'second'
class Session:
trust_env = True
def __enter__(self):
return self
def __exit__(self, *args):
return False
def get(self, *args, **kwargs):
return Response()
with tempfile.TemporaryDirectory() as temp_dir, \
mock.patch.object(scanner.requests, 'Session', return_value=Session()):
path = os.path.join(temp_dir, 'artifact.bin')
with self.assertRaisesRegex(scanner.ScanSlotFatalError, 'download peer fatal'):
scanner.download_file('https://example.invalid/artifact', path, timeout=1)
self.assertEqual(Path(path).read_bytes(), b'first')
def test_completed_network_call_cannot_launch_after_peer_fatal(self):
request_started = threading.Event()
release_request = threading.Event()
response = mock.Mock(status_code=200, headers={}, text='')
def request(*args, **kwargs):
request_started.set()
if not release_request.wait(2):
raise AssertionError('test did not release network request')
return response
def scan_target(target, provider=None, **kwargs):
if target == 'fatal':
if not request_started.wait(2):
raise AssertionError('network request did not start')
scanner._set_scan_slot_fatal('network peer fatal')
release_request.set()
raise scanner.ScanSlotFatalError('network peer fatal')
scanner.api_request('GET', 'https://example.invalid', timeout=1)
scanner.run_command(['must-not-launch'], 5)
return {'findings': [], 'errors': []}
with mock.patch.object(scanner.scan_config, 'max_active_scans', 0), \
mock.patch.object(scanner, 'scan_git_repo', side_effect=scan_target), \
mock.patch.object(scanner.requests, 'request', side_effect=request), \
mock.patch.object(scanner, 'OwnedProcess') as process:
with self.assertRaisesRegex(scanner.ScanSlotFatalError, 'network peer fatal'):
scanner.scan_targets_batch(
['network', 'fatal'], 'git', max_workers=2, persist_results=False,
)
process.assert_not_called()
response.close.assert_called_once()
def test_batch_fatal_kills_owned_child_and_never_launches_queued_target(self):
for max_active in (0, 2):
with self.subTest(max_active=max_active), tempfile.TemporaryDirectory() as temp_dir:
clear_scan_fatal()
started = threading.Event()
killed = threading.Event()
reaped = threading.Event()
process = SlowOwnedProcess(started, killed, reaped)
process._started.clear()
limiter_path = os.path.join(temp_dir, 'limiter.sqlite')
def owned_process(*args, **kwargs):
process._started.set()
return process
def scan_target(target, provider=None, **kwargs):
if target == 'fatal':
if not started.wait(2):
raise AssertionError('slow child did not launch')
scanner._set_scan_slot_fatal('peer fatal')
raise scanner.ScanSlotFatalError('peer fatal')
if target == 'slow':
scanner.run_command(['fixture', target], 30)
else:
scanner.run_command(['fixture', target], 30)
return {'findings': [], 'errors': []}
with mock.patch.object(scanner.scan_config, 'max_active_scans', max_active), \
mock.patch.object(scanner.scan_config, 'scan_limiter_db', limiter_path), \
mock.patch.object(scanner.scan_config, 'scan_slot_wait_sec', 0.01), \
mock.patch.object(scanner.scan_config, 'min_free_gb', 0), \
mock.patch.object(scanner.scan_config, 'trufflehog_job_memory_limit_bytes', 1024 * 1024), \
mock.patch.object(scanner, 'scan_git_repo', side_effect=scan_target), \
mock.patch.object(scanner, 'create_command_work_dir', return_value=temp_dir), \
mock.patch.object(scanner, 'cleanup_command_work_dir'), \
mock.patch.object(scanner, '_shared_staging_owners', return_value=[]), \
mock.patch.object(scanner, 'require_trufflehog_launch_authority'), \
mock.patch.object(scanner, 'write_temp_owner'), \
mock.patch.object(scanner, 'OwnedProcess', side_effect=owned_process) as launches:
began = time.monotonic()
with self.assertRaisesRegex(scanner.ScanSlotFatalError, 'peer fatal'):
scanner.scan_targets_batch(
['slow', 'fatal', 'queued'], 'git', max_workers=2, persist_results=False,
)
elapsed = time.monotonic() - began
self.assertLess(elapsed, 3)
self.assertTrue(killed.wait(1))
self.assertTrue(reaped.wait(1))
self.assertEqual(launches.call_count, 1)
if max_active:
conn = sqlite3.connect(limiter_path)
try:
deadline = time.monotonic() + 2
while conn.execute('SELECT COUNT(*) FROM scan_slots').fetchone()[0] and time.monotonic() < deadline:
time.sleep(0.05)
self.assertEqual(conn.execute('SELECT COUNT(*) FROM scan_slots').fetchone()[0], 0)
finally:
conn.close()
deadline = time.monotonic() + 3
while any(
thread.is_alive() and thread.name.startswith('scan-slot-heartbeat-')
for thread in threading.enumerate()
) and time.monotonic() < deadline:
time.sleep(0.05)
class DockerTagIdentityTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
def setUp(self):
clear_scan_fatal()
self.database_environment = mock.patch.dict(os.environ, {
'SCANNER_DB_URL': '', 'DATABASE_URL': '', 'TRUF_MANAGED_POSTGRES_DSN': '',
})
self.database_environment.start()
def tearDown(self):
clear_scan_fatal()
self.database_environment.stop()
@staticmethod
def response(payload):
response = mock.Mock(status_code=200, headers={}, text='')
response.json.return_value = payload
response.raise_for_status.return_value = None
return response
def fetch_tags(self, *tags, **kwargs):
def resolve_graph(_repo_name, digest, _platform_os, _platform_arch, bearer_token):
return {
'manifest_digest': digest,
'layers': (digest,),
}, bearer_token or 'fixture-token'
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
mock.patch.object(
scanner, 'dockerhub_tag_rate_limit_state',
return_value={'active': False, 'retry_at': None},
), \
mock.patch.object(scanner, 'put_dockerhub_tag_cache') as cache, \
mock.patch.object(scanner, 'resolve_docker_layer_graph', side_effect=resolve_graph), \
mock.patch.object(scanner, 'api_request', return_value=self.response({'results': list(tags)})):
targets, status = scanner.fetch_dockerhub_tags(
'owner/repo', retry_count=0, return_status=True, **kwargs,
)
return targets, status, cache
def test_digest_repoint_changes_queue_identity(self):
digest_one = 'sha256:' + ('1' * 64)
digest_two = 'sha256:' + ('2' * 64)
first, status, _ = self.fetch_tags({'name': 'latest', 'digest': digest_one, 'last_updated': '2026-07-19T00:00:00Z'})
self.assertEqual(status, 'ok')
same, _, _ = self.fetch_tags({'name': 'latest', 'digest': digest_one, 'last_updated': '2026-07-20T00:00:00Z'})
repointed, _, _ = self.fetch_tags({'name': 'latest', 'digest': digest_two, 'last_updated': '2026-07-20T00:00:00Z'})
first = first[0]
same = same[0]
repointed = repointed[0]
self.assertEqual(first, f'owner/repo@{digest_one}')
self.assertEqual(normalize_target(first, 'docker'), normalize_target(same, 'docker'))
self.assertNotEqual(normalize_target(first, 'docker'), normalize_target(repointed, 'docker'))
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [first, same, repointed])
count = db.conn.execute(
"SELECT COUNT(*) AS n FROM target_queue WHERE source = 'dockerhub'"
).fetchone()['n']
self.assertEqual(count, 2)
finally:
db.close()
def test_no_digest_discovery_is_unknown_and_not_cached(self):
targets, status, cache = self.fetch_tags({
'name': 'latest',
'last_updated': '2026-07-20T01:02:03Z',
'images': [{'os': 'linux', 'architecture': 'amd64'}],
})
self.assertEqual((targets, status), ([], 'unknown'))
cache.assert_not_called()
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets(
'dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'],
)
row = db.conn.execute(
"SELECT status, resolver_state FROM target_queue WHERE source = 'dockerhub'"
).fetchone()
self.assertEqual((row['status'], row['resolver_state']), ('deferred', 'pending'))
finally:
db.close()
def test_resolved_bare_repository_rediscovery_preserves_periodic_refresh(self):
digest_a = 'sha256:' + ('a' * 64)
digest_b = 'sha256:' + ('b' * 64)
with tempfile.TemporaryDirectory() as temp_dir, mock.patch.dict(os.environ, {
'DOCKER_RESOLVER_RETRY_SEC': '60',
}):
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' "
"WHERE source = 'dockerhub' AND target = 'owner/repo'"
)
db.conn.commit()
first = db.claim_docker_resolutions('dockerhub', 1, 'resolver-a', 60)[0]
self.assertTrue(db.finish_docker_resolution(
'dockerhub', first['id'], first['resolver_token'],
[f'owner/repo@{digest_a}'],
refresh_interval_sec=86400,
))
resolved = db.conn.execute(
'''SELECT status, resolver_state, resolver_due_at, available_after,
resolver_attempts, resolver_token, completed_at
FROM target_queue WHERE id = ?''',
(first['id'],),
).fetchone()
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
rediscovered = db.conn.execute(
'''SELECT status, resolver_state, resolver_due_at, available_after,
resolver_attempts, resolver_token, completed_at
FROM target_queue WHERE id = ?''',
(first['id'],),
).fetchone()
self.assertEqual(dict(rediscovered), dict(resolved))
self.assertEqual(db.claim_docker_resolutions(
'dockerhub', 1, 'resolver-early', 60, periodic_limit=1,
), [])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' "
"WHERE id = ?",
(first['id'],),
)
db.conn.commit()
second = db.claim_docker_resolutions(
'dockerhub', 1, 'resolver-b', 60, periodic_limit=1, periodic_only=True,
)[0]
self.assertTrue(second['periodic'])
self.assertTrue(db.finish_docker_resolution(
'dockerhub', second['id'], second['resolver_token'],
[f'owner/repo@{digest_b}'],
))
targets = {
row['target'] for row in db.conn.execute(
"SELECT target FROM target_queue WHERE source = 'dockerhub'"
).fetchall()
}
self.assertEqual(targets, {
'owner/repo', f'owner/repo@{digest_a}', f'owner/repo@{digest_b}',
})
finally:
db.close()
def test_rediscovery_preserves_active_resolver_and_scan_leases(self):
digest = 'sha256:' + ('c' * 64)
digest_target = f'owner/active@{digest}'
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' "
"WHERE source = 'dockerhub' AND target = 'owner/repo'"
)
db.conn.commit()
resolving = db.claim_docker_resolutions('dockerhub', 1, 'resolver', 60)[0]
active = db.conn.execute(
'SELECT resolver_due_at, resolver_token, resolver_attempts FROM target_queue WHERE id = ?',
(resolving['id'],),
).fetchone()
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
preserved = db.conn.execute(
'''SELECT resolver_state, resolver_due_at, resolver_token, resolver_attempts
FROM target_queue WHERE id = ?''',
(resolving['id'],),
).fetchone()
self.assertEqual(preserved['resolver_state'], 'resolving')
self.assertEqual(dict(preserved), {
'resolver_state': 'resolving',
'resolver_due_at': active['resolver_due_at'],
'resolver_token': active['resolver_token'],
'resolver_attempts': active['resolver_attempts'],
})
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' WHERE id = ?",
(resolving['id'],),
)
db.conn.commit()
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
expired = db.conn.execute(
'SELECT status, resolver_state, resolver_token FROM target_queue WHERE id = ?',
(resolving['id'],),
).fetchone()
self.assertEqual((expired['status'], expired['resolver_state']), ('deferred', 'pending'))
self.assertIsNone(expired['resolver_token'])
db.enqueue_targets('dockerhub', 'docker', 'q', [digest_target])
scan = db.claim_targets(
'dockerhub', 'docker', 1, 'scanner', 60, max_attempts=3, return_rows=True,
)[0]
db.enqueue_targets(
'dockerhub', 'docker', 'q', [], unresolved_targets=[digest_target],
)
scan_after = db.conn.execute(
'SELECT status, lease_token, resolver_state FROM target_queue WHERE id = ?',
(scan['id'],),
).fetchone()
self.assertEqual(scan_after['status'], 'in_progress')
self.assertEqual(scan_after['lease_token'], scan['lease_token'])
self.assertIsNone(scan_after['resolver_state'])
finally:
db.close()
def test_rediscovered_unchanged_digest_is_deduplicated(self):
digest = 'sha256:' + ('d' * 64)
target = f'owner/repo@{digest}'
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
for owner in ('resolver-a', 'resolver-b'):
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' "
"WHERE source = 'dockerhub' AND target = 'owner/repo'"
)
db.conn.commit()
row = db.claim_docker_resolutions(
'dockerhub', 1, owner, 60, periodic_limit=1,
)[0]
self.assertTrue(db.finish_docker_resolution(
'dockerhub', row['id'], row['resolver_token'], [target],
))
db.enqueue_targets(
'dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'],
)
count = db.conn.execute(
'SELECT COUNT(*) AS n FROM target_queue WHERE source = ?', ('dockerhub',),
).fetchone()['n']
self.assertEqual(count, 2)
finally:
db.close()
def test_matching_platform_digest_is_pinned(self):
digest = 'sha256:' + ('3' * 64)
targets, status, _ = self.fetch_tags({
'name': 'latest',
'images': [
{'os': 'linux', 'architecture': 'arm64', 'digest': 'sha256:' + ('4' * 64)},
{'os': 'linux', 'architecture': 'amd64', 'digest': digest},
],
}, platform_filter_enabled=True)
self.assertEqual((targets, status), ([f'owner/repo@{digest}'], 'ok'))
def test_layer_graph_selector_deduplicates_aliases_and_fills_three_roles(self):
layer = lambda value: 'sha256:' + (value * 64)
candidates = [
{'target': 'newest', 'updated_at': 40, 'source_index': 0,
'layers': (layer('a'), layer('b'), layer('c'))},
{'target': 'newest-alias', 'updated_at': 30, 'source_index': 1,
'layers': (layer('a'), layer('b'), layer('c'))},
{'target': 'recent-small-change', 'updated_at': 35, 'source_index': 2,
'layers': (layer('a'), layer('b'), layer('d'))},
{'target': 'maximum-novelty', 'updated_at': 20, 'source_index': 3,
'layers': (layer('a'), layer('e'), layer('f'))},
{'target': 'oldest-reordered', 'updated_at': 10, 'source_index': 4,
'layers': (layer('c'), layer('b'), layer('a'))},
]
selected = scanner.select_docker_layer_graphs(candidates, 3)
self.assertEqual(
[item['target'] for item in selected],
['newest', 'maximum-novelty', 'oldest-reordered'],
)
for invalid in (False, True, None, '3', 3.0, 0, 11):
with self.subTest(invalid=invalid), self.assertRaises(ValueError):
scanner.docker_images_per_repository_limit(invalid)
def test_registry_resolves_requested_child_and_ordered_layers_with_bearer_auth(self):
index_digest = 'sha256:' + ('1' * 64)
child_digest = 'sha256:' + ('2' * 64)
arm_digest = 'sha256:' + ('3' * 64)
layers = ['sha256:' + ('a' * 64), 'sha256:' + ('b' * 64)]
def registry_response(status, payload, headers=None):
response = mock.Mock(status_code=status, headers=headers or {}, content=b'{}')
response.json.return_value = payload
response.raise_for_status.return_value = None
return response
responses = [
registry_response(401, {}, {
'WWW-Authenticate': (
'Bearer realm="https://auth.docker.io/token",'
'service="registry.docker.io",scope="repository:owner/repo:pull"'
),
}),
registry_response(200, {'token': 'registry-token'}),
registry_response(200, {
'schemaVersion': 2,
'manifests': [
{'digest': arm_digest, 'platform': {'os': 'linux', 'architecture': 'arm64'}},
{'digest': child_digest, 'platform': {'os': 'linux', 'architecture': 'amd64'}},
],
}, {'Docker-Content-Digest': index_digest}),
registry_response(200, {
'schemaVersion': 2,
'layers': [{'digest': digest} for digest in layers],
}, {'Docker-Content-Digest': child_digest}),
]
manager = scanner.DockerTokenManager()
manager.accounts = [
scanner.DockerAccount('account-a', 'user-a', 'secret-a', ''),
]
manager.explicit_pool = True
with mock.patch.object(scanner, 'docker_token_manager', manager), \
mock.patch.object(scanner, 'api_request', side_effect=responses) as request:
graph, token = scanner.resolve_docker_layer_graph('owner/repo', index_digest)
self.assertEqual(graph, {
'manifest_digest': child_digest,
'layers': tuple(layers),
})
self.assertEqual(token.token, 'registry-token')
self.assertEqual(token.account_name, 'account-a')
self.assertEqual(request.call_count, 4)
self.assertEqual(request.call_args_list[1].kwargs['auth'], ('user-a', 'secret-a'))
self.assertEqual(request.call_args_list[1].args[1], 'https://auth.docker.io/token')
child_headers = request.call_args_list[-1].kwargs['headers']
self.assertEqual(child_headers['Authorization'], 'Bearer registry-token')
def test_registry_rejects_oversized_and_invalid_layer_manifests(self):
oversized = self.response({'token': 'ignored'})
oversized.headers = {
'Content-Length': str(scanner.DOCKER_REGISTRY_MANIFEST_MAX_BYTES + 1),
}
with self.assertRaisesRegex(scanner.DockerRegistryResolutionError, 'size limit'):
scanner._bounded_docker_registry_json(oversized, 'fixture')
digest = 'sha256:' + ('9' * 64)
with mock.patch.object(
scanner, 'docker_registry_manifest',
return_value=({'layers': [{'digest': 'not-a-digest'}]}, 'token'),
):
with self.assertRaisesRegex(scanner.DockerRegistryResolutionError, 'layer digest'):
scanner.resolve_docker_layer_graph('owner/repo', digest)
def test_search_resolution_plumbs_ten_image_budget(self):
response = self.response({
'count': 1,
'results': [{'repo_name': 'owner/repo'}],
})
digest = 'sha256:' + ('8' * 64)
with mock.patch.object(scanner, 'api_request', return_value=response), \
mock.patch.object(
scanner, 'fetch_dockerhub_tags',
return_value=([f'owner/repo@{digest}'], 'ok'),
) as fetch_tags:
targets = scanner.fetch_dockerhub_images(
'fixture', pages=1, images_per_repository=10,
)
self.assertEqual(targets, [f'owner/repo@{digest}'])
self.assertEqual(fetch_tags.call_args.args[2], 10)
def test_partial_graph_resolution_emits_subset_without_caching_it(self):
good = 'sha256:' + ('4' * 64)
failed = 'sha256:' + ('5' * 64)
tags = [
{'name': 'good', 'digest': good, 'last_updated': '2026-07-20T02:00:00Z'},
{'name': 'failed', 'digest': failed, 'last_updated': '2026-07-20T01:00:00Z'},
]
def resolve(_repo_name, digest, *_args):
if digest == failed:
raise scanner.DockerRegistryResolutionError('temporary manifest failure')
return {'manifest_digest': digest, 'layers': (digest,)}, 'token'
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
mock.patch.object(
scanner, 'dockerhub_tag_rate_limit_state',
return_value={'active': False, 'retry_at': None},
), \
mock.patch.object(scanner, 'put_dockerhub_tag_cache') as cache, \
mock.patch.object(scanner, 'resolve_docker_layer_graph', side_effect=resolve), \
mock.patch.object(scanner, 'api_request', return_value=self.response({'results': tags})):
targets, status = scanner.fetch_dockerhub_tags(
'owner/repo', limit=3, retry_count=0, return_status=True,
)
self.assertEqual((targets, status), ([f'owner/repo@{good}'], 'partial'))
cache.assert_not_called()
def test_partial_queue_resolution_keeps_bare_repository_retryable(self):
digest = 'sha256:' + ('6' * 64)
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00'"
)
db.conn.commit()
claim = db.claim_docker_resolutions('dockerhub', 1, 'resolver', 60)[0]
self.assertTrue(db.finish_docker_resolution(
'dockerhub', claim['id'], claim['resolver_token'],
[f'owner/repo@{digest}'], 'partial manifest resolution', complete=False,
))
bare = db.conn.execute(
'SELECT status, resolver_state, resolver_due_at FROM target_queue WHERE id = ?',
(claim['id'],),
).fetchone()
self.assertEqual((bare['status'], bare['resolver_state']), ('deferred', 'retry'))
self.assertTrue(bare['resolver_due_at'])
emitted = db.target_queue_item('dockerhub', 'docker', f'owner/repo@{digest}')
self.assertEqual(emitted['status'], 'pending')
finally:
db.close()
def test_global_cooldown_refunds_claim_attempt_and_stale_token_cannot_emit(self):
digest = 'sha256:' + ('7' * 64)
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/retry'])
db.conn.execute(
"UPDATE target_queue SET resolver_attempts = 7, "
"resolver_due_at = '2000-01-01T00:00:00+00:00'"
)
db.conn.commit()
claim = db.claim_docker_resolutions('dockerhub', 1, 'resolver', 60)[0]
self.assertEqual(claim['resolver_attempts_before'], 7)
self.assertEqual(claim['resolver_attempts'], 8)
retry_at = datetime.now(timezone.utc) + timedelta(minutes=5)
retry_at = retry_at.isoformat(timespec='seconds')
self.assertTrue(db.finish_docker_resolution(
'dockerhub', claim['id'], claim['resolver_token'], [],
'shared cooldown', complete=False, retry_at=retry_at,
claim_attempt_consumed=False,
))
row = db.conn.execute(
'''SELECT status, resolver_state, resolver_attempts,
resolver_due_at, available_after
FROM target_queue WHERE id = ?''',
(claim['id'],),
).fetchone()
self.assertEqual((row['status'], row['resolver_state']), ('deferred', 'retry'))
self.assertEqual(row['resolver_attempts'], 7)
self.assertEqual(row['resolver_due_at'], retry_at)
self.assertEqual(row['available_after'], retry_at)
self.assertFalse(db.finish_docker_resolution(
'dockerhub', claim['id'], claim['resolver_token'],
[f'owner/retry@{digest}'], complete=True,
))
self.assertIsNone(db.target_queue_item(
'dockerhub', 'docker', f'owner/retry@{digest}',
))
finally:
db.close()
def test_periodic_repository_refresh_has_reserved_lane_and_deduplicates_digests(self):
digest_a = 'sha256:' + ('8' * 64)
digest_b = 'sha256:' + ('9' * 64)
target_a = f'owner/repo@{digest_a}'
target_b = f'owner/repo@{digest_b}'
with tempfile.TemporaryDirectory() as temp_dir:
db = ScannerDB(db_path=os.path.join(temp_dir, 'queue.db'))
try:
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/repo'])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00'"
)
db.conn.commit()
first = db.claim_docker_resolutions('dockerhub', 1, 'initial', 60)[0]
self.assertTrue(db.finish_docker_resolution(
'dockerhub', first['id'], first['resolver_token'], [target_a],
refresh_interval_sec=86400,
))
anchor = db.conn.execute(
'''SELECT status, resolver_state, resolver_attempts, resolver_due_at
FROM target_queue WHERE id = ?''',
(first['id'],),
).fetchone()
self.assertEqual((anchor['status'], anchor['resolver_state']), ('done', 'resolved'))
self.assertEqual(anchor['resolver_attempts'], 0)
self.assertTrue(anchor['resolver_due_at'])
db.enqueue_targets('dockerhub', 'docker', 'q', [], unresolved_targets=['owner/retry'])
db.conn.execute(
"UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00', "
"available_after = '2000-01-01T00:00:00+00:00' "
"WHERE target IN ('owner/repo', 'owner/retry')"
)
db.conn.commit()
retry = db.claim_docker_resolutions(
'dockerhub', 1, 'retry-first', 60, periodic_limit=1,
)[0]
self.assertEqual(retry['target'], 'owner/retry')
self.assertFalse(retry['periodic'])
self.assertTrue(db.finish_docker_resolution(
'dockerhub', retry['id'], retry['resolver_token'], [],
complete=True, refresh_interval_sec=86400,
))
periodic = db.claim_docker_resolutions(
'dockerhub', 1, 'periodic', 60,
periodic_limit=1, periodic_only=True,
)[0]
self.assertEqual(periodic['target'], 'owner/repo')
self.assertTrue(periodic['periodic'])
self.assertFalse(db.finish_docker_resolution(
'dockerhub', periodic['id'], 'stale-token', [target_b],
refresh_interval_sec=86400,
))
self.assertIsNone(db.target_queue_item('dockerhub', 'docker', target_b))
self.assertTrue(db.finish_docker_resolution(
'dockerhub', periodic['id'], periodic['resolver_token'],
[target_a, target_b], refresh_interval_sec=86400,
))
rows = db.conn.execute(
"SELECT target, status FROM target_queue WHERE source = 'dockerhub' ORDER BY target"
).fetchall()
self.assertEqual(sum(row['target'] == target_a for row in rows), 1)
self.assertEqual(sum(row['target'] == target_b for row in rows), 1)
self.assertEqual(
db.target_queue_item('dockerhub', 'docker', 'owner/repo')['status'], 'done',
)
finally:
db.close()
def test_due_resolver_stops_after_shared_cooldown_without_preclaiming_batch(self):
retry_at = (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(timespec='seconds')
db = mock.Mock()
db.runtime_control_state.return_value = {
'effective_discovery_paused': False,
}
db.claim_docker_resolutions.return_value = [{
'id': 11, 'target': 'owner/repo', 'resolver_token': 'lease', 'periodic': False,
}]
db.finish_docker_resolution.return_value = True
args = SimpleNamespace(
tag_resolve_limit=100,
docker_repository_refresh_max_per_cycle=1,
docker_repository_refresh_interval_sec=86400,
docker_images_per_repository=3,
tag_retry_count=0,
tag_retry_delay=0,
docker_platform_filter_enabled=True,
docker_platform_os='linux',
docker_platform_arch='amd64',
docker_platform_candidate_tags=20,
)
outcome = scanner.DockerTagResolutionOutcome(
tags=(), status='global_cooldown', remote_attempted=False,
retry_at=retry_at, error='shared cooldown',
)
with mock.patch.object(console_runner, 'fetch_dockerhub_tags', return_value=outcome):
console_runner.resolve_due_docker_queue_targets(db, 'dockerhub', args)
self.assertEqual(db.claim_docker_resolutions.call_count, 1)
finish = db.finish_docker_resolution.call_args
self.assertEqual(finish.kwargs['retry_at'], retry_at)
self.assertFalse(finish.kwargs['claim_attempt_consumed'])
def test_due_resolver_persists_resolved_immutable_targets_without_access_probe(self):
target = 'docker.io/acme/public@sha256:' + ('a' * 64)
db = mock.Mock()
db.runtime_control_state.return_value = {
'effective_discovery_paused': False,
}
db.claim_docker_resolutions.side_effect = [[{
'id': 21, 'target': 'acme/public', 'resolver_token': 'lease',
'periodic': False,
}], []]
db.finish_docker_resolution.return_value = True
args = SimpleNamespace(
tag_resolve_limit=1, docker_repository_refresh_max_per_cycle=0,
docker_repository_refresh_interval_sec=86400,
docker_images_per_repository=1, tag_retry_count=0,
tag_retry_delay=0, docker_platform_filter_enabled=True,
docker_platform_os='linux', docker_platform_arch='amd64',
docker_platform_candidate_tags=20,
)
outcome = scanner.DockerTagResolutionOutcome(
tags=(target,), status='ok', remote_attempted=True,
)
with mock.patch.object(
console_runner, 'fetch_dockerhub_tags', return_value=outcome,
):
console_runner.resolve_due_docker_queue_targets(db, 'dockerhub', args)
finish = db.finish_docker_resolution.call_args
self.assertEqual(finish.args[3], [target])
self.assertNotIn('anonymous_verified_targets', finish.kwargs)
self.assertNotIn('anonymous_verification_complete', finish.kwargs)
self.assertTrue(finish.kwargs['complete'])
def test_due_resolver_stops_after_auth_failure_and_consumes_only_one_claim(self):
db = mock.Mock()
db.runtime_control_state.return_value = {
'effective_discovery_paused': False,
}
db.claim_docker_resolutions.return_value = [{
'id': 12, 'target': 'owner/repo', 'resolver_token': 'lease', 'periodic': False,
}]
db.finish_docker_resolution.return_value = True
args = SimpleNamespace(
tag_resolve_limit=100,
docker_repository_refresh_max_per_cycle=1,
docker_repository_refresh_interval_sec=86400,
docker_images_per_repository=3,
tag_retry_count=0,
tag_retry_delay=0,
docker_platform_filter_enabled=True,
docker_platform_os='linux',
docker_platform_arch='amd64',
docker_platform_candidate_tags=20,
)
outcome = scanner.DockerTagResolutionOutcome(
tags=(), status='auth_failed', remote_attempted=False,
error='Docker authentication is unavailable',
)
with mock.patch.object(console_runner, 'fetch_dockerhub_tags', return_value=outcome):
console_runner.resolve_due_docker_queue_targets(db, 'dockerhub', args)
self.assertEqual(db.claim_docker_resolutions.call_count, 1)
finish = db.finish_docker_resolution.call_args
self.assertTrue(finish.kwargs['claim_attempt_consumed'])
self.assertIsNone(finish.kwargs['retry_at'])
def test_due_resolver_reserves_one_periodic_lane_inside_total_limit(self):
retry_rows = [
[{'id': 1, 'target': 'owner/a', 'resolver_token': 'a', 'periodic': False}],
[{'id': 2, 'target': 'owner/b', 'resolver_token': 'b', 'periodic': False}],
[{'id': 3, 'target': 'owner/c', 'resolver_token': 'c', 'periodic': True}],
]
db = mock.Mock()
db.claim_docker_resolutions.side_effect = retry_rows
db.finish_docker_resolution.return_value = True
args = SimpleNamespace(
tag_resolve_limit=3,
docker_repository_refresh_max_per_cycle=1,
docker_repository_refresh_interval_sec=86400,
docker_images_per_repository=3,
tag_retry_count=0,
tag_retry_delay=0,
docker_platform_filter_enabled=True,
docker_platform_os='linux',
docker_platform_arch='amd64',
docker_platform_candidate_tags=20,
)
outcome = scanner.DockerTagResolutionOutcome(
tags=(), status='empty', remote_attempted=True,
)
with mock.patch.object(console_runner, 'fetch_dockerhub_tags', return_value=outcome):
console_runner.resolve_due_docker_queue_targets(db, 'dockerhub', args)
self.assertEqual(db.claim_docker_resolutions.call_count, 3)
self.assertNotIn('periodic_only', db.claim_docker_resolutions.call_args_list[0].kwargs)
self.assertNotIn('periodic_only', db.claim_docker_resolutions.call_args_list[1].kwargs)
self.assertIs(db.claim_docker_resolutions.call_args_list[2].kwargs['periodic_only'], True)
self.assertEqual(db.finish_docker_resolution.call_count, 3)
def test_due_resolver_yields_to_claimable_immutable_scan_backlog(self):
db = mock.Mock(last_error=None)
db.has_claimable_targets_v2.return_value = True
args = SimpleNamespace(target_retry_max_attempts=3)
with mock.patch.object(console_runner, 'resolve_due_docker_queue_targets') as resolve:
processed = console_runner.resolve_due_docker_queue_targets_if_scan_queue_empty(
db, 'dockerhub', args,
)
self.assertEqual(processed, 0)
db.has_claimable_targets_v2.assert_called_once_with(
'dockerhub', 'docker', max_attempts=3,
)
resolve.assert_not_called()
db.has_claimable_targets_v2.return_value = False
with mock.patch.object(
console_runner, 'resolve_due_docker_queue_targets', return_value=7,
) as resolve:
processed = console_runner.resolve_due_docker_queue_targets_if_scan_queue_empty(
db, 'dockerhub', args,
)
self.assertEqual(processed, 7)
resolve.assert_called_once_with(db, 'dockerhub', args)
def test_docker_auth_pool_keeps_account_identity_and_never_falls_back_when_empty(self):
source_config = {
'auth_pool': 'dockerhub_main',
'rate_limit_cooldown': 300,
}
secrets = {'auth_pools': {'dockerhub_main': [
{'name': 'account-a', 'username': 'user-a', 'token': 'secret-a'},
{'name': 'account-b', 'username': 'user-b', 'token': 'secret-b'},
]}}
state = {'sources': {'dockerhub': console_runner.default_source_state()}}
with mock.patch.object(console_runner, 'configure_docker_discovery_accounts') as configure, \
mock.patch.object(console_runner, 'configure_docker_discovery_tokens') as legacy, \
mock.patch('builtins.print'):
console_runner.configure_source_auth(
'dockerhub', source_config, state, secrets,
)
configure.assert_called_once_with(
[
{'name': 'account-a', 'username': 'user-a', 'token': 'secret-a'},
{'name': 'account-b', 'username': 'user-b', 'token': 'secret-b'},
],
cooldown_sec=300,
)
legacy.assert_not_called()
state['sources']['dockerhub']['auth_status']['account-a'] = {
'status': 'dead', 'disabled_until': 'manual', 'disabled_reason': 'auth_invalid',
}
state['sources']['dockerhub']['auth_status']['account-b'] = {
'status': 'dead', 'disabled_until': 'manual', 'disabled_reason': 'auth_invalid',
}
with mock.patch.object(console_runner, 'configure_docker_discovery_accounts') as configure, \
mock.patch.object(console_runner, 'configure_docker_discovery_tokens') as legacy, \
mock.patch('builtins.print'):
console_runner.configure_source_auth(
'dockerhub', source_config, state, secrets,
)
configure.assert_called_once_with([], cooldown_sec=300)
legacy.assert_not_called()
def test_docker_discovery_auth_does_not_require_scanner_runtime_or_create_config(self):
manager = scanner.DockerTokenManager()
accounts = [{'name': 'account-a', 'username': 'user-a', 'token': 'secret-a'}]
with mock.patch.object(scanner, 'docker_token_manager', manager), \
mock.patch.object(
scanner, 'require_scanner_runtime_initialized',
side_effect=RuntimeError('scanner runtime unavailable'),
) as require_runtime, \
mock.patch.object(scanner, 'create_docker_config_dir') as create_config:
scanner.configure_docker_discovery_accounts(accounts, cooldown_sec=300)
self.assertEqual(manager.account_count(), 1)
self.assertEqual(manager.accounts[0].config_dir, '')
create_config.assert_not_called()
require_runtime.assert_not_called()
with self.assertRaisesRegex(RuntimeError, 'scanner runtime unavailable'):
scanner.configure_docker_accounts(accounts, cooldown_sec=300)
require_runtime.assert_called_once_with()
def test_docker_auth_events_persist_only_fixed_token_free_status(self):
source_config = {'auth_pool': 'dockerhub_main', 'rate_limit_cooldown': 300}
secrets = {'auth_pools': {'dockerhub_main': [
{'name': 'account-a', 'username': 'user-a', 'token': 'sentinel-secret'},
]}}
state = {'sources': {'dockerhub': console_runner.default_source_state()}}
events = [{
'name': 'account-a', 'category': 'rate_limit',
'reset_at': '2026-08-29T12:00:00+00:00',
'message': 'Docker hub_tags HTTP 429',
}]
with mock.patch.object(console_runner, 'drain_docker_auth_events', return_value=events):
console_runner.persist_docker_auth_events(
'dockerhub', source_config, state, secrets,
)
status = state['sources']['dockerhub']['auth_status']['account-a']
self.assertEqual(status['disabled_reason'], 'rate_limit')
self.assertEqual(status['last_error'], 'Docker hub_tags HTTP 429')
self.assertNotIn('sentinel-secret', json.dumps(state))
def test_structured_digest_cache_roundtrip_preserves_target(self):
digest = 'sha256:' + ('5' * 64)
repointed_digest = 'sha256:' + ('7' * 64)
target = serialize_docker_tag_target(f'owner/repo:latest@{digest}', '2026-07-20T01:02:03Z')
same_image = serialize_docker_tag_target(f'owner/repo:latest@{digest}', '2026-07-21T01:02:03Z')
repointed = serialize_docker_tag_target(
f'owner/repo:latest@{repointed_digest}', '2026-07-21T01:02:03Z',
)
parsed = parse_docker_target(target)
self.assertEqual(parsed['image'], f'owner/repo:latest@{digest}')
self.assertEqual(parsed['revision'], '2026-07-20T01:02:03Z')
self.assertEqual(normalize_target(target, 'docker'), normalize_target(same_image, 'docker'))
self.assertNotEqual(normalize_target(target, 'docker'), normalize_target(repointed, 'docker'))
with tempfile.TemporaryDirectory() as temp_dir:
path = os.path.join(temp_dir, 'docker-cache.sqlite')
with mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_path', path), \
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_rows', 10), \
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_age_sec', 3600), \
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_bytes', 1024 * 1024), \
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_min_free_bytes', 0):
scanner._dockerhub_tag_cache_initialized.discard(path)
self.assertTrue(scanner.put_dockerhub_tag_cache(
'owner/repo', None, 1, 'ok', [target], 3600,
tag_records=[{'name': 'latest', 'target': target, 'updated_at': 1.0}],
))
self.assertEqual(scanner.get_dockerhub_tag_cache('owner/repo', None, 1), [target])
self.assertTrue(scanner.put_dockerhub_tag_cache(
'owner/repo', None, 1, 'ok', [repointed], 3600,
tag_records=[{'name': 'latest', 'target': repointed, 'updated_at': 2.0}],
))
self.assertEqual(scanner.get_dockerhub_tag_cache('owner/repo', None, 1), [repointed])
self.assertFalse(scanner.put_dockerhub_tag_cache(
'owner/mutable', None, 1, 'ok', ['owner/mutable:latest'], 3600,
))
self.assertIsNone(scanner.get_dockerhub_tag_cache('owner/mutable', None, 1))
def test_concurrent_cache_writes_are_serialized_and_reads_do_not_maintain(self):
with tempfile.TemporaryDirectory() as temp_dir:
path = os.path.join(temp_dir, 'docker-cache.sqlite')
options = (
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_path', path),
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_rows', 100),
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_age_sec', 3600),
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_max_bytes', 16 * 1024 * 1024),
mock.patch.object(scanner.scan_config, 'dockerhub_tag_cache_min_free_bytes', 0),
)
with options[0], options[1], options[2], options[3], options[4]:
scanner._dockerhub_tag_cache_initialized.discard(path)
digest = 'sha256:' + ('8' * 64)
first = f'owner/repo-0@{digest}'
self.assertTrue(scanner.put_dockerhub_tag_cache(
'owner/repo-0', None, 1, 'ok', [first], 3600,
tag_records=[{'name': 'latest', 'target': first, 'updated_at': 1.0}],
))
with mock.patch.object(scanner, 'maintain_dockerhub_tag_cache') as maintain:
self.assertEqual(scanner.get_dockerhub_tag_cache('owner/repo-0', None, 1), [first])
maintain.assert_not_called()
barrier = threading.Barrier(12)
outcomes = []
outcome_lock = threading.Lock()
def write(index):
target = f'owner/repo-{index}@{digest}'
barrier.wait()
result = scanner.put_dockerhub_tag_cache(
f'owner/repo-{index}', None, 1, 'ok', [target], 3600,
tag_records=[{'name': 'latest', 'target': target, 'updated_at': float(index)}],
)
with outcome_lock:
outcomes.append(result)
threads = [threading.Thread(target=write, args=(index,)) for index in range(1, 13)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(10)
self.assertFalse(thread.is_alive())
self.assertEqual(outcomes, [True] * 12)
def test_scan_docker_image_uses_only_digest_pinned_image_argument(self):
digest = 'sha256:' + ('6' * 64)
image = f'registry.example.com:5000/ns/image:Latest@{digest}'
target = serialize_docker_tag_target(image, '2026-07-20T00:00:00Z')
completed = json.dumps({'level': 'info-0', 'msg': 'finished scanning'})
with mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \
mock.patch.object(scanner, 'run_command_streamed', return_value=scanner.streamed_output_from_text('', completed, 0)) as run:
result = scanner.scan_docker_image(target, trufflehog_concurrency=4)
self.assertEqual(result['errors'], [])
command = run.call_args.args[0]
self.assertEqual(command[command.index('--image') + 1], image)
self.assertNotIn(target, command)
self.assertIn('--local-dev', command)
self.assertIn('--no-update', command)
self.assertEqual(command[command.index('--concurrency') + 1], '4')
with mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \
mock.patch.object(scanner, 'run_command_streamed', return_value=scanner.streamed_output_from_text('', '', 0)) as run:
scanner.scan_huggingface_space('owner/space')
self.assertNotIn('--local-dev', run.call_args.args[0])
configured = console_runner.load_config(str(APP_DIR / 'config.yaml'))
args = console_runner.build_args_from_source_config(
'dockerhub', configured['sources']['dockerhub'], configured['global'], 'test',
)
with mock.patch('builtins.print'):
options = console_runner.prepare_scan_options(args, 1)
self.assertEqual(args.timeout, 600)
self.assertEqual(args.docker_images_per_repository, 3)
self.assertEqual(args.docker_repository_refresh_interval_sec, 86400)
self.assertEqual(args.docker_repository_refresh_max_per_cycle, 0)
self.assertEqual(options['trufflehog_concurrency'], 4)
with mock.patch.object(scanner, 'run_command_streamed') as rejected_run:
for mutable in (
'owner/repo',
'owner/repo:latest',
'{"image":"owner/repo:latest","revision":"2026-07-20T00:00:00Z","schema":"docker-tag-v1"}',
):
with self.subTest(mutable=mutable):
rejected = scanner.scan_docker_image(mutable)
self.assertEqual(rejected['error_class'], 'invalid_target')
self.assertFalse(rejected['retryable'])
rejected_run.assert_not_called()
def test_timestamp_cannot_substitute_for_digest_and_config_requires_digest(self):
with self.assertRaisesRegex(ValueError, 'sha256 digest'):
serialize_docker_tag_target('owner/repo:latest', '2026-07-20T01:02:03Z')
configured = console_runner.load_config(str(APP_DIR / 'config.yaml'))
self.assertIs(configured['sources']['dockerhub']['require_digest'], True)
if __name__ == '__main__':
unittest.main()
Reference in New Issue View Git Blame Copy Permalink