from datetime import datetime from pathlib import Path import concurrent.futures import copy import sys import threading import time from types import SimpleNamespace import unittest from unittest import mock ROOT = Path(__file__).resolve().parents[1] APP_DIR = ROOT / 'app' sys.path.insert(0, str(APP_DIR)) import console_runner import scanner class Response: def __init__(self, status_code=200, payload=None, headers=None): self.status_code = status_code self.payload = payload if payload is not None else {} self.headers = headers or {} self.text = '' self.content = b'' def raise_for_status(self): if self.status_code >= 400: raise RuntimeError(f'HTTP {self.status_code}') def json(self): return self.payload def close(self): return None def manager_with_accounts(count=1): manager = scanner.DockerTokenManager() manager.accounts = [ scanner.DockerAccount( f'account-{index}', f'user-{index}', f'fixture-secret-{index}', '', ) for index in range(count) ] manager.explicit_pool = True return manager class DockerHubSearchAuthTests(unittest.TestCase): def test_search_uses_hub_bearer_and_bounded_request_attempts(self): manager = manager_with_accounts() responses = [ Response(payload={'access_token': 'fixture-bearer', 'expires_in': 600}), Response(payload={'count': 0, 'results': []}), ] with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner, 'api_request', side_effect=responses) as request, \ mock.patch.object( manager, 'report_success', wraps=manager.report_success, ) as report_success: response = scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 3, 'page_size': 100}, ) self.assertEqual(response.status_code, 200) self.assertEqual(request.call_count, 2) search_call = request.call_args_list[1] self.assertEqual( search_call.kwargs['headers']['Authorization'], 'Bearer fixture-bearer', ) self.assertEqual(search_call.kwargs['max_retries'], 1) self.assertTrue(any( call.args[1] == 'hub_search' for call in report_success.call_args_list )) self.assertNotIn(('account-0', 'hub_search'), manager.cooldown_until) self.assertNotIn(('account-0', 'hub_tags'), manager.cooldown_until) def test_explicit_empty_pool_fails_without_anonymous_request(self): manager = scanner.DockerTokenManager() manager.explicit_pool = True with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner, 'api_request') as request: with self.assertRaisesRegex( scanner.DockerRemoteAccessError, 'authentication is unavailable', ): scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) request.assert_not_called() def test_search_refreshes_once_after_401(self): manager = manager_with_accounts() responses = [ Response(payload={'access_token': 'old-bearer'}), Response(status_code=401), Response(payload={'access_token': 'new-bearer'}), Response(payload={'count': 0, 'results': []}), ] with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner, 'api_request', side_effect=responses) as request: scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) self.assertEqual(request.call_count, 4) self.assertEqual( request.call_args_list[1].kwargs['headers']['Authorization'], 'Bearer old-bearer', ) self.assertEqual( request.call_args_list[3].kwargs['headers']['Authorization'], 'Bearer new-bearer', ) def test_search_rotates_and_cools_account_after_403_or_429(self): for status_code, expected_category in ((403, 'auth_forbidden'), (429, 'rate_limit')): with self.subTest(status_code=status_code): manager = manager_with_accounts(2) responses = [ Response(payload={'access_token': 'bearer-a'}), Response(status_code=status_code), Response(payload={'access_token': 'bearer-b'}), Response(payload={'count': 0, 'results': []}), ] with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner, 'api_request', side_effect=responses) as request: scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) self.assertEqual(request.call_count, 4) self.assertEqual( manager.cooldown_categories[('account-0', 'hub_search')], expected_category, ) self.assertEqual( request.call_args_list[3].kwargs['headers']['Authorization'], 'Bearer bearer-b', ) def test_transient_page_failure_gets_exactly_two_attempts(self): manager = manager_with_accounts() manager.cache_hub_token('account-0', 'fixture-bearer', 600) timeout = scanner.requests.exceptions.ReadTimeout('fixture timeout') response = Response(payload={'count': 0, 'results': []}) with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \ mock.patch.object( scanner.requests, 'request', side_effect=[timeout, response], ) as request, \ mock.patch.object(scanner, '_wait_or_raise_scan_slot_fatal'): returned = scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) self.assertIs(returned, response) self.assertEqual(request.call_count, 2) def test_page_attempt_budget_does_not_reset_after_account_rotation(self): manager = manager_with_accounts(3) for index in range(3): manager.cache_hub_token(f'account-{index}', f'bearer-{index}', 600) with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object( scanner, 'api_request', side_effect=[ Response(status_code=403), scanner.ApiRequestError('fixture transient failure'), ], ) as request: with self.assertRaises(scanner.ApiRequestError): scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) self.assertEqual(request.call_count, 2) self.assertEqual( request.call_args_list[1].kwargs['headers']['Authorization'], 'Bearer bearer-1', ) def test_search_pool_exhaustion_does_not_set_tag_backoff(self): manager = manager_with_accounts(2) for index in range(2): manager.cache_hub_token(f'account-{index}', f'bearer-{index}', 600) with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object( scanner, 'api_request', side_effect=[Response(status_code=429), Response(status_code=429)], ), \ mock.patch.object(scanner, 'put_dockerhub_tags_rate_limit') as tag_backoff: with self.assertRaisesRegex( scanner.DockerRemoteAccessError, 'accounts are rate-limited', ): scanner.dockerhub_search_response( 'https://hub.docker.com/v2/search/repositories', {'query': 'fixture', 'page': 1, 'page_size': 100}, ) tag_backoff.assert_not_called() self.assertTrue(manager.all_unavailable('hub_search')) self.assertFalse(manager.all_unavailable('hub_tags')) def test_concurrent_pages_share_token_acquisition(self): manager = manager_with_accounts() def token_response(*_args, **_kwargs): time.sleep(0.02) return Response(payload={'access_token': 'shared-bearer'}) with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object( scanner, 'api_request', side_effect=token_response, ) as request, \ concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: futures = [ executor.submit( scanner._docker_hub_access_token, manager.accounts[0], False, 'hub_search', ) for _ in range(2) ] tokens = [future.result() for future in futures] self.assertEqual(tokens, ['shared-bearer', 'shared-bearer']) self.assertEqual(request.call_count, 1) def test_search_event_persists_without_disabling_tag_account(self): source_config = { 'auth_pool': 'dockerhub_main', 'rate_limit_cooldown': 300, } secrets = {'auth_pools': {'dockerhub_main': [{ 'name': 'account-0', 'username': 'user-0', 'token': 'fixture-secret', }]}} state = {'sources': {'dockerhub': console_runner.default_source_state()}} events = [{ 'name': 'account-0', 'endpoint': 'hub_search', 'category': 'rate_limit', 'reset_at': '2099-01-01T00:00:00+00:00', 'message': 'Docker hub_search HTTP 429', }] with mock.patch.object( console_runner, 'drain_docker_auth_events', return_value=events, ): console_runner.persist_docker_auth_events( 'dockerhub', source_config, state, secrets, ) source_state = state['sources']['dockerhub'] self.assertEqual(source_state['auth_status'].get('account-0'), {}) self.assertTrue(console_runner.auth_entry_is_available( 'dockerhub', secrets['auth_pools']['dockerhub_main'][0], state, )) endpoint_state = source_state['auth_endpoint_status']['hub_search']['account-0'] self.assertEqual(endpoint_state['disabled_reason'], 'rate_limit') self.assertNotIn('fixture-secret', repr(state)) manager = manager_with_accounts() manager.restore_endpoint_cooldowns(source_state['auth_endpoint_status']) self.assertTrue(manager.all_unavailable('hub_search')) self.assertFalse(manager.all_unavailable('hub_tags')) def test_auth_invalid_remains_global_after_late_endpoint_events(self): manager = manager_with_accounts() account = manager.accounts[0] manager.report_http_status( account, 'hub_search', 401, Response(status_code=401), 'auth_invalid', ) manager.report_http_status( account, 'hub_tags', 429, Response(status_code=429), 'rate_limit', ) manager.report_success(account, 'registry') self.assertTrue(manager.all_unavailable('hub_search')) self.assertTrue(manager.all_unavailable('hub_tags')) self.assertTrue(manager.all_unavailable('registry')) self.assertEqual( manager.cooldown_categories[('account-0', 'hub_search')], 'auth_invalid', ) self.assertEqual( [(event['endpoint'], event['category']) for event in manager.drain_status_events()], [('hub_search', 'auth_invalid')], ) def test_auth_invalid_account_is_excluded_from_cli_config_rotation(self): manager = scanner.DockerTokenManager() manager.accounts = [ scanner.DockerAccount('account-0', 'user-0', 'secret-0', 'config-0'), scanner.DockerAccount('account-1', 'user-1', 'secret-1', 'config-1'), ] manager.report_http_status( manager.accounts[0], 'hub_search', 401, Response(status_code=401), 'auth_invalid', ) self.assertEqual(manager.get_next_config(), 'config-1') self.assertEqual(manager.get_next_config(), 'config-1') manager.report_http_status( manager.accounts[1], 'registry', 401, Response(status_code=401), 'auth_invalid', ) self.assertIsNone(manager.get_next_config()) def test_persisted_auth_invalid_cannot_be_downgraded(self): source_config = { 'auth_pool': 'dockerhub_main', 'rate_limit_cooldown': 300, } secrets = {'auth_pools': {'dockerhub_main': [{ 'name': 'account-0', 'username': 'user-0', 'token': 'fixture-secret', }]}} state = {'sources': {'dockerhub': console_runner.default_source_state()}} events = [ { 'name': 'account-0', 'endpoint': 'hub_search', 'category': 'auth_invalid', 'reset_at': 'manual', 'message': 'Docker hub_search HTTP 401', }, { 'name': 'account-0', 'endpoint': 'hub_tags', 'category': 'rate_limit', 'reset_at': '2099-01-01T00:00:00+00:00', 'message': 'Docker hub_tags HTTP 429', }, ] with mock.patch.object( console_runner, 'drain_docker_auth_events', return_value=events, ): console_runner.persist_docker_auth_events( 'dockerhub', source_config, state, secrets, ) status = state['sources']['dockerhub']['auth_status']['account-0'] self.assertEqual(status['status'], 'dead') self.assertEqual(status['disabled_until'], 'manual') self.assertEqual(status['disabled_reason'], 'auth_invalid') self.assertNotIn('fixture-secret', repr(state)) class DockerHubSearchPageTests(unittest.TestCase): def test_page_helper_returns_validated_structured_data(self): response = Response(payload={ 'count': '26', 'results': [{ 'repo_name': 'synthetic/repository-a', 'last_updated': '2026-01-02T03:04:05Z', 'url': 'raw-url-marker', 'token': 'raw-token-marker', }], }) with mock.patch.object( scanner, 'dockerhub_search_response', return_value=response, ) as search, mock.patch.object( scanner.time, 'perf_counter', side_effect=[10.0, 10.25], ): result = scanner.fetch_dockerhub_search_page( 'synthetic-query', 2, per_page=25, sort_by='name', sort_order='asc', request_timeout=9, ) self.assertEqual(result, { 'page': 2, 'repositories': [{ 'repo_name': 'synthetic/repository-a', 'last_updated': '2026-01-02T03:04:05Z', }], 'total_count': 26, 'elapsed': 0.25, }) search.assert_called_once_with( 'https://hub.docker.com/v2/search/repositories', { 'query': 'synthetic-query', 'page': 2, 'page_size': 25, 'sort': 'name', 'order': 'asc', }, request_timeout=9, ) self.assertNotIn('raw-url-marker', repr(result)) self.assertNotIn('raw-token-marker', repr(result)) def test_page_helper_rejects_invalid_counts(self): for count in (None, -1, True, 1.5, float('inf'), 'not-a-count'): with self.subTest(count=count), mock.patch.object( scanner, 'dockerhub_search_response', return_value=Response(payload={'count': count, 'results': []}), ): with self.assertRaisesRegex( scanner.DockerHubDiscoveryTransportError, 'invalid result count', ): scanner.fetch_dockerhub_search_page('synthetic-query', 1) def test_page_helper_rejects_count_below_absolute_result_bound(self): response = Response(payload={ 'count': 25, 'results': [{'repo_name': 'synthetic/repository-a'}], }) with mock.patch.object( scanner, 'dockerhub_search_response', return_value=response, ), self.assertRaisesRegex( scanner.DockerHubDiscoveryTransportError, 'incoherent pagination evidence', ): scanner.fetch_dockerhub_search_page( 'synthetic-query', 2, per_page=25, ) def test_page_helper_rejects_malformed_payloads(self): payloads = ( [], {'count': 1}, {'count': 1, 'results': {}}, {'count': 1, 'results': [None]}, {'count': 1, 'results': [{}]}, {'count': 1, 'results': [{'repo_name': ''}]}, {'count': 1, 'results': [{'repo_name': ' '}]}, {'count': 1, 'results': [{'repo_name': 42}]}, ) for payload in payloads: with self.subTest(payload=payload), mock.patch.object( scanner, 'dockerhub_search_response', return_value=Response(payload=payload), ): with self.assertRaisesRegex( scanner.DockerHubDiscoveryTransportError, 'invalid payload', ): scanner.fetch_dockerhub_search_page('synthetic-query', 1) def test_page_helper_deduplicates_names_in_first_seen_order(self): response = Response(payload={ 'count': 3, 'results': [ { 'repo_name': 'synthetic/repository-a', 'last_updated': '2026-01-03T00:00:00Z', }, { 'repo_name': 'synthetic/repository-b', 'last_modified': '2026-01-02T00:00:00Z', }, { 'repo_name': 'synthetic/repository-a', 'last_updated': '2026-01-01T00:00:00Z', }, ], }) with mock.patch.object( scanner, 'dockerhub_search_response', return_value=response, ): result = scanner.fetch_dockerhub_search_page('synthetic-query', 1) self.assertEqual(result['repositories'], [ { 'repo_name': 'synthetic/repository-a', 'last_updated': '2026-01-03T00:00:00Z', }, { 'repo_name': 'synthetic/repository-b', 'last_modified': '2026-01-02T00:00:00Z', }, ]) def test_page_helper_preserves_explicit_pool_fail_closed(self): manager = scanner.DockerTokenManager() manager.explicit_pool = True with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object(scanner, 'api_request') as request: with self.assertRaisesRegex( scanner.DockerHubDiscoveryTransportError, 'failed after bounded attempts', ): scanner.fetch_dockerhub_search_page('synthetic-query', 1) request.assert_not_called() def test_page_helper_preserves_two_get_authenticated_budget(self): manager = manager_with_accounts() manager.cache_hub_token('account-0', 'synthetic-bearer', 600) with mock.patch.object(scanner, 'docker_token_manager', manager), \ mock.patch.object( scanner, 'api_request', side_effect=[ scanner.ApiRequestError('raw-url-marker'), scanner.ApiRequestError('raw-token-marker'), ], ) as request, \ mock.patch.object(scanner, '_wait_or_raise_scan_slot_fatal'): with self.assertRaises( scanner.DockerHubDiscoveryTransportError, ) as raised: scanner.fetch_dockerhub_search_page('synthetic-query', 1) self.assertEqual(request.call_count, 2) self.assertTrue(all( call.kwargs['max_retries'] == 1 for call in request.call_args_list )) self.assertNotIn('raw-url-marker', str(raised.exception)) self.assertNotIn('raw-token-marker', str(raised.exception)) class DockerHubSearchPaginationTests(unittest.TestCase): @staticmethod def response(page, count): return Response(payload={ 'count': count, 'results': [{'repo_name': f'owner/repo-{page}'}], }) def test_search_caps_at_thirty_pages_and_preserves_page_order(self): requested_pages = [] lock = threading.Lock() def search(_url, params, request_timeout=15): page = int(params['page']) with lock: requested_pages.append(page) if page == 2: time.sleep(0.02) return self.response(page, 4000) with mock.patch.object(scanner, 'dockerhub_search_response', side_effect=search): repos = scanner.fetch_dockerhub_images( 'fixture', pages=40, per_page=100, fetch_workers=8, resolve_tags=False, ) self.assertEqual(requested_pages[0], 1) self.assertEqual(sorted(requested_pages), list(range(1, 31))) self.assertEqual( repos, [f'owner/repo-{page}' for page in range(1, 31)], ) def test_page_one_count_limits_expected_request_range(self): requested_pages = [] def search(_url, params, request_timeout=15): page = int(params['page']) requested_pages.append(page) return self.response(page, 250) with mock.patch.object(scanner, 'dockerhub_search_response', side_effect=search): repos = scanner.fetch_dockerhub_images( 'fixture', pages=30, per_page=100, resolve_tags=False, ) self.assertEqual(sorted(requested_pages), [1, 2, 3]) self.assertEqual(repos, [ 'owner/repo-1', 'owner/repo-2', 'owner/repo-3', ]) def test_invalid_page_one_count_fails_closed(self): for payload in ( {'results': []}, {'count': -1, 'results': []}, ): with self.subTest(payload=payload), mock.patch.object( scanner, 'dockerhub_search_response', return_value=Response(payload=payload), ): with self.assertRaises(scanner.DockerHubDiscoveryTransportError): scanner.fetch_dockerhub_images( 'fixture', pages=30, per_page=100, resolve_tags=False, ) def test_expected_page_failure_rejects_all_partial_results(self): def search(_url, params, request_timeout=15): page = int(params['page']) if page == 2: raise scanner.ApiRequestError('bounded fixture failure') return self.response(page, 300) with mock.patch.object(scanner, 'dockerhub_search_response', side_effect=search), \ mock.patch.object(scanner, 'fetch_dockerhub_tags') as tags: with self.assertRaisesRegex( scanner.DockerHubDiscoveryTransportError, 'pagination incomplete', ): scanner.fetch_dockerhub_images( 'fixture', pages=3, per_page=100, resolve_tags=True, ) tags.assert_not_called() def test_recent_search_uses_authenticated_response_path(self): response = Response(payload={'count': 0, 'results': [], 'next': None}) with mock.patch.object( scanner, 'dockerhub_search_response', return_value=response, ) as search: repos = scanner.fetch_recent_dockerhub_images( 'fixture', datetime(2026, 1, 1), pages=30, per_page=100, resolve_tags=False, ) self.assertEqual(repos, []) self.assertEqual(search.call_count, 1) self.assertEqual(search.call_args.args[1]['page'], 1) self.assertEqual(search.call_args.kwargs['request_timeout'], 30) def test_recent_search_preserves_page_update_metadata(self): response = Response(payload={ 'count': 1, 'results': [{ 'repo_name': 'synthetic/repository-a', 'last_updated': '2025-12-31T00:00:00Z', }], }) with mock.patch.object( scanner, 'dockerhub_search_response', return_value=response, ), mock.patch.object(scanner, 'fetch_dockerhub_last_updated') as metadata: repos = scanner.fetch_recent_dockerhub_images( 'synthetic-query', datetime(2026, 1, 1), pages=1, per_page=100, resolve_tags=False, ) self.assertEqual(repos, []) metadata.assert_not_called() class DockerHubSearchCycleTests(unittest.TestCase): class Connection: is_postgres = True def __init__(self, events): self.events = events def rollback(self): self.events.append('rollback') class DB: def __init__(self, events): self.conn = DockerHubSearchCycleTests.Connection(events) self.events = events self.finished = [] def start_source_cycle(self, *args, **kwargs): return 7 def finish_source_cycle(self, *args, **kwargs): self.events.append('finish') self.finished.append((args, kwargs)) @staticmethod def state(): source_state = console_runner.default_source_state() return {'sources': {'dockerhub': source_state}} @staticmethod def patches(args, result): run_cycle = ( mock.patch.object(console_runner, 'run_cycle', side_effect=result) if isinstance(result, BaseException) else mock.patch.object(console_runner, 'run_cycle', return_value=result) ) return ( mock.patch.object(console_runner, 'save_state'), mock.patch.object( console_runner, 'select_auth_entry', return_value={'name': 'docker-1', 'token': 'fixture'}, ), mock.patch.object(console_runner, 'refresh_auth_summary'), mock.patch.object( console_runner, 'build_args_from_source_config', return_value=args, ), mock.patch.object(console_runner, 'configure_source_auth'), mock.patch.object(console_runner, 'persist_docker_auth_events'), mock.patch.object(console_runner, 'queue_counts_for_args', return_value={}), run_cycle, ) def run_source(self, state, db, run_result): args = SimpleNamespace(platform='docker', mode='search') patches = self.patches(args, run_result) with patches[0], patches[1], patches[2], patches[3], \ patches[4], patches[5], patches[6], patches[7]: return console_runner.run_configured_source( 'dockerhub', {'global': {}, 'sources': {'dockerhub': { 'queries': ['first', 'second'], 'mode': 'search', }}}, state, 'state.json', {}, db, run_id=3, ) def test_failed_pagination_finishes_cycle_without_advancing(self): events = [] db = self.DB(events) state = self.state() metrics = self.run_source( state, db, scanner.DockerHubDiscoveryTransportError('bounded failure'), ) self.assertEqual(events, ['rollback', 'finish']) self.assertTrue(metrics['discovery_transport_failed']) self.assertEqual(state['sources']['dockerhub']['query_index'], 0) self.assertEqual(state['sources']['dockerhub']['last_status'], 'failed') self.assertEqual(db.finished[0][0][1], 'failed') def test_successful_cycle_keeps_existing_query_advance(self): events = [] db = self.DB(events) state = self.state() metrics = self.run_source( state, db, {'fetched_count': 0, 'scanned_count': 0}, ) self.assertEqual(metrics['fetched_count'], 0) self.assertEqual(state['sources']['dockerhub']['query_index'], 1) self.assertEqual(state['sources']['dockerhub']['last_status'], 'completed') if __name__ == '__main__': unittest.main()