Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,753 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP_DIR = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP_DIR))
|
||||
|
||||
import console_runner
|
||||
from docker_depth_experiment import (
|
||||
DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
DOCKER_DEPTH_SELECTOR_VERSION,
|
||||
canonical_ordered_query_hash,
|
||||
validate_docker_depth_config,
|
||||
)
|
||||
import scanner_db
|
||||
from scanner_db import ScannerDB
|
||||
|
||||
|
||||
POLICY_SHA256 = 'a' * 64
|
||||
|
||||
|
||||
class DockerDiscoveryProvenanceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.environment = mock.patch.dict(
|
||||
os.environ, {'SCANNER_DB_URL': '', 'DATABASE_URL': ''},
|
||||
)
|
||||
self.environment.start()
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.db = ScannerDB(db_path=os.path.join(self.temp.name, 'scanner.db'))
|
||||
self.run_id = self.db.start_run('test')
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close()
|
||||
self.temp.cleanup()
|
||||
self.environment.stop()
|
||||
|
||||
def cycle(self, query, ordinal, count):
|
||||
return self.db.start_source_cycle(
|
||||
self.run_id, 'dockerhub', 'docker', 'search', query,
|
||||
ordinal + 1, count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def observation(
|
||||
queries, ordinal, cycle_id, *, page_number=1, per_page=100,
|
||||
pass_kind='deep', query_complete=False, total_count=None,
|
||||
):
|
||||
if total_count is None:
|
||||
total_count = (
|
||||
page_number * per_page
|
||||
if query_complete else (page_number + 1) * per_page
|
||||
)
|
||||
return {
|
||||
'cycle_id': cycle_id,
|
||||
'query_ordinal': ordinal,
|
||||
'query_count': len(queries),
|
||||
'page_number': page_number,
|
||||
'page_limit': 30,
|
||||
'per_page': per_page,
|
||||
'total_count': total_count,
|
||||
'policy_sha256': POLICY_SHA256,
|
||||
'pass_kind': pass_kind,
|
||||
'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
'ordered_query_hash': canonical_ordered_query_hash(queries),
|
||||
'query_complete': query_complete,
|
||||
}
|
||||
|
||||
def scalar(self, sql, params=()):
|
||||
return self.db.conn.execute(sql, params).fetchone()[0]
|
||||
|
||||
def test_current_collection_generation_overrides_old_runner_deep_marker(self):
|
||||
queries = ('generation-query',)
|
||||
ordered_hash = canonical_ordered_query_hash(queries)
|
||||
now = datetime.now(timezone.utc)
|
||||
state = console_runner.default_source_state()
|
||||
policies = {'generation-query': {'policy_sha256': POLICY_SHA256}}
|
||||
console_runner.mark_dockerhub_deep_dispatched(
|
||||
state, queries[0], POLICY_SHA256, now,
|
||||
)
|
||||
state_path = Path(self.temp.name) / 'runner-state.json'
|
||||
state_path.write_text(json.dumps({
|
||||
'version': 1, 'sources': {'dockerhub': state},
|
||||
}), encoding='utf-8')
|
||||
state = console_runner.load_state(str(state_path), {
|
||||
'sources': {'dockerhub': {}},
|
||||
})['sources']['dockerhub']
|
||||
complete = self.db.dockerhub_discovery_generation_complete(
|
||||
'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
ordered_hash, 1, POLICY_SHA256,
|
||||
)
|
||||
self.assertFalse(complete)
|
||||
self.assertTrue(console_runner.prepare_dockerhub_discovery_state(
|
||||
state, policies, queries[0], now + timedelta(hours=1),
|
||||
force_deep=not complete,
|
||||
)['deep'])
|
||||
|
||||
timestamp = scanner_db.utc_now_iso()
|
||||
self.db.conn.execute(
|
||||
'''INSERT INTO docker_discovery_passes(
|
||||
pass_token, source, pass_kind, collection_generation,
|
||||
policy_sha256, ordered_queries_sha256, expected_query_count,
|
||||
completed_query_count, state, started_at, completed_at,
|
||||
created_at, updated_at
|
||||
) VALUES ('legacy-generation-pass', 'dockerhub', 'deep', 'legacy',
|
||||
?, ?, 1, 1, 'complete', ?, ?, ?, ?)''',
|
||||
(POLICY_SHA256, ordered_hash, timestamp, timestamp, timestamp, timestamp),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
self.assertFalse(self.db.dockerhub_discovery_generation_complete(
|
||||
'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
ordered_hash, 1, POLICY_SHA256,
|
||||
))
|
||||
current_pass = self.db.conn.execute(
|
||||
'''INSERT INTO docker_discovery_passes(
|
||||
pass_token, source, pass_kind, collection_generation,
|
||||
policy_sha256, ordered_queries_sha256, expected_query_count,
|
||||
completed_query_count, state, started_at, completed_at,
|
||||
created_at, updated_at
|
||||
) VALUES ('current-generation-pass', 'dockerhub', 'deep', ?,
|
||||
?, ?, 1, 1, 'complete', ?, ?, ?, ?) RETURNING id''',
|
||||
(
|
||||
DOCKER_DEPTH_COLLECTION_GENERATION, POLICY_SHA256, ordered_hash,
|
||||
timestamp, timestamp, timestamp, timestamp,
|
||||
),
|
||||
).fetchone()
|
||||
self.db.conn.execute(
|
||||
'''INSERT INTO docker_discovery_pages(
|
||||
pass_id, query, query_ordinal, page_number, result_count,
|
||||
total_count, admitted_count, query_complete, admission_kind,
|
||||
page_sha256, observed_at, created_at
|
||||
) VALUES (?, 'generation-query', 0, 1, 0, 0, 0, 1, 'main',
|
||||
?, ?, ?)''',
|
||||
(current_pass['id'], '7' * 64, timestamp, timestamp),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
complete = self.db.dockerhub_discovery_generation_complete(
|
||||
'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
ordered_hash, 1, POLICY_SHA256,
|
||||
)
|
||||
self.assertTrue(complete)
|
||||
self.assertFalse(console_runner.prepare_dockerhub_discovery_state(
|
||||
state, policies, queries[0], now + timedelta(hours=1),
|
||||
force_deep=not complete,
|
||||
)['deep'])
|
||||
|
||||
def test_main_provenance_failure_rolls_back_page_pass_and_queue_admission(self):
|
||||
queries = ('atomic-main',)
|
||||
cycle_id = self.cycle(queries[0], 0, 1)
|
||||
self.db.conn.execute(
|
||||
'''CREATE TRIGGER reject_fresh_provenance
|
||||
BEFORE INSERT ON docker_repository_query_provenance
|
||||
BEGIN SELECT RAISE(ABORT, 'forced provenance failure'); END'''
|
||||
)
|
||||
self.db.conn.commit()
|
||||
|
||||
with self.assertRaisesRegex(sqlite3.IntegrityError, 'forced provenance failure'):
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/atomic-main'],
|
||||
observation=self.observation(
|
||||
queries, 0, cycle_id, query_complete=True,
|
||||
),
|
||||
)
|
||||
|
||||
for table in (
|
||||
'target_queue', 'docker_discovery_passes', 'docker_discovery_pages',
|
||||
'docker_repository_query_provenance',
|
||||
'docker_repository_query_observations',
|
||||
):
|
||||
self.assertEqual(self.scalar(f'SELECT COUNT(*) FROM {table}'), 0, table)
|
||||
|
||||
def test_main_pages_preserve_shared_query_relations_absolute_rank_and_replay(self):
|
||||
queries = ('first-query', 'second-query')
|
||||
first_cycle = self.cycle(queries[0], 0, len(queries))
|
||||
second_cycle = self.cycle(queries[1], 1, len(queries))
|
||||
first_page = self.observation(queries, 0, first_cycle)
|
||||
second_page = self.observation(
|
||||
queries, 0, first_cycle, page_number=2, query_complete=True,
|
||||
)
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/page-one'], observation=first_page,
|
||||
)
|
||||
page_two_repositories = [
|
||||
'owner/page-two', 'owner/shared', 'owner/shared',
|
||||
]
|
||||
page_two = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], page_two_repositories,
|
||||
observation=second_page,
|
||||
)
|
||||
final = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[1], ['owner/shared'],
|
||||
observation=self.observation(
|
||||
queries, 1, second_cycle, query_complete=True,
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(page_two['pass_id'], final['pass_id'])
|
||||
self.assertTrue(final['pass_complete'])
|
||||
pages = self.db.conn.execute(
|
||||
'''SELECT query, source_cycle_id, retry_work_id, admission_kind
|
||||
FROM docker_discovery_pages
|
||||
WHERE pass_id = ? ORDER BY query_ordinal, page_number''',
|
||||
(final['pass_id'],),
|
||||
).fetchall()
|
||||
self.assertEqual([tuple(row) for row in pages], [
|
||||
('first-query', first_cycle, None, 'main'),
|
||||
('first-query', first_cycle, None, 'main'),
|
||||
('second-query', second_cycle, None, 'main'),
|
||||
])
|
||||
shared_id = self.scalar(
|
||||
"SELECT id FROM target_queue WHERE normalized_target = 'owner/shared'"
|
||||
)
|
||||
self.assertEqual(self.scalar(
|
||||
"SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/shared'"
|
||||
), 1)
|
||||
self.assertEqual(self.scalar(
|
||||
'''SELECT COUNT(*) FROM docker_repository_query_provenance
|
||||
WHERE repository_queue_id = ?''', (shared_id,),
|
||||
), 2)
|
||||
ranks = self.db.conn.execute(
|
||||
'''SELECT query, first_search_rank, best_search_rank, last_search_rank,
|
||||
observation_count, fresh_observation_count,
|
||||
fresh_complete_observation_count, fresh_coverage_eligible
|
||||
FROM docker_repository_query_provenance
|
||||
WHERE repository_queue_id = ? ORDER BY query''',
|
||||
(shared_id,),
|
||||
).fetchall()
|
||||
self.assertEqual(tuple(ranks[0]), (
|
||||
'first-query', 102, 102, 102, 1, 1, 1, 1,
|
||||
))
|
||||
self.assertEqual(tuple(ranks[1]), (
|
||||
'second-query', 1, 1, 1, 1, 1, 1, 1,
|
||||
))
|
||||
|
||||
replay = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], page_two_repositories,
|
||||
observation=second_page,
|
||||
)
|
||||
self.assertEqual(replay['pass_id'], final['pass_id'])
|
||||
self.assertEqual(replay['page_inserted_count'], 0)
|
||||
self.assertEqual(replay['observation_inserted_count'], 0)
|
||||
self.assertEqual(self.scalar(
|
||||
'''SELECT observation_count FROM docker_repository_query_provenance
|
||||
WHERE source = 'dockerhub' AND query = 'first-query'
|
||||
AND repository_queue_id = ?''',
|
||||
(shared_id,),
|
||||
), 1)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, 'page evidence conflicts'):
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/changed'],
|
||||
observation=second_page,
|
||||
)
|
||||
self.assertEqual(self.scalar(
|
||||
"SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/changed'"
|
||||
), 0)
|
||||
|
||||
def test_retry_pages_keep_original_pass_and_policy_under_the_lease(self):
|
||||
queries = ('retry-query',)
|
||||
origin_cycle = self.cycle(queries[0], 0, 1)
|
||||
enqueue = self.db.enqueue_discovery_retry(
|
||||
'dockerhub', queries[0], POLICY_SHA256, 'deep', 'query',
|
||||
page_start=1, page_end=2,
|
||||
observation=self.observation(queries, 0, origin_cycle),
|
||||
)
|
||||
later_cycle = self.cycle(queries[0], 0, 1)
|
||||
coalesced = self.db.enqueue_discovery_retry(
|
||||
'dockerhub', queries[0], POLICY_SHA256, 'deep', 'query',
|
||||
page_start=1, page_end=2,
|
||||
observation=self.observation(queries, 0, later_cycle),
|
||||
)
|
||||
self.assertEqual(coalesced['id'], enqueue['id'])
|
||||
self.assertEqual(coalesced['coalesced_count'], 1)
|
||||
self.assertEqual(coalesced['source_cycle_id'], origin_cycle)
|
||||
claim = self.db.claim_discovery_retries(
|
||||
'dockerhub', {queries[0]: POLICY_SHA256}, 'retry-owner',
|
||||
)[0]
|
||||
self.assertEqual(claim['source_cycle_id'], origin_cycle)
|
||||
unrelated_current_cycle = self.cycle('unrelated-current-query', 0, 1)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, 'retry provenance changed'):
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/wrong-cycle'],
|
||||
retry_id=claim['id'], lease_owner=claim['lease_owner'],
|
||||
lease_token=claim['lease_token'], next_page=2,
|
||||
observation=self.observation(
|
||||
queries, 0, unrelated_current_cycle,
|
||||
),
|
||||
)
|
||||
self.assertEqual(self.scalar(
|
||||
'SELECT next_page FROM discovery_retry_queue WHERE id = ?',
|
||||
(claim['id'],),
|
||||
), 1)
|
||||
self.assertEqual(self.scalar(
|
||||
"SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/wrong-cycle'"
|
||||
), 0)
|
||||
|
||||
page_one = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/retry-one'],
|
||||
retry_id=claim['id'], lease_owner=claim['lease_owner'],
|
||||
lease_token=claim['lease_token'], next_page=2,
|
||||
observation=self.observation(queries, 0, origin_cycle),
|
||||
)
|
||||
page_two = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/retry-two', 'owner/retry-ranked'],
|
||||
retry_id=claim['id'], lease_owner=claim['lease_owner'],
|
||||
lease_token=claim['lease_token'], complete=True,
|
||||
observation=self.observation(
|
||||
queries, 0, origin_cycle, page_number=2, query_complete=True,
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(enqueue['pass_id'], page_one['pass_id'])
|
||||
self.assertEqual(page_one['pass_id'], page_two['pass_id'])
|
||||
self.assertTrue(page_two['pass_complete'])
|
||||
retry_row = self.db.conn.execute(
|
||||
'''SELECT status, source_cycle_id, lease_owner, lease_token,
|
||||
last_error_category, held_at
|
||||
FROM discovery_retry_queue WHERE id = ?''',
|
||||
(claim['id'],),
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(retry_row)[:5], (
|
||||
'held', origin_cycle, None, None, None,
|
||||
))
|
||||
self.assertIsNotNone(retry_row['held_at'])
|
||||
pages = self.db.conn.execute(
|
||||
'''SELECT admission_kind, retry_work_id, source_cycle_id
|
||||
FROM docker_discovery_pages
|
||||
WHERE pass_id = ? ORDER BY page_number''',
|
||||
(page_two['pass_id'],),
|
||||
).fetchall()
|
||||
self.assertEqual([tuple(row) for row in pages], [
|
||||
('retry', claim['id'], origin_cycle),
|
||||
('retry', claim['id'], origin_cycle),
|
||||
])
|
||||
pass_row = self.db.conn.execute(
|
||||
'''SELECT policy_sha256, ordered_queries_sha256, pass_kind, state
|
||||
FROM docker_discovery_passes WHERE id = ?''',
|
||||
(page_two['pass_id'],),
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(pass_row), (
|
||||
POLICY_SHA256, canonical_ordered_query_hash(queries), 'deep', 'complete',
|
||||
))
|
||||
self.assertEqual(self.scalar(
|
||||
'''SELECT search_rank FROM docker_repository_query_observations
|
||||
WHERE repository_queue_id = (
|
||||
SELECT id FROM target_queue
|
||||
WHERE normalized_target = 'owner/retry-ranked'
|
||||
)'''
|
||||
), 102)
|
||||
|
||||
def test_retry_provenance_failure_rolls_back_admission_and_retry_progress(self):
|
||||
queries = ('atomic-retry',)
|
||||
cycle_id = self.cycle(queries[0], 0, 1)
|
||||
retry = self.db.enqueue_discovery_retry(
|
||||
'dockerhub', queries[0], POLICY_SHA256, 'deep', 'page',
|
||||
page_start=1, page_end=1,
|
||||
observation=self.observation(queries, 0, cycle_id),
|
||||
)
|
||||
claim = self.db.claim_discovery_retries(
|
||||
'dockerhub', {queries[0]: POLICY_SHA256}, 'retry-owner',
|
||||
)[0]
|
||||
self.db.conn.execute(
|
||||
'''CREATE TRIGGER reject_retry_provenance
|
||||
BEFORE INSERT ON docker_repository_query_provenance
|
||||
BEGIN SELECT RAISE(ABORT, 'forced retry provenance failure'); END'''
|
||||
)
|
||||
self.db.conn.commit()
|
||||
|
||||
with self.assertRaisesRegex(sqlite3.IntegrityError, 'forced retry provenance failure'):
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/atomic-retry'],
|
||||
retry_id=claim['id'], lease_owner=claim['lease_owner'],
|
||||
lease_token=claim['lease_token'], complete=True,
|
||||
observation=self.observation(
|
||||
queries, 0, cycle_id, query_complete=True,
|
||||
),
|
||||
)
|
||||
|
||||
retry_row = self.db.conn.execute(
|
||||
'''SELECT status, next_page FROM discovery_retry_queue WHERE id = ?''',
|
||||
(retry['id'],),
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(retry_row), ('leased', 1))
|
||||
self.assertEqual(self.scalar(
|
||||
"SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/atomic-retry'"
|
||||
), 0)
|
||||
self.assertEqual(self.scalar('SELECT COUNT(*) FROM docker_discovery_pages'), 0)
|
||||
self.assertEqual(self.scalar(
|
||||
'SELECT completed_query_count FROM docker_discovery_passes WHERE id = ?',
|
||||
(retry['pass_id'],),
|
||||
), 0)
|
||||
|
||||
def test_stale_retry_lease_rolls_back_page_and_preserves_prior_progress(self):
|
||||
queries = ('stale-retry',)
|
||||
origin_cycle = self.cycle(queries[0], 0, 1)
|
||||
retry = self.db.enqueue_discovery_retry(
|
||||
'dockerhub', queries[0], POLICY_SHA256, 'deep', 'range',
|
||||
page_start=1, page_end=2,
|
||||
observation=self.observation(queries, 0, origin_cycle),
|
||||
)
|
||||
stale_claim = self.db.claim_discovery_retries(
|
||||
'dockerhub', {queries[0]: POLICY_SHA256}, 'stale-owner',
|
||||
)[0]
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/stale-first'],
|
||||
retry_id=stale_claim['id'], lease_owner=stale_claim['lease_owner'],
|
||||
lease_token=stale_claim['lease_token'], next_page=2,
|
||||
observation=self.observation(queries, 0, origin_cycle),
|
||||
)
|
||||
self.db.conn.execute(
|
||||
'''UPDATE discovery_retry_queue
|
||||
SET lease_expires_at = '2000-01-01T00:00:00+00:00'
|
||||
WHERE id = ?''',
|
||||
(retry['id'],),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
current_claim = self.db.claim_discovery_retries(
|
||||
'dockerhub', {queries[0]: POLICY_SHA256}, 'current-owner',
|
||||
)[0]
|
||||
|
||||
with self.assertRaises(scanner_db.DiscoveryRetryLeaseError):
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/stale-second'],
|
||||
retry_id=stale_claim['id'], lease_owner=stale_claim['lease_owner'],
|
||||
lease_token=stale_claim['lease_token'], complete=True,
|
||||
observation=self.observation(
|
||||
queries, 0, origin_cycle, page_number=2, query_complete=True,
|
||||
),
|
||||
)
|
||||
|
||||
current = self.db.conn.execute(
|
||||
'''SELECT status, next_page, lease_owner, lease_token, source_cycle_id
|
||||
FROM discovery_retry_queue WHERE id = ?''',
|
||||
(retry['id'],),
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(current), (
|
||||
'leased', 2, 'current-owner', current_claim['lease_token'], origin_cycle,
|
||||
))
|
||||
self.assertEqual(self.scalar('SELECT COUNT(*) FROM docker_discovery_pages'), 1)
|
||||
self.assertEqual(self.scalar(
|
||||
"SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/stale-second'"
|
||||
), 0)
|
||||
|
||||
def test_only_complete_deep_pass_and_not_legacy_evidence_satisfies_coverage(self):
|
||||
queries = ('coverage-query',)
|
||||
deep_cycle = self.cycle(queries[0], 0, 1)
|
||||
ordinary_cycle = self.cycle(queries[0], 0, 1)
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/deep'],
|
||||
observation=self.observation(
|
||||
queries, 0, deep_cycle, query_complete=True,
|
||||
),
|
||||
)
|
||||
self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', queries[0], ['owner/ordinary'],
|
||||
observation=self.observation(
|
||||
queries, 0, ordinary_cycle, pass_kind='ordinary',
|
||||
query_complete=True,
|
||||
),
|
||||
)
|
||||
now = scanner_db.utc_now_iso()
|
||||
legacy_id = self.db.conn.execute(
|
||||
'''INSERT INTO target_queue(
|
||||
source, platform, query, target, normalized_target, status,
|
||||
resolver_state, created_at, updated_at
|
||||
) VALUES ('dockerhub', 'docker', ?, 'owner/legacy', 'owner/legacy',
|
||||
'deferred', 'pending', ?, ?) RETURNING id''',
|
||||
(queries[0], now, now),
|
||||
).fetchone()['id']
|
||||
self.db.conn.execute(
|
||||
'''INSERT INTO docker_repository_query_provenance(
|
||||
source, query, repository_queue_id, provenance_kind,
|
||||
first_observed_at, last_observed_at, created_at, updated_at
|
||||
) VALUES ('dockerhub', ?, ?, 'legacy_queue', ?, ?, ?, ?)''',
|
||||
(queries[0], legacy_id, now, now, now, now),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
|
||||
summary = self.db.dockerhub_discovery_coverage_summary(
|
||||
'dockerhub', queries, canonical_ordered_query_hash(queries),
|
||||
{queries[0]: POLICY_SHA256}, required_repository_count=2,
|
||||
)
|
||||
self.assertEqual(summary['state'], 'collecting')
|
||||
self.assertFalse(summary['planning_allowed'])
|
||||
self.assertEqual(summary['queries'][0]['eligible_repository_count'], 1)
|
||||
ordinary = self.db.conn.execute(
|
||||
'''SELECT fresh_complete_observation_count, fresh_coverage_eligible
|
||||
FROM docker_repository_query_provenance
|
||||
WHERE query = ? AND repository_queue_id = (
|
||||
SELECT id FROM target_queue
|
||||
WHERE normalized_target = 'owner/ordinary'
|
||||
)''',
|
||||
(queries[0],),
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(ordinary), (0, 0))
|
||||
|
||||
def test_coverage_gate_requires_ten_eligible_rows_for_all_61_queries(self):
|
||||
queries = tuple(f'pinned-query-{index:02d}' for index in range(61))
|
||||
ordered_hash = canonical_ordered_query_hash(queries)
|
||||
policies = {query: POLICY_SHA256 for query in queries}
|
||||
for ordinal, query in enumerate(queries):
|
||||
cycle_id = self.cycle(query, ordinal, len(queries))
|
||||
repository_count = 16 if ordinal == 0 else 10
|
||||
repositories = [
|
||||
f'owner/q{ordinal:02d}-repo-{index:02d}'
|
||||
for index in range(repository_count)
|
||||
]
|
||||
result = self.db.persist_dockerhub_discovery_page(
|
||||
'dockerhub', query, repositories,
|
||||
observation=self.observation(
|
||||
queries, ordinal, cycle_id, query_complete=True,
|
||||
),
|
||||
)
|
||||
self.assertTrue(result['pass_complete'])
|
||||
pass_row = self.db.conn.execute(
|
||||
'''SELECT COUNT(*) AS count, MIN(completed_query_count) AS completed,
|
||||
MIN(state) AS state
|
||||
FROM docker_discovery_passes'''
|
||||
).fetchone()
|
||||
self.assertEqual(tuple(pass_row), (1, 61, 'complete'))
|
||||
self.assertTrue(self.db.dockerhub_discovery_generation_complete(
|
||||
'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
ordered_hash, 61, POLICY_SHA256,
|
||||
))
|
||||
|
||||
rows = self.db.conn.execute(
|
||||
'''SELECT id, target FROM target_queue
|
||||
WHERE query = ? ORDER BY target''',
|
||||
(queries[0],),
|
||||
).fetchall()
|
||||
excluded = rows[10:16]
|
||||
digest_target = excluded[0]['target'] + '@sha256:' + ('b' * 64)
|
||||
self.db.conn.execute(
|
||||
'''UPDATE target_queue SET target = ?, normalized_target = ? WHERE id = ?''',
|
||||
(digest_target, digest_target, excluded[0]['id']),
|
||||
)
|
||||
for row, status in zip(excluded[1:5], ('done', 'failed', 'quarantined', 'cold')):
|
||||
self.db.conn.execute(
|
||||
'UPDATE target_queue SET status = ? WHERE id = ?',
|
||||
(status, row['id']),
|
||||
)
|
||||
historical = excluded[5]
|
||||
self.db.conn.execute(
|
||||
'''INSERT INTO target_scans(
|
||||
queue_id, source, query, target, normalized_target, status, created_at
|
||||
) VALUES (?, 'dockerhub', ?, ?, ?, 'clean', ?)''',
|
||||
(
|
||||
historical['id'], queries[0], historical['target'],
|
||||
historical['target'], scanner_db.utc_now_iso(),
|
||||
),
|
||||
)
|
||||
active_id = rows[0]['id']
|
||||
self.db.conn.execute(
|
||||
'''UPDATE target_queue
|
||||
SET lease_owner = 'active-owner', lease_token = 'active-token'
|
||||
WHERE id = ?''',
|
||||
(active_id,),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
|
||||
collecting = self.db.dockerhub_discovery_coverage_summary(
|
||||
'dockerhub', queries, ordered_hash, policies,
|
||||
)
|
||||
self.assertEqual(collecting['state'], 'collecting')
|
||||
self.assertFalse(collecting['planning_allowed'])
|
||||
self.assertEqual(collecting['covered_query_count'], 60)
|
||||
self.assertEqual(collecting['queries'][0]['eligible_repository_count'], 9)
|
||||
|
||||
self.db.conn.execute(
|
||||
'''UPDATE target_queue SET lease_owner = NULL, lease_token = NULL
|
||||
WHERE id = ?''',
|
||||
(active_id,),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
complete = self.db.dockerhub_discovery_coverage_summary(
|
||||
'dockerhub', queries, ordered_hash, policies,
|
||||
)
|
||||
self.assertEqual(complete['state'], 'coverage_complete')
|
||||
self.assertTrue(complete['planning_allowed'])
|
||||
self.assertEqual(complete['covered_query_count'], 61)
|
||||
self.assertEqual(complete['minimum_eligible_repository_count'], 10)
|
||||
|
||||
|
||||
class DockerDiscoveryRunnerMetadataTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def config():
|
||||
queries = [f'runner-query-{index:02d}' for index in range(61)]
|
||||
experiment = {
|
||||
'experiment_key': 'runner-disabled-v1',
|
||||
'enabled': False,
|
||||
'queries': queries,
|
||||
'repositories_per_query': 10,
|
||||
'shallow_images_per_repository': 1,
|
||||
'deep_repositories_per_query': 1,
|
||||
'deep_images_per_repository': 10,
|
||||
'target_limit': 1200,
|
||||
'selector_version': DOCKER_DEPTH_SELECTOR_VERSION,
|
||||
}
|
||||
return {
|
||||
'global': {'sync_file_queues': False},
|
||||
'sources': {'dockerhub': {
|
||||
'queries': queries,
|
||||
'mode': 'search',
|
||||
'pages': 1,
|
||||
'per_page': 100,
|
||||
'require_digest': True,
|
||||
'docker_platform_candidate_tags': 20,
|
||||
'docker_images_per_repository': 3,
|
||||
'docker_depth_experiment': experiment,
|
||||
}},
|
||||
}
|
||||
|
||||
def test_disabled_validated_experiment_still_supplies_page_observation_metadata(self):
|
||||
validated = validate_docker_depth_config(self.config())
|
||||
self.assertFalse(validated.experiment.enabled)
|
||||
args = SimpleNamespace(
|
||||
platform='docker', mode='search', query=validated.experiment.queries[0],
|
||||
pages=1, per_page=100, docker_sort_by='updated_at', sort_order='desc',
|
||||
fetch_timeout=15, dockerhub_discovery_deep=True,
|
||||
)
|
||||
policy = console_runner.dockerhub_discovery_policy(args)
|
||||
args.dockerhub_discovery_policy_sha256 = policy['policy_sha256']
|
||||
console_runner.annotate_dockerhub_experiment_observation_args(
|
||||
args, validated.experiment, cycle_id=77,
|
||||
)
|
||||
|
||||
class DB:
|
||||
observation = None
|
||||
|
||||
@staticmethod
|
||||
def runtime_control_state():
|
||||
return {'effective_discovery_paused': False}
|
||||
|
||||
def persist_dockerhub_discovery_page(self, _source, _query, repositories, **kwargs):
|
||||
self.observation = kwargs.get('observation')
|
||||
return {
|
||||
'attempted_count': len(repositories),
|
||||
'normalized_count': len(repositories),
|
||||
'preexisting_count': 0,
|
||||
'inserted_count': len(repositories),
|
||||
'normalized_repositories': frozenset({'owner/runner'}),
|
||||
'preexisting_repositories': frozenset(),
|
||||
}
|
||||
|
||||
db = DB()
|
||||
response = {
|
||||
'page': 1,
|
||||
'repositories': [{'repo_name': 'owner/runner'}],
|
||||
'total_count': 1,
|
||||
}
|
||||
with mock.patch.object(
|
||||
console_runner, 'fetch_dockerhub_search_page', return_value=response,
|
||||
):
|
||||
console_runner.run_dockerhub_incremental_discovery(args, db)
|
||||
|
||||
self.assertEqual(db.observation, {
|
||||
'cycle_id': 77,
|
||||
'query_ordinal': 0,
|
||||
'query_count': 61,
|
||||
'page_number': 1,
|
||||
'page_limit': 1,
|
||||
'per_page': 100,
|
||||
'total_count': 1,
|
||||
'policy_sha256': policy['policy_sha256'],
|
||||
'pass_kind': 'deep',
|
||||
'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION,
|
||||
'ordered_query_hash': validated.experiment.ordered_query_hash,
|
||||
'query_complete': True,
|
||||
})
|
||||
|
||||
def test_configured_source_integrates_disabled_experiment_before_collection(self):
|
||||
config = self.config()
|
||||
query = config['sources']['dockerhub']['queries'][0]
|
||||
state = {
|
||||
'sources': {'dockerhub': console_runner.default_source_state()},
|
||||
}
|
||||
built_args = SimpleNamespace(platform='docker', mode='search', query=query)
|
||||
seen = []
|
||||
|
||||
class Connection:
|
||||
is_postgres = True
|
||||
|
||||
@staticmethod
|
||||
def rollback():
|
||||
return None
|
||||
|
||||
class DB:
|
||||
conn = Connection()
|
||||
|
||||
@staticmethod
|
||||
def dockerhub_discovery_generation_complete(*_args, **_kwargs):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def start_source_cycle(*_args, **_kwargs):
|
||||
return 77
|
||||
|
||||
@staticmethod
|
||||
def claim_discovery_retries(*_args, **_kwargs):
|
||||
return []
|
||||
|
||||
persist_dockerhub_discovery_page = staticmethod(lambda *_args, **_kwargs: {})
|
||||
finish_discovery_retry = staticmethod(lambda *_args, **_kwargs: {})
|
||||
renew_discovery_retry_lease = staticmethod(lambda *_args, **_kwargs: {})
|
||||
update_discovery_retry = staticmethod(lambda *_args, **_kwargs: {})
|
||||
hold_discovery_retry = staticmethod(lambda *_args, **_kwargs: {})
|
||||
|
||||
def run_cycle(args, *_args, **_kwargs):
|
||||
seen.append(args)
|
||||
return {
|
||||
'cycle_status': 'completed',
|
||||
'deep_dispatch_durable': True,
|
||||
'source_failure_count': 0,
|
||||
'scanned_count': 0,
|
||||
}
|
||||
|
||||
with mock.patch.object(console_runner, 'save_state'), \
|
||||
mock.patch.object(console_runner, 'select_auth_entry', return_value=None), \
|
||||
mock.patch.object(console_runner, 'refresh_auth_summary'), \
|
||||
mock.patch.object(
|
||||
console_runner, 'build_args_from_source_config', return_value=built_args,
|
||||
), \
|
||||
mock.patch.object(console_runner, 'configure_source_auth'), \
|
||||
mock.patch.object(console_runner, 'persist_docker_auth_events'), \
|
||||
mock.patch.object(console_runner, 'queue_counts_for_args', return_value={}), \
|
||||
mock.patch.object(console_runner, 'run_cycle', side_effect=run_cycle):
|
||||
console_runner.run_configured_source(
|
||||
'dockerhub', config, state, 'state.json', {}, DB(), run_id=3,
|
||||
)
|
||||
|
||||
self.assertEqual(len(seen), 1)
|
||||
self.assertTrue(seen[0].docker_depth_collection_only)
|
||||
self.assertEqual(seen[0].dockerhub_discovery_cycle_id, 77)
|
||||
self.assertEqual(seen[0].dockerhub_discovery_query_count, 61)
|
||||
self.assertEqual(
|
||||
seen[0].dockerhub_discovery_ordered_query_hash,
|
||||
canonical_ordered_query_hash(
|
||||
config['sources']['dockerhub']['queries'],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user