sashatrask/truf-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
170dd941b95098efd6d3c06452f866c0e2214833
truf-server/app/scanner_db.py
T
sashatrask 170dd941b9 Initial server source import
2026-09-30 20:30:56 +03:00

30696 lines
1.4 MiB
Plaintext
Raw Blame History

import hashlib
import hmac
import importlib
import ipaddress
import json
import logging
import math
import os
import random
import re
import secrets
import threading
import time
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import urlsplit
from db_backend import (
POSTGRES_APPLICATION_SCHEMA,
connect_host_agent_postgres,
connect_postgres,
connect_sqlite,
database_url_from_env,
is_postgres_url,
redact_database_url,
)
from result_spool import SpoolHashConflictError, prepare_scan_event
from keycheck_candidates import candidate_uid, stored_provider_key_hash
from process_identity import exact_process_identity_state
from target_identity import (
docker_target_identity,
normalize_huggingface_space_id,
parse_dockerhub_digest_target,
parse_docker_target,
postman_target_identity,
validate_docker_image_reference,
)
logger = logging.getLogger(__name__)
REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS = 120
REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS = 60
ADMIN_DISCARDED_QUEUE_REASON = 'discarded stale unassigned backlog by source administrator'
ADMIN_DISCARDED_QUEUE_SQL = (
"target_queue.status = 'quarantined' AND target_queue.last_error = "
f"'{ADMIN_DISCARDED_QUEUE_REASON}'"
)
class RuntimeSafetySchemaError(RuntimeError):
pass
class PipelineCapacityUnavailable(RuntimeError):
pass
class RuntimeControlConflictError(RuntimeError):
pass
class RuntimeControlRevisionConflictError(RuntimeControlConflictError):
def __init__(self, expected_revision, current_state):
self.expected_revision = expected_revision
self.current_state = current_state
super().__init__(
f'runtime control revision changed from {expected_revision} '
f'to {current_state["revision"]}'
)
class RuntimeOperationIdentityConflictError(RuntimeControlConflictError):
pass
class RuntimeOperationTransitionError(RuntimeControlConflictError):
pass
class RuntimeControlTransitionError(RuntimeControlConflictError):
pass
class DiscoveryPausedError(RuntimeError):
def __init__(self, control_state):
self.control_state = dict(control_state)
super().__init__('provider discovery is paused')
class ScanEventConflictError(RuntimeError):
pass
class WorkerObservabilityConflictError(ScanEventConflictError):
pass
class WorkerProgressInactiveError(WorkerObservabilityConflictError):
pass
def _canonical_worker_contract(kind, payload):
validators = {
'progress': ('validate_worker_event', 'validate_progress_event', 'validate_event'),
'diagnostic': (
'validate_worker_diagnostic', 'validate_diagnostic_envelope',
'validate_diagnostic',
),
}
if kind not in validators:
raise ValueError(f'unknown worker contract kind: {kind}')
try:
contracts = importlib.import_module('worker_contracts')
except ModuleNotFoundError as exc:
if exc.name == 'worker_contracts':
raise RuntimeError(
f'worker_contracts is required to persist worker {kind} records'
) from exc
raise RuntimeError(f'worker_contracts import failed: missing {exc.name}') from exc
except ImportError as exc:
raise RuntimeError(f'worker_contracts import failed: {exc}') from exc
codecs = {
'progress': ('decode_worker_event', 'encode_worker_event'),
'diagnostic': ('decode_diagnostic_envelope', 'encode_diagnostic_envelope'),
}
decoder = getattr(contracts, codecs[kind][0], None)
encoder = getattr(contracts, codecs[kind][1], None)
if callable(decoder) and callable(encoder):
try:
raw = json.dumps(
dict(payload or {}), ensure_ascii=True, sort_keys=True,
separators=(',', ':'),
).encode('ascii')
canonical_bytes = encoder(decoder(raw))
normalized = json.loads(canonical_bytes.decode('ascii'))
except (TypeError, ValueError, UnicodeError) as exc:
raise ValueError(f'worker {kind} contract is invalid') from exc
canonical = canonical_bytes.decode('ascii')
return normalized, canonical, hashlib.sha256(canonical_bytes).hexdigest()
validator = next(
(getattr(contracts, name) for name in validators[kind]
if callable(getattr(contracts, name, None))),
None,
)
if validator is None:
expected = ', '.join((*codecs[kind], *validators[kind]))
raise RuntimeError(
f'worker_contracts has no {kind} validator; expected one of: {expected}'
)
normalized = validator(dict(payload or {}))
if normalized is None:
normalized = dict(payload or {})
elif not isinstance(normalized, dict):
to_dict = getattr(normalized, 'to_dict', None)
if not callable(to_dict) or not isinstance((normalized := to_dict()), dict):
raise RuntimeError(f'worker_contracts {kind} validator returned a non-mapping')
try:
canonical = json.dumps(
normalized, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
)
except (TypeError, ValueError) as exc:
raise ValueError(f'worker {kind} contract is not canonical JSON') from exc
return normalized, canonical, hashlib.sha256(canonical.encode('utf-8')).hexdigest()
class DockerCoverageDispositionConflictError(ScanEventConflictError):
pass
class DockerFindingAttributionLimitError(ValueError):
pass
class DiscoveryRetryLeaseError(RuntimeError):
pass
def env_int(name, default):
try:
return int(os.getenv(name, default))
except (TypeError, ValueError):
return default
def env_float(name, default):
try:
return float(os.getenv(name, default))
except (TypeError, ValueError):
return default
DB_FILENAME = 'scanner.db'
SQLITE_BUSY_TIMEOUT_MS = max(1000, env_int('SCANNER_SQLITE_BUSY_TIMEOUT_MS', 15000))
SQLITE_CONNECT_TIMEOUT_SEC = max(1, env_int('SCANNER_SQLITE_CONNECT_TIMEOUT_SEC', max(1, SQLITE_BUSY_TIMEOUT_MS // 1000)))
SQLITE_LOCK_RETRY_ATTEMPTS = max(1, env_int('SCANNER_SQLITE_LOCK_RETRY_ATTEMPTS', 8))
SQLITE_LOCK_RETRY_BASE_SEC = max(0.05, env_float('SCANNER_SQLITE_LOCK_RETRY_BASE_SEC', 0.25))
SQLITE_LOCK_RETRY_MAX_SEC = max(0.5, env_float('SCANNER_SQLITE_LOCK_RETRY_MAX_SEC', 5.0))
STALE_RUN_FINALIZE_BATCH_SIZE = max(1, env_int('SCANNER_STALE_RUN_FINALIZE_BATCH_SIZE', 250))
STALE_RUN_FINALIZE_MAX_BATCHES = max(1, env_int('SCANNER_STALE_RUN_FINALIZE_MAX_BATCHES', 20))
KNOWN_TARGET_LOOKUP_BATCH_SIZE = min(64, max(1, env_int('SCANNER_KNOWN_TARGET_LOOKUP_BATCH_SIZE', 64)))
DISCOVERY_RETRY_MAX_QUERY_CHARS = 256
DISCOVERY_RETRY_MAX_PAGE = 30
DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE = 100
DISCOVERY_RETRY_MAX_ALLOWLIST = 256
DISCOVERY_RETRY_MAX_CLAIM = 10
DISCOVERY_RETRY_MAX_ATTEMPTS = 1000000
DISCOVERY_RETRY_BASE_DELAY_SEC = min(
3600, max(1, env_int('DOCKERHUB_DISCOVERY_RETRY_BASE_SEC', 60)),
)
DISCOVERY_RETRY_MAX_DELAY_SEC = min(86400, max(
DISCOVERY_RETRY_BASE_DELAY_SEC,
env_int('DOCKERHUB_DISCOVERY_RETRY_MAX_SEC', 21600),
))
DISCOVERY_RETRY_MAX_TRUSTED_DELAY_SEC = 86400
DISCOVERY_RETRY_PASS_KINDS = frozenset(('ordinary', 'deep'))
DISCOVERY_RETRY_WORK_KINDS = frozenset(('query', 'page', 'range'))
DISCOVERY_RETRY_ERROR_CATEGORIES = frozenset((
'account_pool_exhausted',
'auth_forbidden',
'auth_invalid',
'auth_unavailable',
'invalid_payload',
'network',
'page_unavailable',
'policy_mismatch',
'provider_cooldown',
'provider_unavailable',
'query_removed',
'rate_limit',
'remote_transient',
'request_failed',
'tail_unavailable',
'transport',
))
DISCOVERY_RETRY_DUE_INDEX_PREDICATE = "status = 'pending'"
DISCOVERY_RETRY_LEASE_INDEX_PREDICATE = "status = 'leased'"
CLAIM_CANDIDATE_MIN = min(1000, max(1, env_int('SCANNER_CLAIM_CANDIDATE_MIN', 64)))
CLAIM_CANDIDATE_MAX = max(CLAIM_CANDIDATE_MIN, min(1000, env_int('SCANNER_CLAIM_CANDIDATE_MAX', 1000)))
RESULT_SPOOL_ADVISORY_CLASS = 1414681926
RESULT_SPOOL_ADVISORY_OBJECT = 1397772111
RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC = max(3600, env_int('RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC', 86400))
PIPELINE_ADVISORY_CLASS = 1414681926
PIPELINE_ADVISORY_OBJECTS = {
'result_ingester': 1768842867,
'jsonl_projector': 1785753445,
}
CI_SOFT_SKIP_REASONS = {
'github_actions': frozenset(('no recent workflow runs', 'no downloadable workflow logs or artifacts')),
'gitlab_ci': frozenset(('no recent pipelines', 'no downloadable job traces or artifacts')),
}
def _ci_cooldown_source_predicate(source):
reasons = ' OR '.join(
f"skipped_reason = '{reason}'" for reason in sorted(CI_SOFT_SKIP_REASONS[source])
)
return f"source = '{source}' AND ({reasons})"
CI_COOLDOWN_INDEX_PREDICATE = "status = 'skipped' AND (" + ' OR '.join(
_ci_cooldown_source_predicate(source) for source in sorted(CI_SOFT_SKIP_REASONS)
) + ')'
TARGET_QUEUE_OBSERVABILITY_STATUSES = (
'pending', 'deferred', 'in_progress', 'done', 'failed', 'quarantined', 'cold',
)
TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT = min(
10000, max(100, env_int('TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT', 100)),
)
TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS = min(
10000, max(1000, env_int('TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS', 5000)),
)
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC = 300
RETIREMENT_STATEMENT_TIMEOUT_MS = 300000
KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES = 3 * 1024 * 1024
HAS_CLAIMABLE_TARGETS_V2_SQL = '''SELECT (
EXISTS (
SELECT 1 FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND status = 'pending'
AND (available_after IS NULL OR available_after <= ?)
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
) OR EXISTS (
SELECT 1 FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND status = 'deferred'
AND available_after IS NOT NULL AND available_after <= ?
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
)
) AS present'''
PACKAGE_CANDIDATE_LOOKUP_MAX_RESULTS = 10000
PACKAGE_CANDIDATE_SCAN_MAX_ROWS = 50000
PACKAGE_CANDIDATE_SCAN_PAGE_SIZE = 250
PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE = 25
PACKAGE_REPO_NONEMPTY_PREDICATE = "repo_url IS NOT NULL AND repo_url <> ''"
DOCKER_LEGACY_PROVENANCE_SEED_MAX_ROWS = 1000000
DOCKER_LEGACY_PROVENANCE_SEED_PAGE_SIZE = 1000
DOCKER_DEPTH_EXPERIMENT_MIGRATION = '20260909_18_docker_depth_experiment'
DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION = '20260910_22_docker_depth_rollout_authority'
DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION = (
'20260910_23_docker_schema_selector_authority'
)
DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION = (
'20260910_24_docker_depth_scarcity_cohort'
)
DOCKER_DEPTH_HOLD_QUERY_INDEX_MIGRATION = (
'20260910_25_docker_depth_hold_query_index'
)
DOCKER_DEPTH_RESOLVER_REFUND_MIGRATION = (
'20260910_26_docker_depth_resolver_attempt_refund'
)
DOCKER_DEPTH_RESOLVER_DISPOSITION_MIGRATION = (
'20260910_27_docker_depth_resolver_disposition'
)
REMOTE_WORKER_MIGRATION = '20260917_28_remote_worker_admission_transport'
OPERATIONS_CONTROL_MIGRATION = '20260919_29_operations_control_audit'
WORKER_OBSERVABILITY_MIGRATION = '20260923_30_worker_observability'
DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION = (
'20260924_31_diagnostic_projection_authority'
)
ASSIGNMENT_UPLOAD_TIMEOUT_MIGRATION = (
'20260924_32_assignment_upload_timeout'
)
REMOTE_ASSIGNMENT_CAPACITY_MIGRATION = (
'20260930_33_remote_assignment_capacity'
)
RUNTIME_CONTROL_TARGET_KIND = 'runtime-control'
RUNTIME_CONTROL_DRAIN_STATES = frozenset(('normal', 'draining', 'drained'))
RUNTIME_CONTROL_ACTION_TARGETS = {
'control.discovery.pause': 'discovery',
'control.discovery.resume': 'discovery',
'control.dispatch.pause': 'dispatch',
'control.dispatch.resume': 'dispatch',
'control.drain.start': 'drain',
'control.drain.cancel': 'drain',
'control.drain.complete': 'drain',
}
RUNTIME_CONTROL_ACTIONS = frozenset(RUNTIME_CONTROL_ACTION_TARGETS)
RUNTIME_CONTROL_MAX_REVISION = 9223372036854775807
RUNTIME_CONTROL_MAX_EXPECTED_REVISION = RUNTIME_CONTROL_MAX_REVISION - 1
RUNTIME_ASYNC_TARGET_KIND = 'runtime-deployment'
RUNTIME_ASYNC_ACTION_TARGETS = {
'apply-config': 'config',
'apply-secrets': 'secrets',
'apply-both': 'config-secrets',
'restart': 'runtime',
}
RUNTIME_ASYNC_ACTIONS = frozenset(RUNTIME_ASYNC_ACTION_TARGETS)
RUNTIME_SOURCE_TARGET_KIND = 'managed-source'
RUNTIME_SOURCE_IDS = frozenset((
'discovery-producer:gitlab',
'discovery-producer:dockerhub',
'discovery-producer:huggingface',
'result-ingester', 'jsonl-projector', 'janitor', 'worker-api',
'keychecks', 'docker-shadow', 'dashboard',
))
RUNTIME_SOURCE_ACTIONS = frozenset((
'start', 'stop', 'restart', 'pause', 'resume', 'set-interval',
'once', 'set-mode', 'set-restart', 'set-restart-delay',
))
RUNTIME_SOURCE_OPERATION_ACTIONS = {
f'supervisor.source.{action}': action for action in RUNTIME_SOURCE_ACTIONS
}
RUNTIME_WORKER_ADMIN_TARGET_KIND = 'worker-admin'
RUNTIME_WORKER_ADMIN_ACTION_TARGETS = {
'workers.user.create': 'user',
'workers.user.set-cap': 'user',
'workers.user.disable': 'user',
'workers.user.enable': 'user',
'workers.device.issue': 'device',
'workers.device.rotate': 'device',
'workers.device.revoke': 'device',
'workers.device.unrevoke': 'device',
'workers.queue.requeue': 'deferred-queue',
}
RUNTIME_DOCUMENT_TARGET_KIND = 'runtime-document'
RUNTIME_DOCUMENT_ACTION_TARGETS = {
'runtime.config.save': 'config',
'runtime.secrets.save': 'secrets',
}
RUNTIME_MANAGED_FILE_TARGET_KIND = 'managed-file'
RUNTIME_MANAGED_FILE_ACTIONS = frozenset((
'files.create', 'files.replace', 'files.delete',
))
RUNTIME_MANAGED_FILE_ADVISORY_CLASS = 1836212590
_RUNTIME_MANAGED_FILE_SQLITE_LOCKS = tuple(
threading.Lock() for _ in range(64)
)
RUNTIME_ASYNC_TERMINAL_RESULTS = frozenset((
'succeeded', 'failed', 'rolled_back', 'failed_hold',
))
RUNTIME_OPERATION_SAFE_CATEGORIES = frozenset((
'agent_failed', 'agent_unavailable', 'apply_failed',
'health_check_failed', 'operation_conflict', 'restart_failed',
'result_invalid', 'revision_conflict', 'rollback_failed',
'supervisor_action_failed', 'validation_failed', 'worker_admin_mutation_failed',
'runtime_document_save_failed', 'managed_file_mutation_failed',
))
RUNTIME_OPERATION_SAFE_DETAILS = frozenset((
'active_hash_mismatch', 'apply_failed', 'bounded_result',
'candidate_hash_mismatch', 'health_check_failed', 'lock_busy',
'restart_failed', 'rollback_failed', 'validation_failed',
))
RUNTIME_AGENT_RESULT_MAX_BYTES = 16 * 1024
RUNTIME_AGENT_RESULT_MAX_DEPTH = 64
PIPELINE_MIGRATION_VERSIONS = (
'20260727_01_pipeline_capacity_bundles',
'20260727_02_projection_queue',
'20260727_03_keycheck_queue',
'20260727_04_normalized_compat',
'20260727_05_provider_canonical_keycheck_identity',
'20260727_06_second_audit_high_fixes',
'20260727_07_final_high_blockers',
'20260727_08_admission_intent_and_capacity_closure',
'20260727_09_serialized_recovery_and_bounded_retirement',
'20260729_10_keycheck_projection_single_writer',
'20260828_11_exact_git_scan_plans',
'20260831_12_projection_findings_index',
'20260901_13_docker_layer_content_scanning',
'20260902_14_adaptive_docker_payload_scanning',
'20260906_15_cold_policy_stale_backlog',
'20260907_16_docker_quarantine_rescan',
'20260909_17_dockerhub_discovery_retry',
DOCKER_DEPTH_EXPERIMENT_MIGRATION,
'20260910_19_docker_retry_provenance',
'20260910_20_docker_depth_resolver',
'20260910_21_docker_finding_layer_attribution',
DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION,
DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION,
DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION,
DOCKER_DEPTH_HOLD_QUERY_INDEX_MIGRATION,
DOCKER_DEPTH_RESOLVER_REFUND_MIGRATION,
DOCKER_DEPTH_RESOLVER_DISPOSITION_MIGRATION,
REMOTE_WORKER_MIGRATION,
OPERATIONS_CONTROL_MIGRATION,
WORKER_OBSERVABILITY_MIGRATION,
DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION,
ASSIGNMENT_UPLOAD_TIMEOUT_MIGRATION,
REMOTE_ASSIGNMENT_CAPACITY_MIGRATION,
)
PIPELINE_QUARANTINE_REVIEW_STATUSES = frozenset((
'pending', 'approved_retry', 'approved_rescan', 'discarded', 'resolved',
))
PIPELINE_QUARANTINE_LEGACY_REVIEW_STATUSES = frozenset((
'pending', 'approved_retry', 'discarded', 'resolved',
))
FINAL_CUTOVER_MARKER = 'postgres-normalized-v2-authority'
PIPELINE_REQUIRED_COLUMNS = {
'runtime_schema_migrations': {'version', 'applied_at', 'code_sha256'},
'runtime_final_cutover': {'id', 'marker', 'checked_at', 'evidence_sha256'},
'runtime_operations': {
'operation_id', 'actor', 'action', 'target_kind', 'target_ref', 'status',
'safe_category', 'safe_detail', 'expected_revision', 'resulting_revision',
'expected_identity_json', 'resulting_identity_json', 'agent_state',
'agent_result_sha256', 'requested_at', 'started_at', 'completed_at',
'agent_reconciled_at', 'updated_at',
},
'runtime_operations_control': {
'id', 'revision', 'discovery_paused', 'dispatch_paused', 'drain_state',
'actor', 'operation_id', 'created_at', 'updated_at',
},
'runtime_audit_events': {
'id', 'operation_id', 'actor', 'action', 'target_kind', 'target_ref',
'result', 'safe_category', 'before_identity_json', 'after_identity_json',
'before_bytes', 'after_bytes', 'previous_event_id',
'previous_event_sha256', 'event_sha256', 'created_at',
},
'pipeline_capacity': {
'id', 'bundle_items', 'bundle_bytes', 'projection_items', 'projection_bytes',
'keycheck_items', 'keycheck_bytes', 'quarantine_items', 'quarantine_bytes', 'updated_at',
},
'result_reservations': {
'id', 'reservation_token', 'bundle_id', 'scan_event_id', 'queue_id', 'state',
'declared_bundle_bytes', 'reserved_bundle_bytes', 'reserved_projection_bytes',
'reserved_candidate_items',
'reserved_candidate_bytes', 'ready_relative_path', 'producer_pid',
'producer_creation_time', 'producer_executable', 'bundle_credit_released',
'cleanup_attempts', 'cleanup_available_after', 'git_scan_plan_json',
'git_scan_plan_sha256', 'docker_layer_plan_json', 'docker_layer_plan_sha256',
'assignment_kind', 'remote_user_id', 'remote_device_id', 'remote_issued_at',
'remote_expires_at', 'remote_effective_config_sha256',
'remote_result_upload_body_timeout_seconds',
'remote_client_compat_sha256', 'remote_resolution_kind',
'remote_execution_snapshot_json', 'remote_execution_snapshot_sha256',
'remote_payload_sha256', 'remote_receipt_id', 'remote_resolution_json',
'remote_resolved_at', 'remote_diagnostic_projection_version',
'remote_diagnostic_count', 'remote_diagnostic_uids_sha256',
},
'worker_progress_events': {
'id', 'reservation_id', 'remote_device_id', 'schema_version', 'sequence',
'event_type', 'phase', 'event_timestamp', 'phase_started_at', 'instance_id',
'slot_id', 'source', 'event_json', 'event_sha256', 'received_at',
},
'worker_diagnostics': {
'id', 'diagnostic_uid', 'reservation_id', 'target_scan_id', 'schema_version',
'scan_event_id', 'slot_id', 'attempt', 'source', 'phase', 'kind', 'category',
'code', 'summary', 'retryable', 'occurred_at', 'captured_at', 'envelope_json',
'envelope_sha256', 'body_payload_json', 'log_payload_json', 'received_at',
},
'admission_intents': {
'reservation_token', 'intent_sha256', 'state', 'reservation_id',
'remote_user_id', 'remote_device_id', 'created_at', 'updated_at',
},
'remote_worker_users': {
'id', 'user_key', 'active_assignment_cap', 'disabled_at', 'created_at', 'updated_at',
},
'remote_worker_devices': {
'id', 'user_id', 'device_key', 'token_sha256', 'revoked_at',
'last_contact_at', 'created_at', 'updated_at',
},
'admission_intent_retirement': {
'id', 'retired_count', 'chain_sha256', 'cursor_token', 'updated_at',
},
'pipeline_artifact_retirement': {
'id', 'retired_count', 'chain_sha256', 'cursor_id', 'updated_at',
},
'result_bundles': {
'reservation_id', 'bundle_id', 'scan_event_id', 'scan_event_hash', 'relative_path',
'actual_bytes', 'state', 'ingest_lease_generation', 'ingest_lease_token',
},
'pipeline_leases': {'worker_name', 'generation', 'lease_token', 'state', 'lease_expires_at'},
'scan_result_compat': {'target_scan_id', 'metadata_json', 'metadata_sha256', 'metadata_bytes'},
'finding_compat_payloads': {'finding_id', 'payload_sha256', 'payload_bytes', 'payload_omitted'},
'projection_jobs': {'id', 'job_kind', 'event_id', 'event_hash', 'status', 'lease_token'},
'projection_streams': {'stream_name', 'base_relative_path', 'current_generation', 'rotation_bytes'},
'projection_cursors': {'stream_name', 'generation', 'committed_offset', 'last_job_id'},
'projection_appends': {'id', 'job_id', 'stream_name', 'byte_offset', 'byte_length', 'state'},
'projection_append_audit': {
'id', 'append_id', 'job_id', 'stream_name', 'state', 'reason_code',
},
'projection_rotations': {'id', 'stream_name', 'from_generation', 'to_generation', 'state'},
'keycheck_credentials': {
'id', 'service', 'credential_hash', 'provider_key_hash', 'candidate_kind',
},
'keycheck_candidates': {
'id', 'candidate_uid', 'credential_id', 'service', 'routed_service',
'secret_hash', 'state',
'lease_token', 'result_projection_reserved_bytes',
'result_projection_credit_transferred',
},
'keycheck_current_state': {'credential_id', 'service', 'status', 'last_result_id', 'state_version'},
'pipeline_quarantine': {
'id', 'subsystem', 'object_type', 'reason_code', 'review_status',
'capacity_credit_applied', 'capacity_items', 'capacity_bytes',
},
'pipeline_artifacts': {
'id', 'subsystem', 'artifact_kind', 'owner_id', 'owner_key',
'relative_path', 'payload_sha256', 'byte_count', 'state', 'updated_at',
'cleanup_attempts', 'cleanup_available_after',
},
'janitor_cursors': {'layout_name', 'last_name', 'wrap_count', 'updated_at'},
'keycheck_recheck_cursors': {
'cursor_key', 'service', 'status_scope', 'last_credential_id',
'wrap_count', 'updated_at',
},
'docker_content_blobs': {
'digest', 'coverage_policy_sha256', 'descriptor_kind', 'declared_bytes', 'media_type', 'state',
'attempts', 'max_attempts', 'available_after', 'lease_reservation_id',
'lease_token', 'lease_plan_sha256', 'lease_expires_at',
'covered_reservation_id', 'covered_scan_event_id', 'covered_policy_sha256',
'verified_bytes', 'covered_at', 'last_error_code', 'last_error_detail',
'created_at', 'updated_at',
},
'docker_image_blob_coverage': {
'queue_id', 'manifest_digest', 'position', 'blob_digest', 'coverage_policy_sha256', 'descriptor_kind',
'selection_policy_sha256', 'plan_sha256', 'reservation_id', 'selected', 'selection_reason',
'coverage_state', 'covered_at', 'last_error_code', 'created_at', 'updated_at',
},
'docker_adaptive_shadow_reports': {
'id', 'report_token', 'evaluator_version', 'state', 'scan_policy_sha256',
'execution_policy_sha256', 'selection_policy_sha256', 'cohort_size',
'completed_pairs', 'full_routed_count', 'adaptive_routed_count',
'routed_intersection_count', 'full_detector_count', 'adaptive_detector_count',
'detector_intersection_count', 'full_slot_ms', 'adaptive_slot_ms',
'omitted_descriptor_count', 'failure_count', 'privacy_violation_count',
'safety_regression_count', 'selection_metrics_json',
'sink_checkpoint_count', 'routed_recall_ppm', 'slot_ratio_ppm',
'recall_threshold_ppm', 'slot_threshold_ppm', 'passed', 'lease_owner',
'lease_token', 'lease_expires_at', 'started_at', 'completed_at',
'created_at', 'updated_at',
},
'discovery_retry_queue': {
'id', 'work_key', 'source', 'query', 'source_cycle_id', 'policy_sha256', 'pass_kind',
'work_kind', 'page_start', 'page_end', 'next_page', 'status', 'attempts',
'available_after', 'lease_owner', 'lease_token', 'leased_at',
'lease_expires_at', 'last_error_category', 'held_at', 'created_at',
'updated_at',
},
'target_queue_policy_events': {
'id', 'queue_id', 'action', 'prior_status', 'next_status', 'source',
'platform', 'query', 'reason_code', 'config_sha256', 'policy_sha256',
'manifest_sha256', 'review_audit_sha256', 'reverses_event_id',
'experiment_id', 'prior_updated_at', 'created_at',
},
}
REDACTED = '***REDACTED***'
SECRET_KEY_PARTS = (
'token',
'secret',
'password',
'authorization',
'credential',
'apikey',
'api_key',
'private_key',
)
DATABASE_SECRET_KEY_PARTS = (
'database_url',
'dashboard_db_url',
'db_url',
'dsn',
'connection_string',
'postgres_url',
)
ENDPOINT_METADATA_MAX_CHARS = 2048
POSTMAN_RESOURCE_MAX_CHARS = 4096
_ENDPOINT_DNS_LABEL_RE = re.compile(r'^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$')
def _validated_endpoint_host(value, require_domain=False):
host = str(value or '').strip().rstrip('.')
if not host or len(host) > 253 or not host.isascii():
return ''
try:
return ipaddress.ip_address(host).compressed.lower()
except ValueError:
labels = host.split('.')
if any(not _ENDPOINT_DNS_LABEL_RE.fullmatch(label) for label in labels):
return ''
if require_domain and len(labels) < 2 and host.lower() != 'localhost':
return ''
if len(labels) > 1 and not (
re.fullmatch(r'[A-Za-z]{2,63}', labels[-1])
or re.fullmatch(r'xn--[A-Za-z0-9-]{1,59}', labels[-1], re.IGNORECASE)
):
return ''
return host.lower()
def _validated_endpoint_port(parsed):
authority = parsed.netloc.rsplit('@', 1)[-1]
try:
port = parsed.port
except ValueError:
return None, False
if authority.startswith('['):
close = authority.find(']')
suffix = authority[close + 1:] if close >= 0 else authority
if close < 0 or (suffix and not re.fullmatch(r':[0-9]+', suffix)):
return None, False
elif ':' in authority:
if authority.count(':') != 1 or not re.fullmatch(r'[0-9]+', authority.rsplit(':', 1)[1]):
return None, False
if port is not None and not 1 <= port <= 65535:
return None, False
return port, True
def _safe_endpoint_path(value):
path = str(value or '')
if not path:
return ''
if not path.startswith('/') or any(
character.isspace() or ord(character) < 32 or ord(character) == 127 or character in '\\?#'
for character in path
):
return ''
return path
def _format_endpoint(scheme, host, port, path):
authority = f'[{host}]' if ':' in host else host
if port is not None:
authority += f':{port}'
prefix = f'{scheme}://{authority}' if scheme else authority
remaining = max(0, ENDPOINT_METADATA_MAX_CHARS - len(prefix))
return prefix + path[:remaining]
def sanitize_endpoint(value):
"""Return bounded endpoint metadata without URL credentials or parameters."""
try:
text = str(value or '').strip()
except Exception:
return ''
if not text or len(text) > 65536 or any(ord(character) < 32 or ord(character) == 127 for character in text):
return ''
scheme_match = re.match(r'^([A-Za-z][A-Za-z0-9+.-]*):\/\/', text)
if scheme_match:
scheme = scheme_match.group(1).lower()
if scheme not in ('http', 'https'):
return ''
try:
parsed = urlsplit(text)
host = _validated_endpoint_host(parsed.hostname)
port, valid_port = _validated_endpoint_port(parsed)
except (TypeError, ValueError):
return ''
if parsed.scheme.lower() != scheme or not parsed.netloc or not host or not valid_port:
return ''
return _format_endpoint(scheme, host, port, _safe_endpoint_path(parsed.path))
if re.match(r'(?i)^https?:', text) or '://' in text:
return ''
base = re.split(r'[?#]', text, maxsplit=1)[0]
if not base or base.startswith('/') and not base.startswith('//'):
return ''
try:
parsed = urlsplit(base if base.startswith('//') else '//' + base)
host = _validated_endpoint_host(parsed.hostname, require_domain=True)
port, valid_port = _validated_endpoint_port(parsed)
except (TypeError, ValueError):
return ''
if not parsed.netloc or not host or not valid_port:
return ''
return _format_endpoint('', host, port, _safe_endpoint_path(parsed.path))
def sanitize_endpoint_host(value):
endpoint = sanitize_endpoint(value)
if not endpoint:
return ''
try:
parsed = urlsplit(endpoint if re.match(r'(?i)^https?://', endpoint) else '//' + endpoint)
return _validated_endpoint_host(parsed.hostname)
except (TypeError, ValueError):
return ''
def _bounded_postman_label(value, max_chars):
text = str(value or '')
if any(ord(character) < 32 or ord(character) == 127 for character in text):
return ''
if re.search(r'(?i)https?://', text) or re.search(
r'(?i)(?:^|[?&])(?:api[_-]?key|token|password|secret|credential)=', text,
):
return ''
return text[:max_chars]
def _sanitize_postman_resource(value):
text = str(value or '').strip()
if not text or any(ord(character) < 32 or ord(character) == 127 for character in text):
return ''
if '://' in text or text.startswith('//') or any(marker in text for marker in ('@', '?', '#')):
return sanitize_endpoint(text)
return text[:POSTMAN_RESOURCE_MAX_CHARS]
def sanitize_postman_context(context):
if not isinstance(context, dict):
return {}
safe = {}
label_limits = {
'provider': 128,
'credential_kind': 128,
'credential_confidence': 128,
'context_location': 128,
'variable_name': 512,
'auth_type': 128,
}
for key, limit in label_limits.items():
if key in context:
safe[key] = _bounded_postman_label(context.get(key), limit)
if 'json_path' in context:
safe['json_path'] = _sanitize_postman_resource(context.get('json_path'))
if 'placeholder' in context:
safe['placeholder'] = bool(context.get('placeholder'))
endpoint = sanitize_endpoint(context.get('endpoint'))
host = sanitize_endpoint_host(endpoint) or sanitize_endpoint_host(context.get('host'))
if 'endpoint' in context or 'host' in context:
safe['endpoint'] = endpoint
safe['host'] = host
return safe
def sanitize_postman_finding(finding):
if not isinstance(finding, dict) or not isinstance(finding.get('PostmanContext'), dict):
return finding
safe = dict(finding)
safe['PostmanContext'] = sanitize_postman_context(finding['PostmanContext'])
return safe
RUNS_COLUMN_SPECS = {
'id': ('id', True), 'started_at': ('text', True), 'ended_at': ('text', False),
'duration_sec': ('real', False), 'status': ('text', True),
'invocation_mode': ('text', False), 'command_line': ('text', False),
'argv_json': ('text', False), 'selected_source': ('text', False),
'selected_platform': ('text', False), 'config_path': ('text', False),
'config_hash': ('text', False), 'enabled_sources_json': ('text', False),
'db_path': ('text', False), 'total_fetched': ('integer', False),
'total_queued_new': ('integer', False), 'total_scan_requested': ('integer', False),
'total_scanned': ('integer', False), 'total_clean': ('integer', False),
'total_found': ('integer', False), 'total_skipped': ('integer', False),
'total_errors': ('integer', False), 'total_findings': ('integer', False),
'total_verified_findings': ('integer', False), 'total_unique_secrets': ('integer', False),
'total_unique_findings': ('integer', False), 'error': ('text', False),
'total_staged': ('integer', True), 'total_quarantined': ('integer', True),
'created_at': ('text', True), 'updated_at': ('text', True),
}
SOURCE_CYCLES_COLUMN_SPECS = {
'id': ('id', True), 'run_id': ('id_ref', False), 'source': ('text', False),
'platform': ('text', False), 'mode': ('text', False), 'query': ('text', False),
'query_index': ('integer', False), 'query_count': ('integer', False),
'auth_name': ('text', False), 'started_at': ('text', True), 'ended_at': ('text', False),
'duration_sec': ('real', False), 'status': ('text', True), 'message': ('text', False),
'config_json': ('text', False), 'queue_todo_before': ('integer', False),
'queue_checked_before': ('integer', False), 'queue_todo_after': ('integer', False),
'queue_checked_after': ('integer', False), 'fetched_count': ('integer', False),
'queued_new_count': ('integer', False), 'queued_updated_count': ('integer', True),
'scan_requested_count': ('integer', False),
'scanned_count': ('integer', False), 'clean_count': ('integer', False),
'found_count': ('integer', False), 'skipped_count': ('integer', False),
'error_count': ('integer', False), 'findings_count': ('integer', False),
'verified_findings_count': ('integer', False), 'unique_secrets_count': ('integer', False),
'unique_findings_count': ('integer', False), 'targets_per_hour': ('real', False),
'hit_rate': ('real', False), 'verified_hit_rate': ('real', False),
'error_rate': ('real', False), 'created_at': ('text', True), 'updated_at': ('text', True),
'staged_count': ('integer', True), 'ingested_count': ('integer', True),
'quarantined_count': ('integer', True),
}
ERRORS_COLUMN_SPECS = {
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
'target_scan_id': ('id_ref', False), 'source': ('text', False), 'query': ('text', False),
'target': ('text', False), 'normalized_target': ('text', False),
'category': ('text', False), 'summary': ('text', False), 'raw_error': ('text', False),
'created_at': ('text', True),
}
QUEUE_SNAPSHOTS_COLUMN_SPECS = {
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
'source': ('text', False), 'phase': ('text', False), 'todo_count': ('integer', False),
'checked_count': ('integer', False), 'todo_file': ('text', False),
'checked_file': ('text', False), 'captured_at': ('text', True),
}
CONFIG_SNAPSHOTS_COLUMN_SPECS = {
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
'scope': ('text', False), 'source': ('text', False), 'config_json': ('text', False),
'captured_at': ('text', True),
}
PACKAGE_REPO_CANDIDATES_COLUMN_SPECS = {
'id': ('id', True), 'package_source': ('text', True), 'package_name': ('text', True),
'package_version': ('text', True), 'query': ('text', False), 'repo_url': ('text', True),
'provider': ('text', False), 'evidence_json': ('text', False),
'confidence': ('text', False), 'first_seen_at': ('text', True),
'last_seen_at': ('text', True), 'last_run_id': ('id_ref', False),
'last_cycle_id': ('id_ref', False),
}
KEYCHECK_COLUMN_SPECS = {
'id': ('id', True),
'service': ('text', True),
'status': ('text', True),
'status_group': ('text', True),
'checked_at': ('text', True),
'key_hash': ('text', False),
'secret_hash': ('text', False),
'key_masked': ('text', False),
'finding_id': ('id_ref', False),
'target_scan_id': ('id_ref', False),
'cycle_id': ('id_ref', False),
'run_id': ('id_ref', False),
'source': ('text', False),
'query': ('text', False),
'target': ('text', False),
'detector_name': ('text', False),
'found_at': ('text', False),
'message': ('text', False),
'metadata_json': ('text', False),
'source_line': ('text', False),
'detector_secret_hash': ('text', False),
'event_id': ('text', False),
'finding_uid': ('text', False),
'link_status': ('text', False),
'link_attempts': ('integer', False),
'linked_at': ('text', False),
'link_error': ('text', False),
'created_at': ('text', True),
'candidate_id': ('id_ref', False),
'credential_id': ('id_ref', False),
'result_source': ('text', True),
}
OUTBOX_COLUMN_SPECS = {
'id': ('id', True),
'target_scan_id': ('id_ref', True),
'payload_json': ('text', True),
'status': ('text', True),
'attempts': ('integer', False),
'last_error': ('text', False),
'lease_owner': ('text', False),
'lease_expires_at': ('text', False),
'available_after': ('text', False),
'created_at': ('text', True),
'delivered_at': ('text', False),
'updated_at': ('text', True),
}
CURSOR_COLUMN_SPECS = {
'source_file': ('text', True),
'file_identity': ('text', True),
'file_size': ('id_ref', True),
'file_mtime_ns': ('id_ref', True),
'source': ('text', True),
'platform': ('text', True),
'byte_offset': ('id_ref', True),
'line_number': ('id_ref', True),
'discarding_oversized': ('integer', True),
'oversized_line_start': ('id_ref', False),
'cumulative_rows': ('id_ref', True),
'cumulative_bytes': ('id_ref', True),
'cumulative_inserted': ('id_ref', True),
'cumulative_rejected': ('id_ref', True),
'completed_at': ('text', False),
'last_report_json': ('text', False),
'updated_at': ('text', True),
}
RECONCILIATION_ISSUE_COLUMN_SPECS = {
'id': ('id', True),
'source_file': ('text', True),
'file_identity': ('text', True),
'source': ('text', True),
'platform': ('text', True),
'line_number': ('id_ref', True),
'byte_offset': ('id_ref', True),
'reason': ('text', True),
'target_preview': ('text', False),
'created_at': ('text', True),
'resolved_at': ('text', False),
}
TARGET_QUEUE_COLUMN_SPECS = {
'id': ('id', True), 'source': ('text', True), 'platform': ('text', True),
'query': ('text', False), 'target': ('text', True), 'normalized_target': ('text', True),
'status': ('text', True), 'attempts': ('integer', False),
'lease_owner': ('text', False), 'lease_token': ('text', False),
'claim_batch': ('text', False), 'leased_at': ('text', False),
'lease_expires_at': ('text', False), 'available_after': ('text', False),
'target_scan_id': ('id_ref', False), 'last_error': ('text', False),
'created_at': ('text', True), 'updated_at': ('text', True), 'completed_at': ('text', False),
'resolver_state': ('text', False), 'resolver_due_at': ('text', False),
'resolver_attempts': ('integer', True), 'resolver_token': ('text', False),
'current_result_reservation_id': ('id_ref', False), 'claim_event_id': ('text', False),
'remote_modified_at': ('text', False), 'scan_remote_modified_at': ('text', False),
'covered_ref': ('text', False), 'covered_head': ('text', False),
}
DISCOVERY_RETRY_QUEUE_COLUMN_SPECS = {
'id': ('id', True),
'work_key': ('text', True),
'source': ('text', True),
'query': ('text', True),
'source_cycle_id': ('id_ref', False),
'policy_sha256': ('text', True),
'pass_kind': ('text', True),
'work_kind': ('text', True),
'page_start': ('integer', True),
'page_end': ('integer', True),
'next_page': ('integer', True),
'status': ('text', True),
'attempts': ('integer', True),
'available_after': ('text', False),
'lease_owner': ('text', False),
'lease_token': ('text', False),
'leased_at': ('text', False),
'lease_expires_at': ('text', False),
'last_error_category': ('text', False),
'held_at': ('text', False),
'created_at': ('text', True),
'updated_at': ('text', True),
}
TARGET_QUEUE_POLICY_EVENT_COLUMN_SPECS = {
'id': ('id', True), 'queue_id': ('id_ref', True),
'action': ('text', True), 'prior_status': ('text', True),
'next_status': ('text', True), 'source': ('text', True),
'platform': ('text', True), 'query': ('text', True),
'reason_code': ('text', True), 'config_sha256': ('text', True),
'policy_sha256': ('text', True), 'manifest_sha256': ('text', True),
'review_audit_sha256': ('text', True),
'reverses_event_id': ('id_ref', False),
'experiment_id': ('id_ref', False),
'prior_updated_at': ('text', True), 'created_at': ('text', True),
}
DOCKER_DEPTH_EXPERIMENT_COLUMN_SPECS = {
'id': ('id', True), 'experiment_key': ('text', True),
'source': ('text', True), 'state': ('text', True),
'collection_generation': ('text', True),
'config_sha256': ('text', True), 'ordered_queries_sha256': ('text', True),
'selector_version': ('text', True), 'selector_sha256': ('text', True),
'provenance_policy_sha256': ('text', True),
'query_count': ('integer', True), 'repositories_per_query': ('integer', True),
'images_per_repository': ('integer', True), 'target_limit': ('integer', True),
'target_count': ('integer', True), 'selection_count': ('integer', True),
'fence_generation': ('id_ref', True), 'fence_owner': ('text', False),
'fence_token': ('text', False), 'fence_expires_at': ('text', False),
'hold_reason_code': ('text', False), 'plan_sha256': ('text', False),
'selection_sha256': ('text', False),
'hold_manifest_sha256': ('text', False), 'created_at': ('text', True),
'updated_at': ('text', True), 'planned_at': ('text', False),
'activated_at': ('text', False), 'draining_at': ('text', False),
'completed_at': ('text', False), 'released_at': ('text', False),
'held_at': ('text', False),
}
DOCKER_DEPTH_EXPERIMENT_QUERY_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'source': ('text', True), 'query_ordinal': ('integer', True),
'query': ('text', True), 'query_sha256': ('text', True),
'required_repository_count': ('integer', True),
'selected_repository_count': ('integer', True), 'created_at': ('text', True),
}
DOCKER_DISCOVERY_PASS_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', False),
'pass_token': ('text', True), 'source': ('text', True),
'pass_kind': ('text', True), 'policy_sha256': ('text', True),
'collection_generation': ('text', True),
'ordered_queries_sha256': ('text', True), 'expected_query_count': ('integer', True),
'completed_query_count': ('integer', True), 'state': ('text', True),
'started_at': ('text', True), 'completed_at': ('text', False),
'created_at': ('text', True), 'updated_at': ('text', True),
}
DOCKER_DISCOVERY_PAGE_COLUMN_SPECS = {
'id': ('id', True), 'pass_id': ('id_ref', True),
'source_cycle_id': ('id_ref', False), 'retry_work_id': ('id_ref', False),
'query': ('text', True), 'query_ordinal': ('integer', True),
'page_number': ('integer', True), 'result_count': ('integer', True),
'total_count': ('id_ref', False),
'admitted_count': ('integer', True), 'query_complete': ('integer', True),
'admission_kind': ('text', True), 'page_sha256': ('text', True),
'observed_at': ('text', True), 'created_at': ('text', True),
}
DOCKER_REPOSITORY_QUERY_PROVENANCE_COLUMN_SPECS = {
'source': ('text', True), 'query': ('text', True),
'repository_queue_id': ('id_ref', True), 'provenance_kind': ('text', True),
'first_observed_at': ('text', True), 'last_observed_at': ('text', True),
'first_search_rank': ('integer', False), 'best_search_rank': ('integer', False),
'last_search_rank': ('integer', False), 'first_cycle_id': ('id_ref', False),
'last_cycle_id': ('id_ref', False), 'first_page_id': ('id_ref', False),
'last_page_id': ('id_ref', False), 'first_policy_sha256': ('text', False),
'last_policy_sha256': ('text', False), 'observation_count': ('integer', True),
'fresh_observation_count': ('integer', True),
'fresh_complete_observation_count': ('integer', True),
'fresh_coverage_eligible': ('integer', True),
'created_at': ('text', True), 'updated_at': ('text', True),
}
DOCKER_REPOSITORY_QUERY_OBSERVATION_COLUMN_SPECS = {
'page_id': ('id_ref', True), 'repository_queue_id': ('id_ref', True),
'source': ('text', True), 'query': ('text', True),
'search_rank': ('integer', True), 'observed_at': ('text', True),
}
DOCKER_IMAGE_MANIFEST_COLUMN_SPECS = {
'id': ('id', True), 'target_queue_id': ('id_ref', True),
'source': ('text', True), 'repository': ('text', True),
'manifest_digest': ('text', True), 'manifest_media_type': ('text', True),
'config_digest': ('text', False), 'graph_sha256': ('text', True),
'manifest_size_bytes': ('id_ref', False), 'layer_count': ('integer', True),
'resolved_at': ('text', True), 'created_at': ('text', True),
}
DOCKER_MANIFEST_LAYER_COLUMN_SPECS = {
'id': ('id', True), 'manifest_id': ('id_ref', True),
'position_from_base': ('integer', True), 'position_from_top': ('integer', True),
'layer_digest': ('text', True), 'media_type': ('text', True),
'layer_size_bytes': ('id_ref', True), 'descriptor_sha256': ('text', True),
'created_at': ('text', True),
}
DOCKER_DEPTH_EXPERIMENT_REPOSITORY_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'query_ordinal': ('integer', True), 'source': ('text', True),
'query': ('text', True), 'repository_queue_id': ('id_ref', True),
'eligibility_page_id': ('id_ref', True), 'repository_rank': ('integer', True),
'planned_is_deep_probe': ('integer', True),
'is_deep_probe': ('integer', True), 'work_state': ('text', True),
'resolver_generation': ('id_ref', True), 'resolver_owner': ('text', False),
'resolver_token': ('text', False), 'resolver_expires_at': ('text', False),
'resolver_attempts': ('integer', True), 'resolver_due_at': ('text', False),
'candidate_distinct_graph_count': ('integer', True),
'selected_image_count': ('integer', True),
'replacement_repository_queue_id': ('id_ref', False),
'replacement_eligibility_page_id': ('id_ref', False),
'replacement_count': ('integer', True),
'replacement_evidence_sha256': ('text', False),
'last_error_code': ('text', False), 'created_at': ('text', True),
'updated_at': ('text', True), 'resolved_at': ('text', False),
}
DOCKER_DEPTH_EXPERIMENT_TARGET_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'target_queue_id': ('id_ref', True), 'manifest_id': ('id_ref', True),
'counter_ordinal': ('integer', True), 'state': ('text', True),
'dispatch_wave': ('integer', True), 'dispatch_order': ('id_ref', True),
'reservation_count': ('integer', True), 'created_at': ('text', True),
'updated_at': ('text', True), 'terminal_at': ('text', False),
}
DOCKER_DEPTH_EXPERIMENT_SELECTION_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'query_ordinal': ('integer', True), 'experiment_repository_id': ('id_ref', True),
'experiment_target_id': ('id_ref', True), 'image_rank': ('integer', True),
'selection_reason': ('text', True), 'selection_evidence_sha256': ('text', True),
'graph_sha256': ('text', True), 'selected_at': ('text', True),
'created_at': ('text', True),
}
DOCKER_DEPTH_EXPERIMENT_SKIP_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'experiment_repository_id': ('id_ref', True),
'repository_queue_id': ('id_ref', True), 'candidate_kind': ('text', True),
'candidate_ordinal': ('integer', True), 'reason_code': ('text', True),
'candidate_identity_sha256': ('text', True),
'evidence_sha256': ('text', True), 'created_at': ('text', True),
}
DOCKER_DEPTH_RESOLVER_REFUND_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'experiment_repository_id': ('id_ref', True),
'repository_queue_id': ('id_ref', True),
'query_ordinal': ('integer', True), 'repository_rank': ('integer', True),
'recovery_kind': ('text', True), 'manifest_sha256': ('text', True),
'entry_evidence_sha256': ('text', True), 'log_sha256': ('text', True),
'target_identity_sha256': ('text', True),
'prior_error_code_sha256': ('text', True),
'prior_work_state': ('text', True), 'next_work_state': ('text', True),
'prior_resolver_attempts': ('integer', True),
'refund_attempts': ('integer', True),
'next_resolver_attempts': ('integer', True),
'confirmed_bug_event_count': ('integer', True),
'applied_at': ('text', True), 'created_at': ('text', True),
}
DOCKER_DEPTH_RESOLVER_DISPOSITION_COLUMN_SPECS = {
'id': ('id', True), 'experiment_id': ('id_ref', True),
'experiment_repository_id': ('id_ref', True),
'prior_repository_queue_id': ('id_ref', True),
'replacement_repository_queue_id': ('id_ref', False),
'replacement_eligibility_page_id': ('id_ref', False),
'replacement_best_search_rank': ('integer', False),
'query_ordinal': ('integer', True), 'repository_rank': ('integer', True),
'disposition_kind': ('text', True), 'outcome': ('text', True),
'manifest_sha256': ('text', True), 'entry_evidence_sha256': ('text', True),
'candidate_snapshot_sha256': ('text', True),
'prior_target_identity_sha256': ('text', True),
'replacement_target_identity_sha256': ('text', False),
'prior_error_code_sha256': ('text', True),
'prior_work_state': ('text', True), 'next_work_state': ('text', True),
'prior_resolver_attempts': ('integer', True),
'next_resolver_attempts': ('integer', True),
'applied_at': ('text', True), 'created_at': ('text', True),
}
DOCKER_DEPTH_EXPERIMENT_SCAN_BINDING_COLUMN_SPECS = {
'id': ('id', True), 'experiment_target_id': ('id_ref', True),
'reservation_id': ('id_ref', True), 'target_scan_id': ('id_ref', False),
'attempt': ('integer', True), 'state': ('text', True),
'bound_at': ('text', True), 'scan_bound_at': ('text', False),
'completed_at': ('text', False), 'created_at': ('text', True),
}
DOCKER_FINDING_LAYER_ATTRIBUTION_COLUMN_SPECS = {
'id': ('id', True), 'scan_binding_id': ('id_ref', True),
'finding_id': ('id_ref', True), 'manifest_layer_id': ('id_ref', False),
'attribution_state': ('text', True), 'reported_layer_digest': ('text', False),
'unattributed_reason': ('text', False),
'position_from_base': ('integer', False), 'position_from_top': ('integer', False),
'created_at': ('text', True),
}
TARGET_SCANS_COLUMN_SPECS = {
'id': ('id', True), 'scan_event_id': ('text', False), 'scan_event_hash': ('text', False),
'queue_id': ('id_ref', False), 'claim_lease_token': ('text', False),
'queue_completion_applied': ('integer', True), 'queue_completion_disposition': ('text', False),
'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False), 'source': ('text', False),
'query': ('text', False), 'target': ('text', False), 'normalized_target': ('text', False),
'scan_type': ('text', False), 'status': ('text', False), 'started_at': ('text', False),
'ended_at': ('text', False), 'duration_sec': ('real', False), 'scan_options_json': ('text', False),
'package_name': ('text', False), 'package_version': ('text', False),
'package_artifact': ('text', False), 'package_date': ('text', False),
'package_filename': ('text', False), 'package_type': ('text', False),
'package_size': ('integer', False), 'findings_count': ('integer', False),
'verified_findings_count': ('integer', False), 'error_count': ('integer', False),
'skipped_reason': ('text', False), 'first_error_summary': ('text', False),
'raw_result_json': ('text', False), 'created_at': ('text', True),
'result_reservation_id': ('id_ref', False),
'compat_schema_version': ('integer', True),
'raw_result_storage': ('text', True),
}
DOCKER_CONTENT_BLOB_COLUMN_SPECS = {
'digest': ('text', True), 'coverage_policy_sha256': ('text', True),
'descriptor_kind': ('text', True),
'declared_bytes': ('id_ref', True), 'media_type': ('text', True),
'state': ('text', True), 'attempts': ('integer', True),
'max_attempts': ('integer', True), 'available_after': ('text', False),
'lease_reservation_id': ('id_ref', False), 'lease_token': ('text', False),
'lease_plan_sha256': ('text', False), 'lease_expires_at': ('text', False),
'covered_reservation_id': ('id_ref', False),
'covered_scan_event_id': ('text', False), 'covered_policy_sha256': ('text', False),
'verified_bytes': ('id_ref', False), 'covered_at': ('text', False),
'last_error_code': ('text', False), 'last_error_detail': ('text', False),
'created_at': ('text', True), 'updated_at': ('text', True),
}
DOCKER_IMAGE_BLOB_COVERAGE_COLUMN_SPECS = {
'queue_id': ('id_ref', True), 'manifest_digest': ('text', True),
'reservation_id': ('id_ref', True), 'position': ('integer', True),
'blob_digest': ('text', True),
'coverage_policy_sha256': ('text', True),
'selection_policy_sha256': ('text', True),
'descriptor_kind': ('text', True), 'plan_sha256': ('text', True),
'selected': ('integer', True),
'selection_reason': ('text', True), 'coverage_state': ('text', True),
'covered_at': ('text', False), 'last_error_code': ('text', False),
'created_at': ('text', True), 'updated_at': ('text', True),
}
DOCKER_ADAPTIVE_SHADOW_REPORT_COLUMN_SPECS = {
'id': ('id', True), 'report_token': ('text', True),
'evaluator_version': ('text', True), 'state': ('text', True),
'scan_policy_sha256': ('text', True), 'execution_policy_sha256': ('text', True),
'selection_policy_sha256': ('text', True), 'cohort_size': ('integer', True),
'completed_pairs': ('integer', True), 'full_routed_count': ('id_ref', True),
'adaptive_routed_count': ('id_ref', True),
'routed_intersection_count': ('id_ref', True),
'full_detector_count': ('id_ref', True), 'adaptive_detector_count': ('id_ref', True),
'detector_intersection_count': ('id_ref', True),
'full_slot_ms': ('id_ref', True), 'adaptive_slot_ms': ('id_ref', True),
'omitted_descriptor_count': ('id_ref', True), 'failure_count': ('integer', True),
'privacy_violation_count': ('integer', True),
'safety_regression_count': ('integer', True),
'selection_metrics_json': ('text', True),
'sink_checkpoint_count': ('integer', True),
'routed_recall_ppm': ('integer', True), 'slot_ratio_ppm': ('integer', True),
'recall_threshold_ppm': ('integer', True), 'slot_threshold_ppm': ('integer', True),
'passed': ('integer', True), 'lease_owner': ('text', True),
'lease_token': ('text', True), 'lease_expires_at': ('text', True),
'started_at': ('text', True), 'completed_at': ('text', False),
'created_at': ('text', True), 'updated_at': ('text', True),
}
FINDINGS_COLUMN_SPECS = {
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
'target_scan_id': ('id_ref', False), 'source': ('text', False), 'query': ('text', False),
'target': ('text', False), 'normalized_target': ('text', False),
'detector_name': ('text', False), 'detector_type': ('text', False),
'verified': ('integer', False), 'raw_secret': ('text', False),
'redacted_secret': ('text', False), 'secret_hash': ('text', False),
'detector_secret_hash': ('text', False), 'finding_fingerprint': ('text', False),
'finding_uid': ('text', False), 'file_path': ('text', False), 'line_number': ('text', False),
'commit_hash': ('text', False), 'source_timestamp': ('text', False),
'source_metadata_type': ('text', False), 'source_metadata_json': ('text', False),
'raw_finding_json': ('text', False), 'provider': ('text', False),
'credential_kind': ('text', False), 'credential_confidence': ('text', False),
'required_context_missing': ('integer', False), 'principal': ('text', False),
'username': ('text', False), 'email': ('text', False), 'project_id': ('text', False),
'tenant_id': ('text', False), 'organization': ('text', False), 'registry': ('text', False),
'endpoint': ('text', False), 'scope': ('text', False), 'resource': ('text', False),
'enrichment_json': ('text', False), 'created_at': ('text', True),
'raw_payload_sha256': ('text', False), 'raw_payload_bytes': ('id_ref', False),
'raw_payload_omitted': ('integer', True),
}
KEYCHECK_EVENT_MAP_COLUMN_SPECS = {
'event_id': ('text', True), 'keycheck_result_id': ('id_ref', False), 'created_at': ('text', True),
}
FINDING_UID_MAP_COLUMN_SPECS = {
'finding_uid': ('text', True), 'finding_id': ('id_ref', True), 'created_at': ('text', True),
}
RUNTIME_OPERATION_COLUMN_SPECS = {
'operation_id': ('text', True), 'actor': ('text', True),
'action': ('text', True), 'target_kind': ('text', True),
'target_ref': ('text', True), 'status': ('text', True),
'safe_category': ('text', False), 'safe_detail': ('text', False),
'expected_revision': ('id_ref', False), 'resulting_revision': ('id_ref', False),
'expected_identity_json': ('text', True), 'resulting_identity_json': ('text', False),
'agent_state': ('text', True), 'agent_result_sha256': ('text', False),
'requested_at': ('text', True), 'started_at': ('text', False),
'completed_at': ('text', False), 'agent_reconciled_at': ('text', False),
'updated_at': ('text', True),
}
RUNTIME_OPERATIONS_CONTROL_COLUMN_SPECS = {
'id': ('integer', True), 'revision': ('id_ref', True),
'discovery_paused': ('integer', True), 'dispatch_paused': ('integer', True),
'drain_state': ('text', True), 'actor': ('text', True),
'operation_id': ('text', False), 'created_at': ('text', True),
'updated_at': ('text', True),
}
RUNTIME_AUDIT_EVENT_COLUMN_SPECS = {
'id': ('id', True), 'operation_id': ('text', False),
'actor': ('text', True), 'action': ('text', True),
'target_kind': ('text', True), 'target_ref': ('text', True),
'result': ('text', True), 'safe_category': ('text', False),
'before_identity_json': ('text', False), 'after_identity_json': ('text', False),
'before_bytes': ('id_ref', False), 'after_bytes': ('id_ref', False),
'previous_event_id': ('id_ref', False), 'previous_event_sha256': ('text', False),
'event_sha256': ('text', True), 'created_at': ('text', True),
}
WORKER_PROGRESS_EVENT_COLUMN_SPECS = {
'id': ('id', True), 'reservation_id': ('id_ref', True),
'remote_device_id': ('id_ref', True), 'schema_version': ('integer', True),
'sequence': ('integer', True), 'event_type': ('text', True),
'phase': ('text', True), 'event_timestamp': ('text', True),
'phase_started_at': ('text', False), 'instance_id': ('text', True),
'slot_id': ('integer', True), 'source': ('text', True),
'event_json': ('text', True), 'event_sha256': ('text', True),
'received_at': ('text', True),
}
WORKER_DIAGNOSTIC_COLUMN_SPECS = {
'id': ('id', True), 'diagnostic_uid': ('text', True),
'reservation_id': ('id_ref', True), 'target_scan_id': ('id_ref', False),
'schema_version': ('integer', True), 'scan_event_id': ('text', False),
'slot_id': ('integer', True), 'attempt': ('integer', True),
'source': ('text', True), 'phase': ('text', True), 'kind': ('text', True),
'category': ('text', True), 'code': ('text', True), 'summary': ('text', True),
'retryable': ('integer', True), 'occurred_at': ('text', True),
'captured_at': ('text', True), 'envelope_json': ('text', True),
'envelope_sha256': ('text', True), 'body_payload_json': ('text', False),
'log_payload_json': ('text', False), 'received_at': ('text', True),
}
RUNTIME_TABLE_SPECS = {
'runtime_operations': RUNTIME_OPERATION_COLUMN_SPECS,
'runtime_operations_control': RUNTIME_OPERATIONS_CONTROL_COLUMN_SPECS,
'runtime_audit_events': RUNTIME_AUDIT_EVENT_COLUMN_SPECS,
'worker_progress_events': WORKER_PROGRESS_EVENT_COLUMN_SPECS,
'worker_diagnostics': WORKER_DIAGNOSTIC_COLUMN_SPECS,
'runs': RUNS_COLUMN_SPECS,
'source_cycles': SOURCE_CYCLES_COLUMN_SPECS,
'target_queue': TARGET_QUEUE_COLUMN_SPECS,
'discovery_retry_queue': DISCOVERY_RETRY_QUEUE_COLUMN_SPECS,
'target_queue_policy_events': TARGET_QUEUE_POLICY_EVENT_COLUMN_SPECS,
'target_scans': TARGET_SCANS_COLUMN_SPECS,
'findings': FINDINGS_COLUMN_SPECS,
'errors': ERRORS_COLUMN_SPECS,
'queue_snapshots': QUEUE_SNAPSHOTS_COLUMN_SPECS,
'config_snapshots': CONFIG_SNAPSHOTS_COLUMN_SPECS,
'package_repo_candidates': PACKAGE_REPO_CANDIDATES_COLUMN_SPECS,
'scan_publication_outbox': OUTBOX_COLUMN_SPECS,
'keycheck_results': KEYCHECK_COLUMN_SPECS,
'keycheck_event_map': KEYCHECK_EVENT_MAP_COLUMN_SPECS,
'finding_uid_map': FINDING_UID_MAP_COLUMN_SPECS,
'target_queue_reconciliation_cursors': CURSOR_COLUMN_SPECS,
'target_queue_reconciliation_issues': RECONCILIATION_ISSUE_COLUMN_SPECS,
'docker_content_blobs': DOCKER_CONTENT_BLOB_COLUMN_SPECS,
'docker_image_blob_coverage': DOCKER_IMAGE_BLOB_COVERAGE_COLUMN_SPECS,
'docker_adaptive_shadow_reports': DOCKER_ADAPTIVE_SHADOW_REPORT_COLUMN_SPECS,
'docker_depth_experiments': DOCKER_DEPTH_EXPERIMENT_COLUMN_SPECS,
'docker_depth_experiment_queries': DOCKER_DEPTH_EXPERIMENT_QUERY_COLUMN_SPECS,
'docker_discovery_passes': DOCKER_DISCOVERY_PASS_COLUMN_SPECS,
'docker_discovery_pages': DOCKER_DISCOVERY_PAGE_COLUMN_SPECS,
'docker_repository_query_provenance': DOCKER_REPOSITORY_QUERY_PROVENANCE_COLUMN_SPECS,
'docker_repository_query_observations': DOCKER_REPOSITORY_QUERY_OBSERVATION_COLUMN_SPECS,
'docker_image_manifests': DOCKER_IMAGE_MANIFEST_COLUMN_SPECS,
'docker_manifest_layers': DOCKER_MANIFEST_LAYER_COLUMN_SPECS,
'docker_depth_experiment_repositories': DOCKER_DEPTH_EXPERIMENT_REPOSITORY_COLUMN_SPECS,
'docker_depth_experiment_targets': DOCKER_DEPTH_EXPERIMENT_TARGET_COLUMN_SPECS,
'docker_depth_experiment_selections': DOCKER_DEPTH_EXPERIMENT_SELECTION_COLUMN_SPECS,
'docker_depth_experiment_candidate_skips': DOCKER_DEPTH_EXPERIMENT_SKIP_COLUMN_SPECS,
'docker_depth_resolver_attempt_refunds': DOCKER_DEPTH_RESOLVER_REFUND_COLUMN_SPECS,
'docker_depth_resolver_dispositions': DOCKER_DEPTH_RESOLVER_DISPOSITION_COLUMN_SPECS,
'docker_depth_experiment_scan_bindings': DOCKER_DEPTH_EXPERIMENT_SCAN_BINDING_COLUMN_SPECS,
'docker_finding_layer_attributions': DOCKER_FINDING_LAYER_ATTRIBUTION_COLUMN_SPECS,
}
DOCKER_DEPTH_EXPERIMENT_TABLES = (
'docker_depth_experiments',
'docker_depth_experiment_queries',
'docker_discovery_passes',
'docker_discovery_pages',
'docker_repository_query_provenance',
'docker_repository_query_observations',
'docker_image_manifests',
'docker_manifest_layers',
'docker_depth_experiment_repositories',
'docker_depth_experiment_targets',
'docker_depth_experiment_selections',
'docker_depth_experiment_candidate_skips',
'docker_depth_resolver_attempt_refunds',
'docker_depth_resolver_dispositions',
'docker_depth_experiment_scan_bindings',
'docker_finding_layer_attributions',
)
PIPELINE_REQUIRED_COLUMNS.update({
table: set(RUNTIME_TABLE_SPECS[table]) for table in DOCKER_DEPTH_EXPERIMENT_TABLES
})
DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS = (
('docker_depth_experiments', 'uq_docker_depth_experiments_key',
('experiment_key',), True, ''),
('docker_depth_experiments', 'idx_docker_depth_experiments_state',
('source', 'state', 'id'), False, ''),
('docker_depth_experiment_queries', 'uq_docker_depth_experiment_queries_ordinal',
('experiment_id', 'query_ordinal'), True, ''),
('docker_depth_experiment_queries', 'uq_docker_depth_experiment_queries_query',
('experiment_id', 'source', 'query'), True, ''),
('docker_discovery_passes', 'uq_docker_discovery_passes_token',
('pass_token',), True, ''),
('docker_discovery_passes', 'idx_docker_discovery_passes_policy',
('source', 'policy_sha256', 'state', 'id'), False, ''),
('docker_discovery_passes', 'idx_docker_discovery_passes_experiment',
('experiment_id', 'state', 'id'), False, ''),
('docker_discovery_pages', 'uq_docker_discovery_pages_position',
('pass_id', 'query_ordinal', 'page_number'), True, ''),
('docker_discovery_pages', 'idx_docker_discovery_pages_query',
('pass_id', 'query', 'query_complete', 'page_number'), False, ''),
('docker_discovery_pages', 'idx_docker_discovery_pages_retry',
('retry_work_id', 'id'), False, 'retry_work_id IS NOT NULL'),
('docker_repository_query_provenance', 'idx_docker_repository_provenance_queue',
('repository_queue_id', 'source', 'query'), False, ''),
('docker_repository_query_provenance', 'idx_docker_repository_provenance_eligible',
('source', 'query', 'fresh_coverage_eligible', 'best_search_rank', 'repository_queue_id'),
False, ''),
('docker_repository_query_observations', 'idx_docker_repository_observations_query',
('source', 'query', 'repository_queue_id', 'page_id'), False, ''),
('docker_image_manifests', 'uq_docker_image_manifests_queue',
('target_queue_id',), True, ''),
('docker_image_manifests', 'idx_docker_image_manifests_digest',
('manifest_digest', 'id'), False, ''),
('docker_manifest_layers', 'uq_docker_manifest_layers_base_position',
('manifest_id', 'position_from_base'), True, ''),
('docker_manifest_layers', 'uq_docker_manifest_layers_top_position',
('manifest_id', 'position_from_top'), True, ''),
('docker_manifest_layers', 'idx_docker_manifest_layers_digest',
('layer_digest', 'manifest_id', 'position_from_base'), False, ''),
('docker_depth_experiment_repositories', 'uq_docker_experiment_repositories_member',
('experiment_id', 'query_ordinal', 'repository_queue_id'), True, ''),
('docker_depth_experiment_repositories', 'uq_docker_experiment_repositories_rank',
('experiment_id', 'query_ordinal', 'repository_rank'), True, ''),
('docker_depth_experiment_repositories', 'idx_docker_experiment_repository_work',
('experiment_id', 'work_state', 'repository_rank', 'query_ordinal', 'id'), False, ''),
('docker_depth_experiment_repositories', 'idx_docker_experiment_repository_due',
('experiment_id', 'work_state', 'resolver_due_at', 'repository_rank',
'query_ordinal', 'id'), False, ''),
('docker_depth_experiment_targets', 'uq_docker_experiment_targets_queue',
('experiment_id', 'target_queue_id'), True, ''),
('docker_depth_experiment_targets', 'uq_docker_experiment_targets_counter',
('experiment_id', 'counter_ordinal'), True, ''),
('docker_depth_experiment_targets', 'idx_docker_experiment_targets_dispatch',
('experiment_id', 'state', 'dispatch_wave', 'dispatch_order', 'id'), False, ''),
('docker_depth_experiment_selections', 'uq_docker_experiment_selections_rank',
('experiment_repository_id', 'image_rank'), True, ''),
('docker_depth_experiment_selections', 'idx_docker_experiment_selections_query',
('experiment_id', 'query_ordinal', 'image_rank', 'experiment_repository_id', 'id'),
False, ''),
('docker_depth_experiment_selections', 'idx_docker_experiment_selections_target',
('experiment_target_id', 'id'), False, ''),
('docker_depth_experiment_candidate_skips', 'uq_docker_experiment_candidate_skip',
('experiment_repository_id', 'candidate_kind', 'candidate_identity_sha256'),
True, ''),
('docker_depth_experiment_candidate_skips', 'idx_docker_experiment_candidate_skips',
('experiment_id', 'experiment_repository_id', 'candidate_kind', 'id'),
False, ''),
('docker_depth_resolver_attempt_refunds', 'uq_docker_depth_resolver_attempt_refund',
('experiment_repository_id', 'recovery_kind'), True, ''),
('docker_depth_resolver_attempt_refunds', 'idx_docker_depth_resolver_attempt_refunds',
('experiment_id', 'id'), False, ''),
('docker_depth_resolver_dispositions', 'uq_docker_depth_resolver_disposition',
('experiment_repository_id', 'disposition_kind'), True, ''),
('docker_depth_resolver_dispositions', 'idx_docker_depth_resolver_dispositions',
('experiment_id', 'id'), False, ''),
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_reservation',
('reservation_id',), True, ''),
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_attempt',
('experiment_target_id', 'attempt'), True, ''),
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_scan',
('target_scan_id',), True, 'target_scan_id IS NOT NULL'),
('docker_depth_experiment_scan_bindings', 'idx_docker_experiment_bindings_target',
('experiment_target_id', 'state', 'id'), False, ''),
('docker_finding_layer_attributions', 'uq_docker_finding_layer_exact',
('finding_id', 'manifest_layer_id'), True, "attribution_state = 'exact'"),
('docker_finding_layer_attributions', 'uq_docker_finding_layer_unattributed',
('finding_id',), True, "attribution_state = 'unattributed'"),
('docker_finding_layer_attributions', 'idx_docker_finding_layer_binding',
('scan_binding_id', 'finding_id', 'id'), False, ''),
('target_queue_policy_events', 'idx_target_queue_policy_events_experiment',
('experiment_id', 'id'), False, 'experiment_id IS NOT NULL'),
)
RUNTIME_PRIMARY_KEYS = {
'runtime_operations': ['operation_id'],
'runtime_operations_control': ['id'],
'runtime_audit_events': ['id'],
'worker_progress_events': ['id'], 'worker_diagnostics': ['id'],
'runs': ['id'], 'source_cycles': ['id'], 'errors': ['id'],
'queue_snapshots': ['id'], 'config_snapshots': ['id'],
'package_repo_candidates': ['id'],
'target_queue': ['id'], 'target_scans': ['id'], 'findings': ['id'],
'discovery_retry_queue': ['id'],
'target_queue_policy_events': ['id'],
'scan_publication_outbox': ['id'], 'keycheck_results': ['id'],
'keycheck_event_map': ['event_id'], 'finding_uid_map': ['finding_uid'],
'target_queue_reconciliation_cursors': ['source_file'],
'target_queue_reconciliation_issues': ['id'],
'docker_content_blobs': ['digest', 'coverage_policy_sha256'],
'docker_image_blob_coverage': ['reservation_id', 'position'],
'docker_adaptive_shadow_reports': ['id'],
'docker_depth_experiments': ['id'],
'docker_depth_experiment_queries': ['id'],
'docker_discovery_passes': ['id'],
'docker_discovery_pages': ['id'],
'docker_repository_query_provenance': ['source', 'query', 'repository_queue_id'],
'docker_repository_query_observations': ['page_id', 'repository_queue_id'],
'docker_image_manifests': ['id'],
'docker_manifest_layers': ['id'],
'docker_depth_experiment_repositories': ['id'],
'docker_depth_experiment_targets': ['id'],
'docker_depth_experiment_selections': ['id'],
'docker_depth_experiment_candidate_skips': ['id'],
'docker_depth_resolver_attempt_refunds': ['id'],
'docker_depth_resolver_dispositions': ['id'],
'docker_depth_experiment_scan_bindings': ['id'],
'docker_finding_layer_attributions': ['id'],
}
RUNTIME_COLUMN_DEFAULTS = {
'runtime_operations': {
'target_ref': '', 'status': 'requested',
'expected_identity_json': '{}', 'agent_state': 'not_required',
},
'runtime_operations_control': {
'revision': '0', 'discovery_paused': '0', 'dispatch_paused': '0',
'drain_state': 'normal',
},
'runtime_audit_events': {'target_ref': ''},
'runs': {
'total_fetched': '0', 'total_queued_new': '0', 'total_scan_requested': '0',
'total_scanned': '0', 'total_clean': '0', 'total_found': '0',
'total_skipped': '0', 'total_errors': '0', 'total_findings': '0',
'total_verified_findings': '0', 'total_unique_secrets': '0',
'total_unique_findings': '0', 'total_staged': '0', 'total_quarantined': '0',
},
'source_cycles': {
'fetched_count': '0', 'queued_new_count': '0', 'queued_updated_count': '0',
'scan_requested_count': '0',
'scanned_count': '0', 'clean_count': '0', 'found_count': '0',
'skipped_count': '0', 'error_count': '0', 'findings_count': '0',
'verified_findings_count': '0', 'unique_secrets_count': '0',
'unique_findings_count': '0', 'targets_per_hour': '0', 'hit_rate': '0',
'verified_hit_rate': '0', 'error_rate': '0', 'staged_count': '0',
'ingested_count': '0', 'quarantined_count': '0',
},
'target_queue': {'status': 'pending', 'attempts': '0', 'resolver_attempts': '0'},
'discovery_retry_queue': {
'page_start': '1', 'page_end': '1', 'next_page': '1',
'status': 'pending', 'attempts': '0',
},
'target_scans': {
'queue_completion_applied': '0', 'findings_count': '0',
'verified_findings_count': '0', 'error_count': '0',
'compat_schema_version': '2', 'raw_result_storage': 'legacy',
},
'findings': {'verified': '0', 'required_context_missing': '0', 'raw_payload_omitted': '0'},
'scan_publication_outbox': {'status': 'pending', 'attempts': '0'},
'keycheck_results': {
'link_status': 'pending', 'link_attempts': '0', 'result_source': 'api_check',
},
'target_queue_reconciliation_cursors': {
'file_size': '0', 'file_mtime_ns': '0', 'byte_offset': '0', 'line_number': '0',
'discarding_oversized': '0', 'cumulative_rows': '0', 'cumulative_bytes': '0',
'cumulative_inserted': '0', 'cumulative_rejected': '0',
},
'docker_content_blobs': {'state': 'pending', 'attempts': '0'},
'docker_image_blob_coverage': {'selection_policy_sha256': ''},
'docker_adaptive_shadow_reports': {
'state': 'running', 'completed_pairs': '0', 'full_routed_count': '0',
'adaptive_routed_count': '0', 'routed_intersection_count': '0',
'full_detector_count': '0', 'adaptive_detector_count': '0',
'detector_intersection_count': '0', 'full_slot_ms': '0',
'adaptive_slot_ms': '0', 'omitted_descriptor_count': '0',
'failure_count': '0', 'privacy_violation_count': '0',
'safety_regression_count': '0', 'selection_metrics_json': '{}',
'sink_checkpoint_count': '0', 'routed_recall_ppm': '0',
'slot_ratio_ppm': '0',
'recall_threshold_ppm': '850000',
'slot_threshold_ppm': '400000',
'passed': '0',
},
'docker_depth_experiments': {
'state': 'collecting', 'target_count': '0', 'selection_count': '0',
'fence_generation': '0',
'collection_generation': 'docker-depth-provenance-v1',
},
'docker_depth_experiment_queries': {
'required_repository_count': '10', 'selected_repository_count': '0',
},
'docker_discovery_passes': {
'completed_query_count': '0', 'state': 'collecting',
'collection_generation': 'docker-depth-provenance-v1',
},
'docker_discovery_pages': {
'result_count': '0', 'admitted_count': '0', 'query_complete': '0',
'admission_kind': 'main',
},
'docker_repository_query_provenance': {
'observation_count': '1', 'fresh_observation_count': '0',
'fresh_complete_observation_count': '0', 'fresh_coverage_eligible': '0',
},
'docker_depth_experiment_repositories': {
'planned_is_deep_probe': '0', 'is_deep_probe': '0',
'work_state': 'pending', 'resolver_generation': '0',
'resolver_attempts': '0', 'candidate_distinct_graph_count': '0',
'selected_image_count': '0', 'replacement_count': '0',
},
'docker_depth_experiment_targets': {'state': 'pending', 'reservation_count': '0'},
'docker_depth_experiment_scan_bindings': {'state': 'reserved'},
}
GENERATED_ID_COLUMNS = {
'runtime_audit_events': 'id',
'worker_progress_events': 'id', 'worker_diagnostics': 'id',
'runs': 'id', 'source_cycles': 'id', 'target_queue': 'id',
'discovery_retry_queue': 'id',
'target_scans': 'id', 'findings': 'id', 'errors': 'id',
'queue_snapshots': 'id', 'config_snapshots': 'id',
'package_repo_candidates': 'id',
'scan_publication_outbox': 'id', 'keycheck_results': 'id',
'target_queue_reconciliation_issues': 'id',
'docker_adaptive_shadow_reports': 'id',
'target_queue_policy_events': 'id',
'docker_depth_experiments': 'id',
'docker_depth_experiment_queries': 'id',
'docker_discovery_passes': 'id',
'docker_discovery_pages': 'id',
'docker_image_manifests': 'id',
'docker_manifest_layers': 'id',
'docker_depth_experiment_repositories': 'id',
'docker_depth_experiment_targets': 'id',
'docker_depth_experiment_selections': 'id',
'docker_depth_experiment_candidate_skips': 'id',
'docker_depth_resolver_attempt_refunds': 'id',
'docker_depth_resolver_dispositions': 'id',
'docker_depth_experiment_scan_bindings': 'id',
'docker_finding_layer_attributions': 'id',
}
REQUIRED_FOREIGN_KEYS = {
'runtime_operations_control': [
(('operation_id',), 'runtime_operations', ('operation_id',)),
],
'runtime_audit_events': [
(('operation_id',), 'runtime_operations', ('operation_id',)),
(('previous_event_id',), 'runtime_audit_events', ('id',)),
],
'worker_progress_events': [
(('reservation_id',), 'result_reservations', ('id',)),
(('remote_device_id',), 'remote_worker_devices', ('id',)),
],
'worker_diagnostics': [
(('reservation_id',), 'result_reservations', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
],
'source_cycles': [(('run_id',), 'runs', ('id',))],
'discovery_retry_queue': [
(('source_cycle_id',), 'source_cycles', ('id',)),
],
'target_scans': [
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
(('queue_id',), 'target_queue', ('id',)),
(('result_reservation_id',), 'result_reservations', ('id',)),
],
'findings': [
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
],
'errors': [
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
],
'queue_snapshots': [
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
],
'config_snapshots': [
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
],
'package_repo_candidates': [
(('last_run_id',), 'runs', ('id',)),
(('last_cycle_id',), 'source_cycles', ('id',)),
],
'target_queue': [
(('target_scan_id',), 'target_scans', ('id',)),
(('current_result_reservation_id',), 'result_reservations', ('id',)),
],
'target_queue_policy_events': [
(('queue_id',), 'target_queue', ('id',)),
(('reverses_event_id',), 'target_queue_policy_events', ('id',)),
(('experiment_id',), 'docker_depth_experiments', ('id',)),
],
'scan_publication_outbox': [(('target_scan_id',), 'target_scans', ('id',))],
'keycheck_results': [
(('finding_id',), 'findings', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
(('run_id',), 'runs', ('id',)),
(('candidate_id',), 'keycheck_candidates', ('id',)),
(('credential_id',), 'keycheck_credentials', ('id',)),
],
'keycheck_event_map': [(('keycheck_result_id',), 'keycheck_results', ('id',))],
'finding_uid_map': [(('finding_id',), 'findings', ('id',))],
'result_reservations': [
(('queue_id',), 'target_queue', ('id',)),
(('run_id',), 'runs', ('id',)),
(('cycle_id',), 'source_cycles', ('id',)),
(('remote_device_id', 'remote_user_id'), 'remote_worker_devices', ('id', 'user_id')),
],
'remote_worker_devices': [(('user_id',), 'remote_worker_users', ('id',))],
'admission_intents': [
(('reservation_id',), 'result_reservations', ('id',)),
(('remote_device_id', 'remote_user_id'), 'remote_worker_devices', ('id', 'user_id')),
],
'docker_content_blobs': [
(('lease_reservation_id',), 'result_reservations', ('id',)),
(('covered_reservation_id',), 'result_reservations', ('id',)),
],
'docker_image_blob_coverage': [
(('queue_id',), 'target_queue', ('id',)),
(
('blob_digest', 'coverage_policy_sha256'),
'docker_content_blobs',
('digest', 'coverage_policy_sha256'),
),
(('reservation_id',), 'result_reservations', ('id',)),
],
'result_bundles': [
(('reservation_id',), 'result_reservations', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
],
'scan_result_compat': [(('target_scan_id',), 'target_scans', ('id',))],
'finding_compat_payloads': [(('finding_id',), 'findings', ('id',))],
'projection_jobs': [
(('target_scan_id',), 'target_scans', ('id',)),
(('keycheck_result_id',), 'keycheck_results', ('id',)),
],
'projection_cursors': [
(('stream_name',), 'projection_streams', ('stream_name',)),
(('last_append_id',), 'projection_appends', ('id',)),
(('last_job_id',), 'projection_jobs', ('id',)),
],
'projection_appends': [
(('job_id',), 'projection_jobs', ('id',)),
(('stream_name',), 'projection_streams', ('stream_name',)),
],
'projection_rotations': [(('stream_name',), 'projection_streams', ('stream_name',))],
'keycheck_candidates': [
(('credential_id',), 'keycheck_credentials', ('id',)),
(('finding_id',), 'findings', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
(('keycheck_result_id',), 'keycheck_results', ('id',)),
],
'keycheck_current_state': [
(('credential_id',), 'keycheck_credentials', ('id',)),
(('last_result_id',), 'keycheck_results', ('id',)),
],
'pipeline_quarantine': [
(('reservation_id',), 'result_reservations', ('id',)),
(('projection_job_id',), 'projection_jobs', ('id',)),
(('keycheck_candidate_id',), 'keycheck_candidates', ('id',)),
],
'docker_depth_experiment_queries': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
],
'docker_discovery_passes': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
],
'docker_discovery_pages': [
(('pass_id',), 'docker_discovery_passes', ('id',)),
(('source_cycle_id',), 'source_cycles', ('id',)),
(('retry_work_id',), 'discovery_retry_queue', ('id',)),
],
'docker_repository_query_provenance': [
(('repository_queue_id',), 'target_queue', ('id',)),
(('first_cycle_id',), 'source_cycles', ('id',)),
(('last_cycle_id',), 'source_cycles', ('id',)),
(('first_page_id',), 'docker_discovery_pages', ('id',)),
(('last_page_id',), 'docker_discovery_pages', ('id',)),
],
'docker_repository_query_observations': [
(('page_id',), 'docker_discovery_pages', ('id',)),
(('repository_queue_id',), 'target_queue', ('id',)),
(
('source', 'query', 'repository_queue_id'),
'docker_repository_query_provenance',
('source', 'query', 'repository_queue_id'),
),
],
'docker_image_manifests': [
(('target_queue_id',), 'target_queue', ('id',)),
],
'docker_manifest_layers': [
(('manifest_id',), 'docker_image_manifests', ('id',)),
],
'docker_depth_experiment_repositories': [
(
('experiment_id', 'query_ordinal', 'source', 'query'),
'docker_depth_experiment_queries',
('experiment_id', 'query_ordinal', 'source', 'query'),
),
(('repository_queue_id',), 'target_queue', ('id',)),
(
('source', 'query', 'repository_queue_id'),
'docker_repository_query_provenance',
('source', 'query', 'repository_queue_id'),
),
(
('eligibility_page_id', 'repository_queue_id', 'source', 'query'),
'docker_repository_query_observations',
('page_id', 'repository_queue_id', 'source', 'query'),
),
(('replacement_repository_queue_id',), 'target_queue', ('id',)),
(
(
'replacement_eligibility_page_id',
'replacement_repository_queue_id',
'source',
'query',
),
'docker_repository_query_observations',
('page_id', 'repository_queue_id', 'source', 'query'),
),
],
'docker_depth_experiment_targets': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
(('target_queue_id',), 'target_queue', ('id',)),
(
('manifest_id', 'target_queue_id'),
'docker_image_manifests',
('id', 'target_queue_id'),
),
],
'docker_depth_experiment_selections': [
(
('experiment_id', 'query_ordinal'),
'docker_depth_experiment_queries',
('experiment_id', 'query_ordinal'),
),
(
('experiment_id', 'query_ordinal', 'experiment_repository_id'),
'docker_depth_experiment_repositories',
('experiment_id', 'query_ordinal', 'id'),
),
(
('experiment_id', 'experiment_target_id'),
'docker_depth_experiment_targets',
('experiment_id', 'id'),
),
],
'docker_depth_experiment_candidate_skips': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
(('repository_queue_id',), 'target_queue', ('id',)),
],
'docker_depth_resolver_attempt_refunds': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
(('repository_queue_id',), 'target_queue', ('id',)),
],
'docker_depth_resolver_dispositions': [
(('experiment_id',), 'docker_depth_experiments', ('id',)),
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
(('prior_repository_queue_id',), 'target_queue', ('id',)),
(('replacement_repository_queue_id',), 'target_queue', ('id',)),
(('replacement_eligibility_page_id',), 'docker_discovery_pages', ('id',)),
],
'docker_depth_experiment_scan_bindings': [
(('experiment_target_id',), 'docker_depth_experiment_targets', ('id',)),
(('reservation_id',), 'result_reservations', ('id',)),
(('target_scan_id',), 'target_scans', ('id',)),
],
'docker_finding_layer_attributions': [
(('scan_binding_id',), 'docker_depth_experiment_scan_bindings', ('id',)),
(('finding_id',), 'findings', ('id',)),
(
(
'manifest_layer_id', 'position_from_base', 'position_from_top',
'reported_layer_digest',
),
'docker_manifest_layers',
('id', 'position_from_base', 'position_from_top', 'layer_digest'),
),
],
}
REQUIRED_FOREIGN_KEY_ACTIONS = {
(table, columns): ('NO ACTION', 'NO ACTION')
for table, foreign_keys in REQUIRED_FOREIGN_KEYS.items()
for columns, _, _ in foreign_keys
}
def _required_foreign_key_actions(table, columns):
return REQUIRED_FOREIGN_KEY_ACTIONS[(table, columns)]
def _foreign_key_actions_match(foreign_key, expected_actions):
update_action, delete_action = expected_actions
return (
str(foreign_key.get('update_action') or '').strip().upper() == update_action
and str(foreign_key.get('delete_action') or '').strip().upper() == delete_action
)
def _schema_type_matches(actual, expected, postgres):
value = re.sub(r'\s+', ' ', str(actual or '').strip().lower())
if expected in ('id', 'id_ref'):
return value == ('bigint' if postgres else 'integer')
if expected == 'integer':
return value == 'integer'
if expected == 'text':
return value == 'text'
if expected == 'real':
return value == 'real'
return value == expected
def _normalized_predicate(value):
text = str(value or '').lower().replace('"', '')
text = re.sub(r'::(?:text|character varying)', '', text)
return re.sub(r'[\s()]', '', text)
def _normalized_default(value):
text = str(value or '').strip().lower()
text = re.sub(r'::(?:text|character varying|integer|bigint|smallint)$', '', text)
while len(text) >= 2 and text[0] == '(' and text[-1] == ')':
text = text[1:-1].strip()
if len(text) >= 2 and text[0] == text[-1] == "'":
text = text[1:-1].replace("''", "'")
return text
def _sql_enum_check(column, values):
return '(' + ' OR '.join(f"{column} = '{value}'" for value in values) + ')'
def _sql_sha256_check(column, nullable=False):
remainder = column
for character in '0123456789abcdef':
remainder = f"replace({remainder}, '{character}', '')"
expression = (
f'(length({column}) = 64 AND {column} = lower({column}) '
f'AND length({remainder}) = 0)'
)
return f'({column} IS NULL OR {expression})' if nullable else expression
DOCKER_DEPTH_EXPERIMENT_STATES = (
'collecting', 'planned', 'holding', 'resolving', 'active', 'draining',
'completed', 'released', 'held',
)
DOCKER_DISCOVERY_PASS_STATES = ('collecting', 'complete', 'held', 'failed')
DOCKER_EXPERIMENT_REPOSITORY_STATES = (
'pending', 'resolving', 'resolved', 'held', 'failed', 'skipped',
)
DOCKER_EXPERIMENT_TARGET_STATES = (
'pending', 'reserved', 'scanning', 'done', 'failed', 'held',
'quarantined', 'skipped',
)
DOCKER_EXPERIMENT_BINDING_STATES = (
'reserved', 'scanning', 'completed', 'failed', 'quarantined', 'released',
)
DOCKER_FINDING_UNATTRIBUTED_REASONS = (
'digest_absent', 'digest_invalid', 'digest_not_in_manifest',
'manifest_mismatch',
)
DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS = {
'discovery_retry_queue': {
'discovery_retry_queue_lifecycle_check': (
"(status = 'leased' AND lease_owner IS NOT NULL AND lease_token IS NOT NULL "
'AND leased_at IS NOT NULL AND lease_expires_at IS NOT NULL AND held_at IS NULL) '
"OR (status = 'pending' AND lease_owner IS NULL AND lease_token IS NULL "
'AND leased_at IS NULL AND lease_expires_at IS NULL AND held_at IS NULL) '
"OR (status = 'held' AND lease_owner IS NULL AND lease_token IS NULL "
'AND leased_at IS NULL AND lease_expires_at IS NULL AND held_at IS NOT NULL)'
),
'discovery_retry_queue_sha256_check': (
f'{_sql_sha256_check("work_key")} '
f'AND {_sql_sha256_check("policy_sha256")}'
),
},
'target_queue_policy_events': {
'target_queue_policy_events_transition_check': (
"(action = 'cold' AND (prior_status = 'pending' OR prior_status = 'deferred') "
"AND next_status = 'cold' AND reverses_event_id IS NULL) OR "
"(action = 'reactivate' AND prior_status = 'cold' "
"AND (next_status = 'pending' OR next_status = 'deferred') "
'AND reverses_event_id IS NOT NULL)'
),
'target_queue_policy_events_hash_check': (
f'{_sql_sha256_check("config_sha256")} '
f'AND {_sql_sha256_check("policy_sha256")} '
f'AND {_sql_sha256_check("manifest_sha256")} '
f'AND {_sql_sha256_check("review_audit_sha256")}'
),
'target_queue_policy_events_experiment_ownership_check': (
"(experiment_id IS NULL "
"AND reason_code <> 'docker_depth_experiment_hold' "
"AND reason_code <> 'docker_depth_experiment_dynamic_hold' "
"AND reason_code <> 'docker_depth_experiment_reviewed_release') OR "
"(experiment_id IS NOT NULL AND ((action = 'cold' AND "
"(reason_code = 'docker_depth_experiment_hold' "
"OR reason_code = 'docker_depth_experiment_dynamic_hold')) "
"OR (action = 'reactivate' "
"AND reason_code = 'docker_depth_experiment_reviewed_release')))"
),
},
'docker_depth_experiments': {
'docker_depth_experiments_state_check': _sql_enum_check(
'state', DOCKER_DEPTH_EXPERIMENT_STATES,
),
'docker_depth_experiments_range_check': (
'query_count >= 1 AND query_count <= 1000 '
'AND repositories_per_query >= 1 AND repositories_per_query <= 39 '
'AND images_per_repository >= 1 AND images_per_repository <= 10 '
'AND target_limit >= 1 AND target_limit <= 2000 '
'AND target_count >= 0 AND target_count <= target_limit '
'AND selection_count >= 0 AND fence_generation >= 0'
),
'docker_depth_experiments_identity_check': (
'length(experiment_key) >= 1 AND length(experiment_key) <= 128 '
'AND length(source) >= 1 AND length(source) <= 64 '
f'AND {_sql_sha256_check("config_sha256")} '
f'AND {_sql_sha256_check("ordered_queries_sha256")} '
'AND length(selector_version) >= 1 AND length(selector_version) <= 128 '
f'AND {_sql_sha256_check("selector_sha256")} '
f'AND {_sql_sha256_check("provenance_policy_sha256")} '
f'AND {_sql_sha256_check("plan_sha256", nullable=True)} '
f'AND {_sql_sha256_check("hold_manifest_sha256", nullable=True)}'
),
'docker_depth_experiments_capacity_check': (
'(query_count * (repositories_per_query + images_per_repository - 1) '
"<= target_limit OR (selector_version = 'docker-rank1-breadth-v1' "
'AND query_count = 61 AND repositories_per_query = 39 '
'AND images_per_repository = 1 AND target_limit = 2000))'
),
'docker_depth_experiments_selection_check': _sql_sha256_check(
'selection_sha256', nullable=True,
),
'docker_depth_experiments_fence_check': (
'(fence_owner IS NULL AND fence_token IS NULL AND fence_expires_at IS NULL) '
'OR (fence_owner IS NOT NULL AND fence_token IS NOT NULL '
'AND fence_expires_at IS NOT NULL)'
),
},
'docker_depth_experiment_queries': {
'docker_depth_experiment_queries_range_check': (
'query_ordinal >= 0 AND required_repository_count >= 1 '
'AND required_repository_count <= 39'
),
'docker_depth_experiment_queries_identity_check': (
'length(source) >= 1 AND length(source) <= 64 '
'AND length(query) >= 1 AND length(query) <= 256 '
f'AND {_sql_sha256_check("query_sha256")}'
),
'docker_depth_experiment_queries_selection_check': (
'selected_repository_count >= 0 '
'AND selected_repository_count <= required_repository_count'
),
},
'docker_discovery_passes': {
'docker_discovery_passes_kind_check': _sql_enum_check(
'pass_kind', ('ordinary', 'deep'),
),
'docker_discovery_passes_state_check': _sql_enum_check(
'state', DOCKER_DISCOVERY_PASS_STATES,
),
'docker_discovery_passes_count_check': (
'expected_query_count >= 1 AND expected_query_count <= 1000 '
'AND completed_query_count >= 0 '
'AND completed_query_count <= expected_query_count'
),
'docker_discovery_passes_identity_check': (
'length(pass_token) >= 16 AND length(pass_token) <= 256 '
'AND length(source) >= 1 AND length(source) <= 64 '
f'AND {_sql_sha256_check("policy_sha256")} '
f'AND {_sql_sha256_check("ordered_queries_sha256")}'
),
'docker_discovery_passes_complete_check': (
"state <> 'complete' OR (completed_query_count = expected_query_count "
'AND completed_at IS NOT NULL)'
),
},
'docker_discovery_pages': {
'docker_discovery_pages_range_check': (
'query_ordinal >= 0 AND page_number >= 1 AND page_number <= 30 '
'AND result_count >= 0 AND admitted_count >= 0 '
'AND admitted_count <= result_count'
),
'docker_discovery_pages_boolean_check': '(query_complete = 0 OR query_complete = 1)',
'docker_discovery_pages_kind_check': _sql_enum_check(
'admission_kind', ('main', 'retry'),
),
'docker_discovery_pages_identity_check': (
'length(query) >= 1 AND length(query) <= 256 '
f'AND {_sql_sha256_check("page_sha256")}'
),
'docker_discovery_pages_evidence_check': (
"(admission_kind = 'main' AND retry_work_id IS NULL) "
"OR admission_kind = 'retry'"
),
'docker_discovery_pages_total_count_check': (
'total_count IS NULL OR (total_count >= 0 AND total_count >= result_count)'
),
},
'docker_repository_query_provenance': {
'docker_repository_query_provenance_kind_check': _sql_enum_check(
'provenance_kind', ('legacy_queue', 'fresh_page'),
),
'docker_repository_query_provenance_range_check': (
'(first_search_rank IS NULL OR first_search_rank >= 1) '
'AND (best_search_rank IS NULL OR best_search_rank >= 1) '
'AND (last_search_rank IS NULL OR last_search_rank >= 1) '
'AND observation_count >= 1 AND fresh_observation_count >= 0 '
'AND fresh_complete_observation_count >= 0'
),
'docker_repository_query_provenance_boolean_check': (
'(fresh_coverage_eligible = 0 OR fresh_coverage_eligible = 1)'
),
'docker_repository_query_provenance_identity_check': (
'length(source) >= 1 AND length(source) <= 64 '
'AND length(query) >= 1 AND length(query) <= 256 '
f'AND {_sql_sha256_check("first_policy_sha256", nullable=True)} '
f'AND {_sql_sha256_check("last_policy_sha256", nullable=True)}'
),
'docker_repository_query_provenance_counts_check': (
'fresh_complete_observation_count <= fresh_observation_count '
'AND fresh_observation_count <= observation_count'
),
'docker_repository_query_provenance_legacy_check': (
"provenance_kind <> 'legacy_queue' OR (fresh_observation_count = 0 "
'AND fresh_complete_observation_count = 0 AND fresh_coverage_eligible = 0 '
'AND first_page_id IS NULL AND last_page_id IS NULL '
'AND first_policy_sha256 IS NULL AND last_policy_sha256 IS NULL)'
),
'docker_repository_query_provenance_fresh_check': (
"provenance_kind <> 'fresh_page' OR (fresh_observation_count >= 1 "
'AND first_search_rank IS NOT NULL AND best_search_rank IS NOT NULL '
'AND last_search_rank IS NOT NULL AND first_page_id IS NOT NULL '
'AND last_page_id IS NOT NULL AND first_policy_sha256 IS NOT NULL '
'AND last_policy_sha256 IS NOT NULL)'
),
'docker_repository_query_provenance_eligibility_check': (
'(fresh_complete_observation_count = 0 AND fresh_coverage_eligible = 0) '
"OR (fresh_complete_observation_count >= 1 AND fresh_coverage_eligible = 1 "
"AND provenance_kind = 'fresh_page')"
),
},
'docker_repository_query_observations': {
'docker_repository_query_observations_rank_check': 'search_rank >= 1',
},
'docker_image_manifests': {
'docker_image_manifests_range_check': (
'layer_count >= 0 AND (manifest_size_bytes IS NULL OR manifest_size_bytes >= 0)'
),
'docker_image_manifests_identity_check': (
'length(source) >= 1 AND length(source) <= 64 '
'AND length(repository) >= 1 AND length(repository) <= 512 '
'AND length(manifest_digest) >= 1 AND length(manifest_digest) <= 256 '
'AND length(manifest_media_type) >= 1 AND length(manifest_media_type) <= 256 '
'AND (config_digest IS NULL OR (length(config_digest) >= 1 '
'AND length(config_digest) <= 256)) '
f'AND {_sql_sha256_check("graph_sha256")}'
),
},
'docker_manifest_layers': {
'docker_manifest_layers_range_check': (
'position_from_base >= 1 AND position_from_top >= 1 AND layer_size_bytes >= 0'
),
'docker_manifest_layers_identity_check': (
'length(layer_digest) >= 1 AND length(layer_digest) <= 256 '
'AND length(media_type) >= 1 AND length(media_type) <= 256 '
f'AND {_sql_sha256_check("descriptor_sha256")}'
),
},
'docker_depth_experiment_repositories': {
'docker_depth_experiment_repositories_range_check': (
'query_ordinal >= 0 AND repository_rank >= 1 AND repository_rank <= 39 '
'AND resolver_generation >= 0 AND resolver_attempts >= 0 '
'AND selected_image_count >= 0 AND selected_image_count <= 10'
),
'docker_depth_experiment_repositories_boolean_check': (
'(is_deep_probe = 0 OR is_deep_probe = 1)'
),
'docker_depth_experiment_repositories_state_check': _sql_enum_check(
'work_state', DOCKER_EXPERIMENT_REPOSITORY_STATES,
),
'docker_depth_experiment_repositories_fence_check': (
"(work_state = 'resolving' AND resolver_owner IS NOT NULL "
'AND resolver_token IS NOT NULL AND resolver_expires_at IS NOT NULL) '
"OR (work_state <> 'resolving' AND resolver_owner IS NULL "
'AND resolver_token IS NULL AND resolver_expires_at IS NULL)'
),
'docker_depth_experiment_repositories_resolver_check': (
'(planned_is_deep_probe = 0 OR planned_is_deep_probe = 1) '
'AND resolver_attempts >= 0 AND resolver_attempts <= 3 '
'AND candidate_distinct_graph_count >= 0 '
'AND candidate_distinct_graph_count <= 100 '
'AND selected_image_count <= candidate_distinct_graph_count '
'AND replacement_count >= 0 AND replacement_count <= 3000'
),
'docker_depth_experiment_repositories_replacement_check': (
'(replacement_count = 0 AND replacement_repository_queue_id IS NULL '
'AND replacement_eligibility_page_id IS NULL '
'AND replacement_evidence_sha256 IS NULL) OR '
'(replacement_count >= 1 AND replacement_repository_queue_id IS NOT NULL '
'AND replacement_eligibility_page_id IS NOT NULL '
f'AND {_sql_sha256_check("replacement_evidence_sha256")})'
),
},
'docker_depth_experiment_targets': {
'docker_depth_experiment_targets_range_check': (
'counter_ordinal >= 1 AND counter_ordinal <= 2000 '
'AND dispatch_wave >= 1 AND dispatch_wave <= 3 '
'AND dispatch_order >= 1 AND reservation_count >= 0'
),
'docker_depth_experiment_targets_state_check': _sql_enum_check(
'state', DOCKER_EXPERIMENT_TARGET_STATES,
),
},
'docker_depth_experiment_selections': {
'docker_depth_experiment_selections_range_check': (
'query_ordinal >= 0 AND image_rank >= 1 AND image_rank <= 10'
),
'docker_depth_experiment_selections_identity_check': (
'length(selection_reason) >= 1 AND length(selection_reason) <= 128 '
f'AND {_sql_sha256_check("selection_evidence_sha256")} '
f'AND {_sql_sha256_check("graph_sha256")}'
),
},
'docker_depth_experiment_candidate_skips': {
'docker_depth_experiment_candidate_skips_kind_check': _sql_enum_check(
'candidate_kind', ('image', 'repository'),
),
'docker_depth_experiment_candidate_skips_range_check': (
'candidate_ordinal >= 1 AND candidate_ordinal <= 3000'
),
'docker_depth_experiment_candidate_skips_identity_check': (
f'{_sql_sha256_check("candidate_identity_sha256")} '
f'AND {_sql_sha256_check("evidence_sha256")}'
),
},
'docker_depth_resolver_attempt_refunds': {
'docker_depth_resolver_attempt_refunds_kind_check': (
"recovery_kind = 'zero_graph_limit_v1'"
),
'docker_depth_resolver_attempt_refunds_state_check': (
"(prior_work_state = 'pending' OR prior_work_state = 'held') "
"AND next_work_state = 'pending'"
),
'docker_depth_resolver_attempt_refunds_range_check': (
'query_ordinal >= 0 AND repository_rank >= 1 '
'AND prior_resolver_attempts >= 2 AND refund_attempts = 2 '
'AND next_resolver_attempts = prior_resolver_attempts - refund_attempts '
'AND next_resolver_attempts >= 0 AND confirmed_bug_event_count = 2'
),
'docker_depth_resolver_attempt_refunds_hash_check': (
f'{_sql_sha256_check("manifest_sha256")} '
f'AND {_sql_sha256_check("entry_evidence_sha256")} '
f'AND {_sql_sha256_check("log_sha256")} '
f'AND {_sql_sha256_check("target_identity_sha256")} '
f'AND {_sql_sha256_check("prior_error_code_sha256")}'
),
},
'docker_depth_resolver_dispositions': {
'docker_depth_resolver_dispositions_kind_check': (
"disposition_kind = 'resolver_attempt_limit_replace_or_skip_v1'"
),
'docker_depth_resolver_dispositions_outcome_check': (
"outcome = 'replaced' OR outcome = 'skipped'"
),
'docker_depth_resolver_dispositions_state_check': (
"prior_work_state = 'held' AND ("
"(outcome = 'replaced' AND next_work_state = 'pending' "
"AND replacement_repository_queue_id IS NOT NULL "
"AND replacement_eligibility_page_id IS NOT NULL "
"AND replacement_best_search_rank IS NOT NULL "
"AND replacement_target_identity_sha256 IS NOT NULL "
"AND next_resolver_attempts = 0) OR "
"(outcome = 'skipped' AND next_work_state = 'skipped' "
"AND replacement_repository_queue_id IS NULL "
"AND replacement_eligibility_page_id IS NULL "
"AND replacement_best_search_rank IS NULL "
"AND replacement_target_identity_sha256 IS NULL "
"AND next_resolver_attempts = prior_resolver_attempts))"
),
'docker_depth_resolver_dispositions_range_check': (
'query_ordinal >= 0 AND repository_rank >= 1 '
'AND prior_resolver_attempts >= 3 AND next_resolver_attempts >= 0 '
'AND (replacement_best_search_rank IS NULL '
'OR replacement_best_search_rank >= 1)'
),
'docker_depth_resolver_dispositions_hash_check': (
f'{_sql_sha256_check("manifest_sha256")} '
f'AND {_sql_sha256_check("entry_evidence_sha256")} '
f'AND {_sql_sha256_check("candidate_snapshot_sha256")} '
f'AND {_sql_sha256_check("prior_target_identity_sha256")} '
f'AND {_sql_sha256_check("prior_error_code_sha256")} '
f'AND (replacement_target_identity_sha256 IS NULL '
f'OR {_sql_sha256_check("replacement_target_identity_sha256")})'
),
},
'docker_depth_experiment_scan_bindings': {
'docker_depth_experiment_scan_bindings_range_check': 'attempt >= 1',
'docker_depth_experiment_scan_bindings_state_check': _sql_enum_check(
'state', DOCKER_EXPERIMENT_BINDING_STATES,
),
},
'docker_finding_layer_attributions': {
'docker_finding_layer_attributions_state_check': _sql_enum_check(
'attribution_state', ('exact', 'unattributed'),
),
'docker_finding_layer_attributions_evidence_check': (
"(attribution_state = 'exact' AND manifest_layer_id IS NOT NULL "
'AND reported_layer_digest IS NOT NULL AND position_from_base IS NOT NULL '
'AND position_from_top IS NOT NULL AND position_from_base >= 1 '
"AND position_from_top >= 1) OR (attribution_state = 'unattributed' "
'AND manifest_layer_id IS NULL AND reported_layer_digest IS NULL '
'AND position_from_base IS NULL AND position_from_top IS NULL)'
),
'docker_finding_layer_attributions_reason_check': (
"(attribution_state = 'exact' AND unattributed_reason IS NULL) OR "
"(attribution_state = 'unattributed' AND unattributed_reason IS NOT NULL AND "
+ _sql_enum_check(
'unattributed_reason', DOCKER_FINDING_UNATTRIBUTED_REASONS,
)
+ ')'
),
},
}
DOCKER_DEPTH_SCHEMA_AUTHORITY_CHECKS = (
('discovery_retry_queue', 'discovery_retry_queue_sha256_check'),
('target_queue_policy_events', 'target_queue_policy_events_transition_check'),
('target_queue_policy_events', 'target_queue_policy_events_hash_check'),
(
'target_queue_policy_events',
'target_queue_policy_events_experiment_ownership_check',
),
('docker_depth_experiments', 'docker_depth_experiments_selection_check'),
('docker_discovery_pages', 'docker_discovery_pages_total_count_check'),
(
'docker_depth_experiment_repositories',
'docker_depth_experiment_repositories_resolver_check',
),
(
'docker_depth_experiment_repositories',
'docker_depth_experiment_repositories_replacement_check',
),
(
'docker_depth_experiment_candidate_skips',
'docker_depth_experiment_candidate_skips_kind_check',
),
(
'docker_depth_experiment_candidate_skips',
'docker_depth_experiment_candidate_skips_range_check',
),
(
'docker_depth_experiment_candidate_skips',
'docker_depth_experiment_candidate_skips_identity_check',
),
)
DOCKER_DEPTH_SCARCITY_COHORT_CHECKS = (
(
'docker_depth_experiment_queries',
'docker_depth_experiment_queries_selection_check',
),
)
_CHECK_TOKEN = re.compile(
r"'(?:''|[^'])*'|<>|>=|<=|!=|=|>|<|[A-Za-z_][A-Za-z0-9_$]*|"
r'\d+|[(),+*\-]'
)
def _normalized_check_expression(value):
text = str(value or '').strip().lower().replace('"', '')
if text.startswith('check'):
text = text[5:].strip()
text = re.sub(
r'::(?:text|character varying|character|varchar|integer|bigint|smallint|numeric)',
'',
text,
)
tokens = []
position = 0
while position < len(text):
if text[position].isspace():
position += 1
continue
match = _CHECK_TOKEN.match(text, position)
if not match:
return ('unparsed', re.sub(r'\s+', '', text.replace('!=', '<>')))
token = match.group(0)
tokens.append('<>' if token == '!=' else token)
position = match.end()
cursor = 0
def combine(operator, values):
flattened = []
for item in values:
if item and item[0] == operator:
flattened.extend(item[1])
else:
flattened.append(item)
return flattened[0] if len(flattened) == 1 else (operator, tuple(flattened))
def parse_primary():
nonlocal cursor
if cursor >= len(tokens):
raise ValueError('missing CHECK expression operand')
token = tokens[cursor]
if token == '(':
cursor += 1
expression = parse_or()
if cursor >= len(tokens) or tokens[cursor] != ')':
raise ValueError('unclosed CHECK expression group')
cursor += 1
return expression
cursor += 1
if re.fullmatch(r'[a-z_][a-z0-9_$]*', token) and (
cursor < len(tokens) and tokens[cursor] == '('
):
cursor += 1
arguments = []
if cursor < len(tokens) and tokens[cursor] != ')':
while True:
arguments.append(parse_or())
if cursor >= len(tokens) or tokens[cursor] != ',':
break
cursor += 1
if cursor >= len(tokens) or tokens[cursor] != ')':
raise ValueError('unclosed CHECK expression function')
cursor += 1
return ('call', token, tuple(arguments))
if token.startswith("'"):
return ('string', token[1:-1].replace("''", "'"))
if token.isdigit():
return ('number', int(token))
return ('name', token)
def parse_multiply():
nonlocal cursor
expression = parse_primary()
while cursor < len(tokens) and tokens[cursor] == '*':
operator = tokens[cursor]
cursor += 1
expression = (operator, expression, parse_primary())
return expression
def parse_add():
nonlocal cursor
expression = parse_multiply()
while cursor < len(tokens) and tokens[cursor] in ('+', '-'):
operator = tokens[cursor]
cursor += 1
expression = (operator, expression, parse_multiply())
return expression
def parse_comparison():
nonlocal cursor
expression = parse_add()
if cursor < len(tokens) and tokens[cursor] == 'is':
cursor += 1
negated = cursor < len(tokens) and tokens[cursor] == 'not'
if negated:
cursor += 1
if cursor >= len(tokens) or tokens[cursor] != 'null':
raise ValueError('unsupported CHECK IS expression')
cursor += 1
return ('is not null' if negated else 'is null', expression)
if cursor < len(tokens) and tokens[cursor] in ('=', '<>', '>=', '<=', '>', '<'):
operator = tokens[cursor]
cursor += 1
return (operator, expression, parse_add())
return expression
def parse_and():
nonlocal cursor
values = [parse_comparison()]
while cursor < len(tokens) and tokens[cursor] == 'and':
cursor += 1
values.append(parse_comparison())
return combine('and', values)
def parse_or():
nonlocal cursor
values = [parse_and()]
while cursor < len(tokens) and tokens[cursor] == 'or':
cursor += 1
values.append(parse_and())
return combine('or', values)
try:
normalized = parse_or()
if cursor != len(tokens):
raise ValueError('unsupported trailing CHECK expression')
return normalized
except ValueError:
return ('unparsed', re.sub(r'\s+', '', text.replace('!=', '<>')))
def _docker_depth_check_constraint_problems(conn):
problems = []
for table, expected_constraints in DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS.items():
if not conn.table_exists(table):
continue
constraints = conn.table_check_constraints(table)
for name, expected_expression in expected_constraints.items():
constraint = constraints.get(name)
if (
not constraint
or not constraint.get('valid', True)
or _normalized_check_expression(constraint.get('expression'))
!= _normalized_check_expression(expected_expression)
):
problems.append(f'check constraint {table}.{name}')
return problems
def _docker_depth_check_clause(table, name):
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
return f'CONSTRAINT {name} CHECK ({expression})'
def _index_usable(index):
return bool(index and index.get('valid', True) and index.get('ready', True) and index.get('live', True))
def _normalized_trigger_sql(value):
return re.sub(r'[\s"]', '', str(value or '').lower())
def _runtime_audit_trigger_problems(conn):
if not conn.table_exists('runtime_audit_events'):
return []
triggers = conn.table_triggers('runtime_audit_events')
if conn.is_postgres:
expected = {
'runtime_audit_events_reject_mutation': (
'before', 'update', 'delete', 'runtime_audit_events',
'executefunctionreject_runtime_audit_event_mutation()',
),
'runtime_audit_events_reject_truncate': (
'before', 'truncate', 'runtime_audit_events',
'executefunctionreject_runtime_audit_event_mutation()',
),
}
else:
expected = {
'runtime_audit_events_reject_update': (
'beforeupdateonruntime_audit_events',
"raise(abort,'runtime_audit_eventsisappend-only')",
),
'runtime_audit_events_reject_delete': (
'beforedeleteonruntime_audit_events',
"raise(abort,'runtime_audit_eventsisappend-only')",
),
}
problems = []
for name, fragments in expected.items():
trigger = triggers.get(name)
sql = _normalized_trigger_sql((trigger or {}).get('sql'))
if (
not trigger
or not trigger.get('enabled', True)
or any(fragment not in sql for fragment in fragments)
or (
conn.is_postgres
and "raiseexception'runtime_audit_eventsisappend-only'"
not in _normalized_trigger_sql(trigger.get('function_sql'))
)
):
problems.append(f'trigger {name}')
return problems
def _column_generates_id(details, postgres):
if not details or not details.get('primary_key') or details.get('generated'):
return False
if not postgres:
return str(details.get('type') or '').strip().lower() == 'integer'
identity = str(details.get('identity') or '')
sequence = str(details.get('sequence') or '')
default_sql = str(details.get('default') or '')
if identity in ('a', 'd'):
return bool(sequence)
return bool(sequence and re.search(r'\bnextval\s*\(', default_sql, re.IGNORECASE))
def utc_now_iso():
return datetime.now(timezone.utc).isoformat(timespec='seconds')
def _worker_contract_timestamp(value):
parsed = datetime.fromisoformat(str(value).replace('Z', '+00:00'))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc).isoformat(
timespec='seconds'
).replace('+00:00', 'Z')
def _canonical_runtime_json(value):
try:
return json.dumps(
value, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
allow_nan=False,
)
except (TypeError, ValueError) as exc:
raise ValueError('runtime authority identity must be canonical JSON data') from exc
def _runtime_operation_id(value):
if not isinstance(value, str) or len(value) != 36:
raise ValueError('runtime operation ID must be a canonical UUID')
try:
parsed = uuid.UUID(value)
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError('runtime operation ID must be a canonical UUID') from exc
if parsed.int == 0 or str(parsed) != value:
raise ValueError('runtime operation ID must be a canonical UUID')
return value
def _runtime_actor(value):
if not isinstance(value, str) or not 1 <= len(value) <= 256:
raise ValueError('runtime operation actor is invalid')
if any(ord(character) < 32 or ord(character) == 127 for character in value):
raise ValueError('runtime operation actor is invalid')
return value
def _runtime_revision(value):
if (
isinstance(value, bool) or not isinstance(value, int)
or not 0 <= value <= RUNTIME_CONTROL_MAX_REVISION
):
raise ValueError('runtime control revision is invalid')
return value
def _runtime_control_identity(value):
if not isinstance(value, dict) or set(value) != {
'discovery_paused', 'dispatch_paused', 'drain_state', 'revision',
}:
raise RuntimeSafetySchemaError('runtime control identity shape is invalid')
if type(value['discovery_paused']) is not bool or type(value['dispatch_paused']) is not bool:
raise RuntimeSafetySchemaError('runtime control pause identity is invalid')
drain_state = value['drain_state']
if drain_state not in RUNTIME_CONTROL_DRAIN_STATES:
raise RuntimeSafetySchemaError('runtime control drain identity is invalid')
try:
revision = _runtime_revision(value['revision'])
except ValueError as exc:
raise RuntimeSafetySchemaError('runtime control revision identity is invalid') from exc
return {
'discovery_paused': value['discovery_paused'],
'dispatch_paused': value['dispatch_paused'],
'drain_state': drain_state,
'revision': revision,
}
def _runtime_control_identity_json(value):
return _canonical_runtime_json(_runtime_control_identity(value))
def _stored_runtime_control_identity(value):
if not isinstance(value, str) or not value:
raise RuntimeSafetySchemaError('runtime control identity JSON is missing')
try:
parsed = json.loads(value)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('runtime control identity JSON is invalid') from exc
canonical = _runtime_control_identity_json(parsed)
if not hmac.compare_digest(canonical, value):
raise RuntimeSafetySchemaError('runtime control identity JSON is not canonical')
return parsed
def _runtime_control_state_from_row(row):
if not row:
raise RuntimeSafetySchemaError('runtime operations control singleton is missing')
try:
revision = row['revision']
discovery_raw = row['discovery_paused']
dispatch_raw = row['dispatch_paused']
except (KeyError, TypeError) as exc:
raise RuntimeSafetySchemaError('runtime operations control state is invalid') from exc
if (
isinstance(revision, bool) or not isinstance(revision, int)
or revision < 0 or revision > RUNTIME_CONTROL_MAX_REVISION
):
raise RuntimeSafetySchemaError('runtime operations control revision is invalid')
if (
isinstance(discovery_raw, bool) or not isinstance(discovery_raw, int)
or isinstance(dispatch_raw, bool) or not isinstance(dispatch_raw, int)
or discovery_raw not in (0, 1) or dispatch_raw not in (0, 1)
):
raise RuntimeSafetySchemaError('runtime operations control pause state is invalid')
drain_state = str(row['drain_state'] or '')
if drain_state not in RUNTIME_CONTROL_DRAIN_STATES:
raise RuntimeSafetySchemaError('runtime operations control drain state is invalid')
actor = str(row['actor'] or '')
try:
_runtime_actor(actor)
except ValueError as exc:
raise RuntimeSafetySchemaError('runtime operations control actor is invalid') from exc
operation_id = row['operation_id']
if operation_id is not None:
try:
operation_id = _runtime_operation_id(str(operation_id))
except ValueError as exc:
raise RuntimeSafetySchemaError('runtime operations control operation ID is invalid') from exc
created_at = str(row['created_at'] or '')
updated_at = str(row['updated_at'] or '')
if not created_at or not updated_at:
raise RuntimeSafetySchemaError('runtime operations control timestamps are invalid')
discovery_paused = bool(discovery_raw)
dispatch_paused = bool(dispatch_raw)
draining = drain_state != 'normal'
return {
'revision': revision,
'discovery_paused': discovery_paused,
'dispatch_paused': dispatch_paused,
'drain_state': drain_state,
'effective_discovery_paused': discovery_paused or draining,
'effective_dispatch_paused': dispatch_paused or draining,
'actor': actor,
'operation_id': operation_id,
'created_at': created_at,
'updated_at': updated_at,
}
def _runtime_control_state_identity(state):
return {
'discovery_paused': state['discovery_paused'],
'dispatch_paused': state['dispatch_paused'],
'drain_state': state['drain_state'],
'revision': state['revision'],
}
def _runtime_control_transition(before, action):
before = _runtime_control_identity(before)
after = dict(before)
if before['revision'] >= RUNTIME_CONTROL_MAX_REVISION:
raise RuntimeControlTransitionError('runtime control revision cannot be advanced')
if action == 'control.discovery.pause':
if before['discovery_paused']:
raise RuntimeControlTransitionError(
'runtime discovery control is already in the requested state'
)
after['discovery_paused'] = True
elif action == 'control.discovery.resume':
if not before['discovery_paused']:
raise RuntimeControlTransitionError(
'runtime discovery control is already in the requested state'
)
after['discovery_paused'] = False
elif action == 'control.dispatch.pause':
if before['dispatch_paused']:
raise RuntimeControlTransitionError(
'runtime dispatch control is already in the requested state'
)
after['dispatch_paused'] = True
elif action == 'control.dispatch.resume':
if not before['dispatch_paused']:
raise RuntimeControlTransitionError(
'runtime dispatch control is already in the requested state'
)
after['dispatch_paused'] = False
elif action == 'control.drain.start':
if before['drain_state'] != 'normal':
raise RuntimeControlTransitionError('runtime drain is already active')
after['drain_state'] = 'draining'
elif action == 'control.drain.cancel':
if before['drain_state'] not in ('draining', 'drained'):
raise RuntimeControlTransitionError('runtime drain is not active')
after['drain_state'] = 'normal'
elif action == 'control.drain.complete':
if before['drain_state'] != 'draining':
raise RuntimeControlTransitionError('runtime drain is not awaiting completion')
after['drain_state'] = 'drained'
else:
raise ValueError('runtime control action is invalid')
after['revision'] = before['revision'] + 1
return after
def _runtime_audit_event_sha256(payload):
previous = payload.get('previous_event_sha256')
if previous is None:
previous_digest = b'\x00' * 32
elif re.fullmatch(r'[a-f0-9]{64}', str(previous or '')):
previous_digest = bytes.fromhex(str(previous))
else:
raise RuntimeSafetySchemaError('runtime audit parent identity is invalid')
encoded = _canonical_runtime_json(payload).encode('ascii')
return hashlib.sha256(
previous_digest + hashlib.sha256(encoded).digest()
).hexdigest()
def _runtime_sha256(value, field):
if not isinstance(value, str) or not re.fullmatch(r'[a-f0-9]{64}', value):
raise ValueError(f'runtime operation {field} is invalid')
return value
def _runtime_safe_code(value, field, *, required=False):
if value is None and not required:
return None
allowed = (
RUNTIME_OPERATION_SAFE_CATEGORIES
if field == 'safe category' else RUNTIME_OPERATION_SAFE_DETAILS
)
if not isinstance(value, str) or value not in allowed:
raise ValueError(f'runtime operation {field} is invalid')
return value
def _runtime_audit_row_hash(row):
event_id = row['id']
if type(event_id) is not int or event_id < 1:
raise ValueError('audit event ID')
operation_id = row['operation_id']
if operation_id is not None:
operation_id = str(operation_id)
actor = str(row['actor'] or '')
action = str(row['action'] or '')
target_kind = str(row['target_kind'] or '')
target_ref = str(row['target_ref'] or '')
result = str(row['result'] or '')
safe_category = row['safe_category']
if safe_category is not None:
safe_category = str(safe_category)
identities = []
for field in ('before_identity_json', 'after_identity_json'):
raw = row[field]
if raw is None:
identities.append(None)
continue
raw = str(raw)
identities.append(raw)
byte_counts = []
for field in ('before_bytes', 'after_bytes'):
value = row[field]
if value is not None and (
type(value) is not int
or value < 0
):
raise ValueError('audit byte count')
byte_counts.append(value)
previous_event_id = row['previous_event_id']
previous_event_sha256 = row['previous_event_sha256']
if previous_event_id is None:
if previous_event_sha256 is not None:
raise ValueError('audit parent')
else:
if (
type(previous_event_id) is not int
or previous_event_id < 1
or previous_event_id >= event_id
):
raise ValueError('audit parent')
previous_event_sha256 = _runtime_sha256(
str(previous_event_sha256 or ''), 'audit parent hash',
)
created_at = str(row['created_at'] or '')
event_sha256 = _runtime_sha256(
str(row['event_sha256'] or ''), 'audit event hash',
)
payload = {
'schema': 'runtime-audit-event-v1',
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': target_kind,
'target_ref': target_ref,
'result': result,
'safe_category': safe_category,
'before_identity_json': identities[0],
'after_identity_json': identities[1],
'before_bytes': byte_counts[0],
'after_bytes': byte_counts[1],
'previous_event_id': previous_event_id,
'previous_event_sha256': previous_event_sha256,
'created_at': created_at,
}
if not hmac.compare_digest(
_runtime_audit_event_sha256(payload), event_sha256,
):
raise ValueError('audit event hash')
return event_id, event_sha256
def _runtime_expected_identity(value, action):
fields = {
'active_config_sha256', 'active_secrets_sha256',
'candidate_config_sha256', 'candidate_secrets_sha256',
}
if not isinstance(value, dict) or set(value) != fields:
raise ValueError('runtime operation expected identity is invalid')
identity = {
'active_config_sha256': _runtime_sha256(
value['active_config_sha256'], 'active config hash',
),
'active_secrets_sha256': _runtime_sha256(
value['active_secrets_sha256'], 'active secrets hash',
),
'candidate_config_sha256': value['candidate_config_sha256'],
'candidate_secrets_sha256': value['candidate_secrets_sha256'],
}
required_candidates = {
'apply-config': (True, False),
'apply-secrets': (False, True),
'apply-both': (True, True),
'restart': (False, False),
}
if action not in required_candidates:
raise ValueError('runtime operation action is invalid')
for key, required in zip(
('candidate_config_sha256', 'candidate_secrets_sha256'),
required_candidates[action],
):
current = identity[key]
if required:
identity[key] = _runtime_sha256(current, key.replace('_', ' '))
elif current is not None:
raise ValueError('runtime operation expected identity is invalid')
return identity
def _runtime_resulting_identity(value):
if value is None:
return None
if not isinstance(value, dict) or set(value) != {
'active_config_sha256', 'active_secrets_sha256',
}:
raise ValueError('runtime operation resulting identity is invalid')
return {
'active_config_sha256': _runtime_sha256(
value['active_config_sha256'], 'resulting config hash',
),
'active_secrets_sha256': _runtime_sha256(
value['active_secrets_sha256'], 'resulting secrets hash',
),
}
def _runtime_success_identity(expected, action):
return {
'active_config_sha256': (
expected['candidate_config_sha256']
if action in ('apply-config', 'apply-both')
else expected['active_config_sha256']
),
'active_secrets_sha256': (
expected['candidate_secrets_sha256']
if action in ('apply-secrets', 'apply-both')
else expected['active_secrets_sha256']
),
}
def _runtime_source_id(value):
if (
type(value) is str
and value != 'all'
and re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:@-]{0,127}', value)
):
return value
return None
def _runtime_source_expected_identity(value, action, target_ref):
source_action = RUNTIME_SOURCE_OPERATION_ACTIONS.get(action)
if source_action is None or _runtime_source_id(target_ref) is None or not isinstance(value, dict):
raise ValueError('runtime source operation identity is invalid')
legacy_fields = {'source_id', 'source_action', 'interval_seconds'}
extended_fields = legacy_fields | {
'mode', 'restart_enabled', 'restart_delay_seconds',
}
if set(value) not in (legacy_fields, extended_fields):
raise ValueError('runtime source operation identity is invalid')
if value.get('source_id') != target_ref or value.get('source_action') != source_action:
raise ValueError('runtime source operation identity is invalid')
interval = value.get('interval_seconds')
if source_action == 'set-interval':
if type(interval) is not int or not 1 <= interval <= 365 * 24 * 60 * 60:
raise ValueError('runtime source operation interval is invalid')
elif interval is not None:
raise ValueError('runtime source operation identity is invalid')
normalized = {
'source_id': target_ref,
'source_action': source_action,
'interval_seconds': interval,
}
if set(value) == legacy_fields:
if source_action not in ('start', 'stop', 'restart', 'pause', 'resume', 'set-interval'):
raise ValueError('runtime source operation identity is invalid')
return normalized
mode = value.get('mode')
restart_enabled = value.get('restart_enabled')
restart_delay = value.get('restart_delay_seconds')
if source_action == 'set-mode':
if mode not in ('loop', 'once', 'repeat'):
raise ValueError('runtime source operation mode is invalid')
elif mode is not None:
raise ValueError('runtime source operation identity is invalid')
if source_action == 'set-restart':
if type(restart_enabled) is not bool:
raise ValueError('runtime source operation restart setting is invalid')
elif restart_enabled is not None:
raise ValueError('runtime source operation identity is invalid')
if source_action == 'set-restart-delay':
if (
type(restart_delay) is not int
or not 1 <= restart_delay <= 365 * 24 * 60 * 60
):
raise ValueError('runtime source operation restart delay is invalid')
elif restart_delay is not None:
raise ValueError('runtime source operation identity is invalid')
if source_action == 'once' and any(
current is not None for current in (mode, restart_enabled, restart_delay)
):
raise ValueError('runtime source operation identity is invalid')
normalized.update({
'mode': mode,
'restart_enabled': restart_enabled,
'restart_delay_seconds': restart_delay,
})
return normalized
def _runtime_source_resulting_identity(value, action, target_ref):
source_action = RUNTIME_SOURCE_OPERATION_ACTIONS.get(action)
if (
source_action is None
or _runtime_source_id(target_ref) is None
or not isinstance(value, dict)
or set(value) != {'source_id', 'source_action', 'outcome'}
or value.get('source_id') != target_ref
or value.get('source_action') != source_action
or value.get('outcome') not in ('completed', 'dependency-blocked')
):
raise ValueError('runtime source operation result is invalid')
return {
'source_id': target_ref,
'source_action': source_action,
'outcome': value['outcome'],
}
def _stored_runtime_source_identity(value, action, target_ref, *, resulting=False):
if not isinstance(value, str) or not value:
raise RuntimeSafetySchemaError('runtime source operation identity is missing')
try:
parsed = json.loads(value)
normalized = (
_runtime_source_resulting_identity(parsed, action, target_ref)
if resulting else
_runtime_source_expected_identity(parsed, action, target_ref)
)
canonical = _canonical_runtime_json(normalized)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('runtime source operation identity is invalid') from exc
if not hmac.compare_digest(canonical, value):
raise RuntimeSafetySchemaError('runtime source operation identity is not canonical')
return normalized
def _runtime_worker_admin_target(action, target_ref):
target_type = RUNTIME_WORKER_ADMIN_ACTION_TARGETS.get(action)
if target_type is None or not isinstance(target_ref, str):
raise ValueError('runtime worker admin operation target is invalid')
if target_type == 'deferred-queue':
if target_ref != 'deferred-queue':
raise ValueError('runtime worker admin operation target is invalid')
elif not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:@-]{0,127}', target_ref):
raise ValueError('runtime worker admin operation target is invalid')
return target_ref
def _runtime_worker_admin_expected_identity(value, action, target_ref):
target_ref = _runtime_worker_admin_target(action, target_ref)
if (
not isinstance(value, dict)
or set(value) != {'action', 'target_ref', 'request_sha256'}
or value.get('action') != action
or value.get('target_ref') != target_ref
):
raise ValueError('runtime worker admin operation identity is invalid')
return {
'action': action,
'target_ref': target_ref,
'request_sha256': _runtime_sha256(
value.get('request_sha256'), 'worker admin request hash',
),
}
def _runtime_worker_admin_resulting_identity(value, action, target_ref):
target_ref = _runtime_worker_admin_target(action, target_ref)
if (
not isinstance(value, dict)
or set(value) != {'action', 'target_ref', 'outcome', 'affected_count'}
or value.get('action') != action
or value.get('target_ref') != target_ref
or value.get('outcome') != 'completed'
or type(value.get('affected_count')) is not int
or not 0 <= value['affected_count'] <= 10000
):
raise ValueError('runtime worker admin operation result is invalid')
return {
'action': action,
'target_ref': target_ref,
'outcome': 'completed',
'affected_count': value['affected_count'],
}
def _stored_runtime_worker_admin_identity(
value, action, target_ref, *, resulting=False,
):
if not isinstance(value, str) or not value:
raise RuntimeSafetySchemaError('runtime worker admin operation identity is missing')
try:
parsed = json.loads(value)
normalized = (
_runtime_worker_admin_resulting_identity(parsed, action, target_ref)
if resulting else
_runtime_worker_admin_expected_identity(parsed, action, target_ref)
)
canonical = _canonical_runtime_json(normalized)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('runtime worker admin operation identity is invalid') from exc
if not hmac.compare_digest(canonical, value):
raise RuntimeSafetySchemaError('runtime worker admin operation identity is not canonical')
return normalized
def _runtime_document_expected_identity(value, action, target_ref):
document = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
if (
document is None or target_ref != document
or not isinstance(value, dict)
or set(value) != {
'document', 'active_config_sha256', 'active_secrets_sha256',
'candidate_config_sha256', 'candidate_secrets_sha256',
'candidate_after_sha256',
'candidate_before_bytes', 'candidate_after_bytes',
'candidate_before_present',
}
or value.get('document') != document
):
raise ValueError('runtime document operation identity is invalid')
before_bytes = value.get('candidate_before_bytes')
after_bytes = value.get('candidate_after_bytes')
before_present = value.get('candidate_before_present')
if (
type(before_bytes) is not int or type(after_bytes) is not int
or type(before_present) is not bool
or not 0 <= before_bytes <= 4 * 1024 * 1024
or not 0 <= after_bytes <= 4 * 1024 * 1024
):
raise ValueError('runtime document operation byte counts are invalid')
return {
'document': document,
'active_config_sha256': _runtime_sha256(
value.get('active_config_sha256'), 'active config hash',
),
'active_secrets_sha256': _runtime_sha256(
value.get('active_secrets_sha256'), 'active secrets hash',
),
'candidate_config_sha256': _runtime_sha256(
value.get('candidate_config_sha256'), 'candidate config hash',
),
'candidate_secrets_sha256': _runtime_sha256(
value.get('candidate_secrets_sha256'), 'candidate secrets hash',
),
'candidate_after_sha256': _runtime_sha256(
value.get('candidate_after_sha256'), 'candidate after hash',
),
'candidate_before_bytes': before_bytes,
'candidate_after_bytes': after_bytes,
'candidate_before_present': before_present,
}
def _runtime_document_resulting_identity(value, action, target_ref):
document = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
if (
document is None or target_ref != document
or not isinstance(value, dict)
or set(value) != {'document', 'outcome', 'candidate_sha256', 'written'}
or value.get('document') != document
or value.get('outcome') != 'completed'
or type(value.get('written')) is not bool
):
raise ValueError('runtime document operation result is invalid')
return {
'document': document,
'outcome': 'completed',
'candidate_sha256': _runtime_sha256(
value.get('candidate_sha256'), 'candidate result hash',
),
'written': value['written'],
}
def _stored_runtime_document_identity(value, action, target_ref, *, resulting=False):
if not isinstance(value, str) or not value:
raise RuntimeSafetySchemaError('runtime document operation identity is missing')
try:
parsed = json.loads(value)
normalized = (
_runtime_document_resulting_identity(parsed, action, target_ref)
if resulting else
_runtime_document_expected_identity(parsed, action, target_ref)
)
canonical = _canonical_runtime_json(normalized)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('runtime document operation identity is invalid') from exc
if not hmac.compare_digest(canonical, value):
raise RuntimeSafetySchemaError('runtime document operation identity is not canonical')
return normalized
def _runtime_managed_file_target(root_id, relative_path, target_ref):
if (
not isinstance(root_id, str)
or root_id != target_ref
or not re.fullmatch(r'[a-z][a-z0-9-]{0,63}', root_id)
or not isinstance(relative_path, str)
):
raise ValueError('runtime managed file operation target is invalid')
try:
encoded_path = relative_path.encode('utf-8', errors='strict')
except UnicodeEncodeError as exc:
raise ValueError('runtime managed file operation target is invalid') from exc
components = relative_path.split('/')
if (
not encoded_path or len(encoded_path) > 4096
or relative_path.startswith('/') or '\\' in relative_path or '\x00' in relative_path
or len(components) > 32
or any(
not component or component in ('.', '..')
or component.startswith('.truf-managed-file-')
or re.fullmatch(r'[A-Za-z]:.*', component)
or len(component.encode('utf-8')) > 255
for component in components
)
):
raise ValueError('runtime managed file operation target is invalid')
return root_id, relative_path
def _runtime_optional_sha256(value, label):
return None if value is None else _runtime_sha256(value, label)
def _runtime_optional_byte_count(value, label):
if value is None:
return None
if type(value) is not int or not 0 <= value <= 64 * 1024 * 1024:
raise ValueError(f'{label} is invalid')
return value
def _runtime_managed_file_expected_identity(value, action, target_ref):
if (
action not in RUNTIME_MANAGED_FILE_ACTIONS
or not isinstance(value, dict)
or set(value) != {
'root_id', 'relative_path', 'expected_sha256',
'proposed_sha256', 'proposed_byte_count',
}
):
raise ValueError('runtime managed file operation identity is invalid')
root_id, relative_path = _runtime_managed_file_target(
value.get('root_id'), value.get('relative_path'), target_ref,
)
expected_sha256 = _runtime_optional_sha256(
value.get('expected_sha256'), 'managed file expected hash',
)
proposed_sha256 = _runtime_optional_sha256(
value.get('proposed_sha256'), 'managed file proposed hash',
)
proposed_byte_count = _runtime_optional_byte_count(
value.get('proposed_byte_count'), 'managed file proposed byte count',
)
if (
action == 'files.create'
and (expected_sha256 is not None or proposed_sha256 is None or proposed_byte_count is None)
or action == 'files.replace'
and (expected_sha256 is None or proposed_sha256 is None or proposed_byte_count is None)
or action == 'files.delete'
and (expected_sha256 is None or proposed_sha256 is not None or proposed_byte_count is not None)
):
raise ValueError('runtime managed file operation identity is invalid')
return {
'root_id': root_id,
'relative_path': relative_path,
'expected_sha256': expected_sha256,
'proposed_sha256': proposed_sha256,
'proposed_byte_count': proposed_byte_count,
}
def _runtime_managed_file_resulting_identity(value, action, target_ref):
if (
action not in RUNTIME_MANAGED_FILE_ACTIONS
or not isinstance(value, dict)
or set(value) != {
'root_id', 'relative_path', 'outcome',
'before_sha256', 'before_byte_count',
'after_sha256', 'after_byte_count', 'written',
}
or value.get('outcome') != 'completed'
or type(value.get('written')) is not bool
):
raise ValueError('runtime managed file operation result is invalid')
root_id, relative_path = _runtime_managed_file_target(
value.get('root_id'), value.get('relative_path'), target_ref,
)
before_sha256 = _runtime_optional_sha256(
value.get('before_sha256'), 'managed file before hash',
)
after_sha256 = _runtime_optional_sha256(
value.get('after_sha256'), 'managed file after hash',
)
before_byte_count = _runtime_optional_byte_count(
value.get('before_byte_count'), 'managed file before byte count',
)
after_byte_count = _runtime_optional_byte_count(
value.get('after_byte_count'), 'managed file after byte count',
)
if (
action == 'files.create'
and (before_sha256 is not None or before_byte_count is not None
or after_sha256 is None or after_byte_count is None or not value['written'])
or action == 'files.replace'
and (before_sha256 is None or after_sha256 is None or after_byte_count is None)
or action == 'files.delete'
and (before_sha256 is None or after_sha256 is not None or after_byte_count is not None
or not value['written'])
):
raise ValueError('runtime managed file operation result is invalid')
return {
'root_id': root_id,
'relative_path': relative_path,
'outcome': 'completed',
'before_sha256': before_sha256,
'before_byte_count': before_byte_count,
'after_sha256': after_sha256,
'after_byte_count': after_byte_count,
'written': value['written'],
}
def _stored_runtime_managed_file_identity(
value, action, target_ref, *, resulting=False,
):
if not isinstance(value, str) or not value:
raise RuntimeSafetySchemaError('runtime managed file operation identity is missing')
try:
parsed = json.loads(value)
normalized = (
_runtime_managed_file_resulting_identity(parsed, action, target_ref)
if resulting else
_runtime_managed_file_expected_identity(parsed, action, target_ref)
)
canonical = _canonical_runtime_json(normalized)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError(
'runtime managed file operation identity is invalid'
) from exc
if not hmac.compare_digest(canonical, value):
raise RuntimeSafetySchemaError(
'runtime managed file operation identity is not canonical'
)
return normalized
def _runtime_result_json_is_too_deep(text):
depth = 0
in_string = False
escaped = False
for character in text:
if in_string:
if escaped:
escaped = False
elif character == '\\':
escaped = True
elif character == '"':
in_string = False
elif character == '"':
in_string = True
elif character in '[{':
depth += 1
if depth > RUNTIME_AGENT_RESULT_MAX_DEPTH:
return True
elif character in ']}' and depth:
depth -= 1
return False
def _runtime_result_envelope(payload, max_bytes):
raw = text = parsed = None
try:
if type(payload) is not bytes:
return None, None, 'runtime operation result must be exact bytes'
if (
isinstance(max_bytes, bool) or not isinstance(max_bytes, int)
or not 1 <= max_bytes <= RUNTIME_AGENT_RESULT_MAX_BYTES
):
return None, None, 'runtime operation result bound is invalid'
raw = payload
if not raw or len(raw) > max_bytes:
return None, None, 'runtime operation result exceeds its byte bound'
digest = hashlib.sha256(raw).hexdigest()
try:
text = raw.decode('utf-8', errors='strict')
except UnicodeDecodeError:
return None, None, 'runtime operation result is not valid UTF-8'
if _runtime_result_json_is_too_deep(text):
return None, None, 'runtime operation result is invalid JSON'
def reject_duplicates(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError('duplicate')
result[key] = value
return result
try:
parsed = json.loads(
text, object_pairs_hook=reject_duplicates,
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('constant')),
)
except (TypeError, ValueError, json.JSONDecodeError, RecursionError):
return None, None, 'runtime operation result is invalid JSON'
if not isinstance(parsed, dict) or set(parsed) != {
'schema', 'operation_id', 'action', 'result', 'safe_category',
'safe_detail', 'resulting_identity',
}:
return None, None, 'runtime operation result shape is invalid'
if parsed['schema'] != 1 or type(parsed['schema']) is not int:
return None, None, 'runtime operation result schema is invalid'
try:
operation_id = _runtime_operation_id(parsed['operation_id'])
action = str(parsed['action']) if isinstance(parsed['action'], str) else ''
if action not in RUNTIME_ASYNC_ACTIONS:
raise ValueError('runtime operation action is invalid')
result = str(parsed['result']) if isinstance(parsed['result'], str) else ''
if result not in RUNTIME_ASYNC_TERMINAL_RESULTS:
raise ValueError('runtime operation result state is invalid')
category = _runtime_safe_code(
parsed['safe_category'], 'safe category', required=result != 'succeeded',
)
detail = _runtime_safe_code(parsed['safe_detail'], 'safe detail')
if result == 'succeeded' and (category is not None or detail is not None):
raise ValueError('successful runtime operation result must not have an error')
identity = _runtime_resulting_identity(parsed['resulting_identity'])
if result in ('succeeded', 'rolled_back') and identity is None:
raise ValueError('runtime operation result identity is required')
except ValueError as exc:
return None, None, str(exc)
return {
'operation_id': operation_id,
'action': action,
'result': result,
'safe_category': category,
'safe_detail': detail,
'resulting_identity': identity,
}, digest, None
finally:
payload = raw = text = parsed = None
def fixed_lease_window(lease_seconds, now=None):
issued_at = now or datetime.now(timezone.utc)
if issued_at.tzinfo is None:
issued_at = issued_at.replace(tzinfo=timezone.utc)
issued_at = issued_at.astimezone(timezone.utc)
expires_at = issued_at + timedelta(seconds=max(60, int(lease_seconds)))
return (
issued_at.isoformat(timespec='seconds'),
expires_at.isoformat(timespec='seconds'),
)
def compact_delivered_scan_publications(conn, now=None):
"""Delivered publication state is represented by absence from the outbox."""
now_dt = now or datetime.now(timezone.utc)
if isinstance(now_dt, str):
now_dt = parse_time(now_dt) or datetime.now(timezone.utc)
now_text = now_dt.isoformat(timespec='seconds')
conn.execute("DELETE FROM scan_publication_outbox WHERE status = 'delivered'")
return now_text
def parse_time(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(str(value).replace('Z', '+00:00'))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
except ValueError:
return None
def json_dumps(value):
return json.dumps(value, ensure_ascii=False, default=str, sort_keys=True)
def canonical_git_scan_plan_bytes(plan):
try:
encoded = json.dumps(
plan, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('utf-8')
except (TypeError, ValueError) as exc:
raise ValueError('Git scan plan must be canonical JSON data') from exc
if len(encoded) > 16 * 1024:
raise ValueError('Git scan plan exceeds its byte bound')
return encoded
def canonical_remote_execution_snapshot_bytes(snapshot):
if not isinstance(snapshot, dict):
raise ValueError('remote execution snapshot must be a JSON object')
# Imported lazily to avoid scanner_db <-> scan_execution import recursion.
from scan_execution import normalize_remote_execution_snapshot
snapshot = normalize_remote_execution_snapshot(snapshot)
try:
encoded = json.dumps(
snapshot, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('utf-8')
except (TypeError, ValueError) as exc:
raise ValueError('remote execution snapshot must be canonical JSON data') from exc
if len(encoded) > 64 * 1024:
raise ValueError('remote execution snapshot exceeds its byte bound')
return encoded
def stored_remote_execution_snapshot(reservation):
stored_json = reservation['remote_execution_snapshot_json']
stored_sha256 = reservation['remote_execution_snapshot_sha256']
if stored_json is None and stored_sha256 is None:
return None
if not stored_json or not re.fullmatch(r'[a-f0-9]{64}', str(stored_sha256 or '')):
raise ScanEventConflictError('remote execution snapshot identity is incomplete')
try:
value = json.loads(str(stored_json))
except (TypeError, ValueError) as exc:
raise ScanEventConflictError('remote execution snapshot JSON is invalid') from exc
encoded = canonical_remote_execution_snapshot_bytes(value)
if encoded.decode('ascii') != str(stored_json) or not hmac.compare_digest(
hashlib.sha256(encoded).hexdigest(), str(stored_sha256),
):
raise ScanEventConflictError('remote execution snapshot identity is invalid')
return value
def stored_git_scan_plan(reservation):
stored_json = reservation['git_scan_plan_json']
stored_sha256 = reservation['git_scan_plan_sha256']
if stored_json is None and stored_sha256 is None:
return None
if stored_json is None or stored_sha256 is None:
raise RuntimeSafetySchemaError('Git scan reservation plan identity is incomplete')
try:
stored_bytes = str(stored_json).encode('ascii')
plan = json.loads(stored_bytes.decode('ascii'))
except (UnicodeEncodeError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('stored Git scan plan JSON is invalid') from exc
if (
hashlib.sha256(stored_bytes).hexdigest() != str(stored_sha256)
or canonical_git_scan_plan_bytes(plan) != stored_bytes
):
raise RuntimeSafetySchemaError('stored Git scan plan identity is invalid')
return plan
def _optional_mapping_value(value, key):
try:
return value[key]
except (KeyError, IndexError):
return None
def remote_assignment_execution_plan(reservation, *, snapshot=None):
# Imported lazily to avoid scanner_db <-> scan_execution import recursion.
from scan_execution import ScanExecutionError
try:
if str(reservation['assignment_kind']) != 'remote':
raise ScanEventConflictError('execution plan requires a remote assignment')
snapshot = (
stored_remote_execution_snapshot(reservation)
if snapshot is None else snapshot
)
if snapshot is None:
raise ScanEventConflictError('remote assignment has no execution snapshot')
encoded = canonical_remote_execution_snapshot_bytes(snapshot)
snapshot = json.loads(encoded.decode('ascii'))
source = str(reservation['source'])
platform = str(reservation['platform'])
target = str(reservation['target'])
normalized_target = str(reservation['normalized_target'])
effective_sha256 = str(
reservation['remote_effective_config_sha256'] or ''
)
if (
snapshot['credential_ref']['source'] != source
or snapshot['execution']['source'] != platform
or snapshot['compatibility']['effective_config_sha256']
!= effective_sha256
or normalize_target(target, platform) != normalized_target
):
raise ScanEventConflictError(
'remote assignment execution context conflicts with its reservation'
)
kind = snapshot['planning']['kind']
git_plan = stored_git_scan_plan(reservation)
has_docker_plan = any(
_optional_mapping_value(reservation, name) is not None
for name in ('docker_layer_plan_json', 'docker_layer_plan_sha256')
)
if kind == 'exact_git_v1':
if source not in ('github', 'gitlab') or platform != source or has_docker_plan:
raise ScanEventConflictError(
'remote Git execution context conflicts with its reservation'
)
if git_plan is not None and normalize_target(
git_plan.get('repo_url'), platform,
) != normalized_target:
raise ScanEventConflictError(
'remote Git plan target conflicts with its reservation'
)
return {
'kind': kind,
'execution_target': target,
'bound_plan': git_plan,
}
if git_plan is not None or has_docker_plan:
raise ScanEventConflictError(
'direct execution context has an unexpected bound plan'
)
if kind == 'docker_direct_v1':
if source != 'dockerhub' or platform != 'docker':
raise ScanEventConflictError(
'remote Docker execution context conflicts with its reservation'
)
parsed = parse_dockerhub_digest_target(target)
if parsed['normalized_target'] != normalized_target:
raise ScanEventConflictError(
'remote Docker target conflicts with its reservation'
)
return {
'kind': kind,
'execution_target': parsed['image'],
'bound_plan': None,
}
if kind == 'huggingface_space_v1':
if source != 'huggingface' or platform != 'huggingface':
raise ScanEventConflictError(
'remote HuggingFace context conflicts with its reservation'
)
execution_target = normalize_huggingface_space_id(target)
if execution_target.lower() != normalized_target:
raise ScanEventConflictError(
'remote HuggingFace target conflicts with its reservation'
)
return {
'kind': kind,
'execution_target': execution_target,
'bound_plan': None,
}
raise ScanEventConflictError('remote assignment planning kind is unsupported')
except ScanEventConflictError:
raise
except (
KeyError, IndexError, TypeError, ValueError,
RuntimeSafetySchemaError, ScanExecutionError,
) as exc:
raise ScanEventConflictError(
'remote assignment execution plan is invalid'
) from exc
def validate_result_git_scan_plan(reservation, metadata):
plan = stored_git_scan_plan(reservation)
metadata_plan = metadata.get('git_scan_plan')
if plan is None:
if metadata_plan is not None:
raise ScanEventConflictError('result has an unbound Git scan plan')
return None
if not isinstance(metadata_plan, dict):
raise ScanEventConflictError('result is missing its bound Git scan plan')
if canonical_git_scan_plan_bytes(metadata_plan) != canonical_git_scan_plan_bytes(plan):
raise ScanEventConflictError('result Git scan plan conflicts with its reservation')
return plan
def validate_remote_result_execution_plan(reservation, result_metadata):
if not isinstance(result_metadata, dict):
raise ScanEventConflictError('remote bundle is missing result metadata')
plan = remote_assignment_execution_plan(reservation)
kind = plan['kind']
if kind == 'exact_git_v1':
if any(
result_metadata.get(name) is not None
for name in ('docker_layer_plan', 'docker_layer_execution')
):
raise ScanEventConflictError(
'remote Git result has unexpected Docker execution metadata'
)
validate_result_git_scan_plan(reservation, result_metadata)
return plan
if any(
result_metadata.get(name) is not None
for name in (
'git_scan_plan', 'git_scan_execution',
'docker_layer_plan', 'docker_layer_execution',
)
):
raise ScanEventConflictError(
'remote direct result has unexpected planning metadata'
)
return plan
def validate_git_resolution(resolved):
resolved = dict(resolved or {})
provider = str(resolved.get('provider') or '').strip().lower()
if provider not in ('github', 'gitlab'):
raise ValueError('Git scan provider must be github or gitlab')
branch = str(resolved.get('branch') or '')
ref = str(resolved.get('ref') or '')
if not branch or ref != f'refs/heads/{branch}' or len(ref) > 1024:
raise ValueError('Git scan resolution has an invalid branch ref')
if (
branch.startswith(('/', '.')) or branch.endswith(('/', '.', '.lock'))
or '..' in branch or '@{' in branch or '\\' in branch
or re.search(r'[\x00-\x20\x7f~^:?*\[]', branch)
or any(part in ('', '.', '..') or part.endswith('.lock') for part in branch.split('/'))
):
raise ValueError('Git scan resolution has an unsafe branch ref')
head_sha = str(resolved.get('head_sha') or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{40}|[a-f0-9]{64}', head_sha):
raise ValueError('Git scan resolution has an invalid head SHA')
repo_url = str(resolved.get('repo_url') or '').strip()
repo_path = str(resolved.get('repo_path') or '').strip()
expected_host = f'{provider}.com'
parsed = urlsplit(repo_url)
if (
parsed.scheme != 'https' or (parsed.hostname or '').lower() != expected_host
or parsed.username is not None or parsed.password is not None
or parsed.port is not None or parsed.query or parsed.fragment
or not repo_path or len(repo_path) > 1024
or parsed.path != f'/{repo_path}.git'
or repo_path.startswith('/') or repo_path.endswith('/') or '\\' in repo_path
or re.search(r'%(?:2f|5c)', repo_path, flags=re.IGNORECASE)
or any(part in ('', '.', '..') for part in repo_path.split('/'))
):
raise ValueError('Git scan resolution has an invalid canonical repository')
ref_source = str(resolved.get('ref_source') or '')
if ref_source not in ('explicit', 'provider_default'):
raise ValueError('Git scan resolution has an invalid ref source')
return {
'provider': provider,
'repo_url': repo_url,
'repo_path': repo_path,
'branch': branch,
'ref': ref,
'head_sha': head_sha,
'ref_source': ref_source,
}
def matching_git_coverage(reservation, metadata, queue_status, error_count):
plan = validate_result_git_scan_plan(reservation, metadata)
if plan is None:
return False, None, None
stored_sha256 = reservation['git_scan_plan_sha256']
execution = metadata.get('git_scan_execution')
if queue_status != 'done' or int(error_count or 0) != 0:
return False, None, None
if not isinstance(execution, dict):
raise ScanEventConflictError('successful Git result is missing execution evidence')
if execution.get('success') is not True or execution.get('pinned') is not True:
return False, None, None
if str(execution.get('plan_sha256') or '') != str(stored_sha256):
raise ScanEventConflictError('Git execution evidence has a conflicting plan hash')
if 'coverage_complete' in execution:
if execution['coverage_complete'] is not True:
return False, None, None
elif metadata.get('degraded') or metadata.get('warnings'):
return False, None, None
plan_mode = str(plan.get('mode') or '')
execution_mode = str(execution.get('mode') or '')
valid_execution = (
(plan_mode == 'baseline' and execution_mode == 'baseline')
or (plan_mode == 'delta' and execution_mode == 'delta')
or (
plan_mode == 'delta' and execution_mode == 'baseline_reset'
and execution.get('continuity_reset') is True
)
or (plan_mode == 'noop' and execution_mode == 'noop')
)
if not valid_execution:
return False, None, None
return True, str(plan['ref']), str(plan['head_sha'])
DOCKER_LAYER_PLAN_MAX_BYTES = 1024 * 1024
DOCKER_LAYER_MAX_DESCRIPTORS = 2048
# Exact duplicate-position attribution is all-or-nothing per result bundle.
DOCKER_DEPTH_MAX_ATTRIBUTION_ROWS_PER_BUNDLE = 100000
DOCKER_BLOB_LEASE_MARGIN_SEC = 300
DOCKER_LAYER_SUPPORTED_MEDIA_TYPES = frozenset((
'application/vnd.oci.image.layer.v1.tar',
'application/vnd.oci.image.layer.v1.tar+gzip',
'application/vnd.oci.image.layer.v1.tar+zstd',
'application/vnd.docker.image.rootfs.diff.tar',
'application/vnd.docker.image.rootfs.diff.tar.gzip',
))
DOCKER_CONFIG_MEDIA_TYPES = frozenset((
'application/vnd.oci.image.config.v1+json',
'application/vnd.docker.container.image.v1+json',
))
DOCKER_LAYER_LIMIT_KEYS = frozenset((
'config_max_bytes', 'layer_max_bytes', 'image_max_bytes', 'max_layers',
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
'blob_timeout_sec', 'filesystem_concurrency', 'blob_max_attempts',
))
DOCKER_ADAPTIVE_SELECTOR_VERSION = 'docker-adaptive-payload-v1'
DOCKER_ADAPTIVE_EXECUTION_VERSION = 'docker-layer-execution-v4'
DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION = 'docker-adaptive-shadow-v1'
DOCKER_ADAPTIVE_GATE_MIN_CONTROLS = 50
DOCKER_ADAPTIVE_GATE_MAX_CONTROLS = 100
DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM = 850000
DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM = 400000
DOCKER_ADAPTIVE_PAYLOAD_CLASSES = frozenset((
'config', 'copy_add', 'app_config_run', 'package_run', 'other_run',
'bulk_data', 'unknown',
))
DOCKER_ADAPTIVE_LAYER_CLASS_ORDER = (
'copy_add', 'app_config_run', 'package_run', 'unknown', 'other_run',
'bulk_data',
)
DOCKER_ADAPTIVE_CHECKPOINT_KEYS = frozenset(('max_blobs', 'max_bytes'))
DOCKER_ADAPTIVE_SELECTION_REASONS = frozenset((
'config_selected', 'already_covered', 'duplicate_digest',
'unsupported_media_type', 'config_too_large', 'layer_too_large',
'image_budget_exhausted', 'layer_limit_exhausted',
*(f'selected_{payload_class}' for payload_class in DOCKER_ADAPTIVE_LAYER_CLASS_ORDER),
))
DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS = (
'selected_config', 'selected_copy_add', 'selected_app_config_run',
'selected_package_run', 'selected_other_run', 'selected_bulk_data',
'selected_unknown', 'reuse_already_covered', 'reuse_duplicate_digest',
'omitted_unsupported_media_type', 'omitted_config_too_large',
'omitted_layer_too_large', 'omitted_image_budget_exhausted',
'omitted_layer_limit_exhausted', 'adaptive_checkpoints',
)
DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS = tuple(
name for name in DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS
if name.startswith('omitted_')
)
def validate_docker_adaptive_policy_hashes(
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
):
values = {
'scan_policy_sha256': str(scan_policy_sha256 or '').lower(),
'execution_policy_sha256': str(execution_policy_sha256 or '').lower(),
'selection_policy_sha256': str(selection_policy_sha256 or '').lower(),
}
for label, value in values.items():
if not re.fullmatch(r'[a-f0-9]{64}', value):
raise ValueError(f'Docker adaptive {label} must be a lowercase SHA-256')
return values
def docker_adaptive_shadow_gate_metrics(values):
names = (
'completed_pairs', 'full_routed_count', 'adaptive_routed_count',
'routed_intersection_count', 'full_detector_count',
'adaptive_detector_count', 'detector_intersection_count',
'full_slot_ms', 'adaptive_slot_ms', 'omitted_descriptor_count',
'failure_count', 'privacy_violation_count', 'safety_regression_count',
)
metrics = {}
for name in names:
raw = values.get(name, 0)
if isinstance(raw, bool):
raise ValueError(f'Docker adaptive shadow {name} must be an integer')
try:
value = int(raw)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f'Docker adaptive shadow {name} must be an integer') from exc
if value < 0 or value > 9223372036854775807:
raise ValueError(f'Docker adaptive shadow {name} is outside the supported range')
metrics[name] = value
if metrics['routed_intersection_count'] > min(
metrics['full_routed_count'], metrics['adaptive_routed_count'],
):
raise ValueError('Docker adaptive routed intersection exceeds its evidence sets')
if metrics['detector_intersection_count'] > min(
metrics['full_detector_count'], metrics['adaptive_detector_count'],
):
raise ValueError('Docker adaptive detector intersection exceeds its evidence sets')
full_routed = metrics['full_routed_count']
full_slot_ms = metrics['full_slot_ms']
metrics['routed_recall_ppm'] = (
metrics['routed_intersection_count'] * 1000000 // full_routed
if full_routed else 0
)
metrics['slot_ratio_ppm'] = (
(metrics['adaptive_slot_ms'] * 1000000 + full_slot_ms - 1) // full_slot_ms
if full_slot_ms else 1000001
)
return metrics
def validate_docker_adaptive_shadow_selection_metrics(value):
if isinstance(value, str):
try:
value = json.loads(value)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise ValueError('Docker adaptive shadow selection metrics are invalid JSON') from exc
if not isinstance(value, dict) or set(value) != set(DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS):
raise ValueError('Docker adaptive shadow selection metrics have invalid fields')
metrics = {}
for name in DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS:
raw = value[name]
if isinstance(raw, bool):
raise ValueError(f'Docker adaptive shadow selection metric {name} must be an integer')
try:
number = int(raw)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(
f'Docker adaptive shadow selection metric {name} must be an integer'
) from exc
if number < 0 or number > 9223372036854775807:
raise ValueError(
f'Docker adaptive shadow selection metric {name} is outside the supported range'
)
metrics[name] = number
return metrics
def docker_content_media_class(kind, media_type):
kind = str(kind or '').strip().lower()
media_type = str(media_type or '').strip().lower()
if kind == 'config' and media_type in DOCKER_CONFIG_MEDIA_TYPES:
return 'config-json'
if kind == 'layer' and media_type in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
if media_type.endswith('+zstd'):
return 'layer-zstd'
if media_type.endswith('+gzip') or media_type.endswith('.gzip'):
return 'layer-gzip'
return 'layer-tar'
return f'{kind}:{media_type}'
def _docker_sha256_digest(value, label='Docker content digest'):
digest = str(value or '').strip().lower()
if not re.fullmatch(r'sha256:[a-f0-9]{64}', digest):
raise ValueError(f'{label} is invalid')
return digest
def _dockerhub_image_parts(target):
parsed = parse_docker_target(target)
image = str(parsed['image']).lower()
image_name, manifest_digest = image.rsplit('@', 1)
manifest_digest = _docker_sha256_digest(manifest_digest, 'Docker manifest digest')
parts = image_name.split('/')
if len(parts) > 1 and ('.' in parts[0] or ':' in parts[0] or parts[0] == 'localhost'):
registry = parts.pop(0)
if registry not in ('docker.io', 'index.docker.io', 'registry-1.docker.io'):
raise ValueError('Docker layer scanning only supports Docker Hub targets')
if not parts or any(not part for part in parts):
raise ValueError('Docker Hub repository is invalid')
repository = '/'.join(parts)
registry_repository = repository if '/' in repository else f'library/{repository}'
return image, repository, registry_repository, manifest_digest
def _normalized_docker_descriptor(value, kind, position=None):
if not isinstance(value, dict) or set(value) != {'digest', 'size', 'media_type'}:
raise ValueError(f'Docker {kind} descriptor has an invalid shape')
digest = _docker_sha256_digest(value.get('digest'), f'Docker {kind} digest')
try:
size = int(value.get('size'))
except (TypeError, ValueError) as exc:
raise ValueError(f'Docker {kind} descriptor size is invalid') from exc
if size < 0 or size > 1024 * 1024 * 1024 * 1024:
raise ValueError(f'Docker {kind} descriptor size is outside its hard bound')
media_type = str(value.get('media_type') or '').strip().lower()
if not media_type or len(media_type) > 256 or any(ord(char) < 32 for char in media_type):
raise ValueError(f'Docker {kind} media type is invalid')
result = {
'digest': digest,
'size': size,
'media_type': media_type,
'kind': kind,
}
if position is not None:
result['position'] = int(position)
return result
def validate_docker_layer_resolution(resolved):
required = {
'version', 'image', 'repository', 'manifest_digest', 'platform_os',
'platform_arch', 'manifest_media_type', 'config', 'layers',
}
if not isinstance(resolved, dict) or set(resolved) != required or resolved.get('version') != 1:
raise ValueError('Docker layer resolution has an invalid shape')
image, _, registry_repository, target_digest = _dockerhub_image_parts(resolved.get('image'))
manifest_digest = _docker_sha256_digest(
resolved.get('manifest_digest'), 'Docker resolved manifest digest',
)
if manifest_digest != target_digest:
raise ValueError('Docker resolved manifest conflicts with the immutable image target')
repository = str(resolved.get('repository') or '').strip().lower()
if repository != registry_repository:
raise ValueError('Docker resolved repository conflicts with the image target')
platform_os = str(resolved.get('platform_os') or '').strip().lower()
platform_arch = str(resolved.get('platform_arch') or '').strip().lower()
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_os):
raise ValueError('Docker resolved platform OS is invalid')
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_arch):
raise ValueError('Docker resolved platform architecture is invalid')
manifest_media_type = str(resolved.get('manifest_media_type') or '').strip().lower()
if not manifest_media_type or len(manifest_media_type) > 256:
raise ValueError('Docker resolved manifest media type is invalid')
config = _normalized_docker_descriptor(resolved.get('config'), 'config', 0)
layers = resolved.get('layers')
if not isinstance(layers, list) or len(layers) > DOCKER_LAYER_MAX_DESCRIPTORS:
raise ValueError('Docker layer descriptor count exceeds its bound')
normalized_layers = [
_normalized_docker_descriptor(descriptor, 'layer', position)
for position, descriptor in enumerate(layers, 1)
]
return {
'version': 1,
'image': image,
'repository': repository,
'manifest_digest': manifest_digest,
'platform_os': platform_os,
'platform_arch': platform_arch,
'manifest_media_type': manifest_media_type,
'config': config,
'layers': normalized_layers,
}
def validate_docker_layer_limits(limits):
if not isinstance(limits, dict) or set(limits) != DOCKER_LAYER_LIMIT_KEYS:
raise ValueError('Docker layer limits have an invalid shape')
bounds = {
'config_max_bytes': (1024, 64 * 1024 * 1024),
'layer_max_bytes': (1024, 8 * 1024 * 1024 * 1024),
'image_max_bytes': (1024, 32 * 1024 * 1024 * 1024),
'max_layers': (1, 256),
'archive_max_size_bytes': (1024, 4 * 1024 * 1024 * 1024),
'archive_max_depth': (1, 16),
'archive_timeout_sec': (1, 600),
'blob_timeout_sec': (10, 3600),
'filesystem_concurrency': (1, 16),
'blob_max_attempts': (1, 10),
}
normalized = {}
for key, (minimum, maximum) in bounds.items():
try:
value = int(limits.get(key))
except (TypeError, ValueError) as exc:
raise ValueError(f'Docker layer limit {key} must be an integer') from exc
if not minimum <= value <= maximum:
raise ValueError(f'Docker layer limit {key} is outside its hard bound')
normalized[key] = value
return normalized
def validate_docker_adaptive_checkpoint(checkpoint):
if not isinstance(checkpoint, dict) or set(checkpoint) != DOCKER_ADAPTIVE_CHECKPOINT_KEYS:
raise ValueError('Docker adaptive checkpoint has an invalid shape')
try:
max_blobs = int(checkpoint.get('max_blobs'))
max_bytes = int(checkpoint.get('max_bytes'))
except (TypeError, ValueError) as exc:
raise ValueError('Docker adaptive checkpoint values must be integers') from exc
if not 1 <= max_blobs <= 32:
raise ValueError('Docker adaptive checkpoint blob count is outside its hard bound')
if not 1024 <= max_bytes <= 32 * 1024 * 1024 * 1024:
raise ValueError('Docker adaptive checkpoint bytes are outside its hard bound')
return {'max_blobs': max_blobs, 'max_bytes': max_bytes}
def docker_layer_coverage_policy_sha256(scan_policy_sha256, limits):
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
raise ValueError('Docker layer scanner policy hash is invalid')
limits = validate_docker_layer_limits(limits)
payload = {
'limits': limits,
'scan_policy_sha256': scan_policy_sha256,
'validation_version': DOCKER_ADAPTIVE_EXECUTION_VERSION,
}
return hashlib.sha256(json.dumps(
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')).hexdigest()
def docker_layer_execution_policy_sha256(scan_policy_sha256, limits):
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
raise ValueError('Docker layer scanner policy hash is invalid')
limits = validate_docker_layer_limits(limits)
payload = {
'archive_semantics': {
key: limits[key] for key in (
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
)
},
'content_media_classes': sorted({
docker_content_media_class('config', media_type)
for media_type in DOCKER_CONFIG_MEDIA_TYPES
} | {
docker_content_media_class('layer', media_type)
for media_type in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES
}),
'scan_policy_sha256': scan_policy_sha256,
'version': DOCKER_ADAPTIVE_EXECUTION_VERSION,
}
return hashlib.sha256(json.dumps(
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')).hexdigest()
def docker_layer_selection_policy_sha256(limits):
limits = validate_docker_layer_limits(limits)
payload = {
'class_order': list(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER),
'limits': {
key: limits[key] for key in (
'config_max_bytes', 'layer_max_bytes', 'image_max_bytes', 'max_layers',
)
},
'supported_config_media_types': sorted(DOCKER_CONFIG_MEDIA_TYPES),
'supported_layer_media_types': sorted(DOCKER_LAYER_SUPPORTED_MEDIA_TYPES),
'tie_breaks': ['position_desc', 'compressed_size_asc', 'digest_asc'],
'version': DOCKER_ADAPTIVE_SELECTOR_VERSION,
}
return hashlib.sha256(json.dumps(
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')).hexdigest()
def canonical_docker_layer_plan_bytes(plan):
if not isinstance(plan, dict):
raise ValueError('Docker layer plan must be an object')
payload = json.dumps(
plan, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')
if len(payload) > DOCKER_LAYER_PLAN_MAX_BYTES:
raise ValueError('Docker layer plan exceeds its byte bound')
return payload
def docker_layer_canary_selected(manifest_digest, basis_points):
manifest_digest = _docker_sha256_digest(manifest_digest, 'Docker canary manifest digest')
basis_points = max(0, min(10000, int(basis_points or 0)))
bucket = int(hashlib.sha256(manifest_digest.encode('ascii')).hexdigest()[:8], 16) % 10000
return bucket < basis_points
def docker_adaptive_canary_selected(manifest_digest, selection_policy_sha256, basis_points):
manifest_digest = _docker_sha256_digest(
manifest_digest, 'Docker adaptive canary manifest digest',
)
selection_policy_sha256 = str(selection_policy_sha256 or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', selection_policy_sha256):
raise ValueError('Docker adaptive selector policy hash is invalid')
basis_points = max(0, min(10000, int(basis_points or 0)))
material = (
f'docker-adaptive-canary-v1:{manifest_digest}:{selection_policy_sha256}'
).encode('ascii')
bucket = int(hashlib.sha256(material).hexdigest()[:8], 16) % 10000
return bucket < basis_points
def validate_docker_layer_plan(plan):
common_required = {
'version', 'image', 'repository', 'manifest_digest', 'platform_os',
'platform_arch', 'manifest_media_type', 'limits',
'selection_policy_sha256', 'scan_policy_sha256', 'descriptors',
}
if not isinstance(plan, dict):
raise ValueError('Docker layer plan has an invalid shape')
version = plan.get('version')
required = common_required if version == 1 else common_required | {
'selector_version', 'execution_policy_sha256', 'checkpoint',
}
if version not in (1, 2) or set(plan) != required:
raise ValueError('Docker layer plan has an invalid shape')
image, _, repository, manifest_digest = _dockerhub_image_parts(plan.get('image'))
if str(plan.get('repository') or '') != repository:
raise ValueError('Docker layer plan repository conflicts with its image')
if _docker_sha256_digest(plan.get('manifest_digest')) != manifest_digest:
raise ValueError('Docker layer plan manifest conflicts with its image')
limits = validate_docker_layer_limits(plan.get('limits'))
selection_policy_sha256 = str(plan.get('selection_policy_sha256') or '').lower()
expected_selection_hash = (
hashlib.sha256(json.dumps(
limits, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')).hexdigest()
if version == 1 else docker_layer_selection_policy_sha256(limits)
)
scan_policy_sha256 = str(plan.get('scan_policy_sha256') or '').lower()
if selection_policy_sha256 != expected_selection_hash:
raise ValueError('Docker layer plan selection policy hash is invalid')
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
raise ValueError('Docker layer plan scanner policy hash is invalid')
selector_version = None
execution_policy_sha256 = None
checkpoint = None
if version == 2:
selector_version = str(plan.get('selector_version') or '')
if selector_version != DOCKER_ADAPTIVE_SELECTOR_VERSION:
raise ValueError('Docker adaptive selector version is invalid')
execution_policy_sha256 = str(
plan.get('execution_policy_sha256') or ''
).lower()
if execution_policy_sha256 != docker_layer_execution_policy_sha256(
scan_policy_sha256, limits,
):
raise ValueError('Docker adaptive execution policy hash is invalid')
checkpoint = validate_docker_adaptive_checkpoint(plan.get('checkpoint'))
descriptors = plan.get('descriptors')
if not isinstance(descriptors, list) or not descriptors or len(descriptors) > DOCKER_LAYER_MAX_DESCRIPTORS + 1:
raise ValueError('Docker layer plan descriptor count is invalid')
positions = set()
normalized_descriptors = []
allowed_coverage = {
'selected', 'leased', 'shared_pending', 'covered', 'terminal_failed', 'skipped',
}
for descriptor in descriptors:
expected = {
'digest', 'size', 'media_type', 'kind', 'position', 'selected',
'selection_reason', 'coverage_state', 'lease_token', 'attempt',
'max_attempts',
}
if version == 2:
expected.add('payload_class')
if not isinstance(descriptor, dict) or set(descriptor) != expected:
raise ValueError('Docker layer plan descriptor has an invalid shape')
kind = str(descriptor.get('kind') or '')
position = int(descriptor.get('position'))
if kind not in ('config', 'layer') or position < 0 or position in positions:
raise ValueError('Docker layer plan descriptor identity is invalid')
if (position == 0) != (kind == 'config'):
raise ValueError('Docker layer plan configuration position is invalid')
positions.add(position)
normalized = _normalized_docker_descriptor({
'digest': descriptor.get('digest'),
'size': descriptor.get('size'),
'media_type': descriptor.get('media_type'),
}, kind, position)
selected = descriptor.get('selected')
coverage_state = str(descriptor.get('coverage_state') or '')
reason = str(descriptor.get('selection_reason') or '')
lease_token = descriptor.get('lease_token')
attempt = int(descriptor.get('attempt'))
max_attempts = int(descriptor.get('max_attempts'))
if max_attempts < 1 or max_attempts > 100 or attempt < 0 or attempt > max_attempts:
raise ValueError('Docker layer plan descriptor attempt bounds are invalid')
if not isinstance(selected, bool) or coverage_state not in allowed_coverage:
raise ValueError('Docker layer plan descriptor state is invalid')
if not reason or len(reason) > 64 or not re.fullmatch(r'[a-z0-9_]+', reason):
raise ValueError('Docker layer plan selection reason is invalid')
payload_class = None
if version == 2:
payload_class = str(descriptor.get('payload_class') or '')
if payload_class not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES:
raise ValueError('Docker layer plan payload class is invalid')
if (kind == 'config') != (payload_class == 'config'):
raise ValueError('Docker layer plan payload class conflicts with descriptor kind')
if reason not in DOCKER_ADAPTIVE_SELECTION_REASONS:
raise ValueError('Docker adaptive selection reason is invalid')
if coverage_state == 'leased':
if not isinstance(lease_token, str) or not 32 <= len(lease_token) <= 128:
raise ValueError('Docker layer plan lease token is invalid')
elif lease_token is not None:
raise ValueError('Docker layer plan has an unexpected lease token')
if selected != (coverage_state != 'skipped'):
raise ValueError('Docker layer plan selected state is inconsistent')
normalized.update({
'selected': selected,
'selection_reason': reason,
'coverage_state': coverage_state,
'lease_token': lease_token,
'attempt': attempt,
'max_attempts': max_attempts,
})
if version == 2:
normalized['payload_class'] = payload_class
normalized_descriptors.append(normalized)
if positions != set(range(len(descriptors))):
raise ValueError('Docker layer plan positions are not contiguous')
platform_os = str(plan.get('platform_os') or '').lower()
platform_arch = str(plan.get('platform_arch') or '').lower()
manifest_media_type = str(plan.get('manifest_media_type') or '').lower()
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_os):
raise ValueError('Docker layer plan platform OS is invalid')
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_arch):
raise ValueError('Docker layer plan platform architecture is invalid')
if not manifest_media_type or len(manifest_media_type) > 256:
raise ValueError('Docker layer plan manifest media type is invalid')
normalized_descriptors.sort(key=lambda item: item['position'])
normalized_plan = {
'version': version,
'image': image,
'repository': repository,
'manifest_digest': manifest_digest,
'platform_os': platform_os,
'platform_arch': platform_arch,
'manifest_media_type': manifest_media_type,
'limits': limits,
'selection_policy_sha256': selection_policy_sha256,
'scan_policy_sha256': scan_policy_sha256,
'descriptors': normalized_descriptors,
}
if version == 2:
normalized_plan.update({
'selector_version': selector_version,
'execution_policy_sha256': execution_policy_sha256,
'checkpoint': checkpoint,
})
return normalized_plan
def docker_layer_plan_coverage_policy_sha256(plan):
if int(plan.get('version') or 0) == 2:
execution_policy_sha256 = str(
plan.get('execution_policy_sha256') or ''
).lower()
expected = docker_layer_execution_policy_sha256(
plan.get('scan_policy_sha256'), plan.get('limits'),
)
if execution_policy_sha256 != expected:
raise ValueError('Docker adaptive execution policy hash is invalid')
return execution_policy_sha256
return docker_layer_coverage_policy_sha256(
plan.get('scan_policy_sha256'), plan.get('limits'),
)
def select_docker_adaptive_payload(descriptors, payload_classes, limits, covered_digests=()):
limits = validate_docker_layer_limits(limits)
descriptors = list(descriptors or [])
payload_classes = list(payload_classes or [])
if (
not descriptors
or descriptors[0].get('kind') != 'config'
or len(payload_classes) != len(descriptors) - 1
or any(value not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES - {'config'} for value in payload_classes)
):
raise ValueError('Docker adaptive payload classes do not match its descriptors')
covered_digests = set(covered_digests or ())
entries = {}
config = descriptors[0]
if config['digest'] in covered_digests:
entries[config['position']] = (config, True, 'already_covered', 'config')
elif config['media_type'] not in DOCKER_CONFIG_MEDIA_TYPES:
entries[config['position']] = (config, False, 'unsupported_media_type', 'config')
elif config['size'] > limits['config_max_bytes']:
entries[config['position']] = (config, False, 'config_too_large', 'config')
else:
entries[config['position']] = (config, True, 'config_selected', 'config')
class_rank = {
payload_class: index
for index, payload_class in enumerate(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER)
}
occurrences = {}
for descriptor, payload_class in zip(descriptors[1:], payload_classes):
occurrences.setdefault(descriptor['digest'], []).append((descriptor, payload_class))
representatives = {}
for digest, values in occurrences.items():
representatives[digest] = min(
values,
key=lambda value: (
class_rank[value[1]], -value[0]['position'], value[0]['size'], digest,
),
)
candidates = []
decisions = {}
for digest, (descriptor, payload_class) in representatives.items():
if digest in covered_digests:
decisions[digest] = (True, 'already_covered')
elif descriptor['media_type'] not in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
decisions[digest] = (False, 'unsupported_media_type')
elif descriptor['size'] > limits['layer_max_bytes']:
decisions[digest] = (False, 'layer_too_large')
else:
candidates.append((descriptor, payload_class))
candidates.sort(key=lambda value: (
class_rank[value[1]], -value[0]['position'], value[0]['size'], value[0]['digest'],
))
all_fit = (
len(candidates) <= limits['max_layers']
and sum(descriptor['size'] for descriptor, _ in candidates) <= limits['image_max_bytes']
)
selected_count = 0
selected_bytes = 0
for descriptor, payload_class in candidates:
if all_fit or (
selected_count < limits['max_layers']
and selected_bytes + descriptor['size'] <= limits['image_max_bytes']
):
decisions[descriptor['digest']] = (True, f'selected_{payload_class}')
selected_count += 1
selected_bytes += descriptor['size']
elif selected_count >= limits['max_layers']:
decisions[descriptor['digest']] = (False, 'layer_limit_exhausted')
else:
decisions[descriptor['digest']] = (False, 'image_budget_exhausted')
for digest, values in occurrences.items():
representative, representative_class = representatives[digest]
selected, reason = decisions[digest]
entries[representative['position']] = (
representative, selected, reason, representative_class,
)
for descriptor, payload_class in values:
if descriptor['position'] == representative['position']:
continue
entries[descriptor['position']] = (
descriptor,
selected,
'duplicate_digest' if selected else reason,
payload_class,
)
return [entries[position] for position in sorted(entries)]
def stored_docker_layer_plan(reservation):
stored_json = reservation['docker_layer_plan_json']
stored_sha256 = reservation['docker_layer_plan_sha256']
if stored_json is None and stored_sha256 is None:
return None, None
if stored_json is None or stored_sha256 is None:
raise RuntimeSafetySchemaError('Docker layer reservation plan identity is incomplete')
try:
stored_bytes = str(stored_json).encode('ascii')
plan = json.loads(stored_bytes.decode('ascii'))
plan = validate_docker_layer_plan(plan)
except (UnicodeEncodeError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc:
raise RuntimeSafetySchemaError('stored Docker layer plan JSON is invalid') from exc
if (
canonical_docker_layer_plan_bytes(plan) != stored_bytes
or hashlib.sha256(stored_bytes).hexdigest() != str(stored_sha256)
):
raise RuntimeSafetySchemaError('stored Docker layer plan identity is invalid')
return plan, str(stored_sha256)
def validate_docker_layer_execution(execution, plan, plan_sha256):
if not isinstance(execution, dict) or set(execution) != {
'version', 'plan_sha256', 'blobs',
}:
raise ValueError('Docker layer execution has an invalid shape')
if (
execution.get('version') != plan['version']
or str(execution.get('plan_sha256') or '') != plan_sha256
):
raise ValueError('Docker layer execution plan identity is invalid')
records = execution.get('blobs')
if not isinstance(records, list) or len(records) > DOCKER_LAYER_MAX_DESCRIPTORS + 1:
raise ValueError('Docker layer execution record count is invalid')
leased = {}
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] == 'leased':
existing = leased.get(descriptor['digest'])
if existing and existing['lease_token'] != descriptor['lease_token']:
raise ValueError('Docker layer plan duplicates a digest with conflicting leases')
leased[descriptor['digest']] = descriptor
normalized = []
seen = set()
statuses = {'covered', 'retryable_failed', 'terminal_failed'}
for record in records:
expected = {
'digest', 'lease_token', 'status', 'verified_bytes', 'transfer_bytes',
'transfer_duration_ms', 'scan_duration_ms', 'finding_count', 'error_code',
}
if not isinstance(record, dict) or set(record) != expected:
raise ValueError('Docker layer execution record has an invalid shape')
digest = _docker_sha256_digest(record.get('digest'), 'execution blob')
descriptor = leased.get(digest)
if not descriptor or digest in seen:
raise ValueError('Docker layer execution has an unclaimed or duplicate blob')
seen.add(digest)
if str(record.get('lease_token') or '') != descriptor['lease_token']:
raise ValueError('Docker layer execution lease token is invalid')
status = str(record.get('status') or '')
if status not in statuses:
raise ValueError('Docker layer execution status is invalid')
numeric = {}
for key, upper in (
('verified_bytes', descriptor['size']),
('transfer_bytes', descriptor['size']),
('transfer_duration_ms', 24 * 60 * 60 * 1000),
('scan_duration_ms', 24 * 60 * 60 * 1000),
('finding_count', 10_000_000),
):
value = record.get(key)
if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > upper:
raise ValueError(f'Docker layer execution {key} is invalid')
numeric[key] = value
error_code = record.get('error_code')
if status == 'covered':
if numeric['verified_bytes'] != descriptor['size'] or error_code is not None:
raise ValueError('successful Docker layer execution is incomplete')
elif (
not isinstance(error_code, str)
or not re.fullmatch(r'[a-z0-9_]{1,64}', error_code)
):
raise ValueError('failed Docker layer execution error code is invalid')
normalized.append({
'digest': digest,
'lease_token': descriptor['lease_token'],
'status': status,
**numeric,
'error_code': error_code,
})
if seen != set(leased):
raise ValueError('Docker layer execution does not account for every leased blob')
normalized.sort(key=lambda item: item['digest'])
return {
'version': plan['version'],
'plan_sha256': plan_sha256,
'blobs': normalized,
}
def docker_layer_canary_metadata_eligible(metadata, target):
if not isinstance(metadata, dict):
return False
raw_plan = metadata.get('docker_layer_plan')
if raw_plan is None:
scan_meta = metadata.get('scan_meta')
scan_meta = scan_meta if isinstance(scan_meta, dict) else {}
return bool(
scan_meta.get('command_timed_out') or metadata.get('error_class') == 'timeout'
)
try:
plan = validate_docker_layer_plan(raw_plan)
if plan['image'] != _dockerhub_image_parts(target)[0]:
return False
plan_sha256 = hashlib.sha256(canonical_docker_layer_plan_bytes(plan)).hexdigest()
execution = validate_docker_layer_execution(
metadata.get('docker_layer_execution'), plan, plan_sha256,
)
except (TypeError, ValueError):
return False
if any(
descriptor['coverage_state'] in ('selected', 'shared_pending', 'retryable_failed')
for descriptor in plan['descriptors']
):
return True
return any(
record['status'] == 'retryable_failed' for record in execution['blobs']
)
def sqlite_lock_error(exc):
message = str(exc or '').lower()
return any(item in message for item in ('locked', 'busy', 'deadlock', 'lock timeout', 'could not serialize access'))
def sqlite_lock_retry_delay(attempt):
base = SQLITE_LOCK_RETRY_BASE_SEC * (2 ** min(int(attempt or 0), 5))
jitter = random.uniform(0, SQLITE_LOCK_RETRY_BASE_SEC)
return min(SQLITE_LOCK_RETRY_MAX_SEC, base + jitter)
def safe_json_loads(value):
if not value:
return None
if isinstance(value, (dict, list)):
return value
try:
return json.loads(str(value))
except (TypeError, ValueError, json.JSONDecodeError):
return None
def sha256_text(value):
if value is None:
return ''
value = str(value)
if not value:
return ''
return hashlib.sha256(value.encode('utf-8', errors='replace')).hexdigest()
def _identity_mapping(value):
if hasattr(value, 'as_dict'):
value = value.as_dict()
value = dict(value or {})
required = ('pid', 'creation_time', 'executable')
if any(not value.get(key) for key in required):
raise ValueError('exact producer identity is incomplete')
return {
'pid': int(value['pid']),
'creation_time': str(value['creation_time']),
'executable': os.path.normcase(os.path.realpath(os.path.abspath(str(value['executable'])))),
}
def hash_file(path):
if not path or not os.path.exists(path):
return None
digest = hashlib.sha256()
with open(path, 'rb') as f:
for chunk in iter(lambda: f.read(1024 * 1024), b''):
digest.update(chunk)
return digest.hexdigest()
def get_database_path(results_dir=None, explicit_path=None):
path = explicit_path or os.getenv('SCANNER_DB_PATH') or os.getenv('SCAN_DB_PATH')
if path:
if results_dir and not os.path.isabs(path):
return os.path.join(results_dir, path)
return path
return os.path.join(results_dir or os.getcwd(), DB_FILENAME)
def get_database_url(explicit_url=None):
return explicit_url or database_url_from_env()
def database_disabled():
return str(os.getenv('SCANNER_DB_DISABLED', '')).strip().lower() in ('1', 'true', 'yes', 'on')
def redact_config(value):
if isinstance(value, dict):
redacted = {}
for key, item in value.items():
key_text = str(key).lower()
if any(part in key_text for part in DATABASE_SECRET_KEY_PARTS):
redacted[key] = redact_database_url(item) if item else item
elif any(part in key_text for part in SECRET_KEY_PARTS):
redacted[key] = REDACTED if item else item
else:
redacted[key] = redact_config(item)
return redacted
if isinstance(value, list):
return [redact_config(item) for item in value]
return value
def redact_argv(argv):
sensitive_flags = {
'--token', '--docker-token', '--password', '--api-key', '--secret',
'--db-url', '--database-url', '--scanner-db-url',
}
output = []
hide_next = False
for value in argv or []:
text = str(value)
if hide_next:
output.append(REDACTED)
hide_next = False
continue
flag = text.split('=', 1)[0].lower()
if flag in sensitive_flags:
if '=' in text:
output.append(text.split('=', 1)[0] + '=' + REDACTED)
else:
output.append(text)
hide_next = True
continue
output.append(redact_database_url(text))
return output
def normalize_target(target, source):
target_text = str(target or '').strip()
source = 'docker' if source == 'dockerhub' else str(source or '').lower()
lowered = target_text.lower()
if source == 'postman':
return postman_target_identity(target)
if source == 'docker':
return docker_target_identity(target)
if source == 'github_archive':
try:
data = json.loads(target_text)
repo_url = str(data.get('url') or data.get('repo_url') or target_text).strip().lower()
branch = str(data.get('branch') or '').strip().lower()
if repo_url.endswith('.git'):
repo_url = repo_url[:-4]
repo_url = repo_url.rstrip('/')
if repo_url.startswith('http://'):
repo_url = 'https://' + repo_url[7:]
return f'{repo_url}#{branch}' if branch else repo_url
except Exception:
pass
if source in ('github', 'github_archive', 'gitlab', 'git'):
if lowered.endswith('.git'):
lowered = lowered[:-4]
lowered = lowered.rstrip('/')
if lowered.startswith('http://'):
lowered = 'https://' + lowered[7:]
return lowered
if source in ('npm', 'pypi'):
try:
data = json.loads(target_text)
name = data.get('name') or ''
version = data.get('version') or ''
return f'{source}:{name}@{version}'.lower()
except Exception:
return target_text.split('|', 1)[0].lower()
return lowered
def _admin_safe_target(target):
text = str(target or '').strip()
try:
parsed = urlsplit(text)
port = parsed.port
except ValueError:
return '[invalid target]'
if parsed.scheme and parsed.hostname:
host = parsed.hostname
if ':' in host and not host.startswith('['):
host = f'[{host}]'
authority = host + (f':{port}' if port is not None else '')
return f'{parsed.scheme.lower()}://{authority}{parsed.path}'[:2048]
if '@' in text:
text = text.rsplit('@', 1)[-1]
return text[:2048]
def _admin_assignment_outcome_sql(reservation='r'):
return f'''CASE {reservation}.remote_resolution_kind
WHEN 'bundle_accepted' THEN 'accepted'
WHEN 'prebundle_report' THEN 'prebundle_failed'
WHEN 'expired' THEN 'expired'
ELSE 'unfinished' END'''
def _admin_scan_outcome_sql(reservation='r', scan='s'):
return (
f"CASE WHEN {reservation}.remote_resolution_kind = 'bundle_accepted' "
f"THEN COALESCE({scan}.status, 'unavailable') ELSE 'unavailable' END"
)
def _validated_admin_worker_filters(value):
filters = dict(value or {})
allowed = {
'source', 'worker', 'assignment_outcome', 'scan_outcome', 'phase',
'category', 'code', 'retryable', 'since',
}
if set(filters) - allowed:
raise ValueError('worker administration filters are invalid')
patterns = {
'source': r'[a-z0-9][a-z0-9_.-]{0,63}',
'phase': r'[a-z][a-z0-9_]{0,63}',
'category': r'[a-z][a-z0-9_]{0,127}',
'code': r'[A-Za-z0-9][A-Za-z0-9._:-]{0,255}',
}
normalized = {}
for name, pattern in patterns.items():
if name not in filters:
continue
item = str(filters[name] or '')
if re.fullmatch(pattern, item) is None:
raise ValueError('worker administration filters are invalid')
normalized[name] = item
if 'worker' in filters:
item = str(filters['worker'] or '').strip()
if not 1 <= len(item) <= 128 or '\x00' in item:
raise ValueError('worker administration filters are invalid')
normalized['worker'] = item
if 'assignment_outcome' in filters:
item = str(filters['assignment_outcome'] or '')
if item not in {'accepted', 'prebundle_failed', 'expired', 'unfinished'}:
raise ValueError('worker administration filters are invalid')
normalized['assignment_outcome'] = item
if 'scan_outcome' in filters:
item = str(filters['scan_outcome'] or '')
if item not in {'clean', 'found', 'degraded', 'error', 'skipped', 'unavailable'}:
raise ValueError('worker administration filters are invalid')
normalized['scan_outcome'] = item
if 'retryable' in filters:
if type(filters['retryable']) is not bool:
raise ValueError('worker administration filters are invalid')
normalized['retryable'] = filters['retryable']
if 'since' in filters:
item = str(filters['since'] or '')
parse_time(item)
normalized['since'] = item
return normalized
def _admin_worker_filter_sql(filters, *, diagnostic_alias=None):
filters = _validated_admin_worker_filters(filters)
conditions = []
parameters = []
if 'source' in filters:
conditions.append('r.source = ?')
parameters.append(filters['source'])
if 'worker' in filters:
conditions.append('d.device_key = ?')
parameters.append(filters['worker'])
if 'assignment_outcome' in filters:
conditions.append(f'({_admin_assignment_outcome_sql()}) = ?')
parameters.append(filters['assignment_outcome'])
if 'scan_outcome' in filters:
conditions.append(f'({_admin_scan_outcome_sql()}) = ?')
parameters.append(filters['scan_outcome'])
if 'since' in filters:
timestamp_column = f'{diagnostic_alias}.occurred_at' if diagnostic_alias else 'r.remote_issued_at'
conditions.append(f'{timestamp_column} >= ?')
parameters.append(filters['since'])
diagnostic_conditions = []
if 'phase' in filters:
if diagnostic_alias:
conditions.append(f'{diagnostic_alias}.phase = ?')
parameters.append(filters['phase'])
else:
conditions.append('''(
p.phase = ? OR EXISTS (
SELECT 1 FROM worker_diagnostics fd
WHERE fd.reservation_id = r.id AND fd.phase = ?
)
)''')
parameters.extend((filters['phase'], filters['phase']))
for name in ('category', 'code'):
if name in filters:
diagnostic_conditions.append(f'fd.{name} = ?')
parameters.append(filters[name])
if 'retryable' in filters:
diagnostic_conditions.append('fd.retryable = ?')
parameters.append(1 if filters['retryable'] else 0)
if diagnostic_conditions:
if diagnostic_alias:
conditions.extend(
condition.replace('fd.', f'{diagnostic_alias}.')
for condition in diagnostic_conditions
)
else:
conditions.append('''EXISTS (
SELECT 1 FROM worker_diagnostics fd
WHERE fd.reservation_id = r.id AND %s
)''' % ' AND '.join(diagnostic_conditions))
return filters, conditions, parameters
def _validated_discovery_retry_source(value):
if not isinstance(value, str) or not re.fullmatch(r'[a-z][a-z0-9_-]{0,63}', value):
raise ValueError('discovery retry source is invalid')
return value
def _validated_discovery_retry_query(value):
if (
not isinstance(value, str)
or not value
or value != value.strip()
or len(value) > DISCOVERY_RETRY_MAX_QUERY_CHARS
or any(ord(character) < 32 or ord(character) == 127 for character in value)
):
raise ValueError('discovery retry query is invalid')
return value
def _validated_discovery_retry_policy(value):
policy = str(value or '')
if not re.fullmatch(r'[a-f0-9]{64}', policy):
raise ValueError('discovery retry policy hash is invalid')
return policy
def _validated_discovery_retry_pages(work_kind, page_start, page_end):
work_kind = str(work_kind or '')
if work_kind not in DISCOVERY_RETRY_WORK_KINDS:
raise ValueError('discovery retry work kind is invalid')
if isinstance(page_start, bool) or isinstance(page_end, bool):
raise ValueError('discovery retry page range is invalid')
try:
start = int(page_start)
end = int(page_end)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry page range is invalid') from None
if not 1 <= start <= end <= DISCOVERY_RETRY_MAX_PAGE:
raise ValueError('discovery retry page range is invalid')
if work_kind == 'query' and start != 1:
raise ValueError('query-level discovery retry work must start at page one')
if work_kind == 'page' and start != end:
raise ValueError('page-level discovery retry work must identify one page')
return work_kind, start, end
def _validated_discovery_retry_error_category(value, required=False):
category = str(value or '')
if not category and not required:
return None
if category not in DISCOVERY_RETRY_ERROR_CATEGORIES:
raise ValueError('discovery retry error category is invalid')
return category
def _validated_discovery_retry_fence(retry_id, lease_owner, lease_token):
if isinstance(retry_id, bool):
raise ValueError('discovery retry lease identity is invalid')
try:
retry_id = int(retry_id)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry lease identity is invalid') from None
owner = str(lease_owner or '')
token = str(lease_token or '')
if (
retry_id < 1
or not 1 <= len(owner) <= 128
or not 1 <= len(token) <= 128
or any(ord(character) < 32 or ord(character) == 127 for character in owner + token)
):
raise ValueError('discovery retry lease identity is invalid')
return retry_id, owner, token
def _validated_discovery_retry_time(value, now=None):
if value is None:
return None
parsed = parse_time(value)
if not parsed:
raise ValueError('discovery retry time is invalid')
now = now or datetime.now(timezone.utc)
if parsed > now + timedelta(seconds=DISCOVERY_RETRY_MAX_TRUSTED_DELAY_SEC):
raise ValueError('discovery retry time exceeds the trusted delay bound')
return max(parsed, now).isoformat(timespec='seconds')
def _discovery_retry_allowlist(configured_query_policies):
if isinstance(configured_query_policies, dict):
entries = list(configured_query_policies.items())
else:
try:
entries = list(configured_query_policies or ())
except TypeError:
raise ValueError('discovery retry query policy allowlist is invalid') from None
if len(entries) > DISCOVERY_RETRY_MAX_ALLOWLIST:
raise ValueError('discovery retry query policy allowlist exceeds its bound')
allowed = {}
for entry in entries:
if not isinstance(entry, (list, tuple)) or len(entry) != 2:
raise ValueError('discovery retry query policy allowlist is invalid')
query = _validated_discovery_retry_query(entry[0])
policy = _validated_discovery_retry_policy(entry[1])
if query in allowed and allowed[query] != policy:
raise ValueError('discovery retry query policy allowlist conflicts')
allowed[query] = policy
return tuple(sorted(allowed.items()))
def discovery_retry_work_key(
source, query, policy_sha256, pass_kind, work_kind, page_start, page_end,
pass_id=None,
):
source = _validated_discovery_retry_source(source)
query = _validated_discovery_retry_query(query)
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
pass_kind = str(pass_kind or '')
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
raise ValueError('discovery retry pass kind is invalid')
work_kind, page_start, page_end = _validated_discovery_retry_pages(
work_kind, page_start, page_end,
)
identity = [
source, query, policy_sha256, pass_kind, work_kind, page_start, page_end,
]
if pass_id is not None:
if isinstance(pass_id, bool):
raise ValueError('discovery retry pass identity is invalid')
try:
pass_id = int(pass_id)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry pass identity is invalid') from None
if not 1 <= pass_id <= 0x7fffffffffffffff:
raise ValueError('discovery retry pass identity is invalid')
identity.append(pass_id)
payload = json.dumps(
identity,
ensure_ascii=True,
separators=(',', ':'),
).encode('utf-8')
digest = hashlib.sha256(payload).hexdigest()
if pass_id is None:
return digest
return f'{pass_id:016x}{digest[:48]}'
def _validated_docker_discovery_observation(value, source, query):
if value is None:
return None
if not isinstance(value, dict):
raise ValueError('DockerHub discovery observation metadata is invalid')
required = {
'cycle_id', 'query_ordinal', 'query_count', 'page_number', 'page_limit',
'per_page', 'total_count', 'policy_sha256', 'pass_kind',
'collection_generation', 'ordered_query_hash', 'query_complete',
}
if set(value) != required:
raise ValueError('DockerHub discovery observation metadata shape is invalid')
numeric = {}
for name in ('query_ordinal', 'query_count', 'page_number', 'page_limit', 'per_page'):
raw = value[name]
if isinstance(raw, bool):
raise ValueError('DockerHub discovery observation bounds are invalid')
try:
numeric[name] = int(raw)
except (TypeError, ValueError, OverflowError):
raise ValueError('DockerHub discovery observation bounds are invalid') from None
if not (
1 <= numeric['query_count'] <= 1000
and 0 <= numeric['query_ordinal'] < numeric['query_count']
and 1 <= numeric['page_number'] <= DISCOVERY_RETRY_MAX_PAGE
and numeric['page_number'] <= numeric['page_limit'] <= DISCOVERY_RETRY_MAX_PAGE
and 1 <= numeric['per_page'] <= DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE
):
raise ValueError('DockerHub discovery observation bounds are invalid')
cycle_id = value['cycle_id']
if cycle_id is not None:
if isinstance(cycle_id, bool):
raise ValueError('DockerHub discovery source cycle identity is invalid')
try:
cycle_id = int(cycle_id)
except (TypeError, ValueError, OverflowError):
raise ValueError('DockerHub discovery source cycle identity is invalid') from None
if cycle_id < 1:
raise ValueError('DockerHub discovery source cycle identity is invalid')
pass_kind = str(value['pass_kind'] or '')
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
raise ValueError('DockerHub discovery pass kind is invalid')
ordered_query_hash = str(value['ordered_query_hash'] or '')
if not re.fullmatch(r'[a-f0-9]{64}', ordered_query_hash):
raise ValueError('DockerHub discovery ordered-query hash is invalid')
if not isinstance(value['query_complete'], bool):
raise ValueError('DockerHub discovery query completion evidence is invalid')
total_count = value['total_count']
if total_count is not None:
if isinstance(total_count, bool):
raise ValueError('DockerHub discovery total-count evidence is invalid')
try:
total_count = int(total_count)
except (TypeError, ValueError, OverflowError):
raise ValueError('DockerHub discovery total-count evidence is invalid') from None
if not 0 <= total_count <= 0x7fffffffffffffff:
raise ValueError('DockerHub discovery total-count evidence is invalid')
if value['query_complete'] and total_count is None:
raise ValueError('DockerHub terminal discovery evidence lacks a total count')
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
collection_generation = str(value['collection_generation'] or '')
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
raise ValueError('DockerHub discovery collection generation is invalid')
return {
'cycle_id': cycle_id,
**numeric,
'policy_sha256': _validated_discovery_retry_policy(value['policy_sha256']),
'pass_kind': pass_kind,
'collection_generation': collection_generation,
'ordered_query_hash': ordered_query_hash,
'total_count': total_count,
'query_complete': value['query_complete'],
'source': source,
'query': query,
}
def _normalized_dockerhub_repositories(repositories):
try:
offered = list(repositories or ())
except TypeError:
raise ValueError('DockerHub discovery repositories must be iterable') from None
if len(offered) > DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE:
raise ValueError('DockerHub discovery page exceeds its repository bound')
normalized = []
observed = []
ordinals = {}
seen = set()
for ordinal, repository in enumerate(offered, 1):
value = repository.get('repo_name') if isinstance(repository, dict) else repository
if not isinstance(value, str) or not value or value != value.strip():
raise ValueError('DockerHub discovery repository is invalid')
try:
bare = validate_docker_image_reference(value, require_digest=False)
except (TypeError, ValueError):
raise ValueError('DockerHub discovery repository is invalid') from None
if '@' in bare or ':' in bare.rsplit('/', 1)[-1]:
raise ValueError('DockerHub discovery repository must be a bare repository anchor')
identity = docker_target_identity(bare)
if not identity:
raise ValueError('DockerHub discovery repository is invalid')
observed.append(identity)
if identity in seen:
continue
seen.add(identity)
normalized.append(identity)
ordinals[identity] = ordinal
return len(offered), normalized, ordinals, observed
def extract_package_metadata(target, result, source):
source = str(source or '').lower()
package = result.get('package') if isinstance(result, dict) else None
if not package and source in ('npm', 'pypi', 'package_git'):
try:
package = json.loads(str(target).strip())
except Exception:
package = {}
if not isinstance(package, dict):
package = {}
return {
'package_name': package.get('name'),
'package_version': package.get('version'),
'package_artifact': package.get('artifact') or package.get('tarball') or package.get('repo_url'),
'repo_url': package.get('repo_url'),
'repo_provider': package.get('provider'),
'package_date': package.get('date'),
'package_filename': package.get('filename'),
'package_type': package.get('packagetype') or package.get('type'),
'package_size': package.get('size'),
}
def extract_raw_secret(finding):
for key in ('RawV2', 'Raw'):
value = finding.get(key)
if value:
return str(value)
structured = finding.get('StructuredData')
if isinstance(structured, dict):
for value in structured.values():
if isinstance(value, str) and value:
return value
return ''
def extract_redacted_secret(finding):
value = finding.get('Redacted')
if value:
return str(value)
return '***REDACTED***' if extract_raw_secret(finding) else ''
def extract_finding_location(finding):
metadata = finding.get('SourceMetadata') or {}
data = metadata.get('Data') if isinstance(metadata, dict) else {}
source_type = ''
details = {}
if isinstance(data, dict):
for key, value in data.items():
if isinstance(value, dict):
source_type = key
details = value
break
if not isinstance(details, dict):
details = {}
return {
'source_metadata_type': source_type,
'file_path': details.get('file') or details.get('path') or details.get('File') or '',
'line_number': str(details.get('line') or details.get('Line') or ''),
'commit_hash': details.get('commit') or details.get('commitHash') or details.get('commit_hash') or '',
'source_timestamp': details.get('timestamp') or details.get('Timestamp') or '',
'source_metadata_json': json_dumps(metadata) if metadata else '',
}
def finding_identity(source, normalized_target, finding):
raw_secret = extract_raw_secret(finding)
secret_hash = sha256_text(raw_secret)
detector = str(finding.get('DetectorName') or finding.get('DetectorType') or '')
location = extract_finding_location(finding)
fallback_hash = sha256_text(json_dumps(finding)) if not secret_hash else ''
detector_secret_hash = sha256_text('|'.join([detector, secret_hash or fallback_hash]))
fingerprint = sha256_text('|'.join([
str(source or ''),
str(normalized_target or ''),
detector,
secret_hash or fallback_hash,
str(location.get('file_path') or ''),
str(location.get('line_number') or ''),
str(location.get('commit_hash') or ''),
str(bool(finding.get('Verified', False))),
]))
return raw_secret, secret_hash, detector_secret_hash, fingerprint, location
def scanner_context(finding):
context = finding.get('ScannerContext')
return context if isinstance(context, dict) else {}
def context_text(finding):
context = scanner_context(finding)
return str(context.get('nearby') or '')
def find_first(patterns, text):
for pattern in patterns:
match = re.search(pattern, text or '', re.IGNORECASE | re.MULTILINE)
if match:
return match.group(1)
return ''
def split_dockerhub_rawv2(rawv2):
if not rawv2 or ':' not in rawv2:
return '', ''
username, token = rawv2.split(':', 1)
return username, token
def split_azure_openai_rawv2(rawv2):
rawv2 = rawv2 or ''
match = re.match(r'^([a-f0-9]{32}):(.+\.openai\.azure\.com)$', rawv2, re.IGNORECASE)
if not match:
return '', ''
return match.group(1), match.group(2)
def split_azure_devops_rawv2(raw, rawv2):
raw = raw or ''
rawv2 = rawv2 or ''
if raw and rawv2.startswith(raw) and len(rawv2) > len(raw):
return rawv2[len(raw):]
return ''
def confidence_for(finding, complete=False, legacy=False, missing=False):
if finding.get('Verified'):
return 'verified'
if legacy:
return 'legacy_unverified'
if missing:
return 'token_only_missing_context'
if complete:
return 'structured_complete'
return 'unverified'
def enrich_finding(finding):
finding = sanitize_postman_finding(finding)
detector = str(finding.get('DetectorName') or '')
detector_key = detector.lower()
raw = str(finding.get('Raw') or '')
rawv2 = str(finding.get('RawV2') or '')
extra = finding.get('ExtraData') if isinstance(finding.get('ExtraData'), dict) else {}
analysis = finding.get('AnalysisInfo') if isinstance(finding.get('AnalysisInfo'), dict) else {}
text = context_text(finding)
out = {
'provider': '',
'credential_kind': detector,
'credential_confidence': confidence_for(finding),
'required_context_missing': 0,
'principal': '',
'username': '',
'email': '',
'project_id': '',
'tenant_id': '',
'organization': '',
'registry': '',
'endpoint': '',
'scope': '',
'resource': '',
'enrichment_json': '',
}
postman_context = finding.get('PostmanContext') if isinstance(finding.get('PostmanContext'), dict) else {}
if postman_context:
postman_kind = postman_context.get('credential_kind') or detector
is_adc = (
detector_key == 'gcpapplicationdefaultcredentials'
or str(postman_kind or '').lower() == 'application_default_credentials'
)
out.update({
'provider': postman_context.get('provider') or out['provider'],
'credential_kind': postman_kind,
'credential_confidence': postman_context.get('credential_confidence') or confidence_for(finding),
'endpoint': postman_context.get('endpoint') or postman_context.get('host') or '',
'resource': '' if is_adc else postman_context.get('json_path') or '',
'scope': postman_context.get('context_location') or '',
'username': postman_context.get('variable_name') or '',
'required_context_missing': 0,
'enrichment_json': json_dumps(postman_context),
})
return out
if detector_key == 'gcp':
data = safe_json_loads(rawv2)
out.update({'provider': 'gcp', 'credential_kind': 'service_account'})
if isinstance(data, dict):
out.update({
'project_id': data.get('project_id') or extra.get('project') or '',
'principal': data.get('client_email') or analysis.get('principal') or '',
'username': data.get('client_email') or '',
'resource': data.get('private_key_id') or '',
'credential_confidence': confidence_for(finding, complete=True),
'enrichment_json': json_dumps({
'type': data.get('type'),
'client_id': data.get('client_id'),
'private_key_id': data.get('private_key_id'),
'client_x509_cert_url': data.get('client_x509_cert_url'),
}),
})
return out
if detector_key == 'gcpapplicationdefaultcredentials':
data = safe_json_loads(text)
if not isinstance(data, dict):
data = {}
project_id = data.get('quota_project_id') or data.get('project_id') or extra.get('project') or ''
out.update({
'provider': 'gcp',
'credential_kind': 'application_default_credentials',
'principal': data.get('client_id') or '',
'project_id': project_id,
'resource': '',
'required_context_missing': 0 if data else 1,
'credential_confidence': confidence_for(finding, complete=bool(data), missing=not bool(data)),
'enrichment_json': json_dumps({
'client_id': data.get('client_id'),
'type': data.get('type'),
'project_id': data.get('project_id'),
'quota_project_id': data.get('quota_project_id'),
'has_client_secret': bool(data.get('client_secret')),
'has_refresh_token': bool(data.get('refresh_token')),
}),
})
return out
if detector_key == 'azurecontainerregistry':
data = safe_json_loads(rawv2)
registry = data.get('username') if isinstance(data, dict) else ''
out.update({
'provider': 'azure',
'credential_kind': 'azure_container_registry',
'registry': registry or find_first([r'([a-z0-9][a-z0-9-]{1,100}[a-z0-9])\.azurecr\.io'], text),
'endpoint': (registry + '.azurecr.io') if registry else '',
'username': registry or '',
'credential_confidence': confidence_for(finding, complete=bool(registry)),
'required_context_missing': 0 if registry else 1,
})
return out
if detector_key == 'azureopenai':
_, endpoint = split_azure_openai_rawv2(rawv2)
endpoint = endpoint or find_first([r'([a-z0-9-]+\.openai\.azure\.com)'], text)
out.update({
'provider': 'azure',
'credential_kind': 'azure_openai_key',
'endpoint': endpoint,
'resource': endpoint.split('.')[0] if endpoint else '',
'credential_confidence': confidence_for(finding, complete=bool(endpoint), missing=not bool(endpoint)),
'required_context_missing': 0 if endpoint else 1,
})
return out
if detector_key == 'azuredevopspersonalaccesstoken':
org = split_azure_devops_rawv2(raw, rawv2) or find_first([
r'https?://dev\.azure\.com/([A-Za-z0-9][A-Za-z0-9-]{1,80})',
r'https?://([A-Za-z0-9][A-Za-z0-9-]{1,80})\.visualstudio\.com',
], text)
out.update({
'provider': 'azure',
'credential_kind': 'azure_devops_pat',
'organization': org,
'endpoint': f'https://dev.azure.com/{org}' if org else '',
'credential_confidence': confidence_for(finding, complete=bool(org), missing=not bool(org)),
'required_context_missing': 0 if org else 1,
})
return out
if detector_key in ('googleai', 'googleaistudio') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() == 'googleaistudio'):
is_aistudio = detector_key == 'googleaistudio' or str(extra.get('name') or '').lower() == 'googleaistudio'
out.update({
'provider': 'google',
'credential_kind': 'google_ai_studio_api_key' if is_aistudio else 'google_ai_api_key',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key in ('qwendashscope', 'qwen_dashscope', 'qwen', 'dashscope') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() in ('qwendashscope', 'qwen_dashscope')):
endpoint = find_first([
r'((?:dashscope(?:-intl|-us)?|cn-hongkong\.dashscope)\.aliyuncs\.com)',
r'(cn-hongkong\.aliyuncs\.com)',
], text)
out.update({
'provider': 'alibaba',
'credential_kind': 'qwen_dashscope_api_key',
'endpoint': endpoint,
'resource': 'dashscope',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key in ('kimimoonshot', 'moonshotai', 'moonshot', 'kimi') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() in ('kimimoonshot', 'moonshotai')):
endpoint = find_first([
r'(api\.moonshot\.ai)',
r'(api\.moonshot\.cn)',
r'(platform\.kimi\.(?:ai|com))',
], text)
out.update({
'provider': 'moonshot',
'credential_kind': 'kimi_moonshot_api_key',
'endpoint': endpoint,
'resource': 'kimi',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'zaiglm' or (detector_key == 'customregex' and str(extra.get('name') or '').lower() == 'zaiglm'):
endpoint = find_first([
r'(api\.z\.ai)',
r'(open\.bigmodel\.cn)',
r'(bigmodel\.cn)',
], text)
out.update({
'provider': 'zai',
'credential_kind': 'glm_api_key',
'endpoint': endpoint or 'api.z.ai',
'resource': 'glm',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'groq':
out.update({
'provider': 'groq',
'credential_kind': 'groq_api_key',
'endpoint': 'api.groq.com',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'replicate':
out.update({
'provider': 'replicate',
'credential_kind': 'replicate_api_token',
'endpoint': 'api.replicate.com',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'xai':
out.update({
'provider': 'xai',
'credential_kind': 'xai_api_key',
'endpoint': 'api.x.ai',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'huggingface':
out.update({
'provider': 'huggingface',
'credential_kind': 'huggingface_user_access_token',
'endpoint': 'huggingface.co',
'credential_confidence': confidence_for(finding, complete=True),
})
return out
if detector_key == 'dockerhub':
username, _ = split_dockerhub_rawv2(rawv2)
username = username or extra.get('hub_username') or analysis.get('username') or find_first([
r'(?i)(?:docker(?:hub)?[_-]?)?(?:user|username|usr|login|id)\s*[:=]\s*["\']?([a-zA-Z0-9][a-zA-Z0-9_.-]{2,60})',
r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})',
], text)
out.update({
'provider': 'dockerhub',
'credential_kind': 'dockerhub_pat',
'username': username,
'email': extra.get('hub_email') or (username if '@' in username else ''),
'scope': extra.get('hub_scope') or '',
'credential_confidence': confidence_for(finding, complete=bool(username), missing=not bool(username)),
'required_context_missing': 0 if username else 1,
'enrichment_json': json_dumps({'2fa_required': extra.get('2fa_required')}),
})
return out
if detector_key in ('github', 'githuboauth2'):
legacy = detector_key == 'githuboauth2' or (raw and not raw.startswith(('ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_')))
out.update({
'provider': 'github',
'credential_kind': 'github_oauth_app' if detector_key == 'githuboauth2' else 'github_token',
'principal': extra.get('username') or analysis.get('key') or '',
'scope': extra.get('scopes') or '',
'credential_confidence': confidence_for(finding, complete=not legacy, legacy=legacy),
})
return out
if detector_key == 'gitlab':
legacy = raw and not raw.startswith(('glpat-', 'gloas-', 'glcbt-', 'glimt-', 'glrt-', 'glft-', 'glsoat-'))
out.update({
'provider': 'gitlab',
'credential_kind': 'gitlab_token',
'principal': analysis.get('key') or '',
'endpoint': analysis.get('host') or '',
'credential_confidence': confidence_for(finding, complete=not legacy, legacy=legacy),
})
return out
return out
def first_error_summary(result):
for error in result.get('errors', []) or []:
for line in str(error).splitlines():
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
for key in ('error', 'msg', 'message'):
if payload.get(key):
return str(payload[key])[:500]
except Exception:
pass
return line[:500]
return ''
def categorize_error(error):
text = str(error or '').lower()
if 'secondary rate limit' in text or 'abuse' in text:
return 'secondary_rate_limit'
if 'rate limit' in text or 'too many requests' in text or ' 429' in text or '429 ' in text:
return 'rate_limit'
if 'auth_invalid' in text or 'authentication failed' in text or 'unauthorized' in text or '401' in text:
return 'auth_invalid'
if 'auth_forbidden' in text or 'forbidden' in text or 'permission denied' in text or 'access denied' in text or '403' in text:
return 'auth_forbidden'
if 'query_invalid' in text or 'validation failed' in text or '422' in text:
return 'query_invalid'
if 'not found' in text or '404' in text:
return 'not_found'
if any(item in text for item in ('server error', '500', '502', '503', '504')):
return 'server_error'
if any(item in text for item in ('no space left', 'not enough free space', 'disk')):
return 'disk_space'
if any(item in text for item in ('timed out', 'timeout', 'deadline exceeded')):
return 'timeout'
if any(item in text for item in ('extract', 'tar', 'zip', 'gzip', 'invalid header')):
return 'extract'
if any(item in text for item in ('download', 'artifact exceeds', 'fetch')):
return 'download'
if any(item in text for item in ('connection', 'dns', 'tls', 'ssl', 'proxy', 'network')):
return 'network'
if 'api' in text or 'http' in text:
return 'api'
if any(item in text for item in ('trufflehog', 'error processing image', 'failed to clone')):
return 'trufflehog'
return 'unknown'
def target_status(result):
if result.get('errors'):
return 'error'
if result.get('skipped'):
return 'skipped'
if result.get('findings'):
return 'found'
if result.get('degraded') or result.get('warnings'):
return 'degraded'
return 'clean'
def summarize_results(results):
summary = {
'scanned_count': len(results or []),
'clean_count': 0,
'found_count': 0,
'skipped_count': 0,
'error_count': 0,
'degraded_count': 0,
'findings_count': 0,
'verified_findings_count': 0,
'unique_secrets_count': 0,
'unique_findings_count': 0,
}
secret_hashes = set()
fingerprints = set()
for result in results or []:
status = target_status(result)
summary[f'{status}_count'] += 1
findings = result.get('findings') or []
summary['findings_count'] += len(findings)
summary['verified_findings_count'] += sum(1 for finding in findings if finding.get('Verified', False))
source = result.get('scan_type') or ''
normalized = normalize_target(result.get('target', ''), source)
for finding in findings:
_, secret_hash, _, fingerprint, _ = finding_identity(source, normalized, finding)
if secret_hash:
secret_hashes.add(secret_hash)
if fingerprint:
fingerprints.add(fingerprint)
summary['unique_secrets_count'] = len(secret_hashes)
summary['unique_findings_count'] = len(fingerprints)
return summary
def count_file_lines(path):
if not path or not os.path.exists(path):
return 0
try:
with open(path, 'r', encoding='utf-8') as f:
return sum(1 for line in f if line.strip())
except OSError:
return 0
def queue_counts(todo_file=None, checked_file=None):
return {
'todo_count': count_file_lines(todo_file),
'checked_count': count_file_lines(checked_file),
'todo_file': todo_file,
'checked_file': checked_file,
}
class ScannerDB:
def __init__(self, results_dir=None, db_path=None, enabled=True, initialize=True, db_url=None):
explicit_url = get_database_url() if db_url is None else db_url
if is_postgres_url(db_path) and not explicit_url:
explicit_url = db_path
db_path = None
self.url = explicit_url if is_postgres_url(explicit_url) else None
self.postgres_required = bool(explicit_url)
self.path = None if self.url else get_database_path(results_dir, db_path)
self.db_display = redact_database_url(self.url) if self.url else self.path
self.conn = None
self._keycheck_result_columns = None
self._last_claim_expectation = None
self._result_spool_publisher_held = False
self._pipeline_advisory_held = set()
self._runtime_managed_file_execution_held = None
self._target_queue_counts_cache = {}
self._target_queue_counts_retry_after = {}
self.last_error = ''
if explicit_url and not self.url:
logger.error(f'Unsupported database URL scheme: {redact_database_url(explicit_url)}')
self.path = None
return
if not enabled or database_disabled():
self.path = None
self.url = None
self.db_display = None
return
try:
if not self.url:
parent = os.path.dirname(self.path)
if parent:
os.makedirs(parent, exist_ok=True)
self.conn = self._new_connection(timeout_sec=SQLITE_CONNECT_TIMEOUT_SEC)
if initialize and self.conn.is_sqlite:
self.conn.execute('PRAGMA journal_mode=WAL')
if initialize and self.conn.is_sqlite:
self.conn.execute('PRAGMA synchronous=NORMAL')
if initialize and self.conn.is_sqlite:
self.initialize_schema()
except Exception as e:
logger.error(f'Observability DB disabled after init failure: {e}')
self.conn = None
@property
def enabled(self):
return self.conn is not None
@classmethod
def host_agent_authority(cls):
database = cls(enabled=False)
database.postgres_required = True
database.db_display = 'fixed host-agent PostgreSQL authority'
connection = connect_host_agent_postgres()
try:
connection.execute("SET application_name = 'truf-host-agent'")
except BaseException:
connection.close()
raise
database.conn = connection
return database
def _new_connection(self, timeout_sec=None):
if self.url:
conn = connect_postgres(self.url)
conn.execute("SET application_name = 'truf-observability'")
return conn
conn = connect_sqlite(self.path, timeout_sec=max(1, int(timeout_sec or SQLITE_CONNECT_TIMEOUT_SEC)))
conn.execute(f'PRAGMA busy_timeout={SQLITE_BUSY_TIMEOUT_MS}')
conn.execute('PRAGMA foreign_keys=ON')
return conn
def _reset_connection(self):
held = self._runtime_managed_file_execution_held
self._runtime_managed_file_execution_held = None
connection = self.conn
self.conn = None
cancellation = None
if held is not None and held[2] is not None:
try:
held[2].release()
except RuntimeError:
pass
except BaseException as exc:
cancellation = exc
try:
if connection:
connection.close()
except BaseException as exc:
if not isinstance(exc, Exception) and cancellation is None:
cancellation = exc
self._result_spool_publisher_held = False
self._pipeline_advisory_held.clear()
if cancellation is not None:
raise cancellation
try:
self.conn = self._new_connection()
self._runtime_safety_schema_validated = False
return True
except Exception as e:
logger.error(f'Observability DB reconnect failed: {e}')
self.conn = None
return False
def close(self):
held = self._runtime_managed_file_execution_held
self._runtime_managed_file_execution_held = None
connection = self.conn
self.conn = None
self._result_spool_publisher_held = False
self._pipeline_advisory_held.clear()
failure = None
if held is not None and held[2] is not None:
try:
held[2].release()
except RuntimeError:
pass
except BaseException as exc:
failure = exc
try:
if connection:
connection.close()
except BaseException as exc:
if (
failure is None
or isinstance(failure, Exception) and not isinstance(exc, Exception)
):
failure = exc
if failure is not None:
raise failure
def set_application_name(self, value):
if not self.conn or not self.conn.is_postgres:
return True
name = str(value or '').strip()
if not name or any(character in name for character in ('\x00', '\r', '\n')):
raise ValueError('PostgreSQL application name is invalid')
self.conn.execute("SELECT set_config('application_name', ?, false)", (name[:63],))
self.conn.commit()
return True
def acquire_runtime_managed_file_execution(self, operation_id):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if self._runtime_managed_file_execution_held is not None:
raise RuntimeOperationTransitionError(
'runtime managed file execution lock is already held'
)
lock_key = int.from_bytes(
hashlib.sha256(operation_id.encode('ascii')).digest()[:4],
'big', signed=True,
)
if self.conn.is_postgres:
self.conn.execute(
'SELECT pg_catalog.pg_advisory_lock(?, ?)',
(RUNTIME_MANAGED_FILE_ADVISORY_CLASS, lock_key),
)
self.conn.commit()
lock = None
else:
lock = _RUNTIME_MANAGED_FILE_SQLITE_LOCKS[
lock_key % len(_RUNTIME_MANAGED_FILE_SQLITE_LOCKS)
]
lock.acquire()
self._runtime_managed_file_execution_held = (
operation_id, lock_key, lock,
)
return True
def release_runtime_managed_file_execution(self, operation_id):
operation_id = _runtime_operation_id(operation_id)
held = self._runtime_managed_file_execution_held
if held is None or held[0] != operation_id:
raise RuntimeOperationTransitionError(
'runtime managed file execution lock is not held'
)
self._runtime_managed_file_execution_held = None
if self.conn.is_postgres:
row = self.conn.execute(
'''SELECT pg_catalog.pg_advisory_unlock(?, ?) AS released''',
(RUNTIME_MANAGED_FILE_ADVISORY_CLASS, held[1]),
).fetchone()
self.conn.commit()
if not row or not bool(row['released']):
raise RuntimeSafetySchemaError(
'runtime managed file execution lock release failed'
)
else:
held[2].release()
return True
def acquire_pipeline_lease(
self, worker_name, supervisor_instance_id, owner_identity,
lease_seconds=30, initial_state='starting',
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('pipeline singleton leases require PostgreSQL')
worker_name = str(worker_name or '')
advisory_object = PIPELINE_ADVISORY_OBJECTS.get(worker_name)
if advisory_object is None:
raise ValueError('unknown pipeline worker lease name')
if worker_name in self._pipeline_advisory_held:
raise RuntimeError(f'pipeline advisory lease is already held: {worker_name}')
identity = _identity_mapping(owner_identity)
token = secrets.token_urlsafe(32)
now = utc_now_iso()
expires = datetime.fromtimestamp(
time.time() + max(5, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
acquired = False
try:
row = self.conn.execute(
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
(PIPELINE_ADVISORY_CLASS, advisory_object),
).fetchone()
acquired = bool(row and row['acquired'])
if not acquired:
self.conn.commit()
return None
current = self.conn.execute(
'SELECT generation FROM pipeline_leases WHERE worker_name = ? FOR UPDATE',
(worker_name,),
).fetchone()
generation = int(current['generation'] or 0) + 1 if current else 1
self.conn.execute(
'''INSERT INTO pipeline_leases(
worker_name, generation, lease_token, supervisor_instance_id,
owner_pid, owner_creation_time, owner_executable, state,
acquired_at, heartbeat_at, lease_expires_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(worker_name) DO UPDATE SET
generation = excluded.generation,
lease_token = excluded.lease_token,
supervisor_instance_id = excluded.supervisor_instance_id,
owner_pid = excluded.owner_pid,
owner_creation_time = excluded.owner_creation_time,
owner_executable = excluded.owner_executable,
state = excluded.state,
acquired_at = excluded.acquired_at,
heartbeat_at = excluded.heartbeat_at,
lease_expires_at = excluded.lease_expires_at,
last_error = NULL,
updated_at = excluded.updated_at''',
(
worker_name, generation, token, str(supervisor_instance_id or ''),
identity['pid'], identity['creation_time'], identity['executable'],
initial_state, now, now, expires, now,
),
)
self.conn.commit()
self._pipeline_advisory_held.add(worker_name)
return {'worker_name': worker_name, 'generation': generation, 'lease_token': token}
except Exception:
self.conn.rollback()
if acquired:
try:
self.conn.execute(
'SELECT pg_advisory_unlock(?, ?)',
(PIPELINE_ADVISORY_CLASS, advisory_object),
)
self.conn.commit()
except Exception:
self.conn.rollback()
raise
def heartbeat_pipeline_lease(
self, worker_name, generation, lease_token, lease_seconds=30,
state='ready', error='',
):
if not self.conn or not self.conn.is_postgres or worker_name not in self._pipeline_advisory_held:
return False
now = utc_now_iso()
expires = datetime.fromtimestamp(
time.time() + max(5, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
cursor = self.conn.execute(
'''UPDATE pipeline_leases SET state = ?, heartbeat_at = ?, lease_expires_at = ?,
last_error = ?, updated_at = ?
WHERE worker_name = ? AND generation = ? AND lease_token = ?''',
(
state, now, expires, first_line(error, 1000) if error else None, now,
worker_name, int(generation), str(lease_token),
),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
def release_pipeline_lease(self, worker_name, generation, lease_token, state='released', error=''):
if not self.conn or not self.conn.is_postgres:
return False
advisory_object = PIPELINE_ADVISORY_OBJECTS.get(worker_name)
if advisory_object is None or worker_name not in self._pipeline_advisory_held:
return False
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE pipeline_leases SET state = ?, heartbeat_at = ?, lease_expires_at = NULL,
lease_token = NULL, last_error = ?, updated_at = ?
WHERE worker_name = ? AND generation = ? AND lease_token = ?''',
(
state, now, first_line(error, 1000) if error else None, now,
worker_name, int(generation), str(lease_token),
),
)
unlocked = self.conn.execute(
'SELECT pg_advisory_unlock(?, ?) AS released',
(PIPELINE_ADVISORY_CLASS, advisory_object),
).fetchone()
self.conn.commit()
if unlocked and unlocked['released']:
self._pipeline_advisory_held.discard(worker_name)
return int(cursor.rowcount or 0) == 1 and bool(unlocked and unlocked['released'])
except Exception:
self.conn.rollback()
raise
def pipeline_worker_health(self, worker_name='result_ingester', supervisor_instance_id=None):
if not self.conn or not self.conn.is_postgres:
return {'healthy': False, 'reason': 'PostgreSQL is unavailable'}
row = self.conn.execute(
'''SELECT worker_name, generation, state, lease_expires_at,
supervisor_instance_id, heartbeat_at, last_error
FROM pipeline_leases WHERE worker_name = ?''',
(worker_name,),
).fetchone()
self.conn.commit()
healthy = bool(
row and row['state'] == 'ready' and row['lease_expires_at']
and str(row['lease_expires_at']) > utc_now_iso()
and (
not supervisor_instance_id
or str(row['supervisor_instance_id'] or '') == str(supervisor_instance_id)
)
)
return {
'healthy': healthy,
'reason': '' if healthy else str((row or {}).get('last_error') if isinstance(row, dict) else '') or 'worker lease is not ready',
**(dict(row) if row else {}),
}
def try_acquire_result_spool_publisher(self):
if not self.conn or not self.conn.is_postgres:
return True
if self._result_spool_publisher_held:
raise RuntimeError('result-spool publisher lease is already held by this database session')
try:
row = self.conn.execute(
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
).fetchone()
self.conn.commit()
self.last_error = ''
acquired = bool(row and row['acquired'])
self._result_spool_publisher_held = acquired
return acquired
except Exception:
connection = self.conn
self.conn = None
self._result_spool_publisher_held = False
try:
if connection:
connection.close()
except Exception:
pass
raise
def result_spool_publisher_state(self):
if not self.conn or not self.conn.is_postgres:
return {
'status': 'held', 'authenticated': True,
'application_name': 'local', 'pid': os.getpid(),
'holder_identity': f'local:{os.getpid()}', 'state': 'active',
}
try:
rows = self.conn.execute(
'''SELECT a.pid, a.application_name, a.state,
a.backend_start::text AS backend_start,
a.backend_type,
COALESCE(a.client_addr::text, '') AS client_addr,
(a.usename = CURRENT_USER) AS same_user,
(a.datname = CURRENT_DATABASE()) AS same_database
FROM pg_catalog.pg_locks l
JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
WHERE l.locktype = 'advisory' AND l.classid = ? AND l.objid = ?
AND l.objsubid = 2 AND l.granted
ORDER BY a.pid''',
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
).fetchall()
self.conn.commit()
if not rows:
return {'status': 'free', 'authenticated': True}
if len(rows) != 1:
raise RuntimeError('result-spool advisory lock has ambiguous multiple holders')
row = rows[0]
application_name = str(row['application_name'] or '')
client_addr = str(row['client_addr'] or '')
try:
client_ip = ipaddress.ip_interface(client_addr).ip
except ValueError:
client_ip = None
authenticated = bool(
re.fullmatch(r'truf-source:[a-z0-9_]+', application_name)
and row['same_user']
and row['same_database']
and row['backend_type'] == 'client backend'
and client_ip in (ipaddress.ip_address('127.0.0.1'), ipaddress.ip_address('::1'))
and int(row['pid'] or 0) > 0
and row['backend_start']
)
if not authenticated:
raise RuntimeError('result-spool advisory lock holder is not an authenticated managed source')
return {
'status': 'held',
'authenticated': True,
'application_name': application_name,
'pid': int(row['pid']),
'state': str(row['state'] or ''),
'holder_identity': f"{int(row['pid'])}:{row['backend_start']}",
}
except Exception:
try:
self.conn.rollback()
except Exception:
pass
raise
def release_result_spool_publisher(self):
if not self.conn or not self.conn.is_postgres:
return True
if not self._result_spool_publisher_held:
return False
try:
row = self.conn.execute(
'SELECT pg_advisory_unlock(?, ?) AS released',
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
).fetchone()
self.conn.commit()
self.last_error = ''
released = bool(row and row['released'])
if released:
self._result_spool_publisher_held = False
return True
connection = self.conn
self.conn = None
self._result_spool_publisher_held = False
try:
connection.close()
except Exception:
pass
return False
except Exception:
connection = self.conn
self.conn = None
self._result_spool_publisher_held = False
try:
if connection:
connection.close()
except Exception:
pass
raise
def result_spool_reservation_progress(self, reservations):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result-spool reservation progress requires PostgreSQL')
reservations = list(reservations or [])
if not reservations or len(reservations) > 10000:
raise RuntimeError('result-spool reservation progress input is outside its bound')
now = time.time()
claims = []
earliest_progress_at = None
for record in reservations:
try:
recover_after = float(record['recover_after'])
expires_at = float(record['expires_at'])
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError('result-spool reservation deadline metadata is invalid') from exc
if not math.isfinite(recover_after) or not math.isfinite(expires_at) or recover_after < expires_at:
raise RuntimeError('result-spool reservation deadlines are indeterminate')
if recover_after <= now:
raise RuntimeError('result-spool recoverable reservation remained in capacity accounting')
if recover_after - now > RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC:
raise RuntimeError('result-spool reservation recovery deadline exceeds its progress bound')
deadline = min(value for value in (expires_at, recover_after) if value > now) if expires_at > now else recover_after
earliest_progress_at = deadline if earliest_progress_at is None else min(earliest_progress_at, deadline)
for claim in record.get('claims') or []:
if claim.get('queue_id') is None or not claim.get('lease_owner') or not claim.get('lease_token') or not claim.get('claim_batch'):
raise RuntimeError('result-spool reservation claim ownership is incomplete')
claims.append((record, claim))
rows_by_id = {}
ids = sorted({int(claim['queue_id']) for _, claim in claims})
for start in range(0, len(ids), 64):
batch = ids[start:start + 64]
placeholders = ','.join('?' for _ in batch)
rows = self.conn.execute(
f'''SELECT id, status, lease_owner, lease_token, claim_batch, lease_expires_at
FROM target_queue WHERE id IN ({placeholders})''',
batch,
).fetchall()
rows_by_id.update({int(row['id']): row for row in rows})
self.conn.commit()
counts = {'exact_live': 0, 'exact_expired': 0, 'stale_or_reassigned': 0, 'unbound': 0}
counts['unbound'] = sum(1 for record in reservations if not (record.get('claims') or []))
for _, claim in claims:
row = rows_by_id.get(int(claim['queue_id']))
exact = bool(
row
and row['status'] == 'in_progress'
and row['lease_owner'] == claim['lease_owner']
and row['lease_token'] == claim['lease_token']
and row['claim_batch'] == claim['claim_batch']
)
if not exact:
counts['stale_or_reassigned'] += 1
continue
lease = parse_time(row['lease_expires_at'])
if lease and lease.timestamp() > now:
counts['exact_live'] += 1
else:
counts['exact_expired'] += 1
return {
'safe_progress': True,
'counts': counts,
'earliest_progress_in_sec': max(0, int((earliest_progress_at or now) - now)),
}
def initialize_schema(self):
if not self.conn:
return
self.conn.executescript(SCHEMA_SQL)
self.conn.execute(
'''CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_credentials_provider_key
ON keycheck_credentials(service, provider_key_hash)'''
)
now = utc_now_iso()
_ensure_runtime_operations_authority(self.conn, now)
self.conn.execute(
'INSERT INTO pipeline_capacity(id, updated_at) VALUES (1, ?) ON CONFLICT(id) DO NOTHING',
(now,),
)
for stream_name, relative_path, rotation_bytes in (
('scan_results', 'scan_results.jsonl', 256 * 1024 * 1024),
('found_secrets', 'found_secrets.jsonl', 128 * 1024 * 1024),
('scan_errors', 'scan_errors.log', 32 * 1024 * 1024),
):
self.conn.execute(
'''INSERT INTO projection_streams(
stream_name, base_relative_path, current_generation,
rotation_bytes, max_generations, created_at, updated_at
) VALUES (?, ?, 0, ?, 16, ?, ?)
ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, relative_path, rotation_bytes, now, now),
)
self.conn.execute(
'''INSERT INTO projection_cursors(stream_name, generation, committed_offset, updated_at)
VALUES (?, 0, 0, ?) ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, now),
)
migration_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
for version in PIPELINE_MIGRATION_VERSIONS:
self.conn.execute(
'''INSERT INTO runtime_schema_migrations(version, applied_at, code_sha256)
VALUES (?, ?, ?) ON CONFLICT(version) DO NOTHING''',
(version, now, migration_code),
)
if self.conn.is_sqlite:
self.conn.execute('PRAGMA user_version=1')
self.conn.commit()
def _safe(self, label, func, default=None):
if not self.conn:
return default
last_error = None
for attempt in range(SQLITE_LOCK_RETRY_ATTEMPTS):
try:
result = func()
self.last_error = ''
if self.conn and self.conn.is_postgres:
try:
self.conn.commit()
except Exception:
pass
return result
except Exception as e:
last_error = e
self.last_error = str(e)
try:
self.conn.rollback()
except Exception:
pass
if isinstance(e, DiscoveryPausedError):
raise
if not sqlite_lock_error(e) or attempt == SQLITE_LOCK_RETRY_ATTEMPTS - 1:
if sqlite_lock_error(e):
self._reset_connection()
else:
logger.error(f'Observability DB write failed during {label}: {e}')
return default
break
if attempt and attempt % 4 == 0:
self._reset_connection()
if not self.conn:
return default
delay = sqlite_lock_retry_delay(attempt)
logger.warning(f'Observability DB locked during {label}; retrying in {delay:.2f}s ({attempt + 1}/{SQLITE_LOCK_RETRY_ATTEMPTS})')
time.sleep(delay)
logger.error(f'Observability DB write failed during {label}: {last_error}')
return default
def _finalize_stale_source_runs(self, source, timestamp):
source = str(source or '').strip()
if not source:
return 0, 0
reason = 'superseded by a new supervised source process'
totals = []
statements = (
(
'''UPDATE source_cycles SET ended_at = ?, status = 'interrupted',
message = COALESCE(NULLIF(message, ''), ?)
WHERE id IN (
SELECT id FROM source_cycles
WHERE source = ? AND status = 'running'
ORDER BY id LIMIT ?
)''',
(timestamp, reason, source, STALE_RUN_FINALIZE_BATCH_SIZE),
'source_cycles',
'source',
),
(
'''UPDATE runs SET ended_at = ?, status = 'interrupted',
error = COALESCE(NULLIF(error, ''), ?), updated_at = ?
WHERE id IN (
SELECT id FROM runs
WHERE selected_source = ? AND status = 'running'
ORDER BY id LIMIT ?
)''',
(timestamp, reason, timestamp, source, STALE_RUN_FINALIZE_BATCH_SIZE),
'runs',
'selected_source',
),
)
for statement, params, table, source_column in statements:
finalized = 0
for _ in range(STALE_RUN_FINALIZE_MAX_BATCHES):
cursor = self.conn.execute(statement, params)
changed = max(0, int(getattr(cursor, 'rowcount', 0) or 0))
finalized += changed
self.conn.commit()
if changed < STALE_RUN_FINALIZE_BATCH_SIZE:
break
remaining = self.conn.execute(
f'''SELECT 1 FROM {table}
WHERE {source_column} = ? AND status = 'running' LIMIT 1''',
(source,),
).fetchone()
if remaining:
raise RuntimeError(
f'bounded stale-run finalization limit reached for {table}:{source}; retry source start'
)
totals.append(finalized)
return tuple(totals)
def start_run(self, invocation_mode, argv=None, selected_source=None, selected_platform=None, config_path=None, config_hash=None, enabled_sources=None, global_config=None):
def op():
now = utc_now_iso()
if selected_source:
self._finalize_stale_source_runs(selected_source, now)
safe_argv = redact_argv(argv)
command_line = ' '.join(safe_argv)
run_id = self.conn.insert_returning_id(
'''INSERT INTO runs (
started_at, status, invocation_mode, command_line, argv_json,
selected_source, selected_platform, config_path, config_hash,
enabled_sources_json, db_path, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
now, 'running', invocation_mode, command_line, json_dumps(safe_argv),
selected_source, selected_platform, config_path, config_hash,
json_dumps(enabled_sources or []), self.db_display, now, now,
),
)
if global_config is not None:
self.record_config_snapshot(run_id, None, 'global', None, global_config)
self.conn.commit()
return run_id
return self._safe('start_run', op)
def finish_run(self, run_id, status='completed', error=None):
if not run_id:
return
def op():
now = utc_now_iso()
row = self.conn.execute('SELECT started_at FROM runs WHERE id = ?', (run_id,)).fetchone()
duration = elapsed_seconds(row['started_at'], now) if row else None
totals = self.conn.execute(
'''SELECT
COALESCE(SUM(fetched_count), 0) AS fetched,
COALESCE(SUM(queued_new_count), 0) AS queued_new,
COALESCE(SUM(scan_requested_count), 0) AS scan_requested,
COALESCE(SUM(scanned_count), 0) AS scanned,
COALESCE(SUM(clean_count), 0) AS clean,
COALESCE(SUM(found_count), 0) AS found,
COALESCE(SUM(skipped_count), 0) AS skipped,
COALESCE(SUM(error_count), 0) AS errors,
COALESCE(SUM(findings_count), 0) AS findings,
COALESCE(SUM(verified_findings_count), 0) AS verified,
COALESCE(SUM(unique_secrets_count), 0) AS unique_secrets,
COALESCE(SUM(unique_findings_count), 0) AS unique_findings
FROM source_cycles WHERE run_id = ?''',
(run_id,),
).fetchone()
self.conn.execute(
'''UPDATE runs SET ended_at = ?, duration_sec = ?, status = ?, error = ?,
total_fetched = ?, total_queued_new = ?, total_scan_requested = ?, total_scanned = ?,
total_clean = ?, total_found = ?, total_skipped = ?, total_errors = ?,
total_findings = ?, total_verified_findings = ?, total_unique_secrets = ?,
total_unique_findings = ?, updated_at = ? WHERE id = ?''',
(
now, duration, status, error,
totals['fetched'], totals['queued_new'], totals['scan_requested'], totals['scanned'],
totals['clean'], totals['found'], totals['skipped'], totals['errors'],
totals['findings'], totals['verified'], totals['unique_secrets'], totals['unique_findings'], now, run_id,
),
)
self.conn.commit()
self._safe('finish_run', op)
def start_source_cycle(self, run_id, source, platform, mode, query, query_index=None, query_count=None, auth_name=None, source_config=None, queue_before=None):
def op():
now = utc_now_iso()
queue_data = queue_before or {}
cycle_id = self.conn.insert_returning_id(
'''INSERT INTO source_cycles (
run_id, source, platform, mode, query, query_index, query_count, auth_name,
started_at, status, config_json, queue_todo_before, queue_checked_before,
created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
run_id, source, platform, mode, query, query_index, query_count, auth_name,
now, 'running', json_dumps(redact_config(source_config or {})),
queue_data.get('todo_count'), queue_data.get('checked_count'), now, now,
),
)
self.record_config_snapshot(run_id, cycle_id, 'source', source, source_config or {})
self.record_queue_snapshot(run_id, cycle_id, source, 'before', queue_data)
self.conn.commit()
return cycle_id
return self._safe('start_source_cycle', op)
def finish_source_cycle(self, cycle_id, status='completed', metrics=None, queue_after=None, message=None):
if not cycle_id:
return
def op():
now = utc_now_iso()
row = self.conn.execute('SELECT started_at, run_id, source FROM source_cycles WHERE id = ?', (cycle_id,)).fetchone()
duration = elapsed_seconds(row['started_at'], now) if row else None
metrics_data = metrics or {}
queue_data = queue_after or {}
if metrics_data.get('authoritative_async'):
staged = int(metrics_data.get('staged_count') or 0)
self.conn.execute(
'''UPDATE source_cycles SET ended_at = ?, duration_sec = ?, status = ?, message = ?,
fetched_count = ?, queued_new_count = ?, queued_updated_count = ?,
scan_requested_count = ?,
staged_count = staged_count + ?, queue_todo_after = ?,
queue_checked_after = ?, updated_at = ? WHERE id = ?''',
(
now, duration, status, message,
metrics_data.get('fetched_count', 0),
metrics_data.get('queued_new_count', 0),
metrics_data.get('queued_updated_count', 0),
metrics_data.get('scan_requested_count', 0), staged,
queue_data.get('todo_count'), queue_data.get('checked_count'), now, cycle_id,
),
)
if row:
self.conn.execute(
'UPDATE runs SET total_staged = total_staged + ?, updated_at = ? WHERE id = ?',
(staged, now, row['run_id']),
)
self.record_queue_snapshot(row['run_id'], cycle_id, row['source'], 'after', queue_data)
self.conn.commit()
return
scanned = int(metrics_data.get('scanned_count') or metrics_data.get('scanned') or 0)
found = int(metrics_data.get('found_count') or 0)
errors = int(metrics_data.get('error_count') or 0)
verified = int(metrics_data.get('verified_findings_count') or 0)
targets_per_hour = scanned / (duration / 3600) if duration and duration > 0 else 0
hit_rate = found / scanned if scanned else 0
verified_hit_rate = verified / scanned if scanned else 0
error_rate = errors / scanned if scanned else 0
self.conn.execute(
'''UPDATE source_cycles SET ended_at = ?, duration_sec = ?, status = ?, message = ?,
fetched_count = ?, queued_new_count = ?, queued_updated_count = ?,
scan_requested_count = ?, scanned_count = ?,
clean_count = ?, found_count = ?, skipped_count = ?, error_count = ?,
findings_count = ?, verified_findings_count = ?, unique_secrets_count = ?, unique_findings_count = ?,
queue_todo_after = ?, queue_checked_after = ?, targets_per_hour = ?, hit_rate = ?,
verified_hit_rate = ?, error_rate = ?, updated_at = ? WHERE id = ?''',
(
now, duration, status, message,
metrics_data.get('fetched_count', 0), metrics_data.get('queued_new_count', 0),
metrics_data.get('queued_updated_count', 0), metrics_data.get('scan_requested_count', 0), scanned,
metrics_data.get('clean_count', 0), found, metrics_data.get('skipped_count', 0), errors,
metrics_data.get('findings_count', 0), verified, metrics_data.get('unique_secrets_count', 0), metrics_data.get('unique_findings_count', 0),
queue_data.get('todo_count'), queue_data.get('checked_count'), targets_per_hour, hit_rate,
verified_hit_rate, error_rate, now, cycle_id,
),
)
if row:
self.record_queue_snapshot(row['run_id'], cycle_id, row['source'], 'after', queue_data)
self.conn.commit()
self._safe('finish_source_cycle', op)
def record_config_snapshot(self, run_id, cycle_id, scope, source, config):
if not self.conn or run_id is None:
return
self.conn.execute(
'INSERT INTO config_snapshots (run_id, cycle_id, scope, source, config_json, captured_at) VALUES (?, ?, ?, ?, ?, ?)',
(run_id, cycle_id, scope, source, json_dumps(redact_config(config or {})), utc_now_iso()),
)
def record_queue_snapshot(self, run_id, cycle_id, source, phase, counts):
if not self.conn or run_id is None:
return
counts = counts or {}
self.conn.execute(
'''INSERT INTO queue_snapshots (
run_id, cycle_id, source, phase, todo_count, checked_count, todo_file, checked_file, captured_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
run_id, cycle_id, source, phase, counts.get('todo_count'), counts.get('checked_count'),
counts.get('todo_file'), counts.get('checked_file'), utc_now_iso(),
),
)
def target_queue_available(self):
if not self.conn or not self.conn.table_exists('target_queue'):
return False
required = {
'id', 'source', 'platform', 'query', 'target', 'normalized_target', 'status', 'attempts',
'lease_owner', 'lease_token', 'claim_batch', 'leased_at', 'lease_expires_at', 'available_after', 'target_scan_id',
'last_error', 'created_at', 'updated_at', 'completed_at',
}
return required.issubset(set(self.conn.table_columns('target_queue')))
def runtime_safety_schema_available(self, commit=True):
if not self.conn:
self.last_error = 'database connection is unavailable'
return False
required = {table: set(specs) for table, specs in RUNTIME_TABLE_SPECS.items()}
try:
missing = []
for table, columns in required.items():
if not self.conn.table_exists(table):
missing.append(f'table {table}')
continue
absent = sorted(columns - set(self.conn.table_columns(table)))
if absent:
missing.append(f'{table} columns {", ".join(absent)}')
table_details = {}
for table, specs in RUNTIME_TABLE_SPECS.items():
if not self.conn.table_exists(table):
continue
details = self.conn.table_column_details(table)
table_details[table] = details
for name, (expected_type, expected_not_null) in specs.items():
actual = details.get(name)
if not actual:
continue
if not _schema_type_matches(actual['type'], expected_type, self.conn.is_postgres):
missing.append(f'{table}.{name} type {actual["type"] or "<empty>"}')
if bool(actual['not_null']) != bool(expected_not_null):
missing.append(f'{table}.{name} nullability')
primary_key = [name for name in specs if details.get(name, {}).get('primary_key')]
if primary_key != RUNTIME_PRIMARY_KEYS[table]:
missing.append(f'{table} primary key')
generated_id = GENERATED_ID_COLUMNS.get(table)
if generated_id and not _column_generates_id(details.get(generated_id), self.conn.is_postgres):
missing.append(f'{table}.{generated_id} generated ID identity/sequence')
for name, actual in details.items():
if name != generated_id and (actual.get('identity') or actual.get('generated')):
missing.append(f'{table}.{name} unexpected generated expression')
expected_defaults = RUNTIME_COLUMN_DEFAULTS.get(table, {})
for name in specs:
if name == generated_id:
continue
actual = details.get(name)
expected_present = name in expected_defaults
expected_default = expected_defaults.get(name, '')
if actual and (
bool(actual.get('has_default', bool(actual.get('default')))) != expected_present
or (
expected_present
and _normalized_default(actual['default']) != _normalized_default(expected_default)
)
):
missing.append(f'{table}.{name} default')
if self.conn.is_postgres:
primary_indexes = [
index for index in self.conn.table_indexes(table).values()
if index.get('primary')
]
if len(primary_indexes) != 1 or primary_indexes[0]['columns'] != RUNTIME_PRIMARY_KEYS[table] or not _index_usable(primary_indexes[0]):
missing.append(f'{table} primary-key index validity/readiness')
queue_indexes = self.conn.table_indexes('target_queue') if self.conn.table_exists('target_queue') else {}
if not any(
index['unique'] and index['columns'] == ['source', 'normalized_target']
and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in queue_indexes.values()
):
missing.append('unique target_queue source/normalized_target index')
retry_indexes = (
self.conn.table_indexes('discovery_retry_queue')
if self.conn.table_exists('discovery_retry_queue') else {}
)
work_key_index = retry_indexes.get('uq_discovery_retry_queue_work_key')
if (
not work_key_index
or not work_key_index['unique']
or work_key_index['columns'] != ['work_key']
or _normalized_predicate(work_key_index['predicate'])
or not _index_usable(work_key_index)
):
missing.append('unique discovery retry work-key index')
scan_indexes = self.conn.table_indexes('target_scans') if self.conn.table_exists('target_scans') else {}
event_index = scan_indexes.get('uq_target_scans_scan_event_id')
if (
not event_index
or not event_index['unique']
or event_index['columns'] != ['scan_event_id']
or _normalized_predicate(event_index['predicate']) != 'scan_event_idisnotnull'
or not _index_usable(event_index)
):
missing.append('unique partial scan-event index')
for table, name, columns in (
(
'worker_progress_events', 'uq_worker_progress_reservation_sequence',
['reservation_id', 'sequence'],
),
('worker_diagnostics', 'uq_worker_diagnostics_uid', ['diagnostic_uid']),
):
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
index = indexes.get(name)
if (
not index or not index['unique'] or index['columns'] != columns
or _normalized_predicate(index['predicate']) or not _index_usable(index)
):
missing.append(f'unique index {name}')
outbox_indexes = self.conn.table_indexes('scan_publication_outbox') if self.conn.table_exists('scan_publication_outbox') else {}
if not any(
index['unique'] and index['columns'] == ['target_scan_id'] and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in outbox_indexes.values()
):
missing.append('unique outbox target-scan index')
status_index = outbox_indexes.get('idx_scan_publication_outbox_status')
if not status_index or status_index['columns'] != ['status', 'available_after', 'id'] or status_index['unique'] or not _index_usable(status_index):
missing.append('outbox status index')
age_index = outbox_indexes.get('idx_scan_publication_outbox_age')
if not age_index or age_index['columns'] != ['status', 'created_at', 'id'] or age_index['unique'] or not _index_usable(age_index):
missing.append('outbox age index')
for table, column in (('keycheck_event_map', 'event_id'), ('finding_uid_map', 'finding_uid')):
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
if not any(
index['unique'] and index['columns'] == [column] and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in indexes.values()
):
missing.append(f'unique {table}.{column} index')
issue_indexes = self.conn.table_indexes('target_queue_reconciliation_issues') if self.conn.table_exists('target_queue_reconciliation_issues') else {}
if not any(
index['unique']
and index['columns'] == ['source_file', 'file_identity', 'line_number', 'byte_offset', 'reason']
and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in issue_indexes.values()
):
missing.append('unique reconciliation issue identity index')
package_indexes = self.conn.table_indexes('package_repo_candidates') if self.conn.table_exists('package_repo_candidates') else {}
if not any(
index['unique']
and index['columns'] == ['package_source', 'package_name', 'package_version', 'repo_url']
and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in package_indexes.values()
):
missing.append('unique package repository candidate identity index')
required_indexes = {
'runs': {
'idx_runs_started_at': ['started_at'],
'idx_runs_status': ['status'],
'idx_runs_selected_source_status': ['selected_source', 'status', 'id'],
},
'source_cycles': {
'idx_source_cycles_run_id': ['run_id'],
'idx_source_cycles_source_query': ['source', 'query'],
'idx_source_cycles_source_status': ['source', 'status', 'id'],
},
'target_queue': {
'idx_target_queue_source_status': ['source', 'status', 'updated_at'],
'idx_target_queue_observe_source_status': ['source', 'status'],
'idx_target_queue_lease': ['source', 'status', 'lease_expires_at'],
'idx_target_queue_platform_status': ['platform', 'status'],
'idx_target_queue_claim_batch': ['claim_batch', 'lease_owner'],
'idx_target_queue_claim': ['source', 'platform', 'status', 'available_after', 'lease_expires_at', 'id'],
'idx_target_queue_resolver_claim': ['source', 'platform', 'status', 'resolver_state', 'resolver_due_at', 'id'],
'idx_target_queue_source_platform_normalized': ['source', 'platform', 'normalized_target'],
'idx_target_queue_claim_pending': ['source', 'platform', 'id', 'attempts', 'available_after'],
'idx_target_queue_claim_deferred': ['source', 'platform', 'available_after', 'id', 'attempts'],
'idx_target_queue_claim_in_progress': ['source', 'platform', 'id', 'attempts', 'available_after', 'lease_expires_at', 'resolver_state'],
'idx_target_queue_active_lease_owner_token': ['lease_owner', 'lease_token'],
'idx_target_queue_exhausted_attempts': ['source', 'platform', 'status', 'attempts', 'id', 'lease_expires_at'],
'idx_target_queue_updated_rescan': ['source', 'platform', 'completed_at', 'remote_modified_at', 'scan_remote_modified_at', 'id'],
'idx_target_queue_cold': ['source', 'platform', 'query', 'id'],
},
'discovery_retry_queue': {
'idx_discovery_retry_queue_due': ['source', 'available_after', 'id'],
'idx_discovery_retry_queue_lease': ['lease_expires_at', 'id'],
'idx_discovery_retry_queue_policy': [
'source', 'query', 'policy_sha256', 'status', 'id',
],
},
'target_queue_policy_events': {
'idx_target_queue_policy_events_queue': ['queue_id', 'id'],
'idx_target_queue_policy_events_manifest': ['manifest_sha256', 'id'],
},
'target_scans': {
'idx_target_scans_cycle_id': ['cycle_id'],
'idx_target_scans_queue_id': ['queue_id'],
'idx_target_scans_source_status': ['source', 'status'],
'idx_target_scans_source_ended': ['source', 'ended_at'],
'idx_target_scans_source_ended_id': ['source', 'ended_at', 'id'],
'idx_target_scans_source_skip_ended': ['source', 'skipped_reason', 'ended_at'],
'idx_target_scans_cooldown_recent': ['source', 'ended_at', 'id'],
'idx_target_scans_normalized_target': ['normalized_target'],
'idx_target_scans_result_reservation': ['result_reservation_id', 'id'],
},
'keycheck_results': {
'idx_keycheck_results_checked': ['checked_at'],
'idx_keycheck_results_service_status': ['service', 'status_group', 'status'],
'idx_keycheck_results_finding': ['finding_id'],
'idx_keycheck_results_cycle': ['cycle_id'],
'idx_keycheck_results_source_query': ['source', 'query'],
'idx_keycheck_results_key_hash': ['key_hash'],
'idx_keycheck_results_secret_hash': ['secret_hash'],
'idx_keycheck_results_link_repair': ['link_status', 'id'],
},
'findings': {
'idx_findings_target_scan_id_id': ['target_scan_id', 'id'],
'idx_findings_cycle_id': ['cycle_id'],
'idx_findings_source': ['source'],
'idx_findings_secret_hash': ['secret_hash'],
'idx_findings_finding_uid': ['finding_uid'],
},
'errors': {
'idx_errors_cycle_id': ['cycle_id'],
'idx_errors_source_category': ['source', 'category'],
'idx_errors_target_scan_id': ['target_scan_id'],
},
'queue_snapshots': {
'idx_queue_snapshots_source_time': ['source', 'captured_at'],
},
'package_repo_candidates': {
'idx_package_repo_candidates_query': ['query'],
'idx_package_repo_candidates_repo': ['repo_url'],
'idx_package_repo_candidates_source_seen': ['package_source', 'last_seen_at'],
'idx_package_repo_candidates_query_seen': ['query', 'last_seen_at', 'id'],
'idx_package_repo_candidates_recent_lookup': ['last_seen_at', 'id', 'package_source', 'query', 'package_name'],
},
'target_queue_reconciliation_issues': {
'idx_reconciliation_issues_open': ['source_file', 'resolved_at', 'id'],
},
'docker_content_blobs': {
'idx_docker_content_blobs_reclaim': [
'state', 'available_after', 'lease_expires_at', 'digest',
],
'idx_docker_content_blobs_reservation': ['lease_reservation_id', 'digest'],
},
'docker_image_blob_coverage': {
'idx_docker_image_blob_coverage_manifest': [
'manifest_digest', 'coverage_state', 'position',
],
'idx_docker_image_blob_coverage_blob': [
'blob_digest', 'coverage_state', 'queue_id',
],
'idx_docker_image_blob_coverage_reservation': ['reservation_id', 'position'],
'idx_docker_image_blob_coverage_selection': [
'queue_id', 'manifest_digest', 'selection_policy_sha256',
'position', 'reservation_id',
],
},
'docker_adaptive_shadow_reports': {
'idx_docker_adaptive_shadow_reports_gate': [
'scan_policy_sha256', 'execution_policy_sha256',
'selection_policy_sha256', 'state', 'completed_at', 'id',
],
},
'runtime_operations': {
'idx_runtime_operations_status_updated': [
'status', 'updated_at', 'operation_id',
],
'idx_runtime_operations_agent_state_updated': [
'agent_state', 'updated_at', 'operation_id',
],
},
'runtime_audit_events': {
'idx_runtime_audit_events_created': ['created_at', 'id'],
'idx_runtime_audit_events_operation': ['operation_id', 'id'],
},
'worker_progress_events': {
'idx_worker_progress_reservation_received': [
'reservation_id', 'received_at', 'id',
],
'idx_worker_progress_device_received': [
'remote_device_id', 'received_at', 'id',
],
'idx_worker_progress_phase_received': ['phase', 'received_at', 'id'],
},
'result_reservations': {
'idx_result_reservations_remote_resolved': [
'remote_resolved_at', 'id',
],
},
'worker_diagnostics': {
'idx_worker_diagnostics_reservation_received': [
'reservation_id', 'received_at', 'id',
],
'idx_worker_diagnostics_scan_received': [
'target_scan_id', 'received_at', 'id',
],
'idx_worker_diagnostics_phase_received': ['phase', 'received_at', 'id'],
'idx_worker_diagnostics_category_code': [
'category', 'code', 'received_at', 'id',
],
'idx_worker_diagnostics_code_received': ['code', 'received_at', 'id'],
'idx_worker_diagnostics_kind_received': ['kind', 'received_at', 'id'],
'idx_worker_diagnostics_retryable_received': [
'retryable', 'received_at', 'id',
],
},
}
for table, expected_indexes in required_indexes.items():
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
for name, columns in expected_indexes.items():
index = indexes.get(name)
if not index or index['columns'] != columns or index['unique'] or not _index_usable(index):
missing.append(f'index {name}')
control = self.conn.execute(
'SELECT id FROM runtime_operations_control WHERE id = 1'
).fetchone() if self.conn.table_exists('runtime_operations_control') else None
if not control:
missing.append('runtime_operations_control singleton row')
missing.extend(_runtime_audit_trigger_problems(self.conn))
experiment_indexes = {}
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
indexes = experiment_indexes.setdefault(
table,
self.conn.table_indexes(table) if self.conn.table_exists(table) else {},
)
index = indexes.get(name)
if (
not index
or index['columns'] != list(columns)
or bool(index['unique']) != bool(unique)
or _normalized_predicate(index['predicate']) != _normalized_predicate(predicate)
or not _index_usable(index)
):
missing.append(f'index {name}')
queue_indexes = self.conn.table_indexes('target_queue') if self.conn.table_exists('target_queue') else {}
for name, predicate in (
('idx_target_queue_claim_pending', "status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
('idx_target_queue_claim_deferred', "status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
('idx_target_queue_claim_in_progress', "status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
('idx_target_queue_active_lease_owner_token', "status = 'in_progress'"),
('idx_target_queue_exhausted_attempts', "status = 'pending' OR status = 'deferred' OR status = 'in_progress'"),
('idx_target_queue_updated_rescan', "status = 'done' AND remote_modified_at IS NOT NULL"),
('idx_target_queue_cold', "status = 'cold'"),
):
if _normalized_predicate((queue_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(predicate):
missing.append(f'predicate {name}')
for name, predicate in (
('idx_discovery_retry_queue_due', DISCOVERY_RETRY_DUE_INDEX_PREDICATE),
('idx_discovery_retry_queue_lease', DISCOVERY_RETRY_LEASE_INDEX_PREDICATE),
):
if _normalized_predicate((retry_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(predicate):
missing.append(f'predicate {name}')
cooldown_index = scan_indexes.get('idx_target_scans_cooldown_recent')
if _normalized_predicate((cooldown_index or {}).get('predicate')) != _normalized_predicate(CI_COOLDOWN_INDEX_PREDICATE):
missing.append('predicate idx_target_scans_cooldown_recent')
for name in (
'idx_package_repo_candidates_query_seen',
'idx_package_repo_candidates_recent_lookup',
):
if _normalized_predicate((package_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(PACKAGE_REPO_NONEMPTY_PREDICATE):
missing.append(f'predicate {name}')
missing.extend(_docker_depth_check_constraint_problems(self.conn))
for table, expected_keys in REQUIRED_FOREIGN_KEYS.items():
if not self.conn.table_exists(table):
continue
foreign_keys = self.conn.table_foreign_keys(table)
for columns, referenced_table, referenced_columns in expected_keys:
authority_matching = [
foreign_key for foreign_key in foreign_keys.values()
if tuple(foreign_key.get('columns') or ()) == columns
and foreign_key.get('referenced_table') == referenced_table
and tuple(foreign_key.get('referenced_columns') or ()) == referenced_columns
and foreign_key.get('referenced_schema') in (
self.conn.application_schema if self.conn.is_postgres else 'main',
)
]
expected_actions = _required_foreign_key_actions(table, columns)
matching = [
foreign_key for foreign_key in authority_matching
if _foreign_key_actions_match(foreign_key, expected_actions)
]
if len(matching) != 1 or not matching[0].get('valid', True):
detail = (
f'foreign key {table}({", ".join(columns)}) -> '
f'{referenced_table}({", ".join(referenced_columns)})'
)
if len(authority_matching) == 1 and not _foreign_key_actions_match(
authority_matching[0], expected_actions,
):
detail += (
f' actions ON UPDATE {expected_actions[0]} '
f'ON DELETE {expected_actions[1]}'
)
missing.append(detail)
if self.conn.is_postgres and commit:
self.conn.commit()
if missing:
self.last_error = 'runtime safety schema is incomplete: ' + '; '.join(missing)
return False
self.last_error = ''
return True
except Exception as exc:
self.last_error = f'unable to validate runtime safety schema: {exc}'
try:
self.conn.rollback()
except Exception:
pass
return False
def require_runtime_safety_schema(self, commit=True):
if not self.runtime_safety_schema_available(commit=commit):
detail = self.last_error or 'runtime safety schema is unavailable'
raise RuntimeSafetySchemaError(f'{detail}; run migrate_runtime_safety.py offline with --apply')
if not self.pipeline_schema_available(commit=commit):
detail = self.last_error or 'pipeline schema is unavailable'
raise RuntimeSafetySchemaError(f'{detail}; run migrate_runtime_safety.py offline with --apply')
return True
def _locked_runtime_control_state(self, shared=False):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
suffix = ''
if self.conn.is_postgres:
suffix = ' FOR SHARE' if shared else ' FOR UPDATE'
row = self.conn.execute(
'SELECT * FROM runtime_operations_control WHERE id = 1' + suffix
).fetchone()
return _runtime_control_state_from_row(row)
def _require_discovery_admission_locked(self):
state = self._locked_runtime_control_state(shared=True)
if state['effective_discovery_paused']:
raise DiscoveryPausedError(state)
return state
def runtime_control_state(self):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
try:
row = self.conn.execute(
'SELECT * FROM runtime_operations_control WHERE id = 1'
).fetchone()
state = _runtime_control_state_from_row(row)
self.conn.commit()
return state
except Exception:
self.conn.rollback()
raise
def _locked_runtime_operation(self, operation_id):
suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
return self.conn.execute(
'SELECT * FROM runtime_operations WHERE operation_id = ?' + suffix,
(operation_id,),
).fetchone()
@staticmethod
def _runtime_operation_state(row):
if not row:
return None
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
actor = _runtime_actor(str(row['actor'] or ''))
action = str(row['action'] or '')
target_kind = str(row['target_kind'] or '')
target_ref = str(row['target_ref'] or '')
status = str(row['status'] or '')
agent_state = str(row['agent_state'] or '')
if action in RUNTIME_ASYNC_ACTIONS:
if (
target_kind != RUNTIME_ASYNC_TARGET_KIND
or target_ref != RUNTIME_ASYNC_ACTION_TARGETS[action]
):
raise RuntimeSafetySchemaError('runtime operation target is invalid')
try:
expected_raw = json.loads(str(row['expected_identity_json'] or ''))
expected = _runtime_expected_identity(expected_raw, action)
expected_json = _canonical_runtime_json(expected)
resulting_json = row['resulting_identity_json']
resulting = None
if resulting_json is not None:
resulting = _runtime_resulting_identity(json.loads(str(resulting_json)))
if _canonical_runtime_json(resulting) != str(resulting_json):
raise ValueError('noncanonical')
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeSafetySchemaError('runtime operation identity is invalid') from exc
if expected_json != str(row['expected_identity_json'] or ''):
raise RuntimeSafetySchemaError('runtime operation identity is not canonical')
elif action in RUNTIME_CONTROL_ACTIONS:
expected = _stored_runtime_control_identity(str(row['expected_identity_json'] or ''))
resulting = None
if row['resulting_identity_json'] is not None:
resulting = _stored_runtime_control_identity(
str(row['resulting_identity_json']),
)
elif action in RUNTIME_SOURCE_OPERATION_ACTIONS:
if target_kind != RUNTIME_SOURCE_TARGET_KIND or _runtime_source_id(target_ref) is None:
raise RuntimeSafetySchemaError('runtime source operation target is invalid')
expected = _stored_runtime_source_identity(
str(row['expected_identity_json'] or ''), action, target_ref,
)
resulting = None
if row['resulting_identity_json'] is not None:
resulting = _stored_runtime_source_identity(
str(row['resulting_identity_json']), action, target_ref,
resulting=True,
)
elif action in RUNTIME_WORKER_ADMIN_ACTION_TARGETS:
if target_kind != RUNTIME_WORKER_ADMIN_TARGET_KIND:
raise RuntimeSafetySchemaError(
'runtime worker admin operation target is invalid'
)
expected = _stored_runtime_worker_admin_identity(
str(row['expected_identity_json'] or ''), action, target_ref,
)
resulting = None
if row['resulting_identity_json'] is not None:
resulting = _stored_runtime_worker_admin_identity(
str(row['resulting_identity_json']), action, target_ref,
resulting=True,
)
elif action in RUNTIME_DOCUMENT_ACTION_TARGETS:
if target_kind != RUNTIME_DOCUMENT_TARGET_KIND:
raise RuntimeSafetySchemaError(
'runtime document operation target is invalid'
)
expected = _stored_runtime_document_identity(
str(row['expected_identity_json'] or ''), action, target_ref,
)
resulting = None
if row['resulting_identity_json'] is not None:
resulting = _stored_runtime_document_identity(
str(row['resulting_identity_json']), action, target_ref,
resulting=True,
)
elif action in RUNTIME_MANAGED_FILE_ACTIONS:
if target_kind != RUNTIME_MANAGED_FILE_TARGET_KIND:
raise RuntimeSafetySchemaError(
'runtime managed file operation target is invalid'
)
expected = _stored_runtime_managed_file_identity(
str(row['expected_identity_json'] or ''), action, target_ref,
)
resulting = None
if row['resulting_identity_json'] is not None:
resulting = _stored_runtime_managed_file_identity(
str(row['resulting_identity_json']), action, target_ref,
resulting=True,
)
else:
raise RuntimeSafetySchemaError('runtime operation action is invalid')
if status not in (
'requested', 'running', 'succeeded', 'failed', 'rolled_back',
'failed_hold', 'canceled',
):
raise RuntimeSafetySchemaError('runtime operation status is invalid')
if agent_state not in (
'not_required', 'pending', 'running', 'succeeded', 'failed',
'rolled_back', 'failed_hold',
):
raise RuntimeSafetySchemaError('runtime operation agent state is invalid')
category = row['safe_category']
detail = row['safe_detail']
try:
category = _runtime_safe_code(category, 'safe category')
detail = _runtime_safe_code(detail, 'safe detail')
except ValueError as exc:
raise RuntimeSafetySchemaError('runtime operation safe result is invalid') from exc
result_sha256 = row['agent_result_sha256']
if result_sha256 is not None:
try:
result_sha256 = _runtime_sha256(str(result_sha256), 'agent result hash')
except ValueError as exc:
raise RuntimeSafetySchemaError('runtime operation result hash is invalid') from exc
return {
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': target_kind,
'target_ref': target_ref,
'status': status,
'safe_category': category,
'safe_detail': detail,
'expected_revision': row['expected_revision'],
'resulting_revision': row['resulting_revision'],
'expected_identity': expected,
'resulting_identity': resulting,
'agent_state': agent_state,
'agent_result_sha256': result_sha256,
'requested_at': str(row['requested_at'] or ''),
'started_at': str(row['started_at']) if row['started_at'] is not None else None,
'completed_at': str(row['completed_at']) if row['completed_at'] is not None else None,
'agent_reconciled_at': (
str(row['agent_reconciled_at'])
if row['agent_reconciled_at'] is not None else None
),
'updated_at': str(row['updated_at'] or ''),
}
def _append_runtime_audit_event_locked(
self, *, operation_id, actor, action, target_kind, target_ref,
result, safe_category=None, before_identity=None, after_identity=None,
before_bytes=None, after_bytes=None, created_at=None,
):
if result not in (
'accepted', 'succeeded', 'rejected', 'failed', 'rolled_back',
'canceled', 'failed_hold',
):
raise ValueError('runtime audit result is invalid')
before_json = (
_canonical_runtime_json(before_identity) if before_identity is not None else None
)
after_json = (
_canonical_runtime_json(after_identity) if after_identity is not None else None
)
for value in (before_bytes, after_bytes):
if value is not None and (
isinstance(value, bool) or not isinstance(value, int) or value < 0
):
raise ValueError('runtime audit byte count is invalid')
tail = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
ORDER BY id DESC LIMIT 1'''
).fetchone()
previous_event_id = int(tail['id']) if tail else None
previous_event_sha256 = str(tail['event_sha256']) if tail else None
now = created_at or utc_now_iso()
payload = {
'schema': 'runtime-audit-event-v1',
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': target_kind,
'target_ref': target_ref,
'result': result,
'safe_category': safe_category,
'before_identity_json': before_json,
'after_identity_json': after_json,
'before_bytes': before_bytes,
'after_bytes': after_bytes,
'previous_event_id': previous_event_id,
'previous_event_sha256': previous_event_sha256,
'created_at': now,
}
event_sha256 = _runtime_audit_event_sha256(payload)
event_id = self.conn.insert_returning_id(
'''INSERT INTO runtime_audit_events(
operation_id, actor, action, target_kind, target_ref,
result, safe_category, before_identity_json,
after_identity_json, before_bytes, after_bytes,
previous_event_id, previous_event_sha256, event_sha256,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
operation_id, actor, action, target_kind, target_ref, result,
safe_category, before_json, after_json, before_bytes, after_bytes,
previous_event_id, previous_event_sha256, event_sha256, now,
),
)
if event_id is None:
raise RuntimeSafetySchemaError('runtime audit insertion failed')
return {
'audit_event_id': int(event_id),
'audit_event_sha256': event_sha256,
}
def runtime_operation(self, operation_id):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
try:
row = self.conn.execute(
'SELECT * FROM runtime_operations WHERE operation_id = ?',
(operation_id,),
).fetchone()
result = self._runtime_operation_state(row)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def recent_runtime_operations(
self, limit=200, *, before_updated_at=None, before_operation_id=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
if type(limit) is not int or not 1 <= limit <= 500:
raise ValueError('runtime operation snapshot limit is out of range')
if (before_updated_at is None) is not (before_operation_id is None):
raise ValueError('runtime operation cursor is incomplete')
if before_updated_at is not None:
if type(before_updated_at) is not str or not before_updated_at:
raise ValueError('runtime operation cursor timestamp is invalid')
before_operation_id = _runtime_operation_id(before_operation_id)
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN')
else:
self.conn.execute(
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY'
)
rows = []
for status in (
'requested', 'running', 'succeeded', 'failed', 'rolled_back',
'failed_hold', 'canceled',
):
if before_updated_at is None:
sql = '''SELECT * FROM runtime_operations WHERE status = ?
ORDER BY updated_at DESC, operation_id DESC LIMIT ?'''
parameters = (status, limit)
else:
sql = '''SELECT * FROM runtime_operations WHERE status = ?
AND (
updated_at < ? OR (
updated_at = ? AND operation_id < ?
)
)
ORDER BY updated_at DESC, operation_id DESC LIMIT ?'''
parameters = (
status, before_updated_at, before_updated_at,
before_operation_id, limit,
)
rows.extend(self.conn.execute(sql, parameters).fetchall())
operations = {}
for row in rows:
operation = self._runtime_operation_state(row)
operations[operation['operation_id']] = operation
result = sorted(
operations.values(),
key=lambda item: (item['updated_at'], item['operation_id']),
reverse=True,
)[:limit]
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def pending_runtime_agent_operations(self, limit=32):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
if type(limit) is not int or not 1 <= limit <= 128:
raise ValueError('runtime agent operation limit is out of range')
try:
rows = self.conn.execute(
'''SELECT * FROM runtime_operations
WHERE target_kind = ?
AND status IN ('requested', 'running')
AND agent_state IN ('pending', 'running')
ORDER BY CASE status WHEN 'running' THEN 0 ELSE 1 END,
updated_at ASC, operation_id ASC
LIMIT ?''',
(RUNTIME_ASYNC_TARGET_KIND, limit),
).fetchall()
result = [self._runtime_operation_state(row) for row in rows]
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def runtime_audit_events(self, before_event_id=None, limit=50):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
if (
before_event_id is not None
and (
type(before_event_id) is not int
or not 1 <= before_event_id <= RUNTIME_CONTROL_MAX_REVISION
)
):
raise ValueError('runtime audit cursor is out of range')
if type(limit) is not int or not 1 <= limit <= 200:
raise ValueError('runtime audit page limit is out of range')
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN')
else:
self.conn.execute(
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY'
)
parameters = []
# The console exposes only events whose identities can be checked
# against a durable typed operation.
clauses = ['operation_id IS NOT NULL']
if before_event_id is not None:
clauses.append('id < ?')
parameters.append(before_event_id)
parameters.append(limit + 1)
rows = self.conn.execute(
'SELECT * FROM runtime_audit_events WHERE ' + ' AND '.join(clauses)
+ ' ORDER BY id DESC LIMIT ?',
tuple(parameters),
).fetchall()
page_rows = rows[:limit]
operation_ids = []
for row in page_rows:
try:
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeSafetySchemaError(
'runtime audit operation identity is invalid'
) from exc
if operation_id not in operation_ids:
operation_ids.append(operation_id)
operations = {}
if operation_ids:
placeholders = ','.join('?' for _value in operation_ids)
operation_rows = self.conn.execute(
'SELECT * FROM runtime_operations WHERE operation_id IN ('
+ placeholders + ')',
tuple(operation_ids),
).fetchall()
for row in operation_rows:
operation = self._runtime_operation_state(row)
operations[operation['operation_id']] = operation
events = []
allowed_results = {
'accepted', 'succeeded', 'rejected', 'failed', 'rolled_back',
'canceled', 'failed_hold',
}
for row in page_rows:
try:
event_id = row['id']
if type(event_id) is not int or event_id < 1:
raise ValueError('event ID')
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
operation = operations.get(operation_id)
if operation is None:
raise ValueError('operation')
actor = _runtime_actor(str(row['actor'] or ''))
action = str(row['action'] or '')
target_kind = str(row['target_kind'] or '')
target_ref = str(row['target_ref'] or '')
result = str(row['result'] or '')
if (
actor != operation['actor']
or action != operation['action']
or target_kind != operation['target_kind']
or target_ref != operation['target_ref']
or result not in allowed_results
):
raise ValueError('event identity')
safe_category = _runtime_safe_code(
row['safe_category'], 'safe category',
)
identities = []
identity_json = []
for field in ('before_identity_json', 'after_identity_json'):
raw = row[field]
if raw is None:
identities.append(None)
identity_json.append(None)
continue
raw = str(raw)
parsed = json.loads(raw)
if not isinstance(parsed, dict) or _canonical_runtime_json(parsed) != raw:
raise ValueError('event identity JSON')
identities.append(parsed)
identity_json.append(raw)
before_identity, after_identity = identities
if before_identity != operation['expected_identity'] or (
after_identity is not None
and after_identity != operation['resulting_identity']
):
raise ValueError('event operation identity')
byte_counts = []
for field in ('before_bytes', 'after_bytes'):
value = row[field]
if value is not None and (
type(value) is not int
or not 0 <= value <= RUNTIME_CONTROL_MAX_REVISION
):
raise ValueError('event byte count')
byte_counts.append(value)
before_bytes, after_bytes = byte_counts
previous_event_id = row['previous_event_id']
previous_event_sha256 = row['previous_event_sha256']
if previous_event_id is None:
if previous_event_sha256 is not None:
raise ValueError('event parent')
elif (
type(previous_event_id) is not int
or previous_event_id < 1
or previous_event_id >= event_id
):
raise ValueError('event parent')
else:
previous_event_sha256 = _runtime_sha256(
str(previous_event_sha256 or ''), 'audit parent hash',
)
event_sha256 = _runtime_sha256(
str(row['event_sha256'] or ''), 'audit event hash',
)
created_at = str(row['created_at'] or '')
if not 1 <= len(created_at) <= 64:
raise ValueError('event time')
payload = {
'schema': 'runtime-audit-event-v1',
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': target_kind,
'target_ref': target_ref,
'result': result,
'safe_category': safe_category,
'before_identity_json': identity_json[0],
'after_identity_json': identity_json[1],
'before_bytes': before_bytes,
'after_bytes': after_bytes,
'previous_event_id': previous_event_id,
'previous_event_sha256': previous_event_sha256,
'created_at': created_at,
}
if not hmac.compare_digest(
_runtime_audit_event_sha256(payload), event_sha256,
):
raise ValueError('event hash')
except (
KeyError, TypeError, ValueError, json.JSONDecodeError,
) as exc:
raise RuntimeSafetySchemaError(
'runtime audit event is invalid'
) from exc
events.append({
'id': event_id,
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': target_kind,
'target_ref': target_ref,
'result': result,
'safe_category': safe_category,
'before_identity': before_identity,
'after_identity': after_identity,
'before_bytes': before_bytes,
'after_bytes': after_bytes,
'previous_event_id': previous_event_id,
'previous_event_sha256': previous_event_sha256,
'event_sha256': event_sha256,
'created_at': created_at,
})
next_before_event_id = (
events[-1]['id'] if len(rows) > limit and events else None
)
self.conn.commit()
return {
'events': events,
'next_before_event_id': next_before_event_id,
}
except Exception:
self.conn.rollback()
raise
def create_runtime_operation(self, *, operation_id, actor, action, expected_identity):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
if not isinstance(action, str) or action not in RUNTIME_ASYNC_ACTIONS:
raise ValueError('runtime operation action is invalid')
expected_identity = _runtime_expected_identity(expected_identity, action)
expected_json = _canonical_runtime_json(expected_identity)
target_ref = RUNTIME_ASYNC_ACTION_TARGETS[action]
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
existing_row = self._locked_runtime_operation(operation_id)
if existing_row:
existing = self._runtime_operation_state(existing_row)
if not (
existing['actor'] == actor
and existing['action'] == action
and existing['target_kind'] == RUNTIME_ASYNC_TARGET_KIND
and existing['target_ref'] == target_ref
and existing['expected_identity'] == expected_identity
):
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
accepted = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = 'accepted'
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(accepted) != 1:
raise RuntimeSafetySchemaError(
'runtime operation accepted audit evidence is incomplete'
)
self.conn.commit()
return {
**existing, 'replayed': True,
'audit_event_id': int(accepted[0]['id']),
'audit_event_sha256': str(accepted[0]['event_sha256']),
}
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_identity_json, agent_state,
requested_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'requested', ?, 'pending', ?, ?)''',
(
operation_id, actor, action, RUNTIME_ASYNC_TARGET_KIND,
target_ref, expected_json, now, now,
),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_ASYNC_TARGET_KIND, target_ref=target_ref,
result='accepted', before_identity=expected_identity,
created_at=now,
)
row = self._locked_runtime_operation(operation_id)
result = self._runtime_operation_state(row)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime operation creation retry budget exhausted')
def mark_runtime_operation_running(self, operation_id):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if current['action'] not in RUNTIME_ASYNC_ACTIONS:
raise RuntimeOperationTransitionError(
'runtime operation does not use agent lifecycle'
)
if current['status'] == 'running':
self.conn.commit()
return {**current, 'replayed': True}
if current['status'] != 'requested':
raise RuntimeOperationTransitionError(
'runtime operation cannot enter running state'
)
now = utc_now_iso()
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = 'running', agent_state = 'running',
started_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'requested'
AND agent_state = 'pending' ''',
(now, now, operation_id),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime operation running transition conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime operation transition retry budget exhausted')
def claim_runtime_operation_execution(self, *, operation_id, action, expected_identity):
"""Atomically bind a host-agent claim to persisted request identity."""
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if not isinstance(action, str) or action not in RUNTIME_ASYNC_ACTIONS:
raise ValueError('runtime operation action is invalid')
expected_identity = _runtime_expected_identity(expected_identity, action)
expected_json = _canonical_runtime_json(expected_identity)
target_ref = RUNTIME_ASYNC_ACTION_TARGETS[action]
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
raise RuntimeOperationTransitionError(
'runtime operation is unavailable'
)
current = self._runtime_operation_state(row)
if not (
current['action'] == action
and current['target_kind'] == RUNTIME_ASYNC_TARGET_KIND
and current['target_ref'] == target_ref
and current['expected_identity'] == expected_identity
):
raise RuntimeOperationIdentityConflictError(
'runtime operation identity does not match the persisted request'
)
if any((
current['expected_revision'] is not None,
current['resulting_identity'] is not None,
current['resulting_revision'] is not None,
current['completed_at'] is not None,
current['agent_reconciled_at'] is not None,
current['safe_category'] is not None,
current['safe_detail'] is not None,
current['agent_result_sha256'] is not None,
)):
raise RuntimeOperationTransitionError(
'runtime operation cannot be claimed for execution'
)
requested = (
current['status'] == 'requested'
and current['agent_state'] == 'pending'
and current['started_at'] is None
)
running = (
current['status'] == 'running'
and current['agent_state'] == 'running'
and current['started_at'] is not None
)
if not requested and not running:
raise RuntimeOperationTransitionError(
'runtime operation cannot be claimed for execution'
)
audits = self.conn.execute(
'''SELECT * FROM runtime_audit_events
WHERE operation_id = ? ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(audits) != 1:
raise RuntimeSafetySchemaError(
'runtime operation accepted audit evidence is incomplete'
)
audit = audits[0]
try:
audit_id = int(audit['id'])
before_json = str(audit['before_identity_json'] or '')
previous_event_id = audit['previous_event_id']
previous_event_sha256 = audit['previous_event_sha256']
if previous_event_id is None:
if previous_event_sha256 is not None:
raise ValueError('audit parent')
else:
if (
type(previous_event_id) is not int
or previous_event_id < 1
or previous_event_id >= audit_id
):
raise ValueError('audit parent')
previous_event_sha256 = _runtime_sha256(
str(previous_event_sha256 or ''), 'audit parent hash',
)
predecessor = self.conn.execute(
'''SELECT * FROM runtime_audit_events
WHERE id < ? ORDER BY id DESC LIMIT 1''',
(audit_id,),
).fetchone()
if predecessor is None:
if previous_event_id is not None:
raise ValueError('audit parent')
else:
predecessor_id, predecessor_sha256 = (
_runtime_audit_row_hash(predecessor)
)
if not (
predecessor_id == previous_event_id
and hmac.compare_digest(
predecessor_sha256,
previous_event_sha256 or '',
)
):
raise ValueError('audit parent')
event_sha256 = _runtime_sha256(
str(audit['event_sha256'] or ''), 'audit event hash',
)
if not (
audit_id >= 1
and str(audit['operation_id'] or '') == operation_id
and str(audit['actor'] or '') == current['actor']
and str(audit['action'] or '') == action
and str(audit['target_kind'] or '') == RUNTIME_ASYNC_TARGET_KIND
and str(audit['target_ref'] or '') == target_ref
and str(audit['result'] or '') == 'accepted'
and audit['safe_category'] is None
and before_json == expected_json
and audit['after_identity_json'] is None
and audit['before_bytes'] is None
and audit['after_bytes'] is None
and str(audit['created_at'] or '') == current['requested_at']
):
raise ValueError('accepted audit')
payload = {
'schema': 'runtime-audit-event-v1',
'operation_id': operation_id,
'actor': current['actor'],
'action': action,
'target_kind': RUNTIME_ASYNC_TARGET_KIND,
'target_ref': target_ref,
'result': 'accepted',
'safe_category': None,
'before_identity_json': expected_json,
'after_identity_json': None,
'before_bytes': None,
'after_bytes': None,
'previous_event_id': previous_event_id,
'previous_event_sha256': previous_event_sha256,
'created_at': current['requested_at'],
}
if not hmac.compare_digest(
_runtime_audit_event_sha256(payload), event_sha256,
):
raise ValueError('audit hash')
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeSafetySchemaError(
'runtime operation accepted audit evidence is invalid'
) from exc
if running:
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': audit_id,
'audit_event_sha256': event_sha256,
}
now = utc_now_iso()
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = 'running', agent_state = 'running',
started_at = ?, updated_at = ?
WHERE operation_id = ? AND action = ?
AND target_kind = ? AND target_ref = ?
AND expected_identity_json = ?
AND status = 'requested' AND agent_state = 'pending' ''',
(
now, now, operation_id, action, RUNTIME_ASYNC_TARGET_KIND,
target_ref, expected_json,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime operation claim conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {
**result, 'replayed': False,
'audit_event_id': audit_id,
'audit_event_sha256': event_sha256,
}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime operation claim retry budget exhausted')
def create_runtime_source_operation(
self, *, operation_id, actor, source_id, source_action,
interval_seconds=None, mode=None, restart_enabled=None,
restart_delay_seconds=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
if _runtime_source_id(source_id) is None or source_action not in RUNTIME_SOURCE_ACTIONS:
raise ValueError('runtime source operation request is invalid')
action = f'supervisor.source.{source_action}'
identity = {
'source_id': source_id,
'source_action': source_action,
'interval_seconds': interval_seconds,
}
if source_action in ('once', 'set-mode', 'set-restart', 'set-restart-delay'):
identity.update({
'mode': mode,
'restart_enabled': restart_enabled,
'restart_delay_seconds': restart_delay_seconds,
})
elif any(
value is not None
for value in (mode, restart_enabled, restart_delay_seconds)
):
raise ValueError('runtime source operation request is invalid')
expected = _runtime_source_expected_identity(identity, action, source_id)
expected_json = _canonical_runtime_json(expected)
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
existing_row = self._locked_runtime_operation(operation_id)
if existing_row:
existing = self._runtime_operation_state(existing_row)
if not (
existing['actor'] == actor
and existing['action'] == action
and existing['target_kind'] == RUNTIME_SOURCE_TARGET_KIND
and existing['target_ref'] == source_id
and existing['expected_identity'] == expected
):
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
accepted = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = 'accepted'
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(accepted) != 1:
raise RuntimeSafetySchemaError(
'runtime source operation accepted audit evidence is incomplete'
)
self.conn.commit()
return {
**existing, 'replayed': True,
'audit_event_id': int(accepted[0]['id']),
'audit_event_sha256': str(accepted[0]['event_sha256']),
}
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_identity_json, agent_state,
requested_at, started_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
(
operation_id, actor, action, RUNTIME_SOURCE_TARGET_KIND,
source_id, expected_json, now, now, now,
),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_SOURCE_TARGET_KIND, target_ref=source_id,
result='accepted', before_identity=expected, created_at=now,
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime source operation creation retry budget exhausted')
def complete_runtime_source_operation(
self, operation_id, *, succeeded, outcome=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if type(succeeded) is not bool:
raise ValueError('runtime source operation result is invalid')
if succeeded:
if outcome not in ('completed', 'dependency-blocked'):
raise ValueError('runtime source operation result is invalid')
elif outcome is not None:
raise ValueError('runtime source operation result is invalid')
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if current['action'] not in RUNTIME_SOURCE_OPERATION_ACTIONS:
raise RuntimeOperationTransitionError(
'runtime operation is not a managed source action'
)
status = 'succeeded' if succeeded else 'failed'
safe_category = None if succeeded else 'supervisor_action_failed'
resulting = None
if succeeded:
resulting = _runtime_source_resulting_identity({
'source_id': current['target_ref'],
'source_action': RUNTIME_SOURCE_OPERATION_ACTIONS[current['action']],
'outcome': outcome,
}, current['action'], current['target_ref'])
if current['status'] in ('succeeded', 'failed'):
if (
current['status'] != status
or current['safe_category'] != safe_category
or current['resulting_identity'] != resulting
):
raise RuntimeOperationIdentityConflictError(
'runtime source operation already has another result'
)
events = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = ?
ORDER BY id LIMIT 2''',
(operation_id, status),
).fetchall()
if len(events) != 1:
raise RuntimeSafetySchemaError(
'runtime source operation terminal audit evidence is incomplete'
)
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': int(events[0]['id']),
'audit_event_sha256': str(events[0]['event_sha256']),
}
if current['status'] != 'running' or current['agent_state'] != 'not_required':
raise RuntimeOperationTransitionError(
'runtime source operation cannot be completed'
)
now = utc_now_iso()
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=current['actor'],
action=current['action'], target_kind=RUNTIME_SOURCE_TARGET_KIND,
target_ref=current['target_ref'], result=status,
safe_category=safe_category,
before_identity=current['expected_identity'],
after_identity=resulting, created_at=now,
)
resulting_json = (
_canonical_runtime_json(resulting) if resulting is not None else None
)
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = ?, safe_category = ?, resulting_identity_json = ?,
completed_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'running'
AND agent_state = 'not_required' ''',
(
status, safe_category, resulting_json, now, now,
operation_id,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime source operation completion conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime source operation completion retry budget exhausted')
def create_runtime_worker_admin_operation(
self, *, operation_id, actor, action, target_ref, request_sha256,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
target_ref = _runtime_worker_admin_target(action, target_ref)
expected = _runtime_worker_admin_expected_identity({
'action': action,
'target_ref': target_ref,
'request_sha256': request_sha256,
}, action, target_ref)
expected_json = _canonical_runtime_json(expected)
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
existing_row = self._locked_runtime_operation(operation_id)
if existing_row:
existing = self._runtime_operation_state(existing_row)
if not (
existing['actor'] == actor
and existing['action'] == action
and existing['target_kind'] == RUNTIME_WORKER_ADMIN_TARGET_KIND
and existing['target_ref'] == target_ref
and existing['expected_identity'] == expected
):
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
accepted = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = 'accepted'
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(accepted) != 1:
raise RuntimeSafetySchemaError(
'runtime worker admin accepted audit evidence is incomplete'
)
self.conn.commit()
return {
**existing, 'replayed': True,
'audit_event_id': int(accepted[0]['id']),
'audit_event_sha256': str(accepted[0]['event_sha256']),
}
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_identity_json, agent_state,
requested_at, started_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
(
operation_id, actor, action, RUNTIME_WORKER_ADMIN_TARGET_KIND,
target_ref, expected_json, now, now, now,
),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_WORKER_ADMIN_TARGET_KIND,
target_ref=target_ref, result='accepted',
before_identity=expected, created_at=now,
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime worker admin operation creation retry budget exhausted'
)
def complete_runtime_worker_admin_operation(
self, operation_id, *, succeeded, affected_count=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if type(succeeded) is not bool:
raise ValueError('runtime worker admin operation result is invalid')
if succeeded:
if type(affected_count) is not int or not 0 <= affected_count <= 10000:
raise ValueError('runtime worker admin operation result is invalid')
elif affected_count is not None:
raise ValueError('runtime worker admin operation result is invalid')
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if current['action'] not in RUNTIME_WORKER_ADMIN_ACTION_TARGETS:
raise RuntimeOperationTransitionError(
'runtime operation is not a worker admin action'
)
status = 'succeeded' if succeeded else 'failed'
safe_category = None if succeeded else 'worker_admin_mutation_failed'
resulting = None
if succeeded:
resulting = _runtime_worker_admin_resulting_identity({
'action': current['action'],
'target_ref': current['target_ref'],
'outcome': 'completed',
'affected_count': affected_count,
}, current['action'], current['target_ref'])
if current['status'] in ('succeeded', 'failed'):
if (
current['status'] != status
or current['safe_category'] != safe_category
or current['resulting_identity'] != resulting
):
raise RuntimeOperationIdentityConflictError(
'runtime worker admin operation already has another result'
)
events = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = ?
ORDER BY id LIMIT 2''',
(operation_id, status),
).fetchall()
if len(events) != 1:
raise RuntimeSafetySchemaError(
'runtime worker admin terminal audit evidence is incomplete'
)
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': int(events[0]['id']),
'audit_event_sha256': str(events[0]['event_sha256']),
}
if current['status'] != 'running' or current['agent_state'] != 'not_required':
raise RuntimeOperationTransitionError(
'runtime worker admin operation cannot be completed'
)
now = utc_now_iso()
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=current['actor'],
action=current['action'],
target_kind=RUNTIME_WORKER_ADMIN_TARGET_KIND,
target_ref=current['target_ref'], result=status,
safe_category=safe_category,
before_identity=current['expected_identity'],
after_identity=resulting, created_at=now,
)
resulting_json = (
_canonical_runtime_json(resulting) if resulting is not None else None
)
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = ?, safe_category = ?, resulting_identity_json = ?,
completed_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'running'
AND agent_state = 'not_required' ''',
(
status, safe_category, resulting_json, now, now,
operation_id,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime worker admin operation completion conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime worker admin operation completion retry budget exhausted'
)
def create_runtime_managed_file_operation(
self, *, operation_id, actor, action, root_id, relative_path,
expected_sha256, proposed_sha256, proposed_byte_count,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
if action not in RUNTIME_MANAGED_FILE_ACTIONS:
raise ValueError('runtime managed file operation action is invalid')
expected = _runtime_managed_file_expected_identity({
'root_id': root_id,
'relative_path': relative_path,
'expected_sha256': expected_sha256,
'proposed_sha256': proposed_sha256,
'proposed_byte_count': proposed_byte_count,
}, action, root_id)
expected_json = _canonical_runtime_json(expected)
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
existing_row = self._locked_runtime_operation(operation_id)
if existing_row:
existing = self._runtime_operation_state(existing_row)
if not (
existing['actor'] == actor
and existing['action'] == action
and existing['target_kind'] == RUNTIME_MANAGED_FILE_TARGET_KIND
and existing['target_ref'] == root_id
and existing['expected_identity'] == expected
):
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
accepted = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = 'accepted'
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(accepted) != 1:
raise RuntimeSafetySchemaError(
'runtime managed file accepted audit evidence is incomplete'
)
self.conn.commit()
return {
**existing, 'replayed': True,
'audit_event_id': int(accepted[0]['id']),
'audit_event_sha256': str(accepted[0]['event_sha256']),
}
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_identity_json, agent_state,
requested_at, started_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
(
operation_id, actor, action, RUNTIME_MANAGED_FILE_TARGET_KIND,
root_id, expected_json, now, now, now,
),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_MANAGED_FILE_TARGET_KIND,
target_ref=root_id, result='accepted',
before_identity=expected,
after_bytes=expected['proposed_byte_count'], created_at=now,
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime managed file operation creation retry budget exhausted'
)
def complete_runtime_managed_file_operation(
self, operation_id, *, succeeded, before_sha256=None,
before_byte_count=None, after_sha256=None, after_byte_count=None,
written=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if type(succeeded) is not bool:
raise ValueError('runtime managed file operation result is invalid')
supplied = (
before_sha256, before_byte_count, after_sha256, after_byte_count, written,
)
if not succeeded and any(value is not None for value in supplied):
raise ValueError('runtime managed file operation result is invalid')
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if current['action'] not in RUNTIME_MANAGED_FILE_ACTIONS:
raise RuntimeOperationTransitionError(
'runtime operation is not a managed file action'
)
status = 'succeeded' if succeeded else 'failed'
safe_category = None if succeeded else 'managed_file_mutation_failed'
resulting = None
if succeeded:
resulting = _runtime_managed_file_resulting_identity({
'root_id': current['target_ref'],
'relative_path': current['expected_identity']['relative_path'],
'outcome': 'completed',
'before_sha256': before_sha256,
'before_byte_count': before_byte_count,
'after_sha256': after_sha256,
'after_byte_count': after_byte_count,
'written': written,
}, current['action'], current['target_ref'])
expected = current['expected_identity']
if (
current['action'] == 'files.create'
and (
resulting['after_sha256'] != expected['proposed_sha256']
or resulting['after_byte_count'] != expected['proposed_byte_count']
)
or current['action'] == 'files.replace'
and (
resulting['before_sha256'] != expected['expected_sha256']
or resulting['after_sha256'] != expected['proposed_sha256']
or resulting['after_byte_count'] != expected['proposed_byte_count']
or not resulting['written'] and (
resulting['before_sha256'] != resulting['after_sha256']
or resulting['before_byte_count'] != resulting['after_byte_count']
)
)
or current['action'] == 'files.delete'
and resulting['before_sha256'] != expected['expected_sha256']
):
raise RuntimeOperationIdentityConflictError(
'runtime managed file result does not match its accepted identity'
)
if current['status'] in ('succeeded', 'failed'):
if (
current['status'] != status
or current['safe_category'] != safe_category
or current['resulting_identity'] != resulting
):
raise RuntimeOperationIdentityConflictError(
'runtime managed file operation already has another result'
)
events = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = ?
ORDER BY id LIMIT 2''',
(operation_id, status),
).fetchall()
if len(events) != 1:
raise RuntimeSafetySchemaError(
'runtime managed file terminal audit evidence is incomplete'
)
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': int(events[0]['id']),
'audit_event_sha256': str(events[0]['event_sha256']),
}
if current['status'] != 'running' or current['agent_state'] != 'not_required':
raise RuntimeOperationTransitionError(
'runtime managed file operation cannot be completed'
)
now = utc_now_iso()
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=current['actor'],
action=current['action'],
target_kind=RUNTIME_MANAGED_FILE_TARGET_KIND,
target_ref=current['target_ref'], result=status,
safe_category=safe_category,
before_identity=current['expected_identity'],
after_identity=resulting,
before_bytes=(
resulting['before_byte_count'] if resulting is not None else None
),
after_bytes=(
resulting['after_byte_count'] if resulting is not None else None
),
created_at=now,
)
resulting_json = (
_canonical_runtime_json(resulting) if resulting is not None else None
)
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = ?, safe_category = ?, resulting_identity_json = ?,
completed_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'running'
AND agent_state = 'not_required' ''',
(
status, safe_category, resulting_json, now, now,
operation_id,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime managed file operation completion conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime managed file operation completion retry budget exhausted'
)
def create_runtime_document_operation(
self, *, operation_id, actor, action, active_config_sha256,
active_secrets_sha256, candidate_config_sha256, candidate_secrets_sha256,
candidate_after_sha256, candidate_before_bytes, candidate_after_bytes,
candidate_before_present,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
target_ref = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
if target_ref is None:
raise ValueError('runtime document operation action is invalid')
expected = _runtime_document_expected_identity({
'document': target_ref,
'active_config_sha256': active_config_sha256,
'active_secrets_sha256': active_secrets_sha256,
'candidate_config_sha256': candidate_config_sha256,
'candidate_secrets_sha256': candidate_secrets_sha256,
'candidate_after_sha256': candidate_after_sha256,
'candidate_before_bytes': candidate_before_bytes,
'candidate_after_bytes': candidate_after_bytes,
'candidate_before_present': candidate_before_present,
}, action, target_ref)
expected_json = _canonical_runtime_json(expected)
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
existing_row = self._locked_runtime_operation(operation_id)
if existing_row:
existing = self._runtime_operation_state(existing_row)
if not (
existing['actor'] == actor
and existing['action'] == action
and existing['target_kind'] == RUNTIME_DOCUMENT_TARGET_KIND
and existing['target_ref'] == target_ref
and existing['expected_identity'] == expected
):
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
accepted = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = 'accepted'
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(accepted) != 1:
raise RuntimeSafetySchemaError(
'runtime document accepted audit evidence is incomplete'
)
self.conn.commit()
return {
**existing, 'replayed': True,
'audit_event_id': int(accepted[0]['id']),
'audit_event_sha256': str(accepted[0]['event_sha256']),
}
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_identity_json, agent_state,
requested_at, started_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
(
operation_id, actor, action, RUNTIME_DOCUMENT_TARGET_KIND,
target_ref, expected_json, now, now, now,
),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_DOCUMENT_TARGET_KIND,
target_ref=target_ref, result='accepted',
before_identity=expected,
before_bytes=expected['candidate_before_bytes'],
after_bytes=expected['candidate_after_bytes'], created_at=now,
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime document operation creation retry budget exhausted'
)
def complete_runtime_document_operation(
self, operation_id, *, succeeded, candidate_sha256=None, written=None,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
if type(succeeded) is not bool:
raise ValueError('runtime document operation result is invalid')
if succeeded:
if type(written) is not bool:
raise ValueError('runtime document operation result is invalid')
candidate_sha256 = _runtime_sha256(
candidate_sha256, 'candidate result hash',
)
elif candidate_sha256 is not None or written is not None:
raise ValueError('runtime document operation result is invalid')
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if current['action'] not in RUNTIME_DOCUMENT_ACTION_TARGETS:
raise RuntimeOperationTransitionError(
'runtime operation is not a document action'
)
if succeeded and not hmac.compare_digest(
candidate_sha256,
current['expected_identity']['candidate_after_sha256'],
):
raise RuntimeOperationIdentityConflictError(
'runtime document result does not match its accepted candidate'
)
status = 'succeeded' if succeeded else 'failed'
safe_category = None if succeeded else 'runtime_document_save_failed'
resulting = None
if succeeded:
resulting = _runtime_document_resulting_identity({
'document': current['target_ref'],
'outcome': 'completed',
'candidate_sha256': candidate_sha256,
'written': written,
}, current['action'], current['target_ref'])
if current['status'] in ('succeeded', 'failed'):
if (
current['status'] != status
or current['safe_category'] != safe_category
or current['resulting_identity'] != resulting
):
raise RuntimeOperationIdentityConflictError(
'runtime document operation already has another result'
)
events = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = ?
ORDER BY id LIMIT 2''',
(operation_id, status),
).fetchall()
if len(events) != 1:
raise RuntimeSafetySchemaError(
'runtime document terminal audit evidence is incomplete'
)
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': int(events[0]['id']),
'audit_event_sha256': str(events[0]['event_sha256']),
}
if current['status'] != 'running' or current['agent_state'] != 'not_required':
raise RuntimeOperationTransitionError(
'runtime document operation cannot be completed'
)
now = utc_now_iso()
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=current['actor'],
action=current['action'], target_kind=RUNTIME_DOCUMENT_TARGET_KIND,
target_ref=current['target_ref'], result=status,
safe_category=safe_category,
before_identity=current['expected_identity'],
after_identity=resulting,
before_bytes=current['expected_identity']['candidate_before_bytes'],
after_bytes=current['expected_identity']['candidate_after_bytes'],
created_at=now,
)
resulting_json = (
_canonical_runtime_json(resulting) if resulting is not None else None
)
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = ?, safe_category = ?, resulting_identity_json = ?,
completed_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'running'
AND agent_state = 'not_required' ''',
(
status, safe_category, resulting_json, now, now,
operation_id,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime document operation completion conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError(
'runtime document operation completion retry budget exhausted'
)
def reconcile_runtime_operation_result(
self, result_envelope, *, max_bytes=RUNTIME_AGENT_RESULT_MAX_BYTES,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
parsed, result_sha256, failure = _runtime_result_envelope(
result_envelope, max_bytes,
)
result_envelope = None
if failure:
raise ValueError(failure)
operation_id = parsed['operation_id']
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._locked_runtime_control_state()
row = self._locked_runtime_operation(operation_id)
if not row:
self.conn.commit()
return None
current = self._runtime_operation_state(row)
if (
current['action'] != parsed['action']
or current['action'] not in RUNTIME_ASYNC_ACTIONS
):
raise RuntimeOperationIdentityConflictError(
'runtime operation result does not match its request'
)
existing_digest = current['agent_result_sha256']
if existing_digest is not None:
if not hmac.compare_digest(existing_digest, result_sha256):
raise RuntimeOperationIdentityConflictError(
'runtime operation already has another result'
)
if (
current['status'] != parsed['result']
or current['agent_state'] != parsed['result']
or current['safe_category'] != parsed['safe_category']
or current['safe_detail'] != parsed['safe_detail']
or current['resulting_identity'] != parsed['resulting_identity']
):
raise RuntimeSafetySchemaError(
'runtime operation replay result is inconsistent'
)
terminal_events = self.conn.execute(
'''SELECT id, event_sha256 FROM runtime_audit_events
WHERE operation_id = ? AND result = ?
ORDER BY id LIMIT 2''',
(operation_id, parsed['result']),
).fetchall()
if len(terminal_events) != 1:
raise RuntimeSafetySchemaError(
'runtime operation terminal audit evidence is incomplete'
)
self.conn.commit()
return {
**current, 'replayed': True,
'audit_event_id': int(terminal_events[0]['id']),
'audit_event_sha256': str(terminal_events[0]['event_sha256']),
}
if current['status'] not in ('requested', 'running'):
raise RuntimeOperationTransitionError(
'runtime operation is already terminal'
)
if parsed['result'] == 'rolled_back':
expected = current['expected_identity']
if parsed['resulting_identity'] != {
'active_config_sha256': expected['active_config_sha256'],
'active_secrets_sha256': expected['active_secrets_sha256'],
}:
raise RuntimeOperationIdentityConflictError(
'rolled-back runtime operation identity is invalid'
)
elif parsed['result'] == 'succeeded':
if parsed['resulting_identity'] != _runtime_success_identity(
current['expected_identity'], current['action'],
):
raise RuntimeOperationIdentityConflictError(
'successful runtime operation identity is invalid'
)
now = utc_now_iso()
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=current['actor'],
action=current['action'], target_kind=current['target_kind'],
target_ref=current['target_ref'], result=parsed['result'],
safe_category=parsed['safe_category'],
before_identity=current['expected_identity'],
after_identity=parsed['resulting_identity'], created_at=now,
)
resulting_json = (
_canonical_runtime_json(parsed['resulting_identity'])
if parsed['resulting_identity'] is not None else None
)
updated = self.conn.execute(
'''UPDATE runtime_operations
SET status = ?, safe_category = ?, safe_detail = ?,
resulting_identity_json = ?, agent_state = ?,
agent_result_sha256 = ?,
started_at = COALESCE(started_at, ?),
completed_at = ?, agent_reconciled_at = ?, updated_at = ?
WHERE operation_id = ? AND status IN ('requested','running')
AND agent_result_sha256 IS NULL''',
(
parsed['result'], parsed['safe_category'], parsed['safe_detail'],
resulting_json, parsed['result'], result_sha256, now, now,
now, now, operation_id,
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeOperationTransitionError(
'runtime operation reconciliation conflicted'
)
result = self._runtime_operation_state(
self._locked_runtime_operation(operation_id),
)
self.conn.commit()
return {**result, 'replayed': False, **audit}
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime operation reconciliation retry budget exhausted')
def _runtime_control_replay_locked(
self, *, operation_id, actor, action, target_ref, expected_revision,
):
operation = self.conn.execute(
'SELECT * FROM runtime_operations WHERE operation_id = ?',
(operation_id,),
).fetchone()
if not operation:
return None
stored_expected_revision = operation['expected_revision']
stored_resulting_revision = operation['resulting_revision']
revisions_valid = (
isinstance(stored_expected_revision, int)
and not isinstance(stored_expected_revision, bool)
and isinstance(stored_resulting_revision, int)
and not isinstance(stored_resulting_revision, bool)
)
immutable_request = (
str(operation['actor'] or '') == actor
and str(operation['action'] or '') == action
and str(operation['target_kind'] or '') == RUNTIME_CONTROL_TARGET_KIND
and str(operation['target_ref'] or '') == target_ref
)
if not immutable_request:
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
if not revisions_valid:
raise RuntimeSafetySchemaError('runtime control replay revisions are invalid')
if stored_expected_revision != expected_revision:
raise RuntimeOperationIdentityConflictError(
'runtime operation ID is already bound to another request'
)
if (
operation['status'] != 'succeeded'
or operation['agent_state'] != 'not_required'
or operation['safe_category'] is not None
or operation['safe_detail'] is not None
or operation['agent_result_sha256'] is not None
or not operation['requested_at']
or not operation['started_at']
or not operation['completed_at']
or not operation['updated_at']
):
raise RuntimeSafetySchemaError('runtime control replay operation is incomplete')
before_json = str(operation['expected_identity_json'] or '')
after_json = str(operation['resulting_identity_json'] or '')
before = _stored_runtime_control_identity(before_json)
after = _stored_runtime_control_identity(after_json)
if (
before['revision'] != expected_revision
or stored_resulting_revision != after['revision']
or after['revision'] != before['revision'] + 1
):
raise RuntimeSafetySchemaError('runtime control replay revisions are invalid')
try:
expected_after = _runtime_control_transition(before, action)
except (RuntimeControlTransitionError, ValueError) as exc:
raise RuntimeSafetySchemaError(
'runtime control replay transition is invalid'
) from exc
if after != expected_after:
raise RuntimeSafetySchemaError('runtime control replay transition does not match')
events = self.conn.execute(
'''SELECT * FROM runtime_audit_events WHERE operation_id = ?
ORDER BY id LIMIT 2''',
(operation_id,),
).fetchall()
if len(events) != 1:
raise RuntimeSafetySchemaError('runtime control replay audit evidence is incomplete')
event = events[0]
if (
str(event['actor'] or '') != actor
or str(event['action'] or '') != action
or str(event['target_kind'] or '') != RUNTIME_CONTROL_TARGET_KIND
or str(event['target_ref'] or '') != target_ref
or event['result'] != 'succeeded'
or event['safe_category'] is not None
or event['before_identity_json'] != before_json
or event['after_identity_json'] != after_json
or event['before_bytes'] is not None
or event['after_bytes'] is not None
or not event['created_at']
):
raise RuntimeSafetySchemaError('runtime control replay audit evidence does not match')
previous_event_id = event['previous_event_id']
previous_event_sha256 = event['previous_event_sha256']
if previous_event_id is None:
if previous_event_sha256 is not None:
raise RuntimeSafetySchemaError('runtime control replay audit parent is invalid')
else:
parent = self.conn.execute(
'SELECT event_sha256 FROM runtime_audit_events WHERE id = ?',
(int(previous_event_id),),
).fetchone()
if (
not parent
or not hmac.compare_digest(
str(parent['event_sha256'] or ''), str(previous_event_sha256 or ''),
)
):
raise RuntimeSafetySchemaError('runtime control replay audit parent is invalid')
payload = {
'schema': 'runtime-audit-event-v1',
'operation_id': operation_id,
'actor': actor,
'action': action,
'target_kind': RUNTIME_CONTROL_TARGET_KIND,
'target_ref': target_ref,
'result': 'succeeded',
'safe_category': None,
'before_identity_json': before_json,
'after_identity_json': after_json,
'before_bytes': None,
'after_bytes': None,
'previous_event_id': int(previous_event_id) if previous_event_id is not None else None,
'previous_event_sha256': previous_event_sha256,
'created_at': str(event['created_at']),
}
expected_sha256 = _runtime_audit_event_sha256(payload)
if not hmac.compare_digest(expected_sha256, str(event['event_sha256'] or '')):
raise RuntimeSafetySchemaError('runtime control replay audit hash is invalid')
return {
'operation_id': operation_id,
'action': action,
'replayed': True,
'before': before,
'after': after,
'audit_event_id': int(event['id']),
'audit_event_sha256': expected_sha256,
}
def _commit_runtime_control_transition_locked(
self, *, state, action, target_ref, actor, operation_id,
):
before = _runtime_control_state_identity(state)
after = _runtime_control_transition(before, action)
before_json = _runtime_control_identity_json(before)
after_json = _runtime_control_identity_json(after)
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_operations(
operation_id, actor, action, target_kind, target_ref,
status, expected_revision, expected_identity_json,
agent_state, requested_at, started_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, 'not_required', ?, ?, ?)''',
(
operation_id, actor, action, RUNTIME_CONTROL_TARGET_KIND,
target_ref, before['revision'], before_json, now, now, now,
),
)
updated = self.conn.execute(
'''UPDATE runtime_operations_control
SET revision = ?, discovery_paused = ?, dispatch_paused = ?,
drain_state = ?, actor = ?, operation_id = ?, updated_at = ?
WHERE id = 1 AND revision = ?''',
(
after['revision'], int(after['discovery_paused']),
int(after['dispatch_paused']), after['drain_state'], actor,
operation_id, now, before['revision'],
),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeControlRevisionConflictError(
before['revision'], self._locked_runtime_control_state(),
)
audit = self._append_runtime_audit_event_locked(
operation_id=operation_id, actor=actor, action=action,
target_kind=RUNTIME_CONTROL_TARGET_KIND, target_ref=target_ref,
result='succeeded', before_identity=before,
after_identity=after, created_at=now,
)
completed = self.conn.execute(
'''UPDATE runtime_operations
SET status = 'succeeded', resulting_revision = ?,
resulting_identity_json = ?, completed_at = ?, updated_at = ?
WHERE operation_id = ? AND status = 'running' ''',
(after['revision'], after_json, now, now, operation_id),
)
if int(completed.rowcount or 0) != 1:
raise RuntimeSafetySchemaError('runtime control operation completion failed')
return {
'operation_id': operation_id,
'action': action,
'replayed': False,
'before': before,
'after': after,
**audit,
}
def _runtime_control_mutation(
self, *, action, target_ref, expected_revision, actor, operation_id,
):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
operation_id = _runtime_operation_id(operation_id)
actor = _runtime_actor(actor)
expected_revision = _runtime_revision(expected_revision)
if expected_revision > RUNTIME_CONTROL_MAX_EXPECTED_REVISION:
raise ValueError('runtime control revision cannot be advanced')
if action not in RUNTIME_CONTROL_ACTIONS:
raise ValueError('runtime control action is invalid')
if RUNTIME_CONTROL_ACTION_TARGETS[action] != target_ref:
raise ValueError('runtime control target is invalid')
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
state = self._locked_runtime_control_state()
replay = self._runtime_control_replay_locked(
operation_id=operation_id, actor=actor, action=action,
target_ref=target_ref, expected_revision=expected_revision,
)
if replay is not None:
self.conn.commit()
return replay
if state['revision'] != expected_revision:
raise RuntimeControlRevisionConflictError(expected_revision, state)
result = self._commit_runtime_control_transition_locked(
state=state, action=action, target_ref=target_ref,
actor=actor, operation_id=operation_id,
)
self.conn.commit()
return result
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime control mutation retry budget exhausted')
def set_runtime_discovery_paused(
self, paused, *, expected_revision, actor, operation_id,
):
if type(paused) is not bool:
raise ValueError('runtime discovery pause state must be a boolean')
return self._runtime_control_mutation(
action='control.discovery.pause' if paused else 'control.discovery.resume',
target_ref='discovery', expected_revision=expected_revision,
actor=actor, operation_id=operation_id,
)
def set_runtime_dispatch_paused(
self, paused, *, expected_revision, actor, operation_id,
):
if type(paused) is not bool:
raise ValueError('runtime dispatch pause state must be a boolean')
return self._runtime_control_mutation(
action='control.dispatch.pause' if paused else 'control.dispatch.resume',
target_ref='dispatch', expected_revision=expected_revision,
actor=actor, operation_id=operation_id,
)
def start_runtime_drain(self, *, expected_revision, actor, operation_id):
return self._runtime_control_mutation(
action='control.drain.start', target_ref='drain',
expected_revision=expected_revision, actor=actor,
operation_id=operation_id,
)
def cancel_runtime_drain(self, *, expected_revision, actor, operation_id):
return self._runtime_control_mutation(
action='control.drain.cancel', target_ref='drain',
expected_revision=expected_revision, actor=actor,
operation_id=operation_id,
)
def _runtime_drain_blockers_locked(self):
row = self.conn.execute(
'''SELECT
(SELECT COUNT(*) FROM result_reservations
WHERE assignment_kind = 'remote'
AND remote_resolved_at IS NULL) AS live_remote_assignments,
(SELECT COUNT(*) FROM result_bundles
WHERE state IN ('ready', 'ingesting')) AS precommit_result_bundles'''
).fetchone()
if not row:
raise RuntimeSafetySchemaError('runtime drain progress is unavailable')
values = {}
for name in ('live_remote_assignments', 'precommit_result_bundles'):
value = row[name]
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise RuntimeSafetySchemaError('runtime drain progress is invalid')
values[name] = value
values['blocker_count'] = sum(values.values())
return values
def runtime_drain_progress(self):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
try:
state = self._locked_runtime_control_state(shared=True)
progress = self._runtime_drain_blockers_locked()
self.conn.commit()
return {**state, **progress}
except Exception:
self.conn.rollback()
raise
def reconcile_runtime_drain(self):
if not self.conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
actor = 'system:drain-reconciler'
operation_id = str(uuid.uuid4())
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
for attempt in range(attempts):
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
state = self._locked_runtime_control_state()
if state['drain_state'] != 'draining':
self.conn.commit()
return None
blockers = self._runtime_drain_blockers_locked()
if blockers['blocker_count']:
self.conn.commit()
return None
result = self._commit_runtime_control_transition_locked(
state=state, action='control.drain.complete',
target_ref='drain', actor=actor, operation_id=operation_id,
)
self.conn.commit()
return result
except Exception as exc:
self.conn.rollback()
if (
self.conn.is_sqlite and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
raise
raise RuntimeSafetySchemaError('runtime drain reconciliation retry budget exhausted')
def final_cutover_status(self):
if not self.conn or not self.conn.is_postgres:
return None
row = self.conn.execute(
'SELECT marker, checked_at, evidence_sha256 FROM runtime_final_cutover WHERE id = 1'
).fetchone()
self.conn.commit()
if not row:
return None
result = dict(row)
if (
result['marker'] != FINAL_CUTOVER_MARKER
or not result['checked_at']
or not re.fullmatch(r'[a-f0-9]{64}', str(result['evidence_sha256'] or ''))
):
return None
return result
def require_final_cutover(self):
if not self.final_cutover_status():
raise RuntimeSafetySchemaError(
'final PostgreSQL v2 cutover marker is absent or invalid; '
'run migrate_runtime_safety.py offline with --apply --sources-stopped'
)
return True
def record_final_cutover(self, evidence):
if not self.conn or not self.conn.is_postgres:
raise RuntimeSafetySchemaError('final cutover authority can only be recorded in PostgreSQL')
encoded = json.dumps(
evidence, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
if len(encoded) > 1024 * 1024:
raise ValueError('final cutover evidence exceeds its byte bound')
digest = hashlib.sha256(encoded).hexdigest()
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO runtime_final_cutover(id, marker, checked_at, evidence_sha256)
VALUES (1, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET marker = excluded.marker,
checked_at = excluded.checked_at,
evidence_sha256 = excluded.evidence_sha256''',
(FINAL_CUTOVER_MARKER, now, digest),
)
self.conn.commit()
return {'marker': FINAL_CUTOVER_MARKER, 'checked_at': now, 'evidence_sha256': digest}
def revoke_final_cutover(self):
if not self.conn or not self.conn.is_postgres:
raise RuntimeSafetySchemaError('final cutover authority can only be revoked in PostgreSQL')
self.conn.execute('DELETE FROM runtime_final_cutover WHERE id = 1')
self.conn.commit()
return True
def pipeline_schema_available(self, commit=True):
if not self.conn:
self.last_error = 'database connection is unavailable'
return False
try:
problems = []
for table, columns in PIPELINE_REQUIRED_COLUMNS.items():
if not self.conn.table_exists(table):
problems.append(f'table {table}')
continue
missing = columns - set(self.conn.table_columns(table))
if missing:
problems.append(f'{table} columns {", ".join(sorted(missing))}')
capacity = self.conn.execute(
'SELECT id FROM pipeline_capacity WHERE id = 1'
).fetchone() if self.conn.table_exists('pipeline_capacity') else None
if not capacity:
problems.append('pipeline_capacity singleton row')
if self.conn.table_exists('runtime_schema_migrations'):
rows = self.conn.execute(
'SELECT version, code_sha256 FROM runtime_schema_migrations'
).fetchall()
applied = {str(row['version']) for row in rows}
missing_versions = set(PIPELINE_MIGRATION_VERSIONS) - applied
if missing_versions:
problems.append('migration versions ' + ', '.join(sorted(missing_versions)))
marker = next(
(
row for row in rows
if str(row['version']) == PIPELINE_MIGRATION_VERSIONS[-1]
),
None,
)
expected_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
if marker and str(marker['code_sha256'] or '') != expected_code:
problems.append('pipeline migration marker code identity')
if self.conn.table_exists('pipeline_quarantine'):
constraint = _pipeline_quarantine_review_status_constraint(self.conn)
if (
constraint.get('statuses') != PIPELINE_QUARANTINE_REVIEW_STATUSES
or not constraint.get('valid')
):
problems.append('pipeline quarantine review-status constraint')
problems.extend(_docker_depth_check_constraint_problems(self.conn))
required_indexes = {
'target_queue': (
'idx_target_queue_current_reservation', 'idx_target_queue_claimable_v2',
'idx_target_queue_cold',
),
'target_queue_policy_events': (
'idx_target_queue_policy_events_queue',
'idx_target_queue_policy_events_manifest',
),
'target_scans': (
'idx_target_scans_event_hash', 'idx_target_scans_queue_id',
'idx_target_scans_result_reservation',
),
'findings': ('idx_findings_target_scan_id_id',),
'result_reservations': (
'uq_result_reservation_queue_lease', 'idx_result_reservations_recovery',
'idx_result_reservations_remote_active_user',
'idx_result_reservations_remote_expiry',
'idx_result_reservations_remote_device_history',
'uq_result_reservations_remote_receipt',
),
'remote_worker_users': ('uq_remote_worker_users_key',),
'remote_worker_devices': (
'uq_remote_worker_devices_key', 'uq_remote_worker_devices_token',
),
'admission_intents': ('idx_admission_intents_remote_device',),
'result_bundles': ('idx_result_bundles_ready', 'idx_result_bundles_ingest_lease'),
'projection_jobs': ('idx_projection_jobs_claim', 'idx_projection_jobs_lease'),
'keycheck_candidates': (
'idx_keycheck_candidates_pending', 'idx_keycheck_candidates_deferred',
'idx_keycheck_candidates_lease',
),
'keycheck_credentials': ('uq_keycheck_credentials_provider_key',),
'pipeline_artifacts': ('idx_pipeline_artifacts_owner',),
'docker_content_blobs': (
'idx_docker_content_blobs_reclaim',
'idx_docker_content_blobs_reservation',
),
'docker_image_blob_coverage': (
'idx_docker_image_blob_coverage_manifest',
'idx_docker_image_blob_coverage_blob',
'idx_docker_image_blob_coverage_reservation',
'idx_docker_image_blob_coverage_selection',
),
'docker_adaptive_shadow_reports': (
'idx_docker_adaptive_shadow_reports_gate',
),
'discovery_retry_queue': (
'uq_discovery_retry_queue_work_key',
'idx_discovery_retry_queue_due',
'idx_discovery_retry_queue_lease',
'idx_discovery_retry_queue_policy',
),
'runtime_operations': (
'idx_runtime_operations_status_updated',
'idx_runtime_operations_agent_state_updated',
),
'runtime_audit_events': (
'idx_runtime_audit_events_created',
'idx_runtime_audit_events_operation',
),
'worker_progress_events': (
'uq_worker_progress_reservation_sequence',
'idx_worker_progress_reservation_received',
'idx_worker_progress_device_received',
'idx_worker_progress_phase_received',
),
'worker_diagnostics': (
'uq_worker_diagnostics_uid',
'idx_worker_diagnostics_reservation_received',
'idx_worker_diagnostics_scan_received',
'idx_worker_diagnostics_phase_received',
'idx_worker_diagnostics_category_code',
'idx_worker_diagnostics_code_received',
'idx_worker_diagnostics_kind_received',
'idx_worker_diagnostics_retryable_received',
),
}
for table, names in required_indexes.items():
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
for name in names:
if not _index_usable(indexes.get(name)):
problems.append(f'index {name}')
experiment_indexes = {}
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
indexes = experiment_indexes.setdefault(
table,
self.conn.table_indexes(table) if self.conn.table_exists(table) else {},
)
index = indexes.get(name)
if (
not index
or index['columns'] != list(columns)
or bool(index['unique']) != bool(unique)
or _normalized_predicate(index['predicate']) != _normalized_predicate(predicate)
or not _index_usable(index)
):
problems.append(f'index {name}')
credential_index = (
self.conn.table_indexes('keycheck_credentials').get('uq_keycheck_credentials_provider_key')
if self.conn.table_exists('keycheck_credentials') else None
)
if not credential_index or not credential_index['unique'] or credential_index['columns'] != [
'service', 'provider_key_hash'
]:
problems.append('unique provider-canonical keycheck credential index')
if self.conn.is_postgres and commit:
self.conn.commit()
if problems:
self.last_error = 'pipeline schema is incomplete: ' + '; '.join(problems)
return False
self.last_error = ''
return True
except Exception as exc:
self.last_error = f'unable to validate pipeline schema: {exc}'
try:
self.conn.rollback()
except Exception:
pass
return False
def provider_routing_finding_rows(self, secret_hash, limit, max_json_chars):
if not self.conn:
raise RuntimeError('database connection is unavailable')
secret_hash = str(secret_hash or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', secret_hash):
raise ValueError('provider routing lookup requires one SHA-256 digest')
limit = min(1025, max(1, int(limit or 0)))
max_json_chars = min(1024 * 1024, max(1024, int(max_json_chars or 0)))
return self.conn.execute(
'''SELECT
detector_name,
SUBSTR(COALESCE(raw_finding_json, ''), 1, ?) AS raw_finding_json,
CASE WHEN LENGTH(COALESCE(raw_finding_json, '')) > ? THEN 1 ELSE 0 END AS truncated
FROM findings
WHERE secret_hash = ?
ORDER BY id DESC
LIMIT ?''',
(max_json_chars, max_json_chars, secret_hash, limit),
).fetchall()
def ensure_scan_publication_outbox(self):
if not self.conn or not self.conn.table_exists('scan_publication_outbox'):
self.last_error = 'scan_publication_outbox is unavailable'
return False
required = {
'id', 'target_scan_id', 'payload_json', 'status', 'attempts', 'last_error',
'lease_owner', 'lease_expires_at', 'available_after', 'created_at', 'delivered_at', 'updated_at',
}
missing = required - set(self.conn.table_columns('scan_publication_outbox'))
if missing:
self.last_error = f'scan_publication_outbox is missing columns: {", ".join(sorted(missing))}'
return False
self.last_error = ''
return True
def scan_publication_backlog_health(self, max_items, max_bytes, max_age_sec, additional_items=0):
if not self.conn or not self.conn.table_exists('scan_publication_outbox'):
return {'healthy': False, 'accepting': False, 'reason': 'publication outbox is unavailable'}
def op():
now = datetime.now(timezone.utc)
payload_size = (
"OCTET_LENGTH(COALESCE(s.raw_result_json, ''))"
if self.conn.is_postgres
else "LENGTH(CAST(COALESCE(s.raw_result_json, '') AS BLOB))"
)
row = self.conn.execute(
f'''SELECT COUNT(*) AS item_count,
COALESCE(SUM({payload_size}), 0) AS payload_bytes,
MIN(o.created_at) AS oldest_at
FROM scan_publication_outbox o
LEFT JOIN target_scans s ON s.id = o.target_scan_id
WHERE o.status IN ('pending', 'delivering', 'dead')'''
).fetchone()
item_count = int(row['item_count'] or 0)
payload_bytes = int(row['payload_bytes'] or 0)
oldest = parse_time(row['oldest_at'])
oldest_age_sec = max(0, int((now - oldest).total_seconds())) if oldest else 0
item_limit = max(1, int(max_items))
byte_limit = max(1, int(max_bytes))
age_limit = max(1, int(max_age_sec))
reasons = []
if item_count > item_limit:
reasons.append(f'items={item_count}>{item_limit}')
if payload_bytes > byte_limit:
reasons.append(f'bytes={payload_bytes}>{byte_limit}')
if oldest_age_sec > age_limit:
reasons.append(f'oldest_age_sec={oldest_age_sec}>{age_limit}')
healthy = not reasons
accepting = healthy and item_count + max(0, int(additional_items or 0)) <= item_limit
if not accepting and not reasons:
reasons.append(f'items={item_count}+{max(0, int(additional_items or 0))}>{item_limit}')
return {
'healthy': healthy,
'accepting': accepting,
'reason': '; '.join(reasons),
'items': item_count,
'bytes': payload_bytes,
'oldest_age_sec': oldest_age_sec,
'max_items': item_limit,
'max_bytes': byte_limit,
'max_age_sec': age_limit,
}
return self._safe('scan_publication_backlog_health', op, {
'healthy': False, 'accepting': False, 'reason': self.last_error or 'publication backlog query failed',
})
def claim_scan_publications(self, lease_owner, limit=100, lease_seconds=300, max_attempts=None):
if not self.conn or not self.conn.table_exists('scan_publication_outbox') or not lease_owner:
return []
def op():
now = utc_now_iso()
compact_delivered_scan_publications(self.conn, now)
lease_until = datetime.fromtimestamp(time.time() + max(60, int(lease_seconds)), timezone.utc).isoformat(timespec='seconds')
self.conn.execute(
'''UPDATE scan_publication_outbox SET status = 'pending', payload_json = '',
lease_owner = NULL, lease_expires_at = NULL, available_after = COALESCE(available_after, ?), updated_at = ?
WHERE status = 'dead' ''',
(now, now),
)
self.conn.execute("UPDATE scan_publication_outbox SET payload_json = '' WHERE payload_json != ''")
self.conn.execute(
'''UPDATE scan_publication_outbox SET status = 'pending', lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
WHERE status = 'delivering' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?''',
(now, now),
)
limit_value = max(1, int(limit or 100))
if self.conn.is_postgres:
rows = self.conn.execute(
'''WITH picked AS (
SELECT id FROM scan_publication_outbox
WHERE status = 'pending' AND (available_after IS NULL OR available_after <= ?)
ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED
)
UPDATE scan_publication_outbox o SET status = 'delivering', lease_owner = ?,
lease_expires_at = ?, attempts = CASE WHEN COALESCE(attempts, 0) < 1000000
THEN COALESCE(attempts, 0) + 1 ELSE COALESCE(attempts, 0) END, updated_at = ?
FROM picked WHERE o.id = picked.id''',
(now, limit_value, lease_owner, lease_until, now),
)
else:
rows = self.conn.execute(
'''SELECT id FROM scan_publication_outbox
WHERE status = 'pending' AND (available_after IS NULL OR available_after <= ?)
ORDER BY id LIMIT ?''',
(now, limit_value),
).fetchall()
for row in rows:
self.conn.execute(
'''UPDATE scan_publication_outbox SET status = 'delivering', lease_owner = ?,
lease_expires_at = ?, attempts = CASE WHEN COALESCE(attempts, 0) < 1000000
THEN COALESCE(attempts, 0) + 1 ELSE COALESCE(attempts, 0) END,
updated_at = ? WHERE id = ?''',
(lease_owner, lease_until, now, row['id']),
)
rows = self.conn.execute(
'''SELECT o.id, o.target_scan_id, COALESCE(s.raw_result_json, '') AS payload_json, o.attempts
FROM scan_publication_outbox o
JOIN target_scans s ON s.id = o.target_scan_id
WHERE o.status = 'delivering' AND o.lease_owner = ?
ORDER BY o.id LIMIT ?''',
(lease_owner, limit_value),
).fetchall()
self.conn.commit()
return rows
return self._safe('claim_scan_publications', op, [])
def renew_scan_publication(self, outbox_id, lease_owner, lease_seconds=300):
if not self.conn or outbox_id is None or not lease_owner:
return False
try:
now = utc_now_iso()
lease_until = datetime.fromtimestamp(
time.time() + max(60, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
cur = self.conn.execute(
'''UPDATE scan_publication_outbox SET lease_expires_at = ?, updated_at = ?
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
(lease_until, now, outbox_id, lease_owner),
)
renewed = int(getattr(cur, 'rowcount', 0) or 0) == 1
self.conn.commit()
self.last_error = ''
return renewed
except Exception as exc:
self.last_error = str(exc)
try:
self.conn.rollback()
except Exception:
pass
logger.error(f'Observability DB write failed during renew_scan_publication: {exc}')
return False
def finish_scan_publication(self, outbox_id, lease_owner, delivered, error=''):
if not self.conn:
return False
def op():
now = utc_now_iso()
if delivered:
cur = self.conn.execute(
'''DELETE FROM scan_publication_outbox
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
(outbox_id, lease_owner),
)
else:
claimed = self.conn.execute(
'''SELECT attempts FROM scan_publication_outbox
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
(outbox_id, lease_owner),
).fetchone()
if not claimed:
self.conn.rollback()
return False
attempts = max(1, int(claimed['attempts'] or 1))
base = max(1, env_int('SCAN_OUTBOX_RETRY_BASE_SEC', 30))
maximum = max(base, env_int('SCAN_OUTBOX_RETRY_MAX_SEC', 3600))
delay = min(maximum, base * (2 ** min(attempts - 1, 20)))
available_after = datetime.fromtimestamp(
time.time() + delay, timezone.utc
).isoformat(timespec='seconds')
cur = self.conn.execute(
'''UPDATE scan_publication_outbox SET status = 'pending', last_error = ?, payload_json = '',
lease_owner = NULL, lease_expires_at = NULL, available_after = ?, updated_at = ?
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
(first_line(error, 500), available_after, now, outbox_id, lease_owner),
)
self.conn.commit()
return int(getattr(cur, 'rowcount', 0) or 0) == 1
return self._safe('finish_scan_publication', op, False)
def sync_target_queue_from_files(self, source, platform, todo_targets=None, checked_targets=None, query=None):
if not self.conn:
return False
def op():
now = utc_now_iso()
for target in checked_targets or []:
normalized = normalize_target(target, platform)
self.conn.execute(
f'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status, created_at, updated_at, completed_at
) VALUES (?, ?, ?, ?, ?, 'done', ?, ?, ?)
ON CONFLICT(source, normalized_target) DO UPDATE SET
status = CASE WHEN target_queue.status IN ('failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status ELSE 'done' END,
target = excluded.target,
platform = excluded.platform,
completed_at = CASE
WHEN target_queue.status IN ('deferred', 'in_progress', 'cold') THEN target_queue.completed_at
ELSE COALESCE(target_queue.completed_at, excluded.completed_at)
END,
lease_owner = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_owner ELSE NULL END,
lease_token = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_token ELSE NULL END,
claim_batch = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.claim_batch ELSE NULL END,
leased_at = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.leased_at ELSE NULL END,
lease_expires_at = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_expires_at ELSE NULL END,
available_after = CASE WHEN target_queue.status IN ('failed', 'deferred', 'in_progress', 'cold') THEN target_queue.available_after ELSE NULL END,
updated_at = excluded.updated_at
WHERE target_queue.status NOT IN ('done', 'failed', 'deferred', 'in_progress', 'cold')
OR target_queue.target <> excluded.target
OR target_queue.platform <> excluded.platform''',
(source, platform, query, target, normalized, now, now, now),
)
for target in todo_targets or []:
normalized = normalize_target(target, platform)
self.conn.execute(
f'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
ON CONFLICT(source, normalized_target) DO UPDATE SET
target = excluded.target,
platform = excluded.platform,
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN excluded.query ELSE COALESCE(target_queue.query, excluded.query) END,
status = CASE
WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 'pending'
WHEN target_queue.status IN ('done', 'failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status
WHEN target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL THEN 'deferred'
ELSE 'pending'
END,
available_after = CASE
WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL
WHEN target_queue.status IN ('pending', 'deferred', 'cold') THEN target_queue.available_after
ELSE NULL
END,
attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN 0 ELSE target_queue.attempts END,
last_error = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.last_error END,
completed_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.completed_at END,
resolver_state = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_state END,
resolver_due_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_due_at END,
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN 0 ELSE target_queue.resolver_attempts END,
resolver_token = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_token END,
updated_at = excluded.updated_at
WHERE ({ADMIN_DISCARDED_QUEUE_SQL})
OR target_queue.target <> excluded.target
OR target_queue.platform <> excluded.platform
OR (target_queue.query IS NULL AND excluded.query IS NOT NULL)
OR (target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL)''',
(source, platform, query, target, normalized, now, now),
)
self.conn.commit()
return True
return self._safe('sync_target_queue_from_files', op, False)
def _docker_discovery_pass_locked(
self, source, query, observation, now, pass_id=None,
):
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
params = (
source,
observation['pass_kind'],
observation['collection_generation'],
observation['policy_sha256'],
observation['ordered_query_hash'],
observation['query_count'],
)
if pass_id is not None:
rows = self.conn.execute(
f'''SELECT * FROM docker_discovery_passes
WHERE id = ? AND source = ? AND pass_kind = ?
AND collection_generation = ? AND policy_sha256 = ?
AND ordered_queries_sha256 = ?
AND expected_query_count = ?
AND state IN ('collecting','complete'){lock_suffix}''',
(pass_id, *params),
).fetchall()
if len(rows) != 1:
raise RuntimeError('DockerHub discovery retry pass identity changed')
candidates = rows
else:
cycle_predicate = (
'page.source_cycle_id IS NULL'
if observation['cycle_id'] is None
else 'page.source_cycle_id = ?'
)
cycle_params = (
() if observation['cycle_id'] is None
else (observation['cycle_id'],)
)
candidates = self.conn.execute(
f'''SELECT * FROM docker_discovery_passes
WHERE source = ? AND pass_kind = ? AND collection_generation = ?
AND policy_sha256 = ?
AND ordered_queries_sha256 = ? AND expected_query_count = ?
AND (
state = 'collecting' OR EXISTS (
SELECT 1 FROM docker_discovery_pages page
WHERE page.pass_id = docker_discovery_passes.id
AND page.query_ordinal = ? AND page.page_number = ?
AND {cycle_predicate}
)
)
ORDER BY id{lock_suffix}''',
(
*params, observation['query_ordinal'],
observation['page_number'], *cycle_params,
),
).fetchall()
empty_candidate = None
selected = None
for candidate in candidates:
pages = self.conn.execute(
'''SELECT query, page_number, source_cycle_id
FROM docker_discovery_pages
WHERE pass_id = ? AND query_ordinal = ? ORDER BY page_number''',
(candidate['id'], observation['query_ordinal']),
).fetchall()
if any(str(page['query']) != query for page in pages):
raise RuntimeError('DockerHub discovery pass query identity conflicts')
exact_page = [
page for page in pages
if int(page['page_number']) == observation['page_number']
]
if pass_id is not None:
if str(candidate['state']) == 'complete' and not exact_page:
raise RuntimeError('DockerHub discovery complete retry pass cannot admit another page')
return candidate
same_cycle = any(
(
page['source_cycle_id'] is None
and observation['cycle_id'] is None
) or (
page['source_cycle_id'] is not None
and observation['cycle_id'] is not None
and int(page['source_cycle_id']) == observation['cycle_id']
)
for page in pages
)
if exact_page and (same_cycle or observation['cycle_id'] is None):
selected = candidate
break
if same_cycle:
if str(candidate['state']) == 'complete':
raise RuntimeError('DockerHub discovery complete pass cannot admit another page')
selected = candidate
break
if not pages and str(candidate['state']) == 'collecting' and empty_candidate is None:
empty_candidate = candidate
selected = selected or empty_candidate
if selected is not None:
return selected
if pass_id is not None:
raise RuntimeError('DockerHub discovery retry pass is unavailable')
token = secrets.token_urlsafe(32)
row = self.conn.execute(
'''INSERT INTO docker_discovery_passes(
experiment_id, pass_token, source, pass_kind,
collection_generation, policy_sha256,
ordered_queries_sha256, expected_query_count,
completed_query_count, state, started_at, completed_at,
created_at, updated_at
) VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, 0, 'collecting', ?, NULL, ?, ?)
RETURNING *''',
(
token, source, observation['pass_kind'],
observation['collection_generation'], observation['policy_sha256'],
observation['ordered_query_hash'], observation['query_count'],
now, now, now,
),
).fetchone()
if not row:
raise RuntimeError('DockerHub discovery pass could not be opened')
return row
def persist_dockerhub_discovery_page(
self, source, query, repositories, retry_id=None, lease_owner=None,
lease_token=None, next_page=None, complete=False, observation=None,
experiment_authority=None, final_cutover=False,
):
"""Durably admit one normalized page without changing existing target state."""
if not self.conn:
raise RuntimeError('database connection is unavailable')
source = _validated_discovery_retry_source(source)
query = _validated_discovery_retry_query(query)
attempted_count, normalized, ordinals, observed = (
_normalized_dockerhub_repositories(repositories)
)
observation = _validated_docker_discovery_observation(
observation, source, query,
)
if observation is not None:
if observation['total_count'] is None:
raise ValueError('DockerHub page admission lacks total-count evidence')
absolute_start = (
observation['page_number'] - 1
) * observation['per_page']
absolute_bound = absolute_start + attempted_count
terminal_by_count = observation['total_count'] <= (
observation['page_number'] * observation['per_page']
)
if (
attempted_count > observation['per_page']
or observation['total_count'] < absolute_bound
or (attempted_count == 0 and observation['total_count'] > absolute_start)
or (
observation['query_complete']
and not terminal_by_count
and observation['page_number'] != observation['page_limit']
)
or (
not observation['query_complete']
and (
terminal_by_count
or observation['page_number'] == observation['page_limit']
)
)
):
raise ValueError('DockerHub page cardinality or continuation evidence conflicts')
if not isinstance(complete, bool):
raise ValueError('discovery retry completion flag is invalid')
retry_requested = any(
value is not None for value in (retry_id, lease_owner, lease_token, next_page)
) or complete
fence = None
if retry_requested:
fence = _validated_discovery_retry_fence(retry_id, lease_owner, lease_token)
if complete and next_page is not None:
raise ValueError('completed discovery retry work cannot retain a next page')
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._require_discovery_admission_locked()
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
page_experiment = None
page_authority = None
if self.conn.is_postgres and source == 'dockerhub':
page_experiment, page_authority, authority_reason = (
self._locked_docker_depth_page_authority(
source, query, observation, experiment_authority,
final_cutover=final_cutover, now=now,
)
)
if authority_reason:
self.conn.commit()
raise ScanEventConflictError(
f'Docker depth page authority drifted: {authority_reason}'
)
retry_row = None
pass_row = None
if fence:
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
retry_row = self.conn.execute(
f'''SELECT work_key, policy_sha256, pass_kind, work_kind,
page_start, page_end, next_page, source_cycle_id
FROM discovery_retry_queue
WHERE id = ? AND source = ? AND query = ? AND status = 'leased'
AND lease_owner = ? AND lease_token = ?
AND lease_expires_at > ?{lock_suffix}''',
(fence[0], source, query, fence[1], fence[2], now),
).fetchone()
if not retry_row:
raise DiscoveryRetryLeaseError('discovery retry page admission lost its lease fence')
if next_page is not None:
if isinstance(next_page, bool):
raise ValueError('discovery retry next page is invalid')
try:
next_page = int(next_page)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry next page is invalid') from None
if not (
int(retry_row['next_page']) <= next_page <= int(retry_row['page_end'])
and 1 <= next_page <= DISCOVERY_RETRY_MAX_PAGE
):
raise ValueError('discovery retry next page is outside its durable range')
retry_identity = (
source, query, retry_row['policy_sha256'], retry_row['pass_kind'],
retry_row['work_kind'], retry_row['page_start'], retry_row['page_end'],
)
legacy_work_key = discovery_retry_work_key(*retry_identity)
if str(retry_row['work_key']) == legacy_work_key:
observation = None
else:
try:
pass_id = int(str(retry_row['work_key'])[:16], 16)
except (TypeError, ValueError, OverflowError):
raise RuntimeError('DockerHub discovery retry pass identity is invalid') from None
if str(retry_row['work_key']) != discovery_retry_work_key(
*retry_identity, pass_id=pass_id,
):
raise RuntimeError('DockerHub discovery retry pass identity conflicts')
if observation is None:
raise RuntimeError('DockerHub discovery retry provenance is unavailable')
retry_cycle_id = (
int(retry_row['source_cycle_id'])
if retry_row['source_cycle_id'] is not None else None
)
if (
observation['policy_sha256'] != str(retry_row['policy_sha256'])
or observation['pass_kind'] != str(retry_row['pass_kind'])
or observation['page_number'] != int(retry_row['next_page'])
or observation['cycle_id'] != retry_cycle_id
):
raise RuntimeError('DockerHub discovery retry provenance changed')
pass_row = self._docker_discovery_pass_locked(
source, query, observation, now, pass_id=pass_id,
)
elif observation is not None:
pass_row = self._docker_discovery_pass_locked(
source, query, observation, now,
)
preexisting = set()
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
for start in range(0, len(normalized), KNOWN_TARGET_LOOKUP_BATCH_SIZE):
values = normalized[start:start + KNOWN_TARGET_LOOKUP_BATCH_SIZE]
rows = self.conn.execute(
'''SELECT normalized_target FROM target_queue
WHERE source = ? AND platform = 'docker'
AND normalized_target IN ({}){}'''.format(
','.join('?' for _ in values), lock_suffix,
),
(source, *values),
).fetchall()
preexisting.update(
str(row['normalized_target']) for row in rows if row['normalized_target']
)
resolver_due = (now_dt + timedelta(
seconds=max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)),
)).isoformat(timespec='seconds')
inserted_count = 0
queue_ids = {}
new_queue_ids = set()
for repository in normalized:
inserted = self.conn.execute(
'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status,
available_after, last_error, resolver_state, resolver_due_at,
created_at, updated_at
) VALUES (?, 'docker', ?, ?, ?, 'deferred', ?,
'Docker tag resolution unresolved', 'pending', ?, ?, ?)
ON CONFLICT(source, normalized_target) DO NOTHING
RETURNING id''',
(
source, query, repository, repository, resolver_due,
resolver_due, now, now,
),
).fetchone()
inserted_count += int(bool(inserted))
if inserted:
queue_ids[repository] = int(inserted['id'])
new_queue_ids.add(int(inserted['id']))
else:
queue_row = self.conn.execute(
f'''SELECT id, platform, status, last_error FROM target_queue
WHERE source = ? AND normalized_target = ?{lock_suffix}''',
(source, repository),
).fetchone()
if not queue_row or str(queue_row['platform']) != 'docker':
raise RuntimeError('DockerHub repository queue identity conflicts')
queue_id = int(queue_row['id'])
queue_ids[repository] = queue_id
if (
str(queue_row['status']) == 'quarantined'
and str(queue_row['last_error'] or '')
== ADMIN_DISCARDED_QUEUE_REASON
):
cursor = self.conn.execute(
'''UPDATE target_queue SET query = ?, target = ?,
status = 'deferred', attempts = 0,
available_after = ?,
last_error = 'Docker tag resolution unresolved',
completed_at = NULL, lease_owner = NULL,
lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL,
resolver_state = 'pending', resolver_due_at = ?,
resolver_attempts = 0, resolver_token = NULL,
claim_event_id = NULL, updated_at = ?
WHERE id = ? AND status = 'quarantined'
AND last_error = ?''',
(
query, repository, resolver_due, resolver_due, now,
queue_id, ADMIN_DISCARDED_QUEUE_REASON,
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError(
'discarded DockerHub target reactivation authority changed'
)
inserted_count += 1
new_queue_ids.add(queue_id)
preexisting.discard(repository)
page_id = None
page_inserted_count = 0
observation_inserted_count = 0
completed_query_count = 0
pass_complete = False
query_complete = False
if pass_row is not None:
page_payload = json.dumps(
{
'per_page': observation['per_page'],
'page_number': observation['page_number'],
'page_limit': observation['page_limit'],
'query': query,
'query_ordinal': observation['query_ordinal'],
'repositories': observed,
'total_count': observation['total_count'],
'collection_generation': observation['collection_generation'],
'schema': 'docker-discovery-page-v2',
'source': source,
},
ensure_ascii=True,
sort_keys=True,
separators=(',', ':'),
).encode('utf-8')
page_sha256 = hashlib.sha256(page_payload).hexdigest()
admission_kind = 'retry' if fence else 'main'
inserted_page = self.conn.execute(
'''INSERT INTO docker_discovery_pages(
pass_id, source_cycle_id, retry_work_id, query,
query_ordinal, page_number, result_count, total_count,
admitted_count,
query_complete, admission_kind, page_sha256,
observed_at, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(pass_id, query_ordinal, page_number) DO NOTHING
RETURNING *''',
(
pass_row['id'], observation['cycle_id'],
fence[0] if fence else None, query,
observation['query_ordinal'], observation['page_number'],
attempted_count, observation['total_count'], len(normalized),
int(observation['query_complete']),
admission_kind, page_sha256, now, now,
),
).fetchone()
page_inserted_count = int(bool(inserted_page))
page_row = inserted_page or self.conn.execute(
f'''SELECT * FROM docker_discovery_pages
WHERE pass_id = ? AND query_ordinal = ? AND page_number = ?{lock_suffix}''',
(
pass_row['id'], observation['query_ordinal'],
observation['page_number'],
),
).fetchone()
if not page_row or (
str(page_row['query']) != query
or int(page_row['result_count']) != attempted_count
or int(page_row['total_count']) != observation['total_count']
or int(page_row['admitted_count']) != len(normalized)
or str(page_row['admission_kind']) != admission_kind
or str(page_row['page_sha256']) != page_sha256
or (
(page_row['retry_work_id'] is None) != (fence is None)
)
or (
fence is not None
and int(page_row['retry_work_id']) != fence[0]
)
or (
(page_row['source_cycle_id'] is None) !=
(observation['cycle_id'] is None)
)
or (
page_row['source_cycle_id'] is not None
and int(page_row['source_cycle_id']) != observation['cycle_id']
)
):
raise RuntimeError('DockerHub discovery page evidence conflicts')
page_id = int(page_row['id'])
page_observed_at = str(page_row['observed_at'])
if observation['query_complete'] and not int(page_row['query_complete']):
self.conn.execute(
'''UPDATE docker_discovery_pages SET query_complete = 1
WHERE id = ? AND query_complete = 0''',
(page_id,),
)
for repository in normalized:
queue_id = queue_ids[repository]
search_rank = (
(observation['page_number'] - 1) * observation['per_page']
+ ordinals[repository]
)
existing_observation = self.conn.execute(
'''SELECT search_rank FROM docker_repository_query_observations
WHERE page_id = ? AND repository_queue_id = ?''',
(page_id, queue_id),
).fetchone()
if existing_observation:
if int(existing_observation['search_rank']) != search_rank:
raise RuntimeError('DockerHub repository observation rank conflicts')
continue
self.conn.execute(
'''INSERT INTO docker_repository_query_provenance(
source, query, repository_queue_id, provenance_kind,
first_observed_at, last_observed_at, first_search_rank,
best_search_rank, last_search_rank, first_cycle_id,
last_cycle_id, first_page_id, last_page_id,
first_policy_sha256, last_policy_sha256,
observation_count, fresh_observation_count,
fresh_complete_observation_count, fresh_coverage_eligible,
created_at, updated_at
) VALUES (?, ?, ?, 'fresh_page', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
1, 1, 0, 0, ?, ?)
ON CONFLICT(source, query, repository_queue_id) DO UPDATE SET
provenance_kind = 'fresh_page',
last_observed_at = excluded.last_observed_at,
first_search_rank = COALESCE(
docker_repository_query_provenance.first_search_rank,
excluded.first_search_rank
),
best_search_rank = CASE
WHEN docker_repository_query_provenance.best_search_rank IS NULL
OR excluded.best_search_rank < docker_repository_query_provenance.best_search_rank
THEN excluded.best_search_rank
ELSE docker_repository_query_provenance.best_search_rank
END,
last_search_rank = excluded.last_search_rank,
first_cycle_id = COALESCE(
docker_repository_query_provenance.first_cycle_id,
excluded.first_cycle_id
),
last_cycle_id = COALESCE(
excluded.last_cycle_id,
docker_repository_query_provenance.last_cycle_id
),
first_page_id = COALESCE(
docker_repository_query_provenance.first_page_id,
excluded.first_page_id
),
last_page_id = excluded.last_page_id,
first_policy_sha256 = COALESCE(
docker_repository_query_provenance.first_policy_sha256,
excluded.first_policy_sha256
),
last_policy_sha256 = excluded.last_policy_sha256,
observation_count = docker_repository_query_provenance.observation_count + 1,
fresh_observation_count = docker_repository_query_provenance.fresh_observation_count + 1,
updated_at = excluded.updated_at''',
(
source, query, queue_id, page_observed_at, page_observed_at,
search_rank, search_rank, search_rank,
observation['cycle_id'], observation['cycle_id'], page_id, page_id,
observation['policy_sha256'], observation['policy_sha256'], now, now,
),
)
self.conn.execute(
'''INSERT INTO docker_repository_query_observations(
page_id, repository_queue_id, source, query,
search_rank, observed_at
) VALUES (?, ?, ?, ?, ?, ?)''',
(page_id, queue_id, source, query, search_rank, page_observed_at),
)
observation_inserted_count += 1
pass_pages = self.conn.execute(
'''SELECT query, query_ordinal, page_number, query_complete
FROM docker_discovery_pages WHERE pass_id = ?
ORDER BY query_ordinal, page_number''',
(pass_row['id'],),
).fetchall()
expected_query_count = int(pass_row['expected_query_count'])
pages_by_query = {}
names_by_query = {}
terminals_by_query = {}
for pass_page in pass_pages:
ordinal = int(pass_page['query_ordinal'])
if not 0 <= ordinal < expected_query_count:
raise RuntimeError('DockerHub discovery pass ordinal is outside its bound')
names_by_query.setdefault(ordinal, set()).add(str(pass_page['query']))
pages_by_query.setdefault(ordinal, set()).add(int(pass_page['page_number']))
if int(pass_page['query_complete']):
terminals_by_query.setdefault(ordinal, set()).add(
int(pass_page['page_number'])
)
if any(len(names) != 1 for names in names_by_query.values()):
raise RuntimeError('DockerHub discovery pass query identity conflicts')
complete_ordinals = set()
for ordinal, terminals in terminals_by_query.items():
pages = pages_by_query.get(ordinal, set())
if any(len({page for page in pages if page <= end}) == end for end in terminals):
complete_ordinals.add(ordinal)
completed_query_count = len(complete_ordinals)
next_pass_state = (
'complete'
if completed_query_count == expected_query_count
else 'collecting'
)
previous_pass_state = str(pass_row['state'])
if previous_pass_state == 'complete' and next_pass_state != 'complete':
raise RuntimeError('DockerHub discovery complete pass lost query evidence')
became_complete = (
previous_pass_state == 'collecting' and next_pass_state == 'complete'
)
cursor = self.conn.execute(
'''UPDATE docker_discovery_passes
SET completed_query_count = ?, state = ?,
completed_at = CASE WHEN ? = 'complete'
THEN COALESCE(completed_at, ?) ELSE NULL END,
updated_at = ?
WHERE id = ? AND state IN ('collecting','complete')''',
(
completed_query_count, next_pass_state, next_pass_state,
now, now, pass_row['id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError('DockerHub discovery pass completion changed')
if became_complete and str(pass_row['pass_kind']) == 'deep':
completed_rows = self.conn.execute(
'''SELECT observation.source, observation.query,
observation.repository_queue_id,
COUNT(*) AS observation_count
FROM docker_repository_query_observations observation
JOIN docker_discovery_pages page ON page.id = observation.page_id
WHERE page.pass_id = ?
GROUP BY observation.source, observation.query,
observation.repository_queue_id''',
(pass_row['id'],),
).fetchall()
for completed_row in completed_rows:
cursor = self.conn.execute(
'''UPDATE docker_repository_query_provenance
SET fresh_complete_observation_count =
fresh_complete_observation_count + ?,
fresh_coverage_eligible = 1,
updated_at = ?
WHERE source = ? AND query = ?
AND repository_queue_id = ?''',
(
int(completed_row['observation_count']), now,
completed_row['source'], completed_row['query'],
completed_row['repository_queue_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError('DockerHub discovery coverage evidence changed')
query_complete = observation['query_ordinal'] in complete_ordinals
pass_complete = next_pass_state == 'complete'
dynamic_hold_count = self._hold_new_docker_depth_repositories_locked(
source, query, new_queue_ids, observation, now,
experiment=page_experiment, authority=page_authority,
)
retry_progress_count = 0
retry_completed_count = 0
if fence and complete:
if page_id is None:
cursor = self.conn.execute(
'''DELETE FROM discovery_retry_queue
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(fence[0], fence[1], fence[2], now),
)
else:
# Keep the inactive work row because admitted pages reference its identity.
cursor = self.conn.execute(
'''UPDATE discovery_retry_queue SET status = 'held',
available_after = NULL, last_error_category = NULL,
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
lease_expires_at = NULL, held_at = ?, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(now, now, fence[0], fence[1], fence[2], now),
)
retry_completed_count = int(cursor.rowcount or 0)
if retry_completed_count != 1:
raise DiscoveryRetryLeaseError('discovery retry completion lost its lease fence')
elif fence and next_page is not None:
cursor = self.conn.execute(
'''UPDATE discovery_retry_queue SET next_page = ?, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(next_page, now, fence[0], fence[1], fence[2], now),
)
retry_progress_count = int(cursor.rowcount or 0)
if retry_progress_count != 1:
raise DiscoveryRetryLeaseError('discovery retry progress lost its lease fence')
self.conn.commit()
return {
'attempted_count': attempted_count,
'normalized_count': len(normalized),
'duplicate_count': attempted_count - len(normalized),
'preexisting_count': len(preexisting),
'inserted_count': inserted_count,
'dynamic_hold_count': dynamic_hold_count,
'retry_progress_count': retry_progress_count,
'retry_completed_count': retry_completed_count,
'normalized_repositories': frozenset(normalized),
'preexisting_repositories': frozenset(preexisting),
'pass_id': int(pass_row['id']) if pass_row is not None else None,
'page_id': page_id,
'page_inserted_count': page_inserted_count,
'observation_inserted_count': observation_inserted_count,
'completed_query_count': completed_query_count,
'query_complete': query_complete,
'pass_complete': pass_complete,
}
except Exception:
self.conn.rollback()
raise
def dockerhub_discovery_generation_complete(
self, source, collection_generation, ordered_query_hash,
expected_query_count, policy_sha256,
):
"""Return durable authority for the generation's first complete deep pass."""
if not self.conn:
raise RuntimeError('database connection is unavailable')
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
source = _validated_discovery_retry_source(source)
collection_generation = str(collection_generation or '')
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
raise ValueError('DockerHub collection generation authority conflicts')
ordered_query_hash = str(ordered_query_hash or '')
if not re.fullmatch(r'[a-f0-9]{64}', ordered_query_hash):
raise ValueError('DockerHub ordered-query generation authority is invalid')
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
if isinstance(expected_query_count, bool):
raise ValueError('DockerHub generation query count is invalid')
try:
expected_query_count = int(expected_query_count)
except (TypeError, ValueError, OverflowError):
raise ValueError('DockerHub generation query count is invalid') from None
if not 1 <= expected_query_count <= 1000:
raise ValueError('DockerHub generation query count is invalid')
row = self.conn.execute(
'''SELECT 1 FROM docker_discovery_passes
WHERE source = ? AND pass_kind = 'deep'
AND collection_generation = ? AND policy_sha256 = ?
AND ordered_queries_sha256 = ? AND expected_query_count = ?
AND completed_query_count = expected_query_count
AND state = 'complete' AND completed_at IS NOT NULL
AND (
SELECT COUNT(DISTINCT terminal.query_ordinal)
FROM docker_discovery_pages terminal
WHERE terminal.pass_id = docker_discovery_passes.id
AND terminal.query_complete = 1
AND terminal.total_count IS NOT NULL
AND terminal.query_ordinal >= 0
AND terminal.query_ordinal < expected_query_count
AND terminal.total_count >= terminal.result_count
AND (
SELECT COUNT(DISTINCT page.page_number)
FROM docker_discovery_pages page
WHERE page.pass_id = terminal.pass_id
AND page.query_ordinal = terminal.query_ordinal
AND page.page_number <= terminal.page_number
) = terminal.page_number
) = expected_query_count
ORDER BY id DESC LIMIT 1''',
(
source, collection_generation, policy_sha256,
ordered_query_hash, expected_query_count,
),
).fetchone()
self.conn.commit()
return bool(row)
def dockerhub_discovery_coverage_summary(
self, source, ordered_queries, ordered_query_hash,
configured_query_policies, required_repository_count=10,
collection_generation=None,
):
"""Return a bounded, read-only gate over complete deep-pass evidence."""
if not self.conn:
raise RuntimeError('database connection is unavailable')
source = _validated_discovery_retry_source(source)
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
collection_generation = str(
collection_generation or DOCKER_DEPTH_COLLECTION_GENERATION
)
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
raise ValueError('DockerHub collection generation authority conflicts')
if isinstance(ordered_queries, (str, bytes)):
raise ValueError('DockerHub discovery ordered queries are invalid')
try:
queries = tuple(ordered_queries)
except TypeError:
raise ValueError('DockerHub discovery ordered queries are invalid') from None
if (
not 1 <= len(queries) <= DISCOVERY_RETRY_MAX_ALLOWLIST
or len(set(queries)) != len(queries)
):
raise ValueError('DockerHub discovery ordered queries are invalid')
queries = tuple(_validated_discovery_retry_query(query) for query in queries)
ordered_query_hash = str(ordered_query_hash or '')
expected_hash = hashlib.sha256(json.dumps(
list(queries), ensure_ascii=True, allow_nan=False, sort_keys=True,
separators=(',', ':'),
).encode('utf-8')).hexdigest()
if ordered_query_hash != expected_hash:
raise ValueError('DockerHub discovery ordered-query hash conflicts')
if isinstance(required_repository_count, bool):
raise ValueError('DockerHub discovery repository requirement is invalid')
try:
required_repository_count = int(required_repository_count)
except (TypeError, ValueError, OverflowError):
raise ValueError('DockerHub discovery repository requirement is invalid') from None
if not 1 <= required_repository_count <= 39:
raise ValueError('DockerHub discovery repository requirement is invalid')
if not isinstance(configured_query_policies, dict):
raise ValueError('DockerHub discovery query policies are invalid')
policy_values = {
query: (
value.get('policy_sha256') if isinstance(value, dict) else value
)
for query, value in configured_query_policies.items()
}
policies = dict(_discovery_retry_allowlist(policy_values))
if set(policies) != set(queries):
raise ValueError('DockerHub discovery query policies do not match ordered queries')
query_summaries = []
try:
for query_ordinal, query in enumerate(queries):
rows = self.conn.execute(
'''SELECT provenance.repository_queue_id,
MIN(observation.search_rank) AS best_search_rank
FROM docker_repository_query_provenance provenance
JOIN docker_repository_query_observations observation
ON observation.source = provenance.source
AND observation.query = provenance.query
AND observation.repository_queue_id = provenance.repository_queue_id
JOIN docker_discovery_pages page ON page.id = observation.page_id
JOIN docker_discovery_passes discovery_pass
ON discovery_pass.id = page.pass_id
JOIN target_queue queue
ON queue.id = provenance.repository_queue_id
WHERE provenance.source = ? AND provenance.query = ?
AND provenance.provenance_kind = 'fresh_page'
AND provenance.fresh_coverage_eligible = 1
AND provenance.fresh_complete_observation_count > 0
AND page.query_ordinal = ? AND page.query = ?
AND discovery_pass.source = ?
AND discovery_pass.pass_kind = 'deep'
AND discovery_pass.collection_generation = ?
AND discovery_pass.policy_sha256 = ?
AND discovery_pass.ordered_queries_sha256 = ?
AND discovery_pass.expected_query_count = ?
AND discovery_pass.state = 'complete'
AND queue.source = ? AND queue.platform = 'docker'
AND queue.status IN ('pending','deferred')
AND queue.target_scan_id IS NULL
AND queue.target NOT LIKE '%@%'
AND queue.normalized_target NOT LIKE '%@%'
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
AND queue.claim_batch IS NULL AND queue.leased_at IS NULL
AND queue.lease_expires_at IS NULL
AND queue.current_result_reservation_id IS NULL
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
AND COALESCE(queue.resolver_state, '') <> 'resolving'
AND NOT EXISTS (
SELECT 1 FROM target_scans scan
WHERE scan.queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1 FROM target_queue_policy_events policy_event
WHERE policy_event.queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1 FROM result_reservations reservation
WHERE reservation.queue_id = queue.id
AND reservation.state IN (
'scanning','ready','ingesting','db_committed'
)
)
AND NOT EXISTS (
SELECT 1
FROM result_reservations reservation
JOIN pipeline_quarantine quarantine
ON quarantine.reservation_id = reservation.id
WHERE reservation.queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1 FROM docker_image_manifests manifest
WHERE manifest.target_queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1
FROM docker_image_blob_coverage coverage
JOIN docker_content_blobs blob
ON blob.digest = coverage.blob_digest
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
WHERE coverage.queue_id = queue.id
AND blob.state IN ('leased','submitted')
)
GROUP BY provenance.repository_queue_id
ORDER BY MIN(observation.search_rank), provenance.repository_queue_id
LIMIT ?''',
(
source, query, query_ordinal, query, source,
collection_generation, policies[query],
ordered_query_hash, len(queries), source,
required_repository_count,
),
).fetchall()
eligible_count = len(rows)
query_summaries.append({
'query_ordinal': query_ordinal,
'query': query,
'policy_sha256': policies[query],
'eligible_repository_count': eligible_count,
'required_repository_count': required_repository_count,
'covered': eligible_count >= required_repository_count,
})
covered_query_count = sum(
int(item['covered']) for item in query_summaries
)
ready = covered_query_count == len(queries)
self.conn.commit()
return {
'source': source,
'ordered_query_hash': ordered_query_hash,
'query_count': len(queries),
'required_repository_count': required_repository_count,
'covered_query_count': covered_query_count,
'minimum_eligible_repository_count': min(
item['eligible_repository_count'] for item in query_summaries
),
'planning_allowed': ready,
'state': 'coverage_complete' if ready else 'collecting',
'counts_capped_at_requirement': True,
'queries': query_summaries,
}
except Exception:
self.conn.rollback()
raise
def enqueue_discovery_retry(
self, source, query, policy_sha256, pass_kind, work_kind,
page_start=1, page_end=None, available_after=None, error_category=None,
observation=None,
):
if not self.conn:
raise RuntimeError('database connection is unavailable')
source = _validated_discovery_retry_source(source)
query = _validated_discovery_retry_query(query)
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
pass_kind = str(pass_kind or '')
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
raise ValueError('discovery retry pass kind is invalid')
if page_end is None:
page_end = DISCOVERY_RETRY_MAX_PAGE if work_kind == 'query' else page_start
work_kind, page_start, page_end = _validated_discovery_retry_pages(
work_kind, page_start, page_end,
)
observation = _validated_docker_discovery_observation(
observation, source, query,
)
if observation is not None and (
observation['policy_sha256'] != policy_sha256
or observation['pass_kind'] != pass_kind
or observation['page_number'] != page_start
):
raise ValueError('discovery retry observation identity conflicts')
error_category = _validated_discovery_retry_error_category(error_category)
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
available_after = _validated_discovery_retry_time(available_after, now_dt)
pass_row = None
source_cycle_id = observation['cycle_id'] if observation is not None else None
if source_cycle_id is not None:
cycle_row = self.conn.execute(
'''SELECT id FROM source_cycles
WHERE id = ? AND source = ? AND query = ?''',
(source_cycle_id, source, query),
).fetchone()
if not cycle_row:
raise RuntimeError('discovery retry source cycle identity conflicts')
if observation is not None:
pass_row = self._docker_discovery_pass_locked(
source, query, observation, now,
)
work_key = discovery_retry_work_key(
source, query, policy_sha256, pass_kind, work_kind,
page_start, page_end,
pass_id=pass_row['id'] if pass_row is not None else None,
)
inserted = self.conn.execute(
'''INSERT INTO discovery_retry_queue (
work_key, source, query, source_cycle_id, policy_sha256,
pass_kind, work_kind,
page_start, page_end, next_page, status, attempts,
available_after, last_error_category, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?)
ON CONFLICT(work_key) DO NOTHING
RETURNING id, status, attempts, next_page''',
(
work_key, source, query, source_cycle_id, policy_sha256,
pass_kind, work_kind,
page_start, page_end, page_start, available_after, error_category,
now, now,
),
).fetchone()
inserted_count = int(bool(inserted))
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
row = inserted or self.conn.execute(
f'''SELECT id, work_key, source, query, policy_sha256, pass_kind,
work_kind, page_start, page_end, next_page, status, attempts,
available_after, source_cycle_id
FROM discovery_retry_queue WHERE work_key = ?{lock_suffix}''',
(work_key,),
).fetchone()
if not row:
raise RuntimeError('discovery retry enqueue lost its durable row')
if not inserted and (
str(row['source']) != source
or str(row['query']) != query
or str(row['policy_sha256']) != policy_sha256
or str(row['pass_kind']) != pass_kind
or str(row['work_kind']) != work_kind
or int(row['page_start']) != page_start
or int(row['page_end']) != page_end
):
raise RuntimeError('discovery retry work-key identity conflict')
if not inserted and row['status'] == 'held':
self.conn.execute(
'''UPDATE discovery_retry_queue SET status = 'pending', next_page = page_start,
available_after = ?, last_error_category = ?, held_at = NULL,
updated_at = ? WHERE id = ? AND status = 'held' ''',
(available_after, error_category, now, row['id']),
)
elif not inserted and row['status'] == 'pending':
self.conn.execute(
'''UPDATE discovery_retry_queue SET
available_after = CASE
WHEN available_after IS NULL OR CAST(? AS TEXT) IS NULL THEN NULL
WHEN available_after <= ? THEN available_after ELSE ? END,
last_error_category = COALESCE(?, last_error_category),
updated_at = ?
WHERE id = ? AND status = 'pending' ''',
(
available_after, available_after, available_after,
error_category, now, row['id'],
),
)
row = self.conn.execute(
'''SELECT id, status, attempts, page_start, page_end, next_page,
source_cycle_id
FROM discovery_retry_queue WHERE work_key = ?''',
(work_key,),
).fetchone()
if not row:
raise RuntimeError('discovery retry enqueue could not verify durability')
self.conn.commit()
return {
'id': int(row['id']),
'work_key': work_key,
'inserted_count': inserted_count,
'coalesced_count': 1 - inserted_count,
'status': str(row['status']),
'attempts': int(row['attempts']),
'page_start': int(row['page_start']),
'page_end': int(row['page_end']),
'next_page': int(row['next_page']),
'source_cycle_id': (
int(row['source_cycle_id'])
if row['source_cycle_id'] is not None else None
),
'pass_id': int(pass_row['id']) if pass_row is not None else None,
}
except Exception:
self.conn.rollback()
raise
def claim_discovery_retries(
self, source, configured_query_policies, lease_owner, limit=1,
lease_seconds=300,
):
if not self.conn:
raise RuntimeError('database connection is unavailable')
source = _validated_discovery_retry_source(source)
allowed = _discovery_retry_allowlist(configured_query_policies)
owner = str(lease_owner or '')
if (
not 1 <= len(owner) <= 128
or any(ord(character) < 32 or ord(character) == 127 for character in owner)
):
raise ValueError('discovery retry lease owner is invalid')
if isinstance(limit, bool) or isinstance(lease_seconds, bool):
raise ValueError('discovery retry claim bounds are invalid')
try:
limit = int(limit)
lease_seconds = int(lease_seconds)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry claim bounds are invalid') from None
if not 1 <= limit <= DISCOVERY_RETRY_MAX_CLAIM or not 1 <= lease_seconds <= 86400:
raise ValueError('discovery retry claim bounds are invalid')
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
state = self._locked_runtime_control_state(shared=True)
if state['effective_discovery_paused']:
self.conn.commit()
return []
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
lease_expires_at = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
timespec='seconds',
)
pair_sql = ' OR '.join(
'(query = ? AND policy_sha256 = ?)' for _ in allowed
) or '0 = 1'
pair_params = tuple(value for pair in allowed for value in pair)
query_sql = ','.join('?' for _ in allowed)
query_params = tuple(query for query, _ in allowed)
category_sql = (
f"CASE WHEN query IN ({query_sql}) THEN 'policy_mismatch' ELSE 'query_removed' END"
if allowed else "'query_removed'"
)
self.conn.execute(
f'''UPDATE discovery_retry_queue SET status = 'held',
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
lease_expires_at = NULL, available_after = NULL,
held_at = ?, last_error_category = {category_sql}, updated_at = ?
WHERE source = ?
AND (status = 'pending' OR (status = 'leased' AND lease_expires_at <= ?))
AND NOT ({pair_sql})''',
(now, *query_params, now, source, now, *pair_params),
)
if not allowed:
self.conn.commit()
return []
lock_suffix = ' FOR UPDATE SKIP LOCKED' if self.conn.is_postgres else ''
rows = self.conn.execute(
f'''SELECT id, work_key, source, query, policy_sha256, pass_kind,
work_kind, page_start, page_end, next_page, attempts,
last_error_category, source_cycle_id
FROM discovery_retry_queue
WHERE source = ? AND ({pair_sql})
AND (
(status = 'pending' AND (available_after IS NULL OR available_after <= ?))
OR (status = 'leased' AND lease_expires_at IS NOT NULL
AND lease_expires_at <= ?)
)
ORDER BY COALESCE(available_after, lease_expires_at, created_at), id
LIMIT ?{lock_suffix}''',
(source, *pair_params, now, now, limit),
).fetchall()
claimed = []
for row in rows:
token = secrets.token_urlsafe(32)
updated = self.conn.execute(
'''UPDATE discovery_retry_queue SET status = 'leased',
lease_owner = ?, lease_token = ?, leased_at = ?,
lease_expires_at = ?, held_at = NULL,
attempts = CASE WHEN attempts < ? THEN attempts + 1 ELSE attempts END,
updated_at = ?
WHERE id = ? AND source = ? AND (
(status = 'pending' AND (available_after IS NULL OR available_after <= ?))
OR (status = 'leased' AND lease_expires_at IS NOT NULL
AND lease_expires_at <= ?)
) RETURNING attempts''',
(
owner, token, now, lease_expires_at, DISCOVERY_RETRY_MAX_ATTEMPTS,
now, row['id'], source, now, now,
),
).fetchone()
if not updated:
raise DiscoveryRetryLeaseError('discovery retry claim lost its selected row')
claimed.append({
'id': int(row['id']),
'work_key': str(row['work_key']),
'source': str(row['source']),
'query': str(row['query']),
'policy_sha256': str(row['policy_sha256']),
'pass_kind': str(row['pass_kind']),
'work_kind': str(row['work_kind']),
'page_start': int(row['page_start']),
'page_end': int(row['page_end']),
'next_page': int(row['next_page']),
'attempts': int(updated['attempts']),
'lease_owner': owner,
'lease_token': token,
'lease_expires_at': lease_expires_at,
'last_error_category': row['last_error_category'],
'source_cycle_id': (
int(row['source_cycle_id'])
if row['source_cycle_id'] is not None else None
),
})
self.conn.commit()
return claimed
except Exception:
self.conn.rollback()
raise
def finish_discovery_retry(self, retry_id, lease_owner, lease_token):
if not self.conn:
raise RuntimeError('database connection is unavailable')
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
retry_id, lease_owner, lease_token,
)
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''DELETE FROM discovery_retry_queue
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(retry_id, lease_owner, lease_token, now),
)
if int(cursor.rowcount or 0) != 1:
raise DiscoveryRetryLeaseError('discovery retry completion lost its lease fence')
self.conn.commit()
return {'id': retry_id, 'deleted_count': 1}
except Exception:
self.conn.rollback()
raise
def renew_discovery_retry_lease(
self, retry_id, lease_owner, lease_token, lease_seconds=300,
):
if not self.conn:
raise RuntimeError('database connection is unavailable')
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
retry_id, lease_owner, lease_token,
)
if isinstance(lease_seconds, bool):
raise ValueError('discovery retry lease duration is invalid')
try:
lease_seconds = int(lease_seconds)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry lease duration is invalid') from None
if not 1 <= lease_seconds <= 86400:
raise ValueError('discovery retry lease duration is invalid')
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
lease_expires_at = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
timespec='seconds',
)
try:
cursor = self.conn.execute(
'''UPDATE discovery_retry_queue SET lease_expires_at = ?, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(
lease_expires_at, now, retry_id, lease_owner, lease_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise DiscoveryRetryLeaseError('discovery retry renewal lost its lease fence')
self.conn.commit()
return {
'id': retry_id,
'status': 'leased',
'lease_expires_at': lease_expires_at,
}
except Exception:
self.conn.rollback()
raise
def update_discovery_retry(
self, retry_id, lease_owner, lease_token, error_category,
retry_at=None, refund_attempt=False, next_page=None,
):
if not self.conn:
raise RuntimeError('database connection is unavailable')
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
retry_id, lease_owner, lease_token,
)
error_category = _validated_discovery_retry_error_category(
error_category, required=True,
)
if not isinstance(refund_attempt, bool):
raise ValueError('discovery retry refund flag is invalid')
if refund_attempt and retry_at is None:
raise ValueError('discovery retry attempt refund requires a trusted retry time')
if next_page is not None:
if isinstance(next_page, bool):
raise ValueError('discovery retry next page is invalid')
try:
next_page = int(next_page)
except (TypeError, ValueError, OverflowError):
raise ValueError('discovery retry next page is invalid') from None
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
row = self.conn.execute(
f'''SELECT attempts, page_start, page_end, next_page
FROM discovery_retry_queue
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?{lock_suffix}''',
(retry_id, lease_owner, lease_token, now),
).fetchone()
if not row:
raise DiscoveryRetryLeaseError('discovery retry update lost its lease fence')
if next_page is None:
next_page = int(row['next_page'])
elif not (
int(row['next_page']) <= next_page <= int(row['page_end'])
and 1 <= next_page <= DISCOVERY_RETRY_MAX_PAGE
):
raise ValueError('discovery retry next page is outside its durable range')
retry_due = _validated_discovery_retry_time(retry_at, now_dt)
attempts = int(row['attempts'])
if retry_due is None:
exponent = min(20, max(0, attempts - 1))
delay = min(
DISCOVERY_RETRY_MAX_DELAY_SEC,
DISCOVERY_RETRY_BASE_DELAY_SEC * (2 ** exponent),
)
retry_due = (now_dt + timedelta(seconds=delay)).isoformat(timespec='seconds')
next_attempts = max(0, attempts - 1) if refund_attempt else attempts
cursor = self.conn.execute(
'''UPDATE discovery_retry_queue SET status = 'pending', attempts = ?,
next_page = ?, available_after = ?, last_error_category = ?,
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
lease_expires_at = NULL, held_at = NULL, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(
next_attempts, next_page, retry_due, error_category, now,
retry_id, lease_owner, lease_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise DiscoveryRetryLeaseError('discovery retry update lost its lease fence')
self.conn.commit()
return {
'id': retry_id,
'status': 'pending',
'attempts': next_attempts,
'next_page': next_page,
'available_after': retry_due,
'last_error_category': error_category,
}
except Exception:
self.conn.rollback()
raise
def hold_discovery_retry(
self, retry_id, lease_owner, lease_token, error_category='policy_mismatch',
):
if not self.conn:
raise RuntimeError('database connection is unavailable')
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
retry_id, lease_owner, lease_token,
)
error_category = _validated_discovery_retry_error_category(
error_category, required=True,
)
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE discovery_retry_queue SET status = 'held',
available_after = NULL, last_error_category = ?, held_at = ?,
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
lease_expires_at = NULL, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(
error_category, now, now, retry_id, lease_owner, lease_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise DiscoveryRetryLeaseError('discovery retry hold lost its lease fence')
self.conn.commit()
return {
'id': retry_id,
'status': 'held',
'last_error_category': error_category,
}
except Exception:
self.conn.rollback()
raise
def enqueue_targets(
self, source, platform, query, targets, requeue_done=False,
unresolved_targets=None, *, discovery_admission=False,
):
targets = list(targets or [])
unresolved_targets = list(unresolved_targets or [])
if not isinstance(discovery_admission, bool):
raise ValueError('discovery admission flag must be boolean')
if not self.conn or (not targets and not unresolved_targets):
return 0
def op():
if discovery_admission:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
self._require_discovery_admission_locked()
now = utc_now_iso()
count = 0
for target in targets or []:
normalized = normalize_target(target, platform)
if requeue_done:
reset_status = f"target_queue.status = 'done' OR ({ADMIN_DISCARDED_QUEUE_SQL})"
status_expr = f"CASE WHEN {reset_status} THEN 'pending' ELSE target_queue.status END"
available_after_expr = f"CASE WHEN {reset_status} THEN NULL ELSE target_queue.available_after END"
attempts_expr = f"CASE WHEN {reset_status} THEN 0 ELSE target_queue.attempts END"
completed_at_expr = f"CASE WHEN {reset_status} THEN NULL ELSE target_queue.completed_at END"
conflict_status_where = reset_status
else:
status_expr = f"CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 'pending' WHEN target_queue.status IN ('done', 'failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status WHEN target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL THEN 'deferred' ELSE 'pending' END"
available_after_expr = f"CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL WHEN target_queue.status IN ('pending', 'deferred', 'cold') THEN target_queue.available_after ELSE NULL END"
attempts_expr = f'CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 0 ELSE target_queue.attempts END'
completed_at_expr = f'CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL ELSE target_queue.completed_at END'
conflict_status_where = f"target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL OR ({ADMIN_DISCARDED_QUEUE_SQL})"
reset_for_discovery = f"({ADMIN_DISCARDED_QUEUE_SQL}) OR (target_queue.status = 'done' AND {str(bool(requeue_done)).upper()})"
self.conn.execute(
f'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
ON CONFLICT(source, normalized_target) DO UPDATE SET
target = excluded.target,
platform = excluded.platform,
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN excluded.query ELSE COALESCE(target_queue.query, excluded.query) END,
status = {status_expr},
lease_owner = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_owner END,
lease_token = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_token END,
claim_batch = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.claim_batch END,
leased_at = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.leased_at END,
lease_expires_at = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_expires_at END,
available_after = {available_after_expr},
attempts = {attempts_expr},
completed_at = {completed_at_expr},
last_error = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.last_error END,
resolver_state = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_state END,
resolver_due_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_due_at END,
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN 0 ELSE target_queue.resolver_attempts END,
resolver_token = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN NULL ELSE target_queue.resolver_token END,
updated_at = excluded.updated_at
WHERE ({conflict_status_where})
OR target_queue.target <> excluded.target
OR target_queue.platform <> excluded.platform
OR (target_queue.query IS NULL AND excluded.query IS NOT NULL)''',
(source, platform, query, target, normalized, now, now),
)
count += 1
for target in unresolved_targets:
normalized = normalize_target(target, platform)
resolver_due = datetime.fromtimestamp(
time.time() + max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)), timezone.utc
).isoformat(timespec='seconds')
self.conn.execute(
f'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status,
available_after, last_error, resolver_state, resolver_due_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'deferred', ?, 'Docker tag resolution unresolved', 'pending', ?, ?, ?)
ON CONFLICT(source, normalized_target) DO UPDATE SET
status = 'deferred', available_after = excluded.available_after,
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN excluded.query ELSE target_queue.query END,
target = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN excluded.target ELSE target_queue.target END,
resolver_state = 'pending', resolver_due_at = excluded.resolver_due_at,
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
THEN 0 ELSE target_queue.resolver_attempts END,
resolver_token = NULL, completed_at = NULL,
last_error = excluded.last_error, updated_at = excluded.updated_at
WHERE target_queue.source = excluded.source
AND target_queue.platform = excluded.platform
AND excluded.platform = 'docker'
AND (
({ADMIN_DISCARDED_QUEUE_SQL})
OR (target_queue.resolver_state IS NULL AND target_queue.status IN ('pending', 'deferred'))
OR (target_queue.resolver_state = 'resolving' AND target_queue.status = 'deferred'
AND target_queue.resolver_due_at IS NOT NULL AND target_queue.resolver_due_at <= ?)
)''',
(source, platform, query, target, normalized, resolver_due, resolver_due, now, now, now),
)
count += 1
self.conn.commit()
return count
return self._safe('enqueue_targets', op, 0)
def observe_discovered_targets(
self, source, platform, query, discoveries, rescan_limit=0, cooldown_seconds=0,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('updated-target observation requires PostgreSQL')
normalized_records = {}
attempted_count = 0
for discovery in discoveries or []:
attempted_count += 1
if isinstance(discovery, dict):
target = str(discovery.get('target') or '').strip()
remote_value = discovery.get('remote_modified_at')
else:
target = str(discovery or '').strip()
remote_value = None
if not target:
continue
if source == 'huggingface':
try:
target = normalize_huggingface_space_id(target)
except (TypeError, ValueError):
continue
normalized = normalize_target(target, platform)
remote_time = parse_time(remote_value)
remote_text = remote_time.isoformat(timespec='seconds') if remote_time else None
current = normalized_records.get(normalized)
if current is None:
normalized_records[normalized] = {
'target': target,
'remote_modified_at': remote_text,
}
elif remote_text and (
not current['remote_modified_at']
or remote_text > current['remote_modified_at']
):
current['target'] = target
current['remote_modified_at'] = remote_text
if not normalized_records:
return {
'attempted_count': attempted_count,
'queued_new_count': 0,
'queued_updated_count': 0,
}
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
cooldown_cutoff = (
now_dt - timedelta(seconds=max(0, int(cooldown_seconds or 0)))
).isoformat(timespec='seconds')
observed_existing_ids = []
queued_new = 0
queued_updated = 0
try:
self._require_discovery_admission_locked()
for normalized in sorted(normalized_records):
record = normalized_records[normalized]
inserted = self.conn.execute(
'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status,
remote_modified_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, ?)
ON CONFLICT(source, normalized_target) DO NOTHING
RETURNING id''',
(
source, platform, query, record['target'], normalized,
record['remote_modified_at'], now, now,
),
).fetchone()
if inserted:
queued_new += 1
continue
row = self.conn.execute(
'''SELECT id, target, platform, query, remote_modified_at,
status, last_error
FROM target_queue
WHERE source = ? AND normalized_target = ?
FOR UPDATE''',
(source, normalized),
).fetchone()
if not row:
raise RuntimeError('discovered target disappeared during observation')
remote_text = record['remote_modified_at']
if (
str(row['status']) == 'quarantined'
and str(row['last_error'] or '') == ADMIN_DISCARDED_QUEUE_REASON
):
cursor = self.conn.execute(
'''UPDATE target_queue SET target = ?, platform = ?, query = ?,
status = 'pending', attempts = 0, available_after = NULL,
last_error = NULL, completed_at = NULL,
lease_owner = NULL, lease_token = NULL,
claim_batch = NULL, leased_at = NULL,
lease_expires_at = NULL, resolver_state = NULL,
resolver_due_at = NULL, resolver_attempts = 0,
resolver_token = NULL, remote_modified_at = ?,
updated_at = ?
WHERE id = ? AND status = 'quarantined' AND last_error = ?''',
(
record['target'], platform, query, remote_text, now,
row['id'], ADMIN_DISCARDED_QUEUE_REASON,
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError('discarded target reactivation authority changed')
queued_updated += 1
continue
remote_advanced = bool(
remote_text
and (
not row['remote_modified_at']
or remote_text > str(row['remote_modified_at'])
)
)
if (
remote_advanced
or str(row['target']) != record['target']
or str(row['platform']) != str(platform)
or (row['query'] is None and query is not None)
):
next_remote = remote_text if remote_advanced else row['remote_modified_at']
self.conn.execute(
'''UPDATE target_queue SET
target = ?, platform = ?, query = COALESCE(query, ?),
remote_modified_at = ?,
updated_at = ?
WHERE id = ?''',
(
record['target'], platform, query, next_remote, now, row['id'],
),
)
if remote_text:
observed_existing_ids.append(int(row['id']))
limit = max(0, int(rescan_limit or 0))
if limit and observed_existing_ids:
placeholders = ','.join('?' for _ in observed_existing_ids)
eligible = self.conn.execute(
f'''SELECT id FROM target_queue
WHERE id IN ({placeholders})
AND source = ? AND platform = ? AND status = 'done'
AND remote_modified_at IS NOT NULL
AND completed_at IS NOT NULL AND completed_at <= ?
AND (
(scan_remote_modified_at IS NULL AND remote_modified_at > completed_at)
OR (scan_remote_modified_at IS NOT NULL AND remote_modified_at > scan_remote_modified_at)
)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND resolver_token IS NULL
AND lease_owner IS NULL AND lease_token IS NULL
AND claim_batch IS NULL AND leased_at IS NULL
AND lease_expires_at IS NULL
AND current_result_reservation_id IS NULL
AND claim_event_id IS NULL
ORDER BY remote_modified_at DESC, id
LIMIT ? FOR UPDATE SKIP LOCKED''',
(*observed_existing_ids, source, platform, cooldown_cutoff, limit),
).fetchall()
eligible_ids = [int(row['id']) for row in eligible]
if eligible_ids:
eligible_placeholders = ','.join('?' for _ in eligible_ids)
promoted = self.conn.execute(
f'''UPDATE target_queue SET
status = 'pending', attempts = 0, available_after = NULL,
last_error = NULL, completed_at = NULL,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL,
current_result_reservation_id = NULL, claim_event_id = NULL,
updated_at = ?
WHERE id IN ({eligible_placeholders})
AND status = 'done' AND current_result_reservation_id IS NULL''',
(now, *eligible_ids),
)
promoted_count = int(promoted.rowcount or 0)
if promoted_count != len(eligible_ids):
raise RuntimeError('updated-target promotion authority changed')
queued_updated += promoted_count
self.conn.commit()
return {
'attempted_count': attempted_count,
'queued_new_count': queued_new,
'queued_updated_count': queued_updated,
}
except Exception:
self.conn.rollback()
raise
@staticmethod
def _admission_intent_sha256(values):
encoded = json.dumps(
values, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
return hashlib.sha256(encoded).hexdigest()
def provision_remote_worker_device(
self, user_key, device_key, token_sha256, active_assignment_cap,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker provisioning requires PostgreSQL')
user_key = str(user_key or '').strip()
device_key = str(device_key or '').strip()
token_sha256 = str(token_sha256 or '').strip().lower()
cap = int(active_assignment_cap)
if not 1 <= len(user_key) <= 128 or not 1 <= len(device_key) <= 128:
raise ValueError('remote worker user and device keys must be 1..128 characters')
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
raise ValueError('remote worker token digest must be lowercase SHA-256')
if not 0 <= cap <= 10000:
raise ValueError('remote worker active assignment cap is out of range')
now = utc_now_iso()
try:
self.conn.execute(
'''INSERT INTO remote_worker_users(
user_key, active_assignment_cap, created_at, updated_at
) VALUES (?, ?, ?, ?)
ON CONFLICT(user_key) DO UPDATE SET
active_assignment_cap = excluded.active_assignment_cap,
updated_at = excluded.updated_at''',
(user_key, cap, now, now),
)
user = self.conn.execute(
'SELECT * FROM remote_worker_users WHERE user_key = ? FOR UPDATE',
(user_key,),
).fetchone()
self.conn.execute(
'''INSERT INTO remote_worker_devices(
user_id, device_key, token_sha256, created_at, updated_at
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(device_key) DO UPDATE SET
token_sha256 = excluded.token_sha256,
updated_at = excluded.updated_at''',
(user['id'], device_key, token_sha256, now, now),
)
device = self.conn.execute(
'''SELECT id, user_id, device_key, revoked_at, created_at, updated_at
FROM remote_worker_devices WHERE device_key = ?''',
(device_key,),
).fetchone()
if int(device['user_id']) != int(user['id']):
raise ScanEventConflictError(
'remote worker device key already belongs to another user'
)
self.conn.commit()
return {
'user_id': int(user['id']), 'user_key': user_key,
'device_id': int(device['id']), 'device_key': device_key,
'active_assignment_cap': cap,
'revoked': device['revoked_at'] is not None,
}
except Exception:
self.conn.rollback()
raise
def create_remote_worker_user(self, user_key, active_assignment_cap):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker user creation requires PostgreSQL')
user_key = str(user_key or '').strip()
cap = int(active_assignment_cap)
if not 1 <= len(user_key) <= 128:
raise ValueError('remote worker user key must be 1..128 characters')
if not 0 <= cap <= 10000:
raise ValueError('remote worker active assignment cap is out of range')
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''INSERT INTO remote_worker_users(
user_key, active_assignment_cap, created_at, updated_at
) VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING''',
(user_key, cap, now, now),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return None
row = self.conn.execute(
'''SELECT id, user_key, active_assignment_cap, disabled_at
FROM remote_worker_users WHERE user_key = ?''',
(user_key,),
).fetchone()
self.conn.commit()
return {
'user_id': int(row['id']), 'user_key': str(row['user_key']),
'active_assignment_cap': int(row['active_assignment_cap']),
'disabled': row['disabled_at'] is not None,
}
except Exception:
self.conn.rollback()
raise
def set_remote_worker_user_cap(self, user_key, active_assignment_cap):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker user update requires PostgreSQL')
user_key = str(user_key or '').strip()
cap = int(active_assignment_cap)
if not 1 <= len(user_key) <= 128:
raise ValueError('remote worker user key must be 1..128 characters')
if not 0 <= cap <= 10000:
raise ValueError('remote worker active assignment cap is out of range')
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE remote_worker_users SET active_assignment_cap = ?, updated_at = ?
WHERE user_key = ?''',
(cap, now, user_key),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
except Exception:
self.conn.rollback()
raise
def set_remote_worker_user_disabled(self, user_key, disabled=True):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker user update requires PostgreSQL')
user_key = str(user_key or '').strip()
if not 1 <= len(user_key) <= 128:
raise ValueError('remote worker user key must be 1..128 characters')
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE remote_worker_users SET disabled_at = ?, updated_at = ?
WHERE user_key = ?''',
(now if disabled else None, now, user_key),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
except Exception:
self.conn.rollback()
raise
def issue_remote_worker_device(
self, user_key, device_key, token_sha256, *, rotate=False,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker device issuance requires PostgreSQL')
user_key = str(user_key or '').strip()
device_key = str(device_key or '').strip()
token_sha256 = str(token_sha256 or '').strip().lower()
if not 1 <= len(user_key) <= 128 or not 1 <= len(device_key) <= 128:
raise ValueError('remote worker user and device keys must be 1..128 characters')
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
raise ValueError('remote worker token digest must be lowercase SHA-256')
now = utc_now_iso()
try:
user = self.conn.execute(
'''SELECT id, user_key FROM remote_worker_users
WHERE user_key = ? FOR SHARE''',
(user_key,),
).fetchone()
if not user:
self.conn.rollback()
return None
if rotate:
cursor = self.conn.execute(
'''UPDATE remote_worker_devices SET token_sha256 = ?, updated_at = ?
WHERE user_id = ? AND device_key = ?''',
(token_sha256, now, user['id'], device_key),
)
else:
cursor = self.conn.execute(
'''INSERT INTO remote_worker_devices(
user_id, device_key, token_sha256, created_at, updated_at
) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING''',
(user['id'], device_key, token_sha256, now, now),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return None
device = self.conn.execute(
'''SELECT id, user_id, device_key, revoked_at
FROM remote_worker_devices WHERE user_id = ? AND device_key = ?''',
(user['id'], device_key),
).fetchone()
if not device:
raise ScanEventConflictError('remote worker device issuance was not durable')
self.conn.commit()
return {
'user_id': int(user['id']), 'user_key': str(user['user_key']),
'device_id': int(device['id']), 'device_key': str(device['device_key']),
'revoked': device['revoked_at'] is not None,
}
except Exception:
self.conn.rollback()
raise
def admin_remote_worker_snapshot(self, limit=200, filters=None):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker admin snapshot requires PostgreSQL')
limit = int(limit)
if not 1 <= limit <= 500:
raise ValueError('remote worker admin snapshot limit is out of range')
filters, filter_conditions, filter_parameters = _admin_worker_filter_sql(
filters,
)
assignment_outcome = _admin_assignment_outcome_sql()
scan_outcome = _admin_scan_outcome_sql()
assignment_where = ' AND '.join((
"r.assignment_kind = 'remote'", *filter_conditions,
))
try:
users = self.conn.execute(
'''SELECT user_key, active_assignment_cap,
(disabled_at IS NOT NULL) AS disabled
FROM remote_worker_users ORDER BY id DESC LIMIT ?''',
(limit,),
).fetchall()
workers = self.conn.execute(
'''SELECT d.device_key, u.user_key, u.active_assignment_cap,
d.last_contact_at,
(d.revoked_at IS NOT NULL) AS revoked,
activity.current_phases,
activity.latest_progress_age_seconds,
activity.known_reasons,
activity.pending_local_recovery,
activity.active_package_identity,
COUNT(r.id) FILTER (
WHERE r.remote_resolution_kind IS NULL
) AS unfinished_count,
COUNT(r.id) FILTER (
WHERE r.remote_resolution_kind IS NULL
) AS active_slot_count,
COUNT(r.id) FILTER (
WHERE r.remote_resolution_kind = 'bundle_accepted'
) AS completed_count,
COUNT(r.id) FILTER (
WHERE r.remote_resolution_kind = 'prebundle_report'
) AS failed_count,
COUNT(r.id) FILTER (
WHERE r.remote_resolution_kind = 'expired'
) AS expired_count
FROM remote_worker_devices d
JOIN remote_worker_users u ON u.id = d.user_id
LEFT JOIN result_reservations r
ON r.remote_device_id = d.id AND r.assignment_kind = 'remote'
LEFT JOIN LATERAL (
SELECT STRING_AGG(
DISTINCT COALESCE(active.phase, 'legacy/unavailable'),
', ' ORDER BY COALESCE(active.phase, 'legacy/unavailable')
) AS current_phases,
MIN(active.progress_age_seconds)
AS latest_progress_age_seconds,
STRING_AGG(
DISTINCT active.known_reason, ', '
ORDER BY active.known_reason
) FILTER (WHERE active.known_reason IS NOT NULL)
AS known_reasons,
COALESCE(BOOL_OR(
active.pending_local_recovery = 'true'
), FALSE) AS pending_local_recovery,
STRING_AGG(
DISTINCT active.package_identity, ', '
ORDER BY active.package_identity
) FILTER (WHERE active.package_identity IS NOT NULL)
AS active_package_identity
FROM (
SELECT ar.id, lp.phase,
CASE WHEN lp.event_timestamp IS NOT NULL
THEN GREATEST(0, EXTRACT(EPOCH FROM (
CURRENT_TIMESTAMP
- lp.event_timestamp::timestamptz
))::BIGINT) END AS progress_age_seconds,
lp.known_reason, lp.pending_local_recovery,
(
ar.remote_execution_snapshot_json::jsonb #>>
'{compatibility,platform_tag}'
) || ':' || LEFT((
ar.remote_execution_snapshot_json::jsonb #>>
'{compatibility,code_manifest_sha256}'
), 12) AS package_identity
FROM result_reservations ar
LEFT JOIN LATERAL (
SELECT e.phase, e.event_timestamp,
e.event_json::jsonb #>> '{progress,reason}'
AS known_reason,
e.event_json::jsonb #>>
'{progress,pending_local_recovery}'
AS pending_local_recovery
FROM worker_progress_events e
WHERE e.reservation_id = ar.id
ORDER BY e.sequence DESC, e.id DESC LIMIT 1
) lp ON TRUE
WHERE ar.remote_device_id = d.id
AND ar.assignment_kind = 'remote'
AND ar.remote_resolution_kind IS NULL
) active
) activity ON TRUE
GROUP BY d.id, d.device_key, u.user_key,
u.active_assignment_cap, d.last_contact_at, d.revoked_at,
activity.current_phases,
activity.latest_progress_age_seconds,
activity.known_reasons,
activity.pending_local_recovery,
activity.active_package_identity
ORDER BY d.id DESC LIMIT ?''',
(limit,),
).fetchall()
assignments = self.conn.execute(
f'''SELECT r.id AS reservation_id, q.id AS queue_id,
u.user_key, u.active_assignment_cap, d.device_key, r.source,
r.normalized_target AS target,
r.remote_issued_at AS issued_at,
r.remote_expires_at AS assignment_deadline_at,
r.remote_result_upload_body_timeout_seconds,
r.remote_resolved_at AS finished_at,
CASE WHEN r.remote_resolved_at IS NOT NULL
AND r.remote_issued_at IS NOT NULL
THEN GREATEST(0, EXTRACT(EPOCH FROM (
r.remote_resolved_at::timestamptz
- r.remote_issued_at::timestamptz
))::BIGINT)
ELSE NULL END AS duration_seconds,
{assignment_outcome} AS assignment_outcome,
{scan_outcome} AS scan_outcome,
COALESCE(
r.remote_resolution_kind = 'bundle_accepted', FALSE
) AS accepted,
COALESCE(
b.committed_at IS NOT NULL OR b.state IN (
'db_committed', 'acknowledged'
), FALSE
) AS ingested,
r.last_error_code AS assignment_code,
s.id AS target_scan_id, s.error_count AS scan_error_count,
s.first_error_summary, s.skipped_reason,
src.metadata_json::jsonb #>> '{{warnings,0}}'
AS scan_warning_summary,
src.metadata_json::jsonb #>> '{{warning_classes,0}}'
AS scan_warning_class,
COALESCE(dg.diagnostic_count, 0) AS diagnostic_count,
dg.diagnostic_categories, dg.diagnostic_codes,
dg.primary_diagnostic,
p.phase AS active_phase, p.phase_started_at,
p.event_timestamp AS last_progress_at,
p.received_at AS last_progress_received_at,
p.slot_id,
p.scan_deadline_at,
p.known_reason, p.pending_local_recovery,
CASE WHEN p.phase_started_at IS NOT NULL
THEN GREATEST(0, EXTRACT(EPOCH FROM (
COALESCE(
r.remote_resolved_at::timestamptz,
CURRENT_TIMESTAMP
) - p.phase_started_at::timestamptz
))::BIGINT) END AS phase_age_seconds,
CASE WHEN p.event_timestamp IS NOT NULL
THEN GREATEST(0, EXTRACT(EPOCH FROM (
COALESCE(
r.remote_resolved_at::timestamptz,
CURRENT_TIMESTAMP
) - p.event_timestamp::timestamptz
))::BIGINT) END AS last_progress_age_seconds,
CASE WHEN r.remote_expires_at IS NOT NULL
THEN EXTRACT(EPOCH FROM (
r.remote_expires_at::timestamptz - COALESCE(
r.remote_resolved_at::timestamptz,
CURRENT_TIMESTAMP
)
))::BIGINT END AS assignment_remaining_seconds,
CASE WHEN p.scan_deadline_at IS NOT NULL
THEN EXTRACT(EPOCH FROM (
p.scan_deadline_at::timestamptz - COALESCE(
r.remote_resolved_at::timestamptz,
CURRENT_TIMESTAMP
)
))::BIGINT END AS scan_remaining_seconds,
b.state AS ingestion_state,
pj.status AS projection_state,
r.remote_diagnostic_projection_version AS diagnostic_projection_version,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,protocol_version}}' AS protocol_version,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,bundle_format_version}}' AS bundle_format_version,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,platform_tag}}' AS platform_tag,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,code_manifest_sha256}}' AS code_manifest_sha256,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,detector_policy_sha256}}' AS detector_policy_sha256,
r.remote_execution_snapshot_json::jsonb #>>
'{{compatibility,effective_config_sha256}}' AS effective_config_sha256
FROM result_reservations r
JOIN remote_worker_users u ON u.id = r.remote_user_id
JOIN remote_worker_devices d ON d.id = r.remote_device_id
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN result_bundles b ON b.reservation_id = r.id
LEFT JOIN target_scans s ON s.id = b.target_scan_id
LEFT JOIN scan_result_compat src ON src.target_scan_id = s.id
LEFT JOIN projection_jobs pj
ON pj.target_scan_id = s.id AND pj.job_kind = 'scan_event'
LEFT JOIN LATERAL (
SELECT e.phase, e.phase_started_at, e.event_timestamp,
e.received_at, e.slot_id,
NULLIF(e.event_json::jsonb ->> 'scan_deadline_at', '')
AS scan_deadline_at,
e.event_json::jsonb #>> '{{progress,reason}}'
AS known_reason,
e.event_json::jsonb #>>
'{{progress,pending_local_recovery}}'
AS pending_local_recovery
FROM worker_progress_events e
WHERE e.reservation_id = r.id
ORDER BY e.sequence DESC, e.id DESC LIMIT 1
) p ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*) AS diagnostic_count,
STRING_AGG(DISTINCT wd.category, ', ' ORDER BY wd.category)
AS diagnostic_categories,
STRING_AGG(DISTINCT wd.code, ', ' ORDER BY wd.code)
AS diagnostic_codes,
(ARRAY_AGG(
wd.category || '/' || wd.code
ORDER BY CASE wd.category
WHEN 'internal' THEN 0
WHEN 'storage' THEN 1
WHEN 'scanner' THEN 2
WHEN 'timeout' THEN 3
ELSE 4 END,
wd.occurred_at DESC, wd.id DESC
))[1] AS primary_diagnostic
FROM worker_diagnostics wd
WHERE wd.reservation_id = r.id
) dg ON TRUE
WHERE {assignment_where}
ORDER BY r.remote_issued_at DESC, r.id DESC LIMIT ?''',
(*filter_parameters, limit),
).fetchall()
deferred = self.conn.execute(
'''SELECT id AS queue_id, source, normalized_target AS target,
available_after
FROM target_queue WHERE status = 'deferred'
ORDER BY updated_at DESC, id DESC LIMIT ?''',
(limit,),
).fetchall()
self.conn.commit()
assignment_rows = [dict(row) for row in assignments]
deferred_rows = [dict(row) for row in deferred]
for row in (*assignment_rows, *deferred_rows):
row['target'] = _admin_safe_target(row.get('target'))
return {
'filters': filters,
'users': [dict(row) for row in users],
'workers': [dict(row) for row in workers],
'assignments': assignment_rows,
'deferred_queue': deferred_rows,
}
except Exception:
self.conn.rollback()
raise
def admin_worker_diagnostic_groups(
self, limit=200, filters=None, *, occurrence_offset=0,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker diagnostic administration requires PostgreSQL')
limit = int(limit)
occurrence_offset = int(occurrence_offset)
if not 1 <= limit <= 500 or occurrence_offset < 0:
raise ValueError('worker diagnostic administration limit is out of range')
filters, conditions, parameters = _admin_worker_filter_sql(
filters, diagnostic_alias='wd',
)
where = ' AND '.join(("r.assignment_kind = 'remote'", *conditions))
try:
rows = self.conn.execute(
f'''WITH filtered AS (
SELECT wd.id, wd.diagnostic_uid, wd.reservation_id,
wd.target_scan_id, wd.source, wd.phase, wd.kind,
wd.category, wd.code, wd.summary, wd.retryable,
wd.occurred_at, wd.received_at, wd.envelope_json,
u.user_key, d.device_key,
{_admin_assignment_outcome_sql()} AS assignment_outcome,
{_admin_scan_outcome_sql()} AS scan_outcome,
COALESCE(
NULLIF(
wd.envelope_json::jsonb #>>
'{{exception,fingerprint}}',
''
),
'taxonomy-v1:' || wd.kind || ':'
|| wd.category || ':' || wd.code
) AS fingerprint
FROM worker_diagnostics wd
JOIN result_reservations r ON r.id = wd.reservation_id
JOIN remote_worker_users u ON u.id = r.remote_user_id
JOIN remote_worker_devices d ON d.id = r.remote_device_id
LEFT JOIN result_bundles b ON b.reservation_id = r.id
LEFT JOIN target_scans s ON s.id = b.target_scan_id
WHERE {where}
), group_stats AS (
SELECT fingerprint, COUNT(*) AS fingerprint_count,
COUNT(DISTINCT reservation_id)
AS affected_assignment_count
FROM filtered GROUP BY fingerprint
), counted AS (
SELECT filtered.*, group_stats.fingerprint_count,
group_stats.affected_assignment_count,
COUNT(*) OVER() AS matched_occurrence_count
FROM filtered
JOIN group_stats USING (fingerprint)
)
SELECT * FROM counted
ORDER BY occurred_at DESC, id DESC
LIMIT ? OFFSET ?''',
(*parameters, limit + 1, occurrence_offset),
).fetchall()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
has_next = len(rows) > limit
page_rows = rows[:limit]
matched_occurrence_count = (
int(page_rows[0]['matched_occurrence_count']) if page_rows else 0
)
grouped = {}
for raw in page_rows:
row = dict(raw)
row.pop('matched_occurrence_count', None)
fingerprint = str(row.pop('fingerprint'))
fingerprint_count = int(row.pop('fingerprint_count'))
affected_assignment_count = int(
row.pop('affected_assignment_count')
)
try:
envelope = json.loads(str(row.pop('envelope_json')))
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise WorkerObservabilityConflictError(
'durable worker diagnostic JSON is invalid'
) from exc
occurrence = {
'diagnostic_uid': str(row['diagnostic_uid']),
'reservation_id': int(row['reservation_id']),
'target_scan_id': (
int(row['target_scan_id'])
if row['target_scan_id'] is not None else None
),
'source': str(row['source']),
'worker': str(row['device_key']),
'user': str(row['user_key']),
'assignment_outcome': str(row['assignment_outcome']),
'scan_outcome': str(row['scan_outcome']),
'phase': str(row['phase']),
'kind': str(row['kind']),
'category': str(row['category']),
'code': str(row['code']),
'summary': str(row['summary']),
'retryable': bool(row['retryable']),
'occurred_at': str(row['occurred_at']),
'received_at': str(row['received_at']),
}
group = grouped.setdefault(fingerprint, {
'fingerprint': fingerprint,
'count': fingerprint_count,
'affected_assignment_count': affected_assignment_count,
'page_occurrence_count': 0,
'affected_assignments': [],
'occurrences': [],
})
group['page_occurrence_count'] += 1
group['occurrences'].append(occurrence)
if occurrence['reservation_id'] not in group['affected_assignments']:
group['affected_assignments'].append(occurrence['reservation_id'])
return {
'filters': filters,
'matched_occurrence_count': matched_occurrence_count,
'occurrence_limit': limit,
'occurrence_offset': occurrence_offset,
'page_occurrence_count': len(page_rows),
'has_previous': occurrence_offset > 0,
'has_next': has_next,
'previous_occurrence_offset': (
max(0, occurrence_offset - limit)
if occurrence_offset > 0 else None
),
'next_occurrence_offset': (
occurrence_offset + limit if has_next else None
),
'truncated': has_next,
'groups': list(grouped.values()),
}
def admin_worker_duration_metrics(self, limit=200, filters=None, *, offset=0):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker duration metrics require PostgreSQL')
limit = int(limit)
offset = int(offset)
if not 1 <= limit <= 500 or offset < 0:
raise ValueError('worker duration metrics limit is out of range')
filters = _validated_admin_worker_filters(filters)
conditions = ["r.assignment_kind = 'remote'"]
parameters = []
if 'source' in filters:
conditions.append('r.source = ?')
parameters.append(filters['source'])
if 'worker' in filters:
conditions.append('d.device_key = ?')
parameters.append(filters['worker'])
if 'assignment_outcome' in filters:
conditions.append(f'({_admin_assignment_outcome_sql()}) = ?')
parameters.append(filters['assignment_outcome'])
if 'scan_outcome' in filters:
conditions.append(f'({_admin_scan_outcome_sql()}) = ?')
parameters.append(filters['scan_outcome'])
diagnostic_conditions = []
for name in ('category', 'code'):
if name in filters:
diagnostic_conditions.append(f'fd.{name} = ?')
parameters.append(filters[name])
if 'retryable' in filters:
diagnostic_conditions.append('fd.retryable = ?')
parameters.append(1 if filters['retryable'] else 0)
if diagnostic_conditions:
conditions.append('''EXISTS (
SELECT 1 FROM worker_diagnostics fd
WHERE fd.reservation_id = r.id AND %s
)''' % ' AND '.join(diagnostic_conditions))
sample_conditions = ['duration_seconds >= 0']
sample_parameters = []
if 'phase' in filters:
sample_conditions.append('phase = ?')
sample_parameters.append(filters['phase'])
if 'since' in filters:
conditions.append(
'(r.remote_resolved_at IS NULL OR r.remote_resolved_at >= ?)'
)
parameters.append(filters['since'])
sample_conditions.append('completed_at >= ?')
sample_parameters.append(filters['since'])
where = ' AND '.join(conditions)
sample_where = ' AND '.join(sample_conditions)
try:
rows = self.conn.execute(
f'''WITH reservation_scope AS (
SELECT r.id, r.source, r.remote_issued_at,
r.remote_resolved_at,
CASE WHEN r.remote_resolution_kind = 'bundle_accepted'
THEN COALESCE(s.status, 'unavailable')
ELSE {_admin_assignment_outcome_sql()} END AS outcome
FROM result_reservations r
JOIN remote_worker_devices d ON d.id = r.remote_device_id
LEFT JOIN result_bundles b ON b.reservation_id = r.id
LEFT JOIN target_scans s ON s.id = b.target_scan_id
WHERE {where}
), ordered_events AS (
SELECT e.reservation_id, e.phase, e.event_timestamp,
e.phase_started_at,
LAG(e.phase) OVER (
PARTITION BY e.reservation_id
ORDER BY e.sequence, e.id
) AS prior_phase,
e.sequence, e.id
FROM worker_progress_events e
JOIN reservation_scope rs ON rs.id = e.reservation_id
), transitions AS (
SELECT reservation_id, phase,
COALESCE(
NULLIF(phase_started_at, ''), event_timestamp
) AS phase_started_at,
sequence, id
FROM ordered_events
WHERE prior_phase IS DISTINCT FROM phase
), phase_edges AS (
SELECT t.reservation_id, t.phase, t.phase_started_at,
LEAD(t.phase_started_at) OVER (
PARTITION BY t.reservation_id
ORDER BY t.sequence, t.id
) AS next_phase_started_at
FROM transitions t
), samples AS (
SELECT rs.source, pe.phase, rs.outcome,
COALESCE(
pe.next_phase_started_at, rs.remote_resolved_at
) AS completed_at,
EXTRACT(EPOCH FROM (
COALESCE(
pe.next_phase_started_at, rs.remote_resolved_at
)::timestamptz
- pe.phase_started_at::timestamptz
))::DOUBLE PRECISION AS duration_seconds
FROM phase_edges pe
JOIN reservation_scope rs ON rs.id = pe.reservation_id
WHERE pe.next_phase_started_at IS NOT NULL
OR rs.remote_resolved_at IS NOT NULL
UNION ALL
SELECT rs.source, 'end_to_end' AS phase, rs.outcome,
rs.remote_resolved_at AS completed_at,
EXTRACT(EPOCH FROM (
rs.remote_resolved_at::timestamptz
- rs.remote_issued_at::timestamptz
))::DOUBLE PRECISION AS duration_seconds
FROM reservation_scope rs
WHERE rs.remote_resolved_at IS NOT NULL
AND rs.remote_issued_at IS NOT NULL
)
, grouped AS (
SELECT source, phase, outcome, COUNT(*) AS sample_count,
PERCENTILE_CONT(0.50) WITHIN GROUP (
ORDER BY duration_seconds
) AS p50_seconds,
PERCENTILE_CONT(0.95) WITHIN GROUP (
ORDER BY duration_seconds
) AS p95_seconds,
PERCENTILE_CONT(0.99) WITHIN GROUP (
ORDER BY duration_seconds
) AS p99_seconds
FROM samples WHERE {sample_where}
GROUP BY source, phase, outcome
)
SELECT grouped.*, COUNT(*) OVER() AS total_group_count
FROM grouped
ORDER BY source, phase, outcome LIMIT ? OFFSET ?''',
(*parameters, *sample_parameters, limit + 1, offset),
).fetchall()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
has_next = len(rows) > limit
page_rows = rows[:limit]
total_group_count = (
int(page_rows[0]['total_group_count']) if page_rows else 0
)
metrics = []
for raw in page_rows:
row = dict(raw)
count = int(row['sample_count'])
metrics.append({
'source': str(row['source']),
'phase': str(row['phase']),
'outcome': str(row['outcome']),
'sample_count': count,
'sufficient': count >= 5,
'minimum_sample_count': 5,
'p50_seconds': float(row['p50_seconds']),
'p95_seconds': float(row['p95_seconds']),
'p99_seconds': float(row['p99_seconds']),
})
return {
'filters': filters,
'metrics': metrics,
'total_group_count': total_group_count,
'metric_limit': limit,
'metric_offset': offset,
'page_group_count': len(metrics),
'has_previous': offset > 0,
'has_next': has_next,
'previous_metric_offset': max(0, offset - limit) if offset > 0 else None,
'next_metric_offset': offset + limit if has_next else None,
'truncated': has_next,
}
def admin_worker_assignment_detail(
self, reservation_id, *, event_limit=500, diagnostic_limit=500,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker assignment detail requires PostgreSQL')
reservation_id = int(reservation_id)
event_limit = int(event_limit)
diagnostic_limit = int(diagnostic_limit)
if (
reservation_id <= 0
or not 1 <= event_limit <= 2000
or not 1 <= diagnostic_limit <= 2000
):
raise ValueError('worker assignment detail request is out of range')
try:
raw_assignment = self.conn.execute(
f'''SELECT r.*, q.id AS queue_id, q.status AS queue_status,
q.completed_at AS queue_settled_at,
u.user_key, u.active_assignment_cap, d.device_key,
d.last_contact_at,
b.state AS bundle_state, b.ready_at AS bundle_ready_at,
b.committed_at AS bundle_committed_at,
b.acknowledged_at AS bundle_acknowledged_at,
b.actual_bytes AS bundle_bytes,
b.finding_count AS bundle_finding_count,
b.error_count AS bundle_error_count,
s.id AS target_scan_id, s.status AS scan_status,
s.started_at AS scan_started_at,
s.ended_at AS scan_ended_at,
s.duration_sec AS scan_duration_seconds,
s.findings_count, s.verified_findings_count,
s.error_count AS scan_error_count, s.skipped_reason,
s.first_error_summary, s.queue_completion_applied,
s.queue_completion_disposition,
src.metadata_json::jsonb #>> '{{warnings,0}}'
AS scan_warning_summary,
src.metadata_json::jsonb #>> '{{warning_classes,0}}'
AS scan_warning_class,
pj.status AS projection_status,
pj.completed_at AS projection_completed_at,
CURRENT_TIMESTAMP AS authority_at,
{ _admin_assignment_outcome_sql() } AS assignment_outcome,
{ _admin_scan_outcome_sql() } AS scan_outcome
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
JOIN remote_worker_users u ON u.id = r.remote_user_id
JOIN remote_worker_devices d ON d.id = r.remote_device_id
LEFT JOIN result_bundles b ON b.reservation_id = r.id
LEFT JOIN target_scans s ON s.id = b.target_scan_id
LEFT JOIN scan_result_compat src ON src.target_scan_id = s.id
LEFT JOIN LATERAL (
SELECT j.status, j.completed_at
FROM projection_jobs j
WHERE j.target_scan_id = s.id
AND j.job_kind = 'scan_event'
ORDER BY j.id DESC LIMIT 1
) pj ON TRUE
WHERE r.id = ? AND r.assignment_kind = 'remote' ''',
(reservation_id,),
).fetchone()
if not raw_assignment:
self.conn.commit()
return None
event_rows = self.conn.execute(
'''SELECT * FROM (
SELECT e.*, COUNT(*) OVER() AS total_event_count
FROM worker_progress_events e
WHERE e.reservation_id = ?
ORDER BY e.sequence DESC, e.id DESC LIMIT ?
) recent
ORDER BY sequence, id''',
(reservation_id, event_limit + 1),
).fetchall()
diagnostic_rows = self.conn.execute(
'''SELECT * FROM worker_diagnostics
WHERE reservation_id = ?
ORDER BY occurred_at, id LIMIT ?''',
(reservation_id, diagnostic_limit + 1),
).fetchall()
target_scan_id = raw_assignment['target_scan_id']
error_rows = []
if target_scan_id is not None:
error_rows = self.conn.execute(
'''SELECT id, category, summary, raw_error, created_at
FROM errors WHERE target_scan_id = ?
ORDER BY id LIMIT ?''',
(int(target_scan_id), diagnostic_limit + 1),
).fetchall()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
assignment_row = dict(raw_assignment)
total_event_count = (
int(event_rows[0]['total_event_count']) if event_rows else 0
)
events_truncated = total_event_count > event_limit
diagnostics_truncated = len(diagnostic_rows) > diagnostic_limit
legacy_errors_truncated = len(error_rows) > diagnostic_limit
events = []
for raw in event_rows[-event_limit:]:
row = dict(raw)
row.pop('total_event_count', None)
events.append(self._worker_observability_row(
row, 'event_json', 'event',
))
diagnostics = []
for raw in diagnostic_rows[:diagnostic_limit]:
row = self._worker_observability_row(
raw, 'envelope_json', 'diagnostic',
)
row['canonical_envelope_json'] = str(row['envelope_json'])
diagnostics.append(row)
legacy_errors = [dict(row) for row in error_rows[:diagnostic_limit]]
def parsed_json(name):
raw = assignment_row.get(name)
if not raw:
return None
try:
value = json.loads(str(raw))
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise WorkerObservabilityConflictError(
'durable remote assignment JSON is invalid'
) from exc
return value
resolution = parsed_json('remote_resolution_json')
execution_snapshot = parsed_json('remote_execution_snapshot_json')
timeline = []
def add_timeline(timestamp, kind, label, **values):
if timestamp:
timeline.append({
'timestamp': str(timestamp), 'kind': kind, 'label': label,
**values,
})
add_timeline(assignment_row.get('remote_issued_at'), 'assignment', 'issued')
for row in events:
event = row['event']
add_timeline(
event.get('timestamp') or row.get('event_timestamp'),
'phase', str(event.get('phase') or row.get('phase') or ''),
sequence=int(row['sequence']),
received_at=str(row['received_at']),
)
add_timeline(assignment_row.get('bundle_ready_at'), 'transport', 'bundle received')
add_timeline(assignment_row.get('remote_resolved_at'), 'receipt', str(
assignment_row.get('remote_resolution_kind') or 'terminal receipt'
))
add_timeline(assignment_row.get('bundle_committed_at'), 'ingestion', 'bundle ingested')
add_timeline(assignment_row.get('queue_settled_at'), 'settlement', 'queue settled')
add_timeline(
assignment_row.get('projection_completed_at'),
'projection', 'projection completed',
)
timeline.sort(key=lambda item: (
parse_time(item['timestamp']), item['kind'], item['label'],
))
durations = []
def duration(label, started, ended, outcome, *, complete=True, authority=None):
if not started or not ended:
return
seconds = max(0.0, (parse_time(ended) - parse_time(started)).total_seconds())
durations.append({
'phase': label, 'duration_seconds': seconds, 'outcome': outcome,
'complete': bool(complete), 'ended_at': str(ended),
'authority': authority or ('transition' if complete else 'database_current_time'),
})
current_authority = (
assignment_row.get('remote_resolved_at')
or assignment_row.get('authority_at')
)
duration(
'end_to_end', assignment_row.get('remote_issued_at'),
current_authority,
assignment_row['assignment_outcome'],
complete=assignment_row.get('remote_resolved_at') is not None,
authority=(
'assignment_resolution'
if assignment_row.get('remote_resolved_at') else 'database_current_time'
),
)
if assignment_row.get('scan_duration_seconds') is not None:
durations.append({
'phase': 'scan_total',
'duration_seconds': float(assignment_row['scan_duration_seconds']),
'outcome': assignment_row['scan_outcome'],
'complete': True,
'ended_at': assignment_row.get('scan_ended_at'),
'authority': 'target_scan',
})
transitions = []
for row in events:
event = row['event']
phase = str(event.get('phase') or row.get('phase') or '')
if not transitions or transitions[-1][0] != phase:
transitions.append((
phase,
str(event.get('phase_started_at') or event.get('timestamp') or ''),
))
for index, (phase, started) in enumerate(transitions):
final = index == len(transitions) - 1
ended = current_authority if final else transitions[index + 1][1]
duration(
phase, started, ended, assignment_row['scan_outcome'],
complete=(not final or assignment_row.get('remote_resolved_at') is not None),
authority=(
'assignment_resolution' if final and assignment_row.get('remote_resolved_at')
else 'database_current_time' if final else 'phase_transition'
),
)
latest_event = events[-1]['event'] if events else None
latest_progress_at = (
latest_event.get('timestamp') if latest_event else None
)
phase_started_at = (
latest_event.get('phase_started_at') if latest_event else None
)
phase_authority = current_authority if latest_event else None
phase_age_seconds = (
max(0.0, (
parse_time(phase_authority) - parse_time(phase_started_at)
).total_seconds())
if phase_started_at and phase_authority else None
)
last_progress_age_seconds = (
max(0.0, (
parse_time(phase_authority) - parse_time(latest_progress_at)
).total_seconds())
if latest_progress_at and phase_authority else None
)
assignment = {
'reservation_id': int(assignment_row['id']),
'queue_id': int(assignment_row['queue_id']),
'source': str(assignment_row['source']),
'target': _admin_safe_target(assignment_row.get('normalized_target')),
'user': str(assignment_row['user_key']),
'worker': str(assignment_row['device_key']),
'active_assignment_cap': int(assignment_row['active_assignment_cap']),
'assignment_outcome': str(assignment_row['assignment_outcome']),
'scan_outcome': str(assignment_row['scan_outcome']),
'issued_at': assignment_row.get('remote_issued_at'),
'resolved_at': assignment_row.get('remote_resolved_at'),
'assignment_code': assignment_row.get('last_error_code'),
'assignment_detail': assignment_row.get('last_error_detail'),
}
resolution_deadlines = (
resolution.get('deadlines') if isinstance(resolution, dict) else None
)
receipt_scan_deadline = (
resolution_deadlines.get('scan_deadline_at')
if isinstance(resolution_deadlines, dict) else None
)
deadlines = {
'assignment_deadline_at': assignment_row.get('remote_expires_at'),
'scan_deadline_at': (
receipt_scan_deadline
or (events[-1]['event'].get('scan_deadline_at') if events else None)
),
'upload_timeout_seconds': assignment_row.get(
'remote_result_upload_body_timeout_seconds'
),
'upload_timeout_availability': (
'persisted at assignment issuance'
if assignment_row.get(
'remote_result_upload_body_timeout_seconds'
) is not None else 'legacy/unavailable'
),
}
compatibility = (execution_snapshot or {}).get('compatibility') or {}
package = {
key: compatibility.get(key) for key in (
'protocol_version', 'bundle_format_version', 'platform_tag',
'code_manifest_sha256', 'detector_policy_sha256',
'effective_config_sha256',
)
}
scan = {
'available': assignment_row.get('target_scan_id') is not None,
'target_scan_id': assignment_row.get('target_scan_id'),
'status': assignment_row.get('scan_status'),
'started_at': assignment_row.get('scan_started_at'),
'ended_at': assignment_row.get('scan_ended_at'),
'duration_seconds': assignment_row.get('scan_duration_seconds'),
'findings_count': assignment_row.get('findings_count'),
'verified_findings_count': assignment_row.get('verified_findings_count'),
'error_count': assignment_row.get('scan_error_count'),
'skipped_reason': assignment_row.get('skipped_reason'),
'first_error_summary': assignment_row.get('first_error_summary'),
'warning_summary': assignment_row.get('scan_warning_summary'),
'warning_class': assignment_row.get('scan_warning_class'),
}
projection_version = assignment_row.get('remote_diagnostic_projection_version')
protocol2 = str(package.get('protocol_version') or '') == '2'
diagnostic_availability = (
'current' if diagnostics or projection_version == 1 or protocol2
else 'legacy/unavailable'
)
return {
'schema': 1,
'assignment': assignment,
'deadlines': deadlines,
'package': package,
'transport': {
'resolution': resolution,
'receipt_id': assignment_row.get('remote_receipt_id'),
'bundle_state': assignment_row.get('bundle_state'),
'bundle_ready_at': assignment_row.get('bundle_ready_at'),
'bundle_committed_at': assignment_row.get('bundle_committed_at'),
'bundle_acknowledged_at': assignment_row.get('bundle_acknowledged_at'),
'queue_status': assignment_row.get('queue_status'),
'queue_settled_at': assignment_row.get('queue_settled_at'),
'projection_status': assignment_row.get('projection_status'),
'projection_completed_at': assignment_row.get('projection_completed_at'),
},
'scan': scan,
'timeline': timeline,
'durations': durations,
'progress': {
'available': bool(events),
'availability': (
'current' if events
else 'unavailable/no persisted progress' if protocol2
else 'legacy/unavailable'
),
'truncated': events_truncated,
'total_event_count': total_event_count,
'omitted_older_event_count': max(0, total_event_count - len(events)),
'current_phase': (
str(latest_event.get('phase')) if latest_event else None
),
'phase_started_at': phase_started_at,
'phase_age_seconds': phase_age_seconds,
'last_progress_at': latest_progress_at,
'last_progress_age_seconds': last_progress_age_seconds,
'age_authority': (
'assignment_resolution'
if assignment_row.get('remote_resolved_at') else 'database_current_time'
) if latest_event else None,
'events': events,
},
'diagnostics': {
'availability': diagnostic_availability,
'projection_version': projection_version,
'declared_count': assignment_row.get('remote_diagnostic_count'),
'truncated': diagnostics_truncated,
'items': diagnostics,
},
'legacy_evidence': {
'available': bool(legacy_errors) or bool(
assignment_row.get('first_error_summary')
),
'explicitly_not_an_envelope': True,
'truncated': legacy_errors_truncated,
'errors': legacy_errors,
'first_error_summary': assignment_row.get('first_error_summary'),
},
}
def admin_worker_diagnostic_envelope(self, reservation_id, diagnostic_uid):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker diagnostic download requires PostgreSQL')
reservation_id = int(reservation_id)
diagnostic_uid = str(diagnostic_uid or '')
if reservation_id <= 0 or re.fullmatch(r'[a-f0-9]{64}', diagnostic_uid) is None:
raise ValueError('worker diagnostic download identity is invalid')
row = self.conn.execute(
'''SELECT envelope_json, envelope_sha256 FROM worker_diagnostics
WHERE reservation_id = ? AND diagnostic_uid = ?''',
(reservation_id, diagnostic_uid),
).fetchone()
self.conn.commit()
if not row:
return None
return {
'canonical_json': str(row['envelope_json']),
'sha256': str(row['envelope_sha256']),
}
def admin_requeue_deferred_targets(self, queue_ids, *, max_items=100):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('admin queue requeue requires PostgreSQL')
maximum = min(500, max(1, int(max_items)))
ids = []
for value in queue_ids or ():
if isinstance(value, bool):
raise ValueError('admin queue IDs must be positive integers')
queue_id = int(value)
if queue_id <= 0 or queue_id > 9223372036854775807:
raise ValueError('admin queue IDs must be positive integers')
ids.append(queue_id)
if not ids or len(ids) > maximum or len(ids) != len(set(ids)):
raise ValueError('admin queue ID selection exceeds its bound')
now = utc_now_iso()
try:
placeholders = ','.join('?' for _ in ids)
cursor = self.conn.execute(
f'''UPDATE target_queue SET status = 'pending', attempts = 0,
available_after = NULL, lease_owner = NULL, lease_token = NULL,
claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
last_error = NULL, completed_at = NULL, updated_at = ?
WHERE id IN ({placeholders}) AND status = 'deferred' ''',
(now, *ids),
)
self.conn.commit()
return int(cursor.rowcount or 0)
except Exception:
self.conn.rollback()
raise
def admin_discard_queued_source(self, source):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('admin source queue discard requires PostgreSQL')
source = str(source or '').strip().lower()
if re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', source) is None:
raise ValueError('admin queue source is invalid')
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'quarantined',
completed_at = ?, available_after = NULL,
lease_owner = NULL, lease_token = NULL,
claim_batch = NULL, leased_at = NULL,
lease_expires_at = NULL, resolver_state = 'resolved',
resolver_due_at = NULL, resolver_attempts = 0,
resolver_token = NULL, claim_event_id = NULL,
last_error = ?,
updated_at = ?
WHERE source = ?
AND status IN ('pending', 'deferred', 'cold')
AND current_result_reservation_id IS NULL
AND target_scan_id IS NULL
AND NOT EXISTS (
SELECT 1 FROM result_reservations reservation
WHERE reservation.queue_id = target_queue.id
)
AND NOT EXISTS (
SELECT 1 FROM target_scans scan
WHERE scan.queue_id = target_queue.id
)''',
(now, ADMIN_DISCARDED_QUEUE_REASON, now, source),
)
self.conn.commit()
return int(cursor.rowcount or 0)
except Exception:
self.conn.rollback()
raise
def authenticate_remote_worker(self, token_sha256):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker authentication requires PostgreSQL')
token_sha256 = str(token_sha256 or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
return None
now = utc_now_iso()
try:
row = self.conn.execute(
'''SELECT d.id AS device_id, d.user_id, d.device_key,
u.user_key, u.active_assignment_cap
FROM remote_worker_devices d
JOIN remote_worker_users u ON u.id = d.user_id
WHERE d.token_sha256 = ? AND d.revoked_at IS NULL
AND u.disabled_at IS NULL FOR UPDATE OF d''',
(token_sha256,),
).fetchone()
if not row:
self.conn.rollback()
return None
self.conn.execute(
'UPDATE remote_worker_devices SET last_contact_at = ?, updated_at = ? WHERE id = ?',
(now, now, row['device_id']),
)
self.conn.commit()
return {
'device_id': int(row['device_id']), 'user_id': int(row['user_id']),
'device_key': str(row['device_key']), 'user_key': str(row['user_key']),
'active_assignment_cap': int(row['active_assignment_cap']),
'token_sha256': token_sha256,
}
except Exception:
self.conn.rollback()
raise
def set_remote_worker_device_revoked(self, device_key, revoked=True):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker revocation requires PostgreSQL')
device_key = str(device_key or '').strip()
if not 1 <= len(device_key) <= 128:
raise ValueError('remote worker device key must be 1..128 characters')
now = utc_now_iso()
cursor = self.conn.execute(
'''UPDATE remote_worker_devices SET revoked_at = ?, updated_at = ?
WHERE device_key = ?''',
(now if revoked else None, now, device_key),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
@staticmethod
def _remote_credential_mapping(device_id, token_sha256):
device_id = int(device_id or 0)
token_sha256 = str(token_sha256 or '').strip().lower()
if device_id <= 0 or not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
raise ValueError('remote worker credential identity is invalid')
return device_id, token_sha256
def _lock_active_remote_credential(self, device_id, token_sha256, user_id=None):
device_id, token_sha256 = self._remote_credential_mapping(
device_id, token_sha256,
)
row = self.conn.execute(
'''SELECT d.id AS device_id, d.user_id
FROM remote_worker_devices d
JOIN remote_worker_users u ON u.id = d.user_id
WHERE d.id = ? AND d.token_sha256 = ? AND d.revoked_at IS NULL
AND u.disabled_at IS NULL
FOR SHARE OF d, u''',
(device_id, token_sha256),
).fetchone()
if row and user_id is not None and int(row['user_id']) != int(user_id):
return None
return row
@staticmethod
def _worker_observability_row(row, json_column, value_key, replayed=False):
result = dict(row)
try:
result[value_key] = json.loads(str(result[json_column]))
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise WorkerObservabilityConflictError(
f'durable worker {value_key} JSON is invalid'
) from exc
result['replayed'] = bool(replayed)
return result
def _record_worker_progress_event(self, reservation_id, device_id, event, received_at=None):
reservation_id = int(reservation_id)
device_id = int(device_id)
event, event_json, event_sha256 = _canonical_worker_contract('progress', event)
schema_version = event.get('schema', event.get('schema_version'))
sequence = event.get('sequence')
slot_id = event.get('slot_id')
if (
isinstance(schema_version, bool) or not isinstance(schema_version, int)
or schema_version <= 0
or isinstance(sequence, bool) or not isinstance(sequence, int) or sequence <= 0
or isinstance(slot_id, bool) or not isinstance(slot_id, int) or slot_id < 0
):
raise ValueError('worker progress numeric metadata is invalid')
if int(event.get('reservation_id') or 0) != reservation_id:
raise WorkerObservabilityConflictError(
'worker progress reservation identity does not match the request'
)
values = {
'event_type': str(event.get('type') or '').strip(),
'phase': str(event.get('phase') or '').strip(),
'event_timestamp': str(event.get('timestamp') or '').strip(),
'phase_started_at': str(event.get('phase_started_at') or '').strip() or None,
'instance_id': str(event.get('instance_id') or '').strip(),
'source': str(event.get('source') or '').strip(),
}
if any(not values[name] for name in (
'event_type', 'phase', 'event_timestamp', 'instance_id', 'source',
)):
raise ValueError('worker progress required metadata is incomplete')
if len(event_json.encode('utf-8')) > 8 * 1024:
raise ValueError('worker progress event exceeds its canonical JSON bound')
lock = ' FOR UPDATE OF r, q' if self.conn.is_postgres else ''
reservation = self.conn.execute(
'''SELECT r.id, r.remote_device_id, r.source, r.state,
r.claim_lease_token, r.scan_event_id, r.remote_expires_at,
r.remote_issued_at, r.remote_resolved_at,
r.remote_resolution_kind, q.status AS queue_status,
q.lease_token AS queue_lease_token,
q.current_result_reservation_id, q.claim_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? AND r.assignment_kind = 'remote' ''' + lock,
(reservation_id,),
).fetchone()
now = str(received_at or utc_now_iso())
if not reservation or (
int(reservation['remote_device_id'] or 0) != device_id
or str(reservation['source']) != values['source']
):
raise WorkerObservabilityConflictError(
'worker progress does not match the owned reservation'
)
existing = self.conn.execute(
'''SELECT * FROM worker_progress_events
WHERE reservation_id = ? AND sequence = ?''',
(reservation_id, sequence),
).fetchone()
if existing:
if (
int(existing['remote_device_id']) != device_id
or str(existing['event_sha256']) != event_sha256
or str(existing['event_json']) != event_json
):
raise WorkerObservabilityConflictError(
'worker progress sequence already has different canonical content'
)
return self._worker_observability_row(
existing, 'event_json', 'event', replayed=True,
)
try:
event_time = datetime.fromisoformat(
str(event['timestamp']).replace('Z', '+00:00')
).astimezone(timezone.utc)
received_time = datetime.fromisoformat(
now.replace('Z', '+00:00')
).astimezone(timezone.utc)
issued_time = datetime.fromisoformat(
str(reservation['remote_issued_at']).replace('Z', '+00:00')
).astimezone(timezone.utc)
except (AttributeError, TypeError, ValueError) as exc:
raise WorkerObservabilityConflictError(
'worker progress authority timestamps are invalid'
) from exc
tolerance = timedelta(seconds=REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS)
if (
event_time < issued_time - tolerance
or event_time > received_time + tolerance
):
raise WorkerObservabilityConflictError(
'worker progress timestamp is outside its authority window'
)
resolved = reservation['remote_resolution_kind'] is not None
if resolved:
try:
resolved_time = datetime.fromisoformat(
str(reservation['remote_resolved_at']).replace('Z', '+00:00')
).astimezone(timezone.utc)
except (AttributeError, TypeError, ValueError) as exc:
raise WorkerObservabilityConflictError(
'resolved worker progress authority timestamp is invalid'
) from exc
if received_time > resolved_time + timedelta(
seconds=REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS
):
raise WorkerProgressInactiveError(
'resolved worker progress grace window elapsed'
)
if event_time > resolved_time + tolerance:
raise WorkerObservabilityConflictError(
'worker progress timestamp is after terminal resolution'
)
if event_time < resolved_time - timedelta(
seconds=(
REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS
+ REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS
)
):
raise WorkerProgressInactiveError(
'worker progress timestamp is too old for terminal grace'
)
if not resolved and (
str(reservation['state']) != 'scanning'
or not reservation['remote_expires_at']
or str(reservation['remote_expires_at']) <= now
or str(reservation['queue_status']) != 'in_progress'
or str(reservation['queue_lease_token'] or '')
!= str(reservation['claim_lease_token'] or '')
or int(reservation['current_result_reservation_id'] or 0) != reservation_id
or str(reservation['claim_event_id'] or '')
!= str(reservation['scan_event_id'] or '')
):
raise WorkerProgressInactiveError(
'worker progress requires the owned current unresolved reservation'
)
latest = self.conn.execute(
'''SELECT sequence, event_timestamp FROM worker_progress_events
WHERE reservation_id = ? ORDER BY sequence DESC LIMIT 1''',
(reservation_id,),
).fetchone()
if latest and sequence <= int(latest['sequence']):
raise WorkerObservabilityConflictError(
'worker progress sequence must strictly increase'
)
if latest:
try:
latest_time = datetime.fromisoformat(
str(latest['event_timestamp']).replace('Z', '+00:00')
).astimezone(timezone.utc)
except (AttributeError, TypeError, ValueError) as exc:
raise WorkerObservabilityConflictError(
'durable worker progress timestamp is invalid'
) from exc
if event_time < latest_time - tolerance:
raise WorkerObservabilityConflictError(
'worker progress timestamp moves backward beyond tolerance'
)
cursor = self.conn.execute(
'''INSERT INTO worker_progress_events(
reservation_id, remote_device_id, schema_version, sequence,
event_type, phase, event_timestamp, phase_started_at, instance_id,
slot_id, source, event_json, event_sha256, received_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(reservation_id, sequence) DO NOTHING''',
(
reservation_id, device_id, schema_version, sequence,
values['event_type'], values['phase'], values['event_timestamp'],
values['phase_started_at'], values['instance_id'], slot_id,
values['source'], event_json, event_sha256, now,
),
)
stored = self.conn.execute(
'''SELECT * FROM worker_progress_events
WHERE reservation_id = ? AND sequence = ?''',
(reservation_id, sequence),
).fetchone()
if not stored or (
int(cursor.rowcount or 0) != 1
and (
int(stored['remote_device_id']) != device_id
or str(stored['event_sha256']) != event_sha256
or str(stored['event_json']) != event_json
)
):
raise WorkerObservabilityConflictError(
'worker progress insert conflicted with different canonical content'
)
return self._worker_observability_row(stored, 'event_json', 'event')
def record_worker_progress_event(self, reservation_id, device_id, event):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker progress persistence requires PostgreSQL')
try:
result = self._record_worker_progress_event(
reservation_id, device_id, event,
)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def record_remote_worker_progress_event(
self, reservation_id, device_id, token_sha256, event,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote worker progress persistence requires PostgreSQL')
try:
if not self._lock_active_remote_credential(device_id, token_sha256):
raise WorkerObservabilityConflictError(
'remote worker credential changed before progress acceptance'
)
result = self._record_worker_progress_event(
reservation_id, device_id, event,
)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def _record_worker_diagnostic(
self, reservation_id, diagnostic, target_scan_id=None, received_at=None,
):
reservation_id = int(reservation_id)
target_scan_id = int(target_scan_id) if target_scan_id is not None else None
diagnostic, envelope_json, envelope_sha256 = _canonical_worker_contract(
'diagnostic', diagnostic,
)
diagnostic_uid = str(
diagnostic.get('diagnostic_uid')
or diagnostic.get('uid')
or diagnostic.get('id')
or ''
).strip()
schema_version = diagnostic.get('schema', diagnostic.get('schema_version'))
slot_id = diagnostic.get('slot_id')
attempt = diagnostic.get('attempt')
retryable = diagnostic.get('retryable')
scan_event_id = diagnostic.get('scan_event_id')
if not diagnostic_uid or len(diagnostic_uid) > 256 or not diagnostic_uid.isascii():
raise ValueError('worker diagnostic UID is invalid')
if (
isinstance(schema_version, bool) or not isinstance(schema_version, int)
or schema_version <= 0
or isinstance(slot_id, bool) or not isinstance(slot_id, int) or slot_id < 0
or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt <= 0
or (
scan_event_id is not None
and (
not isinstance(scan_event_id, str)
or re.fullmatch(r'[a-f0-9]{32,64}', scan_event_id) is None
)
)
or not isinstance(retryable, bool)
):
raise ValueError('worker diagnostic identity or numeric metadata is invalid')
if int(diagnostic.get('reservation_id') or 0) != reservation_id:
raise WorkerObservabilityConflictError(
'worker diagnostic reservation identity does not match the request'
)
assignment_outcome = diagnostic.get('assignment_outcome')
scan_outcome = diagnostic.get('scan_outcome')
if target_scan_id is not None:
if (
assignment_outcome != 'accepted'
or scan_outcome not in {
'clean', 'found', 'degraded', 'error', 'skipped',
}
):
raise WorkerObservabilityConflictError(
'accepted bundle diagnostic outcomes are invalid'
)
elif assignment_outcome == 'accepted':
raise WorkerObservabilityConflictError(
'accepted diagnostic requires an authoritative target scan'
)
elif assignment_outcome in {'prebundle_failed', 'expired'} and (
scan_outcome != 'unavailable'
):
raise WorkerObservabilityConflictError(
'terminal diagnostic scan outcome must be unavailable'
)
values = {
'source': str(diagnostic.get('source') or '').strip(),
'phase': str(diagnostic.get('phase') or '').strip(),
'kind': str(diagnostic.get('kind') or '').strip(),
'category': str(diagnostic.get('category') or '').strip(),
'code': str(diagnostic.get('code') or '').strip(),
'summary': str(diagnostic.get('summary') or '').strip(),
'occurred_at': str(diagnostic.get('occurred_at') or '').strip(),
'captured_at': str(diagnostic.get('captured_at') or '').strip(),
}
if any(not value for value in values.values()):
raise ValueError('worker diagnostic required metadata is incomplete')
if len(envelope_json.encode('utf-8')) > 64 * 1024:
raise ValueError('worker diagnostic exceeds its canonical JSON bound')
reservation = self.conn.execute(
'''SELECT id, source, scan_event_id FROM result_reservations
WHERE id = ?''',
(reservation_id,),
).fetchone()
if not reservation or str(reservation['source']) != values['source']:
raise WorkerObservabilityConflictError(
'worker diagnostic does not match its reservation'
)
if (
scan_event_id is not None
and str(reservation['scan_event_id'] or '') != scan_event_id
):
raise WorkerObservabilityConflictError(
'worker diagnostic scan event does not match its remote reservation'
)
if target_scan_id is not None:
scan = self.conn.execute(
'''SELECT id, result_reservation_id, scan_event_id FROM target_scans
WHERE id = ?''',
(target_scan_id,),
).fetchone()
if not scan or int(scan['result_reservation_id'] or 0) != reservation_id:
raise WorkerObservabilityConflictError(
'worker diagnostic target scan does not belong to its reservation'
)
if scan_event_id is not None and str(scan['scan_event_id'] or '') != scan_event_id:
raise WorkerObservabilityConflictError(
'worker diagnostic scan event does not match its target scan'
)
http = diagnostic.get('http') or {}
process = diagnostic.get('process') or {}
body = diagnostic.get('body', diagnostic.get('raw_body'))
if body is None and isinstance(http, dict):
body = http.get('body')
logs = diagnostic.get('logs', diagnostic.get('log'))
if logs is None and isinstance(process, dict):
logs = {
name: process.get(name) for name in ('stdout', 'stderr')
if process.get(name) is not None
} or None
body_json = (
json.dumps(body, ensure_ascii=True, sort_keys=True, separators=(',', ':'))
if body is not None else None
)
log_json = (
json.dumps(logs, ensure_ascii=True, sort_keys=True, separators=(',', ':'))
if logs is not None else None
)
existing = self.conn.execute(
'SELECT * FROM worker_diagnostics WHERE diagnostic_uid = ?',
(diagnostic_uid,),
).fetchone()
if existing:
if (
int(existing['reservation_id']) != reservation_id
or (
int(existing['target_scan_id'])
if existing['target_scan_id'] is not None else None
) != target_scan_id
or str(existing['envelope_sha256']) != envelope_sha256
or str(existing['envelope_json']) != envelope_json
):
raise WorkerObservabilityConflictError(
'worker diagnostic UID already has different canonical content or authority'
)
return self._worker_observability_row(
existing, 'envelope_json', 'diagnostic', replayed=True,
)
now = str(received_at or utc_now_iso())
cursor = self.conn.execute(
'''INSERT INTO worker_diagnostics(
diagnostic_uid, reservation_id, target_scan_id, schema_version,
scan_event_id, slot_id, attempt, source, phase, kind, category,
code, summary, retryable, occurred_at, captured_at, envelope_json,
envelope_sha256, body_payload_json, log_payload_json, received_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(diagnostic_uid) DO NOTHING''',
(
diagnostic_uid, reservation_id, target_scan_id, schema_version,
scan_event_id, slot_id, attempt, values['source'],
values['phase'], values['kind'], values['category'], values['code'],
values['summary'], int(retryable), values['occurred_at'],
values['captured_at'], envelope_json, envelope_sha256,
body_json, log_json, now,
),
)
stored = self.conn.execute(
'SELECT * FROM worker_diagnostics WHERE diagnostic_uid = ?',
(diagnostic_uid,),
).fetchone()
if not stored or (
int(cursor.rowcount or 0) != 1
and (
int(stored['reservation_id']) != reservation_id
or (
int(stored['target_scan_id'])
if stored['target_scan_id'] is not None else None
) != target_scan_id
or str(stored['envelope_sha256']) != envelope_sha256
or str(stored['envelope_json']) != envelope_json
)
):
raise WorkerObservabilityConflictError(
'worker diagnostic insert conflicted with different canonical content'
)
return self._worker_observability_row(
stored, 'envelope_json', 'diagnostic',
)
def record_worker_diagnostic(
self, reservation_id, diagnostic, target_scan_id=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('worker diagnostic persistence requires PostgreSQL')
try:
result = self._record_worker_diagnostic(
reservation_id, diagnostic, target_scan_id=target_scan_id,
)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def worker_progress_events(self, reservation_id, limit=1000):
if not self.conn:
raise RuntimeError('database connection is unavailable')
limit = int(limit)
if not 1 <= limit <= 10000:
raise ValueError('worker progress query limit is out of range')
rows = self.conn.execute(
'''SELECT e.*, r.remote_user_id, r.assignment_kind,
r.remote_issued_at, r.remote_expires_at,
r.remote_resolution_kind, q.id AS queue_id,
q.status AS queue_status, s.id AS target_scan_id,
s.status AS scan_status, s.started_at AS scan_started_at,
s.ended_at AS scan_ended_at
FROM worker_progress_events e
JOIN result_reservations r ON r.id = e.reservation_id
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN target_scans s ON s.result_reservation_id = r.id
WHERE e.reservation_id = ?
ORDER BY e.sequence, e.id LIMIT ?''',
(int(reservation_id), limit),
).fetchall()
self.conn.commit()
return [
self._worker_observability_row(row, 'event_json', 'event')
for row in rows
]
def worker_diagnostics(self, reservation_id, limit=1000):
if not self.conn:
raise RuntimeError('database connection is unavailable')
limit = int(limit)
if not 1 <= limit <= 10000:
raise ValueError('worker diagnostic query limit is out of range')
rows = self.conn.execute(
'''SELECT d.*, r.remote_device_id, r.remote_user_id, r.assignment_kind,
r.remote_issued_at, r.remote_expires_at,
r.remote_resolution_kind, q.id AS queue_id,
q.status AS queue_status, s.status AS scan_status,
s.started_at AS scan_started_at, s.ended_at AS scan_ended_at
FROM worker_diagnostics d
JOIN result_reservations r ON r.id = d.reservation_id
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN target_scans s ON s.id = d.target_scan_id
WHERE d.reservation_id = ?
ORDER BY d.occurred_at, d.id LIMIT ?''',
(int(reservation_id), limit),
).fetchall()
self.conn.commit()
return [
self._worker_observability_row(row, 'envelope_json', 'diagnostic')
for row in rows
]
@staticmethod
def _remote_assignment_mapping(value):
if value is None:
return None
value = dict(value)
mapped = {
'user_id': int(value.get('user_id') or 0),
'device_id': int(value.get('device_id') or 0),
'effective_config_sha256': str(value.get('effective_config_sha256') or '').lower(),
'client_compat_sha256': str(value.get('client_compat_sha256') or '').lower(),
'token_sha256': str(value.get('token_sha256') or '').lower(),
'result_upload_body_timeout_seconds': value.get(
'result_upload_body_timeout_seconds'
),
}
if mapped['user_id'] <= 0 or mapped['device_id'] <= 0:
raise ValueError('remote assignment requires positive user and device identities')
for key in ('effective_config_sha256', 'client_compat_sha256', 'token_sha256'):
if not re.fullmatch(r'[a-f0-9]{64}', mapped[key]):
raise ValueError(f'remote assignment {key} must be lowercase SHA-256')
timeout = mapped['result_upload_body_timeout_seconds']
if (
isinstance(timeout, bool) or not isinstance(timeout, int)
or not 30 <= timeout <= 86400
):
raise ValueError('remote assignment upload body timeout is invalid')
snapshot_bytes = canonical_remote_execution_snapshot_bytes(
value.get('execution_snapshot'),
)
snapshot = json.loads(snapshot_bytes.decode('ascii'))
if (
snapshot['compatibility']['effective_config_sha256']
!= mapped['effective_config_sha256']
):
raise ValueError(
'remote assignment snapshot effective configuration is invalid'
)
mapped.update({
'execution_snapshot_json': snapshot_bytes.decode('ascii'),
'execution_snapshot_sha256': hashlib.sha256(snapshot_bytes).hexdigest(),
})
return mapped
def ensure_admission_intent(
self, reservation_token, values, remote_user_id=None, remote_device_id=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('v2 admission intent requires PostgreSQL')
token = str(reservation_token or '')
if not token:
raise ValueError('admission intent reservation token is required')
digest = self._admission_intent_sha256(values)
now = utc_now_iso()
try:
self.conn.execute(
'''INSERT INTO admission_intents(
reservation_token, intent_sha256, state, remote_user_id,
remote_device_id, created_at, updated_at
) VALUES (?, ?, 'pending', ?, ?, ?, ?)
ON CONFLICT(reservation_token) DO NOTHING''',
(token, digest, remote_user_id, remote_device_id, now, now),
)
row = self.conn.execute(
'''SELECT intent_sha256, state, remote_user_id, remote_device_id
FROM admission_intents WHERE reservation_token = ?''',
(token,),
).fetchone()
if not row or row['intent_sha256'] != digest or (
str(row['remote_user_id'] or '') != str(remote_user_id or '')
or str(row['remote_device_id'] or '') != str(remote_device_id or '')
):
raise ScanEventConflictError('admission intent token resolves to conflicting identity')
self.conn.commit()
return dict(row)
except Exception:
self.conn.rollback()
raise
def admission_intent_resolution(self, reservation_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('admission intent resolution requires PostgreSQL')
row = self.conn.execute(
'''SELECT state, resolution_detail FROM admission_intents
WHERE reservation_token = ?''',
(str(reservation_token),),
).fetchone()
self.conn.commit()
if not row or str(row['state']) != 'aborted':
return None
return str(row['resolution_detail'] or '') or None
def reserve_and_claim_target(
self, source, platform, producer_identity, supervisor_instance_id,
declared_bundle_bytes, projection_bytes, candidate_items, candidate_bytes,
lease_seconds=3600, max_attempts=3, capacity_limits=None,
reservation_token=None, bundle_id=None, scan_event_id=None, run_id=None,
cycle_id=None, claim_order='oldest', docker_depth_authority=None,
final_cutover=False, remote_assignment=None, reserved_bundle_bytes=None,
remote_max_active=50,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('v2 capacity reservation and target claims require PostgreSQL')
experiment_authority = None
experiment_claim_states = ('active',)
if isinstance(docker_depth_authority, dict) and docker_depth_authority.get(
'enabled'
) is False:
self._hold_disabled_docker_depth_experiment(docker_depth_authority)
if isinstance(docker_depth_authority, dict) and docker_depth_authority.get(
'enabled'
) is True:
if source != 'dockerhub' or platform != 'docker':
raise ValueError(
'Docker depth experiment authority is valid only for DockerHub Docker claims'
)
try:
experiment_authority = self._normalize_docker_depth_resolver_authority(
docker_depth_authority
)
from docker_depth_experiment import (
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
)
if (
experiment_authority['selector_version']
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
):
experiment_claim_states = ('resolving', 'active')
except (TypeError, ValueError):
return None
identity = _identity_mapping(producer_identity)
remote = self._remote_assignment_mapping(remote_assignment)
remote_planning_kind = ''
if remote:
remote_snapshot = json.loads(remote['execution_snapshot_json'])
remote_planning_kind = str(remote_snapshot['planning']['kind'])
if remote_planning_kind == 'docker_direct_v1' and (
source != 'dockerhub' or platform != 'docker'
):
raise ValueError(
'Docker direct assignment requires the DockerHub Docker queue'
)
if remote_planning_kind == 'huggingface_space_v1' and (
source != 'huggingface' or platform != 'huggingface'
):
raise ValueError(
'HuggingFace direct assignment requires the HuggingFace queue'
)
declared_bundle_bytes = max(1, int(declared_bundle_bytes))
reserved_bundle_bytes = (
max(1, int(reserved_bundle_bytes))
if remote and reserved_bundle_bytes is not None
else declared_bundle_bytes
)
remote_max_active = int(remote_max_active)
if remote and not 1 <= remote_max_active <= 10000:
raise ValueError('remote active assignment limit is out of range')
projection_bytes = max(1, int(projection_bytes))
candidate_items = max(0, int(candidate_items))
candidate_bytes = max(0, int(candidate_bytes))
limits = {
'bundle_items': 10000,
'bundle_bytes': 3 * 1024 * 1024 * 1024,
'projection_items': 10000,
'projection_bytes': 2 * 1024 * 1024 * 1024,
'projection_headroom_bytes': 0,
'keycheck_items': 100000,
'keycheck_bytes': 512 * 1024 * 1024,
'quarantine_items': 10000,
'quarantine_bytes': 1024 * 1024 * 1024,
}
limits.update({key: int(value) for key, value in (capacity_limits or {}).items() if key in limits})
if not reservation_token or not bundle_id or not scan_event_id:
raise ValueError('caller must preassign reservation_token, bundle_id, and scan_event_id')
reservation_token = str(reservation_token)
bundle_id = str(bundle_id).lower()
scan_event_id = str(scan_event_id).lower()
if not re.fullmatch(r'[a-f0-9]{32,64}', bundle_id) or not re.fullmatch(r'[a-f0-9]{32,64}', scan_event_id):
raise ValueError('bundle and scan event IDs must be lowercase hexadecimal identities')
claim_order = str(claim_order or 'oldest').strip().lower()
if claim_order not in ('oldest', 'newest', 'balanced'):
raise ValueError('target claim order must be oldest, newest, or balanced')
newest_first = claim_order == 'newest' or (
claim_order == 'balanced' and int(scan_event_id[-1], 16) % 2 == 0
)
claim_direction = 'DESC' if newest_first else 'ASC'
now = utc_now_iso()
owner = (
f'remote:{remote["device_id"]}' if remote
else f'{source}:{identity["pid"]}:{cycle_id or "cycle"}'
)
claim_token = secrets.token_urlsafe(32)
claim_batch = reservation_token
ready_relative_path = f'ready/{bundle_id[:2]}/{bundle_id}.trb'
intent_values = {
'bundle_id': bundle_id,
'scan_event_id': scan_event_id,
'source': str(source),
'platform': str(platform),
'producer_instance_id': str(supervisor_instance_id or ''),
'producer_pid': identity['pid'],
'producer_creation_time': identity['creation_time'],
'producer_executable': identity['executable'],
'declared_bundle_bytes': declared_bundle_bytes,
'reserved_bundle_bytes': reserved_bundle_bytes,
'reserved_projection_items': 1,
'reserved_projection_bytes': projection_bytes,
'reserved_candidate_items': candidate_items,
'reserved_candidate_bytes': candidate_bytes,
'run_id': run_id,
'cycle_id': cycle_id,
'assignment_kind': 'remote' if remote else 'local',
'remote_user_id': remote['user_id'] if remote else None,
'remote_device_id': remote['device_id'] if remote else None,
'remote_effective_config_sha256': (
remote['effective_config_sha256'] if remote else None
),
'remote_client_compat_sha256': remote['client_compat_sha256'] if remote else None,
'remote_result_upload_body_timeout_seconds': (
remote['result_upload_body_timeout_seconds'] if remote else None
),
'remote_execution_snapshot_json': (
remote['execution_snapshot_json'] if remote else None
),
'remote_execution_snapshot_sha256': (
remote['execution_snapshot_sha256'] if remote else None
),
}
self.ensure_admission_intent(
reservation_token, intent_values,
remote_user_id=remote['user_id'] if remote else None,
remote_device_id=remote['device_id'] if remote else None,
)
try:
intent = self.conn.execute(
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
(reservation_token,),
).fetchone()
if not intent or intent['intent_sha256'] != self._admission_intent_sha256(intent_values):
raise ScanEventConflictError('admission intent changed before reservation')
if remote and (
int(intent['remote_user_id'] or 0) != remote['user_id']
or int(intent['remote_device_id'] or 0) != remote['device_id']
):
raise ScanEventConflictError('remote admission intent changed owner identity')
if intent['state'] == 'aborted':
self.conn.commit()
return None
if remote and intent['state'] == 'pending':
control = self._locked_runtime_control_state(shared=True)
if control['effective_dispatch_paused']:
cursor = self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'dispatch_gate_closed',
resolved_at = ?, updated_at = ?
WHERE reservation_token = ? AND state = 'pending' ''',
(now, now, reservation_token),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'dispatch gate lost the pending admission intent fence'
)
self.conn.commit()
return None
experiment = None
if experiment_authority:
experiment, authority_reason = (
self._locked_docker_depth_experiment_authority(
experiment_authority, final_cutover=final_cutover, now=now,
)
)
if not experiment:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = ?, resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(
authority_reason or 'experiment_authority_unavailable',
now, now, reservation_token,
),
)
self.conn.commit()
return None
if str(experiment['state']) == 'released':
experiment = None
experiment_authority = None
remote_user = None
if remote:
remote_user = self.conn.execute(
'''SELECT * FROM remote_worker_users
WHERE id = ? AND disabled_at IS NULL FOR UPDATE''',
(remote['user_id'],),
).fetchone()
credential = self._lock_active_remote_credential(
remote['device_id'], remote['token_sha256'],
user_id=remote['user_id'],
)
if not remote_user or not credential:
raise ScanEventConflictError(
'remote worker credential is disabled, revoked, or rotated'
)
existing = self.conn.execute(
'''SELECT r.*, q.attempts,
binding.id AS experiment_binding_id,
binding.experiment_target_id,
binding.attempt AS experiment_attempt,
target.experiment_id,
target.dispatch_wave, target.dispatch_order,
experiment.experiment_key AS bound_experiment_key
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.reservation_id = r.id
LEFT JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
LEFT JOIN docker_depth_experiments experiment
ON experiment.id = target.experiment_id
WHERE r.reservation_token = ?''',
(reservation_token,),
).fetchone()
if existing:
expected = {
'bundle_id': bundle_id,
'scan_event_id': scan_event_id,
'source': str(source),
'platform': str(platform),
'producer_instance_id': str(supervisor_instance_id or ''),
'producer_pid': identity['pid'],
'producer_creation_time': identity['creation_time'],
'producer_executable': identity['executable'],
'declared_bundle_bytes': declared_bundle_bytes,
'reserved_bundle_bytes': reserved_bundle_bytes,
'reserved_projection_items': 1,
'reserved_projection_bytes': projection_bytes,
'reserved_candidate_items': candidate_items,
'reserved_candidate_bytes': candidate_bytes,
'run_id': run_id,
'cycle_id': cycle_id,
'assignment_kind': 'remote' if remote else 'local',
'remote_user_id': remote['user_id'] if remote else None,
'remote_device_id': remote['device_id'] if remote else None,
'remote_effective_config_sha256': (
remote['effective_config_sha256'] if remote else None
),
'remote_client_compat_sha256': (
remote['client_compat_sha256'] if remote else None
),
'remote_execution_snapshot_json': (
remote['execution_snapshot_json'] if remote else None
),
'remote_execution_snapshot_sha256': (
remote['execution_snapshot_sha256'] if remote else None
),
}
if any(str(existing[key]) != str(value) for key, value in expected.items()):
raise ScanEventConflictError('reservation token resolves to conflicting claim identity')
if experiment_authority and (
existing['experiment_binding_id'] is None
or str(existing['bound_experiment_key'])
!= experiment_authority['experiment_key']
):
raise ScanEventConflictError(
'reservation token is not bound to the active Docker depth experiment'
)
if remote:
remote_assignment_execution_plan(existing)
self.conn.execute(
'''UPDATE admission_intents SET state = 'committed', reservation_id = ?,
resolution_detail = 'existing_reservation', resolved_at = COALESCE(resolved_at, ?),
updated_at = ? WHERE reservation_token = ?''',
(existing['id'], now, now, reservation_token),
)
self.conn.commit()
return self._result_reservation_claim(existing)
if experiment_authority:
if (
str(experiment['state']) == 'held'
and str(experiment['hold_reason_code'] or '')
== 'pipeline_capacity_conflict'
):
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'active', hold_reason_code = NULL,
held_at = NULL, updated_at = ?
WHERE id = ? AND state = 'held'
AND hold_reason_code = 'pipeline_capacity_conflict' ''',
(now, experiment['id']),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth capacity backpressure recovery lost its fence'
)
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ? FOR UPDATE',
(experiment['id'],),
).fetchone()
if str(experiment['state']) not in experiment_claim_states:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = ?, resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(
f"experiment_{experiment['state']}", now, now,
reservation_token,
),
)
self.conn.commit()
return None
health = self.conn.execute(
'''SELECT state, lease_expires_at, supervisor_instance_id
FROM pipeline_leases WHERE worker_name = 'result_ingester' '''
).fetchone()
if not (
health and health['state'] == 'ready'
and str(health['lease_expires_at'] or '') > now
and str(health['supervisor_instance_id'] or '') == str(supervisor_instance_id or '')
):
if experiment:
self._hold_docker_depth_experiment_locked(
experiment, 'ingester_lease_conflict', now,
)
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'ingester_not_ready', resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(now, now, reservation_token),
)
self.conn.commit()
return None
maximum_attempts = max(0, int(max_attempts or 0))
if experiment:
row, selection_reason = self._claim_docker_depth_experiment_target_locked(
experiment, experiment_authority, now, maximum_attempts,
)
if selection_reason:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = ?, resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(selection_reason, now, now, reservation_token),
)
self.conn.commit()
return None
if row:
active = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE id = ? FOR SHARE''',
(experiment['id'],),
).fetchone()
if active and str(active['state']) not in experiment_claim_states:
active = None
final_reason = self._docker_depth_authority_drift_reason(
active, experiment_authority,
) if active else None
if not active:
self.conn.rollback()
return None
if final_reason:
self._hold_docker_depth_experiment_locked(
active, final_reason, now,
)
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = ?, resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(final_reason, now, now, reservation_token),
)
self.conn.commit()
return None
experiment = active
else:
row = self.conn.execute(
f'''SELECT id, query, target, normalized_target, attempts
FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND status = 'pending'
AND (available_after IS NULL OR available_after <= ?)
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id {claim_direction}
LIMIT 1 FOR UPDATE SKIP LOCKED''',
(source, platform, now, maximum_attempts, maximum_attempts),
).fetchone()
if not row:
row = self.conn.execute(
f'''SELECT id, query, target, normalized_target, attempts
FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND status = 'deferred' AND available_after IS NOT NULL
AND available_after <= ?
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY available_after {claim_direction}, id {claim_direction}
LIMIT 1 FOR UPDATE SKIP LOCKED''',
(source, platform, now, maximum_attempts, maximum_attempts),
).fetchone()
if not row:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'no_claimable_target', resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(now, now, reservation_token),
)
self.conn.commit()
if experiment:
self.refresh_docker_depth_experiment_state(
experiment_authority, final_cutover=final_cutover,
)
return None
if remote:
remote_assignment_execution_plan({
'assignment_kind': 'remote',
'source': source,
'platform': platform,
'target': row['target'],
'normalized_target': row['normalized_target'],
'remote_effective_config_sha256': remote[
'effective_config_sha256'
],
'remote_execution_snapshot_json': remote[
'execution_snapshot_json'
],
'remote_execution_snapshot_sha256': remote[
'execution_snapshot_sha256'
],
'git_scan_plan_json': None,
'git_scan_plan_sha256': None,
'docker_layer_plan_json': None,
'docker_layer_plan_sha256': None,
})
# Global accounting is always acquired after the exact workload object.
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if not capacity:
raise RuntimeSafetySchemaError('pipeline capacity singleton is unavailable')
if remote:
active = self.conn.execute(
'''SELECT COUNT(*) AS user_value,
(SELECT COUNT(*) FROM result_reservations
WHERE assignment_kind = 'remote'
AND remote_resolved_at IS NULL) AS global_value
FROM result_reservations
WHERE assignment_kind = 'remote' AND remote_user_id = ?
AND remote_resolved_at IS NULL''',
(remote['user_id'],),
).fetchone()
quota_reason = None
if int(active['user_value'] or 0) >= int(
remote_user['active_assignment_cap'] or 0
):
quota_reason = 'remote_user_quota_closed'
elif int(active['global_value'] or 0) >= remote_max_active:
quota_reason = 'remote_global_quota_closed'
if quota_reason:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = ?, resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(quota_reason, now, now, reservation_token),
)
self.conn.commit()
return None
if (
int(capacity['quarantine_items'] or 0) >= max(0, limits['quarantine_items'])
or int(capacity['quarantine_bytes'] or 0) >= max(0, limits['quarantine_bytes'])
):
if experiment:
self._hold_docker_depth_experiment_locked(
experiment, 'quarantine_capacity_conflict', now,
)
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'quarantine_capacity_conflict',
resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(now, now, reservation_token),
)
self.conn.commit()
return None
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'quarantine_admission_closed',
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
(now, now, reservation_token),
)
self.conn.commit()
return None
requested = {
'bundle_items': 1,
'bundle_bytes': reserved_bundle_bytes,
'projection_items': 1,
'projection_bytes': projection_bytes,
'keycheck_items': candidate_items,
'keycheck_bytes': candidate_bytes,
}
projection_ceiling = max(
0,
limits['projection_bytes']
- max(0, limits['projection_headroom_bytes']),
)
capacity_closed = any(
int(capacity[key] or 0) + requested[key] > max(0, limits[key])
for key in requested if key != 'projection_bytes'
) or (
int(capacity['projection_bytes'] or 0)
+ requested['projection_bytes'] > projection_ceiling
)
if capacity_closed:
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'pipeline_capacity_closed',
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
(now, now, reservation_token),
)
self.conn.commit()
return None
# Ownership starts only after every contended admission lock is held.
now, lease_until = fixed_lease_window(lease_seconds)
reservation_id = self.conn.insert_returning_id(
'''INSERT INTO result_reservations(
reservation_token, bundle_id, scan_event_id, queue_id, run_id, cycle_id,
source, platform, query, target, normalized_target,
claim_lease_owner, claim_lease_token, claim_batch,
producer_instance_id, producer_pid, producer_creation_time, producer_executable,
assignment_kind, remote_user_id, remote_device_id, remote_issued_at,
remote_expires_at, remote_result_upload_body_timeout_seconds,
remote_effective_config_sha256,
remote_client_compat_sha256, remote_execution_snapshot_json,
remote_execution_snapshot_sha256,
declared_bundle_bytes, reserved_bundle_bytes,
reserved_projection_items, reserved_projection_bytes,
reserved_candidate_items, reserved_candidate_bytes, ready_relative_path,
state, producer_lease_expires_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, 1, ?, ?, ?, ?,
'scanning', ?, ?, ?)''',
(
reservation_token, bundle_id, scan_event_id, row['id'], run_id, cycle_id,
source, platform, row['query'], row['target'], row['normalized_target'],
owner, claim_token, claim_batch, str(supervisor_instance_id or ''),
identity['pid'], identity['creation_time'], identity['executable'],
'remote' if remote else 'local', remote['user_id'] if remote else None,
remote['device_id'] if remote else None, now if remote else None,
lease_until if remote else None,
remote['result_upload_body_timeout_seconds'] if remote else None,
remote['effective_config_sha256'] if remote else None,
remote['client_compat_sha256'] if remote else None,
remote['execution_snapshot_json'] if remote else None,
remote['execution_snapshot_sha256'] if remote else None,
declared_bundle_bytes, reserved_bundle_bytes, projection_bytes,
candidate_items, candidate_bytes,
ready_relative_path, lease_until, now, now,
),
)
partial_token = hashlib.sha256(reservation_token.encode('utf-8')).hexdigest()[:24]
for artifact_kind, relative_path, artifact_bytes in (
(
'bundle_partial',
f'tmp/{bundle_id[:2]}/{bundle_id}.{partial_token}.partial',
declared_bundle_bytes,
),
('bundle_ready', ready_relative_path, 0),
):
self.conn.execute(
'''INSERT INTO pipeline_artifacts(
subsystem, artifact_kind, owner_id, owner_key, relative_path,
byte_count, state, created_at, updated_at
) VALUES ('result_bundle', ?, ?, '', ?, ?, 'expected', ?, ?)''',
(
artifact_kind, reservation_id, relative_path,
artifact_bytes, now, now,
),
)
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'in_progress', lease_owner = ?, lease_token = ?,
claim_batch = ?, leased_at = ?, lease_expires_at = ?, attempts = COALESCE(attempts, 0) + 1,
current_result_reservation_id = ?, claim_event_id = ?,
scan_remote_modified_at = remote_modified_at, updated_at = ?
WHERE id = ? AND current_result_reservation_id IS NULL
AND status IN ('pending','deferred')''',
(
owner, claim_token, claim_batch, now, lease_until, reservation_id,
scan_event_id, now, row['id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError('target ownership changed during capacity reservation')
experiment_binding_id = None
experiment_attempt = None
if experiment:
experiment_attempt = int(row['reservation_count']) + 1
experiment_binding_id = self.conn.insert_returning_id(
'''INSERT INTO docker_depth_experiment_scan_bindings(
experiment_target_id, reservation_id, attempt, state,
bound_at, created_at
) VALUES (?, ?, ?, 'reserved', ?, ?)''',
(
row['experiment_target_id'], reservation_id,
experiment_attempt, now, now,
),
)
target_cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_targets
SET state = 'reserved', reservation_count = reservation_count + 1,
terminal_at = NULL, updated_at = ?
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
AND manifest_id = ? AND state = 'pending'
AND reservation_count = ?''',
(
now, row['experiment_target_id'], experiment['id'], row['id'],
row['manifest_id'], row['reservation_count'],
),
)
if not experiment_binding_id or int(target_cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth binding lost its exact target reservation fence'
)
self.conn.execute(
'''UPDATE pipeline_capacity SET
bundle_items = bundle_items + 1,
bundle_bytes = bundle_bytes + ?,
projection_items = projection_items + 1,
projection_bytes = projection_bytes + ?,
keycheck_items = keycheck_items + ?,
keycheck_bytes = keycheck_bytes + ?,
updated_at = ?
WHERE id = 1''',
(
reserved_bundle_bytes, projection_bytes,
candidate_items, candidate_bytes, now,
),
)
self.conn.execute(
'''UPDATE admission_intents SET state = 'committed', reservation_id = ?,
resolution_detail = 'reservation_committed', resolved_at = ?, updated_at = ?
WHERE reservation_token = ?''',
(reservation_id, now, now, reservation_token),
)
self.conn.commit()
created = self.conn.execute(
'''SELECT r.*, q.attempts,
binding.id AS experiment_binding_id,
binding.experiment_target_id,
binding.attempt AS experiment_attempt,
target.experiment_id,
target.dispatch_wave, target.dispatch_order,
experiment.experiment_key AS bound_experiment_key
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.reservation_id = r.id
LEFT JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
LEFT JOIN docker_depth_experiments experiment
ON experiment.id = target.experiment_id
WHERE r.id = ?''',
(reservation_id,),
).fetchone()
self.conn.commit()
return self._result_reservation_claim(created)
except Exception:
self.conn.rollback()
raise
@staticmethod
def _result_reservation_claim(row):
claim = {
'reservation_id': int(row['id']),
'reservation_token': str(row['reservation_token']),
'bundle_id': str(row['bundle_id']),
'scan_event_id': str(row['scan_event_id']),
'queue_id': int(row['queue_id']),
'claim_lease_owner': str(row['claim_lease_owner']),
'claim_lease_token': str(row['claim_lease_token']),
'claim_batch': row['claim_batch'],
'attempts': int(row['attempts'] or 0),
'declared_bundle_bytes': int(row['declared_bundle_bytes']),
'ready_relative_path': str(row['ready_relative_path']),
'source': str(row['source']),
'platform': str(row['platform']),
'query': row['query'],
'target': str(row['target']),
'normalized_target': str(row['normalized_target']),
'run_id': row['run_id'],
'cycle_id': row['cycle_id'],
'producer_instance_id': str(row['producer_instance_id']),
'producer_pid': int(row['producer_pid']),
'producer_creation_time': str(row['producer_creation_time']),
'producer_executable': str(row['producer_executable']),
'assignment_kind': str(row['assignment_kind']),
}
if claim['assignment_kind'] == 'remote':
claim.update({
'remote_user_id': int(row['remote_user_id']),
'remote_device_id': int(row['remote_device_id']),
'remote_issued_at': str(row['remote_issued_at']),
'remote_expires_at': str(row['remote_expires_at']),
'remote_result_upload_body_timeout_seconds': (
int(row['remote_result_upload_body_timeout_seconds'])
if row['remote_result_upload_body_timeout_seconds'] is not None
else None
),
'remote_effective_config_sha256': str(row['remote_effective_config_sha256']),
'remote_client_compat_sha256': str(row['remote_client_compat_sha256']),
})
if 'experiment_binding_id' in row.keys() and row['experiment_binding_id'] is not None:
claim.update({
'experiment_binding_id': int(row['experiment_binding_id']),
'experiment_target_id': int(row['experiment_target_id']),
'experiment_attempt': int(row['experiment_attempt']),
'experiment_id': int(row['experiment_id']),
'experiment_key': str(row['bound_experiment_key']),
'dispatch_wave': int(row['dispatch_wave']),
'dispatch_order': int(row['dispatch_order']),
})
return claim
def docker_layer_timeout_canary_eligible(self, reservation_id, claim_lease_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker layer canary eligibility requires PostgreSQL')
reservation_id = int(reservation_id)
claim_lease_token = str(claim_lease_token or '')
if not claim_lease_token:
raise ValueError('Docker layer canary eligibility requires a claim lease token')
now = utc_now_iso()
try:
row = self.conn.execute(
'''SELECT q.target, src.metadata_json
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN scan_result_compat src ON src.target_scan_id = q.target_scan_id
WHERE r.id = ? AND r.state = 'scanning'
AND r.source = 'dockerhub' AND r.platform = 'docker'
AND r.claim_lease_token = ?
AND r.producer_lease_expires_at > ?
AND q.status = 'in_progress'
AND q.lease_token = ?
AND q.lease_expires_at > ?
AND q.current_result_reservation_id = r.id
AND q.claim_event_id = r.scan_event_id''',
(reservation_id, claim_lease_token, now, claim_lease_token, now),
).fetchone()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
if not row:
raise ScanEventConflictError('Docker layer canary eligibility fence changed')
text = str(row['metadata_json'] or '')
if not text or len(text.encode('utf-8')) > DOCKER_LAYER_PLAN_MAX_BYTES:
return False
try:
metadata = json.loads(text)
except (TypeError, ValueError, json.JSONDecodeError):
return False
return docker_layer_canary_metadata_eligible(metadata, row['target'])
def start_docker_adaptive_shadow_report(
self, scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
cohort_size, lease_owner, lease_seconds=3600,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
policies = validate_docker_adaptive_policy_hashes(
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
)
if isinstance(cohort_size, bool):
raise ValueError('Docker adaptive shadow cohort size must be an integer')
cohort_size = int(cohort_size)
if not DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS:
raise ValueError('Docker adaptive shadow cohort size must be between 50 and 100')
lease_owner = str(lease_owner or '').strip()
if not lease_owner or len(lease_owner) > 256 or not lease_owner.isascii():
raise ValueError('Docker adaptive shadow lease owner is invalid')
if isinstance(lease_seconds, bool):
raise ValueError('Docker adaptive shadow lease duration must be an integer')
lease_seconds = int(lease_seconds)
if not 60 <= lease_seconds <= 86400:
raise ValueError('Docker adaptive shadow lease duration is outside the supported range')
report_token = secrets.token_hex(32)
lease_token = secrets.token_urlsafe(32)
now = utc_now_iso()
lease_expires_at = datetime.fromtimestamp(
time.time() + lease_seconds, timezone.utc,
).isoformat(timespec='seconds')
lock_identity = ':'.join(policies.values())
try:
self.conn.execute(
'SELECT pg_catalog.pg_advisory_xact_lock('
'pg_catalog.hashtextextended(?, 1095982177))',
(lock_identity,),
)
self.conn.execute(
'''UPDATE docker_adaptive_shadow_reports SET
state = 'failed', failure_count = failure_count + 1,
passed = 0, completed_at = ?, updated_at = ?
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
AND selection_policy_sha256 = ? AND state = 'running'
AND lease_expires_at <= ?''',
(
now, now, policies['scan_policy_sha256'],
policies['execution_policy_sha256'],
policies['selection_policy_sha256'], now,
),
)
active = self.conn.execute(
'''SELECT id FROM docker_adaptive_shadow_reports
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
AND selection_policy_sha256 = ? AND state = 'running'
LIMIT 1 FOR UPDATE''',
(
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
policies['selection_policy_sha256'],
),
).fetchone()
if active:
raise ScanEventConflictError('Docker adaptive shadow report is already running')
row = self.conn.execute(
'''INSERT INTO docker_adaptive_shadow_reports(
report_token, evaluator_version, state, scan_policy_sha256,
execution_policy_sha256, selection_policy_sha256, cohort_size,
lease_owner, lease_token, lease_expires_at, started_at,
created_at, updated_at
) VALUES (?, ?, 'running', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
RETURNING id, report_token, evaluator_version, state,
scan_policy_sha256, execution_policy_sha256,
selection_policy_sha256, cohort_size, lease_owner,
lease_token, lease_expires_at, started_at''',
(
report_token, DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION,
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
policies['selection_policy_sha256'], cohort_size, lease_owner,
lease_token, lease_expires_at, now, now, now,
),
).fetchone()
self.conn.commit()
return dict(row)
except Exception:
self.conn.rollback()
raise
def docker_adaptive_shadow_controls(self, scan_policy_sha256, cohort_size, selection_salt):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow controls require PostgreSQL')
scan_policy_sha256 = str(scan_policy_sha256 or '').lower()
selection_salt = str(selection_salt or '').lower()
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
raise ValueError('Docker adaptive shadow scan policy must be a lowercase SHA-256')
if not re.fullmatch(r'[a-f0-9]{64}', selection_salt):
raise ValueError('Docker adaptive shadow selection salt must be a lowercase SHA-256')
if isinstance(cohort_size, bool):
raise ValueError('Docker adaptive shadow cohort size must be an integer')
cohort_size = int(cohort_size)
if not DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS:
raise ValueError('Docker adaptive shadow cohort size must be between 50 and 100')
try:
rows = self.conn.execute(
'''WITH eligible AS (
SELECT ts.id AS target_scan_id, ts.normalized_target,
ROW_NUMBER() OVER (
PARTITION BY ts.normalized_target
ORDER BY ts.ended_at DESC, ts.id DESC
) AS target_rank
FROM target_scans ts
JOIN target_queue q
ON q.id = ts.queue_id AND q.target_scan_id = ts.id
JOIN result_reservations r
ON r.id = ts.result_reservation_id AND r.queue_id = q.id
JOIN result_bundles b
ON b.reservation_id = r.id AND b.target_scan_id = ts.id
JOIN scan_result_compat src ON src.target_scan_id = ts.id
WHERE ts.source = 'dockerhub' AND ts.scan_type = 'docker'
AND ts.status IN ('clean','found')
AND COALESCE(ts.error_count, 0) = 0
AND ts.skipped_reason IS NULL AND ts.ended_at IS NOT NULL
AND ts.queue_completion_applied = 1
AND ts.queue_completion_disposition = 'applied'
AND ts.raw_result_storage = 'normalized_v2'
AND q.source = 'dockerhub' AND q.platform = 'docker'
AND q.status = 'done'
AND r.source = 'dockerhub' AND r.platform = 'docker'
AND r.state = 'acknowledged' AND b.state = 'acknowledged'
AND r.docker_layer_plan_json IS NULL
AND r.docker_layer_plan_sha256 IS NULL
AND ts.normalized_target ~ '@sha256:[a-f0-9]{64}$'
AND (ts.scan_options_json::jsonb ->> 'scan_policy_sha256') = ?
AND (src.metadata_json::jsonb #>>
'{scan_meta,docker_scan_assignment,effective_mode}') = 'full'
)
SELECT target_scan_id, normalized_target
FROM eligible WHERE target_rank = 1
ORDER BY md5(? || ':' || normalized_target), target_scan_id
LIMIT ?''',
(scan_policy_sha256, selection_salt, cohort_size),
).fetchall()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
if len(rows) != cohort_size:
raise RuntimeError('Docker adaptive shadow exact-policy control cohort is incomplete')
controls = []
for row in rows:
normalized_target = str(row['normalized_target'] or '')
try:
parsed = parse_docker_target(normalized_target)
except (TypeError, ValueError) as exc:
raise RuntimeError('Docker adaptive shadow control target is invalid') from exc
if not re.search(r'@sha256:[a-f0-9]{64}$', str(parsed.get('image') or '')):
raise RuntimeError('Docker adaptive shadow control target is not immutable')
controls.append({
'target_scan_id': int(row['target_scan_id']),
'normalized_target': normalized_target,
})
return controls
def docker_adaptive_shadow_control_identities(self, target_scan_id):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow control identities require PostgreSQL')
target_scan_id = int(target_scan_id)
if target_scan_id <= 0:
raise ValueError('Docker adaptive shadow control scan identity is invalid')
try:
routed_rows = self.conn.execute(
'''SELECT c.routed_service, kc.provider_key_hash
FROM keycheck_candidates c
JOIN keycheck_credentials kc ON kc.id = c.credential_id
WHERE c.target_scan_id = ?''',
(target_scan_id,),
).fetchall()
detector_rows = self.conn.execute(
'''SELECT detector_secret_hash FROM findings
WHERE target_scan_id = ?''',
(target_scan_id,),
).fetchall()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
routed = set()
for row in routed_rows:
service = str(row['routed_service'] or '')
provider_key_hash = str(row['provider_key_hash'] or '').lower()
if (
not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', service)
or not re.fullmatch(r'[a-f0-9]{64}', provider_key_hash)
):
raise RuntimeError('Docker adaptive shadow routed identity snapshot is incomplete')
routed.add((service, provider_key_hash))
detectors = set()
for row in detector_rows:
detector_secret_hash = str(row['detector_secret_hash'] or '').lower()
if not re.fullmatch(r'[a-f0-9]{64}', detector_secret_hash):
raise RuntimeError('Docker adaptive shadow detector identity snapshot is incomplete')
detectors.add(detector_secret_hash)
return frozenset(routed), frozenset(detectors)
def finish_docker_adaptive_shadow_report(
self, report_token, lease_owner, lease_token, **values,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
report_token = str(report_token or '')
lease_owner = str(lease_owner or '')
lease_token = str(lease_token or '')
if not report_token or not lease_owner or not lease_token:
raise ValueError('Docker adaptive shadow report fence is required')
metrics = docker_adaptive_shadow_gate_metrics(values)
selection_metrics = validate_docker_adaptive_shadow_selection_metrics(
values.get('selection_metrics')
)
if sum(
selection_metrics[name] for name in DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS
) != metrics['omitted_descriptor_count']:
raise ValueError('Docker adaptive shadow omission metrics do not reconcile')
now = utc_now_iso()
try:
row = self.conn.execute(
'''SELECT * FROM docker_adaptive_shadow_reports
WHERE report_token = ? FOR UPDATE''',
(report_token,),
).fetchone()
if not row:
raise ScanEventConflictError('Docker adaptive shadow report is absent')
cohort_size = int(row['cohort_size'])
if metrics['completed_pairs'] > cohort_size:
raise ValueError('Docker adaptive completed pairs exceed the cohort size')
if (
str(row['state']) != 'running'
or str(row['lease_owner']) != lease_owner
or str(row['lease_token']) != lease_token
or str(row['lease_expires_at']) <= now
):
raise ScanEventConflictError('Docker adaptive shadow report fence changed')
if int(row['sink_checkpoint_count']) != metrics['completed_pairs'] * 2:
raise ValueError('Docker adaptive shadow sink checkpoints do not reconcile')
passed = int(
metrics['completed_pairs'] == cohort_size
and DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS
and metrics['full_routed_count'] > 0
and metrics['full_slot_ms'] > 0
and metrics['adaptive_slot_ms'] > 0
and metrics['routed_recall_ppm'] >= DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
and metrics['slot_ratio_ppm'] <= DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
and metrics['failure_count'] == 0
and metrics['privacy_violation_count'] == 0
and metrics['safety_regression_count'] == 0
)
cursor = self.conn.execute(
'''UPDATE docker_adaptive_shadow_reports SET
state = 'completed', completed_pairs = ?,
full_routed_count = ?, adaptive_routed_count = ?,
routed_intersection_count = ?, full_detector_count = ?,
adaptive_detector_count = ?, detector_intersection_count = ?,
full_slot_ms = ?, adaptive_slot_ms = ?,
omitted_descriptor_count = ?, failure_count = ?,
privacy_violation_count = ?, safety_regression_count = ?,
selection_metrics_json = ?,
routed_recall_ppm = ?, slot_ratio_ppm = ?, passed = ?,
completed_at = ?, updated_at = ?
WHERE id = ? AND state = 'running' AND lease_owner = ?
AND lease_token = ? AND lease_expires_at > ?''',
(
metrics['completed_pairs'], metrics['full_routed_count'],
metrics['adaptive_routed_count'], metrics['routed_intersection_count'],
metrics['full_detector_count'], metrics['adaptive_detector_count'],
metrics['detector_intersection_count'], metrics['full_slot_ms'],
metrics['adaptive_slot_ms'], metrics['omitted_descriptor_count'],
metrics['failure_count'], metrics['privacy_violation_count'],
metrics['safety_regression_count'], json_dumps(selection_metrics),
metrics['routed_recall_ppm'],
metrics['slot_ratio_ppm'], passed, now, now, int(row['id']),
lease_owner, lease_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker adaptive shadow report completion lost its fence')
self.conn.commit()
return {
'report_id': int(row['id']),
'passed': bool(passed),
'cohort_size': cohort_size,
**metrics,
'selection_metrics': selection_metrics,
'completed_at': now,
}
except Exception:
self.conn.rollback()
raise
def checkpoint_docker_adaptive_shadow_report(
self, report_token, lease_owner, lease_token, lease_seconds=3600,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
if isinstance(lease_seconds, bool):
raise ValueError('Docker adaptive shadow lease duration must be an integer')
lease_seconds = int(lease_seconds)
if not 60 <= lease_seconds <= 86400:
raise ValueError('Docker adaptive shadow lease duration is outside the supported range')
now = utc_now_iso()
lease_expires_at = datetime.fromtimestamp(
time.time() + lease_seconds, timezone.utc,
).isoformat(timespec='seconds')
try:
row = self.conn.execute(
'''UPDATE docker_adaptive_shadow_reports SET
sink_checkpoint_count = sink_checkpoint_count + 1,
lease_expires_at = ?, updated_at = ?
WHERE report_token = ? AND state = 'running'
AND lease_owner = ? AND lease_token = ? AND lease_expires_at > ?
RETURNING id, sink_checkpoint_count, lease_expires_at''',
(
lease_expires_at, now, str(report_token or ''),
str(lease_owner or ''), str(lease_token or ''), now,
),
).fetchone()
if not row:
raise ScanEventConflictError('Docker adaptive shadow checkpoint lost its fence')
self.conn.commit()
return {
'report_id': int(row['id']),
'sink_checkpoint_count': int(row['sink_checkpoint_count']),
'lease_expires_at': row['lease_expires_at'],
}
except Exception:
self.conn.rollback()
raise
def fail_docker_adaptive_shadow_report(
self, report_token, lease_owner, lease_token,
privacy_violation_count=0, safety_regression_count=0,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
privacy_violation_count = int(privacy_violation_count)
safety_regression_count = int(safety_regression_count)
if privacy_violation_count < 0 or safety_regression_count < 0:
raise ValueError('Docker adaptive shadow failure counters must be non-negative')
now = utc_now_iso()
try:
cursor = self.conn.execute(
'''UPDATE docker_adaptive_shadow_reports SET
state = 'failed', failure_count = failure_count + 1,
privacy_violation_count = ?, safety_regression_count = ?,
passed = 0, completed_at = ?, updated_at = ?
WHERE report_token = ? AND state = 'running'
AND lease_owner = ? AND lease_token = ? AND lease_expires_at > ?''',
(
privacy_violation_count, safety_regression_count, now, now,
str(report_token or ''), str(lease_owner or ''),
str(lease_token or ''), now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker adaptive shadow report failure lost its fence')
self.conn.commit()
except Exception:
self.conn.rollback()
raise
def docker_adaptive_gate_report(
self, reservation_id, claim_lease_token, scan_policy_sha256,
execution_policy_sha256, selection_policy_sha256, max_age_sec=604800,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker adaptive gate lookup requires PostgreSQL')
reservation_id = int(reservation_id)
claim_lease_token = str(claim_lease_token or '')
if not claim_lease_token:
raise ValueError('Docker adaptive gate lookup requires a claim lease token')
policies = validate_docker_adaptive_policy_hashes(
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
)
if isinstance(max_age_sec, bool):
raise ValueError('Docker adaptive gate freshness must be an integer')
max_age_sec = int(max_age_sec)
if not 60 <= max_age_sec <= 2592000:
raise ValueError('Docker adaptive gate freshness is outside the supported range')
now = utc_now_iso()
cutoff = (datetime.now(timezone.utc) - timedelta(seconds=max_age_sec)).isoformat(
timespec='seconds',
)
try:
claim = self.conn.execute(
'''SELECT r.id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? AND r.state = 'scanning'
AND r.source = 'dockerhub' AND r.platform = 'docker'
AND r.claim_lease_token = ? AND r.producer_lease_expires_at > ?
AND q.status = 'in_progress' AND q.lease_token = ?
AND q.lease_expires_at > ?
AND q.current_result_reservation_id = r.id
AND q.claim_event_id = r.scan_event_id
FOR UPDATE OF r, q''',
(reservation_id, claim_lease_token, now, claim_lease_token, now),
).fetchone()
if not claim:
raise ScanEventConflictError('Docker adaptive gate claim fence changed')
row = self.conn.execute(
'''SELECT * FROM docker_adaptive_shadow_reports
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
AND selection_policy_sha256 = ?
ORDER BY id DESC LIMIT 1''',
(
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
policies['selection_policy_sha256'],
),
).fetchone()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
if not row:
return {'passed': False, 'reason': 'missing'}
result = {
'report_id': int(row['id']),
'passed': False,
'reason': 'failed',
'completed_at': row['completed_at'],
}
if str(row['state']) != 'completed':
result['reason'] = str(row['state'])
return result
if not row['completed_at'] or str(row['completed_at']) < cutoff:
result['reason'] = 'stale'
return result
metrics = docker_adaptive_shadow_gate_metrics(dict(row))
try:
selection_metrics = validate_docker_adaptive_shadow_selection_metrics(
row['selection_metrics_json']
)
except ValueError:
result['reason'] = 'invalid_evidence'
return result
cohort_size = int(row['cohort_size'])
expected_pass = bool(
str(row['evaluator_version']) == DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION
and DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS
and metrics['completed_pairs'] == cohort_size
and int(row['sink_checkpoint_count']) == cohort_size * 2
and sum(
selection_metrics[name]
for name in DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS
) == metrics['omitted_descriptor_count']
and metrics['full_routed_count'] > 0
and metrics['full_slot_ms'] > 0
and metrics['adaptive_slot_ms'] > 0
and metrics['routed_recall_ppm'] >= DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
and metrics['slot_ratio_ppm'] <= DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
and metrics['failure_count'] == 0
and metrics['privacy_violation_count'] == 0
and metrics['safety_regression_count'] == 0
and int(row['recall_threshold_ppm']) == DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
and int(row['slot_threshold_ppm']) == DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
and int(row['routed_recall_ppm']) == metrics['routed_recall_ppm']
and int(row['slot_ratio_ppm']) == metrics['slot_ratio_ppm']
)
if bool(row['passed']) != expected_pass or not expected_pass:
result['reason'] = 'thresholds'
return result
result.update({
'passed': True,
'reason': 'passed',
'cohort_size': cohort_size,
'completed_pairs': metrics['completed_pairs'],
'routed_recall_ppm': metrics['routed_recall_ppm'],
'slot_ratio_ppm': metrics['slot_ratio_ppm'],
})
return result
def _alias_compatible_legacy_docker_coverage(
self, descriptor, coverage_policy_sha256, scan_policy_sha256, limits, now,
):
candidates = self.conn.execute(
'''SELECT blob.*, reservation.state AS reservation_state,
reservation.scan_event_id AS reservation_scan_event_id,
reservation.docker_layer_plan_json,
reservation.docker_layer_plan_sha256
FROM docker_content_blobs blob
JOIN result_reservations reservation
ON reservation.id = blob.covered_reservation_id
WHERE blob.digest = ? AND blob.coverage_policy_sha256 <> ?
AND blob.state = 'covered'
AND blob.covered_policy_sha256 = blob.coverage_policy_sha256
ORDER BY blob.covered_reservation_id, blob.coverage_policy_sha256
FOR UPDATE OF blob''',
(descriptor['digest'], coverage_policy_sha256),
).fetchall()
archive_keys = (
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
)
for candidate in candidates:
try:
plan, plan_sha256 = stored_docker_layer_plan(candidate)
except RuntimeSafetySchemaError:
continue
if (
not plan
or plan['version'] != 1
or str(candidate['reservation_state']) not in ('db_committed', 'acknowledged')
or not candidate['covered_reservation_id']
or not candidate['covered_scan_event_id']
or str(candidate['covered_scan_event_id'])
!= str(candidate['reservation_scan_event_id'])
or not candidate['covered_at']
or int(candidate['verified_bytes'] or -1) != descriptor['size']
or str(candidate['descriptor_kind']) != descriptor['kind']
or int(candidate['declared_bytes']) != descriptor['size']
or docker_content_media_class(
candidate['descriptor_kind'], candidate['media_type'],
) != docker_content_media_class(descriptor['kind'], descriptor['media_type'])
or plan['scan_policy_sha256'] != scan_policy_sha256
or docker_layer_plan_coverage_policy_sha256(plan)
!= str(candidate['coverage_policy_sha256'])
or any(plan['limits'][key] != limits[key] for key in archive_keys)
):
continue
matching_descriptors = [
item for item in plan['descriptors']
if item['digest'] == descriptor['digest']
and item['kind'] == descriptor['kind']
and item['size'] == descriptor['size']
and docker_content_media_class(item['kind'], item['media_type'])
== docker_content_media_class(descriptor['kind'], descriptor['media_type'])
and item['selected']
]
if not matching_descriptors:
continue
evidence = self.conn.execute(
'''SELECT 1 AS present FROM docker_image_blob_coverage
WHERE reservation_id = ? AND blob_digest = ?
AND coverage_policy_sha256 = ? AND plan_sha256 = ?
AND descriptor_kind = ? AND selected = 1
AND coverage_state = 'covered' AND covered_at IS NOT NULL
LIMIT 1''',
(
candidate['covered_reservation_id'], descriptor['digest'],
candidate['coverage_policy_sha256'], plan_sha256, descriptor['kind'],
),
).fetchone()
if not evidence:
continue
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET
state = 'covered', attempts = 0, max_attempts = ?,
available_after = NULL, lease_reservation_id = NULL,
lease_token = NULL, lease_plan_sha256 = NULL, lease_expires_at = NULL,
covered_reservation_id = ?, covered_scan_event_id = ?,
covered_policy_sha256 = ?, verified_bytes = ?, covered_at = ?,
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ?
AND state = 'pending' AND attempts = 0
AND lease_reservation_id IS NULL AND covered_reservation_id IS NULL''',
(
limits['blob_max_attempts'], candidate['covered_reservation_id'],
candidate['covered_scan_event_id'], coverage_policy_sha256,
descriptor['size'], candidate['covered_at'], now,
descriptor['digest'], coverage_policy_sha256,
),
)
return int(cursor.rowcount or 0) == 1
return False
def bind_docker_layer_plan(
self, reservation_id, claim_lease_token, resolved, limits,
scan_policy_sha256, blob_lease_seconds=1800, *, payload_classes=None,
checkpoint=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker layer plan binding requires PostgreSQL')
reservation_id = int(reservation_id)
claim_lease_token = str(claim_lease_token or '')
if not claim_lease_token:
raise ValueError('Docker layer plan binding requires a claim lease token')
resolved = validate_docker_layer_resolution(resolved)
limits = validate_docker_layer_limits(limits)
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
raise ValueError('Docker layer scanner policy hash is invalid')
descriptors = [resolved['config']] + list(resolved['layers'])
adaptive = payload_classes is not None or checkpoint is not None
if adaptive:
payload_classes = list(payload_classes or [])
checkpoint = validate_docker_adaptive_checkpoint(checkpoint)
if (
len(payload_classes) != len(resolved['layers'])
or any(
value not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES - {'config'}
for value in payload_classes
)
):
raise ValueError('Docker adaptive payload classes do not match its resolution')
selection_policy_sha256 = docker_layer_selection_policy_sha256(limits)
coverage_policy_sha256 = docker_layer_execution_policy_sha256(
scan_policy_sha256, limits,
)
else:
policy_bytes = json.dumps(
limits, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
).encode('ascii')
selection_policy_sha256 = hashlib.sha256(policy_bytes).hexdigest()
coverage_policy_sha256 = docker_layer_coverage_policy_sha256(
scan_policy_sha256, limits,
)
blob_lease_seconds = max(
limits['blob_timeout_sec'] + DOCKER_BLOB_LEASE_MARGIN_SEC,
int(blob_lease_seconds or 1800),
)
blob_lease_seconds = max(60, min(7200, blob_lease_seconds))
try:
row = self.conn.execute(
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(reservation_id,),
).fetchone()
if not row:
raise ScanEventConflictError('Docker layer reservation is absent')
now, lease_until = fixed_lease_window(blob_lease_seconds)
if (
str(row['state']) != 'scanning'
or str(row['claim_lease_token']) != claim_lease_token
or str(row['queue_status']) != 'in_progress'
or str(row['queue_lease_token']) != claim_lease_token
or int(row['queue_reservation_id'] or 0) != reservation_id
or str(row['queue_event_id']) != str(row['scan_event_id'])
or str(row['queue_lease_expires_at'] or '') <= now
or str(row['producer_lease_expires_at'] or '') <= now
):
raise ScanEventConflictError('Docker layer reservation is no longer actively fenced')
if str(row['assignment_kind']) == 'remote':
remote_expires_at = str(row['remote_expires_at'] or '')
if remote_expires_at <= now:
raise ScanEventConflictError('remote Docker layer reservation has expired')
lease_until = remote_expires_at
if str(row['platform']).lower() != 'docker' or str(row['source']).lower() != 'dockerhub':
raise ScanEventConflictError('Docker layer plan conflicts with the reservation source')
if docker_target_identity(row['target']) != resolved['image']:
raise ScanEventConflictError('Docker layer resolution conflicts with the reservation target')
stored_json = row['docker_layer_plan_json']
stored_sha256 = row['docker_layer_plan_sha256']
if stored_json is not None or stored_sha256 is not None:
stored_plan, stored_sha256 = stored_docker_layer_plan(row)
stored_resolution = [
{
'digest': descriptor['digest'],
'size': descriptor['size'],
'media_type': descriptor['media_type'],
'kind': descriptor['kind'],
'position': descriptor['position'],
}
for descriptor in stored_plan['descriptors']
]
if (
stored_plan['image'] != resolved['image']
or stored_plan['repository'] != resolved['repository']
or stored_plan['manifest_digest'] != resolved['manifest_digest']
or stored_plan['platform_os'] != resolved['platform_os']
or stored_plan['platform_arch'] != resolved['platform_arch']
or stored_plan['manifest_media_type'] != resolved['manifest_media_type']
or str(stored_plan.get('scan_policy_sha256') or '') != scan_policy_sha256
or docker_layer_plan_coverage_policy_sha256(stored_plan) != coverage_policy_sha256
or stored_plan.get('limits') != limits
or stored_plan.get('selection_policy_sha256') != selection_policy_sha256
or stored_resolution != descriptors
or (stored_plan['version'] == 2) != adaptive
or (
adaptive
and (
stored_plan.get('checkpoint') != checkpoint
or [
descriptor['payload_class']
for descriptor in stored_plan['descriptors'][1:]
] != payload_classes
)
)
):
raise ScanEventConflictError('Docker layer plan replay conflicts with its reservation')
self.conn.commit()
return stored_plan
for digest in sorted({descriptor['digest'] for descriptor in descriptors}):
self.conn.execute(
'SELECT pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtextextended(?, 1764436591))',
(digest,),
)
blob_rows = {}
for descriptor in descriptors:
inserted = self.conn.execute(
'''INSERT INTO docker_content_blobs(
digest, coverage_policy_sha256, descriptor_kind,
declared_bytes, media_type, state,
attempts, max_attempts, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?)
ON CONFLICT(digest, coverage_policy_sha256) DO NOTHING''',
(
descriptor['digest'], coverage_policy_sha256, descriptor['kind'],
descriptor['size'], descriptor['media_type'],
limits['blob_max_attempts'], now, now,
),
)
blob = self.conn.execute(
'''SELECT * FROM docker_content_blobs
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
(descriptor['digest'], coverage_policy_sha256),
).fetchone()
digest_metadata = self.conn.execute(
'''SELECT descriptor_kind, declared_bytes, media_type
FROM docker_content_blobs WHERE digest = ?''',
(descriptor['digest'],),
).fetchall()
conflicting_metadata = any(
str(metadata['descriptor_kind']) != descriptor['kind']
or int(metadata['declared_bytes']) != descriptor['size']
or docker_content_media_class(
metadata['descriptor_kind'], metadata['media_type'],
) != docker_content_media_class(
descriptor['kind'], descriptor['media_type'],
)
for metadata in digest_metadata
)
if not blob or conflicting_metadata or (
str(blob['descriptor_kind']) != descriptor['kind']
or int(blob['declared_bytes']) != descriptor['size']
or docker_content_media_class(
blob['descriptor_kind'], blob['media_type'],
) != docker_content_media_class(
descriptor['kind'], descriptor['media_type'],
)
or (
not adaptive
and int(blob['max_attempts']) != limits['blob_max_attempts']
)
):
raise ScanEventConflictError('Docker content digest resolves to conflicting metadata')
if adaptive and int(inserted.rowcount or 0) == 1:
self._alias_compatible_legacy_docker_coverage(
descriptor, coverage_policy_sha256, scan_policy_sha256, limits, now,
)
blob = self.conn.execute(
'''SELECT * FROM docker_content_blobs
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
(descriptor['digest'], coverage_policy_sha256),
).fetchone()
blob_rows[descriptor['digest']] = blob
first_selection_rows = self.conn.execute(
'''SELECT coverage.position, coverage.blob_digest, coverage.selected,
coverage.selection_reason
FROM docker_image_blob_coverage coverage
JOIN (
SELECT position, MIN(reservation_id) AS reservation_id
FROM docker_image_blob_coverage
WHERE queue_id = ? AND manifest_digest = ?
AND (
(? = 1 AND selection_policy_sha256 = ?)
OR (? = 0 AND coverage_policy_sha256 = ?)
)
GROUP BY position
) first_position
ON first_position.position = coverage.position
AND first_position.reservation_id = coverage.reservation_id
WHERE coverage.queue_id = ? AND coverage.manifest_digest = ?
AND (
(? = 1 AND coverage.selection_policy_sha256 = ?)
OR (? = 0 AND coverage.coverage_policy_sha256 = ?)
)
ORDER BY coverage.position''',
(
row['queue_id'], resolved['manifest_digest'],
1 if adaptive else 0, selection_policy_sha256,
1 if adaptive else 0, coverage_policy_sha256,
row['queue_id'], resolved['manifest_digest'],
1 if adaptive else 0, selection_policy_sha256,
1 if adaptive else 0, coverage_policy_sha256,
),
).fetchall()
first_selection = {
int(selection['position']): selection for selection in first_selection_rows
}
if first_selection:
if len(first_selection) != len(descriptors):
raise ScanEventConflictError('Docker image selection baseline is incomplete')
for descriptor in descriptors:
selection = first_selection.get(descriptor['position'])
if not selection or str(selection['blob_digest']) != descriptor['digest']:
raise ScanEventConflictError('Docker image selection baseline changed')
entries = []
if adaptive:
classes_by_position = {
0: 'config',
**{
position: payload_class
for position, payload_class in enumerate(payload_classes, start=1)
},
}
if first_selection:
entries = [
(
descriptor,
bool(first_selection[descriptor['position']]['selected']),
str(first_selection[descriptor['position']]['selection_reason']),
classes_by_position[descriptor['position']],
)
for descriptor in descriptors
]
else:
covered_digests = {
digest for digest, blob in blob_rows.items()
if str(blob['state']) == 'covered'
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
}
entries = select_docker_adaptive_payload(
descriptors, payload_classes, limits, covered_digests,
)
class_rank = {
payload_class: index
for index, payload_class in enumerate(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER)
}
entries.sort(key=lambda entry: (
0 if entry[0]['kind'] == 'config' else 1 + class_rank[entry[3]],
-entry[0]['position'], entry[0]['size'], entry[0]['digest'],
))
else:
selected_layer_bytes = 0
selected_layer_count = 0
selected_layer_digests = set()
config = descriptors[0]
if first_selection:
selection = first_selection[config['position']]
entries.append((
config, bool(selection['selected']),
str(selection['selection_reason']), None,
))
elif config['media_type'] not in DOCKER_CONFIG_MEDIA_TYPES:
entries.append((config, False, 'unsupported_media_type', None))
elif config['size'] > limits['config_max_bytes']:
entries.append((config, False, 'config_too_large', None))
else:
entries.append((config, True, 'config_selected', None))
for descriptor in reversed(descriptors[1:]):
blob = blob_rows[descriptor['digest']]
if first_selection:
selection = first_selection[descriptor['position']]
entries.append((
descriptor, bool(selection['selected']),
str(selection['selection_reason']), None,
))
continue
globally_covered = (
str(blob['state']) == 'covered'
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
)
if globally_covered:
entries.append((descriptor, True, 'globally_covered', None))
elif descriptor['digest'] in selected_layer_digests:
entries.append((descriptor, True, 'duplicate_selected_content', None))
elif descriptor['media_type'] not in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
entries.append((descriptor, False, 'unsupported_media_type', None))
elif descriptor['size'] > limits['layer_max_bytes']:
entries.append((descriptor, False, 'layer_too_large', None))
elif selected_layer_count >= limits['max_layers']:
entries.append((descriptor, False, 'layer_limit_exhausted', None))
elif selected_layer_bytes + descriptor['size'] > limits['image_max_bytes']:
entries.append((descriptor, False, 'image_budget_exhausted', None))
else:
entries.append((descriptor, True, 'layer_selected', None))
selected_layer_digests.add(descriptor['digest'])
selected_layer_count += 1
selected_layer_bytes += descriptor['size']
planned = []
pending_leases = []
lease_tokens_by_digest = {}
leased_bytes = 0
checkpoint_max_blobs = checkpoint['max_blobs'] if adaptive else 1
checkpoint_max_bytes = checkpoint['max_bytes'] if adaptive else None
for descriptor, selected, reason, payload_class in entries:
blob = blob_rows[descriptor['digest']]
state = str(blob['state'])
effective_max_attempts = min(
int(blob['max_attempts']), limits['blob_max_attempts'],
)
active_other = (
state in ('leased', 'submitted')
and str(blob['lease_expires_at'] or '') > now
and int(blob['lease_reservation_id'] or 0) != reservation_id
)
same_policy_covered = (
state == 'covered'
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
)
lease_token = None
coverage_state = 'skipped'
if selected and same_policy_covered:
coverage_state = 'covered'
if not adaptive:
reason = 'globally_covered'
elif selected and active_other:
coverage_state = 'shared_pending'
if not adaptive:
reason = 'shared_active'
elif selected and (
state == 'failed'
or int(blob['attempts'] or 0) >= effective_max_attempts
):
coverage_state = 'terminal_failed'
if not adaptive:
reason = 'content_attempts_exhausted'
elif selected and str(blob['available_after'] or '') > now:
coverage_state = 'selected'
if not adaptive:
reason = 'content_retry_wait'
elif selected:
lease_token = lease_tokens_by_digest.get(descriptor['digest'])
can_lease = bool(lease_token) or (
len(lease_tokens_by_digest) < checkpoint_max_blobs
and (
checkpoint_max_bytes is None
or leased_bytes + descriptor['size'] <= checkpoint_max_bytes
or not lease_tokens_by_digest
)
)
if can_lease:
coverage_state = 'leased'
else:
coverage_state = 'selected'
if not adaptive:
reason = 'content_checkpoint_pending'
if can_lease and not lease_token:
lease_token = secrets.token_urlsafe(32)
lease_tokens_by_digest[descriptor['digest']] = lease_token
pending_leases.append((descriptor['digest'], lease_token))
leased_bytes += descriptor['size']
planned_descriptor = {
'digest': descriptor['digest'],
'size': descriptor['size'],
'media_type': descriptor['media_type'],
'kind': descriptor['kind'],
'position': descriptor['position'],
'selected': bool(selected),
'selection_reason': reason,
'coverage_state': coverage_state,
'lease_token': lease_token,
'attempt': (
int(blob['attempts'] or 0) + 1
if coverage_state == 'leased'
else min(int(blob['attempts'] or 0), effective_max_attempts)
),
'max_attempts': effective_max_attempts,
}
if adaptive:
planned_descriptor['payload_class'] = payload_class
planned.append(planned_descriptor)
planned.sort(key=lambda item: item['position'])
plan = {
'version': 2 if adaptive else 1,
'image': resolved['image'],
'repository': resolved['repository'],
'manifest_digest': resolved['manifest_digest'],
'platform_os': resolved['platform_os'],
'platform_arch': resolved['platform_arch'],
'manifest_media_type': resolved['manifest_media_type'],
'limits': limits,
'selection_policy_sha256': selection_policy_sha256,
'scan_policy_sha256': scan_policy_sha256,
'descriptors': planned,
}
if adaptive:
plan.update({
'selector_version': DOCKER_ADAPTIVE_SELECTOR_VERSION,
'execution_policy_sha256': coverage_policy_sha256,
'checkpoint': checkpoint,
})
plan_bytes = canonical_docker_layer_plan_bytes(plan)
plan_sha256 = hashlib.sha256(plan_bytes).hexdigest()
for digest, lease_token in pending_leases:
blob = blob_rows[digest]
effective_max = min(int(blob['max_attempts']), limits['blob_max_attempts'])
if (
str(blob['state']) in ('leased', 'submitted')
and blob['lease_reservation_id'] is not None
and int(blob['lease_reservation_id']) != reservation_id
):
self.conn.execute(
'''UPDATE docker_image_blob_coverage
SET coverage_state = 'retryable_failed', covered_at = NULL,
last_error_code = 'content_lease_reclaimed', updated_at = ?
WHERE reservation_id = ? AND plan_sha256 = ?
AND blob_digest = ? AND coverage_policy_sha256 = ?
AND coverage_state IN ('leased','shared_pending')''',
(
now, blob['lease_reservation_id'], blob['lease_plan_sha256'],
digest, coverage_policy_sha256,
),
)
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET
state = 'leased', attempts = attempts + 1, max_attempts = ?,
available_after = NULL, lease_reservation_id = ?, lease_token = ?,
lease_plan_sha256 = ?, lease_expires_at = ?,
covered_reservation_id = NULL, covered_scan_event_id = NULL,
covered_policy_sha256 = NULL, verified_bytes = NULL, covered_at = NULL,
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ?
AND attempts < max_attempts
AND (
state = 'pending'
OR (
state = 'leased'
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
)
)''',
(
effective_max, reservation_id, lease_token, plan_sha256,
lease_until, now, digest, coverage_policy_sha256, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker content lease changed during plan binding')
for descriptor in planned:
conflicting = self.conn.execute(
'''SELECT 1 AS present FROM docker_image_blob_coverage
WHERE queue_id = ? AND position = ? AND (
manifest_digest <> ? OR blob_digest <> ? OR descriptor_kind <> ?
) LIMIT 1 FOR UPDATE''',
(
row['queue_id'], descriptor['position'], resolved['manifest_digest'],
descriptor['digest'], descriptor['kind'],
),
).fetchone()
if conflicting:
raise ScanEventConflictError('Docker image coverage position changed for an immutable target')
self.conn.execute(
'''INSERT INTO docker_image_blob_coverage(
queue_id, manifest_digest, position, blob_digest,
coverage_policy_sha256, selection_policy_sha256,
descriptor_kind, plan_sha256,
reservation_id, selected, selection_reason,
coverage_state, covered_at, last_error_code, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
ON CONFLICT(reservation_id, position) DO UPDATE SET
selected = excluded.selected,
selection_reason = excluded.selection_reason,
coverage_state = excluded.coverage_state,
covered_at = excluded.covered_at,
last_error_code = NULL,
updated_at = excluded.updated_at''',
(
row['queue_id'], resolved['manifest_digest'], descriptor['position'],
descriptor['digest'], coverage_policy_sha256,
selection_policy_sha256, descriptor['kind'], plan_sha256, reservation_id,
1 if descriptor['selected'] else 0, descriptor['selection_reason'],
descriptor['coverage_state'],
now if descriptor['coverage_state'] == 'covered' else None,
now, now,
),
)
cursor = self.conn.execute(
'''UPDATE result_reservations
SET docker_layer_plan_json = ?, docker_layer_plan_sha256 = ?,
producer_lease_expires_at = CASE
WHEN assignment_kind = 'remote' THEN ?
WHEN producer_lease_expires_at < ? THEN ?
ELSE producer_lease_expires_at
END,
updated_at = ?
WHERE id = ? AND state = 'scanning'
AND claim_lease_token = ?
AND docker_layer_plan_json IS NULL AND docker_layer_plan_sha256 IS NULL''',
(
plan_bytes.decode('ascii'), plan_sha256,
lease_until, lease_until, lease_until,
now, reservation_id, claim_lease_token,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker layer plan changed during binding')
cursor = self.conn.execute(
'''UPDATE target_queue SET lease_expires_at = CASE
WHEN ? = 'remote' THEN ?
WHEN lease_expires_at < ? THEN ? ELSE lease_expires_at END,
updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
str(row['assignment_kind']), lease_until, lease_until, lease_until,
now, row['queue_id'], claim_lease_token,
reservation_id, row['scan_event_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker parent lease changed during plan binding')
self.conn.commit()
return plan
except Exception:
self.conn.rollback()
raise
def _submit_docker_blob_leases_locked(self, reservation, now):
plan, plan_sha256 = stored_docker_layer_plan(reservation)
if plan is None:
return 0
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
submitted = 0
seen = set()
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] != 'leased' or descriptor['digest'] in seen:
continue
seen.add(descriptor['digest'])
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET state = 'submitted',
updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ?
AND state IN ('leased','submitted')
AND lease_reservation_id = ? AND lease_token = ?
AND lease_plan_sha256 = ?
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
(
now, descriptor['digest'], coverage_policy_sha256,
int(reservation['id']), descriptor['lease_token'], plan_sha256, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker content lease changed before durable bundle submission'
)
submitted += 1
return submitted
def _release_docker_blob_leases_locked(
self, reservation, reason_code, reason_detail, now, *, refund_attempt=False,
):
plan, plan_sha256 = stored_docker_layer_plan(reservation)
if plan is None:
return 0
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
retry_at = (
datetime.now(timezone.utc) + timedelta(seconds=3600)
).isoformat(timespec='seconds')
released = 0
states = {}
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] != 'leased':
continue
digest = descriptor['digest']
if digest not in states:
blob = self.conn.execute(
'''SELECT * FROM docker_content_blobs
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
(digest, coverage_policy_sha256),
).fetchone()
if not blob:
raise ScanEventConflictError(
'Docker content policy is absent during reservation release'
)
owns_lease = (
str(blob['state']) in ('leased', 'submitted')
and int(blob['lease_reservation_id'] or 0) == int(reservation['id'])
and str(blob['lease_token'] or '') == descriptor['lease_token']
and str(blob['lease_plan_sha256'] or '') == plan_sha256
)
if owns_lease:
attempts = max(
0,
int(blob['attempts'] or 0) - (1 if refund_attempt else 0),
)
terminal = attempts >= int(blob['max_attempts'])
state = 'failed' if terminal else 'pending'
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET state = ?, attempts = ?,
available_after = ?, lease_reservation_id = NULL,
lease_token = NULL, lease_plan_sha256 = NULL,
lease_expires_at = NULL, last_error_code = ?,
last_error_detail = ?, updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ?
AND state IN ('leased','submitted')
AND lease_reservation_id = ? AND lease_token = ?
AND lease_plan_sha256 = ?''',
(
state, attempts, None if terminal else retry_at,
str(reason_code)[:64], first_line(reason_detail, 1000), now,
digest, coverage_policy_sha256, int(reservation['id']),
descriptor['lease_token'], plan_sha256,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker content lease changed during reservation release'
)
states[digest] = {
'coverage_state': 'terminal_failed' if terminal else 'retryable_failed',
'covered_at': None,
'error_code': str(reason_code)[:64],
}
released += 1
elif (
blob['lease_reservation_id'] is not None
and int(blob['lease_reservation_id']) == int(reservation['id'])
):
raise ScanEventConflictError(
'Docker content lease identity changed within its reservation'
)
elif (
str(blob['state']) == 'covered'
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
):
states[digest] = {
'coverage_state': 'covered',
'covered_at': blob['covered_at'],
'error_code': None,
}
elif (
str(blob['state']) == 'failed'
or int(blob['attempts'] or 0) >= int(blob['max_attempts'])
):
states[digest] = {
'coverage_state': 'terminal_failed',
'covered_at': None,
'error_code': str(blob['last_error_code'] or reason_code)[:64],
}
else:
states[digest] = {
'coverage_state': 'retryable_failed',
'covered_at': None,
'error_code': str(reason_code)[:64],
}
disposition = states[digest]
cursor = self.conn.execute(
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
last_error_code = ?, covered_at = ?, updated_at = ?
WHERE queue_id = ? AND position = ? AND reservation_id = ?
AND plan_sha256 = ? AND blob_digest = ?
AND coverage_policy_sha256 = ?
AND coverage_state = 'leased' ''',
(
disposition['coverage_state'], disposition['error_code'],
disposition['covered_at'], now,
int(reservation['queue_id']), descriptor['position'],
int(reservation['id']), plan_sha256, digest,
coverage_policy_sha256,
),
)
if int(cursor.rowcount or 0) != 1:
existing = self.conn.execute(
'''SELECT coverage_state FROM docker_image_blob_coverage
WHERE reservation_id = ? AND position = ? AND plan_sha256 = ?
AND blob_digest = ? AND coverage_policy_sha256 = ?''',
(
int(reservation['id']), descriptor['position'], plan_sha256,
digest, coverage_policy_sha256,
),
).fetchone()
if not existing or str(existing['coverage_state']) != disposition['coverage_state']:
raise ScanEventConflictError(
'Docker image coverage changed during reservation release'
)
return released
def _apply_docker_layer_execution_locked(
self, reservation, metadata, queue_status, now,
):
plan, plan_sha256 = stored_docker_layer_plan(reservation)
metadata_plan = metadata.get('docker_layer_plan')
metadata_execution = metadata.get('docker_layer_execution')
if plan is None:
if metadata_plan is not None or metadata_execution is not None:
raise ScanEventConflictError(
'unbound reservation contains Docker layer execution metadata'
)
return None
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
try:
normalized_metadata_plan = validate_docker_layer_plan(metadata_plan)
metadata_plan_bytes = canonical_docker_layer_plan_bytes(normalized_metadata_plan)
except (TypeError, ValueError) as exc:
raise ScanEventConflictError('Docker layer bundle plan is invalid') from exc
stored_bytes = canonical_docker_layer_plan_bytes(plan)
if (
metadata_plan_bytes != stored_bytes
or hashlib.sha256(metadata_plan_bytes).hexdigest() != plan_sha256
):
raise ScanEventConflictError(
'Docker layer bundle plan does not match its reservation'
)
try:
execution = validate_docker_layer_execution(
metadata_execution, plan, plan_sha256,
)
except (TypeError, ValueError) as exc:
raise ScanEventConflictError('Docker layer bundle execution is invalid') from exc
retry_at = (
datetime.now(timezone.utc) + timedelta(seconds=3600)
).isoformat(timespec='seconds')
plan_by_digest = {}
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] == 'leased':
plan_by_digest.setdefault(descriptor['digest'], descriptor)
for record in execution['blobs']:
descriptor = plan_by_digest[record['digest']]
blob = self.conn.execute(
'''SELECT * FROM docker_content_blobs
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
(record['digest'], coverage_policy_sha256),
).fetchone()
if not blob or (
str(blob['state']) != 'submitted'
or int(blob['lease_reservation_id'] or 0) != int(reservation['id'])
or str(blob['lease_token'] or '') != record['lease_token']
or str(blob['lease_plan_sha256'] or '') != plan_sha256
):
raise ScanEventConflictError(
'Docker layer execution does not own its submitted content lease'
)
terminal = (
record['status'] == 'terminal_failed'
or (
record['status'] == 'retryable_failed'
and int(blob['attempts']) >= int(blob['max_attempts'])
)
)
if record['status'] == 'covered':
blob_state = 'covered'
coverage_state = 'covered'
available_after = None
covered_reservation_id = int(reservation['id'])
covered_scan_event_id = str(reservation['scan_event_id'])
covered_policy_sha256 = coverage_policy_sha256
covered_at = now
error_code = None
else:
blob_state = 'failed' if terminal else 'pending'
coverage_state = 'terminal_failed' if terminal else 'retryable_failed'
available_after = None if terminal else retry_at
covered_reservation_id = None
covered_scan_event_id = None
covered_policy_sha256 = None
covered_at = None
error_code = record['error_code']
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET state = ?, available_after = ?,
lease_reservation_id = NULL, lease_token = NULL,
lease_plan_sha256 = NULL, lease_expires_at = NULL,
covered_reservation_id = ?, covered_scan_event_id = ?,
covered_policy_sha256 = ?, verified_bytes = ?, covered_at = ?,
last_error_code = ?, last_error_detail = NULL, updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ?
AND state = 'submitted'
AND lease_reservation_id = ? AND lease_token = ?
AND lease_plan_sha256 = ?
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
(
blob_state, available_after, covered_reservation_id,
covered_scan_event_id, covered_policy_sha256,
record['verified_bytes'], covered_at, error_code, now,
record['digest'], coverage_policy_sha256,
int(reservation['id']), record['lease_token'], plan_sha256, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker content lease changed during result ingestion'
)
expected_positions = sum(
1 for item in plan['descriptors']
if item['coverage_state'] == 'leased' and item['digest'] == record['digest']
)
cursor = self.conn.execute(
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
covered_at = ?, last_error_code = ?, updated_at = ?
WHERE queue_id = ? AND reservation_id = ? AND plan_sha256 = ?
AND blob_digest = ? AND coverage_policy_sha256 = ?
AND coverage_state = 'leased' ''',
(
coverage_state, covered_at, error_code, now,
int(reservation['queue_id']), int(reservation['id']),
plan_sha256, record['digest'], coverage_policy_sha256,
),
)
if int(cursor.rowcount or 0) != expected_positions:
raise ScanEventConflictError(
'Docker image coverage changed during result ingestion'
)
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] != 'shared_pending':
continue
blob = self.conn.execute(
'''SELECT * FROM docker_content_blobs
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
(descriptor['digest'], coverage_policy_sha256),
).fetchone()
reconciled = None
if (
blob and str(blob['state']) == 'covered'
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
):
reconciled = 'covered'
elif blob and (
str(blob['state']) == 'failed'
or int(blob['attempts'] or 0) >= int(blob['max_attempts'])
):
reconciled = 'terminal_failed'
if reconciled:
cursor = self.conn.execute(
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
covered_at = ?, last_error_code = ?, updated_at = ?
WHERE queue_id = ? AND position = ? AND reservation_id = ?
AND plan_sha256 = ? AND blob_digest = ?
AND coverage_policy_sha256 = ?
AND coverage_state = 'shared_pending' ''',
(
reconciled, now if reconciled == 'covered' else None,
None if reconciled == 'covered' else 'content_attempts_exhausted',
now, int(reservation['queue_id']), descriptor['position'],
int(reservation['id']), plan_sha256, descriptor['digest'],
coverage_policy_sha256,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'shared Docker image coverage changed during result ingestion'
)
coverage_rows = self.conn.execute(
'''SELECT position, selected, coverage_state, blob_digest
FROM docker_image_blob_coverage
WHERE queue_id = ? AND reservation_id = ? AND plan_sha256 = ?
AND coverage_policy_sha256 = ?
ORDER BY position''',
(
int(reservation['queue_id']), int(reservation['id']), plan_sha256,
coverage_policy_sha256,
),
).fetchall()
if len(coverage_rows) != len(plan['descriptors']):
raise ScanEventConflictError('Docker image coverage row count is incomplete')
states = [str(row['coverage_state']) for row in coverage_rows]
if any(state == 'leased' for state in states):
raise ScanEventConflictError('Docker image coverage retained a consumed lease')
if any(state in ('selected', 'shared_pending', 'retryable_failed') for state in states):
image_state = 'retryable'
expected_queue_status = 'deferred'
elif any(state == 'terminal_failed' for state in states):
image_state = 'terminal_incomplete'
expected_queue_status = 'failed'
elif any(state == 'skipped' for state in states):
image_state = 'partial'
expected_queue_status = 'done'
else:
image_state = 'complete'
expected_queue_status = 'done'
frozen_shared_pending = any(
descriptor['coverage_state'] == 'shared_pending'
for descriptor in plan['descriptors']
)
reconciled_shared_completion = (
queue_status == 'deferred'
and expected_queue_status in ('done', 'failed')
and frozen_shared_pending
)
if queue_status != expected_queue_status and not reconciled_shared_completion:
raise DockerCoverageDispositionConflictError(
'Docker image queue disposition conflicts with content coverage'
)
available_after = metadata.get('available_after')
reset_attempts = metadata.get('reset_attempts')
if queue_status == 'deferred':
deferred_at = parse_time(available_after)
authoritative_now = parse_time(now)
expected_reset = (
any(
descriptor['coverage_state'] in ('selected', 'shared_pending')
for descriptor in plan['descriptors']
)
and not any(
record['status'] in ('retryable_failed', 'terminal_failed')
for record in execution['blobs']
)
)
if (
reset_attempts is not expected_reset
or deferred_at is None
or authoritative_now is None
or deferred_at <= authoritative_now
):
raise ScanEventConflictError(
'Docker retry disposition lacks an authoritative future deadline and reset'
)
elif available_after is not None or reset_attempts is not False:
raise ScanEventConflictError(
'terminal Docker image disposition contains retry scheduling metadata'
)
counts = {
state: sum(1 for item in states if item == state)
for state in (
'covered', 'shared_pending', 'retryable_failed',
'terminal_failed', 'skipped',
)
}
return {'state': image_state, **counts}
def bind_git_scan_plan(
self, reservation_id, claim_lease_token, resolved, baseline_depth,
remote_credential=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('exact Git scan plan binding requires PostgreSQL')
reservation_id = int(reservation_id)
claim_lease_token = str(claim_lease_token or '')
if not claim_lease_token:
raise ValueError('Git scan plan binding requires a claim lease token')
resolved = validate_git_resolution(resolved)
try:
baseline_depth = int(baseline_depth)
except (TypeError, ValueError) as exc:
raise ValueError('Git baseline depth must be an integer') from exc
if not 1 <= baseline_depth <= 1000000:
raise ValueError('Git baseline depth must be between 1 and 1000000')
try:
row = self.conn.execute(
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id,
q.covered_ref AS queue_covered_ref,
q.covered_head AS queue_covered_head
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(reservation_id,),
).fetchone()
if not row:
raise ScanEventConflictError('Git scan reservation is absent')
if str(row['assignment_kind']) == 'remote':
credential = dict(remote_credential or {})
if not self._lock_active_remote_credential(
credential.get('device_id'), credential.get('token_sha256'),
user_id=row['remote_user_id'],
) or int(credential.get('device_id') or 0) != int(
row['remote_device_id'] or 0
):
raise ScanEventConflictError(
'remote worker credential changed before Git plan binding'
)
now = utc_now_iso()
if (
str(row['state']) != 'scanning'
or str(row['claim_lease_token']) != claim_lease_token
or str(row['queue_status']) != 'in_progress'
or str(row['queue_lease_token']) != claim_lease_token
or int(row['queue_reservation_id'] or 0) != reservation_id
or str(row['queue_event_id']) != str(row['scan_event_id'])
or str(row['queue_lease_expires_at'] or '') <= now
or str(row['producer_lease_expires_at'] or '') <= now
or (
str(row['assignment_kind']) == 'remote'
and (
row['remote_resolution_kind'] is not None
or str(row['remote_expires_at'] or '') <= now
)
)
):
raise ScanEventConflictError('Git scan reservation is no longer actively fenced')
if str(row['platform']).lower() != resolved['provider']:
raise ScanEventConflictError('Git scan resolution provider conflicts with the reservation')
covered_ref = str(row['queue_covered_ref'] or '')
covered_head = str(row['queue_covered_head'] or '').lower()
if bool(covered_ref) != bool(covered_head):
raise RuntimeSafetySchemaError('Git coverage ref and head must be stored together')
if covered_head and not re.fullmatch(r'[a-f0-9]{40}|[a-f0-9]{64}', covered_head):
raise RuntimeSafetySchemaError('stored Git covered head is invalid')
if covered_ref == resolved['ref'] and covered_head == resolved['head_sha']:
mode = 'noop'
base_sha = covered_head
elif covered_ref == resolved['ref'] and covered_head:
mode = 'delta'
base_sha = covered_head
else:
mode = 'baseline'
base_sha = None
plan = {
'version': 1,
**resolved,
'base_sha': base_sha,
'mode': mode,
'baseline_depth': baseline_depth,
}
plan_bytes = canonical_git_scan_plan_bytes(plan)
plan_json = plan_bytes.decode('ascii')
plan_sha256 = hashlib.sha256(plan_bytes).hexdigest()
stored_json = row['git_scan_plan_json']
stored_sha256 = row['git_scan_plan_sha256']
if stored_json is not None or stored_sha256 is not None:
if str(stored_json or '') != plan_json or str(stored_sha256 or '') != plan_sha256:
raise ScanEventConflictError('Git scan reservation already has a conflicting plan')
self.conn.commit()
return plan
cursor = self.conn.execute(
'''UPDATE result_reservations
SET git_scan_plan_json = ?, git_scan_plan_sha256 = ?, updated_at = ?
WHERE id = ? AND state = 'scanning' AND claim_lease_token = ?
AND git_scan_plan_json IS NULL AND git_scan_plan_sha256 IS NULL''',
(plan_json, plan_sha256, now, reservation_id, claim_lease_token),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Git scan plan binding lost its reservation fence')
self.conn.commit()
return plan
except Exception:
self.conn.rollback()
raise
def remote_bound_git_scan_plan(
self, reservation_id, device_id, claim_lease_token, token_sha256,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote Git scan plan recovery requires PostgreSQL')
reservation_id = int(reservation_id)
device_id = int(device_id)
claim_lease_token = str(claim_lease_token or '')
if reservation_id <= 0 or device_id <= 0 or not claim_lease_token:
raise ValueError('remote Git scan plan recovery identity is invalid')
try:
row = self.conn.execute(
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(reservation_id,),
).fetchone()
if not row or (
str(row['assignment_kind']) != 'remote'
or int(row['remote_device_id'] or 0) != device_id
or str(row['state']) != 'scanning'
or row['remote_resolution_kind'] is not None
or str(row['claim_lease_token']) != claim_lease_token
or str(row['queue_status']) != 'in_progress'
or str(row['queue_lease_token']) != claim_lease_token
or int(row['queue_reservation_id'] or 0) != reservation_id
or str(row['queue_event_id']) != str(row['scan_event_id'])
):
raise ScanEventConflictError('remote Git scan reservation is no longer actively fenced')
if not self._lock_active_remote_credential(
device_id, token_sha256, user_id=row['remote_user_id'],
):
raise ScanEventConflictError(
'remote worker credential changed before Git plan recovery'
)
now = utc_now_iso()
if (
str(row['remote_expires_at'] or '') <= now
or str(row['queue_lease_expires_at'] or '') <= now
or str(row['producer_lease_expires_at'] or '') <= now
):
raise ScanEventConflictError(
'remote Git scan reservation is no longer actively fenced'
)
plan = stored_git_scan_plan(row)
self.conn.commit()
return plan
except Exception:
self.conn.rollback()
raise
def result_reservation_by_token(self, reservation_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result reservation recovery requires PostgreSQL')
row = self.conn.execute(
'SELECT * FROM result_reservations WHERE reservation_token = ?',
(str(reservation_token),),
).fetchone()
self.conn.commit()
return dict(row) if row else None
def recover_result_reservation_claim(self, reservation_token, expected):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result reservation recovery requires PostgreSQL')
token = str(reservation_token)
expected = dict(expected or {})
digest = self._admission_intent_sha256(expected)
now = utc_now_iso()
try:
intent = self.conn.execute(
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
(token,),
).fetchone()
if not intent:
raise RuntimeError('admission intent is absent during exact recovery')
if intent['intent_sha256'] != digest:
raise ScanEventConflictError(
'admission intent token resolves to conflicting recovery identity'
)
if intent['state'] == 'pending':
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'serialized_recovery_observed_no_commit',
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
(now, now, token),
)
self.conn.commit()
return None
if intent['state'] == 'aborted':
self.conn.commit()
return None
row = self.conn.execute(
'''SELECT r.*, q.attempts,
q.id AS bound_queue_id,
q.source AS bound_queue_source,
q.platform AS bound_queue_platform,
q.query AS bound_queue_query,
q.target AS bound_queue_target,
q.normalized_target AS bound_queue_normalized_target,
binding.id AS experiment_binding_id,
binding.experiment_target_id,
binding.attempt AS experiment_attempt,
target.experiment_id,
target.dispatch_wave, target.dispatch_order,
experiment.experiment_key AS bound_experiment_key
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.reservation_id = r.id
LEFT JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
LEFT JOIN docker_depth_experiments experiment
ON experiment.id = target.experiment_id
WHERE r.reservation_token = ? FOR UPDATE OF r, q''',
(token,),
).fetchone()
if not row or int(intent['reservation_id'] or 0) != int(row['id']):
raise RuntimeError('committed admission intent has no exact reservation')
for key, value in expected.items():
if str(row[key]) != str(value):
raise ScanEventConflictError(
'reservation token resolves to conflicting recovered claim identity'
)
queue_identity = {
'id': row['bound_queue_id'],
'source': row['bound_queue_source'],
'platform': row['bound_queue_platform'],
'query': row['bound_queue_query'],
'target': row['bound_queue_target'],
'normalized_target': row['bound_queue_normalized_target'],
}
self._validate_locked_reservation_queue_identity(row, queue_identity)
if str(row['assignment_kind']) == 'remote':
remote_assignment_execution_plan(row)
self.conn.commit()
return self._result_reservation_claim(row)
except Exception:
self.conn.rollback()
raise
def reconcile_remote_assignment_request(
self, reservation_token, device_id, token_sha256,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote assignment request reconciliation requires PostgreSQL')
token = str(reservation_token or '')
device_id, token_sha256 = self._remote_credential_mapping(
device_id, token_sha256,
)
if not re.fullmatch(r'[a-f0-9]{32,64}', token):
raise ValueError('remote assignment request identity is invalid')
now = utc_now_iso()
try:
intent = self.conn.execute(
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
(token,),
).fetchone()
if not intent:
self.conn.commit()
return None
if int(intent['remote_device_id'] or 0) != device_id or not intent['remote_user_id']:
raise ScanEventConflictError(
'remote assignment request belongs to another device'
)
if not self._lock_active_remote_credential(
device_id, token_sha256, user_id=int(intent['remote_user_id']),
):
raise ScanEventConflictError(
'remote worker credential changed before request reconciliation'
)
if intent['state'] == 'pending':
self.conn.execute(
'''UPDATE admission_intents SET state = 'aborted',
resolution_detail = 'serialized_remote_recovery_observed_no_commit',
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
(now, now, token),
)
self.conn.commit()
return {'state': 'aborted'}
if intent['state'] == 'aborted':
self.conn.commit()
return {'state': 'aborted'}
if intent['state'] != 'committed':
raise ScanEventConflictError('remote assignment request state is invalid')
row = self.conn.execute(
'''SELECT r.*, q.attempts,
q.id AS bound_queue_id,
q.source AS bound_queue_source,
q.platform AS bound_queue_platform,
q.query AS bound_queue_query,
q.target AS bound_queue_target,
q.normalized_target AS bound_queue_normalized_target
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.reservation_token = ? FOR UPDATE OF r, q''',
(token,),
).fetchone()
if (
not row or int(intent['reservation_id'] or 0) != int(row['id'])
or str(row['assignment_kind']) != 'remote'
or int(row['remote_user_id'] or 0) != int(intent['remote_user_id'])
or int(row['remote_device_id'] or 0) != device_id
):
raise ScanEventConflictError(
'committed remote assignment request lost its reservation identity'
)
queue_identity = {
'id': row['bound_queue_id'],
'source': row['bound_queue_source'],
'platform': row['bound_queue_platform'],
'query': row['bound_queue_query'],
'target': row['bound_queue_target'],
'normalized_target': row['bound_queue_normalized_target'],
}
self._validate_locked_reservation_queue_identity(row, queue_identity)
snapshot = stored_remote_execution_snapshot(row)
execution_plan = remote_assignment_execution_plan(row, snapshot=snapshot)
receipt = (
self._remote_resolution_from_row(row)
if row['remote_resolution_json'] else None
)
result = {
'state': 'committed',
'claim': self._result_reservation_claim(row),
'execution_snapshot': snapshot,
'execution_snapshot_sha256': str(
row['remote_execution_snapshot_sha256']
),
'execution_plan': execution_plan,
'git_plan': (
execution_plan['bound_plan']
if execution_plan['kind'] == 'exact_git_v1' else None
),
'receipt': receipt,
}
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def renew_result_claim(self, reservation_id, lease_token, lease_seconds=3600):
if not self.conn or not self.conn.is_postgres:
return False
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
reservation = self.conn.execute(
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(int(reservation_id),),
).fetchone()
if not reservation or (
str(reservation['state']) != 'scanning'
or str(reservation['assignment_kind']) == 'remote'
or str(reservation['claim_lease_token']) != str(lease_token)
or str(reservation['producer_lease_expires_at'] or '') <= now
or str(reservation['queue_status']) != 'in_progress'
or str(reservation['queue_lease_token']) != str(lease_token)
or int(reservation['queue_reservation_id'] or 0) != int(reservation_id)
or str(reservation['queue_event_id'] or '') != str(reservation['scan_event_id'])
or str(reservation['queue_lease_expires_at'] or '') <= now
):
self.conn.rollback()
return False
plan, plan_sha256 = stored_docker_layer_plan(reservation)
effective_seconds = max(60, int(lease_seconds))
if plan is not None:
effective_seconds = max(
effective_seconds,
plan['limits']['blob_timeout_sec'] + DOCKER_BLOB_LEASE_MARGIN_SEC,
)
lease_until = datetime.fromtimestamp(
time.time() + effective_seconds, timezone.utc,
).isoformat(timespec='seconds')
if plan is not None:
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
seen = set()
for descriptor in plan['descriptors']:
if descriptor['coverage_state'] != 'leased' or descriptor['digest'] in seen:
continue
seen.add(descriptor['digest'])
cursor = self.conn.execute(
'''UPDATE docker_content_blobs SET lease_expires_at = ?, updated_at = ?
WHERE digest = ? AND coverage_policy_sha256 = ? AND state = 'leased'
AND lease_reservation_id = ? AND lease_token = ?
AND lease_plan_sha256 = ?
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
(
lease_until, now, descriptor['digest'],
coverage_policy_sha256, int(reservation_id),
descriptor['lease_token'], plan_sha256, now,
),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return False
cursor = self.conn.execute(
'''UPDATE result_reservations SET producer_lease_expires_at = ?, updated_at = ?
WHERE id = ? AND claim_lease_token = ? AND state = 'scanning'
AND producer_lease_expires_at > ?''',
(lease_until, now, int(reservation_id), str(lease_token), now),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return False
cursor = self.conn.execute(
'''UPDATE target_queue SET lease_expires_at = ?, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?
AND lease_expires_at > ?''',
(
lease_until, now, reservation['queue_id'], str(lease_token),
int(reservation_id), reservation['scan_event_id'], now,
),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return False
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
@staticmethod
def _validate_result_bundle_reservation_identity(
reservation, values, *, require_header=False,
):
values = dict(values or {})
ready_relative = str(reservation['ready_relative_path']).replace('\\', '/')
def integer_matches(value, expected):
if isinstance(value, bool) or value is None:
return False
try:
return int(value) == int(expected)
except (TypeError, ValueError, OverflowError):
return False
if (
('reservation_id' in values and not integer_matches(
values.get('reservation_id'), reservation['id'],
))
or str(values.get('bundle_id') or '') != str(reservation['bundle_id'])
or str(values.get('scan_event_id') or '')
!= str(reservation['scan_event_id'])
or str(values.get('relative_path') or '').replace('\\', '/')
!= ready_relative
):
raise ValueError('result bundle identity does not match its reservation')
header = values.get('header')
result_metadata = values.get('result_metadata')
if not require_header and header is None and result_metadata is None:
return
if not isinstance(header, dict) or not isinstance(result_metadata, dict):
raise ValueError('result bundle identity metadata is incomplete')
string_header = {
'reservation_token': reservation['reservation_token'],
'bundle_id': reservation['bundle_id'],
'scan_event_id': reservation['scan_event_id'],
'claim_lease_token': reservation['claim_lease_token'],
'source': reservation['source'],
'platform': reservation['platform'],
'query': reservation['query'],
'target': reservation['target'],
'normalized_target': reservation['normalized_target'],
'producer_instance_id': reservation['producer_instance_id'],
'producer_creation_time': reservation['producer_creation_time'],
}
if any(
str(header.get(name) or '') != str(expected or '')
for name, expected in string_header.items()
) or (
str(header.get('ready_relative_path') or '').replace('\\', '/')
!= ready_relative
) or (
str(header.get('producer_executable') or '').replace('\\', '/')
!= str(reservation['producer_executable'] or '').replace('\\', '/')
):
raise ValueError('result bundle header does not match its reservation')
integer_header = {
'format_version': 2,
'reservation_id': reservation['id'],
'queue_id': reservation['queue_id'],
'declared_bytes': reservation['declared_bundle_bytes'],
'producer_pid': reservation['producer_pid'],
}
if any(
not integer_matches(header.get(name), expected)
for name, expected in integer_header.items()
):
raise ValueError('result bundle header does not match its reservation')
for name in ('run_id', 'cycle_id'):
actual = header.get(name)
expected = reservation[name]
if actual is None or expected is None:
matches = actual is None and expected is None
else:
matches = integer_matches(actual, expected)
if not matches:
raise ValueError('result bundle execution identity does not match its reservation')
required_metadata = {
'scan_event_id': reservation['scan_event_id'],
'target': reservation['target'],
'scan_type': reservation['platform'],
}
if any(
str(result_metadata.get(name) or '') != str(expected or '')
for name, expected in required_metadata.items()
):
raise ValueError('result bundle scan identity does not match its reservation')
for name, expected in (
('bundle_id', reservation['bundle_id']),
('source', reservation['source']),
('platform', reservation['platform']),
('query', reservation['query']),
('normalized_target', reservation['normalized_target']),
):
if (
name in result_metadata
and result_metadata[name] is not None
and str(result_metadata[name]) != str(expected)
):
raise ValueError('result bundle scan identity does not match its reservation')
for name, expected in (
('reservation_id', reservation['id']),
('result_reservation_id', reservation['id']),
('queue_id', reservation['queue_id']),
):
if name in result_metadata and not integer_matches(
result_metadata.get(name), expected):
raise ValueError('result bundle scan identity does not match its reservation')
if normalize_target(
result_metadata.get('target'), reservation['platform'],
) != str(reservation['normalized_target']):
raise ValueError('result bundle target identity does not match its reservation')
scan_event_hash = str(values.get('scan_event_hash') or '').lower()
count_values = {
name: values.get(name) for name in (
'actual_bytes', 'frame_count', 'finding_count', 'error_count',
'candidate_count',
)
}
diagnostic_count = values.get('diagnostic_count', 0)
if (
not re.fullmatch(r'[a-f0-9]{64}', scan_event_hash)
or not integer_matches(values.get('reservation_id'), reservation['id'])
):
raise ValueError('result bundle metadata is outside its reservation bounds')
try:
if any(
value is None or isinstance(value, bool)
for value in count_values.values()
):
raise ValueError
counts = {
name: int(value) for name, value in count_values.items()
}
if isinstance(diagnostic_count, bool):
raise ValueError
diagnostic_count = int(diagnostic_count or 0)
except (TypeError, ValueError, OverflowError):
raise ValueError(
'result bundle metadata is outside its reservation bounds'
) from None
if (
counts['actual_bytes'] <= 0
or counts['actual_bytes'] > int(reservation['declared_bundle_bytes'])
or counts['frame_count'] < 3
or counts['frame_count'] != (
counts['finding_count'] + counts['error_count']
+ counts['candidate_count'] + diagnostic_count + 3
)
or diagnostic_count < 0
or any(counts[name] < 0 for name in (
'finding_count', 'error_count', 'candidate_count',
))
):
raise ValueError('result bundle metadata is outside its reservation bounds')
@staticmethod
def _validate_locked_reservation_queue_identity(reservation, queue):
if not queue or (
int(queue['id']) != int(reservation['queue_id'])
or str(queue['source']) != str(reservation['source'])
or str(queue['platform']) != str(reservation['platform'])
or str(queue['query'] or '') != str(reservation['query'] or '')
or str(queue['target']) != str(reservation['target'])
or str(queue['normalized_target'])
!= str(reservation['normalized_target'])
or normalize_target(queue['target'], queue['platform'])
!= str(queue['normalized_target'])
):
raise ScanEventConflictError(
'result reservation lost its exact target queue identity'
)
@staticmethod
def _validate_result_ingestion_arguments(
current_reservation, current_bundle, supplied_reservation, supplied_bundle,
):
supplied_reservation = dict(supplied_reservation or {})
supplied_bundle = dict(supplied_bundle or {})
reservation_fields = (
'bundle_id', 'scan_event_id', 'queue_id', 'source', 'platform',
'query', 'target', 'normalized_target', 'claim_lease_token',
'run_id', 'cycle_id',
)
supplied_reservation_id = supplied_reservation.get(
'id', supplied_reservation.get('reservation_id'),
)
try:
reservation_id_matches = (
not isinstance(supplied_reservation_id, bool)
and int(supplied_reservation_id) == int(current_reservation['id'])
)
except (TypeError, ValueError, OverflowError):
reservation_id_matches = False
if not reservation_id_matches or any(
str(supplied_reservation.get(name) or '')
!= str(current_reservation[name] or '')
for name in reservation_fields
):
raise ScanEventConflictError(
'result ingestion reservation argument lost its exact identity'
)
bundle_fields = (
'reservation_id', 'bundle_id', 'scan_event_id', 'scan_event_hash',
'relative_path', 'actual_bytes', 'frame_count', 'finding_count',
'error_count', 'candidate_count',
)
if any(
str(supplied_bundle.get(name) or '').replace('\\', '/')
!= str(current_bundle[name] or '').replace('\\', '/')
for name in bundle_fields
):
raise ScanEventConflictError(
'result ingestion bundle argument lost its exact identity'
)
def mark_result_bundle_ready(
self, reservation_id, metadata, remote_acceptance=None,
bundle_capacity_bytes=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result bundle readiness requires PostgreSQL')
values = metadata.as_dict() if hasattr(metadata, 'as_dict') else dict(metadata or {})
remote = dict(remote_acceptance or {}) if remote_acceptance is not None else None
diagnostic_projection = None
if remote is not None:
remote = {
'device_id': int(remote.get('device_id') or 0),
'payload_sha256': str(remote.get('payload_sha256') or '').lower(),
'token_sha256': str(remote.get('token_sha256') or '').lower(),
}
if remote['device_id'] <= 0 or not re.fullmatch(
r'[a-f0-9]{64}', remote['payload_sha256'],
) or not re.fullmatch(r'[a-f0-9]{64}', remote['token_sha256']):
raise ValueError('remote bundle acceptance identity is invalid')
diagnostic_projection = {
'version': values.get('effective_diagnostic_projection_version'),
'count': values.get('effective_diagnostic_count'),
'uids_sha256': str(
values.get('effective_diagnostic_uids_sha256') or ''
),
}
if (
diagnostic_projection['version'] != 1
or isinstance(diagnostic_projection['count'], bool)
or not isinstance(diagnostic_projection['count'], int)
or diagnostic_projection['count'] < 0
or not re.fullmatch(
r'[a-f0-9]{64}', diagnostic_projection['uids_sha256'],
)
):
raise ValueError(
'remote diagnostic projection authority is invalid'
)
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
reservation = self.conn.execute(
'''SELECT r.*, q.id AS bound_queue_id,
q.source AS bound_queue_source,
q.platform AS bound_queue_platform,
q.query AS bound_queue_query,
q.target AS bound_queue_target,
q.normalized_target AS bound_queue_normalized_target,
q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(int(reservation_id),),
).fetchone()
if not reservation:
self.conn.rollback()
return False
if remote is not None:
if (
str(reservation['assignment_kind']) != 'remote'
or int(reservation['remote_device_id'] or 0) != remote['device_id']
):
raise ScanEventConflictError('remote bundle owner identity conflicts')
if not self._lock_active_remote_credential(
remote['device_id'], remote['token_sha256'],
user_id=reservation['remote_user_id'],
):
raise ScanEventConflictError(
'remote worker credential changed before bundle acceptance'
)
if reservation['remote_resolution_kind'] is not None:
if (
str(reservation['remote_resolution_kind']) != 'bundle_accepted'
or str(reservation['remote_payload_sha256'] or '')
!= remote['payload_sha256']
):
raise ScanEventConflictError('remote assignment already has a conflicting resolution')
receipt = self._remote_resolution_from_row(reservation)
receipt_diagnostics = receipt.get('diagnostics') or {}
if receipt_diagnostics.get('projection_version') is not None and (
int(reservation['remote_diagnostic_projection_version'] or -1)
!= int(receipt_diagnostics['projection_version'])
or int(reservation['remote_diagnostic_count'] or 0)
!= int(receipt_diagnostics.get('count') or 0)
or str(reservation['remote_diagnostic_uids_sha256'] or '')
!= str(receipt_diagnostics.get('ordered_uid_set_sha256') or '')
):
raise ScanEventConflictError(
'remote diagnostic projection receipt fence is invalid'
)
self.conn.commit()
return receipt
elif str(reservation['assignment_kind']) == 'remote':
raise ScanEventConflictError('remote bundle requires authenticated acceptance')
now = utc_now_iso()
queue_identity = {
'id': reservation['bound_queue_id'],
'source': reservation['bound_queue_source'],
'platform': reservation['bound_queue_platform'],
'query': reservation['bound_queue_query'],
'target': reservation['bound_queue_target'],
'normalized_target': reservation['bound_queue_normalized_target'],
}
self._validate_locked_reservation_queue_identity(
reservation, queue_identity,
)
self._validate_result_bundle_reservation_identity(
reservation, values, require_header='header' in values,
)
if remote is not None:
result_metadata = values.get('result_metadata')
validate_remote_result_execution_plan(
reservation, result_metadata,
)
actual_bytes = int(values['actual_bytes'])
reserved_bytes = int(reservation['reserved_bundle_bytes'])
if actual_bytes > reserved_bytes:
if bundle_capacity_bytes is None:
raise ValueError(
'remote bundle expansion requires a bundle capacity limit'
)
bundle_capacity_bytes = max(0, int(bundle_capacity_bytes))
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
additional_bytes = actual_bytes - reserved_bytes
if (
not capacity
or int(capacity['bundle_bytes']) + additional_bytes
> bundle_capacity_bytes
):
self.conn.rollback()
raise PipelineCapacityUnavailable(
'remote bundle expansion exceeds available capacity'
)
cursor = self.conn.execute(
'''UPDATE result_reservations SET reserved_bundle_bytes = ?,
updated_at = ? WHERE id = ? AND state = 'scanning'
AND reserved_bundle_bytes = ?''',
(actual_bytes, now, reservation_id, reserved_bytes),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'remote bundle capacity expansion lost its reservation fence'
)
self.conn.execute(
'''UPDATE pipeline_capacity SET bundle_bytes = bundle_bytes + ?,
updated_at = ? WHERE id = 1''',
(additional_bytes, now),
)
active_fence = (
str(reservation['state']) == 'scanning'
and str(reservation['queue_status']) == 'in_progress'
and str(reservation['queue_lease_token'] or '') == str(reservation['claim_lease_token'])
and int(reservation['queue_reservation_id'] or 0) == int(reservation_id)
and str(reservation['queue_event_id'] or '') == str(reservation['scan_event_id'])
and str(reservation['queue_lease_expires_at'] or '') > now
and str(reservation['producer_lease_expires_at'] or '') > now
and (
remote is None or str(reservation['remote_expires_at'] or '') > now
)
)
existing = self.conn.execute(
'SELECT scan_event_hash FROM result_bundles WHERE reservation_id = ?',
(int(reservation_id),),
).fetchone()
receipt = None
if remote is not None:
receipt = {
'receipt_id': secrets.token_hex(32),
'resolution': 'bundle_accepted',
'payload_sha256': remote['payload_sha256'],
'reservation_id': int(reservation_id),
'bundle_id': str(reservation['bundle_id']),
'scan_event_id': str(reservation['scan_event_id']),
'accepted_at': now,
}
receipt.update(self._remote_assignment_observability_locked(
reservation,
diagnostic_count=int(
values.get(
'effective_diagnostic_count',
values.get('diagnostic_count') or 0,
)
),
))
receipt['diagnostics'].update({
'projection_version': diagnostic_projection['version'],
'ordered_uid_set_sha256': diagnostic_projection['uids_sha256'],
})
receipt_json = json.dumps(
receipt, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
)
artifact_sha256 = remote['payload_sha256']
else:
receipt_json = None
artifact_sha256 = values.get('scan_event_hash')
if existing:
if str(existing['scan_event_hash']) != str(values.get('scan_event_hash')):
raise ScanEventConflictError('ready bundle replay hash conflicts with its reservation')
if reservation['state'] == 'scanning':
if not active_fence:
self.conn.rollback()
return False
self._submit_docker_blob_leases_locked(reservation, now)
elif reservation['state'] not in ('ready', 'ingesting', 'db_committed', 'acknowledged'):
self.conn.rollback()
return False
if reservation['state'] in ('scanning', 'ready', 'ingesting', 'db_committed'):
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
AND owner_id = ?''',
(now, now, reservation_id),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
byte_count = ?, deleted_at = NULL, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
AND owner_id = ?''',
(
artifact_sha256, int(values.get('actual_bytes') or 0),
now, reservation_id,
),
)
if remote is not None:
cursor = self.conn.execute(
'''UPDATE result_reservations SET
remote_resolution_kind = 'bundle_accepted',
remote_payload_sha256 = ?, remote_receipt_id = ?,
remote_resolution_json = ?, remote_resolved_at = ?,
remote_diagnostic_projection_version = ?,
remote_diagnostic_count = ?,
remote_diagnostic_uids_sha256 = ?, updated_at = ?
WHERE id = ? AND assignment_kind = 'remote'
AND remote_device_id = ? AND remote_resolution_kind IS NULL''',
(
remote['payload_sha256'], receipt['receipt_id'], receipt_json,
now, diagnostic_projection['version'],
diagnostic_projection['count'],
diagnostic_projection['uids_sha256'],
now, reservation_id, remote['device_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('remote bundle receipt lost its reservation fence')
if reservation['state'] in ('scanning', 'ready'):
self._transition_docker_depth_binding_locked(
reservation, 'scanning', 'scanning', now,
)
self.conn.commit()
return receipt if remote is not None else True
if reservation['state'] != 'scanning':
self.conn.rollback()
return False
if not active_fence:
self.conn.rollback()
return False
self._submit_docker_blob_leases_locked(reservation, now)
self.conn.execute(
'''INSERT INTO result_bundles(
reservation_id, bundle_id, scan_event_id, scan_event_hash, format_version,
relative_path, actual_bytes, frame_count, finding_count, error_count,
candidate_count, state, ready_at, updated_at
) VALUES (?, ?, ?, ?, 2, ?, ?, ?, ?, ?, ?, 'ready', ?, ?)''',
(
reservation_id, values['bundle_id'], values['scan_event_id'],
values['scan_event_hash'], values['relative_path'], values['actual_bytes'],
values['frame_count'], values['finding_count'], values['error_count'],
values['candidate_count'], now, now,
),
)
if remote is None:
self.conn.execute(
"UPDATE result_reservations SET state = 'ready', updated_at = ? WHERE id = ?",
(now, reservation_id),
)
else:
cursor = self.conn.execute(
'''UPDATE result_reservations SET state = 'ready',
remote_resolution_kind = 'bundle_accepted',
remote_payload_sha256 = ?, remote_receipt_id = ?,
remote_resolution_json = ?, remote_resolved_at = ?,
remote_diagnostic_projection_version = ?,
remote_diagnostic_count = ?,
remote_diagnostic_uids_sha256 = ?, updated_at = ?
WHERE id = ? AND state = 'scanning' AND assignment_kind = 'remote'
AND remote_device_id = ? AND remote_resolution_kind IS NULL''',
(
remote['payload_sha256'], receipt['receipt_id'], receipt_json,
now, diagnostic_projection['version'],
diagnostic_projection['count'],
diagnostic_projection['uids_sha256'],
now, reservation_id, remote['device_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('remote bundle receipt lost its reservation fence')
self._transition_docker_depth_binding_locked(
reservation, 'scanning', 'scanning', now,
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
AND owner_id = ?''',
(now, now, reservation_id),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
byte_count = ?, deleted_at = NULL, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
AND owner_id = ?''',
(
artifact_sha256, int(values['actual_bytes']), now, reservation_id,
),
)
self.conn.commit()
return receipt if remote is not None else True
except Exception:
self.conn.rollback()
raise
def recover_expired_result_bundle_ready(self, reservation_id, metadata):
"""Recover a validated bundle after its exact producer lease expired."""
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('expired result bundle recovery requires PostgreSQL')
values = metadata.as_dict() if hasattr(metadata, 'as_dict') else dict(metadata or {})
header = values.get('header')
if not isinstance(header, dict):
raise ValueError('expired ready bundle recovery requires its validated header')
try:
now = utc_now_iso()
self._lock_docker_depth_experiment_for_reservation(reservation_id)
reservation = self.conn.execute(
'''SELECT r.*, q.id AS bound_queue_id,
q.source AS bound_queue_source,
q.platform AS bound_queue_platform,
q.query AS bound_queue_query,
q.target AS bound_queue_target,
q.normalized_target AS bound_queue_normalized_target,
q.status AS queue_status, q.lease_token AS queue_lease_token,
q.lease_expires_at AS queue_lease_expires_at,
q.current_result_reservation_id AS queue_reservation_id,
q.claim_event_id AS queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(int(reservation_id),),
).fetchone()
if not reservation:
self.conn.rollback()
return False
if str(reservation['assignment_kind']) == 'remote':
self.conn.rollback()
return False
queue_identity = {
'id': reservation['bound_queue_id'],
'source': reservation['bound_queue_source'],
'platform': reservation['bound_queue_platform'],
'query': reservation['bound_queue_query'],
'target': reservation['bound_queue_target'],
'normalized_target': reservation['bound_queue_normalized_target'],
}
self._validate_locked_reservation_queue_identity(
reservation, queue_identity,
)
self._validate_result_bundle_reservation_identity(
reservation, values, require_header=True,
)
ready_relative = str(reservation['ready_relative_path']).replace('\\', '/')
scan_event_hash = str(values.get('scan_event_hash') or '').lower()
counts = {
name: int(values.get(name) if values.get(name) is not None else -1)
for name in (
'actual_bytes', 'frame_count', 'finding_count', 'error_count',
'candidate_count',
)
}
diagnostic_count = int(values.get('diagnostic_count') or 0)
if (
not re.fullmatch(r'[a-f0-9]{64}', scan_event_hash)
or counts['actual_bytes'] <= 0
or counts['actual_bytes'] > int(reservation['declared_bundle_bytes'])
or counts['frame_count'] < 3
or counts['frame_count'] != (
counts['finding_count'] + counts['error_count']
+ counts['candidate_count'] + diagnostic_count + 3
)
or diagnostic_count < 0
or any(counts[name] < 0 for name in ('finding_count', 'error_count', 'candidate_count'))
):
raise ValueError('expired ready bundle metadata is outside its reservation bounds')
exact_fence = (
str(reservation['state']) == 'scanning'
and str(reservation['queue_status']) == 'in_progress'
and str(reservation['queue_lease_token'] or '') == str(reservation['claim_lease_token'])
and int(reservation['queue_reservation_id'] or 0) == int(reservation_id)
and str(reservation['queue_event_id'] or '') == str(reservation['scan_event_id'])
and bool(reservation['queue_lease_expires_at'])
and str(reservation['queue_lease_expires_at']) <= now
and bool(reservation['producer_lease_expires_at'])
and str(reservation['producer_lease_expires_at']) <= now
)
if not exact_fence:
self.conn.rollback()
return False
if exact_process_identity_state(
reservation['producer_pid'], reservation['producer_creation_time'],
reservation['producer_executable'],
) not in ('dead', 'reused'):
self.conn.rollback()
return False
if reservation['docker_layer_plan_json'] or reservation['docker_layer_plan_sha256']:
self.conn.rollback()
return False
if self.conn.execute(
'''SELECT 1 AS present FROM docker_content_blobs
WHERE lease_reservation_id = ? AND state IN ('leased','submitted') LIMIT 1''',
(int(reservation_id),),
).fetchone():
self.conn.rollback()
return False
if self.conn.execute(
'SELECT 1 AS present FROM result_bundles WHERE reservation_id = ?',
(int(reservation_id),),
).fetchone():
self.conn.rollback()
return False
if self.conn.execute(
'''SELECT 1 AS present FROM pipeline_quarantine
WHERE reservation_id = ? AND review_status = 'pending' LIMIT 1''',
(int(reservation_id),),
).fetchone():
self.conn.rollback()
return False
self._docker_depth_binding_for_reservation_locked(reservation)
self.conn.execute(
'''INSERT INTO result_bundles(
reservation_id, bundle_id, scan_event_id, scan_event_hash, format_version,
relative_path, actual_bytes, frame_count, finding_count, error_count,
candidate_count, state, ready_at, updated_at
) VALUES (?, ?, ?, ?, 2, ?, ?, ?, ?, ?, ?, 'ready', ?, ?)''',
(
int(reservation_id), values['bundle_id'], values['scan_event_id'],
scan_event_hash, ready_relative, counts['actual_bytes'], counts['frame_count'],
counts['finding_count'], counts['error_count'], counts['candidate_count'],
now, now,
),
)
cursor = self.conn.execute(
"UPDATE result_reservations SET state = 'ready', updated_at = ? WHERE id = ? AND state = 'scanning'",
(now, int(reservation_id)),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('expired ready bundle reservation changed during recovery')
self._transition_docker_depth_binding_locked(
reservation, 'scanning', 'scanning', now,
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
AND owner_id = ?''',
(now, now, int(reservation_id)),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
byte_count = ?, deleted_at = NULL, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
AND owner_id = ?''',
(scan_event_hash, counts['actual_bytes'], now, int(reservation_id)),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def active_result_reservations(self, after_id=0, limit=100):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result reservation recovery requires PostgreSQL')
rows = self.conn.execute(
'''SELECT * FROM result_reservations
WHERE id > ? AND state IN ('scanning','ready','ingesting','db_committed')
AND (cleanup_available_after IS NULL OR cleanup_available_after <= ?)
ORDER BY id LIMIT ?''',
(
max(0, int(after_id)), utc_now_iso(),
min(1000, max(1, int(limit))),
),
).fetchall()
self.conn.commit()
return [dict(row) for row in rows]
def defer_result_reservation_cleanup(self, reservation_id, error):
if not self.conn or not self.conn.is_postgres:
return False
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
row = self.conn.execute(
'SELECT cleanup_attempts, state FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if not row:
self.conn.rollback()
return False
attempts = int(row['cleanup_attempts'] or 0) + 1
delay = min(300, 2 ** min(attempts, 8))
available = datetime.fromtimestamp(
time.time() + delay, timezone.utc,
).isoformat(timespec='seconds')
self.conn.execute(
'''UPDATE result_reservations SET cleanup_attempts = ?,
cleanup_available_after = ?, last_error_code = 'storage_cleanup_deferred',
last_error_detail = ?, updated_at = ? WHERE id = ?''',
(attempts, available, first_line(error, 1000), now, int(reservation_id)),
)
bundle = self.conn.execute(
'SELECT target_scan_id, state FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if bundle:
next_state = 'db_committed' if bundle['target_scan_id'] is not None else 'ready'
self.conn.execute(
'''UPDATE result_bundles SET state = ?, available_after = ?,
ingest_lease_generation = NULL, ingest_lease_token = NULL,
ingest_lease_expires_at = NULL, updated_at = ?
WHERE reservation_id = ?''',
(next_state, available, now, int(reservation_id)),
)
self.conn.execute(
'UPDATE result_reservations SET state = ? WHERE id = ?',
(next_state, int(reservation_id)),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def result_bundle_for_reservation(self, reservation_id):
if not self.conn:
return None
row = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ?',
(int(reservation_id),),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
return dict(row) if row else None
def pending_result_bundle_quarantine(self, reservation_id):
if not self.conn or not self.conn.is_postgres:
return None
row = self.conn.execute(
'''SELECT * FROM pipeline_quarantine
WHERE subsystem = 'result_ingester' AND object_type = 'result_bundle'
AND reservation_id = ? AND review_status = 'pending' ''',
(int(reservation_id),),
).fetchone()
self.conn.commit()
return dict(row) if row else None
def claim_ready_result_bundle(self, generation, lease_token, lease_seconds=60):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('result bundle claims require PostgreSQL')
now = utc_now_iso()
expires = datetime.fromtimestamp(
time.time() + max(10, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
ingest_token = secrets.token_urlsafe(32)
try:
owner = self.conn.execute(
'''SELECT 1 AS valid FROM pipeline_leases
WHERE worker_name = 'result_ingester' AND generation = ?
AND lease_token = ? AND state IN ('recovering','ready')
AND lease_expires_at > ? FOR SHARE''',
(int(generation), str(lease_token), now),
).fetchone()
if not owner:
self.conn.rollback()
raise RuntimeError('result ingester singleton fence is not valid')
row = self.conn.execute(
'''SELECT reservation_id FROM result_bundles
WHERE (
state IN ('ready','db_committed')
OR (state = 'ingesting' AND (
ingest_lease_generation IS NULL OR ingest_lease_generation <> ?
OR (ingest_lease_expires_at IS NOT NULL AND ingest_lease_expires_at <= ?)
))
) AND (available_after IS NULL OR available_after <= ?)
ORDER BY CASE WHEN state = 'db_committed' THEN 0 ELSE 1 END,
ready_at, reservation_id
LIMIT 1 FOR UPDATE SKIP LOCKED''',
(int(generation), now, now),
).fetchone()
if not row:
self.conn.rollback()
return None
bundle = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(row['reservation_id'],),
).fetchone()
next_state = 'db_committed' if bundle['state'] == 'db_committed' else 'ingesting'
self.conn.execute(
'''UPDATE result_bundles SET state = ?, ingest_attempts = ingest_attempts + 1,
ingest_lease_generation = ?, ingest_lease_token = ?,
ingest_lease_expires_at = ?, updated_at = ?
WHERE reservation_id = ?''',
(
next_state, int(generation), ingest_token, expires, now,
row['reservation_id'],
),
)
if next_state == 'ingesting':
self.conn.execute(
"UPDATE result_reservations SET state = 'ingesting', updated_at = ? WHERE id = ?",
(now, row['reservation_id']),
)
reservation = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ?',
(row['reservation_id'],),
).fetchone()
updated = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ?',
(row['reservation_id'],),
).fetchone()
self.conn.commit()
return {
'reservation': dict(reservation),
'bundle': dict(updated),
'ingest_lease_token': ingest_token,
}
except Exception:
self.conn.rollback()
raise
def confirm_scan_event(self, event_id, event_hash):
if not self.conn:
raise RuntimeError('database connection is unavailable')
row = self.conn.execute(
'''SELECT id, scan_event_id, scan_event_hash, result_reservation_id,
raw_result_storage, compat_schema_version
FROM target_scans WHERE scan_event_id = ?''',
(str(event_id),),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
if not row:
return None
if str(row['scan_event_hash'] or '') != str(event_hash or ''):
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
return dict(row)
def acknowledge_removed_bundle(self, reservation_id, event_id, event_hash):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('bundle acknowledgement requires PostgreSQL')
now = utc_now_iso()
try:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
reservation = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
bundle = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
event = self.conn.execute(
'SELECT id, scan_event_hash FROM target_scans WHERE scan_event_id = ?',
(str(event_id),),
).fetchone()
if not reservation or not bundle or not event or str(event['scan_event_hash']) != str(event_hash):
self.conn.rollback()
return False
if str(bundle['scan_event_hash']) != str(event_hash):
raise ScanEventConflictError('bundle acknowledgement hash conflicts with committed event')
if not reservation['bundle_credit_released']:
if int(capacity['bundle_items']) < 1 or int(capacity['bundle_bytes']) < int(reservation['reserved_bundle_bytes']):
raise RuntimeError('bundle capacity accounting would become negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET bundle_items = bundle_items - 1,
bundle_bytes = bundle_bytes - ?, updated_at = ? WHERE id = 1''',
(reservation['reserved_bundle_bytes'], now),
)
self.conn.execute(
'''UPDATE result_reservations SET state = 'acknowledged', bundle_credit_released = 1,
released_at = COALESCE(released_at, ?), updated_at = ? WHERE id = ?''',
(now, now, reservation_id),
)
self.conn.execute(
'''UPDATE result_bundles SET state = 'acknowledged', acknowledged_at = COALESCE(acknowledged_at, ?),
ingest_lease_token = NULL, ingest_lease_expires_at = NULL, updated_at = ?
WHERE reservation_id = ?''',
(now, now, reservation_id),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND owner_id = ?
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
(now, now, reservation_id),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
@staticmethod
def _remote_resolution_from_row(row):
raw = str(row['remote_resolution_json'] or '')
try:
value = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise ScanEventConflictError('remote assignment durable receipt is invalid') from exc
if not isinstance(value, dict) or str(value.get('receipt_id') or '') != str(
row['remote_receipt_id'] or ''
):
raise ScanEventConflictError('remote assignment durable receipt identity is invalid')
return value
def _remote_assignment_observability_locked(
self, row, *, diagnostic_count=None,
):
def row_value(name):
return row.get(name) if hasattr(row, 'get') else row[name]
latest = self.conn.execute(
'''SELECT sequence, slot_id, phase, event_timestamp, phase_started_at,
event_json, received_at
FROM worker_progress_events
WHERE reservation_id = ? ORDER BY sequence DESC, id DESC LIMIT 1''',
(int(row['id']),),
).fetchone()
latest_value = None
known_reason = None
scan_deadline_at = None
if latest:
try:
event = json.loads(str(latest['event_json']))
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise WorkerObservabilityConflictError(
'durable worker progress JSON is invalid'
) from exc
latest_value = {
'sequence': int(latest['sequence']),
'slot_id': int(latest['slot_id']),
'phase': str(latest['phase']),
'event_timestamp': str(latest['event_timestamp']),
'phase_started_at': str(latest['phase_started_at'] or ''),
'received_at': str(latest['received_at']),
'progress': dict(event.get('progress') or {}),
}
scan_deadline_at = event.get('scan_deadline_at')
if latest_value['phase'] in {'idle', 'backoff'}:
reason = latest_value['progress'].get('reason')
known_reason = str(reason) if reason is not None else None
if diagnostic_count is None:
count_row = self.conn.execute(
'''SELECT COUNT(*) AS count FROM worker_diagnostics
WHERE reservation_id = ?''',
(int(row['id']),),
).fetchone()
diagnostic_count = int(count_row['count'] or 0) if count_row else 0
accepted_count = row_value('remote_diagnostic_count')
if accepted_count is not None:
diagnostic_count = max(
diagnostic_count, int(accepted_count),
)
timeout_seconds = None
snapshot_raw = row_value('remote_execution_snapshot_json')
if snapshot_raw:
try:
snapshot = json.loads(str(snapshot_raw))
timeout = ((snapshot.get('execution') or {}).get('scan_kwargs') or {}).get(
'timeout_sec'
)
if type(timeout) in (int, float) and not isinstance(timeout, bool):
timeout_seconds = int(timeout)
except (TypeError, ValueError, json.JSONDecodeError):
timeout_seconds = None
return {
'deadlines': {
'assignment_issued_at': (
str(row_value('remote_issued_at'))
if row_value('remote_issued_at') else None
),
'assignment_deadline_at': (
str(row_value('remote_expires_at'))
if row_value('remote_expires_at') else None
),
'scan_deadline_at': scan_deadline_at,
'target_scan_timeout_seconds': timeout_seconds,
'result_upload_body_timeout_seconds': (
int(row_value('remote_result_upload_body_timeout_seconds'))
if row_value('remote_result_upload_body_timeout_seconds') is not None
else None
),
'result_upload_body_timeout_availability': (
'persisted'
if row_value('remote_result_upload_body_timeout_seconds') is not None
else 'legacy/unavailable'
),
'immutable': True,
},
'latest_progress': {
'available': latest_value is not None,
'event': latest_value,
},
'known_reason': known_reason,
'diagnostics': {
'available': int(diagnostic_count) > 0,
'count': int(diagnostic_count),
'projection_version': row_value(
'remote_diagnostic_projection_version'
),
'ordered_uid_set_sha256': row_value(
'remote_diagnostic_uids_sha256'
),
},
}
def remote_assignment_status(self, reservation_id, device_id, token_sha256):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote assignment status requires PostgreSQL')
try:
credential = self._lock_active_remote_credential(
device_id, token_sha256,
)
if not credential:
self.conn.rollback()
return None
row = self.conn.execute(
'''SELECT id, bundle_id, scan_event_id, state, remote_device_id,
remote_issued_at, remote_expires_at,
remote_result_upload_body_timeout_seconds,
remote_execution_snapshot_json,
remote_diagnostic_projection_version,
remote_diagnostic_count,
remote_diagnostic_uids_sha256,
remote_resolution_kind, remote_receipt_id,
remote_resolution_json, remote_payload_sha256, remote_resolved_at
FROM result_reservations
WHERE id = ? AND assignment_kind = 'remote' AND remote_device_id = ?''',
(int(reservation_id), int(device_id)),
).fetchone()
observability = (
self._remote_assignment_observability_locked(row) if row else None
)
self.conn.commit()
except Exception:
self.conn.rollback()
raise
if not row:
return None
if row['remote_resolution_kind'] is not None:
result = dict(self._remote_resolution_from_row(row))
result.setdefault('deadlines', observability['deadlines'])
for name in ('latest_progress', 'known_reason', 'diagnostics'):
result[name] = observability[name]
return result
return {
'reservation_id': int(row['id']),
'bundle_id': str(row['bundle_id']),
'scan_event_id': str(row['scan_event_id']),
'state': str(row['state']),
'expires_at': str(row['remote_expires_at']),
**observability,
}
def remote_assignment_transport(self, reservation_id, device_id, token_sha256):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote assignment transport requires PostgreSQL')
try:
credential = self._lock_active_remote_credential(
device_id, token_sha256,
)
if not credential:
self.conn.rollback()
return None
row = self.conn.execute(
'''SELECT id, reservation_token, bundle_id, scan_event_id, queue_id,
claim_lease_token, declared_bundle_bytes, ready_relative_path,
source, platform, query, target, normalized_target, run_id, cycle_id,
producer_instance_id, producer_pid, producer_creation_time,
producer_executable, state, remote_device_id, remote_expires_at,
remote_resolution_kind, remote_payload_sha256, remote_receipt_id,
remote_resolution_json
FROM result_reservations
WHERE id = ? AND assignment_kind = 'remote' AND remote_device_id = ?''',
(int(reservation_id), int(device_id)),
).fetchone()
self.conn.commit()
except Exception:
self.conn.rollback()
raise
if not row:
return None
result = dict(row)
result['reservation_id'] = int(result['id'])
result['declared_bytes'] = int(result['declared_bundle_bytes'])
result['ready_path'] = str(result['ready_relative_path'])
if result['remote_resolution_kind'] is not None:
result['receipt'] = self._remote_resolution_from_row(row)
return result
def _resolve_remote_scanning_locked(
self, row, resolution_kind, resolution_payload_sha256, detail, now,
receipt_fields=None,
):
reservation_id = int(row['id'])
queue = self.conn.execute(
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
(int(row['queue_id']),),
).fetchone()
if not queue or (
str(queue['status']) != 'in_progress'
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
or int(queue['current_result_reservation_id'] or 0) != reservation_id
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
):
raise ScanEventConflictError('remote assignment lost its exact queue fence')
error_code = (
'remote_assignment_expired'
if resolution_kind == 'expired'
else 'remote_prebundle_infrastructure_failure'
)
self._release_docker_blob_leases_locked(
row, error_code, detail, now, refund_attempt=True,
)
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
deductions = (
int(row['reserved_bundle_bytes']), int(row['reserved_projection_items']),
int(row['reserved_projection_bytes']), int(row['reserved_candidate_items']),
int(row['reserved_candidate_bytes']),
)
if not capacity or (
int(capacity['bundle_items']) < 1
or int(capacity['bundle_bytes']) < deductions[0]
or int(capacity['projection_items']) < deductions[1]
or int(capacity['projection_bytes']) < deductions[2]
or int(capacity['keycheck_items']) < deductions[3]
or int(capacity['keycheck_bytes']) < deductions[4]
):
raise RuntimeError('remote assignment resolution would make capacity accounting negative')
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'pending',
attempts = CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL, available_after = NULL,
current_result_reservation_id = NULL, claim_event_id = NULL,
last_error = ?, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
first_line(detail, 500), now, row['queue_id'], row['claim_lease_token'],
reservation_id, row['scan_event_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('remote assignment queue changed during resolution')
self.conn.execute(
'''UPDATE pipeline_capacity SET
bundle_items = bundle_items - 1, bundle_bytes = bundle_bytes - ?,
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
updated_at = ? WHERE id = 1''',
(*deductions, now),
)
receipt = {
'receipt_id': secrets.token_hex(32),
'resolution': resolution_kind,
'reservation_id': reservation_id,
'bundle_id': str(row['bundle_id']),
'scan_event_id': str(row['scan_event_id']),
'resolved_at': now,
}
receipt.update(self._remote_assignment_observability_locked(row))
if resolution_payload_sha256:
receipt['payload_sha256'] = resolution_payload_sha256
if receipt_fields:
receipt.update(dict(receipt_fields))
receipt_json = json.dumps(
receipt, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
)
cursor = self.conn.execute(
'''UPDATE result_reservations SET state = 'refunded', bundle_credit_released = 1,
last_error_code = ?, last_error_detail = ?, refunded_at = ?,
released_at = ?, remote_resolution_kind = ?, remote_payload_sha256 = ?,
remote_receipt_id = ?, remote_resolution_json = ?, remote_resolved_at = ?,
updated_at = ? WHERE id = ? AND state = 'scanning'
AND assignment_kind = 'remote' AND remote_resolution_kind IS NULL''',
(
error_code, first_line(detail, 1000), now, now, resolution_kind,
resolution_payload_sha256, receipt['receipt_id'], receipt_json,
now, now, reservation_id,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('remote assignment changed during resolution')
self._transition_docker_depth_binding_locked(row, 'released', 'pending', now)
self.conn.execute(
'''UPDATE pipeline_artifacts SET cleanup_attempts = 0,
cleanup_available_after = NULL, cleanup_last_error = NULL,
updated_at = ?
WHERE subsystem = 'result_bundle' AND owner_id = ?
AND artifact_kind IN ('bundle_partial','bundle_ready')
AND state IN ('expected','present')''',
(now, reservation_id),
)
return receipt
def report_remote_prebundle_failure(
self, reservation_id, device_id, token_sha256, report,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote pre-bundle reporting requires PostgreSQL')
report = dict(report or {})
failure_code = str(report.get('failure_code') or '').strip().lower()
if failure_code not in {
'client_process_failed', 'client_storage_failed', 'client_cancelled',
}:
raise ValueError('remote pre-bundle failure code is not permitted')
if set(report) not in (
{'failure_code', 'detail'},
{'failure_code', 'detail', 'diagnostics'},
) or not isinstance(report.get('detail'), str) or len(report['detail']) > 1000:
raise ValueError('remote pre-bundle report shape is invalid')
detail = first_line(report.get('detail') or failure_code, 1000)
normalized_report = {
'failure_code': failure_code,
'detail': report['detail'],
}
diagnostics = report.get('diagnostics')
if diagnostics is not None:
if not isinstance(diagnostics, list) or len(diagnostics) > 32:
raise ValueError('remote pre-bundle diagnostics are invalid')
normalized_report['diagnostics'] = []
diagnostic_uids = set()
for diagnostic in diagnostics:
normalized = _canonical_worker_contract(
'diagnostic', diagnostic,
)[0]
if (
normalized.get('assignment_outcome') != 'prebundle_failed'
or normalized.get('scan_outcome') != 'unavailable'
):
raise ValueError(
'remote pre-bundle diagnostic outcomes are invalid'
)
uid = normalized['diagnostic_uid']
if uid in diagnostic_uids:
raise ValueError(
'remote pre-bundle diagnostic UID is duplicated'
)
diagnostic_uids.add(uid)
normalized_report['diagnostics'].append(normalized)
report_json = json.dumps(
normalized_report,
ensure_ascii=True, sort_keys=True, separators=(',', ':'),
)
if len(report_json.encode('ascii')) > 16 * 1024:
raise ValueError('remote pre-bundle report exceeds its byte bound')
report_sha256 = hashlib.sha256(report_json.encode('utf-8')).hexdigest()
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
row = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if not row or str(row['assignment_kind']) != 'remote' or int(
row['remote_device_id'] or 0
) != int(device_id):
self.conn.rollback()
return None
if not self._lock_active_remote_credential(
device_id, token_sha256, user_id=row['remote_user_id'],
):
raise ScanEventConflictError(
'remote worker credential changed before terminal report'
)
if row['remote_resolution_kind'] is not None:
if (
str(row['remote_resolution_kind']) != 'prebundle_report'
or str(row['remote_payload_sha256'] or '') != report_sha256
):
raise ScanEventConflictError('remote assignment already has a conflicting resolution')
receipt = self._remote_resolution_from_row(row)
self.conn.commit()
return receipt
now = utc_now_iso()
if (
str(row['state']) != 'scanning'
or not row['remote_expires_at']
or str(row['remote_expires_at']) <= now
):
self.conn.rollback()
return None
for diagnostic in normalized_report.get('diagnostics') or ():
self._record_worker_diagnostic(
int(reservation_id), diagnostic, received_at=now,
)
receipt = self._resolve_remote_scanning_locked(
row, 'prebundle_report', report_sha256, detail, now,
receipt_fields={'failure_code': failure_code},
)
self.conn.commit()
return receipt
except Exception:
self.conn.rollback()
raise
def expire_remote_assignment(self, reservation_id, now=None):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote assignment expiry requires PostgreSQL')
now = str(now or utc_now_iso())
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
row = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if not row or (
str(row['assignment_kind']) != 'remote'
or str(row['state']) != 'scanning'
or row['remote_resolution_kind'] is not None
or not row['remote_expires_at']
or str(row['remote_expires_at']) > now
):
self.conn.rollback()
return None
contracts = importlib.import_module('worker_contracts')
observability = self._remote_assignment_observability_locked(row)
latest = observability['latest_progress']['event']
phase_name = (
latest['phase'] if latest is not None
else contracts.WorkerPhase.ASSIGNED.value
)
envelope = contracts.build_diagnostic_envelope(
occurrence_id=(
f'reservation:{int(reservation_id)}:'
f'{row["remote_expires_at"]}:expiry'
),
reservation_id=int(reservation_id),
scan_event_id=None,
slot_id=(latest['slot_id'] if latest is not None else 0),
source=str(row['source']),
phase=contracts.WorkerPhase(phase_name),
kind=contracts.DiagnosticKind.ASSIGNMENT,
category=contracts.DiagnosticCategory.ASSIGNMENT_EXPIRED,
code='assignment.deadline_expired',
summary=(
f'Assignment deadline expired after phase {phase_name}'
if latest is not None else 'Assignment deadline expired without progress'
),
retryable=True,
attempt=1,
assignment_outcome=contracts.AssignmentOutcome.EXPIRED,
scan_outcome=contracts.ScanOutcome.UNAVAILABLE,
occurred_at=_worker_contract_timestamp(row['remote_expires_at']),
captured_at=_worker_contract_timestamp(now),
)
diagnostic = json.loads(
contracts.encode_diagnostic_envelope(envelope).decode('ascii')
)
self._record_worker_diagnostic(
int(reservation_id), diagnostic, received_at=now,
)
receipt = self._resolve_remote_scanning_locked(
row, 'expired', None, 'remote assignment deadline expired', now,
)
self.conn.commit()
return receipt
except Exception:
self.conn.rollback()
raise
def reap_expired_remote_assignments(self, limit=100):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('remote assignment reaper requires PostgreSQL')
now = utc_now_iso()
rows = self.conn.execute(
'''SELECT id FROM result_reservations
WHERE assignment_kind = 'remote' AND state = 'scanning'
AND remote_resolution_kind IS NULL AND remote_expires_at <= ?
ORDER BY remote_expires_at, id LIMIT ?''',
(now, min(1000, max(1, int(limit)))),
).fetchall()
self.conn.commit()
receipts = []
for row in rows:
try:
receipt = self.expire_remote_assignment(row['id'], now=now)
except Exception:
logger.error(
'remote assignment expiry failed for reservation_id=%s',
int(row['id']),
)
continue
if receipt:
receipts.append(receipt)
return receipts
def refund_uncommitted_reservation(
self, reservation_id, exact_producer_identity, reason,
partial_absence_confirmed=False,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('reservation refunds require PostgreSQL')
if partial_absence_confirmed is not True:
raise RuntimeError(
'reservation refund requires tri-state-confirmed deterministic partial absence'
)
identity = _identity_mapping(exact_producer_identity)
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
row = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if not row or row['state'] != 'scanning':
self.conn.rollback()
return False
if str(row['assignment_kind']) != 'local':
self.conn.rollback()
return False
if (
int(row['producer_pid']) != identity['pid']
or str(row['producer_creation_time']) != identity['creation_time']
or os.path.normcase(os.path.realpath(str(row['producer_executable']))) != identity['executable']
):
self.conn.rollback()
return False
queue = self.conn.execute(
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
(int(row['queue_id']),),
).fetchone()
if not queue or (
str(queue['status']) != 'in_progress'
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
or int(queue['current_result_reservation_id'] or 0) != int(row['id'])
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
):
self.conn.rollback()
return False
self._release_docker_blob_leases_locked(
row,
'producer_failed_before_handoff',
reason,
now,
refund_attempt=True,
)
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'pending',
attempts = CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL, available_after = NULL,
current_result_reservation_id = NULL, claim_event_id = NULL,
last_error = ?, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
first_line(reason, 500), now, row['queue_id'], row['claim_lease_token'],
row['id'], row['scan_event_id'],
),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return False
deductions = (
int(row['reserved_bundle_bytes']), int(row['reserved_projection_items']),
int(row['reserved_projection_bytes']), int(row['reserved_candidate_items']),
int(row['reserved_candidate_bytes']),
)
if (
int(capacity['bundle_items']) < 1
or int(capacity['bundle_bytes']) < deductions[0]
or int(capacity['projection_items']) < deductions[1]
or int(capacity['projection_bytes']) < deductions[2]
or int(capacity['keycheck_items']) < deductions[3]
or int(capacity['keycheck_bytes']) < deductions[4]
):
raise RuntimeError('reservation refund would make capacity accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET
bundle_items = bundle_items - 1, bundle_bytes = bundle_bytes - ?,
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
updated_at = ? WHERE id = 1''',
(*deductions, now),
)
self.conn.execute(
'''UPDATE result_reservations SET state = 'refunded', bundle_credit_released = 1,
last_error_code = 'producer_failed_before_handoff', last_error_detail = ?,
refunded_at = ?, released_at = ?, updated_at = ? WHERE id = ?''',
(first_line(reason, 1000), now, now, now, row['id']),
)
self._transition_docker_depth_binding_locked(
row, 'released', 'pending', now,
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND owner_id = ?
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
(now, now, row['id']),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def _finalize_result_bundle_quarantine_locked(
self, reservation, reason_code, reason_detail, source_relative_path,
payload_sha256, byte_count, now,
):
reservation_id = int(reservation['id'])
queue = self.conn.execute(
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
(int(reservation['queue_id']),),
).fetchone()
if str(reservation['state']) == 'quarantined':
if not queue or (
str(queue['status']) != 'quarantined'
or int(queue['current_result_reservation_id'] or 0) != reservation_id
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
or not reservation['bundle_credit_released']
or str(reservation['last_error_code'] or '') != str(reason_code)
or str(reservation['last_error_detail'] or '') != first_line(reason_detail, 2000)
):
raise ScanEventConflictError('quarantined bundle lost its exact queue fence')
bundle = self.conn.execute(
'SELECT state FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(reservation_id,),
).fetchone()
if bundle and str(bundle['state']) != 'quarantined':
raise ScanEventConflictError('quarantined bundle row lost its terminal state')
if source_relative_path:
artifact = self.conn.execute(
'''SELECT state, payload_sha256, byte_count FROM pipeline_artifacts
WHERE subsystem = 'result_bundle'
AND artifact_kind = 'bundle_quarantine' AND owner_id = ?
AND relative_path = ? FOR UPDATE''',
(reservation_id, self._artifact_relative_path(source_relative_path)),
).fetchone()
if not artifact or (
str(artifact['state']) != 'quarantined'
or str(artifact['payload_sha256'] or '') != str(payload_sha256 or '')
or int(artifact['byte_count'] or 0) != max(0, int(byte_count))
):
raise ScanEventConflictError('quarantined bundle artifact evidence changed')
return
else:
if not queue or (
str(queue['status']) != 'in_progress'
or str(queue['lease_token'] or '') != str(reservation['claim_lease_token'])
or int(queue['current_result_reservation_id'] or 0) != reservation_id
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
):
raise ScanEventConflictError('bundle quarantine lost its exact queue fence')
self._release_docker_blob_leases_locked(
reservation,
'bundle_quarantined',
reason_detail or reason_code,
now,
refund_attempt=False,
)
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'quarantined', completed_at = ?, last_error = ?,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
now, first_line(reason_detail or reason_code, 500), now,
reservation['queue_id'], reservation['claim_lease_token'],
reservation_id, reservation['scan_event_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('bundle quarantine queue transition lost its fence')
cursor = self.conn.execute(
'''UPDATE result_reservations SET state = 'quarantined', bundle_credit_released = 1,
last_error_code = ?, last_error_detail = ?, released_at = ?, updated_at = ?
WHERE id = ? AND state = ?''',
(
str(reason_code), first_line(reason_detail, 2000), now, now,
reservation_id, reservation['state'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('bundle quarantine reservation transition lost its fence')
self._transition_docker_depth_binding_locked(
reservation, 'quarantined', 'quarantined', now,
)
self.conn.execute(
'''UPDATE result_bundles SET state = 'quarantined', ingest_lease_token = NULL,
ingest_lease_expires_at = NULL, updated_at = ? WHERE reservation_id = ?''',
(now, reservation_id),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle' AND owner_id = ?
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
(now, now, reservation_id),
)
if source_relative_path:
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'quarantined', payload_sha256 = ?,
byte_count = ?, deleted_at = NULL, updated_at = ?
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_quarantine'
AND owner_id = ? AND relative_path = ?''',
(
str(payload_sha256 or ''), max(0, int(byte_count)), now,
reservation_id, self._artifact_relative_path(source_relative_path),
),
)
def quarantine_result_bundle(
self, reservation_id, reason_code, reason_detail='', source_relative_path='',
payload_sha256='', byte_count=0,
quarantine_max_items=10000, quarantine_max_bytes=1024 * 1024 * 1024,
physical_confirmed=False,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('bundle quarantine requires PostgreSQL')
reason_code = str(reason_code or '').strip()
if not reason_code or len(reason_code) > 64 or not re.fullmatch(r'[a-z0-9_]+', reason_code):
raise ValueError('bundle quarantine reason code is invalid')
reason_detail = first_line(reason_detail, 2000)
source_relative_path = (
self._artifact_relative_path(source_relative_path) if source_relative_path else ''
)
payload_sha256 = str(payload_sha256 or '').strip().lower()
if payload_sha256 and not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
raise ValueError('bundle quarantine payload hash is invalid')
byte_count = max(0, int(byte_count))
now = utc_now_iso()
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
row = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if not row:
self.conn.rollback()
raise ValueError('result reservation is absent')
existing = self.conn.execute(
'''SELECT * FROM pipeline_quarantine
WHERE subsystem = 'result_ingester' AND object_type = 'result_bundle'
AND object_id = ? AND review_status = 'pending' ''',
(int(reservation_id),),
).fetchone()
if existing:
if (
str(existing['source_relative_path'] or '') != str(source_relative_path or '')
or str(existing['payload_sha256'] or '') != str(payload_sha256 or '')
or int(existing['byte_count']) != byte_count
or str(existing['reason_code']) != reason_code
or str(existing['reason_detail'] or '') != reason_detail
or int(existing['reservation_id'] or 0) != int(reservation_id)
or str(existing['event_id'] or '') != str(row['scan_event_id'])
):
raise ScanEventConflictError('bundle quarantine preparation conflicts with existing evidence')
if physical_confirmed:
self._finalize_result_bundle_quarantine_locked(
row, reason_code, reason_detail, source_relative_path,
payload_sha256, byte_count, now,
)
self.conn.commit()
return int(existing['id'])
if str(row['state']) not in ('scanning', 'ready', 'ingesting'):
raise ScanEventConflictError('bundle reservation is not exactly quarantineable')
queue = self.conn.execute(
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
(int(row['queue_id']),),
).fetchone()
if not queue or (
str(queue['status']) != 'in_progress'
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
or int(queue['current_result_reservation_id'] or 0) != int(row['id'])
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
):
raise ScanEventConflictError('bundle quarantine does not own the target queue fence')
bundle = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(int(reservation_id),),
).fetchone()
if str(row['state']) in ('ready', 'ingesting') and (
not bundle
or str(bundle['bundle_id']) != str(row['bundle_id'])
or str(bundle['scan_event_id']) != str(row['scan_event_id'])
or str(bundle['relative_path']).replace('\\', '/')
!= str(row['ready_relative_path']).replace('\\', '/')
or str(bundle['state']) not in ('ready', 'ingesting')
):
raise ScanEventConflictError('bundle quarantine row identity is not exact')
if str(row['state']) == 'scanning' and bundle is not None:
raise ScanEventConflictError('scanning quarantine has an unexpected ready bundle row')
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
projection_items = 0 if row['projection_credit_transferred'] else int(row['reserved_projection_items'])
projection_bytes = 0 if row['projection_credit_transferred'] else int(row['reserved_projection_bytes'])
candidate_items = 0 if row['candidate_credit_transferred'] else int(row['reserved_candidate_items'])
candidate_bytes = 0 if row['candidate_credit_transferred'] else int(row['reserved_candidate_bytes'])
bundle_items = 0 if row['bundle_credit_released'] else 1
bundle_bytes = 0 if row['bundle_credit_released'] else int(row['reserved_bundle_bytes'])
if (
int(capacity['bundle_items']) < bundle_items
or int(capacity['bundle_bytes']) < bundle_bytes
or int(capacity['projection_items']) < projection_items
or int(capacity['projection_bytes']) < projection_bytes
or int(capacity['keycheck_items']) < candidate_items
or int(capacity['keycheck_bytes']) < candidate_bytes
):
raise RuntimeError('bundle quarantine would make capacity accounting negative')
quarantine_capacity_items = bundle_items + projection_items + candidate_items
quarantine_capacity_bytes = bundle_bytes + projection_bytes + candidate_bytes
self.conn.execute(
'''UPDATE pipeline_capacity SET
bundle_items = bundle_items - ?, bundle_bytes = bundle_bytes - ?,
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
quarantine_items = quarantine_items + ?,
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
(
bundle_items, bundle_bytes, projection_items, projection_bytes,
candidate_items, candidate_bytes, quarantine_capacity_items,
quarantine_capacity_bytes, now,
),
)
quarantine_id = self.conn.insert_returning_id(
'''INSERT INTO pipeline_quarantine(
subsystem, object_type, object_id, reservation_id, event_id,
payload_sha256, reason_code, reason_detail, source_relative_path,
byte_count, capacity_items, capacity_bytes, detected_at
) VALUES ('result_ingester', 'result_bundle', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
reservation_id, reservation_id, row['scan_event_id'], payload_sha256,
reason_code, reason_detail, source_relative_path,
byte_count, quarantine_capacity_items,
quarantine_capacity_bytes, now,
),
)
if source_relative_path:
self.conn.execute(
'''INSERT INTO pipeline_artifacts(
subsystem, artifact_kind, owner_id, owner_key, relative_path,
payload_sha256, byte_count, state, created_at, updated_at
) VALUES ('result_bundle','bundle_quarantine',?,'',?,?,?,
'expected',?,?)
ON CONFLICT(subsystem, artifact_kind, owner_id, owner_key)
DO UPDATE SET relative_path = excluded.relative_path,
payload_sha256 = excluded.payload_sha256,
byte_count = excluded.byte_count,
deleted_at = NULL, updated_at = excluded.updated_at''',
(
reservation_id, source_relative_path,
payload_sha256, byte_count, now, now,
),
)
if physical_confirmed:
self._finalize_result_bundle_quarantine_locked(
row, reason_code, reason_detail, source_relative_path,
payload_sha256, byte_count, now,
)
self.conn.commit()
return int(quarantine_id)
except Exception:
self.conn.rollback()
raise
def pipeline_capacity_snapshot(self):
if not self.conn:
return {}
row = self.conn.execute('SELECT * FROM pipeline_capacity WHERE id = 1').fetchone()
if self.conn.is_postgres:
self.conn.commit()
return dict(row) if row else {}
@staticmethod
def _artifact_relative_path(relative_path):
value = str(relative_path or '').replace('\\', '/').strip('/')
if not value or value.startswith('/') or any(part in ('', '.', '..') for part in value.split('/')):
raise ValueError('pipeline artifact path is not a canonical relative path')
return value
def register_pipeline_artifact(
self, subsystem, artifact_kind, owner_id, owner_key, relative_path,
*, state='expected', payload_sha256='', byte_count=0,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('pipeline artifact registration requires PostgreSQL')
if state not in ('expected', 'present', 'quarantined'):
raise ValueError('pipeline artifact registration state is invalid')
relative_path = self._artifact_relative_path(relative_path)
payload_sha256 = str(payload_sha256 or '').lower()
if payload_sha256 and not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
raise ValueError('pipeline artifact payload identity is invalid')
now = utc_now_iso()
try:
row = self.conn.execute(
'''SELECT * FROM pipeline_artifacts
WHERE subsystem = ? AND artifact_kind = ?
AND owner_id = ? AND owner_key = ? FOR UPDATE''',
(str(subsystem), str(artifact_kind), int(owner_id), str(owner_key or '')),
).fetchone()
if row and str(row['relative_path']) != relative_path:
raise ScanEventConflictError('pipeline artifact owner resolves to a conflicting path')
if row and row['payload_sha256'] and payload_sha256 and row['payload_sha256'] != payload_sha256:
raise ScanEventConflictError('pipeline artifact owner resolves to conflicting bytes')
if row:
self.conn.execute(
'''UPDATE pipeline_artifacts SET payload_sha256 = ?, byte_count = ?,
state = ?, deleted_at = NULL, updated_at = ? WHERE id = ?''',
(
payload_sha256 or row['payload_sha256'], max(0, int(byte_count)),
state, now, row['id'],
),
)
artifact_id = int(row['id'])
else:
artifact_id = self.conn.insert_returning_id(
'''INSERT INTO pipeline_artifacts(
subsystem, artifact_kind, owner_id, owner_key, relative_path,
payload_sha256, byte_count, state, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
str(subsystem), str(artifact_kind), int(owner_id), str(owner_key or ''),
relative_path, payload_sha256, max(0, int(byte_count)), state, now, now,
),
)
self.conn.commit()
return int(artifact_id)
except Exception:
self.conn.rollback()
raise
def mark_pipeline_artifact_deleted(self, artifact_id):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('pipeline artifact deletion requires PostgreSQL')
now = utc_now_iso()
cursor = self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, cleanup_attempts = 0, cleanup_available_after = NULL,
cleanup_last_error = NULL, updated_at = ?
WHERE id = ? AND state != 'deleted' ''',
(now, now, int(artifact_id)),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
def defer_pipeline_artifact_cleanup(self, artifact_id, error):
if not self.conn or not self.conn.is_postgres:
return False
now = utc_now_iso()
try:
row = self.conn.execute(
'SELECT cleanup_attempts FROM pipeline_artifacts WHERE id = ? FOR UPDATE',
(int(artifact_id),),
).fetchone()
if not row:
self.conn.rollback()
return False
attempts = int(row['cleanup_attempts'] or 0) + 1
delay = min(300, 2 ** min(attempts, 8))
available = datetime.fromtimestamp(
time.time() + delay, timezone.utc,
).isoformat(timespec='seconds')
self.conn.execute(
'''UPDATE pipeline_artifacts SET cleanup_attempts = ?,
cleanup_available_after = ?, cleanup_last_error = ?, updated_at = ?
WHERE id = ?''',
(attempts, available, first_line(error, 1000), now, int(artifact_id)),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def projection_terminal_temp_artifacts(self, limit=100):
if not self.conn or not self.conn.is_postgres:
return []
rows = self.conn.execute(
'''SELECT a.* FROM pipeline_artifacts a
JOIN projection_jobs j ON j.id = a.owner_id
WHERE a.subsystem = 'jsonl_projector'
AND a.artifact_kind IN ('prepared_stream','projection_tail_temp')
AND a.state IN ('expected','present')
AND j.status IN ('completed','quarantined')
ORDER BY a.id LIMIT ?''',
(min(1000, max(1, int(limit))),),
).fetchall()
self.conn.commit()
return [dict(row) for row in rows]
def bundle_terminal_temp_artifacts(self, limit=100):
if not self.conn or not self.conn.is_postgres:
return []
rows = self.conn.execute(
'''SELECT a.* FROM pipeline_artifacts a
JOIN result_reservations r ON r.id = a.owner_id
WHERE a.subsystem = 'result_bundle'
AND a.artifact_kind IN ('bundle_partial','bundle_ready')
AND a.state IN ('expected','present')
AND r.state IN ('acknowledged','refunded','quarantined')
AND (a.cleanup_available_after IS NULL OR a.cleanup_available_after <= ?)
ORDER BY a.id LIMIT ?''',
(utc_now_iso(), min(1000, max(1, int(limit)))),
).fetchall()
self.conn.commit()
return [dict(row) for row in rows]
def register_projection_tail_quarantine(
self, job_id, append_id, stream_name, relative_path, payload_sha256,
byte_count, max_items, max_bytes,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection tail registration requires PostgreSQL')
relative_path = self._artifact_relative_path(relative_path)
payload_sha256 = str(payload_sha256 or '').lower()
if not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
raise ValueError('projection tail payload identity is invalid')
owner_key = f'{int(append_id)}:{stream_name}:{payload_sha256}'
now = utc_now_iso()
try:
artifact = self.conn.execute(
'''SELECT * FROM pipeline_artifacts
WHERE subsystem = 'jsonl_projector' AND artifact_kind = 'partial_tail'
AND owner_id = ? AND owner_key = ? FOR UPDATE''',
(int(job_id), owner_key),
).fetchone()
if artifact and (
artifact['relative_path'] != relative_path
or artifact['payload_sha256'] != payload_sha256
or int(artifact['byte_count']) != int(byte_count)
):
raise ScanEventConflictError('projection tail artifact identity conflicts')
if not artifact:
artifact_id = self.conn.insert_returning_id(
'''INSERT INTO pipeline_artifacts(
subsystem, artifact_kind, owner_id, owner_key, relative_path,
payload_sha256, byte_count, state, created_at, updated_at
) VALUES ('jsonl_projector','partial_tail',?,?,?,?,?,'expected',?,?)''',
(
int(job_id), owner_key, relative_path, payload_sha256,
max(0, int(byte_count)), now, now,
),
)
else:
artifact_id = int(artifact['id'])
quarantine = self.conn.execute(
'''SELECT id FROM pipeline_quarantine
WHERE subsystem = 'jsonl_projector' AND object_type = 'projection_tail'
AND object_id = ? AND review_status = 'pending' FOR UPDATE''',
(artifact_id,),
).fetchone()
if not quarantine:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
self.conn.execute(
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items + 1,
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
(int(byte_count), now),
)
quarantine_id = self.conn.insert_returning_id(
'''INSERT INTO pipeline_quarantine(
subsystem, object_type, object_id, projection_job_id,
payload_sha256, reason_code, reason_detail, source_relative_path,
byte_count, capacity_items, capacity_bytes, detected_at
) VALUES ('jsonl_projector','projection_tail',?,?,?,?,?,?,?,?,?,?)''',
(
artifact_id, int(job_id), payload_sha256, 'partial_projection_tail',
f'append={int(append_id)} stream={stream_name}', relative_path,
int(byte_count), 1, int(byte_count), now,
),
)
else:
quarantine_id = int(quarantine['id'])
self.conn.commit()
return {'artifact_id': int(artifact_id), 'quarantine_id': int(quarantine_id)}
except Exception:
self.conn.rollback()
raise
def confirm_projection_tail_artifact(self, artifact_id, payload_sha256, byte_count):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection tail confirmation requires PostgreSQL')
now = utc_now_iso()
cursor = self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'quarantined', payload_sha256 = ?,
byte_count = ?, updated_at = ?
WHERE id = ? AND subsystem = 'jsonl_projector'
AND artifact_kind = 'partial_tail'
AND payload_sha256 = ? AND byte_count = ?
AND state IN ('expected','quarantined')''',
(
str(payload_sha256), int(byte_count), now, int(artifact_id),
str(payload_sha256), int(byte_count),
),
)
self.conn.commit()
return int(cursor.rowcount or 0) == 1
def review_pipeline_quarantine(
self, quarantine_id, expected_reason_code, expected_payload_sha256,
action, audit_sha256,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('pipeline quarantine review requires PostgreSQL')
action = str(action or '').lower()
if action not in ('discard', 'rescan', 'retry'):
raise ValueError('quarantine review action must be discard, rescan, or retry')
if not re.fullmatch(r'[a-f0-9]{64}', str(audit_sha256 or '')):
raise ValueError('quarantine review audit identity is invalid')
now = utc_now_iso()
try:
preview = self.conn.execute(
'''SELECT object_type, reservation_id FROM pipeline_quarantine
WHERE id = ?''',
(int(quarantine_id),),
).fetchone()
experiment = None
reservation = None
if (
preview and str(preview['object_type']) == 'result_bundle'
and preview['reservation_id'] is not None
):
experiment = self._lock_docker_depth_experiment_for_reservation(
preview['reservation_id']
)
reservation = self.conn.execute(
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
(preview['reservation_id'],),
).fetchone()
row = self.conn.execute(
'SELECT * FROM pipeline_quarantine WHERE id = ? FOR UPDATE',
(int(quarantine_id),),
).fetchone()
if not row:
raise ValueError('pipeline quarantine row is absent')
if row['review_status'] != 'pending':
if row['review_audit_sha256'] == audit_sha256:
self.conn.commit()
return {'reviewed': True, 'duplicate': True, 'status': row['review_status']}
raise ScanEventConflictError('pipeline quarantine row was already reviewed differently')
if (
str(row['reason_code']) != str(expected_reason_code)
or str(row['payload_sha256'] or '') != str(expected_payload_sha256 or '')
):
raise ScanEventConflictError('pipeline quarantine review evidence does not match')
if row['object_type'] == 'result_bundle':
if not preview or (
str(preview['object_type']) != str(row['object_type'])
or int(preview['reservation_id'] or 0)
!= int(row['reservation_id'] or 0)
):
raise ScanEventConflictError(
'pipeline quarantine review identity changed before locking'
)
bundle = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(row['reservation_id'],),
).fetchone()
if not reservation or reservation['state'] != 'quarantined':
raise RuntimeError('bundle quarantine review requires terminal quarantined reservation state')
if bundle and bundle['state'] != 'quarantined':
raise RuntimeError('bundle quarantine review requires terminal quarantined bundle state')
elif row['object_type'] == 'projection_job':
job_state = self.conn.execute(
'SELECT status FROM projection_jobs WHERE id = ? FOR UPDATE',
(row['projection_job_id'],),
).fetchone()
if not job_state or job_state['status'] != 'quarantined':
raise RuntimeError('projection review requires terminal quarantined job state')
elif row['object_type'] == 'keycheck_candidate':
candidate_state = self.conn.execute(
'SELECT * FROM keycheck_candidates WHERE id = ? FOR UPDATE',
(row['keycheck_candidate_id'],),
).fetchone()
if not candidate_state or candidate_state['state'] != 'quarantined':
raise RuntimeError('keycheck review requires terminal quarantined candidate state')
if action in ('discard', 'rescan') and row['source_relative_path']:
artifact = None
if row['object_type'] == 'projection_tail':
artifact = self.conn.execute(
'SELECT state FROM pipeline_artifacts WHERE id = ? FOR UPDATE',
(row['object_id'],),
).fetchone()
elif row['object_type'] == 'result_bundle':
artifact = self.conn.execute(
'''SELECT state FROM pipeline_artifacts
WHERE subsystem = 'result_bundle'
AND artifact_kind = 'bundle_quarantine' AND owner_id = ? FOR UPDATE''',
(row['reservation_id'],),
).fetchone()
if not artifact or artifact['state'] != 'deleted':
raise RuntimeError(
'physical quarantine artifact must be absent before capacity credit release'
)
if action == 'retry':
if not row['capacity_credit_applied']:
raise RuntimeError('quarantine retry requires exact applied capacity credit')
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['quarantine_items']) < int(row['capacity_items'])
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
):
raise RuntimeError('quarantine retry would make capacity accounting negative')
if row['object_type'] == 'projection_job':
job = self.conn.execute(
'SELECT * FROM projection_jobs WHERE id = ? FOR UPDATE',
(row['projection_job_id'],),
).fetchone()
if not job or job['status'] != 'quarantined' or not job['capacity_released']:
raise RuntimeError('projection quarantine retry state is not exact')
prepared = self.conn.execute(
"SELECT 1 FROM projection_appends WHERE job_id = ? AND state = 'prepared' LIMIT 1",
(job['id'],),
).fetchone()
if prepared:
raise RuntimeError('projection quarantine retry still has a prepared append')
self.conn.execute(
'''UPDATE pipeline_capacity SET
projection_items = projection_items + ?,
projection_bytes = projection_bytes + ?,
quarantine_items = quarantine_items - ?,
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
(
job['capacity_items'], job['capacity_bytes'],
row['capacity_items'], row['capacity_bytes'], now,
),
)
self.conn.execute(
'''UPDATE projection_jobs SET status = 'pending', capacity_released = 0,
available_after = NULL, lease_generation = NULL, lease_token = NULL,
lease_expires_at = NULL, last_error_code = NULL,
last_error_detail = NULL, completed_at = NULL, updated_at = ?
WHERE id = ?''',
(now, job['id']),
)
elif row['object_type'] == 'keycheck_candidate':
candidate = candidate_state
active = self.conn.execute(
'''SELECT 1 FROM keycheck_candidates
WHERE credential_id = ? AND id <> ?
AND state IN ('pending','deferred','leased') LIMIT 1 FOR UPDATE''',
(candidate['credential_id'], candidate['id']),
).fetchone()
if active:
raise RuntimeError('keycheck quarantine retry conflicts with another active credential candidate')
candidate_items = 1
candidate_bytes = int(candidate['capacity_bytes'])
if (
int(row['capacity_items']) < candidate_items
or int(row['capacity_bytes']) < candidate_bytes
):
raise RuntimeError('keycheck quarantine retry has insufficient capacity credit')
self.conn.execute(
'''UPDATE pipeline_capacity SET
keycheck_items = keycheck_items + ?,
keycheck_bytes = keycheck_bytes + ?,
quarantine_items = quarantine_items - ?,
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
(
candidate_items, candidate_bytes,
row['capacity_items'], row['capacity_bytes'], now,
),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET state = 'pending', attempts = 0,
available_after = NULL, capacity_released = 0,
result_projection_credit_transferred = 0,
result_projection_reserved_bytes = 0,
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
last_error = NULL, completed_at = NULL, updated_at = ?
WHERE id = ?''',
(now, candidate['id']),
)
elif row['object_type'] == 'result_bundle':
if reservation['docker_layer_plan_json'] is not None:
raise ValueError(
'Docker layer bundle quarantine requires a fresh parent claim'
)
if not reservation['bundle_credit_released']:
raise RuntimeError('bundle quarantine retry requires released bundle capacity')
bundle_items = 1
bundle_bytes = int(reservation['reserved_bundle_bytes'])
projection_items = (
0 if reservation['projection_credit_transferred']
else int(reservation['reserved_projection_items'])
)
projection_bytes = (
0 if reservation['projection_credit_transferred']
else int(reservation['reserved_projection_bytes'])
)
candidate_items = (
0 if reservation['candidate_credit_transferred']
else int(reservation['reserved_candidate_items'])
)
candidate_bytes = (
0 if reservation['candidate_credit_transferred']
else int(reservation['reserved_candidate_bytes'])
)
expected_items = bundle_items + projection_items + candidate_items
expected_bytes = bundle_bytes + projection_bytes + candidate_bytes
if (
int(row['capacity_items']) != expected_items
or int(row['capacity_bytes']) != expected_bytes
):
raise RuntimeError('bundle quarantine retry capacity evidence is inconsistent')
self.conn.execute(
'''UPDATE pipeline_capacity SET
bundle_items = bundle_items + ?, bundle_bytes = bundle_bytes + ?,
projection_items = projection_items + ?, projection_bytes = projection_bytes + ?,
keycheck_items = keycheck_items + ?, keycheck_bytes = keycheck_bytes + ?,
quarantine_items = quarantine_items - ?,
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
(
bundle_items, bundle_bytes, projection_items, projection_bytes,
candidate_items, candidate_bytes,
row['capacity_items'], row['capacity_bytes'], now,
),
)
self.conn.execute(
'''UPDATE result_reservations SET state = 'scanning',
bundle_credit_released = 0, released_at = NULL,
cleanup_attempts = 0, cleanup_available_after = NULL,
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
WHERE id = ?''',
(now, reservation['id']),
)
if bundle:
self.conn.execute(
'''UPDATE result_bundles SET state = 'ready',
relative_path = ?, available_after = NULL,
ingest_lease_generation = NULL, ingest_lease_token = NULL,
ingest_lease_expires_at = NULL, updated_at = ?
WHERE reservation_id = ?''',
(reservation['ready_relative_path'], now, reservation['id']),
)
self.conn.execute(
'''UPDATE target_queue SET status = 'in_progress', completed_at = NULL,
last_error = NULL, lease_owner = ?, lease_token = ?,
claim_batch = ?, leased_at = ?, lease_expires_at = ?,
claim_event_id = ?, updated_at = ?
WHERE id = ? AND current_result_reservation_id = ?''',
(
reservation['claim_lease_owner'], reservation['claim_lease_token'],
reservation['claim_batch'], now, reservation['producer_lease_expires_at'],
reservation['scan_event_id'], now,
reservation['queue_id'], reservation['id'],
),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
deleted_at = ?, updated_at = ?
WHERE subsystem = 'result_bundle'
AND artifact_kind = 'bundle_quarantine' AND owner_id = ?''',
(now, now, reservation['id']),
)
self.conn.execute(
'''UPDATE pipeline_artifacts SET state = 'present',
relative_path = ?, payload_sha256 = ?, byte_count = ?,
deleted_at = NULL, updated_at = ?
WHERE subsystem = 'result_bundle'
AND artifact_kind = 'bundle_ready' AND owner_id = ?''',
(
reservation['ready_relative_path'], row['payload_sha256'] or '',
int(row['byte_count']), now, reservation['id'],
),
)
self._transition_docker_depth_binding_locked(
reservation, 'reserved', 'reserved', now,
)
else:
raise ValueError('quarantine object type does not support deterministic retry')
review_status = 'approved_retry'
elif action == 'rescan':
if (
row['object_type'] != 'result_bundle'
or reservation['docker_layer_plan_json'] is None
or str(reservation['source']) != 'dockerhub'
or str(reservation['platform']) != 'docker'
):
raise ValueError('quarantine rescan requires a Docker layer result bundle')
if not row['capacity_credit_applied']:
raise RuntimeError('quarantine rescan requires exact applied capacity credit')
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['quarantine_items']) < int(row['capacity_items'])
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
):
raise RuntimeError('quarantine rescan would make capacity accounting negative')
queue = self.conn.execute(
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
(reservation['queue_id'],),
).fetchone()
if not queue or (
str(queue['source']) != 'dockerhub'
or str(queue['platform']) != 'docker'
or str(queue['status']) != 'quarantined'
or int(queue['current_result_reservation_id'] or 0) != int(reservation['id'])
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
):
raise ScanEventConflictError(
'Docker quarantine rescan lost its exact queue fence'
)
active_blob = self.conn.execute(
'''SELECT 1 FROM docker_content_blobs
WHERE lease_reservation_id = ? LIMIT 1 FOR UPDATE''',
(reservation['id'],),
).fetchone()
if active_blob:
raise RuntimeError('Docker quarantine rescan retains active content work')
self.conn.execute(
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items - ?,
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
(row['capacity_items'], row['capacity_bytes'], now),
)
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'pending', attempts = 0,
available_after = NULL, completed_at = NULL, last_error = NULL,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL,
current_result_reservation_id = NULL, claim_event_id = NULL,
updated_at = ?
WHERE id = ? AND status = 'quarantined'
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
now, reservation['queue_id'], reservation['id'],
reservation['scan_event_id'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker quarantine rescan queue transition lost its fence'
)
self._transition_docker_depth_binding_locked(
reservation, 'quarantined', 'pending', now,
)
review_status = 'approved_rescan'
else:
if row['capacity_credit_applied']:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['quarantine_items']) < int(row['capacity_items'])
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
):
raise RuntimeError('quarantine discard would make capacity accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items - ?,
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
(row['capacity_items'], row['capacity_bytes'], now),
)
review_status = 'discarded'
if experiment and action in ('retry', 'rescan'):
self._resume_docker_depth_after_quarantine_review_locked(
experiment, now,
)
self.conn.execute(
'''UPDATE pipeline_quarantine SET review_status = ?, resolved_at = ?,
review_audit_sha256 = ? WHERE id = ?''',
(review_status, now, audit_sha256, row['id']),
)
self.conn.commit()
return {'reviewed': True, 'duplicate': False, 'status': review_status}
except Exception:
self.conn.rollback()
raise
def _resume_docker_depth_after_quarantine_review_locked(self, experiment, now):
if not experiment:
return
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ? FOR UPDATE',
(int(experiment['id']),),
).fetchone()
if not experiment:
raise ScanEventConflictError(
'Docker depth quarantine review lost its experiment authority'
)
state = str(experiment['state'])
if state in ('completed', 'released'):
raise ScanEventConflictError(
'Docker depth terminal experiment cannot reopen quarantined work'
)
if state == 'held':
if str(experiment['hold_reason_code'] or '') != 'scan_target_held':
return
remaining = self.conn.execute(
'''SELECT 1 FROM docker_depth_experiment_targets
WHERE experiment_id = ? AND state IN ('held','quarantined')
LIMIT 1 FOR UPDATE''',
(experiment['id'],),
).fetchone()
if remaining:
return
elif state != 'draining':
return
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'active', hold_reason_code = NULL, held_at = NULL,
draining_at = NULL, updated_at = ?
WHERE id = ? AND state = ?''',
(now, experiment['id'], state),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth quarantine review lost its resume fence'
)
def pipeline_quarantine_for_review(self, quarantine_id):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('pipeline quarantine review lookup requires PostgreSQL')
row = self.conn.execute(
'SELECT * FROM pipeline_quarantine WHERE id = ?', (int(quarantine_id),),
).fetchone()
self.conn.commit()
return dict(row) if row else None
def _hold_docker_depth_experiment_locked(self, experiment, reason, now=None):
now = str(now or utc_now_iso())
reason = first_line(reason, 128)
if not experiment:
return {'status': 'unavailable', 'committed': False, 'reason': reason}
experiment_id = int(experiment['id'])
stable_reason = str(experiment['hold_reason_code'] or reason)
if str(experiment['state']) != 'released':
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = CASE WHEN work_state = 'resolving' THEN 'pending'
ELSE work_state END,
resolver_owner = NULL, resolver_token = NULL,
resolver_expires_at = NULL, resolver_due_at = NULL,
updated_at = ?
WHERE experiment_id = ? AND work_state = 'resolving' ''',
(now, experiment_id),
)
self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'held', hold_reason_code = COALESCE(hold_reason_code, ?),
held_at = COALESCE(held_at, ?), updated_at = ?
WHERE id = ? AND state <> 'released' ''',
(reason, now, now, experiment_id),
)
return {
'status': 'held', 'committed': True, 'reason': stable_reason,
'experiment_id': experiment_id,
}
@staticmethod
def _docker_depth_authority_drift_reason(experiment, authority):
if not experiment:
return 'experiment_authority_absent'
comparisons = (
('experiment_key', 'experiment_key', 'experiment_identity_drift'),
('source', 'source', 'experiment_identity_drift'),
('collection_generation', 'collection_generation', 'collection_generation_drift'),
('config_sha256', 'config_sha256', 'config_hash_drift'),
('ordered_queries_sha256', 'ordered_queries_sha256', 'ordered_query_drift'),
('selector_version', 'selector_version', 'selector_drift'),
('selector_sha256', 'selector_sha256', 'selector_drift'),
('provenance_policy_sha256', 'provenance_policy_sha256', 'provenance_policy_drift'),
)
for column, key, reason in comparisons:
if str(experiment[column]) != str(authority[key]):
return reason
for column, key in (
('query_count', 'query_count'),
('repositories_per_query', 'repositories_per_query'),
('images_per_repository', 'images_per_repository'),
('target_limit', 'target_limit'),
):
if int(experiment[column]) != int(authority[key]):
return 'capacity_limit_drift'
return None
@staticmethod
def _docker_depth_candidate_skip_evidence(
experiment_id, experiment_repository_id, repository_queue_id,
candidate_kind, candidate_ordinal, candidate_identity, reason,
):
candidate_identity_sha256 = hashlib.sha256(json.dumps(
candidate_identity, ensure_ascii=True, allow_nan=False,
sort_keys=True, separators=(',', ':'),
).encode('utf-8')).hexdigest()
evidence = {
'schema': 1,
'type': 'docker-depth-candidate-skip-v1',
'experiment_id': int(experiment_id),
'experiment_repository_id': int(experiment_repository_id),
'repository_queue_id': int(repository_queue_id),
'candidate_kind': str(candidate_kind),
'candidate_ordinal': int(candidate_ordinal),
'candidate_identity_sha256': candidate_identity_sha256,
'reason_code': str(reason),
}
evidence_sha256 = hashlib.sha256(json.dumps(
evidence, ensure_ascii=True, allow_nan=False, sort_keys=True,
separators=(',', ':'),
).encode('utf-8')).hexdigest()
return candidate_identity_sha256, evidence_sha256
def _docker_depth_terminal_repository_evidence_locked(
self, experiment, member,
):
from docker_depth_experiment import (
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
)
state = str(member['work_state'])
resolver_fields = (
'resolver_owner', 'resolver_token', 'resolver_expires_at',
'resolver_due_at',
)
if state == 'resolved':
if (
int(member['selected_image_count']) < 1
or not member['resolved_at']
or member['last_error_code'] is not None
or any(member[name] is not None for name in resolver_fields)
):
return 'repository_state_drift', None
return None, None
if state != 'skipped':
return None, None
current_queue_id = int(
member['replacement_repository_queue_id']
or member['repository_queue_id']
)
skip_reason = str(member['last_error_code'] or '')
if (
int(member['selected_image_count']) != 0
or skip_reason not in (
DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
)
or not member['resolved_at']
or any(member[name] is not None for name in resolver_fields)
):
return 'repository_skip_evidence_drift', None
if self.conn.execute(
'''SELECT 1 FROM docker_depth_experiment_selections
WHERE experiment_repository_id = ? LIMIT 1''',
(member['id'],),
).fetchone():
return 'repository_skip_evidence_drift', None
rows = self.conn.execute(
'''SELECT candidate_ordinal, candidate_identity_sha256,
evidence_sha256
FROM docker_depth_experiment_candidate_skips
WHERE experiment_id = ? AND experiment_repository_id = ?
AND repository_queue_id = ? AND candidate_kind = 'repository'
AND reason_code = ?
ORDER BY id LIMIT 2''',
(
experiment['id'], member['id'], current_queue_id,
skip_reason,
),
).fetchall()
expected_ordinal = int(member['replacement_count']) + 1
identity_sha256, evidence_sha256 = self._docker_depth_candidate_skip_evidence(
experiment['id'], member['id'], current_queue_id, 'repository',
expected_ordinal, {'repository_queue_id': current_queue_id},
skip_reason,
)
if (
len(rows) != 1
or int(rows[0]['candidate_ordinal']) != expected_ordinal
or str(rows[0]['candidate_identity_sha256']) != identity_sha256
or str(rows[0]['evidence_sha256']) != evidence_sha256
):
return 'repository_skip_evidence_drift', None
return None, evidence_sha256
def _docker_depth_runtime_selection_sha256_locked(self, experiment, authority):
from docker_depth_experiment import DOCKER_RANK1_BREADTH_SELECTOR_VERSION
rank1_breadth = (
authority.get('selector_version') == DOCKER_RANK1_BREADTH_SELECTOR_VERSION
)
query_rows = self.conn.execute(
'''SELECT query_ordinal, required_repository_count,
selected_repository_count
FROM docker_depth_experiment_queries
WHERE experiment_id = ? ORDER BY query_ordinal''',
(experiment['id'],),
).fetchall()
if len(query_rows) != authority['query_count']:
raise ScanEventConflictError('Docker depth runtime query selection is incomplete')
selected_counts = []
for query_ordinal, row in enumerate(query_rows):
selected_count = int(row['selected_repository_count'])
if (
int(row['query_ordinal']) != query_ordinal
or int(row['required_repository_count'])
!= authority['repositories_per_query']
or not 0 <= selected_count <= authority['repositories_per_query']
):
raise ScanEventConflictError('Docker depth runtime query selection is invalid')
selected_counts.append(selected_count)
rows = self.conn.execute(
'''SELECT id, query_ordinal, repository_rank, repository_queue_id,
replacement_repository_queue_id,
replacement_eligibility_page_id, replacement_count,
replacement_evidence_sha256, is_deep_probe,
work_state, selected_image_count, last_error_code,
resolved_at, resolver_owner, resolver_token,
resolver_expires_at, resolver_due_at
FROM docker_depth_experiment_repositories
WHERE experiment_id = ?
ORDER BY query_ordinal, repository_rank, id''',
(experiment['id'],),
).fetchall()
expected = sum(selected_counts)
if len(rows) != expected:
raise ScanEventConflictError('Docker depth runtime selection is incomplete')
document = {
'schema': 2,
'type': 'docker-depth-runtime-selection-v2',
'experiment_id': int(experiment['id']),
'plan_sha256': str(experiment['plan_sha256'] or ''),
'collection_generation': authority['collection_generation'],
'queries': [],
}
for query_ordinal in range(authority['query_count']):
members = [
row for row in rows if int(row['query_ordinal']) == query_ordinal
]
terminal_evidence = {}
for row in members:
reason, evidence_sha256 = (
self._docker_depth_terminal_repository_evidence_locked(
experiment, row,
)
)
if reason:
raise ScanEventConflictError(
'Docker depth runtime repository evidence is invalid'
)
terminal_evidence[int(row['id'])] = evidence_sha256
image_bearing = sum(
1 for row in members
if str(row['work_state']) == 'resolved'
and int(row['selected_image_count']) >= 1
)
expected_deep_probe_count = 0 if rank1_breadth else (1 if image_bearing else 0)
if (
len(members) != selected_counts[query_ordinal]
or [int(row['repository_rank']) for row in members]
!= list(range(1, selected_counts[query_ordinal] + 1))
or sum(int(row['is_deep_probe']) for row in members)
!= expected_deep_probe_count
or any(
str(row['work_state']) not in ('resolved', 'skipped')
for row in members
)
):
raise ScanEventConflictError('Docker depth runtime selection is invalid')
document['queries'].append({
'query_ordinal': query_ordinal,
'selected_repository_count': selected_counts[query_ordinal],
'repositories': [{
'member_id': int(row['id']),
'repository_rank': int(row['repository_rank']),
'planned_repository_queue_id': int(row['repository_queue_id']),
'selected_repository_queue_id': int(
row['replacement_repository_queue_id']
or row['repository_queue_id']
),
'replacement_eligibility_page_id': (
int(row['replacement_eligibility_page_id'])
if row['replacement_eligibility_page_id'] is not None else None
),
'replacement_count': int(row['replacement_count']),
'replacement_evidence_sha256': str(
row['replacement_evidence_sha256'] or ''
),
'is_deep_probe': bool(row['is_deep_probe']),
'work_state': str(row['work_state']),
'selected_image_count': int(row['selected_image_count']),
'terminal_reason': (
str(row['last_error_code'])
if str(row['work_state']) == 'skipped' else None
),
'skip_evidence_sha256': terminal_evidence[int(row['id'])],
} for row in members],
})
payload = json.dumps(
document, ensure_ascii=True, allow_nan=False, sort_keys=True,
separators=(',', ':'),
).encode('utf-8')
return hashlib.sha256(payload).hexdigest()
def _freeze_docker_depth_runtime_selection_locked(
self, experiment, authority, now,
):
incomplete = self.conn.execute(
'''SELECT 1 FROM docker_depth_experiment_repositories
WHERE experiment_id = ?
AND work_state NOT IN ('resolved','skipped') LIMIT 1''',
(experiment['id'],),
).fetchone()
if incomplete:
return experiment, None
rows = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_repositories
WHERE experiment_id = ? ORDER BY id''',
(experiment['id'],),
).fetchall()
for row in rows:
reason, _evidence_sha256 = (
self._docker_depth_terminal_repository_evidence_locked(
experiment, row,
)
)
if reason:
return experiment, reason
selection_sha256 = self._docker_depth_runtime_selection_sha256_locked(
experiment, authority,
)
stored = str(experiment['selection_sha256'] or '')
if stored and stored != selection_sha256:
return experiment, 'selection_hash_drift'
if not stored:
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET selection_sha256 = ?, updated_at = ?
WHERE id = ? AND selection_sha256 IS NULL
AND state = 'resolving' ''',
(selection_sha256, now, experiment['id']),
)
if int(cursor.rowcount or 0) != 1:
return experiment, 'selection_hash_drift'
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
return experiment, None
def _docker_depth_hold_event_drift_reason_locked(self, experiment, authority):
from docker_depth_experiment import (
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
DOCKER_DEPTH_HOLD_REASON,
DOCKER_DEPTH_RELEASE_REASON,
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
)
experiment_id = int(experiment['id'])
hold_sha256 = str(experiment['hold_manifest_sha256'] or '')
rows = self.conn.execute(
'''SELECT event.*, queue.status AS queue_status,
queue.source AS queue_source,
queue.platform AS queue_platform,
queue.query AS queue_query,
queue.updated_at AS queue_updated_at
FROM target_queue_policy_events event
JOIN target_queue queue ON queue.id = event.queue_id
WHERE event.experiment_id = ?
ORDER BY event.id LIMIT ?''',
(experiment_id, DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS + 1),
).fetchall()
if len(rows) > DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS:
return 'target_history_drift'
reversed_ids = {
int(row['reverses_event_id']) for row in rows
if row['reverses_event_id'] is not None
}
for row in rows:
if (
int(row['queue_id']) < 1
or str(row['source']) != authority['source']
or str(row['platform']) != 'docker'
or not str(row['query'])
or str(row['queue_source']) != str(row['source'])
or str(row['queue_platform']) != str(row['platform'])
or str(row['queue_query']) != str(row['query'])
or str(row['config_sha256']) != authority['config_sha256']
or str(row['policy_sha256'])
!= authority['provenance_policy_sha256']
):
return 'target_history_drift'
if row['action'] == 'cold':
entry = {
'queue_id': int(row['queue_id']),
'source': str(row['source']),
'platform': str(row['platform']),
'query': str(row['query']),
'prior_status': str(row['prior_status']),
'prior_updated_at': str(row['prior_updated_at']),
}
if (
str(row['manifest_sha256']) != hold_sha256
or str(row['reason_code']) not in (
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
)
or str(row['prior_status']) not in ('pending', 'deferred')
or str(row['next_status']) != 'cold'
or int(row['id']) in reversed_ids
or str(row['queue_status']) != 'cold'
or str(row['queue_updated_at']) != str(row['created_at'])
):
return 'target_history_drift'
elif row['action'] == 'reactivate':
# Reactivation and the completed -> released transition share one
# experiment-row lock, so it is never valid in a pre-release state.
if str(row['reason_code']) != DOCKER_DEPTH_RELEASE_REASON:
return 'target_history_drift'
return 'target_history_drift'
else:
return 'target_history_drift'
if str(row['review_audit_sha256']) != self._target_queue_policy_audit_sha256(
str(row['action']), str(row['manifest_sha256']), entry,
experiment_id,
):
return 'target_history_drift'
return None
def _docker_depth_persisted_drift_reason_locked(self, experiment, authority, now):
from docker_depth_experiment import (
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
_cohort_plan_document,
canonical_docker_depth_plan_hash,
)
experiment_id = int(experiment['id'])
plan_sha256 = str(experiment['plan_sha256'] or '')
hold_sha256 = str(experiment['hold_manifest_sha256'] or '')
if any(
experiment[name] is not None
for name in ('fence_owner', 'fence_token', 'fence_expires_at')
):
return 'experiment_fence_conflict'
if not re.fullmatch(r'[a-f0-9]{64}', plan_sha256):
return 'plan_hash_drift'
if not re.fullmatch(r'[a-f0-9]{64}', hold_sha256):
return 'hold_hash_drift'
query_rows = self.conn.execute(
'''SELECT query_ordinal, source, query, query_sha256,
required_repository_count, selected_repository_count
FROM docker_depth_experiment_queries
WHERE experiment_id = ? ORDER BY query_ordinal LIMIT ?''',
(experiment_id, authority['query_count'] + 1),
).fetchall()
if len(query_rows) != authority['query_count']:
return 'ordered_query_drift'
selected_counts = []
for ordinal, row in enumerate(query_rows):
query = authority['queries'][ordinal]
selected_count = int(row['selected_repository_count'])
if (
int(row['query_ordinal']) != ordinal
or str(row['source']) != authority['source']
or str(row['query']) != query
or str(row['query_sha256']) != hashlib.sha256(
json.dumps(
query, ensure_ascii=True, allow_nan=False, sort_keys=True,
separators=(',', ':'),
).encode('utf-8')
).hexdigest()
or int(row['required_repository_count'])
!= authority['repositories_per_query']
or not 0 <= selected_count <= authority['repositories_per_query']
):
return 'ordered_query_drift'
selected_counts.append(selected_count)
expected_repository_count = sum(selected_counts)
repository_rows = self.conn.execute(
'''SELECT id, query_ordinal, source, query, repository_queue_id,
eligibility_page_id, repository_rank, planned_is_deep_probe,
is_deep_probe, work_state, resolver_owner, resolver_token,
resolver_expires_at, resolver_due_at,
candidate_distinct_graph_count,
selected_image_count, replacement_repository_queue_id,
replacement_eligibility_page_id, replacement_count,
replacement_evidence_sha256, last_error_code, resolved_at
FROM docker_depth_experiment_repositories
WHERE experiment_id = ?
ORDER BY query_ordinal, repository_rank, id LIMIT ?''',
(experiment_id, expected_repository_count + 1),
).fetchall()
if len(repository_rows) != expected_repository_count:
return 'repository_count_drift'
if str(experiment['state']) in ('active', 'draining', 'completed') and any(
str(row['work_state']) not in ('resolved', 'skipped')
for row in repository_rows
):
return 'repository_state_drift'
repositories_by_query = {
ordinal: [] for ordinal in range(authority['query_count'])
}
for row in repository_rows:
ordinal = int(row['query_ordinal'])
if ordinal not in repositories_by_query:
return 'plan_hash_drift'
repositories_by_query[ordinal].append(row)
replacement_count = int(row['replacement_count'])
replacement_present = (
row['replacement_repository_queue_id'] is not None
and row['replacement_eligibility_page_id'] is not None
and bool(re.fullmatch(
r'[a-f0-9]{64}', str(row['replacement_evidence_sha256'] or '')
))
)
if (
replacement_count < 0
or (replacement_count == 0 and (
row['replacement_repository_queue_id'] is not None
or row['replacement_eligibility_page_id'] is not None
or row['replacement_evidence_sha256'] is not None
))
or (replacement_count > 0 and not replacement_present)
):
return 'selection_hash_drift'
if replacement_present and not self.conn.execute(
'''SELECT 1
FROM docker_repository_query_observations observation
JOIN docker_discovery_pages page ON page.id = observation.page_id
JOIN docker_discovery_passes discovery_pass
ON discovery_pass.id = page.pass_id
JOIN target_queue queue
ON queue.id = observation.repository_queue_id
WHERE observation.page_id = ?
AND observation.repository_queue_id = ?
AND observation.source = ? AND observation.query = ?
AND page.query_ordinal = ? AND page.query = ?
AND discovery_pass.source = ?
AND discovery_pass.pass_kind = 'deep'
AND discovery_pass.collection_generation = ?
AND discovery_pass.policy_sha256 = ?
AND discovery_pass.ordered_queries_sha256 = ?
AND discovery_pass.expected_query_count = ?
AND discovery_pass.state = 'complete'
AND queue.source = ? AND queue.platform = 'docker'
AND queue.target NOT LIKE '%@%'
AND queue.normalized_target NOT LIKE '%@%'
LIMIT 1''',
(
row['replacement_eligibility_page_id'],
row['replacement_repository_queue_id'],
authority['source'], authority['queries'][ordinal], ordinal,
authority['queries'][ordinal], authority['source'],
authority['collection_generation'],
authority['provenance_policy_sha256'],
authority['ordered_queries_sha256'], authority['query_count'],
authority['source'],
),
).fetchone():
return 'selection_hash_drift'
resolving = str(row['work_state']) == 'resolving'
if resolving and str(experiment['state']) != 'resolving':
return 'stale_resolver_fence'
if resolving and (
not row['resolver_owner']
or not row['resolver_token']
or not row['resolver_expires_at']
or str(row['resolver_expires_at']) <= now
):
return 'stale_resolver_fence'
if not resolving and any(
row[name] is not None
for name in ('resolver_owner', 'resolver_token', 'resolver_expires_at')
):
return 'stale_resolver_fence'
terminal_reason, _evidence_sha256 = (
self._docker_depth_terminal_repository_evidence_locked(
experiment, row,
)
)
if terminal_reason:
return terminal_reason
planned_queries = []
for ordinal, query_row in enumerate(query_rows):
members = repositories_by_query[ordinal]
selected_count = selected_counts[ordinal]
if (
[int(row['repository_rank']) for row in members]
!= list(range(1, selected_count + 1))
or any(
str(row['source']) != authority['source']
or str(row['query']) != authority['queries'][ordinal]
or int(row['eligibility_page_id']) < 1
for row in members
)
or sum(int(row['planned_is_deep_probe']) for row in members)
!= (1 if selected_count else 0)
):
return 'plan_hash_drift'
all_terminal = all(
str(row['work_state']) in ('resolved', 'skipped') for row in members
)
image_bearing = any(
str(row['work_state']) == 'resolved'
and int(row['selected_image_count']) >= 1
for row in members
)
if (
str(experiment['selector_version'])
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
and all_terminal
):
expected_deep_probe_count = 0
else:
expected_deep_probe_count = (
(1 if image_bearing else 0)
if all_terminal else (1 if selected_count else 0)
)
if sum(int(row['is_deep_probe']) for row in members) != (
expected_deep_probe_count
):
return 'selection_mutation'
planned_queries.append({
'query_ordinal': ordinal,
'query': authority['queries'][ordinal],
'query_sha256': str(query_row['query_sha256']),
'selected_repository_count': selected_count,
'repositories': [{
'repository_queue_id': int(row['repository_queue_id']),
'eligibility_page_id': int(row['eligibility_page_id']),
'repository_rank': int(row['repository_rank']),
'is_deep_probe': bool(row['planned_is_deep_probe']),
} for row in members],
})
if canonical_docker_depth_plan_hash(
_cohort_plan_document(authority, planned_queries)
) != plan_sha256:
return 'plan_hash_drift'
selection_sha256 = str(experiment['selection_sha256'] or '')
if selection_sha256:
if (
not re.fullmatch(r'[a-f0-9]{64}', selection_sha256)
or self._docker_depth_runtime_selection_sha256_locked(
experiment, authority,
) != selection_sha256
):
return 'selection_hash_drift'
elif str(experiment['state']) in ('active', 'draining'):
return 'selection_hash_drift'
target_rows = self.conn.execute(
'''SELECT target.id AS target_id, target.target_queue_id,
target.manifest_id, target.counter_ordinal,
target.state AS target_state, target.dispatch_wave,
target.dispatch_order, target.reservation_count,
queue.source AS queue_source, queue.platform AS queue_platform,
queue.target AS queue_target,
queue.normalized_target AS queue_normalized_target,
queue.status AS queue_status, queue.lease_owner,
queue.lease_token, queue.claim_batch, queue.leased_at,
queue.lease_expires_at, queue.current_result_reservation_id,
queue.claim_event_id, queue.resolver_token,
manifest.repository, manifest.manifest_digest,
manifest.graph_sha256 AS manifest_graph_sha256
FROM docker_depth_experiment_targets target
JOIN target_queue queue ON queue.id = target.target_queue_id
JOIN docker_image_manifests manifest
ON manifest.id = target.manifest_id
AND manifest.target_queue_id = target.target_queue_id
WHERE target.experiment_id = ?
ORDER BY target.id LIMIT ?''',
(experiment_id, authority['target_limit'] + 1),
).fetchall()
if (
len(target_rows) != int(experiment['target_count'])
or len(target_rows) > authority['target_limit']
):
return 'target_counter_drift'
target_by_id = {int(row['target_id']): row for row in target_rows}
if len(target_by_id) != len(target_rows):
return 'target_counter_drift'
selection_rows = self.conn.execute(
'''SELECT selection.id, selection.query_ordinal,
selection.experiment_repository_id,
selection.experiment_target_id, selection.image_rank,
selection.selection_evidence_sha256,
selection.graph_sha256,
member.query_ordinal AS member_query_ordinal,
member.is_deep_probe
FROM docker_depth_experiment_selections selection
JOIN docker_depth_experiment_repositories member
ON member.id = selection.experiment_repository_id
AND member.experiment_id = selection.experiment_id
WHERE selection.experiment_id = ?
ORDER BY selection.id LIMIT ?''',
(experiment_id, authority['theoretical_max_targets'] + 1),
).fetchall()
if (
len(selection_rows) != int(experiment['selection_count'])
or len(selection_rows) > authority['theoretical_max_targets']
):
return 'selection_counter_drift'
selected_by_member = {}
selected_ranks_by_member = {}
dispatch_by_target = {}
selected_target_ids = set()
for selection in selection_rows:
target_id = int(selection['experiment_target_id'])
target = target_by_id.get(target_id)
rank = int(selection['image_rank'])
if (
not target
or int(selection['query_ordinal'])
!= int(selection['member_query_ordinal'])
or (rank > 1 and not bool(selection['is_deep_probe']))
or str(selection['graph_sha256'])
!= str(target['manifest_graph_sha256'])
or not re.fullmatch(
r'[a-f0-9]{64}', str(selection['selection_evidence_sha256'] or '')
)
):
return 'selection_mutation'
member_id = int(selection['experiment_repository_id'])
selected_by_member[member_id] = selected_by_member.get(member_id, 0) + 1
selected_ranks_by_member.setdefault(member_id, []).append(rank)
position = self._docker_depth_dispatch_position(
int(selection['query_ordinal']),
next(
int(row['repository_rank']) for row in repository_rows
if int(row['id']) == member_id
),
rank, authority['query_count'],
)
dispatch_by_target[target_id] = min(
dispatch_by_target.get(target_id, position), position,
)
selected_target_ids.add(target_id)
if selected_target_ids != set(target_by_id):
return 'selection_mutation'
for member in repository_rows:
member_id = int(member['id'])
selected_count = selected_by_member.get(member_id, 0)
if (
selected_count != int(member['selected_image_count'])
or sorted(selected_ranks_by_member.get(member_id, ()))
!= list(range(1, selected_count + 1))
):
return 'selection_counter_drift'
for target_id, target in target_by_id.items():
expected_target = f"{target['repository']}@{target['manifest_digest']}"
if (
(int(target['dispatch_wave']), int(target['dispatch_order']))
!= dispatch_by_target[target_id]
or str(target['queue_source']) != authority['source']
or str(target['queue_platform']) != 'docker'
or str(target['queue_target']) != expected_target
or str(target['queue_normalized_target']) != expected_target
):
return 'selection_mutation'
target_state = str(target['target_state'])
queue_status = str(target['queue_status'])
if target_state in ('held', 'quarantined'):
return 'scan_target_held'
if target_state == 'pending' and queue_status not in ('pending', 'deferred'):
return 'stale_scan_fence'
if target_state in ('reserved', 'scanning') and queue_status != 'in_progress':
return 'stale_scan_fence'
if target_state == 'done' and queue_status != 'done':
return 'stale_scan_fence'
if target_state == 'failed' and queue_status != 'failed':
return 'stale_scan_fence'
if target_state == 'quarantined' and queue_status != 'quarantined':
return 'stale_scan_fence'
if target_state == 'pending' and any(
target[name] is not None for name in (
'lease_owner', 'lease_token', 'claim_batch', 'leased_at',
'lease_expires_at', 'current_result_reservation_id',
'claim_event_id', 'resolver_token',
)
):
return 'stale_scan_fence'
binding_rows = self.conn.execute(
'''SELECT binding.id, binding.experiment_target_id,
binding.reservation_id, binding.target_scan_id,
binding.attempt, binding.state AS binding_state,
target.target_queue_id,
reservation.queue_id AS reservation_queue_id,
reservation.state AS reservation_state,
reservation.claim_lease_token,
reservation.scan_event_id,
reservation.producer_lease_expires_at,
scan.queue_id AS scan_queue_id,
scan.result_reservation_id AS scan_reservation_id
FROM docker_depth_experiment_scan_bindings binding
JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
JOIN result_reservations reservation
ON reservation.id = binding.reservation_id
LEFT JOIN target_scans scan ON scan.id = binding.target_scan_id
WHERE target.experiment_id = ?
ORDER BY binding.experiment_target_id, binding.attempt
LIMIT ?''',
(experiment_id, authority['target_limit'] * 10 + 1),
).fetchall()
if len(binding_rows) > authority['target_limit'] * 10:
return 'binding_count_drift'
binding_count = {}
active_by_target = {}
latest_by_target = {}
for binding in binding_rows:
target_id = int(binding['experiment_target_id'])
state = str(binding['binding_state'])
reservation_state = str(binding['reservation_state'])
if (
target_id not in target_by_id
or int(binding['target_queue_id'])
!= int(binding['reservation_queue_id'])
or int(binding['attempt']) < 1
):
return 'reservation_binding_drift'
has_scan = binding['target_scan_id'] is not None
if has_scan and (
int(binding['scan_queue_id'] or 0) != int(binding['target_queue_id'])
or int(binding['scan_reservation_id'] or 0)
!= int(binding['reservation_id'])
):
return 'reservation_binding_drift'
if state == 'reserved' and (reservation_state != 'scanning' or has_scan):
return 'reservation_binding_drift'
if state == 'scanning' and (
reservation_state not in ('ready', 'ingesting') or has_scan
):
return 'reservation_binding_drift'
if state in ('completed', 'failed') and (
reservation_state not in ('db_committed', 'acknowledged') or not has_scan
):
return 'reservation_binding_drift'
if state == 'released' and (reservation_state != 'refunded' or has_scan):
return 'reservation_binding_drift'
if state == 'quarantined' and reservation_state != 'quarantined':
return 'reservation_binding_drift'
if state in ('reserved', 'scanning'):
if (
state == 'reserved'
and str(binding['producer_lease_expires_at'] or '') <= now
):
return 'stale_scan_fence'
active_by_target.setdefault(target_id, []).append(binding)
binding_count[target_id] = binding_count.get(target_id, 0) + 1
latest_by_target[target_id] = binding
for target_id, target in target_by_id.items():
if int(target['reservation_count']) != binding_count.get(target_id, 0):
return 'binding_count_drift'
active = active_by_target.get(target_id, [])
target_state = str(target['target_state'])
latest = latest_by_target.get(target_id)
if target_state in ('done', 'failed') and (
latest is None
or str(latest['binding_state'])
!= ('completed' if target_state == 'done' else 'failed')
):
return 'reservation_binding_drift'
if target_state in ('reserved', 'scanning'):
if len(active) != 1:
return 'stale_scan_fence'
binding = active[0]
if (
int(target['current_result_reservation_id'] or 0)
!= int(binding['reservation_id'])
or str(target['lease_token'] or '')
!= str(binding['claim_lease_token'])
or str(target['claim_event_id'] or '')
!= str(binding['scan_event_id'])
):
return 'stale_scan_fence'
elif active:
return 'stale_scan_fence'
unbound = self.conn.execute(
'''SELECT
EXISTS(
SELECT 1 FROM result_reservations reservation
JOIN docker_depth_experiment_targets target
ON target.target_queue_id = reservation.queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.experiment_target_id = target.id
AND binding.reservation_id = reservation.id
WHERE target.experiment_id = ? AND binding.id IS NULL
) AS reservation_missing,
EXISTS(
SELECT 1 FROM target_scans scan
JOIN docker_depth_experiment_targets target
ON target.target_queue_id = scan.queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.experiment_target_id = target.id
AND binding.target_scan_id = scan.id
WHERE target.experiment_id = ? AND binding.id IS NULL
) AS scan_missing''',
(experiment_id, experiment_id),
).fetchone()
if bool(unbound['reservation_missing']) or bool(unbound['scan_missing']):
return 'historical_scan_conflict'
return self._docker_depth_hold_event_drift_reason_locked(
experiment, authority,
)
def _locked_docker_depth_experiment_authority(
self, authority, *, final_cutover, now=None, lock=True,
):
now = str(now or utc_now_iso())
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE experiment_key = ? AND source = ?''' + (
' FOR UPDATE' if lock else ''
),
(authority['experiment_key'], authority['source']),
).fetchone()
if not experiment:
return None, 'experiment_authority_absent'
reason = self._docker_depth_authority_drift_reason(experiment, authority)
marker = self.conn.execute(
'''SELECT marker, checked_at, evidence_sha256
FROM runtime_final_cutover WHERE id = 1'''
).fetchone()
if reason is None and (
final_cutover is not True
or not marker
or str(marker['marker']) != FINAL_CUTOVER_MARKER
or not marker['checked_at']
or not re.fullmatch(r'[a-f0-9]{64}', str(marker['evidence_sha256'] or ''))
):
reason = 'final_cutover_unavailable'
if reason is None and str(experiment['state']) in (
'holding', 'resolving', 'active', 'draining', 'completed', 'held',
):
reason = self._docker_depth_persisted_drift_reason_locked(
experiment, authority, now,
)
if reason:
self._hold_docker_depth_experiment_locked(experiment, reason, now)
return None, reason
return experiment, None
def _docker_depth_incomplete_membership_count_locked(
self, experiment_id, *, activation=False,
):
activation_clause = (
"OR target.state <> 'pending' OR queue.status NOT IN ('pending','deferred') "
"OR queue.lease_owner IS NOT NULL OR queue.lease_token IS NOT NULL "
"OR queue.claim_batch IS NOT NULL OR queue.leased_at IS NOT NULL "
"OR queue.lease_expires_at IS NOT NULL "
"OR queue.current_result_reservation_id IS NOT NULL "
"OR queue.claim_event_id IS NOT NULL "
"OR EXISTS (SELECT 1 FROM target_scans scan "
" WHERE scan.queue_id = queue.id) "
"OR EXISTS (SELECT 1 FROM result_reservations reservation "
" WHERE reservation.queue_id = queue.id) "
"OR EXISTS (SELECT 1 FROM target_queue_policy_events event "
" WHERE event.queue_id = queue.id) "
"OR EXISTS (SELECT 1 FROM docker_image_blob_coverage coverage "
" JOIN docker_content_blobs blob "
" ON blob.digest = coverage.blob_digest "
" AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256 "
" WHERE coverage.queue_id = queue.id "
" AND blob.state IN ('leased','submitted'))"
if activation else ''
)
return int(self.conn.execute(
f'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_repositories member
LEFT JOIN docker_depth_experiment_selections selection
ON selection.experiment_repository_id = member.id
AND selection.image_rank = 1
LEFT JOIN docker_depth_experiment_targets target
ON target.id = selection.experiment_target_id
AND target.experiment_id = member.experiment_id
LEFT JOIN target_queue queue ON queue.id = target.target_queue_id
WHERE member.experiment_id = ?
AND member.work_state <> 'skipped'
AND (member.selected_image_count < 1 OR selection.id IS NULL
OR target.id IS NULL OR queue.id IS NULL {activation_clause})''',
(experiment_id,),
).fetchone()['count'])
def _advance_docker_depth_experiment_state_locked(self, experiment, now=None):
now = str(now or utc_now_iso())
experiment_id = int(experiment['id'])
state = str(experiment['state'])
unresolved = int(self.conn.execute(
'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_repositories
WHERE experiment_id = ?
AND work_state NOT IN ('resolved','skipped')''',
(experiment_id,),
).fetchone()['count'])
if state == 'resolving' and unresolved == 0:
from docker_depth_experiment import (
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
)
incremental_scan_profile = (
str(experiment['selector_version'])
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
)
if self._docker_depth_incomplete_membership_count_locked(
experiment_id, activation=not incremental_scan_profile,
):
self._hold_docker_depth_experiment_locked(
experiment, 'cohort_membership_incomplete', now,
)
return self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment_id,),
).fetchone()
experiment, selection_reason = (
self._freeze_docker_depth_runtime_selection_locked(
experiment, authority={
'query_count': int(experiment['query_count']),
'repositories_per_query': int(experiment['repositories_per_query']),
'collection_generation': str(experiment['collection_generation']),
'selector_version': str(experiment['selector_version']),
},
now=now,
)
)
if selection_reason:
self._hold_docker_depth_experiment_locked(
experiment, selection_reason, now,
)
return self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment_id,),
).fetchone()
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'active', activated_at = COALESCE(activated_at, ?),
updated_at = ?
WHERE id = ? AND state = 'resolving' ''',
(now, now, experiment_id),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth activation lost its compare-and-swap fence'
)
state = 'active'
if state == 'active':
nonterminal = int(self.conn.execute(
'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_targets
WHERE experiment_id = ? AND state IN ('pending','reserved','scanning')''',
(experiment_id,),
).fetchone()['count'])
if nonterminal == 0:
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'draining', draining_at = COALESCE(draining_at, ?),
updated_at = ?
WHERE id = ? AND state = 'active' ''',
(now, now, experiment_id),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth draining transition lost its compare-and-swap fence'
)
state = 'draining'
if state == 'draining' and unresolved == 0:
if self._docker_depth_incomplete_membership_count_locked(experiment_id):
self._hold_docker_depth_experiment_locked(
experiment, 'cohort_membership_incomplete', now,
)
return self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment_id,),
).fetchone()
fences = self.conn.execute(
'''SELECT
(SELECT COUNT(*) FROM docker_depth_experiment_targets target
JOIN target_queue queue ON queue.id = target.target_queue_id
WHERE target.experiment_id = ?
AND (target.state NOT IN ('done','failed','skipped')
OR queue.status NOT IN ('done','failed')
OR queue.lease_owner IS NOT NULL
OR queue.lease_token IS NOT NULL
OR queue.claim_batch IS NOT NULL
OR queue.leased_at IS NOT NULL
OR queue.lease_expires_at IS NOT NULL
OR queue.current_result_reservation_id IS NOT NULL
OR queue.claim_event_id IS NOT NULL)) AS queue_fences,
(SELECT COUNT(*) FROM docker_depth_experiment_scan_bindings binding
JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
JOIN result_reservations reservation
ON reservation.id = binding.reservation_id
WHERE target.experiment_id = ?
AND (binding.state IN ('reserved','scanning')
OR reservation.state IN
('scanning','ready','ingesting','db_committed'))) AS reservation_fences,
(SELECT COUNT(*) FROM docker_image_blob_coverage coverage
JOIN docker_depth_experiment_targets target
ON target.target_queue_id = coverage.queue_id
JOIN docker_content_blobs blob
ON blob.digest = coverage.blob_digest
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
WHERE target.experiment_id = ?
AND blob.state IN ('leased','submitted')) AS blob_fences''',
(experiment_id, experiment_id, experiment_id),
).fetchone()
if not any(int(fences[name]) for name in (
'queue_fences', 'reservation_fences', 'blob_fences',
)):
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'completed', completed_at = COALESCE(completed_at, ?),
updated_at = ?
WHERE id = ? AND state = 'draining'
AND fence_owner IS NULL AND fence_token IS NULL
AND fence_expires_at IS NULL''',
(now, now, experiment_id),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth completion lost its compare-and-swap fence'
)
state = 'completed'
if state == str(experiment['state']):
return experiment
return self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment_id,),
).fetchone()
def _hold_disabled_docker_depth_experiment(self, authority):
key = str(authority.get('experiment_key') or '') if isinstance(
authority, dict
) else ''
if not self.conn or not self.conn.is_postgres or not key:
return False
try:
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE experiment_key = ? AND source = 'dockerhub' FOR UPDATE''',
(key,),
).fetchone()
if not experiment or str(experiment['state']) not in (
'holding', 'resolving', 'active', 'draining',
):
self.conn.rollback()
return False
self._hold_docker_depth_experiment_locked(
experiment, 'experiment_disabled', utc_now_iso(),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
return False
def refresh_docker_depth_experiment_state(self, authority, *, final_cutover=False):
if not self.conn or not self.conn.is_postgres:
return {'status': 'unavailable', 'committed': False}
if isinstance(authority, dict) and authority.get('enabled') is False:
held = self._hold_disabled_docker_depth_experiment(authority)
return {
'status': 'held' if held else 'disabled',
'committed': held,
'reason': 'experiment_disabled' if held else None,
}
if not isinstance(authority, dict) or authority.get('enabled') is not True:
return {'status': 'disabled', 'committed': False}
try:
normalized = self._normalize_docker_depth_resolver_authority(authority)
except (TypeError, ValueError):
return {
'status': 'invalid', 'committed': False,
'reason': 'authority_payload_invalid',
}
try:
now = utc_now_iso()
experiment, reason = self._locked_docker_depth_experiment_authority(
normalized, final_cutover=final_cutover, now=now,
)
if not experiment:
self.conn.commit()
return {
'status': 'held' if reason != 'experiment_authority_absent' else 'unavailable',
'committed': reason != 'experiment_authority_absent',
'reason': reason,
}
experiment = self._advance_docker_depth_experiment_state_locked(
experiment, now,
)
self.conn.commit()
return {
'status': str(experiment['state']), 'committed': True,
'experiment_id': int(experiment['id']),
}
except Exception:
self.conn.rollback()
raise
def _docker_depth_candidate_selection_drift_reason(self, candidate, authority):
from docker_depth_experiment import (
canonical_docker_depth_selection_evidence_hash,
canonical_docker_descriptor_hash,
canonical_docker_layer_graph_hash,
)
layers = self.conn.execute(
'''SELECT position_from_base, position_from_top, layer_digest,
media_type, layer_size_bytes, descriptor_sha256
FROM docker_manifest_layers WHERE manifest_id = ?
ORDER BY position_from_base LIMIT 10001''',
(candidate['manifest_id'],),
).fetchall()
if (
len(layers) != int(candidate['layer_count'])
or len(layers) > 10000
):
return 'selection_mutation'
normalized_layers = []
graph = []
for position, layer in enumerate(layers, 1):
digest = str(layer['layer_digest'])
media_type = str(layer['media_type'])
size_bytes = int(layer['layer_size_bytes'])
if (
int(layer['position_from_base']) != position
or int(layer['position_from_top']) != len(layers) - position + 1
or str(layer['descriptor_sha256'])
!= canonical_docker_descriptor_hash(digest, media_type, size_bytes)
):
return 'selection_mutation'
graph.append(digest)
normalized_layers.append({
'digest': digest,
'media_type': media_type,
'size_bytes': size_bytes,
'descriptor_sha256': str(layer['descriptor_sha256']),
'position_from_base': position,
'position_from_top': len(layers) - position + 1,
})
graph_sha256 = canonical_docker_layer_graph_hash(tuple(graph))
if graph_sha256 != str(candidate['graph_sha256']):
return 'selection_mutation'
selections = self.conn.execute(
'''SELECT selection.image_rank, selection.selection_reason,
selection.selection_evidence_sha256,
selection.graph_sha256 AS selection_graph_sha256,
member.candidate_distinct_graph_count
FROM docker_depth_experiment_selections selection
JOIN docker_depth_experiment_repositories member
ON member.id = selection.experiment_repository_id
AND member.experiment_id = selection.experiment_id
WHERE selection.experiment_target_id = ?
ORDER BY selection.id LIMIT ?''',
(candidate['experiment_target_id'], authority['query_count'] + 1),
).fetchall()
if not selections or len(selections) > authority['query_count']:
return 'selection_mutation'
for selection in selections:
evidence = {
'schema': 1,
'type': 'docker-depth-selection-evidence-v1',
'selector_version': authority['selector_version'],
'selector_sha256': authority['selector_sha256'],
'candidate_distinct_graph_count': int(
selection['candidate_distinct_graph_count']
),
'image_rank': int(selection['image_rank']),
'selection_reason': str(selection['selection_reason']),
'target': str(candidate['target']),
'repository': str(candidate['repository']),
'manifest_digest': str(candidate['manifest_digest']),
'manifest_media_type': str(candidate['manifest_media_type']),
'manifest_size_bytes': int(candidate['manifest_size_bytes']),
'config_digest': str(candidate['config_digest']),
'graph_sha256': graph_sha256,
'layers': normalized_layers,
}
if (
str(selection['selection_graph_sha256']) != graph_sha256
or str(selection['selection_evidence_sha256'])
!= canonical_docker_depth_selection_evidence_hash(evidence)
):
return 'selection_mutation'
return None
def _terminalize_exhausted_docker_depth_targets_locked(
self, experiment, authority, now, max_attempts,
):
if max_attempts <= 0:
return 0
rows = self.conn.execute(
'''SELECT target.id AS experiment_target_id,
target.target_queue_id, target.manifest_id,
target.reservation_count,
queue.attempts, queue.target_scan_id AS queue_target_scan_id,
binding.id AS binding_id,
binding.reservation_id AS binding_reservation_id,
binding.target_scan_id AS binding_target_scan_id,
binding.attempt AS binding_attempt,
binding.state AS binding_state,
reservation.id AS reservation_id,
reservation.queue_id AS reservation_queue_id,
reservation.state AS reservation_state,
scan.id AS scan_id, scan.queue_id AS scan_queue_id,
scan.result_reservation_id AS scan_reservation_id
FROM docker_depth_experiment_targets target
JOIN target_queue queue ON queue.id = target.target_queue_id
LEFT JOIN docker_depth_experiment_scan_bindings binding
ON binding.experiment_target_id = target.id
AND binding.attempt = target.reservation_count
LEFT JOIN result_reservations reservation
ON reservation.id = binding.reservation_id
LEFT JOIN target_scans scan ON scan.id = binding.target_scan_id
WHERE target.experiment_id = ? AND target.state = 'pending'
AND target.reservation_count > 0
AND queue.source = ? AND queue.platform = 'docker'
AND queue.status = 'deferred'
AND COALESCE(queue.attempts, 0) >= ?
ORDER BY target.id
FOR UPDATE OF target, queue''',
(experiment['id'], authority['source'], max_attempts),
).fetchall()
for row in rows:
if (
row['binding_id'] is None
or int(row['binding_attempt'] or 0)
!= int(row['reservation_count'])
or str(row['binding_state'] or '') != 'failed'
or row['binding_target_scan_id'] is None
or int(row['reservation_id'] or 0)
!= int(row['binding_reservation_id'] or 0)
or int(row['reservation_queue_id'] or 0)
!= int(row['target_queue_id'])
or str(row['reservation_state'] or '')
not in ('db_committed', 'acknowledged')
or int(row['scan_id'] or 0)
!= int(row['binding_target_scan_id'])
or int(row['scan_queue_id'] or 0)
!= int(row['target_queue_id'])
or int(row['scan_reservation_id'] or 0)
!= int(row['reservation_id'] or 0)
or int(row['queue_target_scan_id'] or 0)
!= int(row['scan_id'] or 0)
):
raise ScanEventConflictError(
'Docker depth exhausted retry recovery conflicts with scan history'
)
queue_cursor = self.conn.execute(
'''UPDATE target_queue
SET status = 'failed', available_after = NULL,
completed_at = COALESCE(completed_at, ?),
last_error = COALESCE(
last_error, 'target retry attempts exhausted'
), updated_at = ?
WHERE id = ? AND source = ? AND platform = 'docker'
AND status = 'deferred' AND COALESCE(attempts, 0) >= ?
AND target_scan_id = ?
AND current_result_reservation_id IS NULL
AND lease_owner IS NULL AND lease_token IS NULL
AND claim_batch IS NULL AND leased_at IS NULL
AND lease_expires_at IS NULL AND claim_event_id IS NULL
AND resolver_token IS NULL
AND (resolver_state IS NULL OR resolver_state = 'resolved')''',
(
now, now, row['target_queue_id'], authority['source'],
max_attempts, row['scan_id'],
),
)
target_cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_targets
SET state = 'failed', terminal_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
AND manifest_id = ? AND state = 'pending'
AND reservation_count = ? AND terminal_at IS NULL''',
(
now, now, row['experiment_target_id'], experiment['id'],
row['target_queue_id'], row['manifest_id'],
row['reservation_count'],
),
)
if (
int(queue_cursor.rowcount or 0) != 1
or int(target_cursor.rowcount or 0) != 1
):
raise ScanEventConflictError(
'Docker depth exhausted retry recovery lost its exact fence'
)
return len(rows)
def _claim_docker_depth_experiment_target_locked(
self, experiment, authority, now, max_attempts,
):
self._terminalize_exhausted_docker_depth_targets_locked(
experiment, authority, now, max_attempts,
)
row = self.conn.execute(
'''SELECT queue.id, queue.query, queue.target,
queue.normalized_target, queue.attempts,
target.id AS experiment_target_id,
target.experiment_id, target.manifest_id,
target.dispatch_wave, target.dispatch_order,
target.reservation_count,
manifest.repository, manifest.manifest_digest,
manifest.manifest_media_type, manifest.manifest_size_bytes,
manifest.config_digest, manifest.graph_sha256,
manifest.layer_count
FROM docker_depth_experiment_targets target
JOIN target_queue queue ON queue.id = target.target_queue_id
JOIN docker_image_manifests manifest
ON manifest.id = target.manifest_id
AND manifest.target_queue_id = queue.id
WHERE target.experiment_id = ? AND target.state = 'pending'
AND queue.source = ? AND queue.platform = 'docker'
AND queue.status IN ('pending','deferred')
AND (queue.status = 'pending'
OR (queue.available_after IS NOT NULL
AND queue.available_after <= ?))
AND (? = 0 OR COALESCE(queue.attempts, 0) < ?)
AND queue.current_result_reservation_id IS NULL
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
AND queue.claim_batch IS NULL AND queue.leased_at IS NULL
AND queue.lease_expires_at IS NULL
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
AND (queue.resolver_state IS NULL OR queue.resolver_state = 'resolved')
AND EXISTS (
SELECT 1 FROM docker_depth_experiment_selections selection
WHERE selection.experiment_id = target.experiment_id
AND selection.experiment_target_id = target.id
)
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets prior
WHERE prior.experiment_id = target.experiment_id
AND prior.dispatch_wave < target.dispatch_wave
AND (
prior.state NOT IN ('done','failed')
OR NOT EXISTS (
SELECT 1
FROM docker_depth_experiment_scan_bindings binding
WHERE binding.experiment_target_id = prior.id
AND binding.attempt = prior.reservation_count
AND (
(prior.state = 'done'
AND binding.state = 'completed')
OR (prior.state = 'failed'
AND binding.state = 'failed')
)
)
)
)
ORDER BY target.dispatch_order, target.id
LIMIT 1 FOR UPDATE OF target, queue SKIP LOCKED''',
(
experiment['id'], authority['source'], now,
max_attempts, max_attempts,
),
).fetchone()
if not row:
return None, None
reason = self._docker_depth_candidate_selection_drift_reason(row, authority)
if reason:
self._hold_docker_depth_experiment_locked(experiment, reason, now)
return None, reason
return row, None
def _docker_depth_experiment_schema_installed(self):
cached = getattr(
self, '_docker_depth_experiment_schema_installed_cache', None,
)
if cached is None:
cached = bool(self.conn.execute(
'''SELECT 1 AS present FROM runtime_schema_migrations
WHERE version = ?''',
(DOCKER_DEPTH_EXPERIMENT_MIGRATION,),
).fetchone())
self._docker_depth_experiment_schema_installed_cache = cached
return cached
def _docker_depth_binding_for_reservation_locked(self, reservation):
if not self._docker_depth_experiment_schema_installed():
return None
reservation_id = int(reservation['id'])
binding = self.conn.execute(
'''SELECT binding.*, target.experiment_id,
target.target_queue_id, target.manifest_id,
target.state AS target_state,
target.reservation_count,
queue.source AS bound_source,
queue.platform AS bound_platform,
queue.query AS bound_query,
queue.target AS bound_target,
queue.normalized_target AS bound_normalized_target,
manifest.source AS manifest_source,
manifest.manifest_digest, manifest.layer_count
FROM docker_depth_experiment_scan_bindings binding
JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
JOIN target_queue queue ON queue.id = target.target_queue_id
JOIN docker_image_manifests manifest
ON manifest.id = target.manifest_id
AND manifest.target_queue_id = target.target_queue_id
WHERE binding.reservation_id = ?
FOR UPDATE OF binding, target, queue, manifest''',
(reservation_id,),
).fetchone()
if binding:
if (
int(binding['target_queue_id']) != int(reservation['queue_id'])
or str(binding['bound_source']) != str(reservation['source'])
or str(binding['bound_platform']) != str(reservation['platform'])
or str(binding['bound_query'] or '') != str(reservation['query'] or '')
or str(binding['bound_target']) != str(reservation['target'])
or str(binding['bound_normalized_target'])
!= str(reservation['normalized_target'])
or str(binding['bound_source']) != 'dockerhub'
or str(binding['bound_platform']) != 'docker'
or str(binding['manifest_source']) != str(binding['bound_source'])
or int(binding['attempt']) != int(binding['reservation_count'])
or str(binding['bound_at']) != str(reservation['created_at'])
or str(binding['created_at']) != str(reservation['created_at'])
):
raise ScanEventConflictError(
'Docker depth binding lost its exact reservation target identity'
)
try:
bound_image = parse_docker_target(binding['bound_target'])['image']
except (TypeError, ValueError):
raise ScanEventConflictError(
'Docker depth binding has an invalid immutable target identity'
) from None
if bound_image.rsplit('@', 1)[-1] != str(binding['manifest_digest']):
raise ScanEventConflictError(
'Docker depth binding manifest identity conflicts with its target'
)
return binding
experiment_target = self.conn.execute(
'''SELECT id FROM docker_depth_experiment_targets
WHERE target_queue_id = ? LIMIT 1 FOR UPDATE''',
(int(reservation['queue_id']),),
).fetchone()
if experiment_target:
raise ScanEventConflictError(
'Docker depth target reservation is missing its atomic binding'
)
return None
def _lock_docker_depth_experiment_for_reservation(self, reservation_id):
if not self._docker_depth_experiment_schema_installed():
return None
identities = self.conn.execute(
'''SELECT DISTINCT identity.experiment_id
FROM (
SELECT target.experiment_id
FROM docker_depth_experiment_scan_bindings binding
JOIN docker_depth_experiment_targets target
ON target.id = binding.experiment_target_id
WHERE binding.reservation_id = ?
UNION
SELECT target.experiment_id
FROM result_reservations reservation
JOIN docker_depth_experiment_targets target
ON target.target_queue_id = reservation.queue_id
WHERE reservation.id = ?
) identity
ORDER BY identity.experiment_id LIMIT 2''',
(int(reservation_id), int(reservation_id)),
).fetchall()
if not identities:
return None
if len(identities) != 1:
raise ScanEventConflictError(
'Docker depth reservation resolves to conflicting experiment authority'
)
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE id = ? FOR UPDATE''',
(int(identities[0]['experiment_id']),),
).fetchone()
if not experiment:
raise ScanEventConflictError(
'Docker depth reservation lost its experiment authority'
)
return experiment
def _transition_docker_depth_binding_locked(
self, reservation, binding_state, target_state, now, *, target_scan_id=None,
):
self._lock_docker_depth_experiment_for_reservation(reservation['id'])
binding = self._docker_depth_binding_for_reservation_locked(reservation)
if not binding:
return None
binding_state = str(binding_state)
target_state = str(target_state)
allowed = {
'reserved': ('reserved', 'quarantined'),
'scanning': ('reserved', 'scanning'),
'completed': ('reserved', 'scanning', 'completed'),
'failed': ('reserved', 'scanning', 'failed'),
'quarantined': ('reserved', 'scanning', 'quarantined'),
'released': ('reserved', 'released'),
}
allowed_targets = {
'reserved': ('reserved', 'quarantined'),
'scanning': ('reserved', 'scanning'),
'done': ('reserved', 'scanning', 'done'),
'failed': ('reserved', 'scanning', 'failed'),
'quarantined': ('reserved', 'scanning', 'quarantined'),
'pending': ('reserved', 'scanning', 'pending', 'quarantined'),
}
if str(binding['state']) not in allowed[binding_state]:
raise ScanEventConflictError(
'Docker depth binding state conflicts with its reservation transition'
)
if str(binding['target_state']) not in allowed_targets[target_state]:
raise ScanEventConflictError(
'Docker depth target state conflicts with its reservation transition'
)
if binding['target_scan_id'] is not None and int(
binding['target_scan_id']
) != int(target_scan_id or 0):
raise ScanEventConflictError(
'Docker depth binding scan identity changed across replay'
)
completed_at = now if binding_state in (
'completed', 'failed', 'quarantined', 'released',
) else None
binding_cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_scan_bindings
SET state = ?, target_scan_id = COALESCE(target_scan_id, ?),
scan_bound_at = COALESCE(scan_bound_at, ?),
completed_at = CASE WHEN ? = 0 THEN NULL
ELSE COALESCE(completed_at, ?) END
WHERE id = ? AND experiment_target_id = ?
AND reservation_id = ? AND attempt = ? AND state = ?''',
(
binding_state, target_scan_id,
now if target_scan_id is not None else None,
1 if completed_at is not None else 0, completed_at,
binding['id'], binding['experiment_target_id'],
reservation['id'], binding['attempt'], binding['state'],
),
)
if int(binding_cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth binding transition lost its exact reservation identity'
)
terminal_at = now if target_state in (
'done', 'failed', 'quarantined', 'held', 'skipped',
) else None
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_targets
SET state = ?, terminal_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
AND manifest_id = ? AND reservation_count = ? AND state = ?''',
(
target_state, terminal_at, now, binding['experiment_target_id'],
binding['experiment_id'], reservation['queue_id'],
binding['manifest_id'], binding['reservation_count'],
binding['target_state'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth target state lost its reservation binding identity'
)
return {
'binding_id': int(binding['id']),
'experiment_id': int(binding['experiment_id']),
'experiment_target_id': int(binding['experiment_target_id']),
'manifest_id': int(binding['manifest_id']),
'attempt': int(binding['attempt']),
}
@staticmethod
def _normalize_docker_depth_resolver_authority(authority):
from docker_depth_experiment import (
DOCKER_DEPTH_COLLECTION_GENERATION,
DOCKER_DEPTH_QUERY_COUNT,
canonical_ordered_query_hash,
canonical_selector_hash,
reviewed_docker_experiment_profile,
)
if not isinstance(authority, dict) or authority.get('enabled') is not True:
raise ValueError('Docker depth resolver authority is disabled or invalid')
queries = authority.get('queries')
if isinstance(queries, (str, bytes)):
raise ValueError('Docker depth resolver query authority is invalid')
try:
queries = tuple(queries)
except TypeError:
raise ValueError('Docker depth resolver query authority is invalid') from None
if (
len(queries) != DOCKER_DEPTH_QUERY_COUNT
or len(set(queries)) != len(queries)
or any(not isinstance(query, str) or not query or query != query.strip()
for query in queries)
):
raise ValueError('Docker depth resolver query authority is invalid')
selector_version = str(authority.get('selector_version') or '')
profile = reviewed_docker_experiment_profile(selector_version)
integer_values = {
'query_count': profile['query_count'],
'repositories_per_query': profile['repositories_per_query'],
'images_per_repository': profile['images_per_repository'],
'target_limit': profile['target_limit'],
'theoretical_max_targets': profile['theoretical_max_targets'],
}
for key, expected in integer_values.items():
value = authority.get(key)
if isinstance(value, bool) or value != expected:
raise ValueError(f'Docker depth resolver {key} authority conflicts')
hashes = (
'config_sha256', 'ordered_queries_sha256', 'selector_sha256',
'provenance_policy_sha256',
)
if any(not re.fullmatch(r'[a-f0-9]{64}', str(authority.get(key) or '')) for key in hashes):
raise ValueError('Docker depth resolver hash authority is invalid')
if authority['ordered_queries_sha256'] != canonical_ordered_query_hash(queries):
raise ValueError('Docker depth resolver ordered-query authority conflicts')
if (
authority.get('selector_version') != selector_version
or authority['selector_sha256'] != canonical_selector_hash(selector_version)
):
raise ValueError('Docker depth resolver selector authority conflicts')
experiment_key = str(authority.get('experiment_key') or '')
if not re.fullmatch(r'[a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?', experiment_key):
raise ValueError('Docker depth resolver experiment key is invalid')
if authority.get('source') != 'dockerhub':
raise ValueError('Docker depth resolver source authority conflicts')
if authority.get('collection_generation') != DOCKER_DEPTH_COLLECTION_GENERATION:
raise ValueError('Docker depth resolver collection generation conflicts')
return {
**integer_values,
'enabled': True,
'experiment_key': experiment_key,
'source': 'dockerhub',
'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION,
'queries': queries,
'config_sha256': str(authority['config_sha256']),
'ordered_queries_sha256': str(authority['ordered_queries_sha256']),
'selector_version': selector_version,
'selector_sha256': str(authority['selector_sha256']),
'provenance_policy_sha256': str(authority['provenance_policy_sha256']),
}
@staticmethod
def _docker_depth_experiment_matches_authority(row, authority):
expected = {
'experiment_key': authority['experiment_key'],
'source': authority['source'],
'collection_generation': authority['collection_generation'],
'config_sha256': authority['config_sha256'],
'ordered_queries_sha256': authority['ordered_queries_sha256'],
'selector_version': authority['selector_version'],
'selector_sha256': authority['selector_sha256'],
'provenance_policy_sha256': authority['provenance_policy_sha256'],
'query_count': authority['query_count'],
'repositories_per_query': authority['repositories_per_query'],
'images_per_repository': authority['images_per_repository'],
'target_limit': authority['target_limit'],
}
return bool(row) and all(
(int(row[key]) if isinstance(value, int) else str(row[key])) == value
for key, value in expected.items()
)
def _terminalize_docker_depth_attempt_limit_locked(
self, experiment, member, authority, now, *, held_recovery=False,
):
from docker_depth_experiment import (
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
)
attempts = int(member['resolver_attempts'])
selected_count = int(member['selected_image_count'])
if attempts < DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS:
raise ScanEventConflictError(
'Docker depth attempt-limit terminalization is premature'
)
stored_selection_count = int(self.conn.execute(
'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_selections
WHERE experiment_repository_id = ?''',
(member['id'],),
).fetchone()['count'])
if stored_selection_count != selected_count:
raise ScanEventConflictError(
'Docker depth attempt-limit selection evidence drifted'
)
current_queue_id = int(
member['replacement_repository_queue_id']
or member['repository_queue_id']
)
self._record_docker_depth_candidate_skip_locked(
experiment, member, current_queue_id, 'repository',
int(member['replacement_count']) + 1,
{'repository_queue_id': current_queue_id},
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON, now,
)
next_state = 'skipped' if selected_count == 0 else 'resolved'
next_error = (
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON
if next_state == 'skipped' else None
)
if held_recovery:
if (
str(experiment['state']) != 'held'
or str(experiment['hold_reason_code'] or '')
!= 'resolver_attempt_limit'
or str(member['work_state']) != 'held'
or str(member['last_error_code'] or '')
!= 'resolver_attempt_limit'
or any(member[name] is not None for name in (
'resolver_owner', 'resolver_token', 'resolver_expires_at',
'resolver_due_at',
))
):
raise ScanEventConflictError(
'Docker depth held attempt-limit evidence drifted'
)
held_count = int(self.conn.execute(
'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_repositories
WHERE experiment_id = ? AND work_state = 'held' ''',
(experiment['id'],),
).fetchone()['count'])
if held_count != 1:
raise ScanEventConflictError(
'Docker depth attempt-limit hold is ambiguous'
)
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = ?, resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = NULL, last_error_code = ?,
resolved_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ? AND work_state = 'held'
AND last_error_code = 'resolver_attempt_limit'
AND resolver_attempts >= ?
AND resolver_owner IS NULL AND resolver_token IS NULL
AND resolver_expires_at IS NULL AND resolver_due_at IS NULL''',
(
next_state, next_error, now, now, member['id'],
experiment['id'], DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth attempt-limit recovery lost its member fence'
)
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'resolving', hold_reason_code = NULL,
held_at = NULL, updated_at = ?
WHERE id = ? AND state = 'held'
AND hold_reason_code = 'resolver_attempt_limit'
AND fence_owner IS NULL AND fence_token IS NULL
AND fence_expires_at IS NULL''',
(now, experiment['id']),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth attempt-limit recovery lost its experiment fence'
)
else:
if (
str(experiment['state']) != 'resolving'
or str(member['work_state']) != 'resolving'
or not member['resolver_owner']
or not member['resolver_token']
or str(member['resolver_expires_at'] or '') <= now
):
raise ScanEventConflictError(
'Docker depth attempt-limit lease evidence drifted'
)
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = ?, resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = NULL, last_error_code = ?,
resolved_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ? AND work_state = 'resolving'
AND resolver_generation = ? AND resolver_owner = ?
AND resolver_token = ? AND resolver_expires_at > ?
AND resolver_attempts >= ?''',
(
next_state, next_error, now, now, member['id'],
experiment['id'], member['resolver_generation'],
member['resolver_owner'], member['resolver_token'], now,
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth attempt-limit terminalization lost its lease fence'
)
if selected_count == 0:
self._finalize_docker_depth_query_breadth_locked(
experiment, member, now,
)
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
experiment, selection_reason = (
self._freeze_docker_depth_runtime_selection_locked(
experiment, authority, now,
)
)
if selection_reason:
result = self._hold_docker_depth_experiment_locked(
experiment, selection_reason, now,
)
result['experiment'] = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
return result
drift_reason = self._docker_depth_persisted_drift_reason_locked(
experiment, authority, now,
)
if drift_reason:
result = self._hold_docker_depth_experiment_locked(
experiment, drift_reason, now,
)
result['experiment'] = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
return result
experiment = self._advance_docker_depth_experiment_state_locked(
experiment, now,
)
return {
'status': 'skipped' if next_state == 'skipped' else 'resolved',
'committed': True,
'reason': DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
'experiment_state': str(experiment['state']),
'experiment': experiment,
}
def claim_docker_depth_experiment_resolutions(
self, source, limit, lease_owner, lease_seconds=300, *, authority=None,
final_cutover=False,
):
if (
not self.conn
or not self.conn.is_postgres
or source != 'dockerhub'
or not lease_owner
):
return []
if isinstance(authority, dict) and authority.get('enabled') is False:
self._hold_disabled_docker_depth_experiment(authority)
return []
if not isinstance(authority, dict) or authority.get('enabled') is not True:
return []
try:
normalized = self._normalize_docker_depth_resolver_authority(authority)
except (TypeError, ValueError):
return []
from docker_depth_experiment import (
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
_require_released_policy_history,
)
if normalized['selector_version'] == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
original_queue_policy_clause = "queue.status IN ('pending','deferred')"
else:
original_queue_policy_clause = '''queue.status IN ('pending','deferred')
AND NOT EXISTS (
SELECT 1 FROM target_queue_policy_events event
WHERE event.queue_id = queue.id
)'''
def op():
state = self._locked_runtime_control_state(shared=True)
if state['effective_discovery_paused']:
self.conn.commit()
return []
now = utc_now_iso()
lease_until = datetime.fromtimestamp(
time.time() + max(60, min(3600, int(lease_seconds or 300))),
timezone.utc,
).isoformat(timespec='seconds')
experiment, _reason = self._locked_docker_depth_experiment_authority(
normalized, final_cutover=final_cutover, now=now,
)
if not experiment:
self.conn.commit()
return []
if (
str(experiment['state']) == 'held'
and str(experiment['hold_reason_code'] or '')
== 'resolver_attempt_limit'
):
held_members = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_repositories
WHERE experiment_id = ? AND work_state = 'held'
ORDER BY id LIMIT 2 FOR UPDATE''',
(experiment['id'],),
).fetchall()
if len(held_members) != 1:
self.conn.commit()
return []
terminal = self._terminalize_docker_depth_attempt_limit_locked(
experiment, held_members[0], normalized, now,
held_recovery=True,
)
experiment = terminal['experiment']
if str(experiment['state']) != 'resolving':
self.conn.commit()
return []
if (
str(experiment['state']) == 'held'
and str(experiment['hold_reason_code'] or '')
== 'stale_resolver_fence'
):
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'resolving', hold_reason_code = NULL,
held_at = NULL, updated_at = ?
WHERE id = ? AND state = 'held'
AND hold_reason_code = 'stale_resolver_fence'
AND fence_owner IS NULL AND fence_token IS NULL
AND fence_expires_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_repositories member
WHERE member.experiment_id = ?
AND (member.work_state = 'resolving'
OR member.resolver_owner IS NOT NULL
OR member.resolver_token IS NOT NULL
OR member.resolver_expires_at IS NOT NULL)
)''',
(now, experiment['id'], experiment['id']),
)
if int(cursor.rowcount or 0) != 1:
self.conn.commit()
return []
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
if str(experiment['state']) == 'holding':
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET state = 'resolving', updated_at = ?
WHERE id = ? AND state = 'holding'
AND plan_sha256 = ? AND hold_manifest_sha256 = ?''',
(
now, experiment['id'], experiment['plan_sha256'],
experiment['hold_manifest_sha256'],
),
)
if int(cursor.rowcount or 0) != 1:
self._hold_docker_depth_experiment_locked(
experiment, 'activation_fence_conflict', now,
)
self.conn.commit()
return []
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
experiment = self._advance_docker_depth_experiment_state_locked(
experiment, now,
)
if str(experiment['state']) != 'resolving':
self.conn.commit()
return []
output = []
maximum = max(1, min(100, int(limit or 1)))
if normalized['selector_version'] == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
maximum = 1
for _ in range(maximum):
row = self.conn.execute(
f'''SELECT member.*, queue.id AS resolution_repository_queue_id,
queue.target
FROM docker_depth_experiment_repositories member
JOIN target_queue queue ON queue.id = COALESCE(
member.replacement_repository_queue_id,
member.repository_queue_id
)
WHERE member.experiment_id = ?
AND member.selected_image_count = 0
AND (
(member.work_state = 'pending'
AND (member.resolver_due_at IS NULL OR member.resolver_due_at <= ?))
OR (member.work_state = 'resolving'
AND member.resolver_expires_at <= ?)
)
AND queue.source = ? AND queue.platform = 'docker'
AND (
({original_queue_policy_clause})
OR (member.replacement_repository_queue_id IS NOT NULL
AND queue.status = 'cold'
AND (SELECT COUNT(*) FROM target_queue_policy_events event
WHERE event.queue_id = queue.id) = 1
AND EXISTS (
SELECT 1 FROM target_queue_policy_events event
WHERE event.queue_id = queue.id
AND event.action = 'cold'
AND event.experiment_id = member.experiment_id
AND event.reason_code IN (
'docker_depth_experiment_hold',
'docker_depth_experiment_dynamic_hold'
)
AND event.config_sha256 = ?
AND event.policy_sha256 = ?
AND event.manifest_sha256 = ?
)))
AND queue.target_scan_id IS NULL
AND queue.target NOT LIKE '%@%' AND queue.normalized_target NOT LIKE '%@%'
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
AND queue.claim_batch IS NULL AND queue.current_result_reservation_id IS NULL
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
AND NOT EXISTS (
SELECT 1 FROM target_scans scan
WHERE scan.queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1 FROM result_reservations reservation
WHERE reservation.queue_id = queue.id
)
ORDER BY member.repository_rank, member.query_ordinal, member.id
LIMIT 1 FOR UPDATE OF member SKIP LOCKED''',
(
experiment['id'], now, now, source,
experiment['config_sha256'],
experiment['provenance_policy_sha256'],
experiment['hold_manifest_sha256'],
),
).fetchone()
stage = 'breadth'
if not row:
unfinished_breadth = self.conn.execute(
'''SELECT COUNT(*) AS count
FROM docker_depth_experiment_repositories
WHERE experiment_id = ? AND selected_image_count = 0
AND work_state IN ('pending','resolving')''',
(experiment['id'],),
).fetchone()['count']
if int(unfinished_breadth):
break
experiment, selection_reason = (
self._freeze_docker_depth_runtime_selection_locked(
experiment, normalized, now,
)
)
if selection_reason:
self._hold_docker_depth_experiment_locked(
experiment, selection_reason, now,
)
break
row = self.conn.execute(
'''SELECT member.*, queue.id AS resolution_repository_queue_id,
queue.target
FROM docker_depth_experiment_repositories member
JOIN target_queue queue ON queue.id = COALESCE(
member.replacement_repository_queue_id,
member.repository_queue_id
)
WHERE member.experiment_id = ? AND member.is_deep_probe = 1
AND member.selected_image_count >= 1
AND member.selected_image_count < member.candidate_distinct_graph_count
AND member.selected_image_count < ?
AND (
(member.work_state = 'pending'
AND (member.resolver_due_at IS NULL OR member.resolver_due_at <= ?))
OR (member.work_state = 'resolving'
AND member.resolver_expires_at <= ?)
)
AND queue.source = ? AND queue.platform = 'docker'
AND (
(queue.status IN ('pending','deferred') AND NOT EXISTS (
SELECT 1 FROM target_queue_policy_events event
WHERE event.queue_id = queue.id
))
OR (member.replacement_repository_queue_id IS NOT NULL
AND queue.status = 'cold'
AND (SELECT COUNT(*) FROM target_queue_policy_events event
WHERE event.queue_id = queue.id) = 1
AND EXISTS (
SELECT 1 FROM target_queue_policy_events event
WHERE event.queue_id = queue.id
AND event.action = 'cold'
AND event.experiment_id = member.experiment_id
AND event.reason_code IN (
'docker_depth_experiment_hold',
'docker_depth_experiment_dynamic_hold'
)
AND event.config_sha256 = ?
AND event.policy_sha256 = ?
AND event.manifest_sha256 = ?
)))
AND queue.target_scan_id IS NULL
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
AND queue.claim_batch IS NULL
AND queue.current_result_reservation_id IS NULL
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
AND NOT EXISTS (
SELECT 1 FROM target_scans scan
WHERE scan.queue_id = queue.id
)
AND NOT EXISTS (
SELECT 1 FROM result_reservations reservation
WHERE reservation.queue_id = queue.id
)
ORDER BY member.query_ordinal, member.repository_rank, member.id
LIMIT 1 FOR UPDATE OF member SKIP LOCKED''',
(
experiment['id'], normalized['images_per_repository'],
now, now, source, experiment['config_sha256'],
experiment['provenance_policy_sha256'],
experiment['hold_manifest_sha256'],
),
).fetchone()
stage = 'deep'
if not row:
break
if (
normalized['selector_version']
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
and row['replacement_repository_queue_id'] is None
):
try:
_require_released_policy_history(
self.conn, [int(row['resolution_repository_queue_id'])],
)
except RuntimeError:
self._hold_docker_depth_experiment_locked(
experiment, 'target_history_drift', now,
)
self.conn.commit()
return []
token = secrets.token_urlsafe(32)
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'resolving',
resolver_generation = resolver_generation + 1,
resolver_owner = ?, resolver_token = ?, resolver_expires_at = ?,
resolver_due_at = NULL, resolver_attempts = resolver_attempts + 1,
updated_at = ?
WHERE id = ?''',
(lease_owner, token, lease_until, now, row['id']),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker depth resolver claim lost its row fence')
output.append({
'id': int(row['id']),
'experiment_id': int(experiment['id']),
'repository_queue_id': int(row['resolution_repository_queue_id']),
'query_ordinal': int(row['query_ordinal']),
'query': str(row['query']),
'repository_rank': int(row['repository_rank']),
'target': str(row['target']),
'stage': stage,
'selection_limit': (
1 if stage == 'breadth'
else normalized['images_per_repository']
),
'resolver_owner': str(lease_owner),
'resolver_token': token,
'resolver_generation': int(row['resolver_generation']) + 1,
'resolver_attempts': int(row['resolver_attempts']) + 1,
'resolver_expires_at': lease_until,
'config_sha256': normalized['config_sha256'],
'ordered_queries_sha256': normalized['ordered_queries_sha256'],
'selector_sha256': normalized['selector_sha256'],
})
self.conn.commit()
return output
return self._safe('claim_docker_depth_experiment_resolutions', op, [])
def renew_docker_depth_experiment_resolution(
self, source, repository_id, resolver_generation, resolver_token, *,
resolver_owner, lease_seconds=300, authority=None, final_cutover=False,
):
if (
not self.conn
or not self.conn.is_postgres
or source != 'dockerhub'
or not resolver_owner
or not resolver_token
):
return {'status': 'unavailable', 'renewed': False}
try:
normalized = self._normalize_docker_depth_resolver_authority(authority)
repository_id = int(repository_id)
resolver_generation = int(resolver_generation)
lease_seconds = int(lease_seconds)
if (
repository_id < 1
or resolver_generation < 1
or not 60 <= lease_seconds <= 3600
):
raise ValueError('Docker depth resolver renewal identity is invalid')
except (TypeError, ValueError, OverflowError):
return {'status': 'invalid', 'renewed': False}
now_dt = datetime.now(timezone.utc)
now = now_dt.isoformat(timespec='seconds')
lease_until = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
timespec='seconds',
)
try:
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
(normalized['experiment_key'], source),
).fetchone()
member = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_repositories
WHERE id = ? AND experiment_id = ? FOR UPDATE''',
(repository_id, experiment['id'] if experiment else 0),
).fetchone()
if not (
experiment
and str(experiment['state']) == 'resolving'
and member
and str(member['work_state']) == 'resolving'
and int(member['resolver_generation']) == resolver_generation
and str(member['resolver_owner']) == str(resolver_owner)
and str(member['resolver_token']) == str(resolver_token)
and str(member['resolver_expires_at'] or '') > now
):
self.conn.rollback()
return {'status': 'stale', 'renewed': False}
checked, reason = self._locked_docker_depth_experiment_authority(
normalized, final_cutover=final_cutover, now=now,
)
if not checked:
self.conn.commit()
return {
'status': 'held', 'renewed': False, 'reason': reason,
}
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET resolver_expires_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ?
AND work_state = 'resolving'
AND resolver_generation = ? AND resolver_owner = ?
AND resolver_token = ? AND resolver_expires_at > ?''',
(
lease_until, now, repository_id, experiment['id'],
resolver_generation, resolver_owner, resolver_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return {'status': 'stale', 'renewed': False}
self.conn.commit()
return {
'status': 'renewed', 'renewed': True,
'resolver_expires_at': lease_until,
}
except Exception:
self.conn.rollback()
raise
def reserve_docker_depth_experiment_target_capacity_locked(
self, experiment_id, target_queue_id, manifest_id,
dispatch_wave, dispatch_order, now=None,
):
"""Reserve one deduplicated target slot inside the caller's transaction."""
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('Docker depth target capacity requires PostgreSQL')
values = (
experiment_id, target_queue_id, manifest_id, dispatch_wave, dispatch_order,
)
if any(isinstance(value, bool) for value in values):
raise ValueError('Docker depth target capacity identity is invalid')
try:
experiment_id, target_queue_id, manifest_id, dispatch_wave, dispatch_order = (
int(value) for value in values
)
except (TypeError, ValueError, OverflowError):
raise ValueError('Docker depth target capacity identity is invalid') from None
if (
min(experiment_id, target_queue_id, manifest_id, dispatch_order) < 1
or dispatch_wave not in (1, 2, 3)
):
raise ValueError('Docker depth target capacity identity is invalid')
experiment = self.conn.execute(
'''SELECT id, state, target_count, target_limit
FROM docker_depth_experiments WHERE id = ? FOR UPDATE''',
(experiment_id,),
).fetchone()
if not experiment or experiment['state'] not in ('resolving', 'active'):
raise ScanEventConflictError('Docker depth target capacity has no active authority')
actual_count = self.conn.execute(
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_targets
WHERE experiment_id = ?''',
(experiment_id,),
).fetchone()['count']
target_count = int(experiment['target_count'])
if int(actual_count) != target_count:
raise ScanEventConflictError('Docker depth target capacity counter drifted')
existing = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_targets
WHERE experiment_id = ? AND target_queue_id = ? FOR UPDATE''',
(experiment_id, target_queue_id),
).fetchone()
if existing:
if int(existing['manifest_id']) != manifest_id:
raise ScanEventConflictError('Docker depth target manifest identity conflicts')
if (dispatch_wave, dispatch_order) < (
int(existing['dispatch_wave']), int(existing['dispatch_order']),
):
existing = self.conn.execute(
'''UPDATE docker_depth_experiment_targets
SET dispatch_wave = ?, dispatch_order = ?, updated_at = ?
WHERE id = ? RETURNING *''',
(dispatch_wave, dispatch_order, str(now or utc_now_iso()), existing['id']),
).fetchone()
return {'target': dict(existing), 'inserted': False}
if target_count >= int(experiment['target_limit']):
raise ScanEventConflictError('Docker depth unique target capacity is exhausted')
now = str(now or utc_now_iso())
target = self.conn.execute(
'''INSERT INTO docker_depth_experiment_targets(
experiment_id, target_queue_id, manifest_id, counter_ordinal,
state, dispatch_wave, dispatch_order, created_at, updated_at
) VALUES (?, ?, ?, ?, 'pending', ?, ?, ?, ?) RETURNING *''',
(
experiment_id, target_queue_id, manifest_id, target_count + 1,
dispatch_wave, dispatch_order, now, now,
),
).fetchone()
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET target_count = target_count + 1, updated_at = ?
WHERE id = ? AND target_count = ? AND target_count < target_limit''',
(now, experiment_id, target_count),
)
if not target or int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker depth target capacity lost its serialized fence')
return {'target': dict(target), 'inserted': True}
@staticmethod
def _normalize_docker_depth_resolution_outcome(outcome, repository, authority, stage):
from docker_depth_experiment import (
canonical_docker_depth_selection_evidence_hash,
canonical_docker_descriptor_hash,
canonical_docker_layer_graph_hash,
)
def value(name, default=None):
if isinstance(outcome, dict):
return outcome.get(name, default)
return getattr(outcome, name, default)
if (
not outcome
or value('selector_version') != authority['selector_version']
or value('selector_hash') != authority['selector_sha256']
or value('fresh_graph_evidence') is not True
or value('cache_bypassed') is not True
):
raise ScanEventConflictError('Docker depth resolver outcome lacks fresh selector evidence')
candidate_count = value('candidate_distinct_graph_count')
if (
isinstance(candidate_count, bool)
or not isinstance(candidate_count, int)
or not 0 <= candidate_count <= 100
):
raise ScanEventConflictError('Docker depth candidate graph count is invalid')
selected_records = value('selection_records', ())
if isinstance(selected_records, (str, bytes)):
raise ScanEventConflictError('Docker depth selection evidence is invalid')
try:
selected_records = list(selected_records)
except TypeError:
raise ScanEventConflictError('Docker depth selection evidence is invalid') from None
candidate_records = value('candidate_records', selected_records)
if not candidate_records:
candidate_records = selected_records
if isinstance(candidate_records, (str, bytes)):
raise ScanEventConflictError('Docker depth candidate evidence is invalid')
try:
candidate_records = list(candidate_records)
except TypeError:
raise ScanEventConflictError('Docker depth candidate evidence is invalid') from None
selected_maximum = 1 if stage == 'breadth' else authority['images_per_repository']
if (
len(selected_records) > selected_maximum
or len(candidate_records) > 100
or candidate_count < len(candidate_records)
or candidate_records[:len(selected_records)] != selected_records
):
raise ScanEventConflictError('Docker depth selection count conflicts with graph evidence')
tags = value('tags', ())
if list(tags or ()) != [
record.get('target') for record in selected_records
if isinstance(record, dict)
]:
raise ScanEventConflictError('Docker depth selected targets conflict with selection evidence')
normalized_records = []
seen_targets = set()
seen_graphs = set()
for expected_rank, raw in enumerate(candidate_records, 1):
if not isinstance(raw, dict):
raise ScanEventConflictError('Docker depth selection record is invalid')
rank = raw.get('image_rank', raw.get('rank'))
reason = str(raw.get('selection_reason', raw.get('reason')) or '')
if (
isinstance(rank, bool)
or rank != expected_rank
or not reason
or len(reason) > 128
):
raise ScanEventConflictError('Docker depth selection rank or reason is invalid')
target = str(raw.get('target') or '').strip()
try:
parsed = parse_docker_target(target)
except (TypeError, ValueError) as exc:
raise ScanEventConflictError('Docker depth immutable target is invalid') from exc
image = str(parsed['image']).lower()
if '@' not in image or parsed['target'] != target:
raise ScanEventConflictError('Docker depth immutable target is not canonical')
target_repository, target_digest = image.rsplit('@', 1)
manifest_digest = str(raw.get('manifest_digest') or '').lower()
if (
target_repository != repository
or str(raw.get('repository') or '').lower() != repository
or manifest_digest != target_digest
or not re.fullmatch(r'sha256:[a-f0-9]{64}', manifest_digest)
):
raise ScanEventConflictError('Docker depth repository or manifest identity conflicts')
manifest_media_type = str(raw.get('manifest_media_type') or '')
config_digest = str(raw.get('config_digest') or '').lower()
manifest_size_bytes = raw.get('manifest_size_bytes')
if (
not manifest_media_type
or manifest_media_type != manifest_media_type.strip().lower()
or len(manifest_media_type) > 256
or not re.fullmatch(r'sha256:[a-f0-9]{64}', config_digest)
or isinstance(manifest_size_bytes, bool)
or not isinstance(manifest_size_bytes, int)
or not 0 <= manifest_size_bytes <= 128 * 1024 * 1024
):
raise ScanEventConflictError('Docker depth manifest metadata is invalid')
layer_metadata = raw.get('layer_metadata')
if isinstance(layer_metadata, (str, bytes)):
raise ScanEventConflictError('Docker depth layer descriptors are invalid')
try:
layer_metadata = list(layer_metadata)
except TypeError:
raise ScanEventConflictError('Docker depth layer descriptors are invalid') from None
if not layer_metadata or len(layer_metadata) > 10000:
raise ScanEventConflictError('Docker depth layer descriptor count is invalid')
layers = []
layer_count = len(layer_metadata)
for position, descriptor in enumerate(layer_metadata, 1):
if not isinstance(descriptor, dict):
raise ScanEventConflictError('Docker depth layer descriptor is invalid')
digest = str(descriptor.get('digest') or '').lower()
media_type = str(descriptor.get('media_type') or '')
size_bytes = descriptor.get('size_bytes')
descriptor_sha256 = str(descriptor.get('descriptor_sha256') or '')
if (
not re.fullmatch(r'sha256:[a-f0-9]{64}', digest)
or not media_type
or media_type != media_type.strip().lower()
or len(media_type) > 256
or isinstance(size_bytes, bool)
or not isinstance(size_bytes, int)
or not 0 <= size_bytes <= 1024 * 1024 * 1024 * 1024
or descriptor.get('position_from_base') != position
or descriptor.get('position_from_top') != layer_count - position + 1
or descriptor_sha256
!= canonical_docker_descriptor_hash(digest, media_type, size_bytes)
):
raise ScanEventConflictError('Docker depth layer descriptor evidence conflicts')
layers.append({
'digest': digest,
'media_type': media_type,
'size_bytes': size_bytes,
'descriptor_sha256': descriptor_sha256,
'position_from_base': position,
'position_from_top': layer_count - position + 1,
})
graph = tuple(layer['digest'] for layer in layers)
raw_graph = tuple(raw.get('graph', raw.get('layers', ())) or ())
graph_sha256 = canonical_docker_layer_graph_hash(graph)
if (
raw_graph != graph
or raw.get('layer_count') != layer_count
or raw.get('graph_sha256', raw.get('graph_hash')) != graph_sha256
):
raise ScanEventConflictError('Docker depth ordered layer graph evidence conflicts')
normalized_target = normalize_target(target, 'docker')
if normalized_target != image or normalized_target in seen_targets or graph in seen_graphs:
raise ScanEventConflictError('Docker depth duplicate target or graph consumed a rank')
seen_targets.add(normalized_target)
seen_graphs.add(graph)
evidence = {
'schema': 1,
'type': 'docker-depth-selection-evidence-v1',
'selector_version': authority['selector_version'],
'selector_sha256': authority['selector_sha256'],
'candidate_distinct_graph_count': candidate_count,
'image_rank': rank,
'selection_reason': reason,
'target': target,
'repository': repository,
'manifest_digest': manifest_digest,
'manifest_media_type': manifest_media_type,
'manifest_size_bytes': manifest_size_bytes,
'config_digest': config_digest,
'graph_sha256': graph_sha256,
'layers': layers,
}
selection_evidence_sha256 = canonical_docker_depth_selection_evidence_hash(evidence)
if str(raw.get('selection_evidence_sha256') or '') != selection_evidence_sha256:
raise ScanEventConflictError('Docker depth selector evidence hash conflicts')
normalized_records.append({
**evidence,
'normalized_target': normalized_target,
'selection_evidence_sha256': selection_evidence_sha256,
})
return candidate_count, normalized_records
def _docker_depth_exact_reactivation_locked(self, experiment_id, queue):
from docker_depth_experiment import (
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
DOCKER_DEPTH_HOLD_REASON,
DOCKER_DEPTH_RELEASE_REASON,
)
experiment = self.conn.execute(
'''SELECT config_sha256, provenance_policy_sha256,
hold_manifest_sha256
FROM docker_depth_experiments WHERE id = ?''',
(experiment_id,),
).fetchone()
events = self.conn.execute(
'''SELECT * FROM target_queue_policy_events
WHERE queue_id = ? ORDER BY id LIMIT 3 FOR UPDATE''',
(queue['id'],),
).fetchall()
if not experiment or len(events) != 2:
return False
cold, reactivation = events
if (
cold['action'] != 'cold'
or reactivation['action'] != 'reactivate'
or int(reactivation['reverses_event_id'] or 0) != int(cold['id'])
or any(
event['experiment_id'] is None
or int(event['experiment_id']) != int(experiment_id)
or int(event['queue_id']) != int(queue['id'])
or str(event['source']) != str(queue['source'])
or str(event['platform']) != str(queue['platform'])
or str(event['query']) != str(queue['query'])
or str(event['config_sha256']) != str(experiment['config_sha256'])
or str(event['policy_sha256'])
!= str(experiment['provenance_policy_sha256'])
for event in events
)
or str(cold['manifest_sha256'])
!= str(experiment['hold_manifest_sha256'])
or str(cold['reason_code']) not in (
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
)
or str(reactivation['reason_code']) != DOCKER_DEPTH_RELEASE_REASON
or str(cold['next_status']) != 'cold'
or str(reactivation['prior_status']) != 'cold'
or str(reactivation['next_status']) != str(queue['status'])
or str(cold['prior_status']) != str(queue['status'])
):
return False
cold_entry = {
'queue_id': int(queue['id']),
'source': str(queue['source']),
'platform': str(queue['platform']),
'query': str(queue['query']),
'prior_status': str(cold['prior_status']),
'prior_updated_at': str(cold['prior_updated_at']),
}
reactivation_entry = {
'queue_id': int(queue['id']),
'source': str(queue['source']),
'platform': str(queue['platform']),
'query': str(queue['query']),
'cold_event_id': int(cold['id']),
'restore_status': str(reactivation['next_status']),
'prior_updated_at': str(reactivation['prior_updated_at']),
}
return (
str(cold['review_audit_sha256'])
== self._target_queue_policy_audit_sha256(
'cold', cold['manifest_sha256'], cold_entry, experiment_id,
)
and str(reactivation['review_audit_sha256'])
== self._target_queue_policy_audit_sha256(
'reactivate', reactivation['manifest_sha256'],
reactivation_entry, experiment_id,
)
)
def _record_docker_depth_candidate_skip_locked(
self, experiment, member, repository_queue_id, candidate_kind,
candidate_ordinal, candidate_identity, reason, now,
):
candidate_identity_sha256, evidence_sha256 = (
self._docker_depth_candidate_skip_evidence(
experiment['id'], member['id'], repository_queue_id,
candidate_kind, candidate_ordinal, candidate_identity, reason,
)
)
self.conn.execute(
'''INSERT INTO docker_depth_experiment_candidate_skips(
experiment_id, experiment_repository_id, repository_queue_id,
candidate_kind, candidate_ordinal, reason_code,
candidate_identity_sha256, evidence_sha256, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(
experiment_repository_id, candidate_kind,
candidate_identity_sha256
) DO NOTHING''',
(
experiment['id'], member['id'], repository_queue_id,
candidate_kind, candidate_ordinal, reason,
candidate_identity_sha256, evidence_sha256, now,
),
)
def _docker_depth_immutable_candidate_conflict_locked(
self, experiment_id, source, record,
):
row = self.conn.execute(
'''SELECT * FROM target_queue
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
(source, record['normalized_target']),
).fetchone()
if not row:
return None, None
experiment_target = self.conn.execute(
'''SELECT id FROM docker_depth_experiment_targets
WHERE experiment_id = ? AND target_queue_id = ?''',
(experiment_id, row['id']),
).fetchone()
historical = self.conn.execute(
'''SELECT
EXISTS(SELECT 1 FROM target_scans WHERE queue_id = ?) AS scanned,
EXISTS(SELECT 1 FROM result_reservations WHERE queue_id = ?) AS reserved,
EXISTS(SELECT 1 FROM target_queue_policy_events WHERE queue_id = ?) AS policy,
EXISTS(
SELECT 1 FROM result_reservations reservation
JOIN pipeline_quarantine quarantine
ON quarantine.reservation_id = reservation.id
WHERE reservation.queue_id = ?
) AS quarantined,
EXISTS(
SELECT 1 FROM docker_image_blob_coverage coverage
JOIN docker_content_blobs blob
ON blob.digest = coverage.blob_digest
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
WHERE coverage.queue_id = ?
AND blob.state IN ('leased','submitted')
) AS blob_fenced''',
(row['id'], row['id'], row['id'], row['id'], row['id']),
).fetchone()
if row['status'] in ('done', 'failed') or row['completed_at'] is not None or bool(
historical['scanned']
):
return 'immutable_target_terminal', int(row['id'])
if row['status'] == 'quarantined' or bool(historical['quarantined']):
return 'immutable_target_quarantined', int(row['id'])
if row['status'] == 'cold' or bool(historical['policy']):
return 'immutable_target_independently_cold', int(row['id'])
fenced = (
row['status'] == 'in_progress'
or bool(historical['reserved'])
or bool(historical['blob_fenced'])
or any(row[name] is not None for name in (
'lease_owner', 'lease_token', 'claim_batch', 'leased_at',
'lease_expires_at', 'current_result_reservation_id',
'claim_event_id', 'resolver_token',
))
)
if fenced:
return 'immutable_target_fenced', int(row['id'])
if experiment_target and row['status'] in ('pending', 'deferred'):
return None, int(row['id'])
return 'immutable_target_existing', int(row['id'])
def _docker_depth_repository_candidate_conflict(
self, experiment, queue_id, *, lock,
):
suffix = ' FOR UPDATE' if lock else ''
queue = self.conn.execute(
f'SELECT * FROM target_queue WHERE id = ?{suffix}',
(queue_id,),
).fetchone()
if not queue:
return 'repository_candidate_absent'
fences = any(queue[name] is not None for name in (
'target_scan_id', 'lease_owner', 'lease_token', 'claim_batch',
'leased_at', 'lease_expires_at', 'current_result_reservation_id',
'claim_event_id', 'resolver_token',
)) or str(queue['resolver_state'] or '') == 'resolving'
if fences or self.conn.execute(
'''SELECT 1 FROM target_scans WHERE queue_id = ?
UNION ALL SELECT 1 FROM result_reservations WHERE queue_id = ?
LIMIT 1''',
(queue_id, queue_id),
).fetchone():
return 'repository_candidate_fenced'
events = self.conn.execute(
'''SELECT * FROM target_queue_policy_events
WHERE queue_id = ? ORDER BY id LIMIT 3''',
(queue_id,),
).fetchall()
if queue['status'] in ('pending', 'deferred') and not events:
return None
if queue['status'] == 'cold' and len(events) == 1:
event = events[0]
if (
event['action'] == 'cold'
and int(event['experiment_id'] or 0) == int(experiment['id'])
and str(event['reason_code']) in (
'docker_depth_experiment_hold',
'docker_depth_experiment_dynamic_hold',
)
and str(event['config_sha256']) == str(experiment['config_sha256'])
and str(event['policy_sha256'])
== str(experiment['provenance_policy_sha256'])
and str(event['manifest_sha256'])
== str(experiment['hold_manifest_sha256'])
and str(event['next_status']) == 'cold'
and str(event['prior_status']) in ('pending', 'deferred')
and str(event['review_audit_sha256'])
== self._target_queue_policy_audit_sha256(
'cold', event['manifest_sha256'], {
'queue_id': int(queue['id']),
'source': str(queue['source']),
'platform': str(queue['platform']),
'query': str(queue['query']),
'prior_status': str(event['prior_status']),
'prior_updated_at': str(event['prior_updated_at']),
}, int(experiment['id']),
)
):
return None
if queue['status'] in ('done', 'failed'):
return 'repository_candidate_terminal'
if queue['status'] == 'quarantined':
return 'repository_candidate_quarantined'
if queue['status'] == 'cold' or events:
return 'repository_candidate_independently_cold'
return 'repository_candidate_ineligible'
def _docker_depth_repository_candidate_conflict_locked(
self, experiment, queue_id,
):
return self._docker_depth_repository_candidate_conflict(
experiment, queue_id, lock=True,
)
def _docker_depth_repository_replacement_candidates(
self, experiment, member, authority, *, lock,
):
candidates = self.conn.execute(
'''SELECT provenance.repository_queue_id,
MIN(observation.search_rank) AS best_search_rank,
MIN(observation.page_id) AS eligibility_page_id,
MIN(queue.normalized_target) AS normalized_target
FROM docker_repository_query_provenance provenance
JOIN docker_repository_query_observations observation
ON observation.source = provenance.source
AND observation.query = provenance.query
AND observation.repository_queue_id = provenance.repository_queue_id
JOIN docker_discovery_pages page ON page.id = observation.page_id
JOIN docker_discovery_passes discovery_pass ON discovery_pass.id = page.pass_id
JOIN target_queue queue ON queue.id = provenance.repository_queue_id
WHERE provenance.source = ? AND provenance.query = ?
AND provenance.provenance_kind = 'fresh_page'
AND provenance.fresh_coverage_eligible = 1
AND provenance.fresh_complete_observation_count > 0
AND page.query_ordinal = ? AND page.query = ?
AND discovery_pass.source = ? AND discovery_pass.pass_kind = 'deep'
AND discovery_pass.collection_generation = ?
AND discovery_pass.policy_sha256 = ?
AND discovery_pass.ordered_queries_sha256 = ?
AND discovery_pass.expected_query_count = ?
AND discovery_pass.state = 'complete'
AND queue.source = ? AND queue.platform = 'docker'
AND queue.target NOT LIKE '%@%' AND queue.normalized_target NOT LIKE '%@%'
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_repositories other
WHERE other.experiment_id = ? AND other.query_ordinal = ?
AND (other.repository_queue_id = queue.id
OR other.replacement_repository_queue_id = queue.id)
)
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_candidate_skips skipped
WHERE skipped.experiment_repository_id = ?
AND skipped.candidate_kind = 'repository'
AND skipped.repository_queue_id = queue.id
)
GROUP BY provenance.repository_queue_id
ORDER BY MIN(observation.search_rank), provenance.repository_queue_id
LIMIT ?''',
(
authority['source'], member['query'], member['query_ordinal'],
member['query'], authority['source'],
authority['collection_generation'], authority['provenance_policy_sha256'],
authority['ordered_queries_sha256'], authority['query_count'],
authority['source'], experiment['id'], member['query_ordinal'],
member['id'], 3000,
),
).fetchall()
if lock and candidates:
ids = sorted({int(row['repository_queue_id']) for row in candidates})
placeholders = ','.join('?' for _ in ids)
self.conn.execute(
f'''SELECT id FROM target_queue WHERE id IN ({placeholders})
ORDER BY id FOR UPDATE''',
tuple(ids),
).fetchall()
inspected = []
for candidate in candidates:
conflict = self._docker_depth_repository_candidate_conflict(
experiment, int(candidate['repository_queue_id']), lock=lock,
)
inspected.append({
'repository_queue_id': int(candidate['repository_queue_id']),
'best_search_rank': int(candidate['best_search_rank']),
'eligibility_page_id': int(candidate['eligibility_page_id']),
'target_identity_sha256': hashlib.sha256(
str(candidate['normalized_target']).encode('utf-8')
).hexdigest(),
'conflict_reason': str(conflict or ''),
})
if not conflict:
break
return inspected
def _finalize_docker_depth_query_breadth_locked(
self, experiment, member, now,
):
query_row = self.conn.execute(
'''SELECT selected_repository_count
FROM docker_depth_experiment_queries
WHERE experiment_id = ? AND query_ordinal = ? FOR UPDATE''',
(experiment['id'], member['query_ordinal']),
).fetchone()
if not query_row:
raise ScanEventConflictError('Docker depth query authority is unavailable')
query_members = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_repositories
WHERE experiment_id = ? AND query_ordinal = ?
ORDER BY repository_rank, repository_queue_id, id
FOR UPDATE''',
(experiment['id'], member['query_ordinal']),
).fetchall()
if len(query_members) != int(query_row['selected_repository_count']):
raise ScanEventConflictError('Docker depth query cohort membership drifted')
if not all(
str(row['work_state']) in ('resolved', 'skipped')
for row in query_members
):
return False
for row in query_members:
reason, _evidence_sha256 = (
self._docker_depth_terminal_repository_evidence_locked(
experiment, row,
)
)
if reason:
raise ScanEventConflictError(
'Docker depth terminal repository evidence drifted'
)
from docker_depth_experiment import DOCKER_RANK1_BREADTH_SELECTOR_VERSION
if str(experiment['selector_version']) == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET is_deep_probe = 0, updated_at = ?
WHERE experiment_id = ? AND query_ordinal = ?
AND is_deep_probe <> 0''',
(now, experiment['id'], member['query_ordinal']),
)
return True
if self.conn.execute(
'''SELECT 1 FROM docker_depth_experiment_selections
WHERE experiment_id = ? AND query_ordinal = ? AND image_rank > 1
LIMIT 1''',
(experiment['id'], member['query_ordinal']),
).fetchone():
return True
eligible = [
row for row in query_members
if str(row['work_state']) == 'resolved'
and int(row['selected_image_count']) >= 1
]
deep_id = None
if eligible:
deep = min(eligible, key=lambda row: (
-int(row['candidate_distinct_graph_count']),
int(row['repository_rank']), int(row['repository_queue_id']),
))
deep_id = int(deep['id'])
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET is_deep_probe = CASE WHEN id = ? THEN 1 ELSE 0 END,
work_state = CASE
WHEN id = ? AND candidate_distinct_graph_count > selected_image_count
AND selected_image_count >= 1
THEN 'pending' ELSE work_state END,
resolved_at = CASE
WHEN id = ? AND candidate_distinct_graph_count > selected_image_count
AND selected_image_count >= 1
THEN NULL ELSE resolved_at END,
updated_at = ?
WHERE experiment_id = ? AND query_ordinal = ?''',
(
deep_id, deep_id, deep_id, now,
experiment['id'], member['query_ordinal'],
),
)
return True
def _replace_docker_depth_repository_locked(
self, experiment, member, authority, reason, now, candidate_count,
):
current_queue_id = int(
member['replacement_repository_queue_id']
or member['repository_queue_id']
)
self._record_docker_depth_candidate_skip_locked(
experiment, member, current_queue_id, 'repository',
int(member['replacement_count']) + 1,
{'repository_queue_id': current_queue_id}, reason, now,
)
candidates = self._docker_depth_repository_replacement_candidates(
experiment, member, authority, lock=True,
)
replacement = None
for candidate in candidates:
candidate_queue_id = candidate['repository_queue_id']
conflict = candidate['conflict_reason']
if conflict:
self._record_docker_depth_candidate_skip_locked(
experiment, member, candidate_queue_id, 'repository',
int(candidate['best_search_rank']),
{'repository_queue_id': candidate_queue_id}, conflict, now,
)
continue
replacement = candidate
break
if not replacement:
from docker_depth_experiment import DOCKER_DEPTH_REPOSITORY_SKIP_REASON
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'skipped', resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = NULL,
candidate_distinct_graph_count = CASE
WHEN candidate_distinct_graph_count >= ?
THEN candidate_distinct_graph_count ELSE ? END,
selected_image_count = 0, last_error_code = ?,
resolved_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ?
AND work_state = 'resolving'
AND resolver_generation = ? AND resolver_owner = ?
AND resolver_token = ? AND resolver_expires_at > ?''',
(
candidate_count, candidate_count,
DOCKER_DEPTH_REPOSITORY_SKIP_REASON, now, now,
member['id'], experiment['id'], member['resolver_generation'],
member['resolver_owner'], member['resolver_token'], now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth repository skip lost its lease fence'
)
self._finalize_docker_depth_query_breadth_locked(
experiment, member, now,
)
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
experiment, selection_reason = (
self._freeze_docker_depth_runtime_selection_locked(
experiment, authority, now,
)
)
if selection_reason:
return self._hold_docker_depth_experiment_locked(
experiment, selection_reason, now,
)
drift_reason = self._docker_depth_persisted_drift_reason_locked(
experiment, authority, now,
)
if drift_reason:
return self._hold_docker_depth_experiment_locked(
experiment, drift_reason, now,
)
experiment = self._advance_docker_depth_experiment_state_locked(
experiment, now,
)
return {
'status': 'skipped', 'committed': True,
'reason': DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
'experiment_state': str(experiment['state']),
}
previous_hash = str(member['replacement_evidence_sha256'] or '')
replacement_document = {
'schema': 1,
'type': 'docker-depth-repository-replacement-v1',
'experiment_id': int(experiment['id']),
'experiment_repository_id': int(member['id']),
'replacement_number': int(member['replacement_count']) + 1,
'from_repository_queue_id': current_queue_id,
'to_repository_queue_id': int(replacement['repository_queue_id']),
'eligibility_page_id': int(replacement['eligibility_page_id']),
'best_search_rank': int(replacement['best_search_rank']),
'previous_evidence_sha256': previous_hash,
'reason_code': str(reason),
}
replacement_sha256 = hashlib.sha256(json.dumps(
replacement_document, ensure_ascii=True, allow_nan=False,
sort_keys=True, separators=(',', ':'),
).encode('utf-8')).hexdigest()
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET replacement_repository_queue_id = ?,
replacement_eligibility_page_id = ?,
replacement_count = replacement_count + 1,
replacement_evidence_sha256 = ?, work_state = 'pending',
resolver_owner = NULL, resolver_token = NULL,
resolver_expires_at = NULL, resolver_due_at = NULL,
resolver_attempts = 0, selected_image_count = 0,
candidate_distinct_graph_count = 0,
last_error_code = 'repository_candidate_replaced',
resolved_at = NULL, updated_at = ? WHERE id = ?''',
(
replacement['repository_queue_id'], replacement['eligibility_page_id'],
replacement_sha256, now, member['id'],
),
)
return {
'status': 'replaced', 'committed': True,
'reason': 'repository_candidate_replaced',
}
@staticmethod
def _rerank_docker_depth_selection_record(record, image_rank):
from docker_depth_experiment import canonical_docker_depth_selection_evidence_hash
evidence_keys = (
'schema', 'type', 'selector_version', 'selector_sha256',
'candidate_distinct_graph_count', 'selection_reason', 'target',
'repository', 'manifest_digest', 'manifest_media_type',
'manifest_size_bytes', 'config_digest', 'graph_sha256', 'layers',
)
evidence = {key: record[key] for key in evidence_keys}
original_rank = int(record['image_rank'])
evidence['image_rank'] = int(image_rank)
if original_rank != image_rank:
evidence['selection_reason'] = 'replacement_after_exclusion'
return {
**record,
**evidence,
'selection_evidence_sha256': canonical_docker_depth_selection_evidence_hash(
evidence
),
}
def _docker_depth_target_queue_locked(
self, experiment_id, source, query, record, now,
):
row = self.conn.execute(
'''SELECT * FROM target_queue
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
(source, record['normalized_target']),
).fetchone()
inserted = False
if not row:
row = self.conn.execute(
'''INSERT INTO target_queue(
source, platform, query, target, normalized_target,
status, created_at, updated_at
) VALUES (?, 'docker', ?, ?, ?, 'pending', ?, ?)
ON CONFLICT(source, normalized_target) DO NOTHING
RETURNING *''',
(
source, query, record['target'], record['normalized_target'],
now, now,
),
).fetchone()
if not row:
row = self.conn.execute(
'''SELECT * FROM target_queue
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
(source, record['normalized_target']),
).fetchone()
else:
inserted = True
experiment_target = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_targets
WHERE experiment_id = ? AND target_queue_id = ? FOR UPDATE''',
(experiment_id, row['id']),
).fetchone() if row else None
audited_reactivation = bool(
row and not inserted and self._docker_depth_exact_reactivation_locked(
experiment_id, row,
)
)
if (
not row
or row['source'] != source
or row['platform'] != 'docker'
or row['status'] not in ('pending', 'deferred')
or normalize_target(row['target'], 'docker') != record['normalized_target']
or row['target_scan_id'] is not None
or row['lease_owner'] is not None
or row['lease_token'] is not None
or row['claim_batch'] is not None
or row['leased_at'] is not None
or row['lease_expires_at'] is not None
or row['current_result_reservation_id'] is not None
or row['claim_event_id'] is not None
or row['completed_at'] is not None
or (not inserted and not experiment_target and not audited_reactivation)
):
raise ScanEventConflictError(
'Docker depth immutable target has prior or conflicting queue state'
)
historical = self.conn.execute(
'''SELECT
EXISTS(SELECT 1 FROM target_scans WHERE queue_id = ?) AS scanned,
EXISTS(SELECT 1 FROM result_reservations WHERE queue_id = ?) AS reserved,
EXISTS(SELECT 1 FROM target_queue_policy_events WHERE queue_id = ?) AS policy,
EXISTS(
SELECT 1 FROM result_reservations reservation
JOIN pipeline_quarantine quarantine
ON quarantine.reservation_id = reservation.id
WHERE reservation.queue_id = ?
) AS quarantined''',
(row['id'], row['id'], row['id'], row['id']),
).fetchone()
if (
bool(historical['scanned'])
or bool(historical['reserved'])
or bool(historical['quarantined'])
or (bool(historical['policy']) and not audited_reactivation)
):
raise ScanEventConflictError(
'Docker depth immutable target has historical or independently cold state'
)
return dict(row)
def _persist_docker_depth_manifest_locked(self, queue_id, source, record, now):
expected = {
'source': source,
'repository': record['repository'],
'manifest_digest': record['manifest_digest'],
'manifest_media_type': record['manifest_media_type'],
'config_digest': record['config_digest'],
'graph_sha256': record['graph_sha256'],
'manifest_size_bytes': record['manifest_size_bytes'],
'layer_count': len(record['layers']),
}
manifest = self.conn.execute(
'''SELECT * FROM docker_image_manifests
WHERE target_queue_id = ? FOR UPDATE''',
(queue_id,),
).fetchone()
if manifest:
if any(
(int(manifest[key]) if isinstance(value, int) else str(manifest[key])) != value
for key, value in expected.items()
):
raise ScanEventConflictError('Docker depth immutable manifest metadata conflicts')
else:
manifest = self.conn.execute(
'''INSERT INTO docker_image_manifests(
target_queue_id, source, repository, manifest_digest,
manifest_media_type, config_digest, graph_sha256,
manifest_size_bytes, layer_count, resolved_at, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *''',
(
queue_id, source, record['repository'], record['manifest_digest'],
record['manifest_media_type'], record['config_digest'],
record['graph_sha256'], record['manifest_size_bytes'],
len(record['layers']), now, now,
),
).fetchone()
for layer in record['layers']:
self.conn.execute(
'''INSERT INTO docker_manifest_layers(
manifest_id, position_from_base, position_from_top,
layer_digest, media_type, layer_size_bytes,
descriptor_sha256, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''',
(
manifest['id'], layer['position_from_base'],
layer['position_from_top'], layer['digest'],
layer['media_type'], layer['size_bytes'],
layer['descriptor_sha256'], now,
),
)
stored_layers = self.conn.execute(
'''SELECT position_from_base, position_from_top, layer_digest,
media_type, layer_size_bytes, descriptor_sha256
FROM docker_manifest_layers WHERE manifest_id = ?
ORDER BY position_from_base''',
(manifest['id'],),
).fetchall()
expected_layers = [(
layer['position_from_base'], layer['position_from_top'], layer['digest'],
layer['media_type'], layer['size_bytes'], layer['descriptor_sha256'],
) for layer in record['layers']]
if [(
int(layer['position_from_base']), int(layer['position_from_top']),
str(layer['layer_digest']), str(layer['media_type']),
int(layer['layer_size_bytes']), str(layer['descriptor_sha256']),
) for layer in stored_layers] != expected_layers:
raise ScanEventConflictError('Docker depth immutable manifest layers conflict')
return int(manifest['id'])
@staticmethod
def _docker_depth_dispatch_position(query_ordinal, repository_rank, image_rank, query_count):
if image_rank == 1:
return 1, (repository_rank - 1) * query_count + query_ordinal + 1
if image_rank <= 3:
return 2, (image_rank - 2) * query_count + query_ordinal + 1
return 3, (image_rank - 4) * query_count + query_ordinal + 1
def _defer_or_hold_docker_depth_resolution_conflict(
self, source, repository_id, resolver_generation, resolver_owner,
resolver_token, authority, error, *, final_cutover,
):
from docker_depth_experiment import (
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS,
)
now = utc_now_iso()
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
(authority['experiment_key'], source),
).fetchone()
member = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_repositories
WHERE id = ? AND experiment_id = ? FOR UPDATE''',
(repository_id, experiment['id'] if experiment else 0),
).fetchone()
if not (
experiment
and str(experiment['state']) == 'resolving'
and member
and member['work_state'] == 'resolving'
and int(member['resolver_generation']) == int(resolver_generation)
and member['resolver_owner'] == resolver_owner
and member['resolver_token'] == resolver_token
and str(member['resolver_expires_at'] or '') > now
):
self.conn.rollback()
return {'status': 'stale', 'committed': False}
checked, authority_reason = self._locked_docker_depth_experiment_authority(
authority, final_cutover=final_cutover, now=now,
)
if not checked:
self.conn.commit()
return {
'status': 'held', 'committed': True,
'reason': authority_reason,
}
attempts = int(member['resolver_attempts'])
capacity_conflict = 'capacity' in str(error).lower()
if capacity_conflict or attempts >= DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS:
reason = (
'resolver_capacity_conflict' if capacity_conflict
else 'resolver_evidence_attempt_limit'
)
self._hold_docker_depth_experiment_locked(experiment, reason, now)
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'held', resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = NULL, last_error_code = ?, updated_at = ?
WHERE id = ?''',
(reason, now, repository_id),
)
self.conn.commit()
return {'status': 'held', 'committed': True, 'reason': reason}
delay = min(
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS
* (2 ** min(8, max(0, attempts - 1))),
)
due = datetime.fromtimestamp(
time.time() + delay, timezone.utc,
).isoformat(timespec='seconds')
self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'pending', resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = ?, last_error_code = 'resolver_evidence_conflict',
updated_at = ? WHERE id = ?''',
(due, now, repository_id),
)
self.conn.commit()
return {
'status': 'conflict_retry', 'committed': True,
'retry_at': due, 'reason': 'resolver_evidence_conflict',
}
def finish_docker_depth_experiment_resolution(
self, source, repository_id, resolver_generation, resolver_token,
outcome=None, error='', complete=None, *, resolver_owner=None,
authority=None, retry_at=None, claim_attempt_consumed=True,
final_cutover=False,
):
if (
not self.conn
or not self.conn.is_postgres
or source != 'dockerhub'
or repository_id is None
or not resolver_token
or not resolver_owner
):
return {'status': 'unavailable', 'committed': False}
if isinstance(authority, dict) and authority.get('enabled') is False:
held = self._hold_disabled_docker_depth_experiment(authority)
return {
'status': 'held' if held else 'unavailable',
'committed': held, 'reason': 'experiment_disabled',
}
if not isinstance(authority, dict) or authority.get('enabled') is not True:
return {'status': 'unavailable', 'committed': False}
try:
normalized = self._normalize_docker_depth_resolver_authority(authority)
except (TypeError, ValueError):
return {
'status': 'invalid', 'committed': False,
'reason': 'authority_payload_invalid',
}
outcome_tags = (
outcome.get('tags', ()) if isinstance(outcome, dict)
else getattr(outcome, 'tags', ())
)
complete = bool(outcome_tags) if complete is None else bool(complete)
if not claim_attempt_consumed and (complete or not retry_at):
return {'status': 'invalid', 'committed': False}
try:
if complete:
self._require_discovery_admission_locked()
now = utc_now_iso()
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
(normalized['experiment_key'], source),
).fetchone()
member = self.conn.execute(
'''SELECT member.*, queue.normalized_target AS repository
FROM docker_depth_experiment_repositories member
JOIN target_queue queue ON queue.id = COALESCE(
member.replacement_repository_queue_id,
member.repository_queue_id
)
WHERE member.id = ? AND member.experiment_id = ?
FOR UPDATE OF member, queue''',
(repository_id, experiment['id'] if experiment else 0),
).fetchone()
if not (
experiment
and str(experiment['state']) == 'resolving'
and member
and member['source'] == source
and member['work_state'] == 'resolving'
and int(member['resolver_generation']) == int(resolver_generation)
and member['resolver_owner'] == resolver_owner
and member['resolver_token'] == resolver_token
and str(member['resolver_expires_at'] or '') > now
):
self.conn.rollback()
return {'status': 'stale', 'committed': False}
experiment, authority_reason = self._locked_docker_depth_experiment_authority(
normalized, final_cutover=final_cutover, now=now,
)
if not experiment:
self.conn.commit()
return {
'status': 'held', 'committed': True,
'reason': authority_reason,
}
stage = 'breadth' if int(member['selected_image_count']) == 0 else 'deep'
if stage == 'deep' and not bool(member['is_deep_probe']):
raise ScanEventConflictError('Docker depth deep resolver authority conflicts')
if not complete:
from docker_depth_experiment import (
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS,
)
if (
claim_attempt_consumed
and int(member['resolver_attempts'])
>= DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS
):
terminal = self._terminalize_docker_depth_attempt_limit_locked(
experiment, member, normalized, now,
)
self.conn.commit()
terminal.pop('experiment', None)
return terminal
now_dt = datetime.now(timezone.utc)
if retry_at:
try:
due_dt = datetime.fromisoformat(str(retry_at).replace('Z', '+00:00'))
if due_dt.tzinfo is None:
due_dt = due_dt.replace(tzinfo=timezone.utc)
due_dt = due_dt.astimezone(timezone.utc)
except (TypeError, ValueError):
self.conn.rollback()
return {'status': 'invalid', 'committed': False}
if due_dt > now_dt + timedelta(seconds=3605):
self.conn.rollback()
return {'status': 'invalid', 'committed': False}
due_dt = max(due_dt, now_dt)
else:
exponent = min(8, max(0, int(member['resolver_attempts']) - 1))
due_dt = datetime.fromtimestamp(
time.time() + min(
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS * (2 ** exponent),
),
timezone.utc,
)
due = due_dt.isoformat(timespec='seconds')
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'pending', resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = ?, resolver_attempts = CASE
WHEN ? = 0 AND resolver_attempts > 0
THEN resolver_attempts - 1 ELSE resolver_attempts END,
last_error_code = ?, updated_at = ?
WHERE id = ? AND experiment_id = ?
AND resolver_generation = ? AND resolver_owner = ?
AND resolver_token = ? AND resolver_expires_at > ?''',
(
due, 1 if claim_attempt_consumed else 0,
first_line(error or 'docker_depth_resolution_deferred', 128),
now, repository_id, experiment['id'], resolver_generation,
resolver_owner, resolver_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
self.conn.rollback()
return {'status': 'stale', 'committed': False}
self.conn.commit()
return {'status': 'deferred', 'committed': True, 'retry_at': due}
candidate_count, candidate_records = self._normalize_docker_depth_resolution_outcome(
outcome, str(member['repository']), normalized, stage,
)
existing_selections = self.conn.execute(
'''SELECT selection.image_rank, selection.graph_sha256,
queue.normalized_target
FROM docker_depth_experiment_selections selection
JOIN docker_depth_experiment_targets target
ON target.id = selection.experiment_target_id
JOIN target_queue queue ON queue.id = target.target_queue_id
WHERE selection.experiment_repository_id = ?
ORDER BY selection.image_rank''',
(repository_id,),
).fetchall()
existing_graphs = {
str(selection['graph_sha256']) for selection in existing_selections
}
existing_targets = {
str(selection['normalized_target']) for selection in existing_selections
}
eligible_records = []
resolution_repository_queue_id = int(
member['replacement_repository_queue_id']
or member['repository_queue_id']
)
for candidate in candidate_records:
if (
candidate['graph_sha256'] in existing_graphs
or candidate['normalized_target'] in existing_targets
):
continue
conflict, _candidate_queue_id = (
self._docker_depth_immutable_candidate_conflict_locked(
experiment['id'], source, candidate,
)
)
if conflict:
self._record_docker_depth_candidate_skip_locked(
experiment, member, resolution_repository_queue_id,
'image', candidate['image_rank'], {
'normalized_target': candidate['normalized_target'],
'graph_sha256': candidate['graph_sha256'],
}, conflict, now,
)
continue
eligible_records.append(candidate)
remaining = normalized['images_per_repository'] - len(existing_selections)
if stage == 'breadth':
remaining = 1
records = [
self._rerank_docker_depth_selection_record(
record, len(existing_selections) + index,
)
for index, record in enumerate(eligible_records[:remaining], 1)
]
if stage == 'breadth' and not records:
replacement = self._replace_docker_depth_repository_locked(
experiment, member, normalized,
'no_eligible_physical_target', now, candidate_count,
)
self.conn.commit()
return replacement
actual_selection_count = int(self.conn.execute(
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_selections
WHERE experiment_id = ?''',
(experiment['id'],),
).fetchone()['count'])
actual_target_count = int(self.conn.execute(
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_targets
WHERE experiment_id = ?''',
(experiment['id'],),
).fetchone()['count'])
if (
actual_selection_count != int(experiment['selection_count'])
or actual_target_count != int(experiment['target_count'])
):
raise ScanEventConflictError('Docker depth counter capacity drifted')
inserted_selections = 0
for record in records:
queue = self._docker_depth_target_queue_locked(
experiment['id'], source, member['query'], record, now,
)
manifest_id = self._persist_docker_depth_manifest_locked(
queue['id'], source, record, now,
)
dispatch_wave, dispatch_order = self._docker_depth_dispatch_position(
int(member['query_ordinal']), int(member['repository_rank']),
record['image_rank'], normalized['query_count'],
)
capacity = self.reserve_docker_depth_experiment_target_capacity_locked(
experiment['id'], queue['id'], manifest_id,
dispatch_wave, dispatch_order, now,
)
selection = self.conn.execute(
'''SELECT * FROM docker_depth_experiment_selections
WHERE experiment_repository_id = ? AND image_rank = ?
FOR UPDATE''',
(repository_id, record['image_rank']),
).fetchone()
expected_selection = {
'experiment_id': int(experiment['id']),
'query_ordinal': int(member['query_ordinal']),
'experiment_target_id': int(capacity['target']['id']),
'selection_reason': record['selection_reason'],
'selection_evidence_sha256': record['selection_evidence_sha256'],
'graph_sha256': record['graph_sha256'],
}
if selection:
if any(
(int(selection[key]) if isinstance(value, int) else str(selection[key]))
!= value for key, value in expected_selection.items()
):
raise ScanEventConflictError('Docker depth persisted selector evidence conflicts')
else:
if actual_selection_count + inserted_selections + 1 > normalized[
'theoretical_max_targets'
]:
raise ScanEventConflictError(
'Docker depth theoretical selection capacity is exhausted'
)
self.conn.execute(
'''INSERT INTO docker_depth_experiment_selections(
experiment_id, query_ordinal, experiment_repository_id,
experiment_target_id, image_rank, selection_reason,
selection_evidence_sha256, graph_sha256,
selected_at, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
experiment['id'], member['query_ordinal'], repository_id,
capacity['target']['id'], record['image_rank'],
record['selection_reason'], record['selection_evidence_sha256'],
record['graph_sha256'], now, now,
),
)
inserted_selections += 1
if inserted_selections:
cursor = self.conn.execute(
'''UPDATE docker_depth_experiments
SET selection_count = selection_count + ?, updated_at = ?
WHERE id = ? AND selection_count = ?
AND selection_count + ? <= ?''',
(
inserted_selections, now, experiment['id'],
actual_selection_count, inserted_selections,
normalized['theoretical_max_targets'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'Docker depth selection capacity lost its serialized fence'
)
selected_image_count = int(self.conn.execute(
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_selections
WHERE experiment_repository_id = ?''',
(repository_id,),
).fetchone()['count'])
cursor = self.conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET work_state = 'resolved', resolver_owner = NULL,
resolver_token = NULL, resolver_expires_at = NULL,
resolver_due_at = NULL,
candidate_distinct_graph_count = CASE
WHEN candidate_distinct_graph_count >= ?
THEN candidate_distinct_graph_count ELSE ? END,
selected_image_count = ?, last_error_code = NULL,
resolved_at = ?, updated_at = ?
WHERE id = ? AND experiment_id = ?
AND resolver_generation = ? AND resolver_owner = ?
AND resolver_token = ? AND resolver_expires_at > ?''',
(
candidate_count, candidate_count, selected_image_count, now, now,
repository_id, experiment['id'], resolver_generation,
resolver_owner, resolver_token, now,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('Docker depth resolver completion lost its lease fence')
if stage == 'breadth':
self._finalize_docker_depth_query_breadth_locked(
experiment, member, now,
)
experiment = self.conn.execute(
'SELECT * FROM docker_depth_experiments WHERE id = ?',
(experiment['id'],),
).fetchone()
experiment, selection_reason = (
self._freeze_docker_depth_runtime_selection_locked(
experiment, normalized, now,
)
)
if selection_reason:
held = self._hold_docker_depth_experiment_locked(
experiment, selection_reason, now,
)
self.conn.commit()
return held
drift_reason = self._docker_depth_persisted_drift_reason_locked(
experiment, normalized, now,
)
if drift_reason:
held = self._hold_docker_depth_experiment_locked(
experiment, drift_reason, now,
)
self.conn.commit()
return held
experiment = self._advance_docker_depth_experiment_state_locked(
experiment, now,
)
self.conn.commit()
return {
'status': 'resolved', 'committed': True,
'stage': stage, 'candidate_distinct_graph_count': candidate_count,
'selected_image_count': selected_image_count,
'inserted_selection_count': inserted_selections,
'experiment_state': str(experiment['state']),
}
except DiscoveryPausedError:
self.conn.rollback()
raise
except Exception as exc:
self.last_error = str(exc)
try:
self.conn.rollback()
result = self._defer_or_hold_docker_depth_resolution_conflict(
source, repository_id, resolver_generation, resolver_owner,
resolver_token, normalized, str(exc),
final_cutover=final_cutover,
)
self.last_error = str(exc)
return result
except Exception as recovery_exc:
self.last_error = f'{exc}; conflict recovery failed: {recovery_exc}'
try:
self.conn.rollback()
except Exception:
pass
return {'status': 'error', 'committed': False}
def _locked_docker_depth_page_authority(
self, source, query, observation, authority, *, final_cutover, now,
):
rows = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE source = ? AND state <> 'released'
ORDER BY id FOR UPDATE''',
(source,),
).fetchall()
if not rows:
return None, None, None
if len(rows) != 1:
for row in rows:
self._hold_docker_depth_experiment_locked(
row, 'dynamic_hold_authority_ambiguous', now,
)
return None, None, 'dynamic_hold_authority_ambiguous'
row = rows[0]
if not isinstance(authority, dict) or authority.get('enabled') is not True:
reason = 'authority_payload_invalid'
if isinstance(authority, dict) and authority.get('enabled') is False:
reason = 'experiment_disabled'
self._hold_docker_depth_experiment_locked(row, reason, now)
return None, None, reason
try:
normalized = self._normalize_docker_depth_resolver_authority(authority)
except (TypeError, ValueError):
return None, None, 'authority_payload_invalid'
if (
str(row['experiment_key']) != normalized['experiment_key']
or str(row['source']) != normalized['source']
):
self._hold_docker_depth_experiment_locked(
row, 'experiment_identity_drift', now,
)
return None, None, 'experiment_identity_drift'
experiment, reason = self._locked_docker_depth_experiment_authority(
normalized, final_cutover=final_cutover, now=now,
)
if not experiment:
return None, None, reason
query_row = self.conn.execute(
'''SELECT query_ordinal FROM docker_depth_experiment_queries
WHERE experiment_id = ? AND source = ? AND query = ?''',
(experiment['id'], source, query),
).fetchone()
if str(experiment['state']) != 'collecting' and (
observation is None
or not query_row
or observation['query_ordinal'] != int(query_row['query_ordinal'])
or observation['query_count'] != normalized['query_count']
or observation['collection_generation']
!= normalized['collection_generation']
or observation['ordered_query_hash']
!= normalized['ordered_queries_sha256']
or observation['policy_sha256']
!= normalized['provenance_policy_sha256']
):
self._hold_docker_depth_experiment_locked(
experiment, 'dynamic_hold_observation_drift', now,
)
return None, None, 'dynamic_hold_observation_drift'
return experiment, normalized, None
def _hold_new_docker_depth_repositories_locked(
self, source, query, queue_ids, observation, now, *, experiment=None,
authority=None,
):
if not queue_ids or not self.conn.is_postgres or not experiment:
return 0
from docker_depth_experiment import (
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
DOCKER_DEPTH_HOLD_ACTIVE_STATES,
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
)
if (
str(experiment['state']) not in DOCKER_DEPTH_HOLD_ACTIVE_STATES
or experiment['hold_manifest_sha256'] is None
):
return 0
query_row = self.conn.execute(
'''SELECT query_ordinal FROM docker_depth_experiment_queries
WHERE experiment_id = ? AND source = ? AND query = ?''',
(experiment['id'], source, query),
).fetchone()
if (
observation is None
or authority is None
or not query_row
or observation['query_ordinal'] != int(query_row['query_ordinal'])
or observation['query_count'] != int(experiment['query_count'])
or observation['collection_generation']
!= experiment['collection_generation']
or observation['ordered_query_hash'] != experiment['ordered_queries_sha256']
or observation['policy_sha256'] != experiment['provenance_policy_sha256']
or str(experiment['config_sha256']) != authority['config_sha256']
or str(experiment['selector_sha256']) != authority['selector_sha256']
):
raise ScanEventConflictError('Docker depth dynamic hold observation drifted')
queue_placeholders = ','.join('?' for _ in queue_ids)
rows = self.conn.execute(
f'''SELECT queue.* FROM target_queue queue
WHERE queue.id IN ({queue_placeholders})
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_repositories member
WHERE member.experiment_id = ?
AND member.repository_queue_id = queue.id
)
ORDER BY queue.id FOR UPDATE''',
(*sorted(queue_ids), experiment['id']),
).fetchall()
entries = []
for row in rows:
if (
row['source'] != source
or row['platform'] != 'docker'
or row['query'] != query
or row['status'] != 'deferred'
or row['target_scan_id'] is not None
or '@' in str(row['target'])
or '@' in str(row['normalized_target'])
):
raise ScanEventConflictError('Docker depth dynamic hold row identity conflicts')
entries.append({
'queue_id': int(row['id']),
'source': str(row['source']),
'platform': str(row['platform']),
'query': str(row['query']),
'prior_status': str(row['status']),
'prior_updated_at': str(row['updated_at']),
})
entries = self._normalize_target_queue_policy_entries(
entries, 'cold', max(1, len(entries)),
) if entries else []
if entries:
active_holds = int(self.conn.execute(
'''SELECT COUNT(*) AS count
FROM target_queue_policy_events event
LEFT JOIN target_queue_policy_events reverse_event
ON reverse_event.reverses_event_id = event.id
WHERE event.experiment_id = ? AND event.action = 'cold'
AND reverse_event.id IS NULL''',
(experiment['id'],),
).fetchone()['count'])
if active_holds + len(entries) > DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS:
raise ScanEventConflictError(
'Docker depth dynamic hold capacity is exhausted'
)
result = self._cold_target_queue_rows_locked(
entries,
reason_code=DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
config_sha256=str(experiment['config_sha256']),
policy_sha256=str(experiment['provenance_policy_sha256']),
manifest_sha256=str(experiment['hold_manifest_sha256']),
experiment_id=int(experiment['id']),
now=now,
)
return int(result['transitioned']) + int(result['duplicates'])
@staticmethod
def _target_queue_policy_audit_sha256(
action, manifest_sha256, entry, experiment_id=None,
):
payload = {
'action': str(action),
'manifest_sha256': str(manifest_sha256),
'entry': dict(entry),
}
if experiment_id is not None:
payload['experiment_id'] = int(experiment_id)
encoded = json.dumps(
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
return hashlib.sha256(encoded).hexdigest()
@staticmethod
def _normalize_target_queue_policy_entries(entries, action, max_rows):
from docker_depth_experiment import DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS
maximum = min(
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
max(1, int(max_rows)),
)
values = [dict(entry) for entry in entries or ()]
if not values or len(values) > maximum:
raise ValueError('target queue policy entry count is outside its bound')
required = {
'queue_id', 'source', 'platform', 'query', 'prior_updated_at',
}
if action == 'cold':
required.add('prior_status')
else:
required.update(('cold_event_id', 'restore_status'))
normalized = []
seen = set()
for value in values:
if set(value) != required:
raise ValueError('target queue policy entry shape is invalid')
queue_id = int(value['queue_id'])
if queue_id <= 0 or queue_id in seen:
raise ValueError('target queue policy queue identity is invalid or duplicated')
seen.add(queue_id)
item = {
'queue_id': queue_id,
'source': str(value['source'] or '').strip(),
'platform': str(value['platform'] or '').strip(),
'query': str(value['query'] or ''),
'prior_updated_at': str(value['prior_updated_at'] or ''),
}
if not all((item['source'], item['platform'], item['query'], item['prior_updated_at'])):
raise ValueError('target queue policy entry contains an empty identity field')
if action == 'cold':
item['prior_status'] = str(value['prior_status'] or '')
if item['prior_status'] not in ('pending', 'deferred'):
raise ValueError('cold transition requires a pending or deferred prior status')
else:
item['cold_event_id'] = int(value['cold_event_id'])
item['restore_status'] = str(value['restore_status'] or '')
if item['cold_event_id'] <= 0 or item['restore_status'] not in ('pending', 'deferred'):
raise ValueError('reactivation transition identity is invalid')
normalized.append(item)
return sorted(normalized, key=lambda item: item['queue_id'])
@staticmethod
def _target_queue_policy_chunks(values, size=500):
values = list(values)
for start in range(0, len(values), size):
yield values[start:start + size]
def _locked_target_queue_policy_rows(self, entries):
rows_by_id = {}
for chunk in self._target_queue_policy_chunks(entries):
placeholders = ','.join('?' for _ in chunk)
rows = self.conn.execute(
f'''SELECT * FROM target_queue
WHERE id IN ({placeholders}) ORDER BY id FOR UPDATE''',
tuple(entry['queue_id'] for entry in chunk),
).fetchall()
for row in rows:
queue_id = int(row['id'])
if queue_id in rows_by_id:
raise ScanEventConflictError(
'target queue policy selection contains duplicate rows'
)
rows_by_id[queue_id] = row
if len(rows_by_id) != len(entries):
raise ScanEventConflictError('target queue policy selection changed')
return [rows_by_id[entry['queue_id']] for entry in entries]
@staticmethod
def _normalize_target_queue_policy_experiment_id(experiment_id):
if experiment_id is None:
return None
if isinstance(experiment_id, bool):
raise ValueError('target queue policy experiment identity is invalid')
try:
experiment_id = int(experiment_id)
except (TypeError, ValueError, OverflowError):
raise ValueError('target queue policy experiment identity is invalid') from None
if experiment_id <= 0:
raise ValueError('target queue policy experiment identity is invalid')
return experiment_id
def _require_target_queue_policy_unfenced(self, row, *, lock_rows=True):
fenced_fields = (
'lease_owner', 'lease_token', 'claim_batch', 'leased_at', 'lease_expires_at',
'current_result_reservation_id', 'claim_event_id', 'resolver_token',
)
if any(row[field] is not None for field in fenced_fields):
raise ScanEventConflictError('target queue policy row has an active claim fence')
if str(row['resolver_state'] or '') == 'resolving':
raise ScanEventConflictError('target queue policy row has an active resolver fence')
lock_suffix = ' FOR UPDATE' if lock_rows else ''
reservation = self.conn.execute(
'''SELECT id FROM result_reservations
WHERE queue_id = ? AND state IN ('scanning','ready','ingesting','db_committed')
LIMIT 1''' + lock_suffix,
(int(row['id']),),
).fetchone()
if reservation:
raise ScanEventConflictError('target queue policy row has an active result reservation')
blob_lock_suffix = ' FOR UPDATE OF blob' if lock_rows else ''
docker_blob = self.conn.execute(
'''SELECT 1
FROM docker_image_blob_coverage coverage
JOIN docker_content_blobs blob
ON blob.digest = coverage.blob_digest
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
WHERE coverage.queue_id = ? AND blob.state IN ('leased','submitted')
LIMIT 1''' + blob_lock_suffix,
(int(row['id']),),
).fetchone()
if docker_blob:
raise ScanEventConflictError('target queue policy row has active Docker content work')
def _target_queue_policy_duplicate_count(
self, action, manifest_sha256, entries, *, reason_code,
config_sha256, policy_sha256, experiment_id=None,
):
entries_by_audit = {}
for entry in entries:
audit_sha256 = self._target_queue_policy_audit_sha256(
action, manifest_sha256, entry, experiment_id,
)
entries_by_audit[audit_sha256] = entry
events_by_audit = {}
for chunk in self._target_queue_policy_chunks(
sorted(entries_by_audit),
):
placeholders = ','.join('?' for _ in chunk)
events = self.conn.execute(
f'''SELECT * FROM target_queue_policy_events
WHERE review_audit_sha256 IN ({placeholders})
ORDER BY id FOR UPDATE''',
tuple(chunk),
).fetchall()
for event in events:
audit_sha256 = str(event['review_audit_sha256'])
if audit_sha256 in events_by_audit:
raise ScanEventConflictError(
'target queue policy audit identity is duplicated'
)
events_by_audit[audit_sha256] = event
if not events_by_audit:
return 0
if len(events_by_audit) != len(entries):
raise ScanEventConflictError(
'target queue policy manifest was only partially applied'
)
reversed_event_ids = set()
if action == 'cold':
event_ids = sorted(int(event['id']) for event in events_by_audit.values())
for chunk in self._target_queue_policy_chunks(event_ids):
placeholders = ','.join('?' for _ in chunk)
reversed_rows = self.conn.execute(
f'''SELECT reverses_event_id FROM target_queue_policy_events
WHERE reverses_event_id IN ({placeholders})
ORDER BY reverses_event_id FOR UPDATE''',
tuple(chunk),
).fetchall()
reversed_event_ids.update(
int(row['reverses_event_id']) for row in reversed_rows
)
queues = {
int(row['id']): row
for row in self._locked_target_queue_policy_rows(entries)
}
for audit_sha256, entry in entries_by_audit.items():
event = events_by_audit[audit_sha256]
expected_next = 'cold' if action == 'cold' else entry['restore_status']
expected_prior = entry['prior_status'] if action == 'cold' else 'cold'
expected_reverse = None if action == 'cold' else entry['cold_event_id']
if (
event['action'] != action
or int(event['queue_id']) != entry['queue_id']
or event['prior_status'] != expected_prior
or event['next_status'] != expected_next
or event['source'] != entry['source']
or event['platform'] != entry['platform']
or event['query'] != entry['query']
or event['reason_code'] != reason_code
or event['config_sha256'] != config_sha256
or event['policy_sha256'] != policy_sha256
or event['manifest_sha256'] != manifest_sha256
or event['prior_updated_at'] != entry['prior_updated_at']
or (
(event['experiment_id'] is None and experiment_id is not None)
or (
event['experiment_id'] is not None
and int(event['experiment_id']) != experiment_id
)
)
or (
(event['reverses_event_id'] is None and expected_reverse is not None)
or (
event['reverses_event_id'] is not None
and int(event['reverses_event_id']) != expected_reverse
)
)
):
raise ScanEventConflictError('target queue policy audit identity conflicts')
queue = queues.get(entry['queue_id'])
if not queue or queue['status'] != expected_next:
raise ScanEventConflictError('target queue policy duplicate state conflicts')
if action == 'cold' and int(event['id']) in reversed_event_ids:
raise ScanEventConflictError('target queue cold event was already reversed')
return len(entries)
def _cold_target_queue_rows_locked(
self, entries, *, reason_code, config_sha256, policy_sha256,
manifest_sha256, experiment_id=None, now=None,
):
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
now = str(now or utc_now_iso())
if not entries:
return {
'transitioned': 0, 'duplicates': 0, 'examined': 0,
'manifest_sha256': manifest_sha256,
}
experiment = None
if experiment_id is not None:
experiment = self.conn.execute(
'''SELECT id FROM docker_depth_experiments
WHERE id = ? FOR UPDATE''',
(experiment_id,),
).fetchone()
if not experiment:
raise ScanEventConflictError(
'target queue policy experiment authority is absent'
)
rows = self._locked_target_queue_policy_rows(entries)
duplicates = self._target_queue_policy_duplicate_count(
'cold', manifest_sha256, entries, reason_code=reason_code,
config_sha256=config_sha256, policy_sha256=policy_sha256,
experiment_id=experiment_id,
)
if duplicates:
return {
'transitioned': 0, 'duplicates': duplicates,
'examined': len(entries), 'manifest_sha256': manifest_sha256,
}
for row, entry in zip(rows, entries):
if (
int(row['id']) != entry['queue_id']
or row['source'] != entry['source']
or row['platform'] != entry['platform']
or row['query'] != entry['query']
or row['status'] != entry['prior_status']
or row['updated_at'] != entry['prior_updated_at']
):
raise ScanEventConflictError('target queue policy evidence does not match')
self._require_target_queue_policy_unfenced(row)
for row, entry in zip(rows, entries):
audit_sha256 = self._target_queue_policy_audit_sha256(
'cold', manifest_sha256, entry, experiment_id,
)
self.conn.execute(
'''INSERT INTO target_queue_policy_events(
queue_id, action, prior_status, next_status, source, platform,
query, reason_code, config_sha256, policy_sha256,
manifest_sha256, review_audit_sha256, reverses_event_id,
experiment_id, prior_updated_at, created_at
) VALUES (?, 'cold', ?, 'cold', ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)''',
(
entry['queue_id'], entry['prior_status'], entry['source'],
entry['platform'], entry['query'], reason_code, config_sha256,
policy_sha256, manifest_sha256, audit_sha256, experiment_id,
entry['prior_updated_at'], now,
),
)
cursor = self.conn.execute(
'''UPDATE target_queue SET status = 'cold', updated_at = ?
WHERE id = ? AND status = ? AND updated_at = ?''',
(now, entry['queue_id'], entry['prior_status'], entry['prior_updated_at']),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('target queue cold transition lost its fence')
return {
'transitioned': len(entries), 'duplicates': 0,
'examined': len(entries), 'manifest_sha256': manifest_sha256,
}
def cold_target_queue_rows(
self, entries, *, reason_code, config_sha256, policy_sha256,
manifest_sha256, max_rows=10000, experiment_id=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('target queue cold transition requires PostgreSQL')
entries = self._normalize_target_queue_policy_entries(entries, 'cold', max_rows)
hashes = (config_sha256, policy_sha256, manifest_sha256)
if not all(re.fullmatch(r'[a-f0-9]{64}', str(value or '')) for value in hashes):
raise ValueError('target queue policy hash identity is invalid')
reason_code = str(reason_code or '').strip()
if not reason_code or len(reason_code) > 128:
raise ValueError('target queue policy reason code is invalid')
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
try:
result = self._cold_target_queue_rows_locked(
entries, reason_code=reason_code, config_sha256=config_sha256,
policy_sha256=policy_sha256, manifest_sha256=manifest_sha256,
experiment_id=experiment_id, now=utc_now_iso(),
)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
def _reactivate_target_queue_rows_locked(
self, entries, *, reason_code, config_sha256, policy_sha256,
manifest_sha256, experiment_id=None, now=None,
):
from docker_depth_experiment import (
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
DOCKER_DEPTH_HOLD_REASON,
)
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
now = str(now or utc_now_iso())
if not entries:
return {
'transitioned': 0, 'duplicates': 0, 'examined': 0,
'manifest_sha256': manifest_sha256,
}
experiment = None
if experiment_id is not None:
experiment = self.conn.execute(
'''SELECT * FROM docker_depth_experiments
WHERE id = ? FOR UPDATE''',
(experiment_id,),
).fetchone()
if not experiment:
raise ScanEventConflictError(
'target queue policy experiment authority is absent'
)
rows = self._locked_target_queue_policy_rows(entries)
duplicates = self._target_queue_policy_duplicate_count(
'reactivate', manifest_sha256, entries, reason_code=reason_code,
config_sha256=config_sha256, policy_sha256=policy_sha256,
experiment_id=experiment_id,
)
if duplicates:
return {
'transitioned': 0, 'duplicates': duplicates,
'examined': len(entries), 'manifest_sha256': manifest_sha256,
}
cold_events = {}
cold_event_ids = sorted(entry['cold_event_id'] for entry in entries)
for chunk in self._target_queue_policy_chunks(cold_event_ids):
placeholders = ','.join('?' for _ in chunk)
event_rows = self.conn.execute(
f'''SELECT * FROM target_queue_policy_events
WHERE id IN ({placeholders}) ORDER BY id FOR UPDATE''',
tuple(chunk),
).fetchall()
cold_events.update((int(event['id']), event) for event in event_rows)
reversed_event_ids = set()
for chunk in self._target_queue_policy_chunks(cold_event_ids):
placeholders = ','.join('?' for _ in chunk)
reversed_rows = self.conn.execute(
f'''SELECT reverses_event_id FROM target_queue_policy_events
WHERE reverses_event_id IN ({placeholders})
ORDER BY reverses_event_id FOR UPDATE''',
tuple(chunk),
).fetchall()
reversed_event_ids.update(
int(event['reverses_event_id']) for event in reversed_rows
)
for row, entry in zip(rows, entries):
cold_event = cold_events.get(entry['cold_event_id'])
cold_experiment_id = (
None if not cold_event or cold_event['experiment_id'] is None
else int(cold_event['experiment_id'])
)
cold_entry = {
'queue_id': entry['queue_id'],
'source': entry['source'],
'platform': entry['platform'],
'query': entry['query'],
'prior_status': entry['restore_status'],
'prior_updated_at': (
str(cold_event['prior_updated_at']) if cold_event else ''
),
}
owned_event_invalid = bool(experiment_id is not None and cold_event) and (
str(cold_event['reason_code']) not in (
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
)
or str(cold_event['config_sha256']) != str(config_sha256)
or str(cold_event['policy_sha256']) != str(policy_sha256)
or str(cold_event['manifest_sha256'])
!= str(experiment['hold_manifest_sha256'])
or str(cold_event['review_audit_sha256'])
!= self._target_queue_policy_audit_sha256(
'cold', cold_event['manifest_sha256'], cold_entry,
experiment_id,
)
)
if (
int(row['id']) != entry['queue_id']
or row['source'] != entry['source']
or row['platform'] != entry['platform']
or row['query'] != entry['query']
or row['status'] != 'cold'
or row['updated_at'] != entry['prior_updated_at']
or not cold_event
or cold_event['action'] != 'cold'
or int(cold_event['queue_id']) != entry['queue_id']
or cold_event['prior_status'] != entry['restore_status']
or cold_event['next_status'] != 'cold'
or cold_experiment_id != experiment_id
or owned_event_invalid
or entry['cold_event_id'] in reversed_event_ids
):
raise ScanEventConflictError('target queue reactivation evidence does not match')
self._require_target_queue_policy_unfenced(row)
for entry in entries:
audit_sha256 = self._target_queue_policy_audit_sha256(
'reactivate', manifest_sha256, entry, experiment_id,
)
self.conn.execute(
'''INSERT INTO target_queue_policy_events(
queue_id, action, prior_status, next_status, source, platform,
query, reason_code, config_sha256, policy_sha256,
manifest_sha256, review_audit_sha256, reverses_event_id,
experiment_id, prior_updated_at, created_at
) VALUES (?, 'reactivate', 'cold', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
entry['queue_id'], entry['restore_status'], entry['source'],
entry['platform'], entry['query'], reason_code, config_sha256,
policy_sha256, manifest_sha256, audit_sha256,
entry['cold_event_id'], experiment_id, entry['prior_updated_at'], now,
),
)
cursor = self.conn.execute(
'''UPDATE target_queue SET status = ?, updated_at = ?
WHERE id = ? AND status = 'cold' AND updated_at = ?''',
(
entry['restore_status'], now, entry['queue_id'],
entry['prior_updated_at'],
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError('target queue reactivation lost its fence')
return {
'transitioned': len(entries), 'duplicates': 0,
'examined': len(entries), 'manifest_sha256': manifest_sha256,
}
def reactivate_cold_target_queue_rows(
self, entries, *, reason_code, config_sha256, policy_sha256,
manifest_sha256, max_rows=10000, experiment_id=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('target queue reactivation requires PostgreSQL')
entries = self._normalize_target_queue_policy_entries(entries, 'reactivate', max_rows)
hashes = (config_sha256, policy_sha256, manifest_sha256)
if not all(re.fullmatch(r'[a-f0-9]{64}', str(value or '')) for value in hashes):
raise ValueError('target queue policy hash identity is invalid')
reason_code = str(reason_code or '').strip()
if not reason_code or len(reason_code) > 128:
raise ValueError('target queue policy reason code is invalid')
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
try:
result = self._reactivate_target_queue_rows_locked(
entries, reason_code=reason_code, config_sha256=config_sha256,
policy_sha256=policy_sha256, manifest_sha256=manifest_sha256,
experiment_id=experiment_id, now=utc_now_iso(),
)
self.conn.commit()
return result
except Exception:
self.conn.rollback()
raise
@staticmethod
def _retirement_chain(previous, rows):
previous = str(previous or '0' * 64)
payload = json.dumps(
[dict(row) for row in rows],
ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
return hashlib.sha256(bytes.fromhex(previous) + hashlib.sha256(payload).digest()).hexdigest()
def retire_admission_intents(self, retention_seconds=30 * 86400, limit=100):
if not self.conn or not self.conn.is_postgres:
return 0
now = utc_now_iso()
cutoff = datetime.fromtimestamp(
time.time() - max(3600, int(retention_seconds)), timezone.utc,
).isoformat(timespec='seconds')
limit = min(500, max(1, int(limit)))
try:
self.conn.execute(f'SET LOCAL statement_timeout = {RETIREMENT_STATEMENT_TIMEOUT_MS}')
self.conn.execute(
'''INSERT INTO admission_intent_retirement(
id, retired_count, chain_sha256, cursor_token, updated_at
) VALUES (1, 0, ?, '', ?) ON CONFLICT(id) DO NOTHING''',
('0' * 64, now),
)
summary = self.conn.execute(
'SELECT * FROM admission_intent_retirement WHERE id = 1 FOR UPDATE'
).fetchone()
rows = self.conn.execute(
'''SELECT i.reservation_token, i.intent_sha256, i.state,
i.resolution_detail, i.created_at, i.updated_at, i.resolved_at
FROM admission_intents i
WHERE i.state = 'aborted' AND i.reservation_id IS NULL
AND i.resolved_at IS NOT NULL AND i.resolved_at <= ?
AND i.reservation_token > ?
AND NOT EXISTS (
SELECT 1 FROM result_reservations r
WHERE r.reservation_token = i.reservation_token
)
ORDER BY i.reservation_token LIMIT ? FOR UPDATE SKIP LOCKED''',
(cutoff, summary['cursor_token'], limit),
).fetchall()
if not rows:
if summary['cursor_token']:
self.conn.execute(
"UPDATE admission_intent_retirement SET cursor_token = '', updated_at = ? WHERE id = 1",
(now,),
)
self.conn.commit()
return 0
tokens = [row['reservation_token'] for row in rows]
placeholders = ','.join('?' for _ in tokens)
deleted = self.conn.execute(
f'''DELETE FROM admission_intents i
WHERE i.reservation_token IN ({placeholders})
AND i.state = 'aborted' AND i.reservation_id IS NULL
AND NOT EXISTS (
SELECT 1 FROM result_reservations r
WHERE r.reservation_token = i.reservation_token
)''',
tokens,
)
count = int(deleted.rowcount or 0)
if count != len(rows):
raise RuntimeError('admission intent retirement lost its exact row fence')
chain = self._retirement_chain(summary['chain_sha256'], rows)
self.conn.execute(
'''UPDATE admission_intent_retirement SET retired_count = retired_count + ?,
chain_sha256 = ?, cursor_token = ?, updated_at = ? WHERE id = 1''',
(count, chain, tokens[-1], now),
)
self.conn.commit()
return count
except Exception:
self.conn.rollback()
raise
def retire_deleted_pipeline_artifacts(self, retention_seconds=30 * 86400, limit=100):
if not self.conn or not self.conn.is_postgres:
return 0
now = utc_now_iso()
cutoff = datetime.fromtimestamp(
time.time() - max(3600, int(retention_seconds)), timezone.utc,
).isoformat(timespec='seconds')
limit = min(500, max(1, int(limit)))
try:
self.conn.execute(f'SET LOCAL statement_timeout = {RETIREMENT_STATEMENT_TIMEOUT_MS}')
self.conn.execute(
'''INSERT INTO pipeline_artifact_retirement(
id, retired_count, chain_sha256, cursor_id, updated_at
) VALUES (1, 0, ?, 0, ?) ON CONFLICT(id) DO NOTHING''',
('0' * 64, now),
)
summary = self.conn.execute(
'SELECT * FROM pipeline_artifact_retirement WHERE id = 1 FOR UPDATE'
).fetchone()
rows = self.conn.execute(
'''SELECT a.id, a.subsystem, a.artifact_kind, a.owner_id, a.owner_key,
a.relative_path, a.payload_sha256, a.created_at, a.updated_at, a.deleted_at
FROM pipeline_artifacts a
WHERE a.state = 'deleted' AND a.deleted_at IS NOT NULL
AND a.deleted_at <= ? AND a.id > ?
AND NOT EXISTS (
SELECT 1 FROM pipeline_quarantine q
WHERE q.review_status = 'pending' AND (
(q.object_type = 'projection_tail' AND q.object_id = a.id)
OR (
q.object_type = 'result_bundle'
AND a.subsystem = 'result_bundle'
AND a.artifact_kind = 'bundle_quarantine'
AND q.reservation_id = a.owner_id
)
)
)
ORDER BY a.id LIMIT ? FOR UPDATE SKIP LOCKED''',
(cutoff, int(summary['cursor_id'] or 0), limit),
).fetchall()
if not rows:
if int(summary['cursor_id'] or 0):
self.conn.execute(
'UPDATE pipeline_artifact_retirement SET cursor_id = 0, updated_at = ? WHERE id = 1',
(now,),
)
self.conn.commit()
return 0
ids = [int(row['id']) for row in rows]
placeholders = ','.join('?' for _ in ids)
deleted = self.conn.execute(
f'''DELETE FROM pipeline_artifacts a
WHERE a.id IN ({placeholders}) AND a.state = 'deleted'
AND NOT EXISTS (
SELECT 1 FROM pipeline_quarantine q
WHERE q.review_status = 'pending' AND (
(q.object_type = 'projection_tail' AND q.object_id = a.id)
OR (
q.object_type = 'result_bundle'
AND a.subsystem = 'result_bundle'
AND a.artifact_kind = 'bundle_quarantine'
AND q.reservation_id = a.owner_id
)
)
)''',
ids,
)
count = int(deleted.rowcount or 0)
if count != len(rows):
raise RuntimeError('pipeline artifact retirement lost its exact row fence')
chain = self._retirement_chain(summary['chain_sha256'], rows)
self.conn.execute(
'''UPDATE pipeline_artifact_retirement SET retired_count = retired_count + ?,
chain_sha256 = ?, cursor_id = ?, updated_at = ? WHERE id = 1''',
(count, chain, ids[-1], now),
)
self.conn.commit()
return count
except Exception:
self.conn.rollback()
raise
def janitor_cursor(self, layout_name):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('janitor cursor authority requires PostgreSQL')
row = self.conn.execute(
'SELECT layout_name, last_name, wrap_count FROM janitor_cursors WHERE layout_name = ?',
(str(layout_name),),
).fetchone()
self.conn.commit()
return dict(row) if row else {
'layout_name': str(layout_name), 'last_name': '', 'wrap_count': 0,
}
def advance_janitor_cursor(self, layout_name, last_name, wrapped=False):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('janitor cursor authority requires PostgreSQL')
now = utc_now_iso()
self.conn.execute(
'''INSERT INTO janitor_cursors(layout_name, last_name, wrap_count, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(layout_name) DO UPDATE SET
last_name = excluded.last_name,
wrap_count = janitor_cursors.wrap_count + excluded.wrap_count,
updated_at = excluded.updated_at''',
(str(layout_name), str(last_name), 1 if wrapped else 0, now),
)
self.conn.commit()
return True
def claim_projection_job(self, generation, lease_token, lease_seconds=120):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection job claims require PostgreSQL')
now = utc_now_iso()
expires = datetime.fromtimestamp(
time.time() + max(10, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
token = secrets.token_urlsafe(32)
try:
owner = self.conn.execute(
'''SELECT 1 AS valid FROM pipeline_leases
WHERE worker_name = 'jsonl_projector' AND generation = ?
AND lease_token = ? AND state = 'ready' AND lease_expires_at > ? FOR SHARE''',
(int(generation), str(lease_token), now),
).fetchone()
if not owner:
raise RuntimeError('JSONL projector singleton fence is not valid')
row = self.conn.execute(
'''SELECT id FROM projection_jobs
WHERE (status = 'pending' AND (available_after IS NULL OR available_after <= ?))
OR (status = 'leased' AND (
lease_generation IS NULL OR lease_generation <> ?
OR (lease_expires_at IS NOT NULL AND lease_expires_at <= ?)
))
ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED''',
(now, int(generation), now),
).fetchone()
if not row:
self.conn.rollback()
return None
self.conn.execute(
'''UPDATE projection_jobs SET status = 'leased', attempts = attempts + 1,
lease_generation = ?, lease_token = ?, lease_expires_at = ?, updated_at = ?
WHERE id = ?''',
(int(generation), token, expires, now, row['id']),
)
claimed = self.conn.execute(
'SELECT * FROM projection_jobs WHERE id = ?', (row['id'],),
).fetchone()
self.conn.commit()
return dict(claimed)
except Exception:
self.conn.rollback()
raise
def expand_projection_job_capacity(
self, job_id, job_lease_token, actual_bytes, projection_max_bytes,
defer_seconds=1,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection capacity expansion requires PostgreSQL')
actual_bytes = max(0, int(actual_bytes))
projection_max_bytes = max(0, int(projection_max_bytes))
now = utc_now_iso()
available_after = datetime.fromtimestamp(
time.time() + max(1, int(defer_seconds)), timezone.utc,
).isoformat(timespec='seconds')
try:
job = self.conn.execute(
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
AND lease_token = ? FOR UPDATE''',
(int(job_id), str(job_lease_token)),
).fetchone()
if not job:
self.conn.rollback()
return None
if actual_bytes <= int(job['capacity_bytes']):
self.conn.commit()
return dict(job)
if self.conn.execute(
'SELECT 1 AS present FROM projection_appends WHERE job_id = ? LIMIT 1',
(int(job_id),),
).fetchone():
raise RuntimeError(
'projection capacity cannot expand after append preparation'
)
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
additional_bytes = actual_bytes - int(job['capacity_bytes'])
if (
not capacity
or actual_bytes > projection_max_bytes
or int(capacity['projection_bytes']) + additional_bytes
> projection_max_bytes
):
cursor = self.conn.execute(
'''UPDATE projection_jobs SET status = 'pending',
available_after = ?, lease_generation = NULL,
lease_token = NULL, lease_expires_at = NULL,
last_error_code = 'projection_capacity_backpressure',
last_error_detail = NULL, updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_token = ?''',
(available_after, now, job_id, job_lease_token),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError(
'projection capacity deferral lost its lease fence'
)
self.conn.commit()
return False
cursor = self.conn.execute(
'''UPDATE projection_jobs SET capacity_bytes = ?,
last_error_code = NULL, last_error_detail = NULL,
updated_at = ?
WHERE id = ? AND status = 'leased' AND lease_token = ?
AND capacity_bytes = ?''',
(
actual_bytes, now, job_id, job_lease_token,
job['capacity_bytes'],
),
)
if int(cursor.rowcount or 0) != 1:
raise RuntimeError(
'projection capacity expansion lost its lease fence'
)
self.conn.execute(
'''UPDATE pipeline_capacity
SET projection_bytes = projection_bytes + ?, updated_at = ?
WHERE id = 1''',
(additional_bytes, now),
)
job = dict(job)
job['capacity_bytes'] = actual_bytes
job['updated_at'] = now
self.conn.commit()
return job
except Exception:
self.conn.rollback()
raise
def projection_stream_state(self, stream_name):
if not self.conn:
raise RuntimeError('database connection is unavailable')
row = self.conn.execute(
'''SELECT s.*, c.generation, c.committed_offset, c.last_append_id,
c.last_job_id, c.last_event_id, c.last_event_hash
FROM projection_streams s
JOIN projection_cursors c ON c.stream_name = s.stream_name
WHERE s.stream_name = ?''',
(str(stream_name),),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
return dict(row) if row else None
def projection_append_for_job(self, job_id, stream_name):
if not self.conn:
return None
row = self.conn.execute(
'SELECT * FROM projection_appends WHERE job_id = ? AND stream_name = ?',
(int(job_id), str(stream_name)),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
return dict(row) if row else None
def initialize_projection_stream_offset(self, stream_name, exact_file_size):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection stream offset initialization requires PostgreSQL')
exact_file_size = max(0, int(exact_file_size))
now = utc_now_iso()
try:
cursor = self.conn.execute(
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
(str(stream_name),),
).fetchone()
if not cursor:
raise ValueError('projection stream cursor is absent')
if int(cursor['committed_offset']) == exact_file_size:
self.conn.commit()
return self.projection_stream_state(stream_name)
append_count = self.conn.execute(
'SELECT COUNT(*) AS count FROM projection_appends WHERE stream_name = ?',
(str(stream_name),),
).fetchone()
if (
int(cursor['committed_offset']) != 0
or cursor['last_append_id'] is not None
or cursor['last_job_id'] is not None
or int(append_count['count'] or 0) != 0
):
raise RuntimeError('projection stream/file mismatch has append history')
updated = self.conn.execute(
'''UPDATE projection_cursors SET committed_offset = ?, updated_at = ?
WHERE stream_name = ? AND committed_offset = 0
AND last_append_id IS NULL AND last_job_id IS NULL''',
(exact_file_size, now, str(stream_name)),
)
if int(updated.rowcount or 0) != 1:
raise RuntimeError('projection stream offset initialization lost its fence')
self.conn.commit()
return self.projection_stream_state(stream_name)
except Exception:
self.conn.rollback()
raise
def prepare_projection_append(
self, job_id, job_lease_token, stream_name, generation,
byte_length, payload_sha256, record_count,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection append preparation requires PostgreSQL')
now = utc_now_iso()
try:
job = self.conn.execute(
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
AND lease_token = ? FOR UPDATE''',
(int(job_id), str(job_lease_token)),
).fetchone()
if not job:
self.conn.rollback()
return None
existing = self.conn.execute(
'''SELECT * FROM projection_appends WHERE job_id = ? AND stream_name = ? FOR UPDATE''',
(int(job_id), str(stream_name)),
).fetchone()
if existing:
self.conn.commit()
return dict(existing)
cursor = self.conn.execute(
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
(str(stream_name),),
).fetchone()
if not cursor or int(cursor['generation']) != int(generation):
raise RuntimeError('projection stream generation changed before append preparation')
append_id = self.conn.insert_returning_id(
'''INSERT INTO projection_appends(
job_id, stream_name, event_id, event_hash, generation,
byte_offset, byte_length, payload_sha256, record_count,
state, prepared_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'prepared', ?)''',
(
job_id, stream_name, job['event_id'], job['event_hash'], generation,
cursor['committed_offset'], max(0, int(byte_length)),
str(payload_sha256), max(0, int(record_count)), now,
),
)
row = self.conn.execute(
'SELECT * FROM projection_appends WHERE id = ?', (append_id,),
).fetchone()
self.conn.commit()
return dict(row)
except Exception:
self.conn.rollback()
raise
def complete_projection_append(self, append_id, job_id, job_lease_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection append completion requires PostgreSQL')
now = utc_now_iso()
try:
job = self.conn.execute(
'''SELECT id FROM projection_jobs WHERE id = ? AND status = 'leased'
AND lease_token = ? FOR UPDATE''',
(int(job_id), str(job_lease_token)),
).fetchone()
append = self.conn.execute(
'SELECT * FROM projection_appends WHERE id = ? AND job_id = ? FOR UPDATE',
(int(append_id), int(job_id)),
).fetchone()
if not job or not append:
self.conn.rollback()
return False
expected_offset = int(append['byte_offset']) + int(append['byte_length'])
cursor = self.conn.execute(
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
(append['stream_name'],),
).fetchone()
if append['state'] == 'appended':
valid = bool(
cursor and int(cursor['generation']) == int(append['generation'])
and int(cursor['committed_offset']) >= expected_offset
)
self.conn.commit()
return valid
if not cursor or int(cursor['generation']) != int(append['generation']) or int(cursor['committed_offset']) != int(append['byte_offset']):
raise RuntimeError('projection cursor no longer matches the prepared append')
self.conn.execute(
"UPDATE projection_appends SET state = 'appended', appended_at = ? WHERE id = ?",
(now, append_id),
)
self.conn.execute(
'''UPDATE projection_cursors SET committed_offset = ?, last_append_id = ?,
last_job_id = ?, last_event_id = ?, last_event_hash = ?, updated_at = ?
WHERE stream_name = ?''',
(
expected_offset, append_id, job_id, append['event_id'],
append['event_hash'], now, append['stream_name'],
),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def complete_projection_job(self, job_id, job_lease_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection job completion requires PostgreSQL')
now = utc_now_iso()
try:
job = self.conn.execute(
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
AND lease_token = ? FOR UPDATE''',
(int(job_id), str(job_lease_token)),
).fetchone()
if not job:
self.conn.rollback()
return False
stream_count = sum(
int(bool(int(job['required_stream_mask']) & mask))
for mask in (1, 2, 4, 8, 16)
)
appended = self.conn.execute(
"SELECT stream_name FROM projection_appends WHERE job_id = ? AND state = 'appended'",
(int(job_id),),
).fetchall()
appended_count = sum(
int(bool(
(row['stream_name'] == 'scan_results' and int(job['required_stream_mask']) & 1)
or (row['stream_name'] == 'found_secrets' and int(job['required_stream_mask']) & 2)
or (row['stream_name'] == 'scan_errors' and int(job['required_stream_mask']) & 4)
or (str(row['stream_name']).startswith('keycheck:')
and str(row['stream_name']).endswith(':results')
and int(job['required_stream_mask']) & 8)
or (str(row['stream_name']).startswith('keycheck:')
and str(row['stream_name']).endswith(':status')
and int(job['required_stream_mask']) & 16)
))
for row in appended
)
if appended_count != stream_count:
raise RuntimeError('projection job does not have every required appended stream')
if not job['capacity_released']:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['projection_items']) < int(job['capacity_items'])
or int(capacity['projection_bytes']) < int(job['capacity_bytes'])
):
raise RuntimeError('projection completion would make capacity accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET projection_items = projection_items - ?,
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
(job['capacity_items'], job['capacity_bytes'], now),
)
self.conn.execute(
'''UPDATE projection_jobs SET status = 'completed', capacity_released = 1,
lease_token = NULL, lease_expires_at = NULL, completed_at = ?, updated_at = ?
WHERE id = ?''',
(now, now, job_id),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def quarantine_projection_job(
self, job_id, job_lease_token, reason_code, detail='',
quarantine_max_items=10000, quarantine_max_bytes=1024 * 1024 * 1024,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection quarantine requires PostgreSQL')
now = utc_now_iso()
try:
job = self.conn.execute(
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
AND lease_token = ? FOR UPDATE''',
(int(job_id), str(job_lease_token)),
).fetchone()
if not job:
self.conn.rollback()
return False
prepared = self.conn.execute(
'''SELECT * FROM projection_appends
WHERE job_id = ? AND state = 'prepared' ORDER BY id FOR UPDATE''',
(int(job_id),),
).fetchall()
for append in prepared:
cursor = self.conn.execute(
'SELECT generation, committed_offset FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
(append['stream_name'],),
).fetchone()
if not cursor or (
int(cursor['generation']) != int(append['generation'])
or int(cursor['committed_offset']) != int(append['byte_offset'])
):
raise RuntimeError('prepared projection append cannot be safely canceled')
self.conn.execute(
'''INSERT INTO projection_append_audit(
append_id, job_id, stream_name, event_id, event_hash, generation,
byte_offset, byte_length, payload_sha256, state,
reason_code, reason_detail, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'quarantined', ?, ?, ?)''',
(
append['id'], append['job_id'], append['stream_name'],
append['event_id'], append['event_hash'], append['generation'],
append['byte_offset'], append['byte_length'], append['payload_sha256'],
str(reason_code), first_line(detail, 2000), now,
),
)
self.conn.execute('DELETE FROM projection_appends WHERE id = ?', (append['id'],))
existing = self.conn.execute(
'''SELECT id FROM pipeline_quarantine
WHERE subsystem = 'jsonl_projector' AND object_type = 'projection_job'
AND object_id = ? AND review_status = 'pending' ''',
(job_id,),
).fetchone()
if not existing:
self.conn.insert_returning_id(
'''INSERT INTO pipeline_quarantine(
subsystem, object_type, object_id, projection_job_id,
event_id, payload_sha256, reason_code, reason_detail,
byte_count, capacity_items, capacity_bytes, detected_at
) VALUES ('jsonl_projector','projection_job',?,?,?,?,?,?,?,?,?,?)''',
(
job_id, job_id, job['event_id'], job['event_hash'], str(reason_code),
first_line(detail, 2000), job['capacity_bytes'],
job['capacity_items'], job['capacity_bytes'], now,
),
)
if not job['capacity_released']:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['projection_items']) < int(job['capacity_items'])
or int(capacity['projection_bytes']) < int(job['capacity_bytes'])
):
raise RuntimeError('projection quarantine would make capacity accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET projection_items = projection_items - ?,
projection_bytes = projection_bytes - ?,
quarantine_items = quarantine_items + ?,
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
(
job['capacity_items'], job['capacity_bytes'], job['capacity_items'],
job['capacity_bytes'], now,
),
)
self.conn.execute(
'''UPDATE projection_jobs SET status = 'quarantined', capacity_released = 1,
last_error_code = ?, last_error_detail = ?, lease_token = NULL,
lease_expires_at = NULL, completed_at = ?, updated_at = ? WHERE id = ?''',
(str(reason_code), first_line(detail, 2000), now, now, job_id),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def prepare_projection_rotation(self, stream_name, source_bytes, segment_relative_path):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection rotation requires PostgreSQL')
now = utc_now_iso()
try:
stream = self.conn.execute(
'SELECT * FROM projection_streams WHERE stream_name = ? FOR UPDATE',
(str(stream_name),),
).fetchone()
cursor = self.conn.execute(
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
(str(stream_name),),
).fetchone()
if not stream or not cursor or int(cursor['committed_offset']) != int(source_bytes):
raise RuntimeError('projection rotation source size does not match its cursor')
to_generation = int(stream['current_generation']) + 1
rotation_id = self.conn.insert_returning_id(
'''INSERT INTO projection_rotations(
stream_name, from_generation, to_generation, source_bytes,
segment_relative_path, state, created_at
) VALUES (?, ?, ?, ?, ?, 'prepared', ?)
ON CONFLICT(stream_name, to_generation) DO NOTHING''',
(
stream_name, stream['current_generation'], to_generation,
int(source_bytes), str(segment_relative_path), now,
),
)
if rotation_id is None:
existing = self.conn.execute(
'SELECT * FROM projection_rotations WHERE stream_name = ? AND to_generation = ?',
(stream_name, to_generation),
).fetchone()
self.conn.commit()
return dict(existing)
row = self.conn.execute(
'SELECT * FROM projection_rotations WHERE id = ?', (rotation_id,),
).fetchone()
self.conn.commit()
return dict(row)
except Exception:
self.conn.rollback()
raise
def complete_projection_rotation(self, rotation_id):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('projection rotation completion requires PostgreSQL')
now = utc_now_iso()
try:
rotation = self.conn.execute(
'SELECT * FROM projection_rotations WHERE id = ? FOR UPDATE',
(int(rotation_id),),
).fetchone()
if not rotation:
self.conn.rollback()
return False
self.conn.execute(
'''UPDATE projection_streams SET current_generation = ?, updated_at = ?
WHERE stream_name = ? AND current_generation = ?''',
(
rotation['to_generation'], now, rotation['stream_name'],
rotation['from_generation'],
),
)
self.conn.execute(
'''UPDATE projection_cursors SET generation = ?, committed_offset = 0,
last_append_id = NULL, updated_at = ? WHERE stream_name = ?''',
(rotation['to_generation'], now, rotation['stream_name']),
)
self.conn.execute(
"UPDATE projection_rotations SET state = 'completed', completed_at = ? WHERE id = ?",
(now, rotation_id),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def pending_projection_rotations(self, limit=100):
if not self.conn:
return []
rows = self.conn.execute(
'''SELECT r.*, s.base_relative_path
FROM projection_rotations r
JOIN projection_streams s ON s.stream_name = r.stream_name
WHERE r.state IN ('prepared','renamed') ORDER BY r.id LIMIT ?''',
(min(1000, max(1, int(limit))),),
).fetchall()
if self.conn.is_postgres:
self.conn.commit()
return [dict(row) for row in rows]
def claim_keycheck_candidate(
self, service, lease_owner, lease_seconds=300,
result_projection_reserve_bytes=KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES,
projection_max_items=10000,
projection_max_bytes=2 * 1024 * 1024 * 1024,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('normal keycheck candidate claims require PostgreSQL')
service = str(service or '').lower()
if not service or not lease_owner:
raise ValueError('keycheck service and lease owner are required')
now = utc_now_iso()
expires = datetime.fromtimestamp(
time.time() + max(30, int(lease_seconds)), timezone.utc,
).isoformat(timespec='seconds')
token = secrets.token_urlsafe(32)
try:
row = self.conn.execute(
'''SELECT id, result_projection_reserved_bytes,
result_projection_credit_transferred
FROM keycheck_candidates
WHERE service = ? AND (
state = 'pending'
OR (state = 'deferred' AND available_after IS NOT NULL AND available_after <= ?)
OR (state = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?)
)
ORDER BY priority DESC, id LIMIT 1 FOR UPDATE SKIP LOCKED''',
(service, now, now),
).fetchone()
if not row:
self.conn.rollback()
return None
reserve_bytes = max(
KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES,
int(result_projection_reserve_bytes),
)
if (
int(row['result_projection_reserved_bytes'] or 0) == 0
and not row['result_projection_credit_transferred']
):
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['projection_items']) + 1 > int(projection_max_items)
or int(capacity['projection_bytes']) + reserve_bytes > int(projection_max_bytes)
):
self.conn.rollback()
return {
'_capacity_blocked': True,
'service': service,
'reason': 'projection_capacity_saturated',
}
self.conn.execute(
'''UPDATE pipeline_capacity SET projection_items = projection_items + 1,
projection_bytes = projection_bytes + ?, updated_at = ? WHERE id = 1''',
(reserve_bytes, now),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET result_projection_reserved_bytes = ?
WHERE id = ?''',
(reserve_bytes, row['id']),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET state = 'leased', attempts = attempts + 1,
lease_owner = ?, lease_token = ?, lease_expires_at = ?, updated_at = ?
WHERE id = ?''',
(str(lease_owner), token, expires, now, row['id']),
)
candidate = self.conn.execute(
'''SELECT c.*, kc.candidate_kind, kc.credential_hash,
kc.provider_key_hash, kc.secret_text,
kc.secret_json, kc.key_masked, kc.endpoint, kc.principal,
kc.metadata_json AS credential_metadata_json
FROM keycheck_candidates c
JOIN keycheck_credentials kc ON kc.id = c.credential_id
WHERE c.id = ?''',
(row['id'],),
).fetchone()
finding = None
if candidate['finding_id'] is not None:
finding_row = self.conn.execute(
'''SELECT f.*, cp.raw_value, cp.raw_v2_value, cp.structured_data_json,
cp.extra_data_json, cp.analysis_info_json, cp.extension_json,
cp.payload_sha256, cp.payload_bytes, cp.payload_omitted
FROM findings f
LEFT JOIN finding_compat_payloads cp ON cp.finding_id = f.id
WHERE f.id = ?''',
(candidate['finding_id'],),
).fetchone()
if finding_row:
if finding_row['payload_omitted']:
finding = {
'finding_uid': finding_row['finding_uid'],
'DetectorName': finding_row['detector_name'],
'finding_omitted': True,
'payload_sha256': finding_row['payload_sha256'],
}
else:
finding = safe_json_loads(finding_row['extension_json']) or {}
finding.update({
'finding_uid': finding_row['finding_uid'],
'DetectorName': finding_row['detector_name'],
'DetectorType': finding_row['detector_type'],
'Verified': bool(finding_row['verified']),
'Raw': finding_row['raw_value'],
'RawV2': finding_row['raw_v2_value'],
})
for column, key in (
('structured_data_json', 'StructuredData'),
('extra_data_json', 'ExtraData'),
('analysis_info_json', 'AnalysisInfo'),
):
value = safe_json_loads(finding_row[column])
if value is not None:
finding[key] = value
output = dict(candidate)
output['finding'] = finding or {}
self.conn.commit()
return output
except Exception:
self.conn.rollback()
raise
def keycheck_candidate_cached_status(self, candidate_id, lease_token):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('cached keycheck status lookup requires PostgreSQL')
try:
row = self.conn.execute(
'''SELECT c.service, c.state, c.lease_token, c.metadata_json,
s.status, s.status_group, s.last_result_id, s.result_source,
s.checked_at, s.state_version,
EXISTS(
SELECT 1 FROM pipeline_quarantine q
WHERE q.keycheck_candidate_id = c.id
AND q.review_status = 'approved_retry'
) AS approved_retry
FROM keycheck_candidates c
LEFT JOIN keycheck_current_state s ON s.credential_id = c.credential_id
WHERE c.id = ?''',
(int(candidate_id),),
).fetchone()
if (
not row or row['state'] != 'leased'
or str(row['lease_token'] or '') != str(lease_token)
):
self.conn.rollback()
return None
metadata = safe_json_loads(row['metadata_json']) or {}
reason = ''
if row['service'] == 'provider_resolver':
reason = 'provider_resolution_required'
elif 'recheck_of_status' in metadata or 'recheck_generation' in metadata:
reason = 'explicit_recheck'
elif row['approved_retry']:
reason = 'approved_quarantine_retry'
elif row['state_version'] is None:
reason = 'no_current_state'
self.conn.commit()
if reason:
return {'probe_required': True, 'reason': reason}
return {
'probe_required': False,
'status': row['status'],
'status_group': row['status_group'],
'last_result_id': int(row['last_result_id']),
'result_source': row['result_source'],
'checked_at': row['checked_at'],
'state_version': int(row['state_version']),
}
except Exception:
self.conn.rollback()
raise
def defer_keycheck_candidate(self, candidate_id, lease_token, error='', delay_seconds=60):
if not self.conn or not self.conn.is_postgres:
return False
now = utc_now_iso()
available = datetime.fromtimestamp(
time.time() + max(1, int(delay_seconds)), timezone.utc,
).isoformat(timespec='seconds')
try:
candidate = self.conn.execute(
'''SELECT * FROM keycheck_candidates
WHERE id = ? AND state = 'leased' AND lease_token = ? FOR UPDATE''',
(int(candidate_id), str(lease_token)),
).fetchone()
if not candidate:
self.conn.rollback()
return False
reserved = (
int(candidate['result_projection_reserved_bytes'] or 0)
if not candidate['result_projection_credit_transferred'] else 0
)
if reserved:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if int(capacity['projection_items']) < 1 or int(capacity['projection_bytes']) < reserved:
raise RuntimeError('keycheck defer would make projection capacity negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET projection_items = projection_items - 1,
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
(reserved, now),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET state = 'deferred', available_after = ?,
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
result_projection_reserved_bytes = 0,
last_error = ?, updated_at = ? WHERE id = ?''',
(available, first_line(error, 1000), now, int(candidate_id)),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def quarantine_keycheck_candidate(self, candidate_id, lease_token, reason_code, detail=''):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('keycheck candidate quarantine requires PostgreSQL')
now = utc_now_iso()
try:
candidate = self.conn.execute(
'''SELECT * FROM keycheck_candidates
WHERE id = ? AND state = 'leased' AND lease_token = ? FOR UPDATE''',
(int(candidate_id), str(lease_token)),
).fetchone()
if not candidate:
self.conn.rollback()
return False
existing = self.conn.execute(
'''SELECT id FROM pipeline_quarantine
WHERE subsystem = 'keycheck' AND object_type = 'keycheck_candidate'
AND object_id = ? AND review_status = 'pending' FOR UPDATE''',
(int(candidate_id),),
).fetchone()
if existing:
self.conn.commit()
return True
projection_bytes = (
int(candidate['result_projection_reserved_bytes'] or 0)
if not candidate['result_projection_credit_transferred'] else 0
)
quarantine_items = (0 if candidate['capacity_released'] else 1) + int(projection_bytes > 0)
quarantine_bytes = (
(0 if candidate['capacity_released'] else int(candidate['capacity_bytes']))
+ projection_bytes
)
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if (
int(capacity['keycheck_items']) < (0 if candidate['capacity_released'] else 1)
or int(capacity['keycheck_bytes']) < (
0 if candidate['capacity_released'] else int(candidate['capacity_bytes'])
)
or int(capacity['projection_items']) < int(projection_bytes > 0)
or int(capacity['projection_bytes']) < projection_bytes
):
raise RuntimeError('keycheck quarantine would make capacity accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
quarantine_items = quarantine_items + ?,
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
(
0 if candidate['capacity_released'] else 1,
0 if candidate['capacity_released'] else candidate['capacity_bytes'],
int(projection_bytes > 0), projection_bytes,
quarantine_items, quarantine_bytes, now,
),
)
self.conn.insert_returning_id(
'''INSERT INTO pipeline_quarantine(
subsystem, object_type, object_id, keycheck_candidate_id,
reason_code, reason_detail, byte_count, capacity_items,
capacity_bytes, detected_at
) VALUES ('keycheck','keycheck_candidate',?,?,?,?,?,?,?,?)''',
(
candidate_id, candidate_id, str(reason_code), first_line(detail, 2000),
0, quarantine_items, quarantine_bytes, now,
),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET state = 'quarantined', capacity_released = 1,
result_projection_credit_transferred = 1,
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
last_error = ?, completed_at = ?, updated_at = ? WHERE id = ?''',
(first_line(detail or reason_code, 1000), now, now, candidate_id),
)
self.conn.commit()
return True
except Exception:
self.conn.rollback()
raise
def complete_keycheck_candidate(
self, candidate_id, lease_token, event_id, status, status_group,
checked_at=None, message='', metadata=None, result_source='api_check',
resolved_service='', cached_state_version=None, cached_last_result_id=None,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('normal keycheck completion requires PostgreSQL')
metadata = dict(metadata or {})
resolved_service = str(resolved_service or '').strip().lower()
cached_completion = (
cached_state_version is not None or cached_last_result_id is not None
)
if cached_completion:
if cached_state_version is None or cached_last_result_id is None:
raise ValueError('cached keycheck completion requires an exact current-state fence')
if str(result_source or '') != 'cached_status':
raise ValueError('cached keycheck completion requires cached_status result source')
if resolved_service:
raise ValueError('provider resolution cannot use cached keycheck completion')
if (
int(metadata.get('cached_state_version') or -1) != int(cached_state_version)
or int(metadata.get('cached_result_id') or -1) != int(cached_last_result_id)
):
raise ValueError('cached keycheck metadata conflicts with its current-state fence')
if resolved_service:
if not re.fullmatch(r'[a-z][a-z0-9_]{1,63}', resolved_service):
raise ValueError('resolved keycheck service is invalid')
if str(metadata.get('resolved_provider') or '').strip().lower() != resolved_service:
raise ValueError('resolved keycheck service conflicts with result metadata')
if str(metadata.get('provider_resolution') or '') != 'matched':
raise ValueError('resolved keycheck service requires a matched provider resolution')
if not metadata.get('authenticated') and str(status).upper() not in ('VALID', 'ALIVE'):
raise ValueError('resolved keycheck service has no authenticated provider evidence')
event_payload = json.dumps({
'candidate_id': int(candidate_id),
'event_id': str(event_id or ''),
'status': str(status).upper(),
'status_group': str(status_group).lower(),
'checked_at': str(checked_at or ''),
'message': first_line(message, 1000),
'metadata': metadata,
'result_source': str(result_source or 'api_check'),
}, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str).encode('utf-8')
metadata['_event_hash'] = hashlib.sha256(event_payload).hexdigest()
encoded_metadata = json.dumps(
metadata, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
if len(encoded_metadata) > 1024 * 1024:
raise ValueError('keycheck result metadata exceeds its byte bound')
event_id = str(event_id or '')
if not re.fullmatch(r'[a-f0-9]{64}', event_id):
raise ValueError('keycheck event ID must be a stable SHA-256 identity')
now = utc_now_iso()
checked = str(checked_at or now)
try:
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
candidate = self.conn.execute(
'''SELECT c.*, kc.credential_hash, kc.provider_key_hash,
kc.key_masked, kc.secret_text, kc.secret_json,
kc.candidate_kind, kc.endpoint, kc.principal,
kc.metadata_json AS credential_metadata_json
FROM keycheck_candidates c
JOIN keycheck_credentials kc ON kc.id = c.credential_id
WHERE c.id = ? FOR UPDATE OF c''',
(int(candidate_id),),
).fetchone()
if not candidate:
raise ValueError('keycheck candidate is absent')
existing_map = self.conn.execute(
'SELECT keycheck_result_id FROM keycheck_event_map WHERE event_id = ? FOR UPDATE',
(event_id,),
).fetchone()
if existing_map:
if int(candidate['keycheck_result_id'] or 0) != int(existing_map['keycheck_result_id'] or 0):
raise ScanEventConflictError('keycheck event replay conflicts with candidate completion')
existing_result = self.conn.execute(
'SELECT metadata_json FROM keycheck_results WHERE id = ?',
(existing_map['keycheck_result_id'],),
).fetchone()
existing_metadata = safe_json_loads(existing_result['metadata_json'] if existing_result else '') or {}
if existing_metadata.get('_event_hash') != metadata['_event_hash']:
raise ScanEventConflictError('keycheck event replay has a different payload hash')
self.conn.commit()
return {
'completed': True, 'duplicate': True,
'keycheck_result_id': existing_map['keycheck_result_id'], 'event_id': event_id,
}
if candidate['state'] != 'leased' or str(candidate['lease_token'] or '') != str(lease_token):
self.conn.rollback()
return None
candidate = dict(candidate)
if cached_completion:
candidate_metadata = safe_json_loads(candidate['metadata_json']) or {}
approved_retry = self.conn.execute(
'''SELECT 1 FROM pipeline_quarantine
WHERE keycheck_candidate_id = ? AND review_status = 'approved_retry'
LIMIT 1''',
(int(candidate_id),),
).fetchone()
if (
candidate['service'] == 'provider_resolver'
or 'recheck_of_status' in candidate_metadata
or 'recheck_generation' in candidate_metadata
or approved_retry
):
self.conn.rollback()
return {'completed': False, 'probe_required': True}
current_state = self.conn.execute(
'''SELECT service, status, status_group, last_result_id, state_version
FROM keycheck_current_state WHERE credential_id = ? FOR UPDATE''',
(candidate['credential_id'],),
).fetchone()
if not current_state:
self.conn.rollback()
return {'completed': False, 'probe_required': True}
if (
current_state['service'] != candidate['service']
or int(current_state['state_version']) != int(cached_state_version)
or int(current_state['last_result_id']) != int(cached_last_result_id)
or str(current_state['status']).upper() != str(status).upper()
or str(current_state['status_group']).lower() != str(status_group).lower()
):
self.conn.rollback()
return {'completed': False, 'cached_state_changed': True}
if resolved_service and resolved_service != candidate['service']:
allowed_services = {'provider_resolver', 'qwen', 'deepseek', 'kimi', 'zai'}
if candidate['service'] not in allowed_services or resolved_service not in allowed_services - {'provider_resolver'}:
raise ValueError('provider resolution cannot reassign this keycheck service')
if candidate['candidate_kind'] != 'provider_key' or not candidate['secret_text']:
raise ValueError('provider resolution requires a text provider-key credential')
probe_material = str(candidate['secret_text'])
expected_provider_hash = hashlib.sha256(probe_material.encode('utf-8')).hexdigest()
if expected_provider_hash != candidate['provider_key_hash']:
raise ScanEventConflictError('provider resolution credential material conflicts with its key hash')
resolved_credential_hash = hashlib.sha256('|'.join((
'truf-credential-v2', resolved_service, probe_material,
)).encode('utf-8')).hexdigest()
resolved_credential_id = self.conn.insert_returning_id(
'''INSERT INTO keycheck_credentials(
service, credential_hash, provider_key_hash, candidate_kind,
secret_text, secret_json, key_masked, endpoint, principal,
metadata_json, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(service, provider_key_hash) DO NOTHING''',
(
resolved_service, resolved_credential_hash, candidate['provider_key_hash'],
candidate['candidate_kind'], candidate['secret_text'], candidate['secret_json'],
candidate['key_masked'], candidate['endpoint'], candidate['principal'],
candidate['credential_metadata_json'], now, now,
),
)
if resolved_credential_id is None:
resolved_credential = self.conn.execute(
'''SELECT id, credential_hash, secret_text, secret_json
FROM keycheck_credentials
WHERE service = ? AND provider_key_hash = ? FOR UPDATE''',
(resolved_service, candidate['provider_key_hash']),
).fetchone()
if not resolved_credential or (
resolved_credential['credential_hash'] != resolved_credential_hash
or resolved_credential['secret_text'] != candidate['secret_text']
or resolved_credential['secret_json'] != candidate['secret_json']
):
raise ScanEventConflictError('resolved provider credential conflicts with canonical material')
resolved_credential_id = resolved_credential['id']
self.conn.execute(
'''UPDATE keycheck_candidates SET credential_id = ?, service = ?, updated_at = ?
WHERE id = ? AND state = 'leased' AND lease_token = ?''',
(
resolved_credential_id, resolved_service, now,
int(candidate_id), str(lease_token),
),
)
candidate['credential_id'] = resolved_credential_id
candidate['credential_hash'] = resolved_credential_hash
candidate['service'] = resolved_service
result_message = first_line(message, 1000)
projected_result = {
'event_id': event_id,
'service': candidate['service'],
'status': str(status).upper(),
'status_group': str(status_group).lower(),
'checked_at': checked,
'key_hash': candidate['provider_key_hash'],
'secret_hash': candidate['secret_hash'],
'key_masked': candidate['key_masked'],
'finding_uid': candidate['finding_uid'],
'detector': candidate['detector_name'],
'source': candidate['source'],
'message': result_message,
'metadata': metadata,
'result_source': str(result_source or 'api_check'),
}
result_projection_bytes = len(json.dumps(
projected_result, ensure_ascii=False, sort_keys=True,
separators=(',', ':'), default=str,
).encode('utf-8')) + 1
projection_capacity_bytes = result_projection_bytes
reserved_projection_bytes = int(candidate['result_projection_reserved_bytes'] or 0)
if (
reserved_projection_bytes < KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES
or candidate['result_projection_credit_transferred']
):
raise RuntimeError('keycheck result has no exact pre-probe projection reservation')
if projection_capacity_bytes > reserved_projection_bytes:
raise ValueError(
'keycheck result compatibility output exceeds its pre-reserved byte capacity'
)
result_id = self.conn.insert_returning_id(
'''INSERT INTO keycheck_results(
service, status, status_group, checked_at, key_hash, secret_hash,
key_masked, finding_id, target_scan_id, source, query, target,
detector_name, found_at, message, metadata_json, event_id, finding_uid,
link_status, link_attempts, linked_at, link_error,
candidate_id, credential_id, result_source, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
'linked', 0, ?, '', ?, ?, ?, ?)''',
(
candidate['service'], str(status).upper(), str(status_group).lower(), checked,
candidate['provider_key_hash'], candidate['secret_hash'],
candidate['key_masked'], candidate['finding_id'], candidate['target_scan_id'],
candidate['source'], candidate['query'], candidate['target'],
candidate['detector_name'], candidate['found_at'], result_message,
encoded_metadata.decode('utf-8'), event_id, candidate['finding_uid'], now,
candidate_id, candidate['credential_id'], str(result_source or 'api_check'), now,
),
)
self.conn.execute(
'''INSERT INTO keycheck_event_map(event_id, keycheck_result_id, created_at)
VALUES (?, ?, ?)''',
(event_id, result_id, now),
)
if not cached_completion:
self.conn.execute(
'''INSERT INTO keycheck_current_state(
credential_id, service, status, status_group, last_result_id,
result_source, checked_at, recheck_after, state_version,
metadata_json, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
ON CONFLICT(credential_id) DO UPDATE SET
service = excluded.service, status = excluded.status,
status_group = excluded.status_group, last_result_id = excluded.last_result_id,
result_source = excluded.result_source, checked_at = excluded.checked_at,
recheck_after = excluded.recheck_after,
state_version = keycheck_current_state.state_version + 1,
metadata_json = excluded.metadata_json, updated_at = excluded.updated_at''',
(
candidate['credential_id'], candidate['service'], str(status).upper(),
str(status_group).lower(), result_id, str(result_source or 'api_check'),
checked, metadata.get('recheck_after'), encoded_metadata.decode('utf-8'), now,
),
)
if not candidate['capacity_released']:
if (
int(capacity['keycheck_items']) < 1
or int(capacity['keycheck_bytes']) < int(candidate['capacity_bytes'])
or int(capacity['projection_items']) < 1
or int(capacity['projection_bytes']) < reserved_projection_bytes
):
raise RuntimeError('keycheck completion would make candidate capacity negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items - 1,
keycheck_bytes = keycheck_bytes - ?,
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
(
candidate['capacity_bytes'],
reserved_projection_bytes - projection_capacity_bytes, now,
),
)
for stream_name, relative_path in ((
f'keycheck:{candidate["service"]}:results',
f'{candidate["service"]}/{candidate["service"]}Results.jsonl',
),):
self.conn.execute(
'''INSERT INTO projection_streams(
stream_name, base_relative_path, current_generation,
rotation_bytes, max_generations, created_at, updated_at
) VALUES (?, ?, 0, ?, 16, ?, ?)
ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, relative_path, 32 * 1024 * 1024, now, now),
)
self.conn.execute(
'''INSERT INTO projection_cursors(
stream_name, generation, committed_offset, updated_at
) VALUES (?, 0, 0, ?) ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, now),
)
projection_id = self.conn.insert_returning_id(
'''INSERT INTO projection_jobs(
job_kind, event_id, event_hash, keycheck_result_id, status,
required_stream_mask, capacity_items, capacity_bytes,
created_at, updated_at
) VALUES ('keycheck_event', ?, ?, ?, 'pending', 8, 1, ?, ?, ?)''',
(
event_id, hashlib.sha256(encoded_metadata + event_id.encode('ascii')).hexdigest(),
result_id, projection_capacity_bytes, now, now,
),
)
self.conn.execute(
'''UPDATE keycheck_candidates SET state = 'completed', keycheck_result_id = ?,
capacity_released = 1, lease_owner = NULL, lease_token = NULL,
lease_expires_at = NULL, result_projection_credit_transferred = 1,
completed_at = ?, updated_at = ? WHERE id = ?''',
(result_id, now, now, candidate_id),
)
self.conn.commit()
return {
'completed': True, 'duplicate': False, 'keycheck_result_id': result_id,
'projection_job_id': projection_id, 'event_id': event_id,
}
except Exception:
self.conn.rollback()
raise
def keycheck_result_for_projection(self, keycheck_result_id):
if not self.conn:
return None
row = self.conn.execute(
'''SELECT kr.*, kc.candidate_kind, kc.key_masked AS credential_masked,
kc.secret_text AS credential_secret_text,
kc.secret_json AS credential_secret_json,
c.candidate_uid
FROM keycheck_results kr
LEFT JOIN keycheck_credentials kc ON kc.id = kr.credential_id
LEFT JOIN keycheck_candidates c ON c.id = kr.candidate_id
WHERE kr.id = ?''',
(int(keycheck_result_id),),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
return dict(row) if row else None
def enqueue_keycheck_rechecks(
self, service, status_groups=None, max_items=1000,
queue_max_items=100000, queue_max_bytes=512 * 1024 * 1024,
):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('keycheck recheck generation requires PostgreSQL')
service = str(service or '').lower()
groups = sorted({str(value).lower() for value in (status_groups or []) if value})
max_items = min(10000, max(1, int(max_items)))
now = utc_now_iso()
status_scope = ','.join(groups) if groups else '*'
cursor_key = hashlib.sha256(
f'truf-keycheck-recheck-cursor-v1|{service}|{status_scope}'.encode('utf-8')
).hexdigest()
try:
self.conn.execute(
'''INSERT INTO keycheck_recheck_cursors(
cursor_key, service, status_scope, last_credential_id,
wrap_count, updated_at
) VALUES (?, ?, ?, 0, 0, ?) ON CONFLICT(cursor_key) DO NOTHING''',
(cursor_key, service, status_scope, now),
)
cursor = self.conn.execute(
'SELECT * FROM keycheck_recheck_cursors WHERE cursor_key = ? FOR UPDATE',
(cursor_key,),
).fetchone()
clauses = [
's.service = ?',
'''NOT EXISTS (
SELECT 1 FROM pipeline_quarantine q
JOIN keycheck_candidates blocked ON blocked.id = q.keycheck_candidate_id
WHERE blocked.credential_id = s.credential_id
AND q.review_status = 'pending'
AND q.reason_code IN (
'provider_candidate_unconsumed',
'candidate_provider_route_mismatch'
)
)''',
]
params = [service]
if groups:
clauses.append('s.status_group IN ({})'.format(','.join('?' for _ in groups)))
params.extend(groups)
def select_rows(after_id):
return self.conn.execute(
f'''SELECT s.credential_id, s.state_version, s.status, s.status_group,
c.credential_hash, c.candidate_kind, c.secret_text, c.secret_json,
r.secret_hash, r.source, r.query, r.target, r.detector_name,
r.found_at, r.finding_uid
FROM keycheck_current_state s
JOIN keycheck_credentials c ON c.id = s.credential_id
JOIN keycheck_results r ON r.id = s.last_result_id
WHERE {' AND '.join(clauses)} AND s.credential_id > ?
ORDER BY s.credential_id LIMIT ?''',
(*params, int(after_id), max_items),
).fetchall()
rows = select_rows(cursor['last_credential_id'])
wrapped = False
if not rows and int(cursor['last_credential_id'] or 0) > 0:
rows = select_rows(0)
wrapped = True
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
inserted = 0
inserted_bytes = 0
last_scanned = 0 if wrapped else int(cursor['last_credential_id'] or 0)
generation = int(cursor['wrap_count'] or 0) + (1 if wrapped else 0)
for row in rows:
uid = hashlib.sha256('|'.join((
'truf-keycheck-recheck-v2', service, status_scope,
str(generation), str(row['credential_id']), str(row['state_version']),
)).encode('utf-8')).hexdigest()
capacity_bytes = len(str(row['secret_text'] or row['secret_json'] or '').encode('utf-8')) + 512
if (
int(capacity['keycheck_items']) + inserted + 1 > int(queue_max_items)
or int(capacity['keycheck_bytes']) + inserted_bytes + capacity_bytes > int(queue_max_bytes)
):
break
candidate_id = self.conn.insert_returning_id(
'''INSERT INTO keycheck_candidates(
candidate_uid, credential_id, service, routed_service, secret_hash,
source, query, target, detector_name, found_at, finding_uid,
metadata_json, state,
capacity_bytes, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
ON CONFLICT(candidate_uid) DO NOTHING''',
(
uid, row['credential_id'], service, service, row['secret_hash'],
row['source'], row['query'], row['target'], row['detector_name'],
row['found_at'], row['finding_uid'],
json_dumps({
'provider_hint': service,
'recheck_of_status': row['status'],
'state_version': row['state_version'],
'recheck_generation': generation,
}),
capacity_bytes, now, now,
),
)
if candidate_id is not None:
inserted += 1
inserted_bytes += capacity_bytes
last_scanned = int(row['credential_id'])
if inserted:
self.conn.execute(
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items + ?,
keycheck_bytes = keycheck_bytes + ?, updated_at = ? WHERE id = 1''',
(inserted, inserted_bytes, now),
)
self.conn.execute(
'''UPDATE keycheck_recheck_cursors SET last_credential_id = ?,
wrap_count = wrap_count + ?, updated_at = ? WHERE cursor_key = ?''',
(last_scanned, 1 if wrapped else 0, now, cursor_key),
)
self.conn.commit()
return inserted
except Exception:
self.conn.rollback()
raise
def keycheck_service_has_work(self, service):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('keycheck work inspection requires PostgreSQL')
now = utc_now_iso()
row = self.conn.execute(
'''SELECT (
EXISTS (
SELECT 1 FROM keycheck_candidates
WHERE service = ? AND state = 'pending'
) OR EXISTS (
SELECT 1 FROM keycheck_candidates
WHERE service = ? AND state = 'deferred'
AND available_after IS NOT NULL AND available_after <= ?
) OR EXISTS (
SELECT 1 FROM keycheck_candidates
WHERE service = ? AND state = 'leased'
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
)
) AS present''',
(str(service).lower(), str(service).lower(), now, str(service).lower(), now),
).fetchone()
self.conn.commit()
return bool(row and row['present'])
def has_claimable_targets(self, source, platform, max_attempts=0):
if not self.conn:
return False
def op():
now = utc_now_iso()
max_attempts_value = max(0, int(max_attempts or 0))
row = self.conn.execute(
'''SELECT 1 AS present FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND (status = 'pending'
OR (status = 'deferred' AND available_after IS NOT NULL AND available_after <= ?)
OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (available_after IS NULL OR available_after <= ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id LIMIT 1''',
(source, platform, now, now, max_attempts_value, max_attempts_value, now),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
return bool(row)
return bool(self._safe('has_claimable_targets', op, False))
def has_claimable_targets_v2(self, source, platform, max_attempts=0):
if not self.conn:
return False
try:
now = utc_now_iso()
maximum = max(0, int(max_attempts or 0))
row = self.conn.execute(
HAS_CLAIMABLE_TARGETS_V2_SQL,
(
source, platform, now, maximum, maximum,
source, platform, now, maximum, maximum,
),
).fetchone()
if self.conn.is_postgres:
self.conn.commit()
self.last_error = ''
return bool(row and row['present'])
except Exception as exc:
self.last_error = str(exc)
self.conn.rollback()
return False
def claim_targets(
self, source, platform, limit, lease_owner=None, lease_seconds=86400,
max_attempts=0, return_rows=False, claim_batch=None,
):
if not self.conn:
return None
batch = str(claim_batch or secrets.token_urlsafe(24))
self._last_claim_expectation = None
commit_state = {'started': False}
def commit_claim():
commit_state['started'] = True
self.conn.commit()
commit_state['started'] = False
def remember_expected(owner, rows):
self._last_claim_expectation = {
'claim_batch': batch,
'lease_owner': str(owner),
'claims': [
{
'id': int(row['id']),
'lease_token': str(row['lease_token']),
}
for row in rows
],
}
def op():
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
now = utc_now_iso()
lease_until = datetime.fromtimestamp(time.time() + max(60, int(lease_seconds or 86400)), timezone.utc).isoformat(timespec='seconds')
limit_value = max(1, int(limit or 1000000))
max_attempts_value = max(0, int(max_attempts or 0))
owner = lease_owner or f'{source}:{os.getpid()}'
existing = self.conn.execute(
'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
FROM target_queue WHERE source = ? AND platform = ? AND status = 'in_progress'
AND current_result_reservation_id IS NULL
AND lease_owner = ? AND claim_batch = ? ORDER BY id''',
(source, platform, owner, batch),
).fetchall()
if existing:
remember_expected(owner, existing)
commit_claim()
return existing if return_rows else [row['target'] for row in existing]
candidate_limit = min(
CLAIM_CANDIDATE_MAX,
max(CLAIM_CANDIDATE_MIN, limit_value * 8),
)
if max_attempts_value:
if self.conn.is_postgres:
exhausted = self.conn.execute(
'''WITH candidates AS MATERIALIZED (
SELECT id FROM (
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'pending'
AND current_result_reservation_id IS NULL
AND attempts >= ?
ORDER BY id LIMIT ?)
UNION ALL
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'deferred'
AND current_result_reservation_id IS NULL
AND attempts >= ?
ORDER BY id LIMIT ?)
UNION ALL
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'in_progress'
AND current_result_reservation_id IS NULL
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
AND attempts >= ?
ORDER BY id LIMIT ?)
) AS status_candidates
ORDER BY id LIMIT ?
)
SELECT q.id FROM candidates c
JOIN target_queue q ON q.id = c.id
ORDER BY q.id FOR UPDATE OF q SKIP LOCKED''',
(
source, platform, max_attempts_value, candidate_limit,
source, platform, max_attempts_value, candidate_limit,
source, platform, now, max_attempts_value, candidate_limit,
candidate_limit,
),
).fetchall()
exhausted_ids = [row['id'] for row in exhausted]
if exhausted_ids:
placeholders = ','.join('?' for _ in exhausted_ids)
self.conn.execute(
f'''UPDATE target_queue SET status = 'failed', completed_at = COALESCE(completed_at, ?),
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
available_after = NULL, last_error = COALESCE(last_error, 'target retry attempts exhausted'), updated_at = ?
WHERE id IN ({placeholders})''',
(now, now, *exhausted_ids),
)
else:
self.conn.execute(
'''UPDATE target_queue SET status = 'failed', completed_at = COALESCE(completed_at, ?),
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
available_after = NULL, last_error = COALESCE(last_error, 'target retry attempts exhausted'), updated_at = ?
WHERE source = ? AND platform = ? AND COALESCE(attempts, 0) >= ?
AND current_result_reservation_id IS NULL
AND (status IN ('pending', 'deferred')
OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))''',
(now, now, source, platform, max_attempts_value, now),
)
if self.conn.is_postgres:
rows = self.conn.execute(
'''WITH candidates AS MATERIALIZED (
SELECT id FROM (
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'pending'
AND current_result_reservation_id IS NULL
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (available_after IS NULL OR available_after <= ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id LIMIT ?)
UNION ALL
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'deferred'
AND current_result_reservation_id IS NULL
AND available_after IS NOT NULL AND available_after <= ?
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id LIMIT ?)
UNION ALL
(SELECT id FROM target_queue
WHERE source = ? AND platform = ? AND status = 'in_progress'
AND current_result_reservation_id IS NULL
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (available_after IS NULL OR available_after <= ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id LIMIT ?)
) AS status_candidates
ORDER BY id LIMIT ?
)
SELECT q.id, q.target FROM candidates c
JOIN target_queue q ON q.id = c.id
ORDER BY q.id LIMIT ? FOR UPDATE OF q SKIP LOCKED''',
(
source, platform, max_attempts_value, max_attempts_value, now, candidate_limit,
source, platform, now, max_attempts_value, max_attempts_value, candidate_limit,
source, platform, now, max_attempts_value, max_attempts_value, now, candidate_limit,
candidate_limit, limit_value,
),
).fetchall()
else:
rows = self.conn.execute(
'''SELECT id, target
FROM target_queue
WHERE source = ? AND platform = ?
AND current_result_reservation_id IS NULL
AND (status = 'pending' OR (status = 'deferred' AND available_after IS NOT NULL AND available_after <= ?) OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))
AND (? = 0 OR COALESCE(attempts, 0) < ?)
AND (available_after IS NULL OR available_after <= ?)
AND (resolver_state IS NULL OR resolver_state = 'resolved')
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_targets experiment_target
WHERE experiment_target.target_queue_id = target_queue.id
)
ORDER BY id LIMIT ?''',
(source, platform, now, now, max_attempts_value, max_attempts_value, now, limit_value),
).fetchall()
ids = [row['id'] for row in rows]
for row_id in ids:
lease_token = secrets.token_urlsafe(32)
self.conn.execute(
'''UPDATE target_queue SET status = 'in_progress', lease_owner = ?, lease_token = ?, claim_batch = ?, leased_at = ?, lease_expires_at = ?,
attempts = COALESCE(attempts, 0) + 1,
scan_remote_modified_at = remote_modified_at, updated_at = ? WHERE id = ?''',
(owner, lease_token, batch, now, lease_until, now, row_id),
)
claimed_rows = None
if return_rows and ids:
placeholders = ','.join('?' for _ in ids)
claimed_rows = self.conn.execute(
f'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
FROM target_queue WHERE id IN ({placeholders}) ORDER BY id''',
ids,
).fetchall()
expected_rows = claimed_rows if return_rows and ids else self.conn.execute(
f'''SELECT id, lease_token FROM target_queue
WHERE claim_batch = ? AND lease_owner = ? ORDER BY id''',
(batch, owner),
).fetchall()
remember_expected(owner, expected_rows)
commit_claim()
if return_rows and ids:
return claimed_rows
return [row['target'] for row in rows]
last_error = None
for attempt in range(SQLITE_LOCK_RETRY_ATTEMPTS):
commit_state['started'] = False
try:
result = op()
self.last_error = ''
return result
except Exception as exc:
last_error = exc
self.last_error = str(exc)
try:
self.conn.rollback()
except Exception:
pass
if commit_state['started']:
raise
if not sqlite_lock_error(exc) or attempt == SQLITE_LOCK_RETRY_ATTEMPTS - 1:
raise
time.sleep(sqlite_lock_retry_delay(attempt))
raise last_error
def claim_recovery_expectation(self, claim_batch, lease_owner):
expectation = self._last_claim_expectation
if not expectation:
return None
if (
str(expectation.get('claim_batch') or '') != str(claim_batch or '')
or str(expectation.get('lease_owner') or '') != str(lease_owner or '')
):
return None
return [dict(claim) for claim in expectation.get('claims') or []]
def recover_claim_batch(self, claim_batch, lease_owner):
if not claim_batch or not lease_owner:
return []
if not self.conn and not self._reset_connection():
return []
def op():
rows = self.conn.execute(
'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
FROM target_queue WHERE status = 'in_progress' AND claim_batch = ?
AND lease_owner = ? ORDER BY id''',
(str(claim_batch), str(lease_owner)),
).fetchall()
if self.conn.is_postgres:
self.conn.commit()
return rows
rows = self._safe('recover_claim_batch', op, [])
if self.last_error:
first_error = self.last_error
if not self._reset_connection():
raise RuntimeError(f'unable to reconnect for committed claim batch recovery: {first_error}')
rows = self._safe('recover_claim_batch_after_reconnect', op, [])
if self.last_error:
raise RuntimeError(f'unable to recover committed claim batch: {self.last_error}')
return rows
def reclaim_target_leases(self, source, platform, lease_owner=None):
if not self.conn:
return 0
def op():
now = utc_now_iso()
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'pending', lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL,
lease_expires_at = NULL, updated_at = ?
WHERE source = ? AND platform = ? AND status = 'in_progress'
AND current_result_reservation_id IS NULL
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?''',
(now, source, platform, now),
)
self.conn.commit()
return int(getattr(cur, 'rowcount', 0) or 0)
return self._safe('reclaim_target_leases', op, 0)
def complete_target_queue_item(self, source, platform, target, target_scan_id=None, status='done', error=None, available_after=None, lease_owner=None, reset_attempts=False, queue_id=None, lease_token=None):
if not self.conn or queue_id is None or not lease_token:
return False
def op():
now = utc_now_iso()
completed = now if status in ('done', 'failed') else None
params = [status, target_scan_id, first_line(error, 500) if error else None, available_after, 1 if reset_attempts else 0, completed, now]
params.extend((queue_id, lease_token))
cur = self.conn.execute(
'''UPDATE target_queue SET
status = ?, target_scan_id = COALESCE(?, target_scan_id), last_error = ?,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
available_after = ?, attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
completed_at = COALESCE(?, completed_at), updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
params,
)
self.conn.commit()
return getattr(cur, 'rowcount', 0) != 0
return self._safe('complete_target_queue_item', op, False)
def renew_target_leases(self, lease_owner, lease_seconds=1800, lease_tokens=None):
tokens = [str(token) for token in (lease_tokens or []) if token]
if not self.conn or not lease_owner or not tokens:
return 0
def op():
now = utc_now_iso()
lease_until = datetime.fromtimestamp(
time.time() + max(60, int(lease_seconds or 1800)), timezone.utc
).isoformat(timespec='seconds')
placeholders = ','.join('?' for _ in tokens)
cur = self.conn.execute(
f'''UPDATE target_queue SET lease_expires_at = ?, updated_at = ?
WHERE status = 'in_progress' AND lease_owner = ?
AND lease_token IN ({placeholders})''',
(lease_until, now, lease_owner, *tokens),
)
self.conn.commit()
return int(getattr(cur, 'rowcount', 0) or 0)
return self._safe('renew_target_leases', op, 0)
def active_target_lease_tokens(self, lease_owner, lease_tokens=None):
tokens = [str(token) for token in (lease_tokens or []) if token]
if not self.conn or not lease_owner or not tokens:
return set()
def op():
placeholders = ','.join('?' for _ in tokens)
rows = self.conn.execute(
f'''SELECT lease_token FROM target_queue
WHERE status = 'in_progress' AND lease_owner = ?
AND lease_token IN ({placeholders})''',
(lease_owner, *tokens),
).fetchall()
if self.conn.is_postgres:
self.conn.commit()
return {str(row['lease_token']) for row in rows if row['lease_token']}
return self._safe('active_target_lease_tokens', op, set())
def refund_target_claims(self, claims, error='infrastructure persistence failure'):
normalized = []
for claim in claims or []:
queue_id = claim.get('id') if isinstance(claim, dict) else claim['id']
lease_token = claim.get('lease_token') if isinstance(claim, dict) else claim['lease_token']
if queue_id is None or not lease_token:
return False
item = (int(queue_id), str(lease_token))
if item not in normalized:
normalized.append(item)
if not self.conn or not normalized:
return False
def op():
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
now = utc_now_iso()
for queue_id, lease_token in normalized:
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'pending',
attempts = CASE WHEN COALESCE(attempts, 0) > 0 THEN attempts - 1 ELSE 0 END,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL,
available_after = NULL, completed_at = NULL, last_error = ?, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
(first_line(error, 500), now, queue_id, lease_token),
)
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
self.conn.rollback()
return False
self.conn.commit()
return True
return self._safe('refund_target_claims', op, False)
def refund_target_claim(self, queue_id, lease_token, error='infrastructure persistence failure'):
return self.refund_target_claims(
[{'id': queue_id, 'lease_token': lease_token}], error,
)
def refund_stopped_result_spool_claims(self, reservation, error='supervisor controlled source stop'):
claims = [dict(claim) for claim in (reservation or {}).get('claims') or []]
if not self.conn or len(claims) > 10000:
return False
if not claims:
return True
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
rows = {}
for claim in claims:
queue_id = int(claim.get('queue_id'))
row = self.conn.execute(
f'''SELECT id, status, lease_owner, lease_token, claim_batch
FROM target_queue WHERE id = ?{lock_suffix}''',
(queue_id,),
).fetchone()
if not (
row
and row['status'] == 'in_progress'
and row['lease_owner'] == claim.get('lease_owner')
and row['lease_token'] == claim.get('lease_token')
and row['claim_batch'] == claim.get('claim_batch')
and str(claim.get('claim_batch') or '') == str((reservation or {}).get('reservation_id') or '')
and str(claim.get('lease_owner') or '') == str((reservation or {}).get('owner') or '')
):
self.conn.rollback()
return False
rows[queue_id] = row
now = utc_now_iso()
for queue_id, row in rows.items():
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'pending',
attempts = CASE WHEN COALESCE(attempts, 0) > 0 THEN attempts - 1 ELSE 0 END,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL,
available_after = NULL, completed_at = NULL, last_error = ?, updated_at = ?
WHERE id = ? AND status = 'in_progress'
AND lease_owner = ? AND lease_token = ? AND claim_batch = ?''',
(
first_line(error, 500), now, queue_id,
row['lease_owner'], row['lease_token'], row['claim_batch'],
),
)
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
self.conn.rollback()
return False
self.conn.commit()
return True
except Exception:
try:
self.conn.rollback()
except Exception:
pass
raise
def claim_docker_resolutions(
self, source, limit, lease_owner, lease_seconds=300, periodic_limit=0,
periodic_only=False, allowed_queries=None,
):
if not self.conn or not source or not lease_owner:
return []
query_values = None
if allowed_queries is not None:
query_values = []
for value in allowed_queries:
query = str(value or '').strip()
if not query:
raise ValueError('Docker resolver allowed query is empty')
if query not in query_values:
query_values.append(query)
def op():
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
state = self._locked_runtime_control_state(shared=True)
if state['effective_discovery_paused']:
self.conn.commit()
return []
now = utc_now_iso()
lease_until = datetime.fromtimestamp(
time.time() + max(60, int(lease_seconds or 300)), timezone.utc
).isoformat(timespec='seconds')
lock_suffix = ' FOR UPDATE SKIP LOCKED' if self.conn.is_postgres else ''
total_limit = max(1, int(limit or 1))
query_clause = ''
query_params = ()
if query_values is not None:
if not query_values:
self.conn.commit()
return []
query_clause = ' AND query IN (' + ','.join('?' for _ in query_values) + ')'
query_params = tuple(query_values)
retry_rows = [] if periodic_only else self.conn.execute(
f'''SELECT id, target, COALESCE(resolver_attempts, 0) AS resolver_attempts_before
FROM target_queue
WHERE source = ? AND platform = 'docker' AND status = 'deferred'
AND resolver_state IN ('pending', 'retry', 'resolving')
AND resolver_due_at IS NOT NULL AND resolver_due_at <= ?{query_clause}
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_repositories member
JOIN docker_depth_experiments experiment
ON experiment.id = member.experiment_id
WHERE (member.repository_queue_id = target_queue.id
OR member.replacement_repository_queue_id = target_queue.id)
AND experiment.state IN (
'planned','holding','resolving','active','draining','held'
)
)
ORDER BY resolver_due_at, id LIMIT ?{lock_suffix}''',
(source, now, *query_params, total_limit),
).fetchall()
rows = [(row, False) for row in retry_rows]
periodic_count = min(
max(0, int(periodic_limit or 0)),
max(0, total_limit - len(rows)),
)
if periodic_count:
periodic_rows = self.conn.execute(
f'''SELECT id, target, COALESCE(resolver_attempts, 0) AS resolver_attempts_before
FROM target_queue
WHERE source = ? AND platform = 'docker' AND status = 'done'
AND resolver_state = 'resolved' AND resolver_token IS NULL
AND (resolver_due_at IS NULL OR resolver_due_at <= ?){query_clause}
AND NOT EXISTS (
SELECT 1 FROM docker_depth_experiment_repositories member
JOIN docker_depth_experiments experiment
ON experiment.id = member.experiment_id
WHERE (member.repository_queue_id = target_queue.id
OR member.replacement_repository_queue_id = target_queue.id)
AND experiment.state IN (
'planned','holding','resolving','active','draining','held'
)
)
ORDER BY COALESCE(resolver_due_at, ''), id
LIMIT ?{lock_suffix}''',
(source, now, *query_params, periodic_count),
).fetchall()
rows.extend((row, True) for row in periodic_rows)
output = []
for row, periodic in rows:
token = secrets.token_urlsafe(32)
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'resolving',
resolver_due_at = ?, available_after = ?,
resolver_token = ?, resolver_attempts = COALESCE(resolver_attempts, 0) + 1,
updated_at = ? WHERE id = ?''',
(lease_until, lease_until, token, now, row['id']),
)
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
self.conn.rollback()
return []
attempts_before = int(row['resolver_attempts_before'] or 0)
output.append({
'id': row['id'],
'target': row['target'],
'resolver_token': token,
'resolver_attempts_before': attempts_before,
'resolver_attempts': attempts_before + 1,
'periodic': periodic,
})
self.conn.commit()
return output
return self._safe('claim_docker_resolutions', op, [])
def finish_docker_resolution(
self, source, queue_id, resolver_token, tagged_targets=None, error='', complete=None,
*, retry_at=None, claim_attempt_consumed=True, refresh_interval_sec=0,
):
if not self.conn or queue_id is None or not resolver_token:
return False
tagged_targets = list(tagged_targets or [])
complete = bool(tagged_targets) if complete is None else bool(complete)
if not claim_attempt_consumed and (complete or not retry_at):
return False
def op():
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
if tagged_targets:
self._require_discovery_admission_locked()
now = utc_now_iso()
now_dt = datetime.now(timezone.utc)
retry_due = None
if retry_at:
try:
retry_dt = datetime.fromisoformat(str(retry_at).replace('Z', '+00:00'))
if retry_dt.tzinfo is None:
retry_dt = retry_dt.replace(tzinfo=timezone.utc)
retry_dt = retry_dt.astimezone(timezone.utc)
except (TypeError, ValueError):
self.conn.rollback()
return False
if retry_dt > now_dt + timedelta(seconds=3605):
self.conn.rollback()
return False
if retry_dt < now_dt:
retry_dt = now_dt
retry_due = retry_dt.isoformat(timespec='seconds')
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
row = self.conn.execute(
f'''SELECT query, resolver_attempts FROM target_queue
WHERE id = ? AND source = ? AND platform = 'docker'
AND resolver_state = 'resolving' AND resolver_token = ?{lock_suffix}''',
(queue_id, source, resolver_token),
).fetchone()
if not row:
self.conn.rollback()
return False
if tagged_targets:
for target in tagged_targets:
normalized = normalize_target(target, 'docker')
if not normalized:
continue
self.conn.execute(
'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status,
created_at, updated_at
) VALUES (?, 'docker', ?, ?, ?, 'pending', ?, ?)
ON CONFLICT(source, normalized_target) DO NOTHING''',
(source, row['query'], target, normalized, now, now),
)
if complete:
refresh_interval = max(0, int(refresh_interval_sec or 0))
next_due = (
datetime.fromtimestamp(
time.time() + refresh_interval, timezone.utc,
).isoformat(timespec='seconds')
if refresh_interval else None
)
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'done', resolver_state = 'resolved',
resolver_due_at = ?, resolver_token = NULL, resolver_attempts = 0,
available_after = NULL, last_error = NULL,
completed_at = COALESCE(completed_at, ?), updated_at = ?
WHERE id = ? AND resolver_token = ?''',
(next_due, now, now, queue_id, resolver_token),
)
elif retry_due is not None:
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'retry',
resolver_due_at = ?, resolver_token = NULL, available_after = ?,
resolver_attempts = CASE
WHEN ? = 0 AND COALESCE(resolver_attempts, 0) > 0
THEN resolver_attempts - 1 ELSE COALESCE(resolver_attempts, 0) END,
last_error = ?, updated_at = ?
WHERE id = ? AND resolver_token = ?''',
(
retry_due, retry_due, 1 if claim_attempt_consumed else 0,
first_line(error or 'Docker tag resolution deferred', 500),
now, queue_id, resolver_token,
),
)
else:
base_delay = max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600))
max_delay = max(base_delay, env_int('DOCKER_RESOLVER_RETRY_MAX_SEC', 86400))
exponent = min(16, max(0, int(row['resolver_attempts'] or 1) - 1))
delay = min(max_delay, base_delay * (2 ** exponent))
due = datetime.fromtimestamp(time.time() + delay, timezone.utc).isoformat(timespec='seconds')
cur = self.conn.execute(
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'retry',
resolver_due_at = ?, resolver_token = NULL, available_after = ?,
last_error = ?, updated_at = ? WHERE id = ? AND resolver_token = ?''',
(
due, due,
first_line(error or 'Docker tag resolution unresolved', 500),
now, queue_id, resolver_token,
),
)
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
self.conn.rollback()
return False
self.conn.commit()
return True
return self._safe('finish_docker_resolution', op, False)
def requeue_target_ids(self, queue_ids):
ids = [int(value) for value in (queue_ids or []) if value is not None]
if not self.conn or not ids:
return 0
def op():
now = utc_now_iso()
placeholders = ','.join('?' for _ in ids)
cur = self.conn.execute(
f'''UPDATE target_queue SET status = 'pending', attempts = 0, available_after = NULL,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
last_error = NULL, completed_at = NULL, updated_at = ?
WHERE id IN ({placeholders}) AND status = 'deferred' ''',
[now, *ids],
)
self.conn.commit()
return int(getattr(cur, 'rowcount', 0) or 0)
return self._safe('requeue_target_ids', op, 0)
def target_queue_item(self, source, platform, target):
if not self.conn:
return None
def op():
normalized = normalize_target(target, platform)
return self.conn.execute(
'''SELECT id, status, attempts, available_after, lease_owner, lease_token, lease_expires_at
FROM target_queue WHERE source = ? AND normalized_target = ?''',
(source, normalized),
).fetchone()
return self._safe('target_queue_item', op)
def target_queue_counts(self, source=None):
if not self.conn:
return {
'counts': {}, 'degraded': True, 'stale': True,
'reason': 'database_unavailable', 'truncated_statuses': [],
}
cache_key = str(source or '*')
cache = getattr(self, '_target_queue_counts_cache', None)
if cache is None:
cache = {}
self._target_queue_counts_cache = cache
retry_after_by_key = getattr(self, '_target_queue_counts_retry_after', None)
if retry_after_by_key is None:
retry_after_by_key = {}
self._target_queue_counts_retry_after = retry_after_by_key
now = time.monotonic()
retry_after = float(retry_after_by_key.get(cache_key) or 0)
if retry_after > now:
cached = cache.get(cache_key)
snapshot = cached or {
'counts': {},
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
'sampled_rows': 0,
'truncated_statuses': [],
}
return dict(
snapshot,
counts=dict(snapshot.get('counts') or {}),
degraded=True,
stale=True,
reason='bounded_count_retry_backoff',
retry_after_sec=max(1, min(
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC,
int(math.ceil(retry_after - now)),
)),
truncated_statuses=list(snapshot.get('truncated_statuses') or []),
)
try:
counts = {}
truncated = []
if self.conn.is_postgres:
self.conn.execute(
f'SET LOCAL statement_timeout = {TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS}'
)
for status in TARGET_QUEUE_OBSERVABILITY_STATUSES:
if source:
row = self.conn.execute(
'''SELECT COUNT(*) AS count FROM (
SELECT 1 FROM target_queue
WHERE source = ? AND status = ? LIMIT ?
) AS sampled''',
(source, status, TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT),
).fetchone()
else:
row = self.conn.execute(
'''SELECT COUNT(*) AS count FROM (
SELECT 1 FROM target_queue
WHERE status = ? LIMIT ?
) AS sampled''',
(status, TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT),
).fetchone()
count = int(row['count'] or 0)
if count:
counts[status] = count
if count >= TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT:
truncated.append(status)
if self.conn.is_postgres:
self.conn.commit()
snapshot = {
'counts': counts,
'degraded': bool(truncated),
'stale': False,
'reason': 'bounded_status_sample' if truncated else '',
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
'sampled_rows': sum(counts.values()),
'retry_after_sec': 0,
'truncated_statuses': truncated,
}
cache[cache_key] = snapshot
retry_after_by_key.pop(cache_key, None)
self.last_error = ''
return dict(snapshot, counts=dict(counts), truncated_statuses=list(truncated))
except Exception as exc:
self.last_error = str(exc)
rollback_succeeded = False
try:
self.conn.rollback()
rollback_succeeded = True
except Exception:
pass
if rollback_succeeded:
retry_after_by_key[cache_key] = (
time.monotonic() + TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC
)
cached = cache.get(cache_key)
logger.warning('Target queue observability degraded after bounded count query failure: %s', type(exc).__name__)
if cached:
return dict(
cached,
counts=dict(cached.get('counts') or {}),
degraded=True,
stale=True,
reason='bounded_count_query_failed',
retry_after_sec=(
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC if rollback_succeeded else 0
),
truncated_statuses=list(cached.get('truncated_statuses') or []),
)
return {
'counts': {},
'degraded': True,
'stale': True,
'reason': 'bounded_count_query_failed',
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
'sampled_rows': 0,
'retry_after_sec': (
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC if rollback_succeeded else 0
),
'truncated_statuses': [],
}
def admin_target_queue_health(self, sources):
if (
not isinstance(sources, (tuple, list)) or not 1 <= len(sources) <= 16
or any(type(source) is not str or not source for source in sources)
or len(set(sources)) != len(sources)
):
raise ValueError('admin queue sources are invalid')
snapshots = [self.target_queue_counts(source) for source in sources]
counts = {}
truncated = set()
for snapshot in snapshots:
for status, count in (snapshot.get('counts') or {}).items():
counts[status] = counts.get(status, 0) + int(count)
truncated.update(snapshot.get('truncated_statuses') or ())
degraded = any(snapshot.get('degraded') is True for snapshot in snapshots)
stale = any(snapshot.get('stale') is True for snapshot in snapshots)
return {
'counts': counts,
'degraded': degraded,
'stale': stale,
'reason': 'bounded_core_source_sample' if degraded else '',
'sample_limit_per_status': (
TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT * len(sources)
),
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
'sampled_rows': sum(counts.values()),
'retry_after_sec': max(
(int(snapshot.get('retry_after_sec') or 0) for snapshot in snapshots),
default=0,
),
'truncated_statuses': sorted(truncated),
}
def known_target_normalizations(self, source, platform=None):
if not self.conn:
return set()
def op():
if platform:
rows = self.conn.execute(
'''SELECT normalized_target FROM target_queue
WHERE source = ? AND platform = ?''',
(source, platform),
).fetchall()
else:
rows = self.conn.execute(
'SELECT normalized_target FROM target_queue WHERE source = ?',
(source,),
).fetchall()
return {str(row['normalized_target']) for row in rows if row['normalized_target']}
return self._safe('known_target_normalizations', op, set())
def known_target_normalizations_for(self, source, platform, targets, batch_size=KNOWN_TARGET_LOOKUP_BATCH_SIZE):
"""Return only known identities from the offered bounded discovery batch."""
if not self.conn:
return set()
normalized = []
seen = set()
for target in targets or []:
value = normalize_target(target, platform)
if value and value not in seen:
seen.add(value)
normalized.append(value)
if not normalized:
return set()
def op():
known = set()
size = max(1, min(KNOWN_TARGET_LOOKUP_BATCH_SIZE, int(batch_size or KNOWN_TARGET_LOOKUP_BATCH_SIZE)))
for start in range(0, len(normalized), size):
values = normalized[start:start + size]
rows = self.conn.execute(
'''SELECT normalized_target FROM target_queue
WHERE source = ? AND platform = ? AND normalized_target IN ({})'''.format(
','.join('?' for _ in values)
),
(source, platform, *values),
).fetchall()
known.update(str(row['normalized_target']) for row in rows if row['normalized_target'])
return known
return self._safe('known_target_normalizations_for', op, set())
def record_package_repo_candidates(self, run_id, cycle_id, query, candidates):
offered = list(candidates or [])
report = {
'offered_rows': len(offered),
'committed_rows': 0,
'skipped_rows': 0,
'failed': False,
'failed_chunk_rows': 0,
'connection_usable': bool(self.conn),
}
if not offered or not self.conn:
report['skipped_rows'] = len(offered)
report['failed'] = bool(offered)
return report
for start in range(0, len(offered), PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE):
chunk = offered[start:start + PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE]
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
failure = None
for attempt in range(attempts):
try:
now = utc_now_iso()
for candidate in chunk:
self.conn.execute(
'''INSERT INTO package_repo_candidates (
package_source, package_name, package_version, query, repo_url, provider,
evidence_json, confidence, first_seen_at, last_seen_at, last_run_id, last_cycle_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(package_source, package_name, package_version, repo_url)
DO UPDATE SET
query = excluded.query,
provider = excluded.provider,
evidence_json = excluded.evidence_json,
confidence = excluded.confidence,
last_seen_at = excluded.last_seen_at,
last_run_id = excluded.last_run_id,
last_cycle_id = excluded.last_cycle_id''',
(
candidate.get('package_source'),
candidate.get('name') or candidate.get('package_name'),
candidate.get('version') or candidate.get('package_version') or '',
query,
candidate.get('repo_url'),
candidate.get('provider'),
json_dumps(candidate.get('evidence') or []),
candidate.get('confidence') or 'medium',
now,
now,
run_id,
cycle_id,
),
)
self.conn.commit()
report['committed_rows'] += len(chunk)
failure = None
break
except Exception as exc:
failure = exc
try:
self.conn.rollback()
except Exception:
report['connection_usable'] = self._reset_connection()
if (
self.conn
and self.conn.is_sqlite
and sqlite_lock_error(exc)
and attempt < attempts - 1
):
time.sleep(sqlite_lock_retry_delay(attempt))
continue
break
if failure is not None:
self.last_error = str(failure)
report.update({
'skipped_rows': len(offered) - report['committed_rows'],
'failed': True,
'failed_chunk_rows': len(chunk),
'connection_usable': bool(self.conn) and report['connection_usable'],
})
logger.warning(
'Optional package candidate cache write stopped after a failed bounded chunk: '
'committed=%s skipped=%s chunk_rows=%s error=%s',
report['committed_rows'], report['skipped_rows'], len(chunk), type(failure).__name__,
)
return report
self.last_error = ''
return report
def _bounded_package_candidate_rows(self, query, sources, limit):
scan_limit = min(
PACKAGE_CANDIDATE_SCAN_MAX_ROWS,
max(5000, int(limit) * 4),
)
source_set = set(sources)
exact = []
if query:
scope = PACKAGE_REPO_NONEMPTY_PREDICATE
source_params = []
if sources:
scope += ' AND package_source IN ({})'.format(','.join('?' for _ in sources))
source_params.extend(sources)
try:
if self.conn.is_postgres:
self.conn.execute('SET LOCAL enable_seqscan = off')
exact = self.conn.execute(
f'''SELECT id, package_source, package_name AS name,
package_version AS version, repo_url, provider,
evidence_json, confidence, last_seen_at
FROM package_repo_candidates
WHERE {scope} AND query = ?
ORDER BY last_seen_at DESC, id DESC LIMIT ?''',
(*source_params, query, limit),
).fetchall()
except Exception as exc:
if self.conn.is_postgres:
self.conn.rollback()
logger.warning(
'Exact package candidate branch failed; bounded substring results remain available: %s',
type(exc).__name__,
)
matched_ids = []
examined = 0
cursor_seen = None
cursor_id = None
more_rows = False
while examined < scan_limit and len(matched_ids) < limit:
page_limit = min(
PACKAGE_CANDIDATE_SCAN_PAGE_SIZE,
scan_limit - examined,
)
cursor_clause = ''
params = []
if cursor_seen is not None and cursor_id is not None:
cursor_clause = 'AND (last_seen_at, id) < (?, ?)'
params.extend((cursor_seen, cursor_id))
params.append(page_limit)
page = self.conn.execute(
f'''SELECT id, package_source, package_name, query, last_seen_at
FROM package_repo_candidates
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE} {cursor_clause}
ORDER BY last_seen_at DESC, id DESC
LIMIT ?''',
params,
).fetchall()
if not page:
more_rows = False
break
examined += len(page)
for row in page:
if source_set and str(row['package_source'] or '').lower() not in source_set:
continue
if not query or (
str(row['query'] or '') != query
and query in str(row['package_name'] or '')
):
matched_ids.append(int(row['id']))
if len(matched_ids) >= limit:
break
cursor_seen = page[-1]['last_seen_at']
cursor_id = page[-1]['id']
more_rows = len(page) == page_limit
if len(page) < page_limit:
break
if len(matched_ids) < limit and examined >= scan_limit and more_rows:
probe = self.conn.execute(
f'''SELECT 1 FROM package_repo_candidates
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE}
AND (last_seen_at, id) < (?, ?)
ORDER BY last_seen_at DESC, id DESC
LIMIT 1''',
(cursor_seen, cursor_id),
).fetchone()
if probe:
logger.warning(
'Package candidate substring lookup reached its bounded %s-row metadata scan; '
'older candidates were not examined',
scan_limit,
)
rows_by_id = {}
for start in range(0, len(matched_ids), 64):
batch = matched_ids[start:start + 64]
placeholders = ','.join('?' for _ in batch)
rows = self.conn.execute(
f'''SELECT id, package_source, package_name AS name, package_version AS version,
repo_url, provider, evidence_json, confidence, last_seen_at
FROM package_repo_candidates WHERE id IN ({placeholders})''',
batch,
).fetchall()
rows_by_id.update({int(row['id']): row for row in rows})
combined = {
int(row['id']): row
for row in (*exact, *(rows_by_id[row_id] for row_id in matched_ids if row_id in rows_by_id))
}
return sorted(
combined.values(),
key=lambda row: (str(row['last_seen_at'] or ''), int(row['id'])),
reverse=True,
)[:limit]
def get_package_repo_candidates(self, query=None, package_sources=None, limit=1000):
if not self.conn:
return []
def op():
requested_limit = max(1, int(limit or 1000))
limit_value = min(PACKAGE_CANDIDATE_LOOKUP_MAX_RESULTS, requested_limit)
if requested_limit > limit_value:
logger.warning(
'Package candidate lookup capped requested result limit from %s to %s',
requested_limit, limit_value,
)
sources = [item.strip().lower() for item in (package_sources or []) if item.strip()]
search = str(query or '')
rows = self._bounded_package_candidate_rows(
search, sources, limit_value,
)
candidates = []
for row in rows:
try:
evidence = json.loads(row['evidence_json'] or '[]')
except json.JSONDecodeError:
evidence = []
candidates.append({
'source': 'package_git',
'package_source': row['package_source'],
'name': row['name'],
'version': row['version'],
'repo_url': row['repo_url'],
'provider': row['provider'],
'evidence': evidence,
'confidence': row['confidence'],
})
return candidates
return self._safe('get_package_repo_candidates', op, [])
@staticmethod
def _compat_payload(finding, max_bytes=16 * 1024 * 1024):
mapped = {
'DetectorName', 'DetectorType', 'Verified', 'Raw', 'RawV2', 'Redacted',
'StructuredData', 'ExtraData', 'AnalysisInfo', 'finding_uid',
}
extension = {key: value for key, value in finding.items() if key not in mapped}
payload = {
'raw_value': finding.get('Raw'),
'raw_v2_value': finding.get('RawV2'),
'structured_data_json': json_dumps(finding.get('StructuredData')) if finding.get('StructuredData') is not None else None,
'extra_data_json': json_dumps(finding.get('ExtraData')) if finding.get('ExtraData') is not None else None,
'analysis_info_json': json_dumps(finding.get('AnalysisInfo')) if finding.get('AnalysisInfo') is not None else None,
'extension_json': json_dumps(extension) if extension else None,
}
encoded = json.dumps(
finding, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
omitted = len(encoded) > max(1, int(max_bytes))
if omitted:
payload = {key: None for key in payload}
payload.update({
'payload_sha256': hashlib.sha256(encoded).hexdigest(),
'payload_bytes': len(encoded),
'payload_omitted': 1 if omitted else 0,
})
return payload
def _insert_normalized_finding(
self, run_id, cycle_id, target_scan_id, source, query, target,
normalized, finding,
):
finding = sanitize_postman_finding(finding)
raw_secret, secret_hash, detector_secret_hash, fingerprint, location = finding_identity(
source, normalized, finding,
)
finding_uid = str(finding.get('finding_uid') or fingerprint)
enrichment = enrich_finding(finding)
compat = self._compat_payload(finding)
finding_id = self.conn.insert_returning_id(
'''INSERT INTO findings(
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
detector_name, detector_type, verified, raw_secret, redacted_secret, secret_hash,
detector_secret_hash, finding_fingerprint, finding_uid, file_path, line_number,
commit_hash, source_timestamp, source_metadata_type, source_metadata_json,
raw_finding_json, provider, credential_kind, credential_confidence,
required_context_missing, principal, username, email, project_id, tenant_id,
organization, registry, endpoint, scope, resource, enrichment_json,
raw_payload_sha256, raw_payload_bytes, raw_payload_omitted, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
run_id, cycle_id, target_scan_id, source, query, target, normalized,
finding.get('DetectorName'), finding.get('DetectorType'),
1 if finding.get('Verified', False) else 0, raw_secret,
extract_redacted_secret(finding), secret_hash, detector_secret_hash,
fingerprint, finding_uid, location.get('file_path'), location.get('line_number'),
location.get('commit_hash'), location.get('source_timestamp'),
location.get('source_metadata_type'), location.get('source_metadata_json'),
enrichment.get('provider'), enrichment.get('credential_kind'),
enrichment.get('credential_confidence'), enrichment.get('required_context_missing'),
enrichment.get('principal'), enrichment.get('username'), enrichment.get('email'),
enrichment.get('project_id'), enrichment.get('tenant_id'),
enrichment.get('organization'), enrichment.get('registry'),
enrichment.get('endpoint'), enrichment.get('scope'), enrichment.get('resource'),
enrichment.get('enrichment_json'), compat['payload_sha256'],
compat['payload_bytes'], compat['payload_omitted'], utc_now_iso(),
),
)
if not finding_id:
raise RuntimeError('normalized finding insert returned no identity')
self.conn.execute(
'''INSERT INTO finding_compat_payloads(
finding_id, raw_value, raw_v2_value, structured_data_json,
extra_data_json, analysis_info_json, extension_json,
payload_sha256, payload_bytes, payload_omitted, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
finding_id, compat['raw_value'], compat['raw_v2_value'],
compat['structured_data_json'], compat['extra_data_json'],
compat['analysis_info_json'], compat['extension_json'],
compat['payload_sha256'], compat['payload_bytes'], compat['payload_omitted'],
utc_now_iso(),
),
)
self.conn.execute(
'''INSERT INTO finding_uid_map(finding_uid, finding_id, created_at)
VALUES (?, ?, ?) ON CONFLICT(finding_uid) DO NOTHING''',
(finding_uid, finding_id, utc_now_iso()),
)
return finding_id, finding_uid
@staticmethod
def _docker_finding_layer_evidence(finding, manifest_digest):
metadata = finding.get('SourceMetadata')
data = metadata.get('Data') if isinstance(metadata, dict) else None
if not isinstance(data, dict):
return None, 'digest_absent'
docker = data.get('Docker')
if 'Docker' in data and not isinstance(docker, dict):
return None, 'digest_invalid'
if isinstance(docker, dict) and 'layer' in docker:
digest = docker.get('layer')
if not isinstance(digest, str) or not re.fullmatch(
r'sha256:[0-9a-f]{64}', digest):
return None, 'digest_invalid'
return digest, None
if 'DockerContent' not in data:
return None, 'digest_absent'
docker_content = data.get('DockerContent')
if not isinstance(docker_content, dict):
return None, 'digest_invalid'
if docker_content.get('descriptor_kind') != 'layer':
return None, 'digest_absent'
digest = docker_content.get('blob_digest')
reported_manifest = docker_content.get('manifest_digest')
if (
not isinstance(digest, str)
or not re.fullmatch(r'sha256:[0-9a-f]{64}', digest)
or not isinstance(reported_manifest, str)
or not re.fullmatch(r'sha256:[0-9a-f]{64}', reported_manifest)
):
return None, 'digest_invalid'
if reported_manifest != manifest_digest:
return None, 'manifest_mismatch'
return digest, None
def _insert_docker_finding_layer_attributions_locked(
self, binding, target_scan_id, finding_id, finding,
):
if not binding:
return 0
if binding['target_scan_id'] is not None and int(
binding['target_scan_id']) != int(target_scan_id):
raise ScanEventConflictError(
'Docker finding attribution binding has a conflicting scan identity'
)
finding_row = self.conn.execute(
'''SELECT finding.id
FROM findings finding
JOIN target_scans scan ON scan.id = finding.target_scan_id
WHERE finding.id = ? AND finding.target_scan_id = ?
AND scan.queue_id = ? AND scan.result_reservation_id = ?
FOR UPDATE OF finding, scan''',
(
int(finding_id), int(target_scan_id),
int(binding['target_queue_id']), int(binding['reservation_id']),
),
).fetchone()
if not finding_row:
raise ScanEventConflictError(
'Docker finding attribution lost its exact reservation scan identity'
)
digest, reason = self._docker_finding_layer_evidence(
finding, str(binding['manifest_digest']),
)
layers = []
if digest is not None:
layers = self.conn.execute(
'''SELECT id, position_from_base, position_from_top, layer_digest
FROM docker_manifest_layers
WHERE manifest_id = ? AND layer_digest = ?
ORDER BY position_from_base, id FOR UPDATE''',
(int(binding['manifest_id']), digest),
).fetchall()
if not layers:
digest = None
reason = 'digest_not_in_manifest'
if digest is None:
expected_rows = ((
int(binding['id']), None, 'unattributed', None, reason, None, None,
),)
else:
expected_rows = tuple(
(
int(binding['id']), int(layer['id']), 'exact',
str(layer['layer_digest']), None,
int(layer['position_from_base']), int(layer['position_from_top']),
)
for layer in layers
)
now = utc_now_iso()
for row in expected_rows:
self.conn.execute(
'''INSERT INTO docker_finding_layer_attributions(
scan_binding_id, finding_id, manifest_layer_id,
attribution_state, reported_layer_digest,
unattributed_reason, position_from_base,
position_from_top, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT DO NOTHING''',
(row[0], int(finding_id), *row[1:], now),
)
actual_rows = self.conn.execute(
'''SELECT scan_binding_id, manifest_layer_id, attribution_state,
reported_layer_digest, unattributed_reason,
position_from_base, position_from_top
FROM docker_finding_layer_attributions
WHERE finding_id = ? ORDER BY id FOR UPDATE''',
(int(finding_id),),
).fetchall()
actual = {
(
int(row['scan_binding_id']),
int(row['manifest_layer_id']) if row['manifest_layer_id'] is not None else None,
str(row['attribution_state']), row['reported_layer_digest'],
row['unattributed_reason'],
int(row['position_from_base']) if row['position_from_base'] is not None else None,
int(row['position_from_top']) if row['position_from_top'] is not None else None,
)
for row in actual_rows
}
expected = set(expected_rows)
if len(actual_rows) != len(expected_rows) or actual != expected:
raise ScanEventConflictError(
'Docker finding layer attribution conflicts with persisted evidence'
)
return len(expected_rows)
def _precompute_docker_finding_attribution_rows_locked(self, reader, binding):
if not binding:
return 0
layers = self.conn.execute(
'''SELECT id, layer_digest, position_from_base, position_from_top
FROM docker_manifest_layers
WHERE manifest_id = ?
ORDER BY position_from_base, id FOR UPDATE''',
(int(binding['manifest_id']),),
).fetchall()
if len(layers) != int(binding['layer_count']):
raise ScanEventConflictError(
'Docker attribution manifest layer authority is incomplete'
)
positions_by_digest = {}
for layer in layers:
positions_by_digest[str(layer['layer_digest'])] = (
positions_by_digest.get(str(layer['layer_digest']), 0) + 1
)
projected_rows = 0
finding_count = 0
for finding in reader.iter_findings():
finding_count += 1
digest, _reason = self._docker_finding_layer_evidence(
finding, str(binding['manifest_digest']),
)
projected_rows += max(1, positions_by_digest.get(digest, 0))
if projected_rows > DOCKER_DEPTH_MAX_ATTRIBUTION_ROWS_PER_BUNDLE:
raise DockerFindingAttributionLimitError(
'Docker finding attribution exceeds its strict per-bundle row limit'
)
if finding_count != int(reader.validate().finding_count):
raise ScanEventConflictError(
'Docker attribution finding count conflicts with its bundle'
)
return projected_rows
def _existing_normalized_finding_for_replay_locked(
self, target_scan_id, source, normalized, finding,
):
finding = sanitize_postman_finding(finding)
_, _, _, fingerprint, _ = finding_identity(source, normalized, finding)
finding_uid = str(finding.get('finding_uid') or fingerprint)
payload = self._compat_payload(finding)
rows = self.conn.execute(
'''SELECT id, finding_fingerprint, raw_payload_sha256
FROM findings
WHERE target_scan_id = ? AND finding_uid = ?
ORDER BY id LIMIT 2 FOR UPDATE''',
(int(target_scan_id), finding_uid),
).fetchall()
if len(rows) != 1 or (
str(rows[0]['finding_fingerprint']) != fingerprint
or str(rows[0]['raw_payload_sha256']) != payload['payload_sha256']
):
raise ScanEventConflictError(
'replayed Docker finding identity conflicts with persisted evidence'
)
return int(rows[0]['id']), finding_uid
def _insert_bundle_candidate(
self, frame, reservation, target_scan_id, finding_ids,
capacity_allocation=None, routed_service_supported=True,
):
if not isinstance(frame, dict):
raise ValueError('keycheck candidate frame must be an object')
service = str(frame.get('service') or '').lower()
credential_hash = str(frame.get('credential_hash') or '').lower()
provider_key_hash = str(frame.get('provider_key_hash') or '').lower()
secret_hash = str(frame.get('secret_hash') or '').lower()
if (
not service
or not re.fullmatch(r'[a-f0-9]{64}', credential_hash)
or not re.fullmatch(r'[a-f0-9]{64}', provider_key_hash)
or not re.fullmatch(r'[a-f0-9]{64}', secret_hash)
):
raise ValueError('keycheck candidate identity is invalid')
secret_text = frame.get('secret_text')
secret_json = frame.get('secret_json')
if (secret_text is None) == (secret_json is None):
raise ValueError('keycheck candidate must contain exactly one secret representation')
metadata = frame.get('metadata') if isinstance(frame.get('metadata'), dict) else {}
attribution = frame.get('attribution') if isinstance(frame.get('attribution'), dict) else {}
now = utc_now_iso()
credential_id = self.conn.insert_returning_id(
'''INSERT INTO keycheck_credentials(
service, credential_hash, provider_key_hash,
candidate_kind, secret_text, secret_json,
key_masked, endpoint, principal, metadata_json, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(service, provider_key_hash) DO NOTHING''',
(
service, credential_hash, provider_key_hash,
str(frame.get('candidate_kind') or 'raw'),
secret_text, secret_json, str(frame.get('key_masked') or ''),
str(frame.get('endpoint') or ''), str(frame.get('principal') or ''),
json_dumps(metadata), now, now,
),
)
if credential_id is None:
credential = self.conn.execute(
'''SELECT id, provider_key_hash, secret_text, secret_json
FROM keycheck_credentials
WHERE service = ? AND provider_key_hash = ?''',
(service, provider_key_hash),
).fetchone()
if not credential or (
credential['provider_key_hash'] != provider_key_hash
or credential['secret_text'] != secret_text
or credential['secret_json'] != secret_json
):
raise ScanEventConflictError('provider credential identity resolves to conflicting secret material')
credential_id = credential['id']
finding_uid_value = str(attribution.get('finding_uid') or metadata.get('finding_uid') or '')
finding_id = finding_ids.get(finding_uid_value)
origin = finding_uid_value or str(attribution.get('origin') or frame.get('candidate_kind') or 'structured')
uid = candidate_uid(reservation['scan_event_id'], origin, service, credential_hash)
serialized_bytes = len(json.dumps(
frame, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8'))
capacity_bytes = max(serialized_bytes, int(capacity_allocation or serialized_bytes))
if routed_service_supported:
candidate_sql = '''INSERT INTO keycheck_candidates(
candidate_uid, credential_id, service, routed_service, secret_hash,
finding_id, target_scan_id,
scan_event_id, finding_uid, source, query, target, detector_name,
found_at, metadata_json, state, capacity_bytes, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
ON CONFLICT(candidate_uid) DO NOTHING'''
candidate_values = (
uid, credential_id, service, service, secret_hash, finding_id, target_scan_id,
reservation['scan_event_id'], finding_uid_value, reservation['source'],
reservation['query'], reservation['target'], metadata.get('detector_name'),
now, json_dumps(metadata), capacity_bytes, now, now,
)
else:
candidate_sql = '''INSERT INTO keycheck_candidates(
candidate_uid, credential_id, service, secret_hash,
finding_id, target_scan_id,
scan_event_id, finding_uid, source, query, target, detector_name,
found_at, metadata_json, state, capacity_bytes, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
ON CONFLICT(candidate_uid) DO NOTHING'''
candidate_values = (
uid, credential_id, service, secret_hash, finding_id, target_scan_id,
reservation['scan_event_id'], finding_uid_value, reservation['source'],
reservation['query'], reservation['target'], metadata.get('detector_name'),
now, json_dumps(metadata), capacity_bytes, now, now,
)
candidate_id = self.conn.insert_returning_id(candidate_sql, candidate_values)
return (1, capacity_bytes) if candidate_id is not None else (0, 0)
def ingest_result_bundle(self, reader, reservation, bundle_row):
if not self.conn or not self.conn.is_postgres:
raise RuntimeError('normalized result bundle ingestion requires PostgreSQL')
validated = reader.validate()
reservation_id = int(reservation['id'] if 'id' in reservation else reservation['reservation_id'])
event_id = str(validated.scan_event_id)
event_hash = str(validated.scan_event_hash)
metadata = reader.metadata()
effective_diagnostics = reader.effective_diagnostics()
contracts = importlib.import_module('worker_contracts')
diagnostic_uid_set_sha256 = contracts.ordered_diagnostic_uid_set_sha256(
effective_diagnostics
)
if any(key in metadata for key in ('findings', 'errors')):
raise ValueError('bundle metadata must not duplicate findings or errors')
metadata_bytes = json.dumps(
metadata, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
).encode('utf-8')
if len(metadata_bytes) > 16 * 1024 * 1024:
raise ValueError('scan compatibility metadata exceeds its byte bound')
routed_service_supported = (
'routed_service' in self.conn.table_columns('keycheck_candidates')
)
try:
self._lock_docker_depth_experiment_for_reservation(reservation_id)
current_reservation = self.conn.execute(
'''SELECT r.*, q.id AS bound_queue_id,
q.source AS bound_queue_source,
q.platform AS bound_queue_platform,
q.query AS bound_queue_query,
q.target AS bound_queue_target,
q.normalized_target AS bound_queue_normalized_target,
q.status AS bound_queue_status,
q.lease_token AS bound_queue_lease_token,
q.current_result_reservation_id AS bound_queue_reservation_id,
q.claim_event_id AS bound_queue_event_id
FROM result_reservations r
JOIN target_queue q ON q.id = r.queue_id
WHERE r.id = ? FOR UPDATE OF r, q''',
(reservation_id,),
).fetchone()
current_bundle = self.conn.execute(
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
(reservation_id,),
).fetchone()
if not current_reservation or not current_bundle:
raise ValueError('bundle reservation state is absent')
self._validate_result_ingestion_arguments(
current_reservation, current_bundle, reservation, bundle_row,
)
queue_identity = {
'id': current_reservation['bound_queue_id'],
'source': current_reservation['bound_queue_source'],
'platform': current_reservation['bound_queue_platform'],
'query': current_reservation['bound_queue_query'],
'target': current_reservation['bound_queue_target'],
'normalized_target': current_reservation[
'bound_queue_normalized_target'
],
}
self._validate_locked_reservation_queue_identity(
current_reservation, queue_identity,
)
validated_identity = validated.as_dict()
validated_identity['relative_path'] = current_bundle['relative_path']
self._validate_result_ingestion_arguments(
current_reservation, current_bundle,
current_reservation, validated_identity,
)
self._validate_result_bundle_reservation_identity(
current_reservation, validated_identity, require_header=True,
)
if str(current_reservation['assignment_kind']) == 'remote':
validate_remote_result_execution_plan(
current_reservation, metadata,
)
projection_version = current_reservation[
'remote_diagnostic_projection_version'
]
if int(projection_version or 0) == 0:
cursor = self.conn.execute(
'''UPDATE result_reservations
SET remote_diagnostic_projection_version = 1,
remote_diagnostic_count = ?,
remote_diagnostic_uids_sha256 = ?, updated_at = ?
WHERE id = ? AND remote_resolution_kind = 'bundle_accepted'
AND COALESCE(remote_diagnostic_projection_version, 0) = 0''',
(
len(effective_diagnostics),
diagnostic_uid_set_sha256, utc_now_iso(), reservation_id,
),
)
if int(cursor.rowcount or 0) != 1:
raise ScanEventConflictError(
'legacy accepted diagnostic projection fence changed'
)
elif (
int(projection_version) != contracts.DIAGNOSTIC_PROJECTION_VERSION
or int(current_reservation['remote_diagnostic_count'] or 0)
!= len(effective_diagnostics)
or str(current_reservation['remote_diagnostic_uids_sha256'] or '')
!= diagnostic_uid_set_sha256
):
raise ScanEventConflictError(
'accepted diagnostic projection differs from receipt authority'
)
if str(current_bundle['scan_event_hash']) != event_hash:
raise ScanEventConflictError(
'validated bundle identity conflicts with its database reservation'
)
docker_depth_binding = self._docker_depth_binding_for_reservation_locked(
current_reservation,
)
self._precompute_docker_finding_attribution_rows_locked(
reader, docker_depth_binding,
)
existing = self.conn.execute(
'''SELECT id, scan_event_hash, result_reservation_id, queue_id,
claim_lease_token, source, query, target,
normalized_target, scan_type, raw_result_storage,
compat_schema_version
FROM target_scans WHERE scan_event_id = ? FOR UPDATE''',
(event_id,),
).fetchone()
if existing:
if str(existing['scan_event_hash']) != event_hash:
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
if int(existing['result_reservation_id'] or 0) != reservation_id:
raise ScanEventConflictError('scan event replay belongs to a different reservation')
if (
int(existing['queue_id'] or 0)
!= int(current_reservation['queue_id'])
or str(existing['claim_lease_token'] or '')
!= str(current_reservation['claim_lease_token'])
or str(existing['source'] or '')
!= str(current_reservation['source'])
or str(existing['query'] or '')
!= str(current_reservation['query'] or '')
or str(existing['target'] or '')
!= str(current_reservation['target'])
or str(existing['normalized_target'] or '')
!= str(current_reservation['normalized_target'])
or str(existing['scan_type'] or '')
!= str(current_reservation['platform'])
or str(existing['raw_result_storage'] or '') != 'normalized_v2'
or int(existing['compat_schema_version'] or 0) != 2
):
raise ScanEventConflictError(
'authoritative scan replay lost its exact reservation identity'
)
if docker_depth_binding:
if (
docker_depth_binding['target_scan_id'] is None
or int(docker_depth_binding['target_scan_id']) != int(existing['id'])
):
raise ScanEventConflictError(
'Docker depth binding scan identity conflicts with replay'
)
replay_finding_ids = {}
for finding in reader.iter_findings():
finding_id, finding_uid_value = (
self._existing_normalized_finding_for_replay_locked(
existing['id'], str(current_reservation['source']),
str(current_reservation['normalized_target']), finding,
)
)
replay_finding_ids[finding_uid_value] = finding_id
self._insert_docker_finding_layer_attributions_locked(
docker_depth_binding, existing['id'], finding_id, finding,
)
if len(replay_finding_ids) != validated.finding_count:
raise ScanEventConflictError(
'replayed Docker finding identities are missing or duplicated'
)
diagnostic_received_at = utc_now_iso()
for diagnostic in effective_diagnostics:
self._record_worker_diagnostic(
reservation_id, diagnostic,
target_scan_id=int(existing['id']),
received_at=diagnostic_received_at,
)
self.conn.execute(
"UPDATE result_bundles SET state = 'db_committed', target_scan_id = ?, updated_at = ? WHERE reservation_id = ?",
(existing['id'], utc_now_iso(), reservation_id),
)
self.conn.execute(
"UPDATE result_reservations SET state = 'db_committed', updated_at = ? WHERE id = ?",
(utc_now_iso(), reservation_id),
)
self.conn.commit()
return {
'ingested': True, 'duplicate': True, 'target_scan_id': existing['id'],
'scan_event_id': event_id, 'scan_event_hash': event_hash,
}
if current_reservation['state'] not in ('ready', 'ingesting'):
raise RuntimeError('bundle reservation is not ingestible')
if current_bundle['state'] not in ('ready', 'ingesting'):
raise RuntimeError('bundle row is not ingestible')
if (
str(current_reservation['bound_queue_status']) != 'in_progress'
or str(current_reservation['bound_queue_lease_token'] or '')
!= str(current_reservation['claim_lease_token'])
or int(current_reservation['bound_queue_reservation_id'] or 0)
!= reservation_id
or str(current_reservation['bound_queue_event_id'] or '') != event_id
):
raise ScanEventConflictError(
'result ingestion lost its exact target queue fence'
)
source = str(current_reservation['source'])
target = str(current_reservation['target'])
normalized = str(current_reservation['normalized_target'])
run_id = current_reservation['run_id']
cycle_id = current_reservation['cycle_id']
query = current_reservation['query']
package = extract_package_metadata(target, metadata, source)
status = str(metadata.get('status') or 'clean')
timestamp = str(metadata.get('timestamp') or utc_now_iso())
target_scan_id = self.conn.insert_returning_id(
'''INSERT INTO target_scans(
scan_event_id, scan_event_hash, queue_id, claim_lease_token,
queue_completion_applied, queue_completion_disposition,
run_id, cycle_id, source, query, target, normalized_target, scan_type,
status, started_at, ended_at, duration_sec, scan_options_json,
package_name, package_version, package_artifact, package_date,
package_filename, package_type, package_size, findings_count,
verified_findings_count, error_count, skipped_reason, first_error_summary,
raw_result_json, result_reservation_id, compat_schema_version,
raw_result_storage, created_at
) VALUES (?, ?, ?, ?, 0, 'pending', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, 2, 'normalized_v2', ?)''',
(
event_id, event_hash, current_reservation['queue_id'],
current_reservation['claim_lease_token'], run_id, cycle_id, source, query,
target, normalized, metadata.get('scan_type'), status,
metadata.get('scan_started_at') or timestamp, timestamp,
metadata.get('duration_sec'), json_dumps(redact_config(metadata.get('scan_options') or {})),
package.get('package_name'), package.get('package_version'),
package.get('package_artifact'), package.get('package_date'),
package.get('package_filename'), package.get('package_type'),
package.get('package_size'), validated.finding_count,
int(metadata.get('verified_findings_count') or 0), validated.error_count,
metadata.get('skipped'), metadata.get('first_error_summary'),
reservation_id, utc_now_iso(),
),
)
if not target_scan_id:
raise RuntimeError('normalized target scan insert returned no identity')
self.conn.execute(
'''INSERT INTO scan_result_compat(
target_scan_id, schema_version, metadata_json, metadata_sha256,
metadata_bytes, reconstruction_status, created_at
) VALUES (?, 2, ?, ?, ?, 'bounded', ?)''',
(
target_scan_id, metadata_bytes.decode('utf-8'),
hashlib.sha256(metadata_bytes).hexdigest(), len(metadata_bytes), utc_now_iso(),
),
)
diagnostic_received_at = utc_now_iso()
for diagnostic in effective_diagnostics:
self._record_worker_diagnostic(
reservation_id, diagnostic,
target_scan_id=target_scan_id,
received_at=diagnostic_received_at,
)
finding_ids = {}
verified_count = 0
for finding in reader.iter_findings():
finding_id, finding_uid_value = self._insert_normalized_finding(
run_id, cycle_id, target_scan_id, source, query, target, normalized, finding,
)
finding_ids[finding_uid_value] = finding_id
self._insert_docker_finding_layer_attributions_locked(
docker_depth_binding, target_scan_id, finding_id, finding,
)
verified_count += int(bool(finding.get('Verified')))
if len(finding_ids) != validated.finding_count:
raise ValueError('bundle finding identities are missing or duplicated')
for error in reader.iter_errors():
self._insert_error(
run_id, cycle_id, target_scan_id, source, query, target, normalized, error,
)
actual_candidate_items = 0
actual_candidate_bytes = 0
per_candidate_capacity = (
int(current_reservation['reserved_candidate_bytes']) // validated.candidate_count
if validated.candidate_count else 0
)
for frame in reader.iter_candidates():
inserted_items, inserted_bytes = self._insert_bundle_candidate(
frame, current_reservation, target_scan_id, finding_ids,
capacity_allocation=per_candidate_capacity,
routed_service_supported=routed_service_supported,
)
actual_candidate_items += inserted_items
actual_candidate_bytes += inserted_bytes
reserved_candidate_items = int(current_reservation['reserved_candidate_items'])
reserved_candidate_bytes = int(current_reservation['reserved_candidate_bytes'])
if actual_candidate_items > reserved_candidate_items or actual_candidate_bytes > reserved_candidate_bytes:
raise ValueError('bundle candidates exceed their pre-reserved capacity')
now = utc_now_iso()
queue_status = str(metadata.get('queue_status') or ('failed' if status == 'error' else 'done'))
if queue_status not in ('done', 'failed', 'deferred'):
raise ValueError('bundle queue disposition is invalid')
self._apply_docker_layer_execution_locked(
current_reservation, metadata, queue_status, now,
)
advance_git_coverage, covered_ref, covered_head = matching_git_coverage(
current_reservation, metadata, queue_status, validated.error_count,
)
completed_at = now if queue_status in ('done', 'failed') else None
queue_cursor = self.conn.execute(
'''UPDATE target_queue SET status = ?, target_scan_id = ?, last_error = ?,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
leased_at = NULL, lease_expires_at = NULL, available_after = ?,
attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
completed_at = COALESCE(?, completed_at),
covered_ref = CASE WHEN ? != 0 THEN ? ELSE covered_ref END,
covered_head = CASE WHEN ? != 0 THEN ? ELSE covered_head END,
current_result_reservation_id = NULL, claim_event_id = NULL, updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
AND current_result_reservation_id = ? AND claim_event_id = ?''',
(
queue_status, target_scan_id, first_line(metadata.get('queue_error'), 500),
metadata.get('available_after'), 1 if metadata.get('reset_attempts') else 0,
completed_at,
1 if advance_git_coverage else 0, covered_ref,
1 if advance_git_coverage else 0, covered_head,
now, current_reservation['queue_id'],
current_reservation['claim_lease_token'], reservation_id, event_id,
),
)
if int(queue_cursor.rowcount or 0) != 1:
raise ScanEventConflictError('fenced target queue completion did not match the reservation')
self.conn.execute(
'''UPDATE target_scans SET queue_completion_applied = 1,
queue_completion_disposition = 'applied',
verified_findings_count = ? WHERE id = ?''',
(verified_count, target_scan_id),
)
self._transition_docker_depth_binding_locked(
current_reservation,
'completed' if queue_status == 'done' else 'failed',
queue_status if queue_status in ('done', 'failed') else 'pending',
now,
target_scan_id=target_scan_id,
)
self._insert_derived_postman_targets(metadata.get('derived_postman_targets'), now)
projection_job_id = self.conn.insert_returning_id(
'''INSERT INTO projection_jobs(
job_kind, event_id, event_hash, target_scan_id, status,
required_stream_mask, capacity_items, capacity_bytes,
created_at, updated_at
) VALUES ('scan_event', ?, ?, ?, 'pending', ?, ?, ?, ?, ?)''',
(
event_id, event_hash, target_scan_id,
1 | (2 if validated.finding_count else 0) | (4 if validated.error_count else 0),
current_reservation['reserved_projection_items'],
current_reservation['reserved_projection_bytes'], now, now,
),
)
if not projection_job_id:
raise RuntimeError('projection job insert returned no identity')
unused_items = reserved_candidate_items - actual_candidate_items
unused_bytes = reserved_candidate_bytes - actual_candidate_bytes
# Workload rows are complete before taking the global accounting lock.
capacity = self.conn.execute(
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
).fetchone()
if int(capacity['keycheck_items']) < unused_items or int(capacity['keycheck_bytes']) < unused_bytes:
raise RuntimeError('candidate capacity transfer would make accounting negative')
self.conn.execute(
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items - ?,
keycheck_bytes = keycheck_bytes - ?, updated_at = ? WHERE id = 1''',
(unused_items, unused_bytes, now),
)
self.conn.execute(
'''UPDATE result_reservations SET state = 'db_committed',
projection_credit_transferred = 1, candidate_credit_transferred = 1,
updated_at = ? WHERE id = ?''',
(now, reservation_id),
)
self.conn.execute(
'''UPDATE result_bundles SET state = 'db_committed', target_scan_id = ?,
committed_at = ?, updated_at = ? WHERE reservation_id = ?''',
(target_scan_id, now, now, reservation_id),
)
found_count = 1 if validated.finding_count else 0
clean_count = 1 if not validated.finding_count and not validated.error_count and not metadata.get('skipped') else 0
skipped_count = 1 if metadata.get('skipped') else 0
if cycle_id is not None:
self.conn.execute(
'''UPDATE source_cycles SET ingested_count = ingested_count + 1,
scanned_count = scanned_count + 1, clean_count = clean_count + ?,
found_count = found_count + ?, skipped_count = skipped_count + ?,
error_count = error_count + ?, findings_count = findings_count + ?,
verified_findings_count = verified_findings_count + ?, updated_at = ?
WHERE id = ?''',
(
clean_count, found_count, skipped_count, validated.error_count,
validated.finding_count, verified_count, now, cycle_id,
),
)
if run_id is not None:
self.conn.execute(
'''UPDATE runs SET total_scanned = total_scanned + 1,
total_clean = total_clean + ?, total_found = total_found + ?,
total_skipped = total_skipped + ?, total_errors = total_errors + ?,
total_findings = total_findings + ?,
total_verified_findings = total_verified_findings + ?, updated_at = ?
WHERE id = ?''',
(
clean_count, found_count, skipped_count, validated.error_count,
validated.finding_count, verified_count, now, run_id,
),
)
self.conn.commit()
return {
'ingested': True, 'duplicate': False, 'target_scan_id': target_scan_id,
'projection_job_id': projection_job_id,
'candidate_count': actual_candidate_items,
'scan_event_id': event_id, 'scan_event_hash': event_hash,
}
except Exception:
self.conn.rollback()
raise
@staticmethod
def _compat_finding_from_row(row):
if row['payload_omitted']:
return {
'finding_uid': row['finding_uid'],
'DetectorName': row['detector_name'],
'Verified': bool(row['verified']),
'finding_omitted': True,
'payload_sha256': row['payload_sha256'],
'payload_bytes': row['payload_bytes'],
}
finding = safe_json_loads(row['extension_json']) or {}
finding.update({
'finding_uid': row['finding_uid'],
'DetectorName': row['detector_name'],
'DetectorType': row['detector_type'],
'Verified': bool(row['verified']),
'Raw': row['raw_value'],
'RawV2': row['raw_v2_value'],
'Redacted': row['redacted_secret'],
})
for column, key in (
('structured_data_json', 'StructuredData'),
('extra_data_json', 'ExtraData'),
('analysis_info_json', 'AnalysisInfo'),
):
value = safe_json_loads(row[column])
if value is not None:
finding[key] = value
return finding
def projection_scan_header(self, target_scan_id, max_bytes=16 * 1024 * 1024):
if not self.conn:
raise RuntimeError('database connection is unavailable')
raw_size_sql = (
'OCTET_LENGTH(ts.raw_result_json)' if self.conn.is_postgres
else "LENGTH(CAST(ts.raw_result_json AS BLOB))"
)
scan = self.conn.execute(
f'''SELECT ts.id, ts.scan_event_id, ts.target, ts.scan_type, ts.ended_at,
ts.raw_result_storage,
CASE WHEN ts.raw_result_json IS NULL THEN 0
ELSE {raw_size_sql} END AS raw_result_bytes,
sc.metadata_sha256, sc.metadata_bytes, sc.reconstruction_status
FROM target_scans ts
LEFT JOIN scan_result_compat sc ON sc.target_scan_id = ts.id
WHERE ts.id = ?''',
(int(target_scan_id),),
).fetchone()
if not scan:
return None
if scan['raw_result_storage'] != 'normalized_v2':
raw_bytes = int(scan['raw_result_bytes'] or 0)
if raw_bytes <= 0 or raw_bytes > max(1, int(max_bytes)):
raise ValueError('legacy compatibility result exceeds its reconstruction byte bound')
payload_size_sql = (
'OCTET_LENGTH(raw_result_json)' if self.conn.is_postgres
else 'LENGTH(CAST(raw_result_json AS BLOB))'
)
payload = self.conn.execute(
f'''SELECT raw_result_json FROM target_scans
WHERE id = ? AND raw_result_storage != 'normalized_v2'
AND raw_result_json IS NOT NULL
AND {payload_size_sql} = ?''',
(int(target_scan_id), raw_bytes),
).fetchone()
if not payload:
raise RuntimeError('legacy compatibility result changed during bounded reconstruction')
legacy = safe_json_loads(payload['raw_result_json'])
if isinstance(legacy, dict):
legacy.setdefault(
'scan_event_id', scan['scan_event_id'] or f'legacy-target-scan-{scan["id"]}',
)
if self.conn.is_postgres:
self.conn.commit()
return {'storage': 'legacy', 'result': legacy}
if self.conn.is_postgres:
self.conn.commit()
return None
if int(scan['metadata_bytes'] or 0) > max(1, int(max_bytes)):
raise ValueError('normalized compatibility metadata exceeds its reconstruction byte bound')
metadata_size_sql = (
'OCTET_LENGTH(metadata_json)' if self.conn.is_postgres
else 'LENGTH(CAST(metadata_json AS BLOB))'
)
metadata = self.conn.execute(
f'''SELECT metadata_json FROM scan_result_compat
WHERE target_scan_id = ? AND metadata_bytes = ?
AND {metadata_size_sql} <= ?''',
(
int(target_scan_id), int(scan['metadata_bytes'] or 0),
max(1, int(max_bytes)),
),
).fetchone()
if not metadata:
raise RuntimeError('normalized compatibility metadata changed during bounded reconstruction')
result = safe_json_loads(metadata['metadata_json']) or {}
if not isinstance(result, dict):
raise ValueError('normalized compatibility metadata is invalid')
result['scan_event_id'] = scan['scan_event_id']
result['target'] = scan['target']
result['scan_type'] = scan['scan_type']
result['timestamp'] = scan['ended_at']
result.pop('findings', None)
result.pop('errors', None)
if self.conn.is_postgres:
self.conn.commit()
return {'storage': 'normalized_v2', 'result': result}
def iter_projection_findings(self, target_scan_id, max_findings=20000, page_size=4):
last_id = 0
count = 0
page_size = min(64, max(1, int(page_size)))
while True:
rows = self.conn.execute(
'''SELECT f.*, cp.raw_value, cp.raw_v2_value, cp.structured_data_json,
cp.extra_data_json, cp.analysis_info_json, cp.extension_json,
cp.payload_sha256, cp.payload_bytes, cp.payload_omitted
FROM findings f
LEFT JOIN finding_compat_payloads cp ON cp.finding_id = f.id
WHERE f.target_scan_id = ? AND f.id > ? ORDER BY f.id LIMIT ?''',
(int(target_scan_id), last_id, page_size),
).fetchall()
if self.conn.is_postgres:
self.conn.commit()
if not rows:
return
for row in rows:
count += 1
if count > max(0, int(max_findings)):
raise ValueError('normalized compatibility reconstruction exceeds its finding bound')
last_id = int(row['id'])
yield self._compat_finding_from_row(row)
def iter_projection_errors(self, target_scan_id, max_errors=20000, page_size=64):
last_id = 0
count = 0
page_size = min(256, max(1, int(page_size)))
while True:
rows = self.conn.execute(
'''SELECT id, raw_error FROM errors
WHERE target_scan_id = ? AND id > ? ORDER BY id LIMIT ?''',
(int(target_scan_id), last_id, page_size),
).fetchall()
if self.conn.is_postgres:
self.conn.commit()
if not rows:
return
for row in rows:
count += 1
if count > max(0, int(max_errors)):
raise ValueError('normalized compatibility reconstruction exceeds its error bound')
last_id = int(row['id'])
yield row['raw_error']
def reconstruct_scan_result(
self, target_scan_id, max_bytes=192 * 1024 * 1024,
max_findings=20000, max_errors=20000,
):
header = self.projection_scan_header(target_scan_id, max_bytes=max_bytes)
if not header:
return None
result = header['result']
if header['storage'] == 'normalized_v2':
byte_limit = max(1, int(max_bytes))
used = len(json.dumps(
result, ensure_ascii=True, separators=(',', ':'), default=str,
).encode('utf-8')) + 64
findings = []
for finding in self.iter_projection_findings(target_scan_id, max_findings):
used += len(json.dumps(
finding, ensure_ascii=True, separators=(',', ':'), default=str,
).encode('utf-8')) + 1
if used > byte_limit:
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
findings.append(finding)
errors = []
for error in self.iter_projection_errors(target_scan_id, max_errors):
used += len(json.dumps(
error, ensure_ascii=True, separators=(',', ':'), default=str,
).encode('utf-8')) + 1
if used > byte_limit:
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
errors.append(error)
result['findings'] = findings
result['errors'] = errors
encoded = json.dumps(result, ensure_ascii=True, separators=(',', ':'), default=str).encode('utf-8')
if len(encoded) > max(1, int(max_bytes)):
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
return result
def record_target_result(self, run_id, cycle_id, source, query, target, result, scan_options=None):
if not run_id or not cycle_id:
return None
def op():
target_scan_id = self._insert_target_result(run_id, cycle_id, source, query, target, result, scan_options)
self.conn.commit()
return target_scan_id
return self._safe('record_target_result', op)
def _insert_target_result(
self, run_id, cycle_id, source, query, target, result, scan_options=None,
scan_event_id=None, scan_event_hash_value=None, queue_id=None, claim_lease_token=None,
queue_completion_applied=0, queue_completion_disposition=None,
):
status = target_status(result)
normalized = normalize_target(target, source)
findings = [sanitize_postman_finding(finding) for finding in (result.get('findings') or [])]
persisted_result = dict(result)
if 'findings' in result or findings:
persisted_result['findings'] = findings
errors = result.get('errors') or []
package = extract_package_metadata(target, result, source)
timestamp = result.get('timestamp') or utc_now_iso()
conflict_clause = ' ON CONFLICT DO NOTHING' if scan_event_id else ''
target_scan_id = self.conn.insert_returning_id(
'''INSERT INTO target_scans (
scan_event_id, scan_event_hash, queue_id, claim_lease_token,
queue_completion_applied, queue_completion_disposition,
run_id, cycle_id, source, query, target, normalized_target, scan_type, status,
started_at, ended_at, duration_sec, scan_options_json, package_name, package_version,
package_artifact, package_date, package_filename, package_type, package_size,
findings_count, verified_findings_count, error_count, skipped_reason, first_error_summary,
raw_result_json, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' + conflict_clause,
(
scan_event_id, scan_event_hash_value, queue_id, claim_lease_token,
1 if queue_completion_applied else 0, queue_completion_disposition,
run_id, cycle_id, source, query, target, normalized, result.get('scan_type'), status,
result.get('scan_started_at') or timestamp, timestamp, result.get('duration_sec'), json_dumps(redact_config(scan_options or {})),
package.get('package_name'), package.get('package_version'), package.get('package_artifact'), package.get('package_date'),
package.get('package_filename'), package.get('package_type'), package.get('package_size'),
len(findings), sum(1 for finding in findings if finding.get('Verified', False)), len(errors),
result.get('skipped'), first_error_summary(result), json_dumps(persisted_result), utc_now_iso(),
),
)
if target_scan_id is None and scan_event_id:
return None
for finding in findings:
self._insert_finding(run_id, cycle_id, target_scan_id, source, query, target, normalized, finding)
for error in errors:
self._insert_error(run_id, cycle_id, target_scan_id, source, query, target, normalized, error)
return target_scan_id
@staticmethod
def _unique_violation(exc):
code = getattr(exc, 'sqlstate', None) or getattr(exc, 'pgcode', None)
text = str(exc or '').lower()
return code == '23505' or 'unique constraint' in text or 'duplicate key' in text
def _confirmed_scan_event(self, event_id, event_hash):
row = self.conn.execute(
'''SELECT id, scan_event_hash, queue_completion_applied, queue_completion_disposition
FROM target_scans WHERE scan_event_id = ?''',
(event_id,),
).fetchone()
if not row:
return None
if str(row['scan_event_hash'] or '') != event_hash:
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
outbox = self.conn.execute(
'SELECT id FROM scan_publication_outbox WHERE target_scan_id = ?',
(row['id'],),
).fetchone()
# Delivery deletes the reference row. Its absence on replay therefore
# means the authoritative scan event was already projected.
self.conn.commit()
applied = bool(row['queue_completion_applied'])
return {
'ingested': True,
'duplicate': True,
'stale': not applied,
'queue_completion_applied': applied,
'queue_completion_disposition': row['queue_completion_disposition'],
'target_scan_id': row['id'],
'outbox_id': outbox['id'] if outbox else None,
'scan_event_id': event_id,
'scan_event_hash': event_hash,
}
def confirmed_scan_event(self, event_id, event_hash):
if not self.conn:
raise RuntimeError('database connection is unavailable')
return self._confirmed_scan_event(str(event_id or ''), str(event_hash or ''))
def _insert_derived_postman_targets(self, targets, now):
seen = set()
for target in targets or []:
normalized = normalize_target(target, 'postman')
if not normalized or normalized in seen:
continue
seen.add(normalized)
self.conn.execute(
'''INSERT INTO target_queue (
source, platform, query, target, normalized_target, status, created_at, updated_at
) VALUES ('postman', 'postman', 'harvested', ?, ?, 'pending', ?, ?)
ON CONFLICT(source, normalized_target) DO NOTHING''',
(target, normalized, now, now),
)
def ingest_scan_event(self, envelope):
if not self.conn:
raise RuntimeError('database connection is unavailable')
self.require_runtime_safety_schema()
try:
event = prepare_scan_event(envelope)
except SpoolHashConflictError as exc:
raise ScanEventConflictError(str(exc)) from exc
event_id = event['scan_event_id']
event_hash = event['scan_event_hash']
result = event.get('result')
if not isinstance(result, dict):
raise ValueError('scan event result must be an object')
if str(result.get('scan_event_id') or '') != event_id:
raise ValueError('scan event result ID does not match its envelope')
queue_id = event.get('queue_id')
lease_token = str(event.get('claim_lease_token') or '')
if queue_id is None or not lease_token:
raise ValueError('scan event requires queue_id and claim_lease_token')
queue_status = str(event.get('queue_status') or '')
if queue_status not in ('done', 'failed', 'deferred'):
raise ValueError(f'invalid scan event queue status: {queue_status}')
run_id = event.get('run_id')
cycle_id = event.get('cycle_id')
if run_id is None or cycle_id is None:
raise ValueError('scan event requires run_id and cycle_id')
try:
if self.conn.is_sqlite:
self.conn.execute('BEGIN IMMEDIATE')
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
existing = self.conn.execute(
f'''SELECT id, scan_event_hash, queue_completion_applied, queue_completion_disposition
FROM target_scans WHERE scan_event_id = ?{lock_suffix}''',
(event_id,),
).fetchone()
if existing:
if str(existing['scan_event_hash'] or '') != event_hash:
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
self.conn.commit()
return self._confirmed_scan_event(event_id, event_hash)
target = str(event.get('target') or result.get('target') or '')
source = str(event.get('source') or '')
query = event.get('query')
target_scan_id = self._insert_target_result(
run_id, cycle_id, source, query, target, result, event.get('scan_options') or {},
scan_event_id=event_id,
scan_event_hash_value=event_hash,
queue_id=queue_id,
claim_lease_token=lease_token,
queue_completion_applied=0,
queue_completion_disposition='pending',
)
if target_scan_id is None:
confirmed = self._confirmed_scan_event(event_id, event_hash)
if confirmed:
return confirmed
raise RuntimeError(f'scan event {event_id} conflict returned no authoritative row')
now = utc_now_iso()
completed = now if queue_status in ('done', 'failed') else None
cur = self.conn.execute(
'''UPDATE target_queue SET
status = ?, target_scan_id = ?, last_error = ?,
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
available_after = ?, attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
completed_at = COALESCE(?, completed_at), updated_at = ?
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
(
queue_status, target_scan_id,
first_line(event.get('queue_error'), 500) if event.get('queue_error') else None,
event.get('available_after'), 1 if event.get('reset_attempts') else 0,
completed, now, queue_id, lease_token,
),
)
applied = int(getattr(cur, 'rowcount', 0) or 0) == 1
disposition = 'applied' if applied else 'stale_lease'
self.conn.execute(
'''UPDATE target_scans SET queue_completion_applied = ?, queue_completion_disposition = ?
WHERE id = ?''',
(1 if applied else 0, disposition, target_scan_id),
)
self._insert_derived_postman_targets(event.get('derived_postman_targets'), now)
outbox_id = self.conn.insert_returning_id(
'''INSERT INTO scan_publication_outbox (
target_scan_id, payload_json, status, attempts, created_at, updated_at
) VALUES (?, ?, 'pending', 0, ?, ?)''',
(target_scan_id, '', now, now),
)
self.conn.commit()
self.last_error = ''
return {
'ingested': True,
'duplicate': False,
'stale': not applied,
'queue_completion_applied': applied,
'queue_completion_disposition': disposition,
'target_scan_id': target_scan_id,
'outbox_id': outbox_id,
'scan_event_id': event_id,
'scan_event_hash': event_hash,
}
except ScanEventConflictError:
try:
self.conn.rollback()
except Exception:
pass
raise
except Exception as exc:
self.last_error = str(exc)
try:
self.conn.rollback()
except Exception:
pass
if self._unique_violation(exc):
confirmed = self._confirmed_scan_event(event_id, event_hash)
if confirmed:
return confirmed
raise
def record_and_complete_target_result(
self, run_id, cycle_id, source, query, target, result, scan_options,
queue_id, lease_owner, queue_status, queue_error=None, available_after=None,
reset_attempts=False, derived_postman_targets=None, lease_token=None,
):
event_id = str(result.get('scan_event_id') or '')
if not self.conn or not event_id or queue_id is None or not lease_token:
return None
event = prepare_scan_event({
'version': 1,
'scan_event_id': event_id,
'run_id': run_id,
'cycle_id': cycle_id,
'source': source,
'query': query,
'target': target,
'result': result,
'scan_options': scan_options or {},
'queue_id': queue_id,
'claim_lease_token': lease_token,
'claim_lease_owner': lease_owner,
'queue_status': queue_status,
'queue_error': queue_error,
'available_after': available_after,
'reset_attempts': bool(reset_attempts),
'derived_postman_targets': list(derived_postman_targets or []),
})
return self.ingest_scan_event(event)
def record_target_results(self, run_id, cycle_id, source, query, results, scan_options=None):
output = {}
for result in results or []:
target = result.get('target', '')
target_scan_id = self.record_target_result(run_id, cycle_id, source, query, target, result, scan_options)
if target_scan_id is None:
output[normalize_target(target, source)] = None
output[str(target)] = None
else:
output[normalize_target(target, source)] = target_scan_id
output[str(target)] = target_scan_id
return output
def _insert_finding(self, run_id, cycle_id, target_scan_id, source, query, target, normalized, finding):
finding = sanitize_postman_finding(finding)
raw_secret, secret_hash, detector_secret_hash, fingerprint, location = finding_identity(source, normalized, finding)
finding_uid = finding.get('finding_uid') or fingerprint
enrichment = enrich_finding(finding)
finding_id = self.conn.insert_returning_id(
'''INSERT INTO findings (
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
detector_name, detector_type, verified, raw_secret, redacted_secret, secret_hash,
detector_secret_hash, finding_fingerprint, finding_uid, file_path, line_number, commit_hash,
source_timestamp, source_metadata_type, source_metadata_json, raw_finding_json,
provider, credential_kind, credential_confidence, required_context_missing,
principal, username, email, project_id, tenant_id, organization, registry,
endpoint, scope, resource, enrichment_json, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
run_id, cycle_id, target_scan_id, source, query, target, normalized,
finding.get('DetectorName'), finding.get('DetectorType'), 1 if finding.get('Verified', False) else 0,
raw_secret, extract_redacted_secret(finding), secret_hash, detector_secret_hash, fingerprint, finding_uid,
location.get('file_path'), location.get('line_number'), location.get('commit_hash'), location.get('source_timestamp'),
location.get('source_metadata_type'), location.get('source_metadata_json'), json_dumps(finding),
enrichment.get('provider'), enrichment.get('credential_kind'), enrichment.get('credential_confidence'),
enrichment.get('required_context_missing'), enrichment.get('principal'), enrichment.get('username'),
enrichment.get('email'), enrichment.get('project_id'), enrichment.get('tenant_id'), enrichment.get('organization'),
enrichment.get('registry'), enrichment.get('endpoint'), enrichment.get('scope'), enrichment.get('resource'),
enrichment.get('enrichment_json'), utc_now_iso(),
),
)
if finding_uid and finding_id:
self.conn.execute(
'''INSERT INTO finding_uid_map (finding_uid, finding_id, created_at)
VALUES (?, ?, ?)
ON CONFLICT(finding_uid) DO NOTHING''',
(finding_uid, finding_id, utc_now_iso()),
)
def _insert_error(self, run_id, cycle_id, target_scan_id, source, query, target, normalized, error):
raw_error = str(error)
self.conn.execute(
'''INSERT INTO errors (
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
category, summary, raw_error, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
(
run_id, cycle_id, target_scan_id, source, query, target, normalized,
categorize_error(raw_error), first_line(raw_error), raw_error, utc_now_iso(),
),
)
def record_keycheck_result(self, service, status, status_group, checked_at=None, key_hash='', secret_hash='', key_masked='', detector_name='', message='', metadata=None, link_findings=True, source_line='', detector_secret_hash=''):
if not self.conn:
return
self.require_runtime_safety_schema()
def keycheck_result_columns():
if self._keycheck_result_columns is None:
try:
self._keycheck_result_columns = self.conn.table_columns('keycheck_results')
except Exception:
self._keycheck_result_columns = set()
return self._keycheck_result_columns
def matching_findings(hash_value):
if not hash_value:
return []
clauses = ['secret_hash = ?']
params = [hash_value]
if detector_name:
if str(detector_name).lower() == 'googleaistudio':
extra_name = self.conn.json_extract('raw_finding_json', '$.ExtraData.name')
clauses.append("""(
LOWER(detector_name) = LOWER(?)
OR (
LOWER(detector_name) = 'customregex'
AND LOWER(COALESCE({extra_name}, '')) = LOWER(?)
)
)""".format(extra_name=extra_name))
params.extend([detector_name, detector_name])
else:
clauses.append('LOWER(detector_name) = LOWER(?)')
params.append(detector_name)
row = self.conn.execute(
f'''SELECT id, run_id, cycle_id, target_scan_id, source, query, target, detector_name, created_at
FROM findings WHERE {' AND '.join(clauses)} ORDER BY id DESC LIMIT 1''',
params,
).fetchone()
return [row] if row else []
def op():
now = utc_now_iso()
checked = checked_at or now
# Inline linking is intentionally disabled for live keychecks. It can
# misattribute duplicate secrets and should be handled by the bounded
# repair/linker pipeline instead.
rows = []
if not rows:
rows = [None]
has_link_columns = {'source_line', 'detector_secret_hash', 'link_status', 'link_attempts', 'linked_at', 'link_error'}.issubset(keycheck_result_columns())
has_event_columns = {'event_id', 'finding_uid'}.issubset(keycheck_result_columns())
for row in rows:
base_values = (
service, status, status_group, checked, key_hash or '', secret_hash or '', key_masked or '',
row['id'] if row else None,
row['target_scan_id'] if row else None,
row['cycle_id'] if row else None,
row['run_id'] if row else None,
row['source'] if row else None,
row['query'] if row else None,
row['target'] if row else None,
row['detector_name'] if row else detector_name,
row['created_at'] if row else None,
first_line(message, 1000),
json_dumps(metadata or {}),
)
if has_link_columns and has_event_columns:
self.conn.execute(
'''INSERT INTO keycheck_results (
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
found_at, message, metadata_json, source_line, detector_secret_hash,
event_id, finding_uid, link_status, link_attempts, linked_at, link_error, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
base_values + (
source_line or (metadata or {}).get('source_line') or '',
detector_secret_hash or '',
(metadata or {}).get('event_id') or '',
(metadata or {}).get('finding_uid') or '',
'linked' if row else 'pending',
0,
now if row else None,
'',
now,
),
)
elif has_link_columns:
self.conn.execute(
'''INSERT INTO keycheck_results (
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
found_at, message, metadata_json, source_line, detector_secret_hash,
link_status, link_attempts, linked_at, link_error, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
base_values + (
source_line or (metadata or {}).get('source_line') or '',
detector_secret_hash or '',
'linked' if row else 'pending',
0,
now if row else None,
'',
now,
),
)
else:
self.conn.execute(
'''INSERT INTO keycheck_results (
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
found_at, message, metadata_json, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
base_values + (now,),
)
self.conn.commit()
return True
return bool(self._safe('record_keycheck_result', op, False))
def _migration_add_columns(conn, table, additions):
existing = conn.table_columns(table)
for name, ddl in additions.items():
if name not in existing:
conn.execute(f'ALTER TABLE {table} ADD COLUMN {name} {ddl}')
def _migration_seed_legacy_docker_provenance(
conn,
max_rows=DOCKER_LEGACY_PROVENANCE_SEED_MAX_ROWS,
page_size=DOCKER_LEGACY_PROVENANCE_SEED_PAGE_SIZE,
):
max_rows = max(1, int(max_rows))
page_size = max(1, min(int(page_size), max_rows, 10000))
after_id = 0
scanned = 0
inserted = 0
predicate = """platform = 'docker' AND query IS NOT NULL AND query <> ''
AND (resolver_state IS NOT NULL OR target NOT LIKE '%@%')"""
while scanned < max_rows:
rows = conn.execute(
f'''SELECT id, source, query, target, created_at, updated_at
FROM target_queue
WHERE id > ? AND {predicate}
ORDER BY id LIMIT ?''',
(after_id, min(page_size, max_rows - scanned)),
).fetchall()
if not rows:
break
for row in rows:
after_id = int(row['id'])
target = str(row['target'] or '').strip()
if not target or '@' in target or ':' in target.rsplit('/', 1)[-1]:
continue
cursor = conn.execute(
'''INSERT INTO docker_repository_query_provenance(
source, query, repository_queue_id, provenance_kind,
first_observed_at, last_observed_at, created_at, updated_at
) VALUES (?, ?, ?, 'legacy_queue', ?, ?, ?, ?)
ON CONFLICT(source, query, repository_queue_id) DO NOTHING''',
(
row['source'], row['query'], row['id'], row['created_at'],
row['created_at'], row['created_at'], row['updated_at'],
),
)
inserted += max(0, int(getattr(cursor, 'rowcount', 0) or 0))
scanned += len(rows)
if scanned >= max_rows and conn.execute(
f'''SELECT 1 AS present FROM target_queue
WHERE id > ? AND {predicate} ORDER BY id LIMIT 1''',
(after_id,),
).fetchone():
raise RuntimeSafetySchemaError(
f'legacy Docker provenance seed exceeds the reviewed {max_rows}-row bound'
)
return {'scanned_count': scanned, 'inserted_count': inserted}
def _migration_backfill_keycheck_hashes(conn, max_rows=1000000):
processed = 0
while True:
rows = conn.execute(
'''SELECT id, service, candidate_kind, secret_text, secret_json, endpoint, principal
FROM keycheck_credentials WHERE provider_key_hash = '' ORDER BY id LIMIT 250'''
).fetchall()
if not rows:
break
for row in rows:
try:
digest = stored_provider_key_hash(
row['service'], row['candidate_kind'], row['secret_text'], row['secret_json'],
row['endpoint'], row['principal'],
)
except ValueError as exc:
raise RuntimeSafetySchemaError(
f'keycheck credential {row["id"]} cannot be migrated to provider-canonical identity'
) from exc
conn.execute(
"UPDATE keycheck_credentials SET provider_key_hash = ? WHERE id = ? AND provider_key_hash = ''",
(digest, row['id']),
)
processed += 1
if processed > max_rows:
raise RuntimeSafetySchemaError('keycheck credential identity backfill exceeded its row bound')
processed = 0
while True:
rows = conn.execute(
'''SELECT c.id,
(SELECT NULLIF(f.secret_hash, '') FROM findings f WHERE f.id = c.finding_id) AS finding_hash,
(SELECT NULLIF(r.secret_hash, '') FROM keycheck_results r
WHERE r.candidate_id = c.id ORDER BY r.id DESC LIMIT 1) AS result_hash,
kc.provider_key_hash
FROM keycheck_candidates c
JOIN keycheck_credentials kc ON kc.id = c.credential_id
WHERE c.secret_hash = '' ORDER BY c.id LIMIT 250'''
).fetchall()
if not rows:
break
for row in rows:
digest = str(row['finding_hash'] or row['result_hash'] or row['provider_key_hash'] or '').lower()
if not re.fullmatch(r'[a-f0-9]{64}', digest):
raise RuntimeSafetySchemaError(
f'keycheck candidate {row["id"]} cannot be migrated to detector secret identity'
)
conn.execute(
"UPDATE keycheck_candidates SET secret_hash = ? WHERE id = ? AND secret_hash = ''",
(digest, row['id']),
)
processed += 1
if processed > max_rows:
raise RuntimeSafetySchemaError('keycheck candidate identity backfill exceeded its row bound')
processed = 0
while True:
rows = conn.execute(
'''SELECT r.id, kc.provider_key_hash, c.secret_hash
FROM keycheck_results r
JOIN keycheck_candidates c ON c.id = r.candidate_id
JOIN keycheck_credentials kc ON kc.id = c.credential_id
WHERE COALESCE(r.key_hash, '') != kc.provider_key_hash
OR COALESCE(r.secret_hash, '') != c.secret_hash
ORDER BY r.id LIMIT 250'''
).fetchall()
if not rows:
break
for row in rows:
conn.execute(
'UPDATE keycheck_results SET key_hash = ?, secret_hash = ? WHERE id = ?',
(row['provider_key_hash'], row['secret_hash'], row['id']),
)
processed += 1
if processed > max_rows:
raise RuntimeSafetySchemaError('keycheck result identity backfill exceeded its row bound')
duplicate = conn.execute(
'''SELECT service, provider_key_hash, COUNT(*) AS count
FROM keycheck_credentials WHERE provider_key_hash != ''
GROUP BY service, provider_key_hash HAVING COUNT(*) > 1 LIMIT 1'''
).fetchone()
if duplicate:
raise RuntimeSafetySchemaError(
'provider-canonical keycheck credentials contain duplicate persisted identities; '
'reviewed credential merge is required before cutover'
)
def _migration_backfill_docker_selection_policies(conn, max_reservations=1000000):
processed = 0
while True:
reservations = conn.execute(
'''SELECT DISTINCT reservation_id
FROM docker_image_blob_coverage
WHERE COALESCE(selection_policy_sha256, '') = ?
ORDER BY reservation_id LIMIT 250''',
('',),
).fetchall()
if not reservations:
break
for identity in reservations:
reservation_id = int(identity['reservation_id'])
reservation = conn.execute(
'''SELECT id, docker_layer_plan_json, docker_layer_plan_sha256
FROM result_reservations WHERE id = ?''',
(reservation_id,),
).fetchone()
if not reservation:
raise RuntimeSafetySchemaError(
f'Docker coverage reservation {reservation_id} is absent during selector backfill'
)
plan, plan_sha256 = stored_docker_layer_plan(reservation)
if plan is None:
raise RuntimeSafetySchemaError(
f'Docker coverage reservation {reservation_id} has no bound plan'
)
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
expected = {
descriptor['position']: descriptor for descriptor in plan['descriptors']
}
rows = conn.execute(
'''SELECT position, blob_digest, coverage_policy_sha256,
selection_policy_sha256, descriptor_kind, plan_sha256,
selected, selection_reason
FROM docker_image_blob_coverage
WHERE reservation_id = ? ORDER BY position''',
(reservation_id,),
).fetchall()
if len(rows) != len(expected):
raise RuntimeSafetySchemaError(
f'Docker coverage reservation {reservation_id} is incomplete during selector backfill'
)
empty_count = 0
for row in rows:
descriptor = expected.get(int(row['position']))
existing_selector = str(row['selection_policy_sha256'] or '')
if (
not descriptor
or str(row['blob_digest']) != descriptor['digest']
or str(row['coverage_policy_sha256']) != coverage_policy_sha256
or existing_selector not in ('', plan['selection_policy_sha256'])
or str(row['descriptor_kind']) != descriptor['kind']
or str(row['plan_sha256']) != plan_sha256
or bool(row['selected']) != descriptor['selected']
or str(row['selection_reason']) != descriptor['selection_reason']
):
raise RuntimeSafetySchemaError(
f'Docker coverage reservation {reservation_id} conflicts with its bound plan'
)
empty_count += 1 if not existing_selector else 0
cursor = conn.execute(
'''UPDATE docker_image_blob_coverage SET selection_policy_sha256 = ?
WHERE reservation_id = ? AND COALESCE(selection_policy_sha256, '') = ?''',
(plan['selection_policy_sha256'], reservation_id, ''),
)
if int(cursor.rowcount or 0) != empty_count:
raise RuntimeSafetySchemaError(
f'Docker coverage reservation {reservation_id} changed during selector backfill'
)
processed += 1
if processed > max_reservations:
raise RuntimeSafetySchemaError(
'Docker selection-policy backfill exceeded its reservation bound'
)
remaining = conn.execute(
'''SELECT reservation_id FROM docker_image_blob_coverage
WHERE COALESCE(selection_policy_sha256, '') = ? LIMIT 1''',
('',),
).fetchone()
if remaining:
raise RuntimeSafetySchemaError('Docker selection-policy backfill is incomplete')
def _pipeline_quarantine_review_status_constraint(conn):
if conn.is_postgres:
rows = conn.execute(
'''SELECT con.conname AS name,
pg_catalog.pg_get_constraintdef(con.oid, true) AS definition,
con.convalidated AS is_valid
FROM pg_catalog.pg_constraint con
JOIN pg_catalog.pg_class tbl ON tbl.oid = con.conrelid
JOIN pg_catalog.pg_namespace n ON n.oid = tbl.relnamespace
WHERE con.contype = 'c' AND n.nspname = ?
AND tbl.relname = 'pipeline_quarantine' ''',
(conn.application_schema,),
).fetchall()
matching = [
row for row in rows
if 'review_status' in str(row['definition'] or '').lower()
]
if len(matching) != 1:
return {'name': '', 'statuses': None, 'valid': False}
row = matching[0]
statuses = frozenset(re.findall(r"'([^']+)'", str(row['definition'] or '')))
return {
'name': str(row['name'] or ''),
'statuses': statuses,
'valid': bool(row['is_valid']),
}
row = conn.execute(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'pipeline_quarantine'"
).fetchone()
sql = str(row['sql'] if row else '')
match = re.search(
r'review_status\s+TEXT.*?CHECK\s*\(\s*review_status\s+IN\s*\((.*?)\)\s*\)',
sql, re.IGNORECASE | re.DOTALL,
)
statuses = frozenset(re.findall(r"'([^']+)'", match.group(1))) if match else None
return {'name': '', 'statuses': statuses, 'valid': statuses is not None}
def _migration_ensure_pipeline_quarantine_review_status_constraint(conn):
constraint = _pipeline_quarantine_review_status_constraint(conn)
statuses = constraint.get('statuses')
if statuses == PIPELINE_QUARANTINE_REVIEW_STATUSES:
if conn.is_postgres and not constraint.get('valid'):
conn.execute(
'ALTER TABLE pipeline_quarantine VALIDATE CONSTRAINT '
+ _quoted_pg_name(constraint['name'])
)
return
if not conn.is_postgres:
raise RuntimeSafetySchemaError(
'SQLite pipeline_quarantine review-status constraint requires a reviewed table rebuild'
)
if statuses not in (None, PIPELINE_QUARANTINE_LEGACY_REVIEW_STATUSES):
raise RuntimeSafetySchemaError(
'PostgreSQL pipeline_quarantine has an unexpected review-status constraint; '
'manual reviewed repair is required'
)
if statuses is not None:
conn.execute(
'ALTER TABLE pipeline_quarantine DROP CONSTRAINT '
+ _quoted_pg_name(constraint['name'])
)
conn.execute(
'''ALTER TABLE pipeline_quarantine
ADD CONSTRAINT pipeline_quarantine_review_status_check
CHECK (review_status IN (
'pending','approved_retry','approved_rescan','discarded','resolved'
))'''
)
def _migration_require_pipeline_quiescence(conn):
if not conn.table_exists('runtime_schema_migrations'):
return
applied = {
str(row['version']) for row in conn.execute(
'SELECT version FROM runtime_schema_migrations'
).fetchall()
}
if set(PIPELINE_MIGRATION_VERSIONS).issubset(applied) or not all(
conn.table_exists(table) for table in (
'pipeline_leases', 'result_reservations', 'target_queue', 'docker_content_blobs',
)
):
return
capacity_backfill_only = (
set(PIPELINE_MIGRATION_VERSIONS[:-1]).issubset(applied)
and REMOTE_ASSIGNMENT_CAPACITY_MIGRATION not in applied
)
active = {
'worker_leases': conn.execute(
"SELECT COUNT(*) AS count FROM pipeline_leases WHERE state NOT IN ('released','failed')"
).fetchone()['count'],
'result_reservations': 0 if capacity_backfill_only else conn.execute(
"""SELECT COUNT(*) AS count FROM result_reservations
WHERE state IN ('scanning','ready','ingesting','db_committed')"""
).fetchone()['count'],
'queue_leases': 0 if capacity_backfill_only else conn.execute(
"""SELECT COUNT(*) AS count FROM target_queue q
LEFT JOIN result_reservations r ON r.id = q.current_result_reservation_id
WHERE q.status = 'in_progress'
OR r.state IN ('scanning','ready','ingesting','db_committed')"""
).fetchone()['count'],
'blob_leases': conn.execute(
"""SELECT COUNT(*) AS count FROM docker_content_blobs
WHERE state IN ('leased','submitted') OR lease_reservation_id IS NOT NULL"""
).fetchone()['count'],
}
if conn.table_exists('discovery_retry_queue'):
active['discovery_retry_leases'] = conn.execute(
"SELECT COUNT(*) AS count FROM discovery_retry_queue WHERE status = 'leased'"
).fetchone()['count']
queue_columns = set(conn.table_columns('target_queue'))
if {'resolver_state', 'resolver_token'} <= queue_columns:
active['docker_resolvers'] = conn.execute(
"""SELECT COUNT(*) AS count FROM target_queue
WHERE resolver_state = 'resolving' OR resolver_token IS NOT NULL"""
).fetchone()['count']
if conn.table_exists('docker_depth_experiments'):
active['experiment_authority_fences'] = conn.execute(
"""SELECT COUNT(*) AS count FROM docker_depth_experiments
WHERE fence_token IS NOT NULL
OR state IN ('holding','resolving','active','draining')"""
).fetchone()['count']
if conn.table_exists('docker_depth_experiment_repositories'):
active['experiment_resolvers'] = conn.execute(
"""SELECT COUNT(*) AS count FROM docker_depth_experiment_repositories
WHERE work_state = 'resolving' OR resolver_token IS NOT NULL"""
).fetchone()['count']
if (
conn.table_exists('target_queue_policy_events')
and 'experiment_id' in conn.table_columns('target_queue_policy_events')
):
active['experiment_holds'] = conn.execute(
"""SELECT COUNT(*) AS count
FROM target_queue_policy_events event
LEFT JOIN docker_depth_experiments experiment
ON experiment.id = event.experiment_id
WHERE event.experiment_id IS NOT NULL AND event.action = 'cold'
AND NOT EXISTS (
SELECT 1 FROM target_queue_policy_events reversal
WHERE reversal.reverses_event_id = event.id
)
AND (
experiment.id IS NULL
OR experiment.state NOT IN ('held','completed','released')
OR experiment.fence_token IS NOT NULL
)"""
).fetchone()['count']
if any(int(value or 0) for value in active.values()):
summary = ', '.join(
f'{name}={int(value or 0)}' for name, value in active.items()
if int(value or 0)
)
raise RuntimeSafetySchemaError(
f'pipeline schema migration requires stopped, reconciled runtime ({summary})'
)
def _migration_postgres_column_shape(conn, table, specs):
if not conn.is_postgres:
return
type_sql = {'id': 'BIGINT', 'id_ref': 'BIGINT', 'integer': 'INTEGER', 'text': 'TEXT', 'real': 'REAL'}
details = conn.table_column_details(table)
for name, (expected_type, expected_not_null) in specs.items():
actual = details.get(name)
if not actual:
raise RuntimeSafetySchemaError(f'migration did not create {table}.{name}')
if not _schema_type_matches(actual['type'], expected_type, True):
target_type = type_sql[expected_type]
actual_type = re.sub(r'\s+', ' ', str(actual['type'] or '').strip().lower())
safe_sources = {
'BIGINT': {'integer', 'smallint'},
'INTEGER': {'smallint'},
'TEXT': {'character varying', 'character', 'varchar'},
'REAL': set(),
}
if actual_type not in safe_sources[target_type]:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{name} type {actual_type or "<empty>"} cannot be safely '
f'changed to {target_type}; manual reviewed conversion is required'
)
conn.execute(
f'ALTER TABLE {table} ALTER COLUMN {name} TYPE {target_type} USING {name}::{target_type}'
)
if expected_not_null and not actual['not_null']:
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} SET NOT NULL')
elif not expected_not_null and actual['not_null'] and not actual['primary_key']:
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} DROP NOT NULL')
def _migration_require_sqlite_column_shape(conn, table, specs):
if not conn.is_sqlite:
return
details = conn.table_column_details(table)
problems = []
for name, (expected_type, expected_not_null) in specs.items():
actual = details.get(name)
if not actual:
problems.append(f'missing {name}')
continue
if not _schema_type_matches(actual['type'], expected_type, False):
problems.append(f'{name} type {actual["type"] or "<empty>"}')
if bool(actual['not_null']) != bool(expected_not_null):
problems.append(f'{name} nullability')
if problems:
raise RuntimeSafetySchemaError(
f'SQLite table {table} has a non-additively-repairable schema ({"; ".join(problems)}); '
'manually rebuild this table from a reviewed backup with the exact runtime schema'
)
def _migration_ensure_defaults(conn, table, specs, defaults, generated_id=None):
details = conn.table_column_details(table)
for name in specs:
if name == generated_id:
continue
expected_present = name in defaults
expected = defaults.get(name, '')
actual = details.get(name)
actual_present = bool(actual and actual.get('has_default', bool(actual.get('default'))))
if actual and actual_present == expected_present and (
not expected_present or _normalized_default(actual['default']) == _normalized_default(expected)
):
continue
if conn.is_sqlite:
raise RuntimeSafetySchemaError(
f'SQLite table {table}.{name} has default {actual["default"] or "<none>"}; '
'manually rebuild the table with the exact runtime default'
)
if not expected_present:
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} DROP DEFAULT')
else:
default_sql = str(int(expected)) if str(expected).isdigit() else "'" + str(expected).replace("'", "''") + "'"
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} SET DEFAULT {default_sql}')
def _migration_require_primary_key(conn, table, specs, expected):
details = conn.table_column_details(table)
actual = [name for name in specs if details.get(name, {}).get('primary_key')]
if actual != list(expected):
raise RuntimeSafetySchemaError(
f'{table} primary key is {actual or "absent"}, expected {list(expected)}; '
'manual reviewed table rebuild is required'
)
if conn.is_postgres:
primary_indexes = [
index for index in conn.table_indexes(table).values()
if index.get('primary')
]
if (
len(primary_indexes) != 1
or primary_indexes[0].get('columns') != list(expected)
or not _index_usable(primary_indexes[0])
):
raise RuntimeSafetySchemaError(
f'PostgreSQL {table} primary-key index is absent, invalid, or unready; '
'manual reviewed primary-key constraint rebuild is required'
)
def _quoted_pg_name(value):
parts = str(value or '').split('.')
if not parts or any(not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part) for part in parts):
raise RuntimeSafetySchemaError(f'unsafe PostgreSQL catalog identifier: {value!r}')
return '.'.join('"' + part.replace('"', '""') + '"' for part in parts)
def _migration_ensure_generated_id(conn, table, column):
details = conn.table_column_details(table).get(column)
if _column_generates_id(details, conn.is_postgres):
return
if not conn.is_postgres:
raise RuntimeSafetySchemaError(
f'SQLite {table}.{column} does not auto-generate an INTEGER PRIMARY KEY; '
'manually rebuild this table from a reviewed backup'
)
if not details or details.get('type') != 'bigint' or not details.get('primary_key'):
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{column} is not a BIGINT PRIMARY KEY; manual reviewed table repair is required'
)
if details.get('identity') or details.get('generated') or details.get('default') or details.get('sequence'):
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{column} has malformed identity/sequence metadata; '
'manually repair its generated-ID definition before retrying'
)
try:
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {column} ADD GENERATED BY DEFAULT AS IDENTITY')
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{column} cannot be safely converted to an identity column; '
'manually add a generated sequence/identity and reseed it above MAX(id)'
) from exc
repaired = conn.table_column_details(table).get(column) or {}
sequence = repaired.get('sequence')
if not _column_generates_id(repaired, True) or not sequence:
raise RuntimeSafetySchemaError(
f'PostgreSQL did not create a usable identity sequence for {table}.{column}; '
'manual identity repair is required'
)
row = conn.execute(f'SELECT COALESCE(MAX({column}), 0) + 1 AS next_id FROM {table}').fetchone()
next_id = max(1, int(row['next_id'] if row else 1))
try:
conn.execute(f'ALTER SEQUENCE {_quoted_pg_name(sequence)} RESTART WITH {next_id}')
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL identity sequence {sequence} could not be reseeded to {next_id}; '
f'manually restart it above MAX({table}.{column})'
) from exc
def _migration_add_docker_depth_schema_authority_checks(conn, marker_applied):
if not conn.is_postgres or marker_applied:
return
for table, name in DOCKER_DEPTH_SCHEMA_AUTHORITY_CHECKS:
if name in conn.table_check_constraints(table):
continue
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
try:
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
)
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
f'{_quoted_pg_name(name)}'
)
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{name} cannot be validated against existing rows; '
'manually repair malformed safety data before retrying migration'
) from exc
def _migration_add_docker_depth_scarcity_checks(conn, marker_applied):
if not conn.is_postgres or marker_applied:
return
for table, name in DOCKER_DEPTH_SCARCITY_COHORT_CHECKS:
if name in conn.table_check_constraints(table):
continue
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
try:
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
)
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
f'{_quoted_pg_name(name)}'
)
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{name} cannot be validated against existing rows; '
'manually repair malformed scarcity cohort data before retrying migration'
) from exc
def _migration_require_docker_depth_check_constraints(conn):
problems = _docker_depth_check_constraint_problems(conn)
prior_reviewed = {
('docker_depth_experiments', 'docker_depth_experiments_range_check'): (
'query_count >= 1 AND query_count <= 1000 '
'AND repositories_per_query >= 1 AND repositories_per_query <= 10 '
'AND images_per_repository >= 1 AND images_per_repository <= 10 '
'AND target_limit >= 1 AND target_limit <= 1200 '
'AND target_count >= 0 AND target_count <= target_limit '
'AND selection_count >= 0 AND fence_generation >= 0'
),
('docker_depth_experiments', 'docker_depth_experiments_capacity_check'): (
'query_count * (repositories_per_query + images_per_repository - 1) '
'<= target_limit'
),
(
'docker_depth_experiment_queries',
'docker_depth_experiment_queries_range_check',
): (
'query_ordinal >= 0 AND required_repository_count >= 1 '
'AND required_repository_count <= 10'
),
(
'docker_depth_experiment_repositories',
'docker_depth_experiment_repositories_range_check',
): (
'query_ordinal >= 0 AND repository_rank >= 1 AND repository_rank <= 10 '
'AND resolver_generation >= 0 AND resolver_attempts >= 0 '
'AND selected_image_count >= 0 AND selected_image_count <= 10'
),
(
'docker_depth_experiment_targets',
'docker_depth_experiment_targets_range_check',
): (
'counter_ordinal >= 1 AND counter_ordinal <= 1200 '
'AND dispatch_wave >= 1 AND dispatch_wave <= 3 '
'AND dispatch_order >= 1 AND reservation_count >= 0'
),
}
if conn.is_postgres and problems:
for table, name in prior_reviewed:
label = f'check constraint {table}.{name}'
if label not in problems:
continue
constraint = conn.table_check_constraints(table).get(name)
if (
not constraint
or not constraint.get('valid', True)
or _normalized_check_expression(constraint.get('expression'))
!= _normalized_check_expression(prior_reviewed[(table, name)])
):
continue
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
try:
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} '
f'DROP CONSTRAINT {_quoted_pg_name(name)}'
)
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
)
conn.execute(
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
f'{_quoted_pg_name(name)}'
)
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table}.{name} reviewed widening failed validation'
) from exc
problems = _docker_depth_check_constraint_problems(conn)
if problems:
raise RuntimeSafetySchemaError(
'Docker depth experiment schema has missing, invalid, or noncanonical '
+ '; '.join(problems)
+ '; only the exact prior reviewed CHECK constraints can be widened'
)
def _migration_ensure_foreign_keys(conn):
for table, expected_keys in REQUIRED_FOREIGN_KEYS.items():
foreign_keys = conn.table_foreign_keys(table)
for columns, referenced_table, referenced_columns in expected_keys:
authority_matching = [
(name, foreign_key)
for name, foreign_key in foreign_keys.items()
if tuple(foreign_key.get('columns') or ()) == columns
and foreign_key.get('referenced_table') == referenced_table
and tuple(foreign_key.get('referenced_columns') or ()) == referenced_columns
and foreign_key.get('referenced_schema') in (
conn.application_schema if conn.is_postgres else 'main',
)
]
if len(authority_matching) > 1:
raise RuntimeSafetySchemaError(
f'{table} has duplicate foreign keys for ({", ".join(columns)}); '
'manual reviewed constraint cleanup is required'
)
expected_actions = _required_foreign_key_actions(table, columns)
if authority_matching and not _foreign_key_actions_match(
authority_matching[0][1], expected_actions,
):
raise RuntimeSafetySchemaError(
f'{table}({", ".join(columns)}) foreign key actions are not '
f'ON UPDATE {expected_actions[0]} ON DELETE {expected_actions[1]}; '
'manual reviewed constraint repair is required'
)
if authority_matching:
name, foreign_key = authority_matching[0]
if conn.is_postgres and not foreign_key.get('valid', True):
try:
conn.execute(f'ALTER TABLE {table} VALIDATE CONSTRAINT {_quoted_pg_name(name)}')
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table} foreign key {name} cannot be validated against existing rows; '
'manually repair orphaned references before retrying migration'
) from exc
continue
conflicting = [
foreign_key for foreign_key in foreign_keys.values()
if tuple(foreign_key.get('columns') or ()) == columns
]
if conflicting:
raise RuntimeSafetySchemaError(
f'{table}({", ".join(columns)}) has a foreign key to the wrong authority; '
'manual reviewed constraint repair is required'
)
if not conn.is_postgres:
raise RuntimeSafetySchemaError(
f'SQLite {table} is missing foreign key ({", ".join(columns)}) -> '
f'{referenced_table}({", ".join(referenced_columns)}); '
'manually rebuild this table from a reviewed backup'
)
name = f'fk_{table}_{"_".join(columns)}'
try:
conn.execute(
f'ALTER TABLE {table} ADD CONSTRAINT {name} '
f'FOREIGN KEY ({", ".join(columns)}) REFERENCES '
f'{POSTGRES_APPLICATION_SCHEMA}.{referenced_table} ({", ".join(referenced_columns)}) '
f'ON UPDATE {expected_actions[0]} ON DELETE {expected_actions[1]}'
)
except Exception as exc:
raise RuntimeSafetySchemaError(
f'PostgreSQL {table} cannot add foreign key ({", ".join(columns)}) because '
'existing rows do not match the referenced authority; manually repair orphaned rows'
) from exc
foreign_keys = conn.table_foreign_keys(table)
def _migration_ensure_index(
conn, table, name, sql, columns, unique=False, predicate='', required_sql_fragments=(),
):
current = conn.table_indexes(table).get(name)
current_sql = str((current or {}).get('sql') or '').lower()
valid = bool(
current
and bool(current['unique']) == bool(unique)
and current['columns'] == list(columns)
and _normalized_predicate(current['predicate']) == _normalized_predicate(predicate)
and _index_usable(current)
and all(str(fragment).lower() in current_sql for fragment in required_sql_fragments)
)
if valid:
return
if current:
conn.execute(f'DROP INDEX {name}')
conn.execute(sql)
def _ensure_runtime_operations_authority(conn, now=None):
timestamp = now or utc_now_iso()
conn.execute(
'''INSERT INTO runtime_operations_control(
id, revision, discovery_paused, dispatch_paused, drain_state,
actor, operation_id, created_at, updated_at
) VALUES (1, 0, 0, 0, 'normal', 'system:migration', NULL, ?, ?)
ON CONFLICT(id) DO NOTHING''',
(timestamp, timestamp),
)
if conn.is_postgres:
conn.execute('''
CREATE OR REPLACE FUNCTION reject_runtime_audit_event_mutation()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
RAISE EXCEPTION 'runtime_audit_events is append-only';
END;
$$
''')
conn.execute(
'DROP TRIGGER IF EXISTS runtime_audit_events_reject_mutation '
'ON runtime_audit_events'
)
conn.execute('''
CREATE TRIGGER runtime_audit_events_reject_mutation
BEFORE UPDATE OR DELETE ON runtime_audit_events
FOR EACH ROW EXECUTE FUNCTION reject_runtime_audit_event_mutation()
''')
conn.execute(
'DROP TRIGGER IF EXISTS runtime_audit_events_reject_truncate '
'ON runtime_audit_events'
)
conn.execute('''
CREATE TRIGGER runtime_audit_events_reject_truncate
BEFORE TRUNCATE ON runtime_audit_events
FOR EACH STATEMENT EXECUTE FUNCTION reject_runtime_audit_event_mutation()
''')
return
conn.execute('DROP TRIGGER IF EXISTS runtime_audit_events_reject_update')
conn.execute('''
CREATE TRIGGER runtime_audit_events_reject_update
BEFORE UPDATE ON runtime_audit_events
BEGIN
SELECT RAISE(ABORT, 'runtime_audit_events is append-only');
END
''')
conn.execute('DROP TRIGGER IF EXISTS runtime_audit_events_reject_delete')
conn.execute('''
CREATE TRIGGER runtime_audit_events_reject_delete
BEFORE DELETE ON runtime_audit_events
BEGIN
SELECT RAISE(ABORT, 'runtime_audit_events is append-only');
END
''')
def _migration_retain_pg_trgm_without_name_index(conn):
if not conn.is_postgres:
return
extension = conn.execute(
'''SELECT EXISTS (
SELECT 1 FROM pg_catalog.pg_available_extensions WHERE name = ?
) AS available''',
('pg_trgm',),
).fetchone()
if extension and extension['available']:
conn.execute('SAVEPOINT package_candidate_trgm')
try:
conn.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm')
conn.execute('RELEASE SAVEPOINT package_candidate_trgm')
except Exception as exc:
conn.execute('ROLLBACK TO SAVEPOINT package_candidate_trgm')
conn.execute('RELEASE SAVEPOINT package_candidate_trgm')
logger.warning(
'pg_trgm extension retention failed; bounded ordered lookup remains active: %s',
type(exc).__name__,
)
conn.execute('DROP INDEX IF EXISTS idx_package_repo_candidates_name_trgm')
def _migration_reconcile_pipeline_capacity(conn, page_size=1000):
totals = {
'bundle_items': 0, 'bundle_bytes': 0,
'projection_items': 0, 'projection_bytes': 0,
'keycheck_items': 0, 'keycheck_bytes': 0,
'quarantine_items': 0, 'quarantine_bytes': 0,
}
def pages(table, columns, where):
after_id = 0
while True:
rows = conn.execute(
f'''SELECT id, {columns} FROM {table}
WHERE id > ? AND ({where}) ORDER BY id LIMIT ?''',
(after_id, max(1, int(page_size))),
).fetchall()
if not rows:
return
for row in rows:
after_id = int(row['id'])
yield row
for row in pages(
'result_reservations',
'''reserved_bundle_bytes, reserved_projection_items, reserved_projection_bytes,
reserved_candidate_items, reserved_candidate_bytes, bundle_credit_released,
projection_credit_transferred, candidate_credit_transferred''',
"state IN ('scanning','ready','ingesting','db_committed')",
):
if not row['bundle_credit_released']:
totals['bundle_items'] += 1
totals['bundle_bytes'] += int(row['reserved_bundle_bytes'])
if not row['projection_credit_transferred']:
totals['projection_items'] += int(row['reserved_projection_items'])
totals['projection_bytes'] += int(row['reserved_projection_bytes'])
if not row['candidate_credit_transferred']:
totals['keycheck_items'] += int(row['reserved_candidate_items'])
totals['keycheck_bytes'] += int(row['reserved_candidate_bytes'])
for row in pages(
'projection_jobs', 'capacity_items, capacity_bytes',
"status IN ('pending','leased') AND capacity_released = 0",
):
totals['projection_items'] += int(row['capacity_items'])
totals['projection_bytes'] += int(row['capacity_bytes'])
for row in pages(
'keycheck_candidates',
'capacity_bytes, result_projection_reserved_bytes, result_projection_credit_transferred',
"state IN ('pending','leased','deferred') AND capacity_released = 0",
):
totals['keycheck_items'] += 1
totals['keycheck_bytes'] += int(row['capacity_bytes'])
if (
int(row['result_projection_reserved_bytes'] or 0) > 0
and not row['result_projection_credit_transferred']
):
totals['projection_items'] += 1
totals['projection_bytes'] += int(row['result_projection_reserved_bytes'])
for row in pages(
'pipeline_quarantine', 'capacity_items, capacity_bytes',
"review_status = 'pending' AND capacity_credit_applied = 1",
):
totals['quarantine_items'] += int(row['capacity_items'])
totals['quarantine_bytes'] += int(row['capacity_bytes'])
conn.execute(
'''UPDATE pipeline_capacity SET bundle_items = ?, bundle_bytes = ?,
projection_items = ?, projection_bytes = ?, keycheck_items = ?,
keycheck_bytes = ?, quarantine_items = ?, quarantine_bytes = ?,
updated_at = ? WHERE id = 1''',
(
totals['bundle_items'], totals['bundle_bytes'], totals['projection_items'],
totals['projection_bytes'], totals['keycheck_items'], totals['keycheck_bytes'],
totals['quarantine_items'], totals['quarantine_bytes'], utc_now_iso(),
),
)
return totals
def migrate_runtime_safety_schema(db, initialize_base=False):
conn = getattr(db, 'conn', None)
if not conn:
raise RuntimeSafetySchemaError('database connection is unavailable')
try:
if initialize_base:
conn.executescript(SCHEMA_SQL)
for table in ('target_queue', 'target_scans', 'findings'):
if not conn.table_exists(table):
raise RuntimeSafetySchemaError(f'base schema table is missing: {table}')
_migration_require_pipeline_quiescence(conn)
_migration_add_columns(conn, 'target_queue', {
'lease_token': 'TEXT',
'claim_batch': 'TEXT',
'resolver_state': 'TEXT',
'resolver_due_at': 'TEXT',
'resolver_attempts': 'INTEGER NOT NULL DEFAULT 0',
'resolver_token': 'TEXT',
'current_result_reservation_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
'claim_event_id': 'TEXT',
'remote_modified_at': 'TEXT',
'scan_remote_modified_at': 'TEXT',
'covered_ref': 'TEXT',
'covered_head': 'TEXT',
})
id_type = 'BIGINT' if conn.is_postgres else 'INTEGER'
_migration_add_columns(conn, 'target_scans', {
'scan_event_id': 'TEXT',
'scan_event_hash': 'TEXT',
'queue_id': id_type,
'claim_lease_token': 'TEXT',
'queue_completion_applied': 'INTEGER NOT NULL DEFAULT 0',
'queue_completion_disposition': 'TEXT',
'result_reservation_id': id_type,
'compat_schema_version': 'INTEGER NOT NULL DEFAULT 2',
'raw_result_storage': "TEXT NOT NULL DEFAULT 'legacy'",
})
_migration_add_columns(conn, 'findings', {
'detector_type': 'TEXT',
'verified': 'INTEGER DEFAULT 0',
'raw_secret': 'TEXT',
'redacted_secret': 'TEXT',
'secret_hash': 'TEXT',
'finding_uid': 'TEXT',
'detector_secret_hash': 'TEXT',
'finding_fingerprint': 'TEXT',
'file_path': 'TEXT',
'line_number': 'TEXT',
'commit_hash': 'TEXT',
'source_timestamp': 'TEXT',
'source_metadata_type': 'TEXT',
'raw_finding_json': 'TEXT',
'source_metadata_json': 'TEXT',
'provider': 'TEXT',
'credential_kind': 'TEXT',
'credential_confidence': 'TEXT',
'required_context_missing': 'INTEGER DEFAULT 0',
'principal': 'TEXT',
'username': 'TEXT',
'email': 'TEXT',
'project_id': 'TEXT',
'tenant_id': 'TEXT',
'organization': 'TEXT',
'registry': 'TEXT',
'endpoint': 'TEXT',
'scope': 'TEXT',
'resource': 'TEXT',
'enrichment_json': 'TEXT',
'raw_payload_sha256': 'TEXT',
'raw_payload_bytes': id_type,
'raw_payload_omitted': 'INTEGER NOT NULL DEFAULT 0',
})
_migration_add_columns(conn, 'runs', {
'total_staged': 'INTEGER NOT NULL DEFAULT 0',
'total_quarantined': 'INTEGER NOT NULL DEFAULT 0',
})
_migration_add_columns(conn, 'source_cycles', {
'queued_updated_count': 'INTEGER NOT NULL DEFAULT 0',
'staged_count': 'INTEGER NOT NULL DEFAULT 0',
'ingested_count': 'INTEGER NOT NULL DEFAULT 0',
'quarantined_count': 'INTEGER NOT NULL DEFAULT 0',
})
conn.execute(f'''CREATE TABLE IF NOT EXISTS finding_uid_map (
finding_uid TEXT PRIMARY KEY,
finding_id {id_type} NOT NULL,
created_at TEXT NOT NULL,
FOREIGN KEY(finding_id) REFERENCES findings(id)
)''')
_migration_postgres_column_shape(conn, 'finding_uid_map', {
'finding_uid': ('text', True), 'finding_id': ('id_ref', True), 'created_at': ('text', True),
})
outbox_id = 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' if conn.is_postgres else 'INTEGER PRIMARY KEY AUTOINCREMENT'
conn.execute(f'''CREATE TABLE IF NOT EXISTS scan_publication_outbox (
id {outbox_id},
target_scan_id {id_type} NOT NULL UNIQUE,
payload_json TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER DEFAULT 0,
last_error TEXT,
lease_owner TEXT,
lease_expires_at TEXT,
available_after TEXT,
created_at TEXT NOT NULL,
delivered_at TEXT,
updated_at TEXT NOT NULL,
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
)''')
_migration_add_columns(conn, 'scan_publication_outbox', {
'payload_json': "TEXT NOT NULL DEFAULT ''",
'status': "TEXT NOT NULL DEFAULT 'pending'",
'attempts': 'INTEGER DEFAULT 0',
'last_error': 'TEXT',
'lease_owner': 'TEXT',
'lease_expires_at': 'TEXT',
'available_after': 'TEXT',
'created_at': "TEXT NOT NULL DEFAULT ''",
'delivered_at': 'TEXT',
'updated_at': "TEXT NOT NULL DEFAULT ''",
})
_migration_postgres_column_shape(conn, 'scan_publication_outbox', OUTBOX_COLUMN_SPECS)
compact_delivered_scan_publications(conn)
conn.execute(
'''UPDATE scan_publication_outbox SET payload_json = '',
status = CASE WHEN status = 'delivering' THEN 'delivering' ELSE 'pending' END,
lease_owner = CASE WHEN status = 'delivering' THEN lease_owner ELSE NULL END,
lease_expires_at = CASE WHEN status = 'delivering' THEN lease_expires_at ELSE NULL END,
available_after = CASE WHEN status = 'dead' THEN NULL ELSE available_after END
WHERE payload_json != '' OR status NOT IN ('pending', 'delivering')'''
)
cursor_integer = 'BIGINT' if conn.is_postgres else 'INTEGER'
conn.execute(f'''CREATE TABLE IF NOT EXISTS target_queue_reconciliation_cursors (
source_file TEXT PRIMARY KEY,
file_identity TEXT NOT NULL,
file_size {cursor_integer} NOT NULL DEFAULT 0,
file_mtime_ns {cursor_integer} NOT NULL DEFAULT 0,
source TEXT NOT NULL,
platform TEXT NOT NULL,
byte_offset {cursor_integer} NOT NULL DEFAULT 0,
line_number {cursor_integer} NOT NULL DEFAULT 0,
discarding_oversized INTEGER NOT NULL DEFAULT 0,
oversized_line_start {cursor_integer},
cumulative_rows {cursor_integer} NOT NULL DEFAULT 0,
cumulative_bytes {cursor_integer} NOT NULL DEFAULT 0,
cumulative_inserted {cursor_integer} NOT NULL DEFAULT 0,
cumulative_rejected {cursor_integer} NOT NULL DEFAULT 0,
completed_at TEXT,
last_report_json TEXT,
updated_at TEXT NOT NULL
)''')
_migration_add_columns(conn, 'target_queue_reconciliation_cursors', {
'file_size': f'{cursor_integer} NOT NULL DEFAULT 0',
'file_mtime_ns': f'{cursor_integer} NOT NULL DEFAULT 0',
'discarding_oversized': 'INTEGER NOT NULL DEFAULT 0',
'oversized_line_start': cursor_integer,
'cumulative_rows': f'{cursor_integer} NOT NULL DEFAULT 0',
'cumulative_bytes': f'{cursor_integer} NOT NULL DEFAULT 0',
'cumulative_inserted': f'{cursor_integer} NOT NULL DEFAULT 0',
'cumulative_rejected': f'{cursor_integer} NOT NULL DEFAULT 0',
'completed_at': 'TEXT',
})
_migration_postgres_column_shape(conn, 'target_queue_reconciliation_cursors', CURSOR_COLUMN_SPECS)
issue_id = 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' if conn.is_postgres else 'INTEGER PRIMARY KEY AUTOINCREMENT'
conn.execute(f'''CREATE TABLE IF NOT EXISTS target_queue_reconciliation_issues (
id {issue_id},
source_file TEXT NOT NULL,
file_identity TEXT NOT NULL,
source TEXT NOT NULL,
platform TEXT NOT NULL,
line_number {cursor_integer} NOT NULL,
byte_offset {cursor_integer} NOT NULL,
reason TEXT NOT NULL,
target_preview TEXT,
created_at TEXT NOT NULL,
resolved_at TEXT
)''')
_migration_postgres_column_shape(conn, 'target_queue_reconciliation_issues', RECONCILIATION_ISSUE_COLUMN_SPECS)
conn.executescript(KEYCHECK_RESULTS_SQL)
if 'id' not in conn.table_columns('keycheck_results'):
raise RuntimeSafetySchemaError('keycheck_results.id is missing and cannot be repaired additively')
_migration_add_columns(conn, 'keycheck_results', {
'source_line': 'TEXT',
'detector_secret_hash': 'TEXT',
'event_id': 'TEXT',
'finding_uid': 'TEXT',
'link_status': "TEXT DEFAULT 'pending'",
'link_attempts': 'INTEGER DEFAULT 0',
'linked_at': 'TEXT',
'link_error': 'TEXT',
'candidate_id': id_type,
'credential_id': id_type,
'result_source': "TEXT NOT NULL DEFAULT 'api_check'",
})
conn.execute(f'''CREATE TABLE IF NOT EXISTS keycheck_event_map (
event_id TEXT PRIMARY KEY,
keycheck_result_id {id_type},
created_at TEXT NOT NULL,
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
)''')
_migration_postgres_column_shape(conn, 'keycheck_results', KEYCHECK_COLUMN_SPECS)
_migration_postgres_column_shape(conn, 'keycheck_event_map', {
'event_id': ('text', True), 'keycheck_result_id': ('id_ref', False), 'created_at': ('text', True),
})
conn.executescript(KEYCHECK_RESULTS_SQL)
# The canonical schema creates this index, so an existing marker-13 table
# needs the additive selector column before the schema script reaches it.
if conn.table_exists('docker_image_blob_coverage'):
_migration_add_columns(conn, 'docker_image_blob_coverage', {
'selection_policy_sha256': "TEXT NOT NULL DEFAULT ''",
})
if conn.table_exists('target_queue_policy_events'):
experiment_reference = (
'BIGINT' if conn.is_postgres
else 'INTEGER REFERENCES docker_depth_experiments(id)'
)
_migration_add_columns(conn, 'target_queue_policy_events', {
'experiment_id': experiment_reference,
})
if conn.table_exists('discovery_retry_queue'):
source_cycle_reference = (
'BIGINT' if conn.is_postgres
else 'INTEGER REFERENCES source_cycles(id)'
)
_migration_add_columns(conn, 'discovery_retry_queue', {
'source_cycle_id': source_cycle_reference,
})
rollout_authority_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
(DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION,),
).fetchone()
schema_selector_authority_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
(DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION,),
).fetchone()
scarcity_cohort_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
(DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION,),
).fetchone()
if conn.table_exists('docker_depth_experiments'):
_migration_add_columns(conn, 'docker_depth_experiments', {
'collection_generation': (
"TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1'"
),
'selection_sha256': 'TEXT',
})
if not rollout_authority_migration:
conn.execute(
"UPDATE docker_depth_experiments "
"SET collection_generation = 'legacy'"
)
if conn.table_exists('docker_discovery_passes'):
_migration_add_columns(conn, 'docker_discovery_passes', {
'collection_generation': (
"TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1'"
),
})
if not rollout_authority_migration:
conn.execute(
"UPDATE docker_discovery_passes "
"SET collection_generation = 'legacy'"
)
if conn.table_exists('docker_discovery_pages'):
count_type = 'BIGINT' if conn.is_postgres else 'INTEGER'
_migration_add_columns(conn, 'docker_discovery_pages', {
'total_count': count_type,
})
if (
conn.table_exists('docker_finding_layer_attributions')
and 'unattributed_reason' not in conn.table_columns(
'docker_finding_layer_attributions'
)):
if conn.execute(
"SELECT id FROM docker_finding_layer_attributions "
"WHERE attribution_state = 'unattributed' LIMIT 1"
).fetchone():
raise RuntimeSafetySchemaError(
'existing unattributed Docker findings require a reviewed reason backfill'
)
reason_check = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[
'docker_finding_layer_attributions'
]['docker_finding_layer_attributions_reason_check']
_migration_add_columns(conn, 'docker_finding_layer_attributions', {
'unattributed_reason': (
'TEXT CONSTRAINT docker_finding_layer_attributions_reason_check '
f'CHECK ({reason_check})'
),
})
if conn.table_exists('docker_depth_experiment_repositories'):
resolver_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
('20260910_20_docker_depth_resolver',),
).fetchone()
_migration_add_columns(conn, 'docker_depth_experiment_repositories', {
'planned_is_deep_probe': 'INTEGER NOT NULL DEFAULT 0',
'resolver_due_at': 'TEXT',
'candidate_distinct_graph_count': 'INTEGER NOT NULL DEFAULT 0',
'replacement_repository_queue_id': (
'BIGINT' if conn.is_postgres else 'INTEGER'
),
'replacement_eligibility_page_id': (
'BIGINT' if conn.is_postgres else 'INTEGER'
),
'replacement_count': 'INTEGER NOT NULL DEFAULT 0',
'replacement_evidence_sha256': 'TEXT',
})
if not resolver_migration:
conn.execute(
'''UPDATE docker_depth_experiment_repositories
SET planned_is_deep_probe = is_deep_probe'''
)
if conn.table_exists('docker_depth_experiment_queries'):
selection_check = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[
'docker_depth_experiment_queries'
]['docker_depth_experiment_queries_selection_check']
_migration_add_columns(conn, 'docker_depth_experiment_queries', {
'selected_repository_count': (
'INTEGER NOT NULL DEFAULT 0 '
'CONSTRAINT docker_depth_experiment_queries_selection_check '
f'CHECK ({selection_check})'
),
})
if not scarcity_cohort_migration:
conn.execute(
'''UPDATE docker_depth_experiment_queries
SET selected_repository_count = required_repository_count'''
)
# Existing installations must gain columns before the schema script
# creates partial indexes that reference them. Fresh installs create
# the complete tables directly from PIPELINE_SCHEMA_SQL below.
remote_capacity_applied = False
if conn.table_exists('result_reservations'):
remote_capacity_applied = bool(conn.execute(
'''SELECT version FROM runtime_schema_migrations
WHERE version = ?''',
(REMOTE_ASSIGNMENT_CAPACITY_MIGRATION,),
).fetchone()) if conn.table_exists('runtime_schema_migrations') else False
projection_authority_applied = bool(conn.execute(
'''SELECT version FROM runtime_schema_migrations
WHERE version = ?''',
(DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION,),
).fetchone()) if conn.table_exists('runtime_schema_migrations') else False
_migration_add_columns(conn, 'result_reservations', {
'reserved_bundle_bytes': (
'BIGINT CHECK (reserved_bundle_bytes > 0)'
),
'assignment_kind': "TEXT NOT NULL DEFAULT 'local' CHECK (assignment_kind IN ('local','remote'))",
'remote_user_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
'remote_device_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
'remote_issued_at': 'TEXT', 'remote_expires_at': 'TEXT',
'remote_result_upload_body_timeout_seconds': (
'INTEGER CHECK ('
'remote_result_upload_body_timeout_seconds IS NULL OR '
'remote_result_upload_body_timeout_seconds BETWEEN 30 AND 86400)'
),
'remote_effective_config_sha256': 'TEXT',
'remote_client_compat_sha256': 'TEXT',
'remote_execution_snapshot_json': 'TEXT',
'remote_execution_snapshot_sha256': 'TEXT',
'remote_resolution_kind': 'TEXT', 'remote_payload_sha256': 'TEXT',
'remote_receipt_id': 'TEXT', 'remote_resolution_json': 'TEXT',
'remote_resolved_at': 'TEXT',
'remote_diagnostic_projection_version': 'INTEGER',
'remote_diagnostic_count': 'INTEGER',
'remote_diagnostic_uids_sha256': 'TEXT',
})
if not remote_capacity_applied:
conn.execute(
'''UPDATE result_reservations
SET reserved_bundle_bytes = declared_bundle_bytes
WHERE reserved_bundle_bytes IS NULL'''
)
if conn.is_postgres:
conn.execute(
'''ALTER TABLE result_reservations
ALTER COLUMN reserved_bundle_bytes SET NOT NULL'''
)
if not projection_authority_applied:
conn.execute(
'''UPDATE result_reservations
SET remote_diagnostic_projection_version = 0
WHERE assignment_kind = 'remote'
AND remote_resolution_kind = 'bundle_accepted'
AND remote_diagnostic_projection_version IS NULL'''
)
if conn.table_exists('admission_intents'):
_migration_add_columns(conn, 'admission_intents', {
'remote_user_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
'remote_device_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
})
conn.executescript(PIPELINE_SCHEMA_SQL)
_ensure_runtime_operations_authority(conn)
_migration_ensure_pipeline_quarantine_review_status_constraint(conn)
_migration_add_docker_depth_schema_authority_checks(
conn, bool(schema_selector_authority_migration),
)
_migration_add_docker_depth_scarcity_checks(
conn, bool(scarcity_cohort_migration),
)
_migration_require_docker_depth_check_constraints(conn)
_migration_add_columns(conn, 'pipeline_quarantine', {
'capacity_credit_applied': 'INTEGER NOT NULL DEFAULT 1',
'capacity_items': 'BIGINT NOT NULL DEFAULT 1',
'capacity_bytes': 'BIGINT NOT NULL DEFAULT 0',
})
_migration_add_columns(conn, 'result_reservations', {
'cleanup_attempts': 'INTEGER NOT NULL DEFAULT 0',
'cleanup_available_after': 'TEXT',
'git_scan_plan_json': 'TEXT',
'git_scan_plan_sha256': 'TEXT',
'docker_layer_plan_json': 'TEXT',
'docker_layer_plan_sha256': 'TEXT',
})
_migration_add_columns(conn, 'docker_content_blobs', {
'coverage_policy_sha256': 'TEXT',
})
_migration_add_columns(conn, 'docker_image_blob_coverage', {
'coverage_policy_sha256': 'TEXT',
'selection_policy_sha256': "TEXT NOT NULL DEFAULT ''",
})
_migration_backfill_docker_selection_policies(conn)
_migration_add_columns(conn, 'pipeline_artifacts', {
'cleanup_attempts': 'INTEGER NOT NULL DEFAULT 0',
'cleanup_available_after': 'TEXT',
'cleanup_last_error': 'TEXT',
})
conn.execute(
'''UPDATE pipeline_quarantine SET capacity_bytes = byte_count
WHERE capacity_credit_applied = 1 AND capacity_bytes = 0 AND byte_count > 0'''
)
_migration_add_columns(conn, 'keycheck_credentials', {
'provider_key_hash': "TEXT NOT NULL DEFAULT ''",
})
_migration_add_columns(conn, 'keycheck_candidates', {
'secret_hash': "TEXT NOT NULL DEFAULT ''",
'routed_service': "TEXT NOT NULL DEFAULT ''",
'result_projection_reserved_bytes': 'BIGINT NOT NULL DEFAULT 0',
'result_projection_credit_transferred': 'INTEGER NOT NULL DEFAULT 0',
})
_migration_add_columns(conn, 'docker_adaptive_shadow_reports', {
'selection_metrics_json': "TEXT NOT NULL DEFAULT '{}'",
'sink_checkpoint_count': 'INTEGER NOT NULL DEFAULT 0',
})
_migration_backfill_keycheck_hashes(conn)
conn.execute(
'''CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_credentials_provider_key
ON keycheck_credentials(service, provider_key_hash)'''
)
now = utc_now_iso()
conn.execute(
'''INSERT INTO pipeline_capacity(id, updated_at) VALUES (1, ?)
ON CONFLICT(id) DO NOTHING''',
(now,),
)
for stream_name, relative_path, rotation_bytes in (
('scan_results', 'scan_results.jsonl', 256 * 1024 * 1024),
('found_secrets', 'found_secrets.jsonl', 128 * 1024 * 1024),
('scan_errors', 'scan_errors.log', 32 * 1024 * 1024),
):
conn.execute(
'''INSERT INTO projection_streams(
stream_name, base_relative_path, current_generation,
rotation_bytes, max_generations, created_at, updated_at
) VALUES (?, ?, 0, ?, 16, ?, ?)
ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, relative_path, rotation_bytes, now, now),
)
conn.execute(
'''INSERT INTO projection_cursors(
stream_name, generation, committed_offset, updated_at
) VALUES (?, 0, 0, ?)
ON CONFLICT(stream_name) DO NOTHING''',
(stream_name, now),
)
migration_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
capacity_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
(PIPELINE_MIGRATION_VERSIONS[0],),
).fetchone()
if not capacity_migration or not remote_capacity_applied:
_migration_reconcile_pipeline_capacity(conn)
single_writer_migration = conn.execute(
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
('20260729_10_keycheck_projection_single_writer',),
).fetchone()
if not single_writer_migration:
prepared_status = conn.execute(
'''SELECT a.id FROM projection_appends a
JOIN projection_jobs j ON j.id = a.job_id
WHERE j.job_kind = 'keycheck_event'
AND j.status IN ('pending','leased')
AND (j.required_stream_mask & 16) <> 0
AND a.state = 'prepared' AND a.stream_name LIKE 'keycheck:%:status'
LIMIT 1'''
).fetchone()
if prepared_status:
raise RuntimeSafetySchemaError(
'keycheck status projection cutover requires reviewed prepared-append recovery'
)
conn.execute(
'''UPDATE projection_jobs
SET required_stream_mask = required_stream_mask - 16, updated_at = ?
WHERE job_kind = 'keycheck_event' AND status IN ('pending','leased')
AND (required_stream_mask & 16) <> 0''',
(now,),
)
for version in PIPELINE_MIGRATION_VERSIONS:
conn.execute(
'''INSERT INTO runtime_schema_migrations(version, applied_at, code_sha256)
VALUES (?, ?, ?) ON CONFLICT(version) DO NOTHING''',
(version, now, migration_code),
)
conn.execute(
'''UPDATE runtime_schema_migrations SET applied_at = ?, code_sha256 = ?
WHERE version = ?''',
(now, migration_code, PIPELINE_MIGRATION_VERSIONS[-1]),
)
for table, specs in RUNTIME_TABLE_SPECS.items():
if conn.is_postgres:
_migration_postgres_column_shape(conn, table, specs)
else:
_migration_require_sqlite_column_shape(conn, table, specs)
_migration_require_primary_key(conn, table, specs, RUNTIME_PRIMARY_KEYS[table])
generated_id = GENERATED_ID_COLUMNS.get(table)
if generated_id:
_migration_ensure_generated_id(conn, table, generated_id)
_migration_ensure_defaults(
conn, table, specs, RUNTIME_COLUMN_DEFAULTS.get(table, {}), generated_id,
)
_migration_ensure_foreign_keys(conn)
_migration_seed_legacy_docker_provenance(conn)
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
statement = (
f'CREATE {"UNIQUE " if unique else ""}INDEX {name} '
f'ON {table}({", ".join(columns)})'
)
if predicate:
statement += f' WHERE {predicate}'
_migration_ensure_index(
conn, table, name, statement, list(columns),
unique=unique, predicate=predicate,
)
now = utc_now_iso()
unresolved = conn.execute(
'''SELECT id, target FROM target_queue
WHERE platform = 'docker' AND status IN ('pending', 'deferred')
AND resolver_state IS NULL'''
).fetchall()
resolver_due = datetime.fromtimestamp(
time.time() + max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)), timezone.utc
).isoformat(timespec='seconds')
for row in unresolved:
target = str(row['target'] or '').strip()
if target and '@' not in target and ':' not in target.rsplit('/', 1)[-1]:
conn.execute(
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'pending', resolver_due_at = ?,
available_after = COALESCE(available_after, ?), updated_at = ? WHERE id = ?''',
(resolver_due, resolver_due, now, row['id']),
)
conn.execute(
'''UPDATE target_queue SET
resolver_state = CASE WHEN resolver_state = 'resolving' THEN 'retry' ELSE resolver_state END,
resolver_due_at = ?, resolver_token = CASE WHEN resolver_state = 'resolving' THEN NULL ELSE resolver_token END,
available_after = COALESCE(available_after, ?), updated_at = ?
WHERE platform = 'docker' AND status = 'deferred'
AND resolver_state IN ('pending', 'retry', 'resolving')
AND resolver_due_at IS NULL''',
(resolver_due, resolver_due, now),
)
_migration_ensure_index(
conn, 'target_scans', 'uq_target_scans_scan_event_id',
'''CREATE UNIQUE INDEX uq_target_scans_scan_event_id
ON target_scans(scan_event_id) WHERE scan_event_id IS NOT NULL''',
['scan_event_id'], unique=True, predicate='scan_event_id IS NOT NULL',
)
_migration_ensure_index(
conn, 'target_scans', 'idx_target_scans_event_hash',
'CREATE INDEX idx_target_scans_event_hash ON target_scans(scan_event_id, scan_event_hash)',
['scan_event_id', 'scan_event_hash'],
)
_migration_ensure_index(
conn, 'target_scans', 'idx_target_scans_queue_id',
'CREATE INDEX idx_target_scans_queue_id ON target_scans(queue_id)',
['queue_id'],
)
_migration_ensure_index(
conn, 'target_scans', 'idx_target_scans_result_reservation',
'''CREATE INDEX idx_target_scans_result_reservation
ON target_scans(result_reservation_id, id)''',
['result_reservation_id', 'id'],
)
_migration_ensure_index(
conn, 'target_queue', 'idx_target_queue_current_reservation',
'CREATE INDEX idx_target_queue_current_reservation ON target_queue(current_result_reservation_id)',
['current_result_reservation_id'],
)
_migration_ensure_index(
conn, 'target_queue', 'idx_target_queue_claimable_v2',
'''CREATE INDEX idx_target_queue_claimable_v2
ON target_queue(source, platform, status, available_after, id)
WHERE current_result_reservation_id IS NULL
AND (status = 'pending' OR status = 'deferred')''',
['source', 'platform', 'status', 'available_after', 'id'],
predicate="current_result_reservation_id IS NULL AND (status = 'pending' OR status = 'deferred')",
)
_migration_ensure_index(
conn, 'discovery_retry_queue', 'uq_discovery_retry_queue_work_key',
'''CREATE UNIQUE INDEX uq_discovery_retry_queue_work_key
ON discovery_retry_queue(work_key)''',
['work_key'], unique=True,
)
_migration_ensure_index(
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_due',
'''CREATE INDEX idx_discovery_retry_queue_due
ON discovery_retry_queue(source, available_after, id)
WHERE status = 'pending' ''',
['source', 'available_after', 'id'],
predicate=DISCOVERY_RETRY_DUE_INDEX_PREDICATE,
)
_migration_ensure_index(
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_lease',
'''CREATE INDEX idx_discovery_retry_queue_lease
ON discovery_retry_queue(lease_expires_at, id)
WHERE status = 'leased' ''',
['lease_expires_at', 'id'],
predicate=DISCOVERY_RETRY_LEASE_INDEX_PREDICATE,
)
_migration_ensure_index(
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_policy',
'''CREATE INDEX idx_discovery_retry_queue_policy
ON discovery_retry_queue(source, query, policy_sha256, status, id)''',
['source', 'query', 'policy_sha256', 'status', 'id'],
)
target_queue_indexes = conn.table_indexes('target_queue')
if not any(
index['unique'] and index['columns'] == ['source', 'normalized_target']
and not _normalized_predicate(index['predicate'])
for index in target_queue_indexes.values()
):
try:
_migration_ensure_index(
conn, 'target_queue', 'uq_target_queue_source_normalized',
'CREATE UNIQUE INDEX uq_target_queue_source_normalized ON target_queue(source, normalized_target)',
['source', 'normalized_target'], unique=True,
)
except Exception as exc:
raise RuntimeSafetySchemaError(
'target_queue cannot be made unique on (source, normalized_target); '
'manually resolve duplicate rows before retrying migration'
) from exc
_migration_ensure_index(
conn, 'scan_publication_outbox', 'uq_scan_publication_outbox_target_scan_id',
'CREATE UNIQUE INDEX uq_scan_publication_outbox_target_scan_id ON scan_publication_outbox(target_scan_id)',
['target_scan_id'], unique=True,
)
_migration_ensure_index(
conn, 'keycheck_event_map', 'uq_keycheck_event_map_event_id',
'CREATE UNIQUE INDEX uq_keycheck_event_map_event_id ON keycheck_event_map(event_id)',
['event_id'], unique=True,
)
_migration_ensure_index(
conn, 'finding_uid_map', 'uq_finding_uid_map_finding_uid',
'CREATE UNIQUE INDEX uq_finding_uid_map_finding_uid ON finding_uid_map(finding_uid)',
['finding_uid'], unique=True,
)
package_indexes = conn.table_indexes('package_repo_candidates')
if not any(
index['unique']
and index['columns'] == ['package_source', 'package_name', 'package_version', 'repo_url']
and not _normalized_predicate(index['predicate'])
and _index_usable(index)
for index in package_indexes.values()
):
try:
_migration_ensure_index(
conn, 'package_repo_candidates', 'uq_package_repo_candidates_identity',
'''CREATE UNIQUE INDEX uq_package_repo_candidates_identity
ON package_repo_candidates(package_source, package_name, package_version, repo_url)''',
['package_source', 'package_name', 'package_version', 'repo_url'], unique=True,
)
except Exception as exc:
raise RuntimeSafetySchemaError(
'package_repo_candidates cannot be made unique on its package/repository identity; '
'manually resolve duplicate rows before retrying migration'
) from exc
index_specs = (
('runs', 'idx_runs_started_at', 'started_at'),
('runs', 'idx_runs_status', 'status'),
('runs', 'idx_runs_selected_source_status', 'selected_source, status, id'),
('source_cycles', 'idx_source_cycles_run_id', 'run_id'),
('source_cycles', 'idx_source_cycles_source_query', 'source, query'),
('source_cycles', 'idx_source_cycles_source_status', 'source, status, id'),
('target_queue', 'idx_target_queue_source_status', 'source, status, updated_at'),
('target_queue', 'idx_target_queue_observe_source_status', 'source, status'),
('target_queue', 'idx_target_queue_lease', 'source, status, lease_expires_at'),
('target_queue', 'idx_target_queue_platform_status', 'platform, status'),
('target_queue', 'idx_target_queue_claim_batch', 'claim_batch, lease_owner'),
('target_queue', 'idx_target_queue_claim', 'source, platform, status, available_after, lease_expires_at, id'),
('target_queue', 'idx_target_queue_resolver_claim', 'source, platform, status, resolver_state, resolver_due_at, id'),
('target_queue', 'idx_target_queue_source_platform_normalized', 'source, platform, normalized_target'),
('scan_publication_outbox', 'idx_scan_publication_outbox_status', 'status, available_after, id'),
('scan_publication_outbox', 'idx_scan_publication_outbox_age', 'status, created_at, id'),
('target_scans', 'idx_target_scans_cycle_id', 'cycle_id'),
('target_scans', 'idx_target_scans_source_status', 'source, status'),
('target_scans', 'idx_target_scans_source_ended', 'source, ended_at'),
('target_scans', 'idx_target_scans_source_ended_id', 'source, ended_at, id'),
('target_scans', 'idx_target_scans_source_skip_ended', 'source, skipped_reason, ended_at'),
('target_scans', 'idx_target_scans_normalized_target', 'normalized_target'),
('keycheck_results', 'idx_keycheck_results_checked', 'checked_at'),
('keycheck_results', 'idx_keycheck_results_service_status', 'service, status_group, status'),
('keycheck_results', 'idx_keycheck_results_finding', 'finding_id'),
('keycheck_results', 'idx_keycheck_results_cycle', 'cycle_id'),
('keycheck_results', 'idx_keycheck_results_source_query', 'source, query'),
('keycheck_results', 'idx_keycheck_results_key_hash', 'key_hash'),
('keycheck_results', 'idx_keycheck_results_secret_hash', 'secret_hash'),
('keycheck_results', 'idx_keycheck_results_link_repair', 'link_status, id'),
('findings', 'idx_findings_finding_uid', 'finding_uid'),
('findings', 'idx_findings_target_scan_id_id', 'target_scan_id, id'),
('findings', 'idx_findings_cycle_id', 'cycle_id'),
('findings', 'idx_findings_source', 'source'),
('findings', 'idx_findings_secret_hash', 'secret_hash'),
('findings', 'idx_findings_provider', 'provider'),
('findings', 'idx_findings_confidence', 'credential_confidence'),
('errors', 'idx_errors_cycle_id', 'cycle_id'),
('errors', 'idx_errors_source_category', 'source, category'),
('errors', 'idx_errors_target_scan_id', 'target_scan_id'),
('queue_snapshots', 'idx_queue_snapshots_source_time', 'source, captured_at'),
('package_repo_candidates', 'idx_package_repo_candidates_query', 'query'),
('package_repo_candidates', 'idx_package_repo_candidates_repo', 'repo_url'),
('package_repo_candidates', 'idx_package_repo_candidates_source_seen', 'package_source, last_seen_at'),
('target_queue_reconciliation_issues', 'idx_reconciliation_issues_open', 'source_file, resolved_at, id'),
('target_queue_policy_events', 'idx_target_queue_policy_events_queue', 'queue_id, id'),
('target_queue_policy_events', 'idx_target_queue_policy_events_manifest', 'manifest_sha256, id'),
('docker_content_blobs', 'idx_docker_content_blobs_reclaim', 'state, available_after, lease_expires_at, digest'),
('docker_content_blobs', 'idx_docker_content_blobs_reservation', 'lease_reservation_id, digest'),
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_manifest', 'manifest_digest, coverage_state, position'),
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_blob', 'blob_digest, coverage_state, queue_id'),
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_reservation', 'reservation_id, position'),
(
'docker_image_blob_coverage',
'idx_docker_image_blob_coverage_selection',
'queue_id, manifest_digest, selection_policy_sha256, position, reservation_id',
),
(
'docker_adaptive_shadow_reports',
'idx_docker_adaptive_shadow_reports_gate',
'scan_policy_sha256, execution_policy_sha256, selection_policy_sha256, '
'state, completed_at, id',
),
)
for table, name, column_sql in index_specs:
columns = [value.strip() for value in column_sql.split(',')]
_migration_ensure_index(
conn, table, name, f'CREATE INDEX {name} ON {table}({column_sql})', columns,
)
_migration_ensure_index(
conn,
'target_scans',
'idx_target_scans_cooldown_recent',
f'''CREATE INDEX idx_target_scans_cooldown_recent
ON target_scans(source, ended_at DESC, id DESC)
WHERE {CI_COOLDOWN_INDEX_PREDICATE}''',
['source', 'ended_at', 'id'],
predicate=CI_COOLDOWN_INDEX_PREDICATE,
)
for name, columns in (
(
'idx_package_repo_candidates_query_seen',
'query, last_seen_at DESC, id DESC',
),
(
'idx_package_repo_candidates_recent_lookup',
'last_seen_at DESC, id DESC, package_source, query, package_name',
),
):
_migration_ensure_index(
conn,
'package_repo_candidates',
name,
f'''CREATE INDEX {name} ON package_repo_candidates({columns})
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE}''',
[value.strip().split()[0] for value in columns.split(',')],
predicate=PACKAGE_REPO_NONEMPTY_PREDICATE,
)
_migration_retain_pg_trgm_without_name_index(conn)
for name, columns, predicate in (
(
'idx_target_queue_claim_pending',
'source, platform, id, attempts, available_after',
"status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')",
),
(
'idx_target_queue_claim_deferred',
'source, platform, available_after, id, attempts',
"status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')",
),
(
'idx_target_queue_claim_in_progress',
'source, platform, id, attempts, available_after, lease_expires_at, resolver_state',
"status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved')",
),
(
'idx_target_queue_active_lease_owner_token',
'lease_owner, lease_token',
"status = 'in_progress'",
),
(
'idx_target_queue_exhausted_attempts',
'source, platform, status, attempts, id, lease_expires_at',
"status = 'pending' OR status = 'deferred' OR status = 'in_progress'",
),
(
'idx_target_queue_updated_rescan',
'source, platform, completed_at, remote_modified_at, scan_remote_modified_at, id',
"status = 'done' AND remote_modified_at IS NOT NULL",
),
(
'idx_target_queue_cold',
'source, platform, query, id',
"status = 'cold'",
),
):
_migration_ensure_index(
conn,
'target_queue',
name,
f'CREATE INDEX {name} ON target_queue({columns}) WHERE {predicate}',
[value.strip() for value in columns.split(',')],
predicate=predicate,
)
if conn.is_postgres:
conn.execute(
'''CREATE STATISTICS IF NOT EXISTS st_target_queue_claim_selectivity
(dependencies, mcv)
ON source, platform, status, resolver_state, attempts
FROM target_queue'''
)
conn.execute('ANALYZE target_queue')
conn.execute('ANALYZE target_scans')
conn.execute('ANALYZE findings')
conn.execute('ANALYZE package_repo_candidates')
_migration_ensure_index(
conn, 'target_queue_reconciliation_issues', 'uq_reconciliation_issue_identity',
'''CREATE UNIQUE INDEX uq_reconciliation_issue_identity
ON target_queue_reconciliation_issues(
source_file, file_identity, line_number, byte_offset, reason
)''',
['source_file', 'file_identity', 'line_number', 'byte_offset', 'reason'], unique=True,
)
db._runtime_safety_schema_validated = False
db.require_runtime_safety_schema(commit=False)
conn.commit()
db._docker_depth_experiment_schema_installed_cache = True
return True
except Exception:
try:
conn.rollback()
except Exception:
pass
raise
def first_line(value, limit=500):
for line in str(value or '').splitlines():
line = line.strip()
if line:
return line[:limit]
return ''
def elapsed_seconds(start, end):
start_dt = parse_time(start)
end_dt = parse_time(end)
if not start_dt or not end_dt:
return None
return max(0.0, (end_dt - start_dt).total_seconds())
KEYCHECK_RESULTS_SQL = r'''
CREATE TABLE IF NOT EXISTS keycheck_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
service TEXT NOT NULL,
status TEXT NOT NULL,
status_group TEXT NOT NULL,
checked_at TEXT NOT NULL,
key_hash TEXT,
secret_hash TEXT,
key_masked TEXT,
finding_id INTEGER,
target_scan_id INTEGER,
cycle_id INTEGER,
run_id INTEGER,
source TEXT,
query TEXT,
target TEXT,
detector_name TEXT,
found_at TEXT,
message TEXT,
metadata_json TEXT,
source_line TEXT,
detector_secret_hash TEXT,
event_id TEXT,
finding_uid TEXT,
link_status TEXT DEFAULT 'pending',
link_attempts INTEGER DEFAULT 0,
linked_at TEXT,
link_error TEXT,
candidate_id INTEGER,
credential_id INTEGER,
result_source TEXT NOT NULL DEFAULT 'api_check',
created_at TEXT NOT NULL,
FOREIGN KEY(finding_id) REFERENCES findings(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(candidate_id) REFERENCES keycheck_candidates(id),
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id)
);
CREATE TABLE IF NOT EXISTS keycheck_event_map (
event_id TEXT PRIMARY KEY,
keycheck_result_id INTEGER,
created_at TEXT NOT NULL,
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_checked ON keycheck_results(checked_at);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_service_status ON keycheck_results(service, status_group, status);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_finding ON keycheck_results(finding_id);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_cycle ON keycheck_results(cycle_id);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_source_query ON keycheck_results(source, query);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_key_hash ON keycheck_results(key_hash);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_secret_hash ON keycheck_results(secret_hash);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_link_repair ON keycheck_results(link_status, id);
DROP VIEW IF EXISTS keycheck_latest_state;
CREATE VIEW IF NOT EXISTS keycheck_latest_state AS
WITH normalized AS (
SELECT
kr.*,
COALESCE(NULLIF(kr.key_hash, ''), NULLIF(kr.secret_hash, ''), NULLIF(kr.key_masked, '')) AS key_identity,
CASE
WHEN NULLIF(kr.key_hash, '') IS NOT NULL THEN 'key_hash'
WHEN NULLIF(kr.secret_hash, '') IS NOT NULL THEN 'secret_hash'
WHEN NULLIF(kr.key_masked, '') IS NOT NULL THEN 'key_masked'
ELSE 'none'
END AS key_identity_kind
FROM keycheck_results kr
), ranked AS (
SELECT
normalized.*,
ROW_NUMBER() OVER (
PARTITION BY service, key_identity
ORDER BY checked_at DESC, id DESC
) AS latest_rank
FROM normalized
WHERE key_identity IS NOT NULL
)
SELECT *
FROM ranked
WHERE latest_rank = 1;
'''
DOCKER_DEPTH_EXPERIMENT_SCHEMA_SQL = f'''
CREATE TABLE IF NOT EXISTS docker_depth_experiments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_key TEXT NOT NULL,
source TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'collecting',
collection_generation TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1',
config_sha256 TEXT NOT NULL,
ordered_queries_sha256 TEXT NOT NULL,
selector_version TEXT NOT NULL,
selector_sha256 TEXT NOT NULL,
provenance_policy_sha256 TEXT NOT NULL,
query_count INTEGER NOT NULL,
repositories_per_query INTEGER NOT NULL,
images_per_repository INTEGER NOT NULL,
target_limit INTEGER NOT NULL,
target_count INTEGER NOT NULL DEFAULT 0,
selection_count INTEGER NOT NULL DEFAULT 0,
fence_generation INTEGER NOT NULL DEFAULT 0,
fence_owner TEXT,
fence_token TEXT,
fence_expires_at TEXT,
hold_reason_code TEXT,
plan_sha256 TEXT,
selection_sha256 TEXT,
hold_manifest_sha256 TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
planned_at TEXT,
activated_at TEXT,
draining_at TEXT,
completed_at TEXT,
released_at TEXT,
held_at TEXT,
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_state_check')},
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_range_check')},
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_identity_check')},
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_capacity_check')},
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_selection_check')},
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_fence_check')}
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_queries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
source TEXT NOT NULL,
query_ordinal INTEGER NOT NULL,
query TEXT NOT NULL,
query_sha256 TEXT NOT NULL,
required_repository_count INTEGER NOT NULL DEFAULT 10,
selected_repository_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
UNIQUE(experiment_id, query_ordinal),
UNIQUE(experiment_id, source, query),
UNIQUE(experiment_id, query_ordinal, source, query),
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_identity_check')},
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_selection_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
);
CREATE TABLE IF NOT EXISTS docker_discovery_passes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER,
pass_token TEXT NOT NULL,
source TEXT NOT NULL,
pass_kind TEXT NOT NULL,
collection_generation TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1',
policy_sha256 TEXT NOT NULL,
ordered_queries_sha256 TEXT NOT NULL,
expected_query_count INTEGER NOT NULL,
completed_query_count INTEGER NOT NULL DEFAULT 0,
state TEXT NOT NULL DEFAULT 'collecting',
started_at TEXT NOT NULL,
completed_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_kind_check')},
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_state_check')},
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_count_check')},
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_identity_check')},
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_complete_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
);
CREATE TABLE IF NOT EXISTS docker_discovery_pages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
pass_id INTEGER NOT NULL,
source_cycle_id INTEGER,
retry_work_id INTEGER,
query TEXT NOT NULL,
query_ordinal INTEGER NOT NULL,
page_number INTEGER NOT NULL,
result_count INTEGER NOT NULL DEFAULT 0,
total_count INTEGER,
admitted_count INTEGER NOT NULL DEFAULT 0,
query_complete INTEGER NOT NULL DEFAULT 0,
admission_kind TEXT NOT NULL DEFAULT 'main',
page_sha256 TEXT NOT NULL,
observed_at TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE(pass_id, query_ordinal, page_number),
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_range_check')},
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_boolean_check')},
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_kind_check')},
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_identity_check')},
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_evidence_check')},
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_total_count_check')},
FOREIGN KEY(pass_id) REFERENCES docker_discovery_passes(id),
FOREIGN KEY(source_cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(retry_work_id) REFERENCES discovery_retry_queue(id)
);
CREATE TABLE IF NOT EXISTS docker_repository_query_provenance (
source TEXT NOT NULL,
query TEXT NOT NULL,
repository_queue_id INTEGER NOT NULL,
provenance_kind TEXT NOT NULL,
first_observed_at TEXT NOT NULL,
last_observed_at TEXT NOT NULL,
first_search_rank INTEGER,
best_search_rank INTEGER,
last_search_rank INTEGER,
first_cycle_id INTEGER,
last_cycle_id INTEGER,
first_page_id INTEGER,
last_page_id INTEGER,
first_policy_sha256 TEXT,
last_policy_sha256 TEXT,
observation_count INTEGER NOT NULL DEFAULT 1,
fresh_observation_count INTEGER NOT NULL DEFAULT 0,
fresh_complete_observation_count INTEGER NOT NULL DEFAULT 0,
fresh_coverage_eligible INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY(source, query, repository_queue_id),
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_kind_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_range_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_boolean_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_identity_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_counts_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_legacy_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_fresh_check')},
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_eligibility_check')},
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(first_cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(last_cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(first_page_id) REFERENCES docker_discovery_pages(id),
FOREIGN KEY(last_page_id) REFERENCES docker_discovery_pages(id)
);
CREATE TABLE IF NOT EXISTS docker_repository_query_observations (
page_id INTEGER NOT NULL,
repository_queue_id INTEGER NOT NULL,
source TEXT NOT NULL,
query TEXT NOT NULL,
search_rank INTEGER NOT NULL,
observed_at TEXT NOT NULL,
PRIMARY KEY(page_id, repository_queue_id),
UNIQUE(page_id, repository_queue_id, source, query),
{_docker_depth_check_clause('docker_repository_query_observations', 'docker_repository_query_observations_rank_check')},
FOREIGN KEY(page_id) REFERENCES docker_discovery_pages(id),
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(source, query, repository_queue_id)
REFERENCES docker_repository_query_provenance(source, query, repository_queue_id)
);
CREATE TABLE IF NOT EXISTS docker_image_manifests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target_queue_id INTEGER NOT NULL,
source TEXT NOT NULL,
repository TEXT NOT NULL,
manifest_digest TEXT NOT NULL,
manifest_media_type TEXT NOT NULL,
config_digest TEXT,
graph_sha256 TEXT NOT NULL,
manifest_size_bytes INTEGER,
layer_count INTEGER NOT NULL,
resolved_at TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE(id, target_queue_id),
{_docker_depth_check_clause('docker_image_manifests', 'docker_image_manifests_range_check')},
{_docker_depth_check_clause('docker_image_manifests', 'docker_image_manifests_identity_check')},
FOREIGN KEY(target_queue_id) REFERENCES target_queue(id)
);
CREATE TABLE IF NOT EXISTS docker_manifest_layers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
manifest_id INTEGER NOT NULL,
position_from_base INTEGER NOT NULL,
position_from_top INTEGER NOT NULL,
layer_digest TEXT NOT NULL,
media_type TEXT NOT NULL,
layer_size_bytes INTEGER NOT NULL,
descriptor_sha256 TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE(id, position_from_base, position_from_top, layer_digest),
{_docker_depth_check_clause('docker_manifest_layers', 'docker_manifest_layers_range_check')},
{_docker_depth_check_clause('docker_manifest_layers', 'docker_manifest_layers_identity_check')},
FOREIGN KEY(manifest_id) REFERENCES docker_image_manifests(id)
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_repositories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
query_ordinal INTEGER NOT NULL,
source TEXT NOT NULL,
query TEXT NOT NULL,
repository_queue_id INTEGER NOT NULL,
eligibility_page_id INTEGER NOT NULL,
repository_rank INTEGER NOT NULL,
is_deep_probe INTEGER NOT NULL DEFAULT 0,
planned_is_deep_probe INTEGER NOT NULL DEFAULT 0,
work_state TEXT NOT NULL DEFAULT 'pending',
resolver_generation INTEGER NOT NULL DEFAULT 0,
resolver_owner TEXT,
resolver_token TEXT,
resolver_expires_at TEXT,
resolver_attempts INTEGER NOT NULL DEFAULT 0,
resolver_due_at TEXT,
candidate_distinct_graph_count INTEGER NOT NULL DEFAULT 0,
selected_image_count INTEGER NOT NULL DEFAULT 0,
replacement_repository_queue_id INTEGER,
replacement_eligibility_page_id INTEGER,
replacement_count INTEGER NOT NULL DEFAULT 0,
replacement_evidence_sha256 TEXT,
last_error_code TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
resolved_at TEXT,
UNIQUE(experiment_id, query_ordinal, id),
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_boolean_check')},
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_state_check')},
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_fence_check')},
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_resolver_check')},
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_replacement_check')},
FOREIGN KEY(experiment_id, query_ordinal, source, query)
REFERENCES docker_depth_experiment_queries(
experiment_id, query_ordinal, source, query
),
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(source, query, repository_queue_id)
REFERENCES docker_repository_query_provenance(source, query, repository_queue_id),
FOREIGN KEY(eligibility_page_id, repository_queue_id, source, query)
REFERENCES docker_repository_query_observations(
page_id, repository_queue_id, source, query
),
FOREIGN KEY(replacement_repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(
replacement_eligibility_page_id, replacement_repository_queue_id, source, query
) REFERENCES docker_repository_query_observations(
page_id, repository_queue_id, source, query
)
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_targets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
target_queue_id INTEGER NOT NULL,
manifest_id INTEGER NOT NULL,
counter_ordinal INTEGER NOT NULL,
state TEXT NOT NULL DEFAULT 'pending',
dispatch_wave INTEGER NOT NULL,
dispatch_order INTEGER NOT NULL,
reservation_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
terminal_at TEXT,
UNIQUE(experiment_id, id),
{_docker_depth_check_clause('docker_depth_experiment_targets', 'docker_depth_experiment_targets_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_targets', 'docker_depth_experiment_targets_state_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
FOREIGN KEY(target_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(manifest_id, target_queue_id)
REFERENCES docker_image_manifests(id, target_queue_id)
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_selections (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
query_ordinal INTEGER NOT NULL,
experiment_repository_id INTEGER NOT NULL,
experiment_target_id INTEGER NOT NULL,
image_rank INTEGER NOT NULL,
selection_reason TEXT NOT NULL,
selection_evidence_sha256 TEXT NOT NULL,
graph_sha256 TEXT NOT NULL,
selected_at TEXT NOT NULL,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_depth_experiment_selections', 'docker_depth_experiment_selections_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_selections', 'docker_depth_experiment_selections_identity_check')},
FOREIGN KEY(experiment_id, query_ordinal)
REFERENCES docker_depth_experiment_queries(experiment_id, query_ordinal),
FOREIGN KEY(experiment_id, query_ordinal, experiment_repository_id)
REFERENCES docker_depth_experiment_repositories(experiment_id, query_ordinal, id),
FOREIGN KEY(experiment_id, experiment_target_id)
REFERENCES docker_depth_experiment_targets(experiment_id, id)
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_candidate_skips (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
experiment_repository_id INTEGER NOT NULL,
repository_queue_id INTEGER NOT NULL,
candidate_kind TEXT NOT NULL,
candidate_ordinal INTEGER NOT NULL,
reason_code TEXT NOT NULL,
candidate_identity_sha256 TEXT NOT NULL,
evidence_sha256 TEXT NOT NULL,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_kind_check')},
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_identity_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
FOREIGN KEY(experiment_repository_id)
REFERENCES docker_depth_experiment_repositories(id),
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id)
);
CREATE TABLE IF NOT EXISTS docker_depth_resolver_attempt_refunds (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
experiment_repository_id INTEGER NOT NULL,
repository_queue_id INTEGER NOT NULL,
query_ordinal INTEGER NOT NULL,
repository_rank INTEGER NOT NULL,
recovery_kind TEXT NOT NULL,
manifest_sha256 TEXT NOT NULL,
entry_evidence_sha256 TEXT NOT NULL,
log_sha256 TEXT NOT NULL,
target_identity_sha256 TEXT NOT NULL,
prior_error_code_sha256 TEXT NOT NULL,
prior_work_state TEXT NOT NULL,
next_work_state TEXT NOT NULL,
prior_resolver_attempts INTEGER NOT NULL,
refund_attempts INTEGER NOT NULL,
next_resolver_attempts INTEGER NOT NULL,
confirmed_bug_event_count INTEGER NOT NULL,
applied_at TEXT NOT NULL,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_kind_check')},
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_state_check')},
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_range_check')},
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_hash_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
FOREIGN KEY(experiment_repository_id)
REFERENCES docker_depth_experiment_repositories(id),
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id)
);
CREATE TABLE IF NOT EXISTS docker_depth_resolver_dispositions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_id INTEGER NOT NULL,
experiment_repository_id INTEGER NOT NULL,
prior_repository_queue_id INTEGER NOT NULL,
replacement_repository_queue_id INTEGER,
replacement_eligibility_page_id INTEGER,
replacement_best_search_rank INTEGER,
query_ordinal INTEGER NOT NULL,
repository_rank INTEGER NOT NULL,
disposition_kind TEXT NOT NULL,
outcome TEXT NOT NULL,
manifest_sha256 TEXT NOT NULL,
entry_evidence_sha256 TEXT NOT NULL,
candidate_snapshot_sha256 TEXT NOT NULL,
prior_target_identity_sha256 TEXT NOT NULL,
replacement_target_identity_sha256 TEXT,
prior_error_code_sha256 TEXT NOT NULL,
prior_work_state TEXT NOT NULL,
next_work_state TEXT NOT NULL,
prior_resolver_attempts INTEGER NOT NULL,
next_resolver_attempts INTEGER NOT NULL,
applied_at TEXT NOT NULL,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_kind_check')},
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_outcome_check')},
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_state_check')},
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_range_check')},
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_hash_check')},
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
FOREIGN KEY(experiment_repository_id)
REFERENCES docker_depth_experiment_repositories(id),
FOREIGN KEY(prior_repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(replacement_repository_queue_id) REFERENCES target_queue(id),
FOREIGN KEY(replacement_eligibility_page_id) REFERENCES docker_discovery_pages(id)
);
CREATE TABLE IF NOT EXISTS docker_depth_experiment_scan_bindings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
experiment_target_id INTEGER NOT NULL,
reservation_id INTEGER NOT NULL,
target_scan_id INTEGER,
attempt INTEGER NOT NULL,
state TEXT NOT NULL DEFAULT 'reserved',
bound_at TEXT NOT NULL,
scan_bound_at TEXT,
completed_at TEXT,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_depth_experiment_scan_bindings', 'docker_depth_experiment_scan_bindings_range_check')},
{_docker_depth_check_clause('docker_depth_experiment_scan_bindings', 'docker_depth_experiment_scan_bindings_state_check')},
FOREIGN KEY(experiment_target_id) REFERENCES docker_depth_experiment_targets(id),
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS docker_finding_layer_attributions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scan_binding_id INTEGER NOT NULL,
finding_id INTEGER NOT NULL,
manifest_layer_id INTEGER,
attribution_state TEXT NOT NULL,
reported_layer_digest TEXT,
unattributed_reason TEXT,
position_from_base INTEGER,
position_from_top INTEGER,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_state_check')},
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_evidence_check')},
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_reason_check')},
FOREIGN KEY(scan_binding_id) REFERENCES docker_depth_experiment_scan_bindings(id),
FOREIGN KEY(finding_id) REFERENCES findings(id),
FOREIGN KEY(
manifest_layer_id, position_from_base, position_from_top, reported_layer_digest
) REFERENCES docker_manifest_layers(
id, position_from_base, position_from_top, layer_digest
)
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiments_key
ON docker_depth_experiments(experiment_key);
CREATE INDEX IF NOT EXISTS idx_docker_depth_experiments_state
ON docker_depth_experiments(source, state, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiment_queries_ordinal
ON docker_depth_experiment_queries(experiment_id, query_ordinal);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiment_queries_query
ON docker_depth_experiment_queries(experiment_id, source, query);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_discovery_passes_token
ON docker_discovery_passes(pass_token);
CREATE INDEX IF NOT EXISTS idx_docker_discovery_passes_policy
ON docker_discovery_passes(source, policy_sha256, state, id);
CREATE INDEX IF NOT EXISTS idx_docker_discovery_passes_experiment
ON docker_discovery_passes(experiment_id, state, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_discovery_pages_position
ON docker_discovery_pages(pass_id, query_ordinal, page_number);
CREATE INDEX IF NOT EXISTS idx_docker_discovery_pages_query
ON docker_discovery_pages(pass_id, query, query_complete, page_number);
CREATE INDEX IF NOT EXISTS idx_docker_discovery_pages_retry
ON docker_discovery_pages(retry_work_id, id) WHERE retry_work_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_docker_repository_provenance_queue
ON docker_repository_query_provenance(repository_queue_id, source, query);
CREATE INDEX IF NOT EXISTS idx_docker_repository_provenance_eligible
ON docker_repository_query_provenance(
source, query, fresh_coverage_eligible, best_search_rank, repository_queue_id
);
CREATE INDEX IF NOT EXISTS idx_docker_repository_observations_query
ON docker_repository_query_observations(source, query, repository_queue_id, page_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_image_manifests_queue
ON docker_image_manifests(target_queue_id);
CREATE INDEX IF NOT EXISTS idx_docker_image_manifests_digest
ON docker_image_manifests(manifest_digest, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_manifest_layers_base_position
ON docker_manifest_layers(manifest_id, position_from_base);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_manifest_layers_top_position
ON docker_manifest_layers(manifest_id, position_from_top);
CREATE INDEX IF NOT EXISTS idx_docker_manifest_layers_digest
ON docker_manifest_layers(layer_digest, manifest_id, position_from_base);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_repositories_member
ON docker_depth_experiment_repositories(experiment_id, query_ordinal, repository_queue_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_repositories_rank
ON docker_depth_experiment_repositories(experiment_id, query_ordinal, repository_rank);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_repository_work
ON docker_depth_experiment_repositories(
experiment_id, work_state, repository_rank, query_ordinal, id
);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_repository_due
ON docker_depth_experiment_repositories(
experiment_id, work_state, resolver_due_at, repository_rank, query_ordinal, id
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_targets_queue
ON docker_depth_experiment_targets(experiment_id, target_queue_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_targets_counter
ON docker_depth_experiment_targets(experiment_id, counter_ordinal);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_targets_dispatch
ON docker_depth_experiment_targets(experiment_id, state, dispatch_wave, dispatch_order, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_selections_rank
ON docker_depth_experiment_selections(experiment_repository_id, image_rank);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_selections_query
ON docker_depth_experiment_selections(
experiment_id, query_ordinal, image_rank, experiment_repository_id, id
);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_selections_target
ON docker_depth_experiment_selections(experiment_target_id, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_candidate_skip
ON docker_depth_experiment_candidate_skips(
experiment_repository_id, candidate_kind, candidate_identity_sha256
);
CREATE INDEX IF NOT EXISTS idx_docker_experiment_candidate_skips
ON docker_depth_experiment_candidate_skips(
experiment_id, experiment_repository_id, candidate_kind, id
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_resolver_attempt_refund
ON docker_depth_resolver_attempt_refunds(experiment_repository_id, recovery_kind);
CREATE INDEX IF NOT EXISTS idx_docker_depth_resolver_attempt_refunds
ON docker_depth_resolver_attempt_refunds(experiment_id, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_resolver_disposition
ON docker_depth_resolver_dispositions(experiment_repository_id, disposition_kind);
CREATE INDEX IF NOT EXISTS idx_docker_depth_resolver_dispositions
ON docker_depth_resolver_dispositions(experiment_id, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_reservation
ON docker_depth_experiment_scan_bindings(reservation_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_attempt
ON docker_depth_experiment_scan_bindings(experiment_target_id, attempt);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_scan
ON docker_depth_experiment_scan_bindings(target_scan_id) WHERE target_scan_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_docker_experiment_bindings_target
ON docker_depth_experiment_scan_bindings(experiment_target_id, state, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_finding_layer_exact
ON docker_finding_layer_attributions(finding_id, manifest_layer_id) WHERE attribution_state = 'exact';
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_finding_layer_unattributed
ON docker_finding_layer_attributions(finding_id) WHERE attribution_state = 'unattributed';
CREATE INDEX IF NOT EXISTS idx_docker_finding_layer_binding
ON docker_finding_layer_attributions(scan_binding_id, finding_id, id);
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_experiment
ON target_queue_policy_events(experiment_id, id) WHERE experiment_id IS NOT NULL;
'''
PIPELINE_SCHEMA_SQL = f'''
CREATE TABLE IF NOT EXISTS runtime_schema_migrations (
version TEXT PRIMARY KEY,
applied_at TEXT NOT NULL,
code_sha256 TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS discovery_retry_queue (
id INTEGER PRIMARY KEY AUTOINCREMENT,
work_key TEXT NOT NULL,
source TEXT NOT NULL,
query TEXT NOT NULL,
source_cycle_id INTEGER,
policy_sha256 TEXT NOT NULL,
pass_kind TEXT NOT NULL,
work_kind TEXT NOT NULL,
page_start INTEGER NOT NULL DEFAULT 1,
page_end INTEGER NOT NULL DEFAULT 1,
next_page INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
available_after TEXT,
lease_owner TEXT,
lease_token TEXT,
leased_at TEXT,
lease_expires_at TEXT,
last_error_category TEXT,
held_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
CONSTRAINT discovery_retry_queue_identity_check CHECK (
length(work_key) = 64 AND work_key = lower(work_key)
AND length(source) BETWEEN 1 AND 64
AND length(query) BETWEEN 1 AND 256
AND length(policy_sha256) = 64 AND policy_sha256 = lower(policy_sha256)
AND pass_kind IN ('ordinary','deep')
AND work_kind IN ('query','page','range')
),
CONSTRAINT discovery_retry_queue_page_check CHECK (
page_start BETWEEN 1 AND 30
AND page_end BETWEEN page_start AND 30
AND next_page BETWEEN page_start AND page_end
AND (work_kind <> 'query' OR page_start = 1)
AND (work_kind <> 'page' OR page_start = page_end)
),
CONSTRAINT discovery_retry_queue_status_check CHECK (
status IN ('pending','leased','held')
AND attempts BETWEEN 0 AND 1000000
),
CONSTRAINT discovery_retry_queue_error_check CHECK (
last_error_category IS NULL OR last_error_category IN (
'account_pool_exhausted','auth_forbidden','auth_invalid','auth_unavailable',
'invalid_payload','network','page_unavailable','policy_mismatch',
'provider_cooldown','provider_unavailable','query_removed','rate_limit',
'remote_transient','request_failed','tail_unavailable','transport'
)
),
{_docker_depth_check_clause('discovery_retry_queue', 'discovery_retry_queue_lifecycle_check')},
{_docker_depth_check_clause('discovery_retry_queue', 'discovery_retry_queue_sha256_check')},
FOREIGN KEY(source_cycle_id) REFERENCES source_cycles(id)
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_discovery_retry_queue_work_key
ON discovery_retry_queue(work_key);
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_due
ON discovery_retry_queue(source, available_after, id) WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_lease
ON discovery_retry_queue(lease_expires_at, id) WHERE status = 'leased';
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_policy
ON discovery_retry_queue(source, query, policy_sha256, status, id);
CREATE TABLE IF NOT EXISTS runtime_final_cutover (
id INTEGER PRIMARY KEY CHECK (id = 1),
marker TEXT NOT NULL,
checked_at TEXT NOT NULL,
evidence_sha256 TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS runtime_operations (
operation_id TEXT PRIMARY KEY CHECK (length(operation_id) = 36),
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
action TEXT NOT NULL CHECK (length(action) BETWEEN 1 AND 128),
target_kind TEXT NOT NULL CHECK (length(target_kind) BETWEEN 1 AND 64),
target_ref TEXT NOT NULL DEFAULT '' CHECK (length(target_ref) <= 512),
status TEXT NOT NULL DEFAULT 'requested' CHECK (
status IN ('requested','running','succeeded','failed','rolled_back','failed_hold','canceled')
),
safe_category TEXT CHECK (safe_category IS NULL OR length(safe_category) <= 128),
safe_detail TEXT CHECK (safe_detail IS NULL OR length(safe_detail) <= 2048),
expected_revision INTEGER CHECK (expected_revision IS NULL OR expected_revision >= 0),
resulting_revision INTEGER CHECK (resulting_revision IS NULL OR resulting_revision >= 0),
expected_identity_json TEXT NOT NULL DEFAULT '{{}}'
CHECK (length(expected_identity_json) <= 65536),
resulting_identity_json TEXT CHECK (
resulting_identity_json IS NULL OR length(resulting_identity_json) <= 65536
),
agent_state TEXT NOT NULL DEFAULT 'not_required' CHECK (
agent_state IN (
'not_required','pending','running','succeeded','failed','rolled_back','failed_hold'
)
),
agent_result_sha256 TEXT CHECK (
agent_result_sha256 IS NULL OR (
length(agent_result_sha256) = 64 AND agent_result_sha256 = lower(agent_result_sha256)
)
),
requested_at TEXT NOT NULL,
started_at TEXT,
completed_at TEXT,
agent_reconciled_at TEXT,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS runtime_operations_control (
id INTEGER PRIMARY KEY CHECK (id = 1),
revision INTEGER NOT NULL DEFAULT 0 CHECK (revision >= 0),
discovery_paused INTEGER NOT NULL DEFAULT 0 CHECK (discovery_paused IN (0,1)),
dispatch_paused INTEGER NOT NULL DEFAULT 0 CHECK (dispatch_paused IN (0,1)),
drain_state TEXT NOT NULL DEFAULT 'normal'
CHECK (drain_state IN ('normal','draining','drained')),
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
operation_id TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
FOREIGN KEY(operation_id) REFERENCES runtime_operations(operation_id)
);
CREATE TABLE IF NOT EXISTS runtime_audit_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
operation_id TEXT,
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
action TEXT NOT NULL CHECK (length(action) BETWEEN 1 AND 128),
target_kind TEXT NOT NULL CHECK (length(target_kind) BETWEEN 1 AND 64),
target_ref TEXT NOT NULL DEFAULT '' CHECK (length(target_ref) <= 512),
result TEXT NOT NULL CHECK (
result IN (
'accepted','succeeded','rejected','failed','rolled_back','canceled','failed_hold'
)
),
safe_category TEXT CHECK (safe_category IS NULL OR length(safe_category) <= 128),
before_identity_json TEXT CHECK (
before_identity_json IS NULL OR length(before_identity_json) <= 65536
),
after_identity_json TEXT CHECK (
after_identity_json IS NULL OR length(after_identity_json) <= 65536
),
before_bytes INTEGER CHECK (before_bytes IS NULL OR before_bytes >= 0),
after_bytes INTEGER CHECK (after_bytes IS NULL OR after_bytes >= 0),
previous_event_id INTEGER UNIQUE,
previous_event_sha256 TEXT,
event_sha256 TEXT NOT NULL UNIQUE CHECK (
length(event_sha256) = 64 AND event_sha256 = lower(event_sha256)
),
created_at TEXT NOT NULL,
CONSTRAINT runtime_audit_events_chain_check CHECK (
(previous_event_id IS NULL AND previous_event_sha256 IS NULL)
OR (previous_event_id IS NOT NULL AND length(previous_event_sha256) = 64
AND previous_event_sha256 = lower(previous_event_sha256))
),
FOREIGN KEY(operation_id) REFERENCES runtime_operations(operation_id),
FOREIGN KEY(previous_event_id) REFERENCES runtime_audit_events(id)
);
CREATE TABLE IF NOT EXISTS pipeline_capacity (
id INTEGER PRIMARY KEY CHECK (id = 1),
bundle_items BIGINT NOT NULL DEFAULT 0 CHECK (bundle_items >= 0),
bundle_bytes BIGINT NOT NULL DEFAULT 0 CHECK (bundle_bytes >= 0),
projection_items BIGINT NOT NULL DEFAULT 0 CHECK (projection_items >= 0),
projection_bytes BIGINT NOT NULL DEFAULT 0 CHECK (projection_bytes >= 0),
keycheck_items BIGINT NOT NULL DEFAULT 0 CHECK (keycheck_items >= 0),
keycheck_bytes BIGINT NOT NULL DEFAULT 0 CHECK (keycheck_bytes >= 0),
quarantine_items BIGINT NOT NULL DEFAULT 0 CHECK (quarantine_items >= 0),
quarantine_bytes BIGINT NOT NULL DEFAULT 0 CHECK (quarantine_bytes >= 0),
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS remote_worker_users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_key TEXT NOT NULL UNIQUE,
active_assignment_cap INTEGER NOT NULL DEFAULT 0
CHECK (active_assignment_cap BETWEEN 0 AND 10000),
disabled_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS remote_worker_devices (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
device_key TEXT NOT NULL UNIQUE,
token_sha256 TEXT NOT NULL UNIQUE CHECK (
length(token_sha256) = 64 AND token_sha256 = lower(token_sha256)
),
revoked_at TEXT,
last_contact_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(id, user_id),
FOREIGN KEY(user_id) REFERENCES remote_worker_users(id)
);
CREATE TABLE IF NOT EXISTS admission_intents (
reservation_token TEXT PRIMARY KEY,
intent_sha256 TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('pending','committed','aborted')),
reservation_id BIGINT,
resolution_detail TEXT,
remote_user_id INTEGER,
remote_device_id INTEGER,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
resolved_at TEXT,
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(remote_device_id, remote_user_id)
REFERENCES remote_worker_devices(id, user_id)
);
CREATE TABLE IF NOT EXISTS admission_intent_retirement (
id INTEGER PRIMARY KEY CHECK (id = 1),
retired_count BIGINT NOT NULL DEFAULT 0,
chain_sha256 TEXT NOT NULL,
cursor_token TEXT NOT NULL DEFAULT '',
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS pipeline_artifact_retirement (
id INTEGER PRIMARY KEY CHECK (id = 1),
retired_count BIGINT NOT NULL DEFAULT 0,
chain_sha256 TEXT NOT NULL,
cursor_id BIGINT NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS result_reservations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
reservation_token TEXT NOT NULL UNIQUE,
bundle_id TEXT NOT NULL UNIQUE,
scan_event_id TEXT NOT NULL UNIQUE,
queue_id INTEGER NOT NULL,
run_id INTEGER,
cycle_id INTEGER,
source TEXT NOT NULL,
platform TEXT NOT NULL,
query TEXT,
target TEXT NOT NULL,
normalized_target TEXT NOT NULL,
claim_lease_owner TEXT NOT NULL,
claim_lease_token TEXT NOT NULL,
claim_batch TEXT,
producer_instance_id TEXT NOT NULL,
producer_pid BIGINT NOT NULL,
producer_creation_time TEXT NOT NULL,
producer_executable TEXT NOT NULL,
assignment_kind TEXT NOT NULL DEFAULT 'local'
CHECK (assignment_kind IN ('local','remote')),
remote_user_id INTEGER,
remote_device_id INTEGER,
remote_issued_at TEXT,
remote_expires_at TEXT,
remote_result_upload_body_timeout_seconds INTEGER CHECK (
remote_result_upload_body_timeout_seconds IS NULL
OR remote_result_upload_body_timeout_seconds BETWEEN 30 AND 86400
),
remote_effective_config_sha256 TEXT,
remote_client_compat_sha256 TEXT,
remote_execution_snapshot_json TEXT CHECK (
remote_execution_snapshot_json IS NULL OR length(remote_execution_snapshot_json) <= 65536
),
remote_execution_snapshot_sha256 TEXT,
remote_resolution_kind TEXT CHECK (
remote_resolution_kind IS NULL OR remote_resolution_kind IN (
'bundle_accepted','prebundle_report','expired'
)
),
remote_payload_sha256 TEXT,
remote_receipt_id TEXT,
remote_resolution_json TEXT CHECK (
remote_resolution_json IS NULL OR length(remote_resolution_json) <= 16384
),
remote_resolved_at TEXT,
remote_diagnostic_projection_version INTEGER CHECK (
remote_diagnostic_projection_version IS NULL
OR remote_diagnostic_projection_version IN (0,1)
),
remote_diagnostic_count INTEGER CHECK (
remote_diagnostic_count IS NULL OR remote_diagnostic_count >= 0
),
remote_diagnostic_uids_sha256 TEXT,
declared_bundle_bytes BIGINT NOT NULL CHECK (declared_bundle_bytes > 0),
reserved_bundle_bytes BIGINT NOT NULL CHECK (reserved_bundle_bytes > 0),
reserved_projection_items BIGINT NOT NULL DEFAULT 1,
reserved_projection_bytes BIGINT NOT NULL,
reserved_candidate_items BIGINT NOT NULL,
reserved_candidate_bytes BIGINT NOT NULL,
ready_relative_path TEXT NOT NULL,
state TEXT NOT NULL CHECK (
state IN ('scanning','ready','ingesting','db_committed','acknowledged','refunded','quarantined')
),
producer_lease_expires_at TEXT NOT NULL,
bundle_credit_released INTEGER NOT NULL DEFAULT 0,
projection_credit_transferred INTEGER NOT NULL DEFAULT 0,
candidate_credit_transferred INTEGER NOT NULL DEFAULT 0,
last_error_code TEXT,
last_error_detail TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
refunded_at TEXT,
released_at TEXT,
cleanup_attempts INTEGER NOT NULL DEFAULT 0,
cleanup_available_after TEXT,
git_scan_plan_json TEXT,
git_scan_plan_sha256 TEXT,
docker_layer_plan_json TEXT,
docker_layer_plan_sha256 TEXT,
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(remote_device_id, remote_user_id)
REFERENCES remote_worker_devices(id, user_id)
);
CREATE TABLE IF NOT EXISTS worker_progress_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
reservation_id INTEGER NOT NULL,
remote_device_id INTEGER NOT NULL,
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
sequence INTEGER NOT NULL CHECK (sequence > 0),
event_type TEXT NOT NULL CHECK (length(event_type) BETWEEN 1 AND 128),
phase TEXT NOT NULL CHECK (length(phase) BETWEEN 1 AND 64),
event_timestamp TEXT NOT NULL,
phase_started_at TEXT,
instance_id TEXT NOT NULL CHECK (length(instance_id) BETWEEN 1 AND 256),
slot_id INTEGER NOT NULL CHECK (slot_id >= 0),
source TEXT NOT NULL CHECK (length(source) BETWEEN 1 AND 64),
event_json TEXT NOT NULL CHECK (length(event_json) BETWEEN 2 AND 65536),
event_sha256 TEXT NOT NULL CHECK (
length(event_sha256) = 64 AND event_sha256 = lower(event_sha256)
),
received_at TEXT NOT NULL,
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(remote_device_id) REFERENCES remote_worker_devices(id)
);
CREATE TABLE IF NOT EXISTS worker_diagnostics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
diagnostic_uid TEXT NOT NULL CHECK (length(diagnostic_uid) BETWEEN 1 AND 256),
reservation_id INTEGER NOT NULL,
target_scan_id INTEGER,
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
scan_event_id TEXT CHECK (
scan_event_id IS NULL OR (
length(scan_event_id) BETWEEN 32 AND 64
AND scan_event_id = lower(scan_event_id)
)
),
slot_id INTEGER NOT NULL CHECK (slot_id >= 0),
attempt INTEGER NOT NULL CHECK (attempt > 0),
source TEXT NOT NULL CHECK (length(source) BETWEEN 1 AND 64),
phase TEXT NOT NULL CHECK (length(phase) BETWEEN 1 AND 64),
kind TEXT NOT NULL CHECK (length(kind) BETWEEN 1 AND 64),
category TEXT NOT NULL CHECK (length(category) BETWEEN 1 AND 128),
code TEXT NOT NULL CHECK (length(code) BETWEEN 1 AND 256),
summary TEXT NOT NULL CHECK (length(summary) BETWEEN 1 AND 2048),
retryable INTEGER NOT NULL CHECK (retryable IN (0,1)),
occurred_at TEXT NOT NULL,
captured_at TEXT NOT NULL,
envelope_json TEXT NOT NULL CHECK (length(envelope_json) BETWEEN 2 AND 65536),
envelope_sha256 TEXT NOT NULL CHECK (
length(envelope_sha256) = 64 AND envelope_sha256 = lower(envelope_sha256)
),
body_payload_json TEXT CHECK (
body_payload_json IS NULL OR length(body_payload_json) <= 65536
),
log_payload_json TEXT CHECK (
log_payload_json IS NULL OR length(log_payload_json) <= 65536
),
received_at TEXT NOT NULL,
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS docker_content_blobs (
digest TEXT NOT NULL,
coverage_policy_sha256 TEXT NOT NULL,
descriptor_kind TEXT NOT NULL CHECK (descriptor_kind IN ('config','layer')),
declared_bytes INTEGER NOT NULL CHECK (declared_bytes >= 0),
media_type TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending','leased','submitted','covered','failed')),
attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
max_attempts INTEGER NOT NULL CHECK (max_attempts > 0),
available_after TEXT,
lease_reservation_id INTEGER,
lease_token TEXT,
lease_plan_sha256 TEXT,
lease_expires_at TEXT,
covered_reservation_id INTEGER,
covered_scan_event_id TEXT,
covered_policy_sha256 TEXT,
verified_bytes INTEGER,
covered_at TEXT,
last_error_code TEXT,
last_error_detail TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY(digest, coverage_policy_sha256),
FOREIGN KEY(lease_reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(covered_reservation_id) REFERENCES result_reservations(id)
);
CREATE TABLE IF NOT EXISTS docker_image_blob_coverage (
queue_id INTEGER NOT NULL,
manifest_digest TEXT NOT NULL,
position INTEGER NOT NULL CHECK (position >= 0),
blob_digest TEXT NOT NULL,
coverage_policy_sha256 TEXT NOT NULL,
selection_policy_sha256 TEXT NOT NULL DEFAULT '',
descriptor_kind TEXT NOT NULL CHECK (descriptor_kind IN ('config','layer')),
plan_sha256 TEXT NOT NULL,
reservation_id INTEGER NOT NULL,
selected INTEGER NOT NULL CHECK (selected IN (0,1)),
selection_reason TEXT NOT NULL,
coverage_state TEXT NOT NULL CHECK (
coverage_state IN (
'selected','leased','shared_pending','covered',
'retryable_failed','terminal_failed','skipped'
)
),
covered_at TEXT,
last_error_code TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY(reservation_id, position),
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
FOREIGN KEY(blob_digest, coverage_policy_sha256)
REFERENCES docker_content_blobs(digest, coverage_policy_sha256),
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id)
);
CREATE TABLE IF NOT EXISTS docker_adaptive_shadow_reports (
id INTEGER PRIMARY KEY AUTOINCREMENT,
report_token TEXT NOT NULL UNIQUE,
evaluator_version TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'running'
CHECK (state IN ('running','completed','failed')),
scan_policy_sha256 TEXT NOT NULL,
execution_policy_sha256 TEXT NOT NULL,
selection_policy_sha256 TEXT NOT NULL,
cohort_size INTEGER NOT NULL CHECK (cohort_size >= 50 AND cohort_size <= 100),
completed_pairs INTEGER NOT NULL DEFAULT 0 CHECK (completed_pairs >= 0),
full_routed_count INTEGER NOT NULL DEFAULT 0 CHECK (full_routed_count >= 0),
adaptive_routed_count INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_routed_count >= 0),
routed_intersection_count INTEGER NOT NULL DEFAULT 0 CHECK (routed_intersection_count >= 0),
full_detector_count INTEGER NOT NULL DEFAULT 0 CHECK (full_detector_count >= 0),
adaptive_detector_count INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_detector_count >= 0),
detector_intersection_count INTEGER NOT NULL DEFAULT 0
CHECK (detector_intersection_count >= 0),
full_slot_ms INTEGER NOT NULL DEFAULT 0 CHECK (full_slot_ms >= 0),
adaptive_slot_ms INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_slot_ms >= 0),
omitted_descriptor_count INTEGER NOT NULL DEFAULT 0 CHECK (omitted_descriptor_count >= 0),
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
privacy_violation_count INTEGER NOT NULL DEFAULT 0 CHECK (privacy_violation_count >= 0),
safety_regression_count INTEGER NOT NULL DEFAULT 0 CHECK (safety_regression_count >= 0),
selection_metrics_json TEXT NOT NULL DEFAULT '{{}}',
sink_checkpoint_count INTEGER NOT NULL DEFAULT 0 CHECK (sink_checkpoint_count >= 0),
routed_recall_ppm INTEGER NOT NULL DEFAULT 0 CHECK (routed_recall_ppm >= 0),
slot_ratio_ppm INTEGER NOT NULL DEFAULT 0 CHECK (slot_ratio_ppm >= 0),
recall_threshold_ppm INTEGER NOT NULL DEFAULT 850000
CHECK (recall_threshold_ppm = 850000),
slot_threshold_ppm INTEGER NOT NULL DEFAULT 400000
CHECK (slot_threshold_ppm = 400000),
passed INTEGER NOT NULL DEFAULT 0 CHECK (passed IN (0,1)),
lease_owner TEXT NOT NULL,
lease_token TEXT NOT NULL,
lease_expires_at TEXT NOT NULL,
started_at TEXT NOT NULL,
completed_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS target_queue_policy_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
queue_id INTEGER NOT NULL,
action TEXT NOT NULL,
prior_status TEXT NOT NULL,
next_status TEXT NOT NULL,
source TEXT NOT NULL,
platform TEXT NOT NULL,
query TEXT NOT NULL,
reason_code TEXT NOT NULL,
config_sha256 TEXT NOT NULL,
policy_sha256 TEXT NOT NULL,
manifest_sha256 TEXT NOT NULL,
review_audit_sha256 TEXT NOT NULL UNIQUE,
reverses_event_id INTEGER UNIQUE,
experiment_id INTEGER,
prior_updated_at TEXT NOT NULL,
created_at TEXT NOT NULL,
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_transition_check')},
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_hash_check')},
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_experiment_ownership_check')},
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
FOREIGN KEY(reverses_event_id) REFERENCES target_queue_policy_events(id),
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
);
CREATE TABLE IF NOT EXISTS result_bundles (
reservation_id INTEGER PRIMARY KEY,
bundle_id TEXT NOT NULL UNIQUE,
scan_event_id TEXT NOT NULL UNIQUE,
scan_event_hash TEXT NOT NULL,
format_version INTEGER NOT NULL CHECK (format_version = 2),
relative_path TEXT NOT NULL UNIQUE,
actual_bytes BIGINT NOT NULL CHECK (actual_bytes > 0),
frame_count INTEGER NOT NULL,
finding_count INTEGER NOT NULL,
error_count INTEGER NOT NULL,
candidate_count INTEGER NOT NULL,
state TEXT NOT NULL CHECK (state IN ('ready','ingesting','db_committed','acknowledged','quarantined')),
available_after TEXT,
ingest_attempts INTEGER NOT NULL DEFAULT 0,
ingest_lease_generation BIGINT,
ingest_lease_token TEXT,
ingest_lease_expires_at TEXT,
target_scan_id INTEGER,
ready_at TEXT NOT NULL,
committed_at TEXT,
acknowledged_at TEXT,
updated_at TEXT NOT NULL,
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS pipeline_leases (
worker_name TEXT PRIMARY KEY,
generation BIGINT NOT NULL DEFAULT 0,
lease_token TEXT,
supervisor_instance_id TEXT,
owner_pid BIGINT,
owner_creation_time TEXT,
owner_executable TEXT,
state TEXT NOT NULL DEFAULT 'released'
CHECK (state IN ('starting','recovering','ready','stopping','released','failed')),
acquired_at TEXT,
heartbeat_at TEXT,
lease_expires_at TEXT,
last_error TEXT,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS scan_result_compat (
target_scan_id INTEGER PRIMARY KEY,
schema_version INTEGER NOT NULL CHECK (schema_version = 2),
metadata_json TEXT NOT NULL,
metadata_sha256 TEXT NOT NULL,
metadata_bytes BIGINT NOT NULL,
reconstruction_status TEXT NOT NULL CHECK (reconstruction_status IN ('exact','bounded','legacy')),
omitted_payload_sha256 TEXT,
created_at TEXT NOT NULL,
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS finding_compat_payloads (
finding_id INTEGER PRIMARY KEY,
raw_value TEXT,
raw_v2_value TEXT,
structured_data_json TEXT,
extra_data_json TEXT,
analysis_info_json TEXT,
extension_json TEXT,
payload_sha256 TEXT NOT NULL,
payload_bytes BIGINT NOT NULL,
payload_omitted INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
FOREIGN KEY(finding_id) REFERENCES findings(id)
);
CREATE TABLE IF NOT EXISTS projection_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_kind TEXT NOT NULL CHECK (job_kind IN ('scan_event','keycheck_event','rebuild')),
event_id TEXT NOT NULL,
event_hash TEXT NOT NULL,
target_scan_id INTEGER,
keycheck_result_id INTEGER,
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','leased','completed','quarantined')),
required_stream_mask INTEGER NOT NULL,
capacity_items BIGINT NOT NULL,
capacity_bytes BIGINT NOT NULL,
capacity_released INTEGER NOT NULL DEFAULT 0,
attempts INTEGER NOT NULL DEFAULT 0,
available_after TEXT,
lease_generation BIGINT,
lease_token TEXT,
lease_expires_at TEXT,
last_error_code TEXT,
last_error_detail TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT,
UNIQUE(job_kind, event_id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
);
CREATE TABLE IF NOT EXISTS projection_streams (
stream_name TEXT PRIMARY KEY,
base_relative_path TEXT NOT NULL UNIQUE,
current_generation BIGINT NOT NULL DEFAULT 0,
rotation_bytes BIGINT NOT NULL,
max_generations INTEGER NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS projection_cursors (
stream_name TEXT PRIMARY KEY,
generation BIGINT NOT NULL,
committed_offset BIGINT NOT NULL,
last_append_id INTEGER,
last_job_id INTEGER,
last_event_id TEXT,
last_event_hash TEXT,
updated_at TEXT NOT NULL,
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name),
FOREIGN KEY(last_append_id) REFERENCES projection_appends(id),
FOREIGN KEY(last_job_id) REFERENCES projection_jobs(id)
);
CREATE TABLE IF NOT EXISTS projection_appends (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id INTEGER NOT NULL,
stream_name TEXT NOT NULL,
event_id TEXT NOT NULL,
event_hash TEXT NOT NULL,
generation BIGINT NOT NULL,
byte_offset BIGINT NOT NULL,
byte_length BIGINT NOT NULL,
payload_sha256 TEXT NOT NULL,
record_count INTEGER NOT NULL,
state TEXT NOT NULL CHECK (state IN ('prepared','appended')),
prepared_at TEXT NOT NULL,
appended_at TEXT,
UNIQUE(job_id, stream_name),
UNIQUE(stream_name, generation, byte_offset),
FOREIGN KEY(job_id) REFERENCES projection_jobs(id),
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
);
CREATE TABLE IF NOT EXISTS projection_append_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
append_id BIGINT NOT NULL,
job_id INTEGER NOT NULL,
stream_name TEXT NOT NULL,
event_id TEXT NOT NULL,
event_hash TEXT NOT NULL,
generation BIGINT NOT NULL,
byte_offset BIGINT NOT NULL,
byte_length BIGINT NOT NULL,
payload_sha256 TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('canceled','quarantined')),
reason_code TEXT NOT NULL,
reason_detail TEXT,
created_at TEXT NOT NULL,
FOREIGN KEY(job_id) REFERENCES projection_jobs(id),
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
);
CREATE TABLE IF NOT EXISTS projection_rotations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stream_name TEXT NOT NULL,
from_generation BIGINT NOT NULL,
to_generation BIGINT NOT NULL,
source_bytes BIGINT NOT NULL,
segment_relative_path TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('prepared','renamed','completed')),
created_at TEXT NOT NULL,
completed_at TEXT,
UNIQUE(stream_name, to_generation),
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
);
CREATE TABLE IF NOT EXISTS keycheck_credentials (
id INTEGER PRIMARY KEY AUTOINCREMENT,
service TEXT NOT NULL,
credential_hash TEXT NOT NULL,
provider_key_hash TEXT NOT NULL,
candidate_kind TEXT NOT NULL,
secret_text TEXT,
secret_json TEXT,
key_masked TEXT,
endpoint TEXT,
principal TEXT,
metadata_json TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(service, credential_hash)
);
CREATE TABLE IF NOT EXISTS keycheck_candidates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
candidate_uid TEXT NOT NULL UNIQUE,
credential_id INTEGER NOT NULL,
service TEXT NOT NULL,
routed_service TEXT NOT NULL DEFAULT '',
secret_hash TEXT NOT NULL,
finding_id INTEGER,
target_scan_id INTEGER,
scan_event_id TEXT,
finding_uid TEXT,
source TEXT,
query TEXT,
target TEXT,
detector_name TEXT,
found_at TEXT,
metadata_json TEXT,
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending','leased','deferred','completed','quarantined')),
priority INTEGER NOT NULL DEFAULT 0,
attempts INTEGER NOT NULL DEFAULT 0,
available_after TEXT,
lease_owner TEXT,
lease_token TEXT,
lease_expires_at TEXT,
keycheck_result_id INTEGER,
capacity_bytes BIGINT NOT NULL,
capacity_released INTEGER NOT NULL DEFAULT 0,
result_projection_reserved_bytes BIGINT NOT NULL DEFAULT 0,
result_projection_credit_transferred INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT,
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id),
FOREIGN KEY(finding_id) REFERENCES findings(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
);
CREATE TABLE IF NOT EXISTS keycheck_current_state (
credential_id INTEGER PRIMARY KEY,
service TEXT NOT NULL,
status TEXT NOT NULL,
status_group TEXT NOT NULL,
last_result_id INTEGER NOT NULL,
result_source TEXT NOT NULL,
checked_at TEXT NOT NULL,
recheck_after TEXT,
state_version BIGINT NOT NULL DEFAULT 1,
metadata_json TEXT,
updated_at TEXT NOT NULL,
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id),
FOREIGN KEY(last_result_id) REFERENCES keycheck_results(id)
);
CREATE TABLE IF NOT EXISTS pipeline_quarantine (
id INTEGER PRIMARY KEY AUTOINCREMENT,
subsystem TEXT NOT NULL,
object_type TEXT NOT NULL,
object_id INTEGER,
reservation_id INTEGER,
projection_job_id INTEGER,
keycheck_candidate_id INTEGER,
event_id TEXT,
payload_sha256 TEXT,
reason_code TEXT NOT NULL,
reason_detail TEXT,
source_relative_path TEXT,
byte_count BIGINT NOT NULL DEFAULT 0,
capacity_items BIGINT NOT NULL DEFAULT 1,
capacity_bytes BIGINT NOT NULL DEFAULT 0,
capacity_credit_applied INTEGER NOT NULL DEFAULT 1,
review_status TEXT NOT NULL DEFAULT 'pending'
CHECK (review_status IN ('pending','approved_retry','approved_rescan','discarded','resolved')),
detected_at TEXT NOT NULL,
resolved_at TEXT,
review_audit_sha256 TEXT,
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
FOREIGN KEY(projection_job_id) REFERENCES projection_jobs(id),
FOREIGN KEY(keycheck_candidate_id) REFERENCES keycheck_candidates(id)
);
CREATE TABLE IF NOT EXISTS pipeline_artifacts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
subsystem TEXT NOT NULL,
artifact_kind TEXT NOT NULL,
owner_id BIGINT NOT NULL,
owner_key TEXT NOT NULL DEFAULT '',
relative_path TEXT NOT NULL,
payload_sha256 TEXT,
byte_count BIGINT NOT NULL DEFAULT 0,
state TEXT NOT NULL CHECK (state IN ('expected','present','quarantined','deleted')),
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
deleted_at TEXT,
cleanup_attempts INTEGER NOT NULL DEFAULT 0,
cleanup_available_after TEXT,
cleanup_last_error TEXT,
UNIQUE(subsystem, artifact_kind, owner_id, owner_key),
UNIQUE(subsystem, relative_path)
);
CREATE TABLE IF NOT EXISTS janitor_cursors (
layout_name TEXT PRIMARY KEY,
last_name TEXT NOT NULL DEFAULT '',
wrap_count BIGINT NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS keycheck_recheck_cursors (
cursor_key TEXT PRIMARY KEY,
service TEXT NOT NULL,
status_scope TEXT NOT NULL,
last_credential_id BIGINT NOT NULL DEFAULT 0,
wrap_count BIGINT NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_result_reservation_queue_lease
ON result_reservations(queue_id, claim_lease_token);
CREATE UNIQUE INDEX IF NOT EXISTS uq_worker_progress_reservation_sequence
ON worker_progress_events(reservation_id, sequence);
CREATE INDEX IF NOT EXISTS idx_worker_progress_reservation_received
ON worker_progress_events(reservation_id, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_progress_device_received
ON worker_progress_events(remote_device_id, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_progress_phase_received
ON worker_progress_events(phase, received_at, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_worker_diagnostics_uid
ON worker_diagnostics(diagnostic_uid);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_reservation_received
ON worker_diagnostics(reservation_id, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_scan_received
ON worker_diagnostics(target_scan_id, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_phase_received
ON worker_diagnostics(phase, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_category_code
ON worker_diagnostics(category, code, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_code_received
ON worker_diagnostics(code, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_kind_received
ON worker_diagnostics(kind, received_at, id);
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_retryable_received
ON worker_diagnostics(retryable, received_at, id);
CREATE INDEX IF NOT EXISTS idx_runtime_operations_status_updated
ON runtime_operations(status, updated_at, operation_id);
CREATE INDEX IF NOT EXISTS idx_runtime_operations_agent_state_updated
ON runtime_operations(agent_state, updated_at, operation_id);
CREATE INDEX IF NOT EXISTS idx_runtime_audit_events_created
ON runtime_audit_events(created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_runtime_audit_events_operation
ON runtime_audit_events(operation_id, id DESC);
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_users_key
ON remote_worker_users(user_key);
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_devices_key
ON remote_worker_devices(device_key);
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_devices_token
ON remote_worker_devices(token_sha256);
CREATE INDEX IF NOT EXISTS idx_admission_intents_remote_device
ON admission_intents(remote_device_id, created_at) WHERE remote_device_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_active_user
ON result_reservations(remote_user_id, id)
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_expiry
ON result_reservations(remote_expires_at, id)
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NULL AND state = 'scanning';
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_device_history
ON result_reservations(remote_device_id, remote_issued_at, id)
WHERE assignment_kind = 'remote';
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_resolved
ON result_reservations(remote_resolved_at, id)
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS uq_result_reservations_remote_receipt
ON result_reservations(remote_receipt_id) WHERE remote_receipt_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_result_reservations_recovery
ON result_reservations(state, producer_lease_expires_at, id)
WHERE state IN ('scanning','ready','ingesting','db_committed');
CREATE INDEX IF NOT EXISTS idx_result_reservations_queue_history
ON result_reservations(queue_id, id DESC);
CREATE INDEX IF NOT EXISTS idx_result_bundles_ready
ON result_bundles(state, available_after, ready_at, reservation_id)
WHERE state IN ('ready','ingesting','db_committed');
CREATE INDEX IF NOT EXISTS idx_result_bundles_ingest_lease
ON result_bundles(state, ingest_lease_expires_at)
WHERE state = 'ingesting';
CREATE INDEX IF NOT EXISTS idx_docker_content_blobs_reclaim
ON docker_content_blobs(state, available_after, lease_expires_at, digest);
CREATE INDEX IF NOT EXISTS idx_docker_content_blobs_reservation
ON docker_content_blobs(lease_reservation_id, digest);
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_manifest
ON docker_image_blob_coverage(manifest_digest, coverage_state, position);
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_blob
ON docker_image_blob_coverage(blob_digest, coverage_state, queue_id);
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_reservation
ON docker_image_blob_coverage(reservation_id, position);
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_selection
ON docker_image_blob_coverage(
queue_id, manifest_digest, selection_policy_sha256, position, reservation_id
);
CREATE INDEX IF NOT EXISTS idx_docker_adaptive_shadow_reports_gate
ON docker_adaptive_shadow_reports(
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
state, completed_at, id
);
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_queue
ON target_queue_policy_events(queue_id, id DESC);
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_manifest
ON target_queue_policy_events(manifest_sha256, id);
CREATE INDEX IF NOT EXISTS idx_projection_jobs_claim
ON projection_jobs(status, available_after, id) WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_projection_jobs_lease
ON projection_jobs(status, lease_expires_at, id) WHERE status = 'leased';
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_pending
ON keycheck_candidates(service, priority DESC, id) WHERE state = 'pending';
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_deferred
ON keycheck_candidates(service, available_after, id) WHERE state = 'deferred';
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_lease
ON keycheck_candidates(service, lease_expires_at, id) WHERE state = 'leased';
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_credential
ON keycheck_candidates(credential_id, state, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_candidate_finding_credential
ON keycheck_candidates(service, finding_id, credential_id) WHERE finding_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_pipeline_quarantine_pending
ON pipeline_quarantine(subsystem, review_status, id) WHERE review_status = 'pending';
CREATE UNIQUE INDEX IF NOT EXISTS uq_pipeline_quarantine_open_object
ON pipeline_quarantine(subsystem, object_type, object_id)
WHERE review_status = 'pending' AND object_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_pipeline_artifacts_owner
ON pipeline_artifacts(subsystem, owner_id, state, id);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_candidate ON keycheck_results(candidate_id);
CREATE INDEX IF NOT EXISTS idx_keycheck_results_credential_checked
ON keycheck_results(credential_id, checked_at DESC, id DESC);
''' + DOCKER_DEPTH_EXPERIMENT_SCHEMA_SQL
SCHEMA_SQL = r'''
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at TEXT NOT NULL,
ended_at TEXT,
duration_sec REAL,
status TEXT NOT NULL,
invocation_mode TEXT,
command_line TEXT,
argv_json TEXT,
selected_source TEXT,
selected_platform TEXT,
config_path TEXT,
config_hash TEXT,
enabled_sources_json TEXT,
db_path TEXT,
total_fetched INTEGER DEFAULT 0,
total_queued_new INTEGER DEFAULT 0,
total_scan_requested INTEGER DEFAULT 0,
total_scanned INTEGER DEFAULT 0,
total_clean INTEGER DEFAULT 0,
total_found INTEGER DEFAULT 0,
total_skipped INTEGER DEFAULT 0,
total_errors INTEGER DEFAULT 0,
total_findings INTEGER DEFAULT 0,
total_verified_findings INTEGER DEFAULT 0,
total_unique_secrets INTEGER DEFAULT 0,
total_unique_findings INTEGER DEFAULT 0,
total_staged INTEGER NOT NULL DEFAULT 0,
total_quarantined INTEGER NOT NULL DEFAULT 0,
error TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS source_cycles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER,
source TEXT,
platform TEXT,
mode TEXT,
query TEXT,
query_index INTEGER,
query_count INTEGER,
auth_name TEXT,
started_at TEXT NOT NULL,
ended_at TEXT,
duration_sec REAL,
status TEXT NOT NULL,
message TEXT,
config_json TEXT,
queue_todo_before INTEGER,
queue_checked_before INTEGER,
queue_todo_after INTEGER,
queue_checked_after INTEGER,
fetched_count INTEGER DEFAULT 0,
queued_new_count INTEGER DEFAULT 0,
queued_updated_count INTEGER NOT NULL DEFAULT 0,
scan_requested_count INTEGER DEFAULT 0,
scanned_count INTEGER DEFAULT 0,
clean_count INTEGER DEFAULT 0,
found_count INTEGER DEFAULT 0,
skipped_count INTEGER DEFAULT 0,
error_count INTEGER DEFAULT 0,
findings_count INTEGER DEFAULT 0,
verified_findings_count INTEGER DEFAULT 0,
unique_secrets_count INTEGER DEFAULT 0,
unique_findings_count INTEGER DEFAULT 0,
targets_per_hour REAL DEFAULT 0,
hit_rate REAL DEFAULT 0,
verified_hit_rate REAL DEFAULT 0,
error_rate REAL DEFAULT 0,
staged_count INTEGER NOT NULL DEFAULT 0,
ingested_count INTEGER NOT NULL DEFAULT 0,
quarantined_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id)
);
CREATE TABLE IF NOT EXISTS target_scans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scan_event_id TEXT,
scan_event_hash TEXT,
queue_id INTEGER,
claim_lease_token TEXT,
queue_completion_applied INTEGER NOT NULL DEFAULT 0,
queue_completion_disposition TEXT,
run_id INTEGER,
cycle_id INTEGER,
source TEXT,
query TEXT,
target TEXT,
normalized_target TEXT,
scan_type TEXT,
status TEXT,
started_at TEXT,
ended_at TEXT,
duration_sec REAL,
scan_options_json TEXT,
package_name TEXT,
package_version TEXT,
package_artifact TEXT,
package_date TEXT,
package_filename TEXT,
package_type TEXT,
package_size INTEGER,
findings_count INTEGER DEFAULT 0,
verified_findings_count INTEGER DEFAULT 0,
error_count INTEGER DEFAULT 0,
skipped_reason TEXT,
first_error_summary TEXT,
raw_result_json TEXT,
result_reservation_id INTEGER,
compat_schema_version INTEGER NOT NULL DEFAULT 2,
raw_result_storage TEXT NOT NULL DEFAULT 'legacy',
created_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
FOREIGN KEY(result_reservation_id) REFERENCES result_reservations(id)
);
CREATE TABLE IF NOT EXISTS findings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER,
cycle_id INTEGER,
target_scan_id INTEGER,
source TEXT,
query TEXT,
target TEXT,
normalized_target TEXT,
detector_name TEXT,
detector_type TEXT,
verified INTEGER DEFAULT 0,
raw_secret TEXT,
redacted_secret TEXT,
secret_hash TEXT,
detector_secret_hash TEXT,
finding_fingerprint TEXT,
finding_uid TEXT,
file_path TEXT,
line_number TEXT,
commit_hash TEXT,
source_timestamp TEXT,
source_metadata_type TEXT,
source_metadata_json TEXT,
raw_finding_json TEXT,
provider TEXT,
credential_kind TEXT,
credential_confidence TEXT,
required_context_missing INTEGER DEFAULT 0,
principal TEXT,
username TEXT,
email TEXT,
project_id TEXT,
tenant_id TEXT,
organization TEXT,
registry TEXT,
endpoint TEXT,
scope TEXT,
resource TEXT,
enrichment_json TEXT,
raw_payload_sha256 TEXT,
raw_payload_bytes INTEGER,
raw_payload_omitted INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS finding_uid_map (
finding_uid TEXT PRIMARY KEY,
finding_id INTEGER NOT NULL,
created_at TEXT NOT NULL,
FOREIGN KEY(finding_id) REFERENCES findings(id)
);
CREATE TABLE IF NOT EXISTS errors (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER,
cycle_id INTEGER,
target_scan_id INTEGER,
source TEXT,
query TEXT,
target TEXT,
normalized_target TEXT,
category TEXT,
summary TEXT,
raw_error TEXT,
created_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS queue_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER,
cycle_id INTEGER,
source TEXT,
phase TEXT,
todo_count INTEGER,
checked_count INTEGER,
todo_file TEXT,
checked_file TEXT,
captured_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id)
);
CREATE TABLE IF NOT EXISTS config_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER,
cycle_id INTEGER,
scope TEXT,
source TEXT,
config_json TEXT,
captured_at TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES runs(id),
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id)
);
CREATE TABLE IF NOT EXISTS package_repo_candidates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
package_source TEXT NOT NULL,
package_name TEXT NOT NULL,
package_version TEXT NOT NULL,
query TEXT,
repo_url TEXT NOT NULL,
provider TEXT,
evidence_json TEXT,
confidence TEXT,
first_seen_at TEXT NOT NULL,
last_seen_at TEXT NOT NULL,
last_run_id INTEGER,
last_cycle_id INTEGER,
UNIQUE(package_source, package_name, package_version, repo_url),
FOREIGN KEY(last_run_id) REFERENCES runs(id),
FOREIGN KEY(last_cycle_id) REFERENCES source_cycles(id)
);
CREATE TABLE IF NOT EXISTS target_queue (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
platform TEXT NOT NULL,
query TEXT,
target TEXT NOT NULL,
normalized_target TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER DEFAULT 0,
lease_owner TEXT,
lease_token TEXT,
claim_batch TEXT,
leased_at TEXT,
lease_expires_at TEXT,
available_after TEXT,
target_scan_id INTEGER,
last_error TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT,
resolver_state TEXT,
resolver_due_at TEXT,
resolver_attempts INTEGER NOT NULL DEFAULT 0,
resolver_token TEXT,
current_result_reservation_id INTEGER,
claim_event_id TEXT,
remote_modified_at TEXT,
scan_remote_modified_at TEXT,
covered_ref TEXT,
covered_head TEXT,
UNIQUE(source, normalized_target),
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
FOREIGN KEY(current_result_reservation_id) REFERENCES result_reservations(id)
);
CREATE TABLE IF NOT EXISTS scan_publication_outbox (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target_scan_id INTEGER NOT NULL UNIQUE,
payload_json TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER DEFAULT 0,
last_error TEXT,
lease_owner TEXT,
lease_expires_at TEXT,
available_after TEXT,
created_at TEXT NOT NULL,
delivered_at TEXT,
updated_at TEXT NOT NULL,
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
);
CREATE TABLE IF NOT EXISTS target_queue_reconciliation_cursors (
source_file TEXT PRIMARY KEY,
file_identity TEXT NOT NULL,
file_size INTEGER NOT NULL DEFAULT 0,
file_mtime_ns INTEGER NOT NULL DEFAULT 0,
source TEXT NOT NULL,
platform TEXT NOT NULL,
byte_offset INTEGER NOT NULL DEFAULT 0,
line_number INTEGER NOT NULL DEFAULT 0,
discarding_oversized INTEGER NOT NULL DEFAULT 0,
oversized_line_start INTEGER,
cumulative_rows INTEGER NOT NULL DEFAULT 0,
cumulative_bytes INTEGER NOT NULL DEFAULT 0,
cumulative_inserted INTEGER NOT NULL DEFAULT 0,
cumulative_rejected INTEGER NOT NULL DEFAULT 0,
completed_at TEXT,
last_report_json TEXT,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS target_queue_reconciliation_issues (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_file TEXT NOT NULL,
file_identity TEXT NOT NULL,
source TEXT NOT NULL,
platform TEXT NOT NULL,
line_number INTEGER NOT NULL,
byte_offset INTEGER NOT NULL,
reason TEXT NOT NULL,
target_preview TEXT,
created_at TEXT NOT NULL,
resolved_at TEXT
);
''' + KEYCHECK_RESULTS_SQL + PIPELINE_SCHEMA_SQL + r'''
CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
CREATE INDEX IF NOT EXISTS idx_runs_selected_source_status ON runs(selected_source, status, id);
CREATE INDEX IF NOT EXISTS idx_source_cycles_run_id ON source_cycles(run_id);
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_started ON source_cycles(source, started_at);
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_query ON source_cycles(source, query);
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_status ON source_cycles(source, status, id);
CREATE INDEX IF NOT EXISTS idx_target_scans_cycle_id ON target_scans(cycle_id);
CREATE INDEX IF NOT EXISTS idx_target_scans_queue_id ON target_scans(queue_id);
CREATE INDEX IF NOT EXISTS idx_target_scans_result_reservation ON target_scans(result_reservation_id, id);
CREATE INDEX IF NOT EXISTS idx_target_scans_source_status ON target_scans(source, status);
CREATE INDEX IF NOT EXISTS idx_target_scans_source_ended ON target_scans(source, ended_at DESC);
CREATE INDEX IF NOT EXISTS idx_target_scans_source_ended_id ON target_scans(source, ended_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_target_scans_normalized_target ON target_scans(normalized_target);
CREATE INDEX IF NOT EXISTS idx_target_scans_source_skip_ended ON target_scans(source, skipped_reason, ended_at DESC);
CREATE INDEX IF NOT EXISTS idx_target_scans_cooldown_recent ON target_scans(source, ended_at DESC, id DESC) WHERE status = 'skipped' AND ((source = 'github_actions' AND (skipped_reason = 'no downloadable workflow logs or artifacts' OR skipped_reason = 'no recent workflow runs')) OR (source = 'gitlab_ci' AND (skipped_reason = 'no downloadable job traces or artifacts' OR skipped_reason = 'no recent pipelines')));
CREATE UNIQUE INDEX IF NOT EXISTS uq_target_scans_scan_event_id ON target_scans(scan_event_id) WHERE scan_event_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_target_scans_event_hash ON target_scans(scan_event_id, scan_event_hash);
CREATE INDEX IF NOT EXISTS idx_findings_cycle_id ON findings(cycle_id);
CREATE INDEX IF NOT EXISTS idx_findings_target_scan_id_id ON findings(target_scan_id, id);
CREATE INDEX IF NOT EXISTS idx_findings_source ON findings(source);
CREATE INDEX IF NOT EXISTS idx_findings_detector ON findings(detector_name);
CREATE INDEX IF NOT EXISTS idx_findings_secret_hash ON findings(secret_hash);
CREATE INDEX IF NOT EXISTS idx_findings_detector_secret_hash ON findings(detector_secret_hash);
CREATE INDEX IF NOT EXISTS idx_findings_fingerprint ON findings(finding_fingerprint);
CREATE INDEX IF NOT EXISTS idx_findings_finding_uid ON findings(finding_uid);
CREATE INDEX IF NOT EXISTS idx_errors_cycle_id ON errors(cycle_id);
CREATE INDEX IF NOT EXISTS idx_errors_source_category ON errors(source, category);
CREATE INDEX IF NOT EXISTS idx_errors_target_scan_id ON errors(target_scan_id);
CREATE INDEX IF NOT EXISTS idx_queue_snapshots_source_time ON queue_snapshots(source, captured_at);
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_query ON package_repo_candidates(query);
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_repo ON package_repo_candidates(repo_url);
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_provider_seen ON package_repo_candidates(LOWER(provider), last_seen_at DESC);
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_source_seen ON package_repo_candidates(package_source, last_seen_at);
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_query_seen ON package_repo_candidates(query, last_seen_at DESC, id DESC) WHERE repo_url IS NOT NULL AND repo_url <> '';
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_recent_lookup ON package_repo_candidates(last_seen_at DESC, id DESC, package_source, query, package_name) WHERE repo_url IS NOT NULL AND repo_url <> '';
CREATE INDEX IF NOT EXISTS idx_target_queue_source_status ON target_queue(source, status, updated_at);
CREATE INDEX IF NOT EXISTS idx_target_queue_observe_source_status ON target_queue(source, status);
CREATE INDEX IF NOT EXISTS idx_target_queue_lease ON target_queue(source, status, lease_expires_at);
CREATE INDEX IF NOT EXISTS idx_target_queue_platform_status ON target_queue(platform, status);
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_batch ON target_queue(claim_batch, lease_owner);
CREATE INDEX IF NOT EXISTS idx_target_queue_claim ON target_queue(source, platform, status, available_after, lease_expires_at, id);
CREATE INDEX IF NOT EXISTS idx_target_queue_resolver_claim ON target_queue(source, platform, status, resolver_state, resolver_due_at, id);
CREATE INDEX IF NOT EXISTS idx_target_queue_source_platform_normalized ON target_queue(source, platform, normalized_target);
CREATE INDEX IF NOT EXISTS idx_target_queue_cold ON target_queue(source, platform, query, id) WHERE status = 'cold';
CREATE INDEX IF NOT EXISTS idx_target_queue_current_reservation ON target_queue(current_result_reservation_id);
CREATE INDEX IF NOT EXISTS idx_target_queue_claimable_v2 ON target_queue(source, platform, status, available_after, id) WHERE current_result_reservation_id IS NULL AND (status = 'pending' OR status = 'deferred');
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_pending ON target_queue(source, platform, id, attempts, available_after) WHERE status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved');
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_deferred ON target_queue(source, platform, available_after, id, attempts) WHERE status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved');
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_in_progress ON target_queue(source, platform, id, attempts, available_after, lease_expires_at, resolver_state) WHERE status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved');
CREATE INDEX IF NOT EXISTS idx_target_queue_active_lease_owner_token ON target_queue(lease_owner, lease_token) WHERE status = 'in_progress';
CREATE INDEX IF NOT EXISTS idx_target_queue_exhausted_attempts ON target_queue(source, platform, status, attempts, id, lease_expires_at) WHERE status = 'pending' OR status = 'deferred' OR status = 'in_progress';
CREATE INDEX IF NOT EXISTS idx_target_queue_updated_rescan ON target_queue(source, platform, completed_at, remote_modified_at, scan_remote_modified_at, id) WHERE status = 'done' AND remote_modified_at IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_scan_publication_outbox_status ON scan_publication_outbox(status, available_after, id);
CREATE INDEX IF NOT EXISTS idx_scan_publication_outbox_age ON scan_publication_outbox(status, created_at, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_scan_publication_outbox_target_scan_id ON scan_publication_outbox(target_scan_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_event_map_event_id ON keycheck_event_map(event_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_finding_uid_map_finding_uid ON finding_uid_map(finding_uid);
CREATE INDEX IF NOT EXISTS idx_reconciliation_issues_open ON target_queue_reconciliation_issues(source_file, resolved_at, id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_reconciliation_issue_identity ON target_queue_reconciliation_issues(source_file, file_identity, line_number, byte_offset, reason);
'''
Reference in New Issue View Git Blame Copy Permalink