30696 lines
1.4 MiB
Plaintext
30696 lines
1.4 MiB
Plaintext
import hashlib
|
|
import hmac
|
|
import importlib
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import math
|
|
import os
|
|
import random
|
|
import re
|
|
import secrets
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from urllib.parse import urlsplit
|
|
|
|
from db_backend import (
|
|
POSTGRES_APPLICATION_SCHEMA,
|
|
connect_host_agent_postgres,
|
|
connect_postgres,
|
|
connect_sqlite,
|
|
database_url_from_env,
|
|
is_postgres_url,
|
|
redact_database_url,
|
|
)
|
|
from result_spool import SpoolHashConflictError, prepare_scan_event
|
|
from keycheck_candidates import candidate_uid, stored_provider_key_hash
|
|
from process_identity import exact_process_identity_state
|
|
from target_identity import (
|
|
docker_target_identity,
|
|
normalize_huggingface_space_id,
|
|
parse_dockerhub_digest_target,
|
|
parse_docker_target,
|
|
postman_target_identity,
|
|
validate_docker_image_reference,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS = 120
|
|
REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS = 60
|
|
ADMIN_DISCARDED_QUEUE_REASON = 'discarded stale unassigned backlog by source administrator'
|
|
ADMIN_DISCARDED_QUEUE_SQL = (
|
|
"target_queue.status = 'quarantined' AND target_queue.last_error = "
|
|
f"'{ADMIN_DISCARDED_QUEUE_REASON}'"
|
|
)
|
|
|
|
|
|
class RuntimeSafetySchemaError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class PipelineCapacityUnavailable(RuntimeError):
|
|
pass
|
|
|
|
|
|
class RuntimeControlConflictError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class RuntimeControlRevisionConflictError(RuntimeControlConflictError):
|
|
def __init__(self, expected_revision, current_state):
|
|
self.expected_revision = expected_revision
|
|
self.current_state = current_state
|
|
super().__init__(
|
|
f'runtime control revision changed from {expected_revision} '
|
|
f'to {current_state["revision"]}'
|
|
)
|
|
|
|
|
|
class RuntimeOperationIdentityConflictError(RuntimeControlConflictError):
|
|
pass
|
|
|
|
|
|
class RuntimeOperationTransitionError(RuntimeControlConflictError):
|
|
pass
|
|
|
|
|
|
class RuntimeControlTransitionError(RuntimeControlConflictError):
|
|
pass
|
|
|
|
|
|
class DiscoveryPausedError(RuntimeError):
|
|
def __init__(self, control_state):
|
|
self.control_state = dict(control_state)
|
|
super().__init__('provider discovery is paused')
|
|
|
|
|
|
class ScanEventConflictError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class WorkerObservabilityConflictError(ScanEventConflictError):
|
|
pass
|
|
|
|
|
|
class WorkerProgressInactiveError(WorkerObservabilityConflictError):
|
|
pass
|
|
|
|
|
|
def _canonical_worker_contract(kind, payload):
|
|
validators = {
|
|
'progress': ('validate_worker_event', 'validate_progress_event', 'validate_event'),
|
|
'diagnostic': (
|
|
'validate_worker_diagnostic', 'validate_diagnostic_envelope',
|
|
'validate_diagnostic',
|
|
),
|
|
}
|
|
if kind not in validators:
|
|
raise ValueError(f'unknown worker contract kind: {kind}')
|
|
try:
|
|
contracts = importlib.import_module('worker_contracts')
|
|
except ModuleNotFoundError as exc:
|
|
if exc.name == 'worker_contracts':
|
|
raise RuntimeError(
|
|
f'worker_contracts is required to persist worker {kind} records'
|
|
) from exc
|
|
raise RuntimeError(f'worker_contracts import failed: missing {exc.name}') from exc
|
|
except ImportError as exc:
|
|
raise RuntimeError(f'worker_contracts import failed: {exc}') from exc
|
|
codecs = {
|
|
'progress': ('decode_worker_event', 'encode_worker_event'),
|
|
'diagnostic': ('decode_diagnostic_envelope', 'encode_diagnostic_envelope'),
|
|
}
|
|
decoder = getattr(contracts, codecs[kind][0], None)
|
|
encoder = getattr(contracts, codecs[kind][1], None)
|
|
if callable(decoder) and callable(encoder):
|
|
try:
|
|
raw = json.dumps(
|
|
dict(payload or {}), ensure_ascii=True, sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('ascii')
|
|
canonical_bytes = encoder(decoder(raw))
|
|
normalized = json.loads(canonical_bytes.decode('ascii'))
|
|
except (TypeError, ValueError, UnicodeError) as exc:
|
|
raise ValueError(f'worker {kind} contract is invalid') from exc
|
|
canonical = canonical_bytes.decode('ascii')
|
|
return normalized, canonical, hashlib.sha256(canonical_bytes).hexdigest()
|
|
validator = next(
|
|
(getattr(contracts, name) for name in validators[kind]
|
|
if callable(getattr(contracts, name, None))),
|
|
None,
|
|
)
|
|
if validator is None:
|
|
expected = ', '.join((*codecs[kind], *validators[kind]))
|
|
raise RuntimeError(
|
|
f'worker_contracts has no {kind} validator; expected one of: {expected}'
|
|
)
|
|
normalized = validator(dict(payload or {}))
|
|
if normalized is None:
|
|
normalized = dict(payload or {})
|
|
elif not isinstance(normalized, dict):
|
|
to_dict = getattr(normalized, 'to_dict', None)
|
|
if not callable(to_dict) or not isinstance((normalized := to_dict()), dict):
|
|
raise RuntimeError(f'worker_contracts {kind} validator returned a non-mapping')
|
|
try:
|
|
canonical = json.dumps(
|
|
normalized, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError(f'worker {kind} contract is not canonical JSON') from exc
|
|
return normalized, canonical, hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
|
|
|
|
|
class DockerCoverageDispositionConflictError(ScanEventConflictError):
|
|
pass
|
|
|
|
|
|
class DockerFindingAttributionLimitError(ValueError):
|
|
pass
|
|
|
|
|
|
class DiscoveryRetryLeaseError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def env_int(name, default):
|
|
try:
|
|
return int(os.getenv(name, default))
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def env_float(name, default):
|
|
try:
|
|
return float(os.getenv(name, default))
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
DB_FILENAME = 'scanner.db'
|
|
SQLITE_BUSY_TIMEOUT_MS = max(1000, env_int('SCANNER_SQLITE_BUSY_TIMEOUT_MS', 15000))
|
|
SQLITE_CONNECT_TIMEOUT_SEC = max(1, env_int('SCANNER_SQLITE_CONNECT_TIMEOUT_SEC', max(1, SQLITE_BUSY_TIMEOUT_MS // 1000)))
|
|
SQLITE_LOCK_RETRY_ATTEMPTS = max(1, env_int('SCANNER_SQLITE_LOCK_RETRY_ATTEMPTS', 8))
|
|
SQLITE_LOCK_RETRY_BASE_SEC = max(0.05, env_float('SCANNER_SQLITE_LOCK_RETRY_BASE_SEC', 0.25))
|
|
SQLITE_LOCK_RETRY_MAX_SEC = max(0.5, env_float('SCANNER_SQLITE_LOCK_RETRY_MAX_SEC', 5.0))
|
|
STALE_RUN_FINALIZE_BATCH_SIZE = max(1, env_int('SCANNER_STALE_RUN_FINALIZE_BATCH_SIZE', 250))
|
|
STALE_RUN_FINALIZE_MAX_BATCHES = max(1, env_int('SCANNER_STALE_RUN_FINALIZE_MAX_BATCHES', 20))
|
|
KNOWN_TARGET_LOOKUP_BATCH_SIZE = min(64, max(1, env_int('SCANNER_KNOWN_TARGET_LOOKUP_BATCH_SIZE', 64)))
|
|
DISCOVERY_RETRY_MAX_QUERY_CHARS = 256
|
|
DISCOVERY_RETRY_MAX_PAGE = 30
|
|
DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE = 100
|
|
DISCOVERY_RETRY_MAX_ALLOWLIST = 256
|
|
DISCOVERY_RETRY_MAX_CLAIM = 10
|
|
DISCOVERY_RETRY_MAX_ATTEMPTS = 1000000
|
|
DISCOVERY_RETRY_BASE_DELAY_SEC = min(
|
|
3600, max(1, env_int('DOCKERHUB_DISCOVERY_RETRY_BASE_SEC', 60)),
|
|
)
|
|
DISCOVERY_RETRY_MAX_DELAY_SEC = min(86400, max(
|
|
DISCOVERY_RETRY_BASE_DELAY_SEC,
|
|
env_int('DOCKERHUB_DISCOVERY_RETRY_MAX_SEC', 21600),
|
|
))
|
|
DISCOVERY_RETRY_MAX_TRUSTED_DELAY_SEC = 86400
|
|
DISCOVERY_RETRY_PASS_KINDS = frozenset(('ordinary', 'deep'))
|
|
DISCOVERY_RETRY_WORK_KINDS = frozenset(('query', 'page', 'range'))
|
|
DISCOVERY_RETRY_ERROR_CATEGORIES = frozenset((
|
|
'account_pool_exhausted',
|
|
'auth_forbidden',
|
|
'auth_invalid',
|
|
'auth_unavailable',
|
|
'invalid_payload',
|
|
'network',
|
|
'page_unavailable',
|
|
'policy_mismatch',
|
|
'provider_cooldown',
|
|
'provider_unavailable',
|
|
'query_removed',
|
|
'rate_limit',
|
|
'remote_transient',
|
|
'request_failed',
|
|
'tail_unavailable',
|
|
'transport',
|
|
))
|
|
DISCOVERY_RETRY_DUE_INDEX_PREDICATE = "status = 'pending'"
|
|
DISCOVERY_RETRY_LEASE_INDEX_PREDICATE = "status = 'leased'"
|
|
CLAIM_CANDIDATE_MIN = min(1000, max(1, env_int('SCANNER_CLAIM_CANDIDATE_MIN', 64)))
|
|
CLAIM_CANDIDATE_MAX = max(CLAIM_CANDIDATE_MIN, min(1000, env_int('SCANNER_CLAIM_CANDIDATE_MAX', 1000)))
|
|
RESULT_SPOOL_ADVISORY_CLASS = 1414681926
|
|
RESULT_SPOOL_ADVISORY_OBJECT = 1397772111
|
|
RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC = max(3600, env_int('RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC', 86400))
|
|
PIPELINE_ADVISORY_CLASS = 1414681926
|
|
PIPELINE_ADVISORY_OBJECTS = {
|
|
'result_ingester': 1768842867,
|
|
'jsonl_projector': 1785753445,
|
|
}
|
|
CI_SOFT_SKIP_REASONS = {
|
|
'github_actions': frozenset(('no recent workflow runs', 'no downloadable workflow logs or artifacts')),
|
|
'gitlab_ci': frozenset(('no recent pipelines', 'no downloadable job traces or artifacts')),
|
|
}
|
|
|
|
|
|
def _ci_cooldown_source_predicate(source):
|
|
reasons = ' OR '.join(
|
|
f"skipped_reason = '{reason}'" for reason in sorted(CI_SOFT_SKIP_REASONS[source])
|
|
)
|
|
return f"source = '{source}' AND ({reasons})"
|
|
|
|
|
|
CI_COOLDOWN_INDEX_PREDICATE = "status = 'skipped' AND (" + ' OR '.join(
|
|
_ci_cooldown_source_predicate(source) for source in sorted(CI_SOFT_SKIP_REASONS)
|
|
) + ')'
|
|
TARGET_QUEUE_OBSERVABILITY_STATUSES = (
|
|
'pending', 'deferred', 'in_progress', 'done', 'failed', 'quarantined', 'cold',
|
|
)
|
|
TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT = min(
|
|
10000, max(100, env_int('TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT', 100)),
|
|
)
|
|
TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS = min(
|
|
10000, max(1000, env_int('TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS', 5000)),
|
|
)
|
|
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC = 300
|
|
RETIREMENT_STATEMENT_TIMEOUT_MS = 300000
|
|
KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES = 3 * 1024 * 1024
|
|
HAS_CLAIMABLE_TARGETS_V2_SQL = '''SELECT (
|
|
EXISTS (
|
|
SELECT 1 FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND status = 'pending'
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
) OR EXISTS (
|
|
SELECT 1 FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND status = 'deferred'
|
|
AND available_after IS NOT NULL AND available_after <= ?
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
)
|
|
) AS present'''
|
|
PACKAGE_CANDIDATE_LOOKUP_MAX_RESULTS = 10000
|
|
PACKAGE_CANDIDATE_SCAN_MAX_ROWS = 50000
|
|
PACKAGE_CANDIDATE_SCAN_PAGE_SIZE = 250
|
|
PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE = 25
|
|
PACKAGE_REPO_NONEMPTY_PREDICATE = "repo_url IS NOT NULL AND repo_url <> ''"
|
|
DOCKER_LEGACY_PROVENANCE_SEED_MAX_ROWS = 1000000
|
|
DOCKER_LEGACY_PROVENANCE_SEED_PAGE_SIZE = 1000
|
|
DOCKER_DEPTH_EXPERIMENT_MIGRATION = '20260909_18_docker_depth_experiment'
|
|
DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION = '20260910_22_docker_depth_rollout_authority'
|
|
DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION = (
|
|
'20260910_23_docker_schema_selector_authority'
|
|
)
|
|
DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION = (
|
|
'20260910_24_docker_depth_scarcity_cohort'
|
|
)
|
|
DOCKER_DEPTH_HOLD_QUERY_INDEX_MIGRATION = (
|
|
'20260910_25_docker_depth_hold_query_index'
|
|
)
|
|
DOCKER_DEPTH_RESOLVER_REFUND_MIGRATION = (
|
|
'20260910_26_docker_depth_resolver_attempt_refund'
|
|
)
|
|
DOCKER_DEPTH_RESOLVER_DISPOSITION_MIGRATION = (
|
|
'20260910_27_docker_depth_resolver_disposition'
|
|
)
|
|
REMOTE_WORKER_MIGRATION = '20260917_28_remote_worker_admission_transport'
|
|
OPERATIONS_CONTROL_MIGRATION = '20260919_29_operations_control_audit'
|
|
WORKER_OBSERVABILITY_MIGRATION = '20260923_30_worker_observability'
|
|
DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION = (
|
|
'20260924_31_diagnostic_projection_authority'
|
|
)
|
|
ASSIGNMENT_UPLOAD_TIMEOUT_MIGRATION = (
|
|
'20260924_32_assignment_upload_timeout'
|
|
)
|
|
REMOTE_ASSIGNMENT_CAPACITY_MIGRATION = (
|
|
'20260930_33_remote_assignment_capacity'
|
|
)
|
|
RUNTIME_CONTROL_TARGET_KIND = 'runtime-control'
|
|
RUNTIME_CONTROL_DRAIN_STATES = frozenset(('normal', 'draining', 'drained'))
|
|
RUNTIME_CONTROL_ACTION_TARGETS = {
|
|
'control.discovery.pause': 'discovery',
|
|
'control.discovery.resume': 'discovery',
|
|
'control.dispatch.pause': 'dispatch',
|
|
'control.dispatch.resume': 'dispatch',
|
|
'control.drain.start': 'drain',
|
|
'control.drain.cancel': 'drain',
|
|
'control.drain.complete': 'drain',
|
|
}
|
|
RUNTIME_CONTROL_ACTIONS = frozenset(RUNTIME_CONTROL_ACTION_TARGETS)
|
|
RUNTIME_CONTROL_MAX_REVISION = 9223372036854775807
|
|
RUNTIME_CONTROL_MAX_EXPECTED_REVISION = RUNTIME_CONTROL_MAX_REVISION - 1
|
|
RUNTIME_ASYNC_TARGET_KIND = 'runtime-deployment'
|
|
RUNTIME_ASYNC_ACTION_TARGETS = {
|
|
'apply-config': 'config',
|
|
'apply-secrets': 'secrets',
|
|
'apply-both': 'config-secrets',
|
|
'restart': 'runtime',
|
|
}
|
|
RUNTIME_ASYNC_ACTIONS = frozenset(RUNTIME_ASYNC_ACTION_TARGETS)
|
|
RUNTIME_SOURCE_TARGET_KIND = 'managed-source'
|
|
RUNTIME_SOURCE_IDS = frozenset((
|
|
'discovery-producer:gitlab',
|
|
'discovery-producer:dockerhub',
|
|
'discovery-producer:huggingface',
|
|
'result-ingester', 'jsonl-projector', 'janitor', 'worker-api',
|
|
'keychecks', 'docker-shadow', 'dashboard',
|
|
))
|
|
RUNTIME_SOURCE_ACTIONS = frozenset((
|
|
'start', 'stop', 'restart', 'pause', 'resume', 'set-interval',
|
|
'once', 'set-mode', 'set-restart', 'set-restart-delay',
|
|
))
|
|
RUNTIME_SOURCE_OPERATION_ACTIONS = {
|
|
f'supervisor.source.{action}': action for action in RUNTIME_SOURCE_ACTIONS
|
|
}
|
|
RUNTIME_WORKER_ADMIN_TARGET_KIND = 'worker-admin'
|
|
RUNTIME_WORKER_ADMIN_ACTION_TARGETS = {
|
|
'workers.user.create': 'user',
|
|
'workers.user.set-cap': 'user',
|
|
'workers.user.disable': 'user',
|
|
'workers.user.enable': 'user',
|
|
'workers.device.issue': 'device',
|
|
'workers.device.rotate': 'device',
|
|
'workers.device.revoke': 'device',
|
|
'workers.device.unrevoke': 'device',
|
|
'workers.queue.requeue': 'deferred-queue',
|
|
}
|
|
RUNTIME_DOCUMENT_TARGET_KIND = 'runtime-document'
|
|
RUNTIME_DOCUMENT_ACTION_TARGETS = {
|
|
'runtime.config.save': 'config',
|
|
'runtime.secrets.save': 'secrets',
|
|
}
|
|
RUNTIME_MANAGED_FILE_TARGET_KIND = 'managed-file'
|
|
RUNTIME_MANAGED_FILE_ACTIONS = frozenset((
|
|
'files.create', 'files.replace', 'files.delete',
|
|
))
|
|
RUNTIME_MANAGED_FILE_ADVISORY_CLASS = 1836212590
|
|
_RUNTIME_MANAGED_FILE_SQLITE_LOCKS = tuple(
|
|
threading.Lock() for _ in range(64)
|
|
)
|
|
RUNTIME_ASYNC_TERMINAL_RESULTS = frozenset((
|
|
'succeeded', 'failed', 'rolled_back', 'failed_hold',
|
|
))
|
|
RUNTIME_OPERATION_SAFE_CATEGORIES = frozenset((
|
|
'agent_failed', 'agent_unavailable', 'apply_failed',
|
|
'health_check_failed', 'operation_conflict', 'restart_failed',
|
|
'result_invalid', 'revision_conflict', 'rollback_failed',
|
|
'supervisor_action_failed', 'validation_failed', 'worker_admin_mutation_failed',
|
|
'runtime_document_save_failed', 'managed_file_mutation_failed',
|
|
))
|
|
RUNTIME_OPERATION_SAFE_DETAILS = frozenset((
|
|
'active_hash_mismatch', 'apply_failed', 'bounded_result',
|
|
'candidate_hash_mismatch', 'health_check_failed', 'lock_busy',
|
|
'restart_failed', 'rollback_failed', 'validation_failed',
|
|
))
|
|
RUNTIME_AGENT_RESULT_MAX_BYTES = 16 * 1024
|
|
RUNTIME_AGENT_RESULT_MAX_DEPTH = 64
|
|
PIPELINE_MIGRATION_VERSIONS = (
|
|
'20260727_01_pipeline_capacity_bundles',
|
|
'20260727_02_projection_queue',
|
|
'20260727_03_keycheck_queue',
|
|
'20260727_04_normalized_compat',
|
|
'20260727_05_provider_canonical_keycheck_identity',
|
|
'20260727_06_second_audit_high_fixes',
|
|
'20260727_07_final_high_blockers',
|
|
'20260727_08_admission_intent_and_capacity_closure',
|
|
'20260727_09_serialized_recovery_and_bounded_retirement',
|
|
'20260729_10_keycheck_projection_single_writer',
|
|
'20260828_11_exact_git_scan_plans',
|
|
'20260831_12_projection_findings_index',
|
|
'20260901_13_docker_layer_content_scanning',
|
|
'20260902_14_adaptive_docker_payload_scanning',
|
|
'20260906_15_cold_policy_stale_backlog',
|
|
'20260907_16_docker_quarantine_rescan',
|
|
'20260909_17_dockerhub_discovery_retry',
|
|
DOCKER_DEPTH_EXPERIMENT_MIGRATION,
|
|
'20260910_19_docker_retry_provenance',
|
|
'20260910_20_docker_depth_resolver',
|
|
'20260910_21_docker_finding_layer_attribution',
|
|
DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION,
|
|
DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION,
|
|
DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION,
|
|
DOCKER_DEPTH_HOLD_QUERY_INDEX_MIGRATION,
|
|
DOCKER_DEPTH_RESOLVER_REFUND_MIGRATION,
|
|
DOCKER_DEPTH_RESOLVER_DISPOSITION_MIGRATION,
|
|
REMOTE_WORKER_MIGRATION,
|
|
OPERATIONS_CONTROL_MIGRATION,
|
|
WORKER_OBSERVABILITY_MIGRATION,
|
|
DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION,
|
|
ASSIGNMENT_UPLOAD_TIMEOUT_MIGRATION,
|
|
REMOTE_ASSIGNMENT_CAPACITY_MIGRATION,
|
|
)
|
|
PIPELINE_QUARANTINE_REVIEW_STATUSES = frozenset((
|
|
'pending', 'approved_retry', 'approved_rescan', 'discarded', 'resolved',
|
|
))
|
|
PIPELINE_QUARANTINE_LEGACY_REVIEW_STATUSES = frozenset((
|
|
'pending', 'approved_retry', 'discarded', 'resolved',
|
|
))
|
|
FINAL_CUTOVER_MARKER = 'postgres-normalized-v2-authority'
|
|
PIPELINE_REQUIRED_COLUMNS = {
|
|
'runtime_schema_migrations': {'version', 'applied_at', 'code_sha256'},
|
|
'runtime_final_cutover': {'id', 'marker', 'checked_at', 'evidence_sha256'},
|
|
'runtime_operations': {
|
|
'operation_id', 'actor', 'action', 'target_kind', 'target_ref', 'status',
|
|
'safe_category', 'safe_detail', 'expected_revision', 'resulting_revision',
|
|
'expected_identity_json', 'resulting_identity_json', 'agent_state',
|
|
'agent_result_sha256', 'requested_at', 'started_at', 'completed_at',
|
|
'agent_reconciled_at', 'updated_at',
|
|
},
|
|
'runtime_operations_control': {
|
|
'id', 'revision', 'discovery_paused', 'dispatch_paused', 'drain_state',
|
|
'actor', 'operation_id', 'created_at', 'updated_at',
|
|
},
|
|
'runtime_audit_events': {
|
|
'id', 'operation_id', 'actor', 'action', 'target_kind', 'target_ref',
|
|
'result', 'safe_category', 'before_identity_json', 'after_identity_json',
|
|
'before_bytes', 'after_bytes', 'previous_event_id',
|
|
'previous_event_sha256', 'event_sha256', 'created_at',
|
|
},
|
|
'pipeline_capacity': {
|
|
'id', 'bundle_items', 'bundle_bytes', 'projection_items', 'projection_bytes',
|
|
'keycheck_items', 'keycheck_bytes', 'quarantine_items', 'quarantine_bytes', 'updated_at',
|
|
},
|
|
'result_reservations': {
|
|
'id', 'reservation_token', 'bundle_id', 'scan_event_id', 'queue_id', 'state',
|
|
'declared_bundle_bytes', 'reserved_bundle_bytes', 'reserved_projection_bytes',
|
|
'reserved_candidate_items',
|
|
'reserved_candidate_bytes', 'ready_relative_path', 'producer_pid',
|
|
'producer_creation_time', 'producer_executable', 'bundle_credit_released',
|
|
'cleanup_attempts', 'cleanup_available_after', 'git_scan_plan_json',
|
|
'git_scan_plan_sha256', 'docker_layer_plan_json', 'docker_layer_plan_sha256',
|
|
'assignment_kind', 'remote_user_id', 'remote_device_id', 'remote_issued_at',
|
|
'remote_expires_at', 'remote_effective_config_sha256',
|
|
'remote_result_upload_body_timeout_seconds',
|
|
'remote_client_compat_sha256', 'remote_resolution_kind',
|
|
'remote_execution_snapshot_json', 'remote_execution_snapshot_sha256',
|
|
'remote_payload_sha256', 'remote_receipt_id', 'remote_resolution_json',
|
|
'remote_resolved_at', 'remote_diagnostic_projection_version',
|
|
'remote_diagnostic_count', 'remote_diagnostic_uids_sha256',
|
|
},
|
|
'worker_progress_events': {
|
|
'id', 'reservation_id', 'remote_device_id', 'schema_version', 'sequence',
|
|
'event_type', 'phase', 'event_timestamp', 'phase_started_at', 'instance_id',
|
|
'slot_id', 'source', 'event_json', 'event_sha256', 'received_at',
|
|
},
|
|
'worker_diagnostics': {
|
|
'id', 'diagnostic_uid', 'reservation_id', 'target_scan_id', 'schema_version',
|
|
'scan_event_id', 'slot_id', 'attempt', 'source', 'phase', 'kind', 'category',
|
|
'code', 'summary', 'retryable', 'occurred_at', 'captured_at', 'envelope_json',
|
|
'envelope_sha256', 'body_payload_json', 'log_payload_json', 'received_at',
|
|
},
|
|
'admission_intents': {
|
|
'reservation_token', 'intent_sha256', 'state', 'reservation_id',
|
|
'remote_user_id', 'remote_device_id', 'created_at', 'updated_at',
|
|
},
|
|
'remote_worker_users': {
|
|
'id', 'user_key', 'active_assignment_cap', 'disabled_at', 'created_at', 'updated_at',
|
|
},
|
|
'remote_worker_devices': {
|
|
'id', 'user_id', 'device_key', 'token_sha256', 'revoked_at',
|
|
'last_contact_at', 'created_at', 'updated_at',
|
|
},
|
|
'admission_intent_retirement': {
|
|
'id', 'retired_count', 'chain_sha256', 'cursor_token', 'updated_at',
|
|
},
|
|
'pipeline_artifact_retirement': {
|
|
'id', 'retired_count', 'chain_sha256', 'cursor_id', 'updated_at',
|
|
},
|
|
'result_bundles': {
|
|
'reservation_id', 'bundle_id', 'scan_event_id', 'scan_event_hash', 'relative_path',
|
|
'actual_bytes', 'state', 'ingest_lease_generation', 'ingest_lease_token',
|
|
},
|
|
'pipeline_leases': {'worker_name', 'generation', 'lease_token', 'state', 'lease_expires_at'},
|
|
'scan_result_compat': {'target_scan_id', 'metadata_json', 'metadata_sha256', 'metadata_bytes'},
|
|
'finding_compat_payloads': {'finding_id', 'payload_sha256', 'payload_bytes', 'payload_omitted'},
|
|
'projection_jobs': {'id', 'job_kind', 'event_id', 'event_hash', 'status', 'lease_token'},
|
|
'projection_streams': {'stream_name', 'base_relative_path', 'current_generation', 'rotation_bytes'},
|
|
'projection_cursors': {'stream_name', 'generation', 'committed_offset', 'last_job_id'},
|
|
'projection_appends': {'id', 'job_id', 'stream_name', 'byte_offset', 'byte_length', 'state'},
|
|
'projection_append_audit': {
|
|
'id', 'append_id', 'job_id', 'stream_name', 'state', 'reason_code',
|
|
},
|
|
'projection_rotations': {'id', 'stream_name', 'from_generation', 'to_generation', 'state'},
|
|
'keycheck_credentials': {
|
|
'id', 'service', 'credential_hash', 'provider_key_hash', 'candidate_kind',
|
|
},
|
|
'keycheck_candidates': {
|
|
'id', 'candidate_uid', 'credential_id', 'service', 'routed_service',
|
|
'secret_hash', 'state',
|
|
'lease_token', 'result_projection_reserved_bytes',
|
|
'result_projection_credit_transferred',
|
|
},
|
|
'keycheck_current_state': {'credential_id', 'service', 'status', 'last_result_id', 'state_version'},
|
|
'pipeline_quarantine': {
|
|
'id', 'subsystem', 'object_type', 'reason_code', 'review_status',
|
|
'capacity_credit_applied', 'capacity_items', 'capacity_bytes',
|
|
},
|
|
'pipeline_artifacts': {
|
|
'id', 'subsystem', 'artifact_kind', 'owner_id', 'owner_key',
|
|
'relative_path', 'payload_sha256', 'byte_count', 'state', 'updated_at',
|
|
'cleanup_attempts', 'cleanup_available_after',
|
|
},
|
|
'janitor_cursors': {'layout_name', 'last_name', 'wrap_count', 'updated_at'},
|
|
'keycheck_recheck_cursors': {
|
|
'cursor_key', 'service', 'status_scope', 'last_credential_id',
|
|
'wrap_count', 'updated_at',
|
|
},
|
|
'docker_content_blobs': {
|
|
'digest', 'coverage_policy_sha256', 'descriptor_kind', 'declared_bytes', 'media_type', 'state',
|
|
'attempts', 'max_attempts', 'available_after', 'lease_reservation_id',
|
|
'lease_token', 'lease_plan_sha256', 'lease_expires_at',
|
|
'covered_reservation_id', 'covered_scan_event_id', 'covered_policy_sha256',
|
|
'verified_bytes', 'covered_at', 'last_error_code', 'last_error_detail',
|
|
'created_at', 'updated_at',
|
|
},
|
|
'docker_image_blob_coverage': {
|
|
'queue_id', 'manifest_digest', 'position', 'blob_digest', 'coverage_policy_sha256', 'descriptor_kind',
|
|
'selection_policy_sha256', 'plan_sha256', 'reservation_id', 'selected', 'selection_reason',
|
|
'coverage_state', 'covered_at', 'last_error_code', 'created_at', 'updated_at',
|
|
},
|
|
'docker_adaptive_shadow_reports': {
|
|
'id', 'report_token', 'evaluator_version', 'state', 'scan_policy_sha256',
|
|
'execution_policy_sha256', 'selection_policy_sha256', 'cohort_size',
|
|
'completed_pairs', 'full_routed_count', 'adaptive_routed_count',
|
|
'routed_intersection_count', 'full_detector_count', 'adaptive_detector_count',
|
|
'detector_intersection_count', 'full_slot_ms', 'adaptive_slot_ms',
|
|
'omitted_descriptor_count', 'failure_count', 'privacy_violation_count',
|
|
'safety_regression_count', 'selection_metrics_json',
|
|
'sink_checkpoint_count', 'routed_recall_ppm', 'slot_ratio_ppm',
|
|
'recall_threshold_ppm', 'slot_threshold_ppm', 'passed', 'lease_owner',
|
|
'lease_token', 'lease_expires_at', 'started_at', 'completed_at',
|
|
'created_at', 'updated_at',
|
|
},
|
|
'discovery_retry_queue': {
|
|
'id', 'work_key', 'source', 'query', 'source_cycle_id', 'policy_sha256', 'pass_kind',
|
|
'work_kind', 'page_start', 'page_end', 'next_page', 'status', 'attempts',
|
|
'available_after', 'lease_owner', 'lease_token', 'leased_at',
|
|
'lease_expires_at', 'last_error_category', 'held_at', 'created_at',
|
|
'updated_at',
|
|
},
|
|
'target_queue_policy_events': {
|
|
'id', 'queue_id', 'action', 'prior_status', 'next_status', 'source',
|
|
'platform', 'query', 'reason_code', 'config_sha256', 'policy_sha256',
|
|
'manifest_sha256', 'review_audit_sha256', 'reverses_event_id',
|
|
'experiment_id', 'prior_updated_at', 'created_at',
|
|
},
|
|
}
|
|
REDACTED = '***REDACTED***'
|
|
SECRET_KEY_PARTS = (
|
|
'token',
|
|
'secret',
|
|
'password',
|
|
'authorization',
|
|
'credential',
|
|
'apikey',
|
|
'api_key',
|
|
'private_key',
|
|
)
|
|
DATABASE_SECRET_KEY_PARTS = (
|
|
'database_url',
|
|
'dashboard_db_url',
|
|
'db_url',
|
|
'dsn',
|
|
'connection_string',
|
|
'postgres_url',
|
|
)
|
|
ENDPOINT_METADATA_MAX_CHARS = 2048
|
|
POSTMAN_RESOURCE_MAX_CHARS = 4096
|
|
_ENDPOINT_DNS_LABEL_RE = re.compile(r'^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$')
|
|
|
|
|
|
def _validated_endpoint_host(value, require_domain=False):
|
|
host = str(value or '').strip().rstrip('.')
|
|
if not host or len(host) > 253 or not host.isascii():
|
|
return ''
|
|
try:
|
|
return ipaddress.ip_address(host).compressed.lower()
|
|
except ValueError:
|
|
labels = host.split('.')
|
|
if any(not _ENDPOINT_DNS_LABEL_RE.fullmatch(label) for label in labels):
|
|
return ''
|
|
if require_domain and len(labels) < 2 and host.lower() != 'localhost':
|
|
return ''
|
|
if len(labels) > 1 and not (
|
|
re.fullmatch(r'[A-Za-z]{2,63}', labels[-1])
|
|
or re.fullmatch(r'xn--[A-Za-z0-9-]{1,59}', labels[-1], re.IGNORECASE)
|
|
):
|
|
return ''
|
|
return host.lower()
|
|
|
|
|
|
def _validated_endpoint_port(parsed):
|
|
authority = parsed.netloc.rsplit('@', 1)[-1]
|
|
try:
|
|
port = parsed.port
|
|
except ValueError:
|
|
return None, False
|
|
if authority.startswith('['):
|
|
close = authority.find(']')
|
|
suffix = authority[close + 1:] if close >= 0 else authority
|
|
if close < 0 or (suffix and not re.fullmatch(r':[0-9]+', suffix)):
|
|
return None, False
|
|
elif ':' in authority:
|
|
if authority.count(':') != 1 or not re.fullmatch(r'[0-9]+', authority.rsplit(':', 1)[1]):
|
|
return None, False
|
|
if port is not None and not 1 <= port <= 65535:
|
|
return None, False
|
|
return port, True
|
|
|
|
|
|
def _safe_endpoint_path(value):
|
|
path = str(value or '')
|
|
if not path:
|
|
return ''
|
|
if not path.startswith('/') or any(
|
|
character.isspace() or ord(character) < 32 or ord(character) == 127 or character in '\\?#'
|
|
for character in path
|
|
):
|
|
return ''
|
|
return path
|
|
|
|
|
|
def _format_endpoint(scheme, host, port, path):
|
|
authority = f'[{host}]' if ':' in host else host
|
|
if port is not None:
|
|
authority += f':{port}'
|
|
prefix = f'{scheme}://{authority}' if scheme else authority
|
|
remaining = max(0, ENDPOINT_METADATA_MAX_CHARS - len(prefix))
|
|
return prefix + path[:remaining]
|
|
|
|
|
|
def sanitize_endpoint(value):
|
|
"""Return bounded endpoint metadata without URL credentials or parameters."""
|
|
try:
|
|
text = str(value or '').strip()
|
|
except Exception:
|
|
return ''
|
|
if not text or len(text) > 65536 or any(ord(character) < 32 or ord(character) == 127 for character in text):
|
|
return ''
|
|
|
|
scheme_match = re.match(r'^([A-Za-z][A-Za-z0-9+.-]*):\/\/', text)
|
|
if scheme_match:
|
|
scheme = scheme_match.group(1).lower()
|
|
if scheme not in ('http', 'https'):
|
|
return ''
|
|
try:
|
|
parsed = urlsplit(text)
|
|
host = _validated_endpoint_host(parsed.hostname)
|
|
port, valid_port = _validated_endpoint_port(parsed)
|
|
except (TypeError, ValueError):
|
|
return ''
|
|
if parsed.scheme.lower() != scheme or not parsed.netloc or not host or not valid_port:
|
|
return ''
|
|
return _format_endpoint(scheme, host, port, _safe_endpoint_path(parsed.path))
|
|
|
|
if re.match(r'(?i)^https?:', text) or '://' in text:
|
|
return ''
|
|
base = re.split(r'[?#]', text, maxsplit=1)[0]
|
|
if not base or base.startswith('/') and not base.startswith('//'):
|
|
return ''
|
|
try:
|
|
parsed = urlsplit(base if base.startswith('//') else '//' + base)
|
|
host = _validated_endpoint_host(parsed.hostname, require_domain=True)
|
|
port, valid_port = _validated_endpoint_port(parsed)
|
|
except (TypeError, ValueError):
|
|
return ''
|
|
if not parsed.netloc or not host or not valid_port:
|
|
return ''
|
|
return _format_endpoint('', host, port, _safe_endpoint_path(parsed.path))
|
|
|
|
|
|
def sanitize_endpoint_host(value):
|
|
endpoint = sanitize_endpoint(value)
|
|
if not endpoint:
|
|
return ''
|
|
try:
|
|
parsed = urlsplit(endpoint if re.match(r'(?i)^https?://', endpoint) else '//' + endpoint)
|
|
return _validated_endpoint_host(parsed.hostname)
|
|
except (TypeError, ValueError):
|
|
return ''
|
|
|
|
|
|
def _bounded_postman_label(value, max_chars):
|
|
text = str(value or '')
|
|
if any(ord(character) < 32 or ord(character) == 127 for character in text):
|
|
return ''
|
|
if re.search(r'(?i)https?://', text) or re.search(
|
|
r'(?i)(?:^|[?&])(?:api[_-]?key|token|password|secret|credential)=', text,
|
|
):
|
|
return ''
|
|
return text[:max_chars]
|
|
|
|
|
|
def _sanitize_postman_resource(value):
|
|
text = str(value or '').strip()
|
|
if not text or any(ord(character) < 32 or ord(character) == 127 for character in text):
|
|
return ''
|
|
if '://' in text or text.startswith('//') or any(marker in text for marker in ('@', '?', '#')):
|
|
return sanitize_endpoint(text)
|
|
return text[:POSTMAN_RESOURCE_MAX_CHARS]
|
|
|
|
|
|
def sanitize_postman_context(context):
|
|
if not isinstance(context, dict):
|
|
return {}
|
|
safe = {}
|
|
label_limits = {
|
|
'provider': 128,
|
|
'credential_kind': 128,
|
|
'credential_confidence': 128,
|
|
'context_location': 128,
|
|
'variable_name': 512,
|
|
'auth_type': 128,
|
|
}
|
|
for key, limit in label_limits.items():
|
|
if key in context:
|
|
safe[key] = _bounded_postman_label(context.get(key), limit)
|
|
if 'json_path' in context:
|
|
safe['json_path'] = _sanitize_postman_resource(context.get('json_path'))
|
|
if 'placeholder' in context:
|
|
safe['placeholder'] = bool(context.get('placeholder'))
|
|
|
|
endpoint = sanitize_endpoint(context.get('endpoint'))
|
|
host = sanitize_endpoint_host(endpoint) or sanitize_endpoint_host(context.get('host'))
|
|
if 'endpoint' in context or 'host' in context:
|
|
safe['endpoint'] = endpoint
|
|
safe['host'] = host
|
|
return safe
|
|
|
|
|
|
def sanitize_postman_finding(finding):
|
|
if not isinstance(finding, dict) or not isinstance(finding.get('PostmanContext'), dict):
|
|
return finding
|
|
safe = dict(finding)
|
|
safe['PostmanContext'] = sanitize_postman_context(finding['PostmanContext'])
|
|
return safe
|
|
|
|
RUNS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'started_at': ('text', True), 'ended_at': ('text', False),
|
|
'duration_sec': ('real', False), 'status': ('text', True),
|
|
'invocation_mode': ('text', False), 'command_line': ('text', False),
|
|
'argv_json': ('text', False), 'selected_source': ('text', False),
|
|
'selected_platform': ('text', False), 'config_path': ('text', False),
|
|
'config_hash': ('text', False), 'enabled_sources_json': ('text', False),
|
|
'db_path': ('text', False), 'total_fetched': ('integer', False),
|
|
'total_queued_new': ('integer', False), 'total_scan_requested': ('integer', False),
|
|
'total_scanned': ('integer', False), 'total_clean': ('integer', False),
|
|
'total_found': ('integer', False), 'total_skipped': ('integer', False),
|
|
'total_errors': ('integer', False), 'total_findings': ('integer', False),
|
|
'total_verified_findings': ('integer', False), 'total_unique_secrets': ('integer', False),
|
|
'total_unique_findings': ('integer', False), 'error': ('text', False),
|
|
'total_staged': ('integer', True), 'total_quarantined': ('integer', True),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
SOURCE_CYCLES_COLUMN_SPECS = {
|
|
'id': ('id', True), 'run_id': ('id_ref', False), 'source': ('text', False),
|
|
'platform': ('text', False), 'mode': ('text', False), 'query': ('text', False),
|
|
'query_index': ('integer', False), 'query_count': ('integer', False),
|
|
'auth_name': ('text', False), 'started_at': ('text', True), 'ended_at': ('text', False),
|
|
'duration_sec': ('real', False), 'status': ('text', True), 'message': ('text', False),
|
|
'config_json': ('text', False), 'queue_todo_before': ('integer', False),
|
|
'queue_checked_before': ('integer', False), 'queue_todo_after': ('integer', False),
|
|
'queue_checked_after': ('integer', False), 'fetched_count': ('integer', False),
|
|
'queued_new_count': ('integer', False), 'queued_updated_count': ('integer', True),
|
|
'scan_requested_count': ('integer', False),
|
|
'scanned_count': ('integer', False), 'clean_count': ('integer', False),
|
|
'found_count': ('integer', False), 'skipped_count': ('integer', False),
|
|
'error_count': ('integer', False), 'findings_count': ('integer', False),
|
|
'verified_findings_count': ('integer', False), 'unique_secrets_count': ('integer', False),
|
|
'unique_findings_count': ('integer', False), 'targets_per_hour': ('real', False),
|
|
'hit_rate': ('real', False), 'verified_hit_rate': ('real', False),
|
|
'error_rate': ('real', False), 'created_at': ('text', True), 'updated_at': ('text', True),
|
|
'staged_count': ('integer', True), 'ingested_count': ('integer', True),
|
|
'quarantined_count': ('integer', True),
|
|
}
|
|
|
|
ERRORS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
|
|
'target_scan_id': ('id_ref', False), 'source': ('text', False), 'query': ('text', False),
|
|
'target': ('text', False), 'normalized_target': ('text', False),
|
|
'category': ('text', False), 'summary': ('text', False), 'raw_error': ('text', False),
|
|
'created_at': ('text', True),
|
|
}
|
|
|
|
QUEUE_SNAPSHOTS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
|
|
'source': ('text', False), 'phase': ('text', False), 'todo_count': ('integer', False),
|
|
'checked_count': ('integer', False), 'todo_file': ('text', False),
|
|
'checked_file': ('text', False), 'captured_at': ('text', True),
|
|
}
|
|
|
|
CONFIG_SNAPSHOTS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
|
|
'scope': ('text', False), 'source': ('text', False), 'config_json': ('text', False),
|
|
'captured_at': ('text', True),
|
|
}
|
|
|
|
PACKAGE_REPO_CANDIDATES_COLUMN_SPECS = {
|
|
'id': ('id', True), 'package_source': ('text', True), 'package_name': ('text', True),
|
|
'package_version': ('text', True), 'query': ('text', False), 'repo_url': ('text', True),
|
|
'provider': ('text', False), 'evidence_json': ('text', False),
|
|
'confidence': ('text', False), 'first_seen_at': ('text', True),
|
|
'last_seen_at': ('text', True), 'last_run_id': ('id_ref', False),
|
|
'last_cycle_id': ('id_ref', False),
|
|
}
|
|
|
|
KEYCHECK_COLUMN_SPECS = {
|
|
'id': ('id', True),
|
|
'service': ('text', True),
|
|
'status': ('text', True),
|
|
'status_group': ('text', True),
|
|
'checked_at': ('text', True),
|
|
'key_hash': ('text', False),
|
|
'secret_hash': ('text', False),
|
|
'key_masked': ('text', False),
|
|
'finding_id': ('id_ref', False),
|
|
'target_scan_id': ('id_ref', False),
|
|
'cycle_id': ('id_ref', False),
|
|
'run_id': ('id_ref', False),
|
|
'source': ('text', False),
|
|
'query': ('text', False),
|
|
'target': ('text', False),
|
|
'detector_name': ('text', False),
|
|
'found_at': ('text', False),
|
|
'message': ('text', False),
|
|
'metadata_json': ('text', False),
|
|
'source_line': ('text', False),
|
|
'detector_secret_hash': ('text', False),
|
|
'event_id': ('text', False),
|
|
'finding_uid': ('text', False),
|
|
'link_status': ('text', False),
|
|
'link_attempts': ('integer', False),
|
|
'linked_at': ('text', False),
|
|
'link_error': ('text', False),
|
|
'created_at': ('text', True),
|
|
'candidate_id': ('id_ref', False),
|
|
'credential_id': ('id_ref', False),
|
|
'result_source': ('text', True),
|
|
}
|
|
|
|
OUTBOX_COLUMN_SPECS = {
|
|
'id': ('id', True),
|
|
'target_scan_id': ('id_ref', True),
|
|
'payload_json': ('text', True),
|
|
'status': ('text', True),
|
|
'attempts': ('integer', False),
|
|
'last_error': ('text', False),
|
|
'lease_owner': ('text', False),
|
|
'lease_expires_at': ('text', False),
|
|
'available_after': ('text', False),
|
|
'created_at': ('text', True),
|
|
'delivered_at': ('text', False),
|
|
'updated_at': ('text', True),
|
|
}
|
|
|
|
CURSOR_COLUMN_SPECS = {
|
|
'source_file': ('text', True),
|
|
'file_identity': ('text', True),
|
|
'file_size': ('id_ref', True),
|
|
'file_mtime_ns': ('id_ref', True),
|
|
'source': ('text', True),
|
|
'platform': ('text', True),
|
|
'byte_offset': ('id_ref', True),
|
|
'line_number': ('id_ref', True),
|
|
'discarding_oversized': ('integer', True),
|
|
'oversized_line_start': ('id_ref', False),
|
|
'cumulative_rows': ('id_ref', True),
|
|
'cumulative_bytes': ('id_ref', True),
|
|
'cumulative_inserted': ('id_ref', True),
|
|
'cumulative_rejected': ('id_ref', True),
|
|
'completed_at': ('text', False),
|
|
'last_report_json': ('text', False),
|
|
'updated_at': ('text', True),
|
|
}
|
|
|
|
RECONCILIATION_ISSUE_COLUMN_SPECS = {
|
|
'id': ('id', True),
|
|
'source_file': ('text', True),
|
|
'file_identity': ('text', True),
|
|
'source': ('text', True),
|
|
'platform': ('text', True),
|
|
'line_number': ('id_ref', True),
|
|
'byte_offset': ('id_ref', True),
|
|
'reason': ('text', True),
|
|
'target_preview': ('text', False),
|
|
'created_at': ('text', True),
|
|
'resolved_at': ('text', False),
|
|
}
|
|
|
|
TARGET_QUEUE_COLUMN_SPECS = {
|
|
'id': ('id', True), 'source': ('text', True), 'platform': ('text', True),
|
|
'query': ('text', False), 'target': ('text', True), 'normalized_target': ('text', True),
|
|
'status': ('text', True), 'attempts': ('integer', False),
|
|
'lease_owner': ('text', False), 'lease_token': ('text', False),
|
|
'claim_batch': ('text', False), 'leased_at': ('text', False),
|
|
'lease_expires_at': ('text', False), 'available_after': ('text', False),
|
|
'target_scan_id': ('id_ref', False), 'last_error': ('text', False),
|
|
'created_at': ('text', True), 'updated_at': ('text', True), 'completed_at': ('text', False),
|
|
'resolver_state': ('text', False), 'resolver_due_at': ('text', False),
|
|
'resolver_attempts': ('integer', True), 'resolver_token': ('text', False),
|
|
'current_result_reservation_id': ('id_ref', False), 'claim_event_id': ('text', False),
|
|
'remote_modified_at': ('text', False), 'scan_remote_modified_at': ('text', False),
|
|
'covered_ref': ('text', False), 'covered_head': ('text', False),
|
|
}
|
|
|
|
DISCOVERY_RETRY_QUEUE_COLUMN_SPECS = {
|
|
'id': ('id', True),
|
|
'work_key': ('text', True),
|
|
'source': ('text', True),
|
|
'query': ('text', True),
|
|
'source_cycle_id': ('id_ref', False),
|
|
'policy_sha256': ('text', True),
|
|
'pass_kind': ('text', True),
|
|
'work_kind': ('text', True),
|
|
'page_start': ('integer', True),
|
|
'page_end': ('integer', True),
|
|
'next_page': ('integer', True),
|
|
'status': ('text', True),
|
|
'attempts': ('integer', True),
|
|
'available_after': ('text', False),
|
|
'lease_owner': ('text', False),
|
|
'lease_token': ('text', False),
|
|
'leased_at': ('text', False),
|
|
'lease_expires_at': ('text', False),
|
|
'last_error_category': ('text', False),
|
|
'held_at': ('text', False),
|
|
'created_at': ('text', True),
|
|
'updated_at': ('text', True),
|
|
}
|
|
|
|
TARGET_QUEUE_POLICY_EVENT_COLUMN_SPECS = {
|
|
'id': ('id', True), 'queue_id': ('id_ref', True),
|
|
'action': ('text', True), 'prior_status': ('text', True),
|
|
'next_status': ('text', True), 'source': ('text', True),
|
|
'platform': ('text', True), 'query': ('text', True),
|
|
'reason_code': ('text', True), 'config_sha256': ('text', True),
|
|
'policy_sha256': ('text', True), 'manifest_sha256': ('text', True),
|
|
'review_audit_sha256': ('text', True),
|
|
'reverses_event_id': ('id_ref', False),
|
|
'experiment_id': ('id_ref', False),
|
|
'prior_updated_at': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_key': ('text', True),
|
|
'source': ('text', True), 'state': ('text', True),
|
|
'collection_generation': ('text', True),
|
|
'config_sha256': ('text', True), 'ordered_queries_sha256': ('text', True),
|
|
'selector_version': ('text', True), 'selector_sha256': ('text', True),
|
|
'provenance_policy_sha256': ('text', True),
|
|
'query_count': ('integer', True), 'repositories_per_query': ('integer', True),
|
|
'images_per_repository': ('integer', True), 'target_limit': ('integer', True),
|
|
'target_count': ('integer', True), 'selection_count': ('integer', True),
|
|
'fence_generation': ('id_ref', True), 'fence_owner': ('text', False),
|
|
'fence_token': ('text', False), 'fence_expires_at': ('text', False),
|
|
'hold_reason_code': ('text', False), 'plan_sha256': ('text', False),
|
|
'selection_sha256': ('text', False),
|
|
'hold_manifest_sha256': ('text', False), 'created_at': ('text', True),
|
|
'updated_at': ('text', True), 'planned_at': ('text', False),
|
|
'activated_at': ('text', False), 'draining_at': ('text', False),
|
|
'completed_at': ('text', False), 'released_at': ('text', False),
|
|
'held_at': ('text', False),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_QUERY_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'source': ('text', True), 'query_ordinal': ('integer', True),
|
|
'query': ('text', True), 'query_sha256': ('text', True),
|
|
'required_repository_count': ('integer', True),
|
|
'selected_repository_count': ('integer', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DISCOVERY_PASS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', False),
|
|
'pass_token': ('text', True), 'source': ('text', True),
|
|
'pass_kind': ('text', True), 'policy_sha256': ('text', True),
|
|
'collection_generation': ('text', True),
|
|
'ordered_queries_sha256': ('text', True), 'expected_query_count': ('integer', True),
|
|
'completed_query_count': ('integer', True), 'state': ('text', True),
|
|
'started_at': ('text', True), 'completed_at': ('text', False),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DISCOVERY_PAGE_COLUMN_SPECS = {
|
|
'id': ('id', True), 'pass_id': ('id_ref', True),
|
|
'source_cycle_id': ('id_ref', False), 'retry_work_id': ('id_ref', False),
|
|
'query': ('text', True), 'query_ordinal': ('integer', True),
|
|
'page_number': ('integer', True), 'result_count': ('integer', True),
|
|
'total_count': ('id_ref', False),
|
|
'admitted_count': ('integer', True), 'query_complete': ('integer', True),
|
|
'admission_kind': ('text', True), 'page_sha256': ('text', True),
|
|
'observed_at': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_REPOSITORY_QUERY_PROVENANCE_COLUMN_SPECS = {
|
|
'source': ('text', True), 'query': ('text', True),
|
|
'repository_queue_id': ('id_ref', True), 'provenance_kind': ('text', True),
|
|
'first_observed_at': ('text', True), 'last_observed_at': ('text', True),
|
|
'first_search_rank': ('integer', False), 'best_search_rank': ('integer', False),
|
|
'last_search_rank': ('integer', False), 'first_cycle_id': ('id_ref', False),
|
|
'last_cycle_id': ('id_ref', False), 'first_page_id': ('id_ref', False),
|
|
'last_page_id': ('id_ref', False), 'first_policy_sha256': ('text', False),
|
|
'last_policy_sha256': ('text', False), 'observation_count': ('integer', True),
|
|
'fresh_observation_count': ('integer', True),
|
|
'fresh_complete_observation_count': ('integer', True),
|
|
'fresh_coverage_eligible': ('integer', True),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_REPOSITORY_QUERY_OBSERVATION_COLUMN_SPECS = {
|
|
'page_id': ('id_ref', True), 'repository_queue_id': ('id_ref', True),
|
|
'source': ('text', True), 'query': ('text', True),
|
|
'search_rank': ('integer', True), 'observed_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_IMAGE_MANIFEST_COLUMN_SPECS = {
|
|
'id': ('id', True), 'target_queue_id': ('id_ref', True),
|
|
'source': ('text', True), 'repository': ('text', True),
|
|
'manifest_digest': ('text', True), 'manifest_media_type': ('text', True),
|
|
'config_digest': ('text', False), 'graph_sha256': ('text', True),
|
|
'manifest_size_bytes': ('id_ref', False), 'layer_count': ('integer', True),
|
|
'resolved_at': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_MANIFEST_LAYER_COLUMN_SPECS = {
|
|
'id': ('id', True), 'manifest_id': ('id_ref', True),
|
|
'position_from_base': ('integer', True), 'position_from_top': ('integer', True),
|
|
'layer_digest': ('text', True), 'media_type': ('text', True),
|
|
'layer_size_bytes': ('id_ref', True), 'descriptor_sha256': ('text', True),
|
|
'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_REPOSITORY_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'query_ordinal': ('integer', True), 'source': ('text', True),
|
|
'query': ('text', True), 'repository_queue_id': ('id_ref', True),
|
|
'eligibility_page_id': ('id_ref', True), 'repository_rank': ('integer', True),
|
|
'planned_is_deep_probe': ('integer', True),
|
|
'is_deep_probe': ('integer', True), 'work_state': ('text', True),
|
|
'resolver_generation': ('id_ref', True), 'resolver_owner': ('text', False),
|
|
'resolver_token': ('text', False), 'resolver_expires_at': ('text', False),
|
|
'resolver_attempts': ('integer', True), 'resolver_due_at': ('text', False),
|
|
'candidate_distinct_graph_count': ('integer', True),
|
|
'selected_image_count': ('integer', True),
|
|
'replacement_repository_queue_id': ('id_ref', False),
|
|
'replacement_eligibility_page_id': ('id_ref', False),
|
|
'replacement_count': ('integer', True),
|
|
'replacement_evidence_sha256': ('text', False),
|
|
'last_error_code': ('text', False), 'created_at': ('text', True),
|
|
'updated_at': ('text', True), 'resolved_at': ('text', False),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_TARGET_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'target_queue_id': ('id_ref', True), 'manifest_id': ('id_ref', True),
|
|
'counter_ordinal': ('integer', True), 'state': ('text', True),
|
|
'dispatch_wave': ('integer', True), 'dispatch_order': ('id_ref', True),
|
|
'reservation_count': ('integer', True), 'created_at': ('text', True),
|
|
'updated_at': ('text', True), 'terminal_at': ('text', False),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_SELECTION_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'query_ordinal': ('integer', True), 'experiment_repository_id': ('id_ref', True),
|
|
'experiment_target_id': ('id_ref', True), 'image_rank': ('integer', True),
|
|
'selection_reason': ('text', True), 'selection_evidence_sha256': ('text', True),
|
|
'graph_sha256': ('text', True), 'selected_at': ('text', True),
|
|
'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_SKIP_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'experiment_repository_id': ('id_ref', True),
|
|
'repository_queue_id': ('id_ref', True), 'candidate_kind': ('text', True),
|
|
'candidate_ordinal': ('integer', True), 'reason_code': ('text', True),
|
|
'candidate_identity_sha256': ('text', True),
|
|
'evidence_sha256': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_RESOLVER_REFUND_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'experiment_repository_id': ('id_ref', True),
|
|
'repository_queue_id': ('id_ref', True),
|
|
'query_ordinal': ('integer', True), 'repository_rank': ('integer', True),
|
|
'recovery_kind': ('text', True), 'manifest_sha256': ('text', True),
|
|
'entry_evidence_sha256': ('text', True), 'log_sha256': ('text', True),
|
|
'target_identity_sha256': ('text', True),
|
|
'prior_error_code_sha256': ('text', True),
|
|
'prior_work_state': ('text', True), 'next_work_state': ('text', True),
|
|
'prior_resolver_attempts': ('integer', True),
|
|
'refund_attempts': ('integer', True),
|
|
'next_resolver_attempts': ('integer', True),
|
|
'confirmed_bug_event_count': ('integer', True),
|
|
'applied_at': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_RESOLVER_DISPOSITION_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_id': ('id_ref', True),
|
|
'experiment_repository_id': ('id_ref', True),
|
|
'prior_repository_queue_id': ('id_ref', True),
|
|
'replacement_repository_queue_id': ('id_ref', False),
|
|
'replacement_eligibility_page_id': ('id_ref', False),
|
|
'replacement_best_search_rank': ('integer', False),
|
|
'query_ordinal': ('integer', True), 'repository_rank': ('integer', True),
|
|
'disposition_kind': ('text', True), 'outcome': ('text', True),
|
|
'manifest_sha256': ('text', True), 'entry_evidence_sha256': ('text', True),
|
|
'candidate_snapshot_sha256': ('text', True),
|
|
'prior_target_identity_sha256': ('text', True),
|
|
'replacement_target_identity_sha256': ('text', False),
|
|
'prior_error_code_sha256': ('text', True),
|
|
'prior_work_state': ('text', True), 'next_work_state': ('text', True),
|
|
'prior_resolver_attempts': ('integer', True),
|
|
'next_resolver_attempts': ('integer', True),
|
|
'applied_at': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_SCAN_BINDING_COLUMN_SPECS = {
|
|
'id': ('id', True), 'experiment_target_id': ('id_ref', True),
|
|
'reservation_id': ('id_ref', True), 'target_scan_id': ('id_ref', False),
|
|
'attempt': ('integer', True), 'state': ('text', True),
|
|
'bound_at': ('text', True), 'scan_bound_at': ('text', False),
|
|
'completed_at': ('text', False), 'created_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_FINDING_LAYER_ATTRIBUTION_COLUMN_SPECS = {
|
|
'id': ('id', True), 'scan_binding_id': ('id_ref', True),
|
|
'finding_id': ('id_ref', True), 'manifest_layer_id': ('id_ref', False),
|
|
'attribution_state': ('text', True), 'reported_layer_digest': ('text', False),
|
|
'unattributed_reason': ('text', False),
|
|
'position_from_base': ('integer', False), 'position_from_top': ('integer', False),
|
|
'created_at': ('text', True),
|
|
}
|
|
|
|
TARGET_SCANS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'scan_event_id': ('text', False), 'scan_event_hash': ('text', False),
|
|
'queue_id': ('id_ref', False), 'claim_lease_token': ('text', False),
|
|
'queue_completion_applied': ('integer', True), 'queue_completion_disposition': ('text', False),
|
|
'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False), 'source': ('text', False),
|
|
'query': ('text', False), 'target': ('text', False), 'normalized_target': ('text', False),
|
|
'scan_type': ('text', False), 'status': ('text', False), 'started_at': ('text', False),
|
|
'ended_at': ('text', False), 'duration_sec': ('real', False), 'scan_options_json': ('text', False),
|
|
'package_name': ('text', False), 'package_version': ('text', False),
|
|
'package_artifact': ('text', False), 'package_date': ('text', False),
|
|
'package_filename': ('text', False), 'package_type': ('text', False),
|
|
'package_size': ('integer', False), 'findings_count': ('integer', False),
|
|
'verified_findings_count': ('integer', False), 'error_count': ('integer', False),
|
|
'skipped_reason': ('text', False), 'first_error_summary': ('text', False),
|
|
'raw_result_json': ('text', False), 'created_at': ('text', True),
|
|
'result_reservation_id': ('id_ref', False),
|
|
'compat_schema_version': ('integer', True),
|
|
'raw_result_storage': ('text', True),
|
|
}
|
|
|
|
DOCKER_CONTENT_BLOB_COLUMN_SPECS = {
|
|
'digest': ('text', True), 'coverage_policy_sha256': ('text', True),
|
|
'descriptor_kind': ('text', True),
|
|
'declared_bytes': ('id_ref', True), 'media_type': ('text', True),
|
|
'state': ('text', True), 'attempts': ('integer', True),
|
|
'max_attempts': ('integer', True), 'available_after': ('text', False),
|
|
'lease_reservation_id': ('id_ref', False), 'lease_token': ('text', False),
|
|
'lease_plan_sha256': ('text', False), 'lease_expires_at': ('text', False),
|
|
'covered_reservation_id': ('id_ref', False),
|
|
'covered_scan_event_id': ('text', False), 'covered_policy_sha256': ('text', False),
|
|
'verified_bytes': ('id_ref', False), 'covered_at': ('text', False),
|
|
'last_error_code': ('text', False), 'last_error_detail': ('text', False),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_IMAGE_BLOB_COVERAGE_COLUMN_SPECS = {
|
|
'queue_id': ('id_ref', True), 'manifest_digest': ('text', True),
|
|
'reservation_id': ('id_ref', True), 'position': ('integer', True),
|
|
'blob_digest': ('text', True),
|
|
'coverage_policy_sha256': ('text', True),
|
|
'selection_policy_sha256': ('text', True),
|
|
'descriptor_kind': ('text', True), 'plan_sha256': ('text', True),
|
|
'selected': ('integer', True),
|
|
'selection_reason': ('text', True), 'coverage_state': ('text', True),
|
|
'covered_at': ('text', False), 'last_error_code': ('text', False),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
DOCKER_ADAPTIVE_SHADOW_REPORT_COLUMN_SPECS = {
|
|
'id': ('id', True), 'report_token': ('text', True),
|
|
'evaluator_version': ('text', True), 'state': ('text', True),
|
|
'scan_policy_sha256': ('text', True), 'execution_policy_sha256': ('text', True),
|
|
'selection_policy_sha256': ('text', True), 'cohort_size': ('integer', True),
|
|
'completed_pairs': ('integer', True), 'full_routed_count': ('id_ref', True),
|
|
'adaptive_routed_count': ('id_ref', True),
|
|
'routed_intersection_count': ('id_ref', True),
|
|
'full_detector_count': ('id_ref', True), 'adaptive_detector_count': ('id_ref', True),
|
|
'detector_intersection_count': ('id_ref', True),
|
|
'full_slot_ms': ('id_ref', True), 'adaptive_slot_ms': ('id_ref', True),
|
|
'omitted_descriptor_count': ('id_ref', True), 'failure_count': ('integer', True),
|
|
'privacy_violation_count': ('integer', True),
|
|
'safety_regression_count': ('integer', True),
|
|
'selection_metrics_json': ('text', True),
|
|
'sink_checkpoint_count': ('integer', True),
|
|
'routed_recall_ppm': ('integer', True), 'slot_ratio_ppm': ('integer', True),
|
|
'recall_threshold_ppm': ('integer', True), 'slot_threshold_ppm': ('integer', True),
|
|
'passed': ('integer', True), 'lease_owner': ('text', True),
|
|
'lease_token': ('text', True), 'lease_expires_at': ('text', True),
|
|
'started_at': ('text', True), 'completed_at': ('text', False),
|
|
'created_at': ('text', True), 'updated_at': ('text', True),
|
|
}
|
|
|
|
FINDINGS_COLUMN_SPECS = {
|
|
'id': ('id', True), 'run_id': ('id_ref', False), 'cycle_id': ('id_ref', False),
|
|
'target_scan_id': ('id_ref', False), 'source': ('text', False), 'query': ('text', False),
|
|
'target': ('text', False), 'normalized_target': ('text', False),
|
|
'detector_name': ('text', False), 'detector_type': ('text', False),
|
|
'verified': ('integer', False), 'raw_secret': ('text', False),
|
|
'redacted_secret': ('text', False), 'secret_hash': ('text', False),
|
|
'detector_secret_hash': ('text', False), 'finding_fingerprint': ('text', False),
|
|
'finding_uid': ('text', False), 'file_path': ('text', False), 'line_number': ('text', False),
|
|
'commit_hash': ('text', False), 'source_timestamp': ('text', False),
|
|
'source_metadata_type': ('text', False), 'source_metadata_json': ('text', False),
|
|
'raw_finding_json': ('text', False), 'provider': ('text', False),
|
|
'credential_kind': ('text', False), 'credential_confidence': ('text', False),
|
|
'required_context_missing': ('integer', False), 'principal': ('text', False),
|
|
'username': ('text', False), 'email': ('text', False), 'project_id': ('text', False),
|
|
'tenant_id': ('text', False), 'organization': ('text', False), 'registry': ('text', False),
|
|
'endpoint': ('text', False), 'scope': ('text', False), 'resource': ('text', False),
|
|
'enrichment_json': ('text', False), 'created_at': ('text', True),
|
|
'raw_payload_sha256': ('text', False), 'raw_payload_bytes': ('id_ref', False),
|
|
'raw_payload_omitted': ('integer', True),
|
|
}
|
|
|
|
KEYCHECK_EVENT_MAP_COLUMN_SPECS = {
|
|
'event_id': ('text', True), 'keycheck_result_id': ('id_ref', False), 'created_at': ('text', True),
|
|
}
|
|
|
|
FINDING_UID_MAP_COLUMN_SPECS = {
|
|
'finding_uid': ('text', True), 'finding_id': ('id_ref', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
RUNTIME_OPERATION_COLUMN_SPECS = {
|
|
'operation_id': ('text', True), 'actor': ('text', True),
|
|
'action': ('text', True), 'target_kind': ('text', True),
|
|
'target_ref': ('text', True), 'status': ('text', True),
|
|
'safe_category': ('text', False), 'safe_detail': ('text', False),
|
|
'expected_revision': ('id_ref', False), 'resulting_revision': ('id_ref', False),
|
|
'expected_identity_json': ('text', True), 'resulting_identity_json': ('text', False),
|
|
'agent_state': ('text', True), 'agent_result_sha256': ('text', False),
|
|
'requested_at': ('text', True), 'started_at': ('text', False),
|
|
'completed_at': ('text', False), 'agent_reconciled_at': ('text', False),
|
|
'updated_at': ('text', True),
|
|
}
|
|
|
|
RUNTIME_OPERATIONS_CONTROL_COLUMN_SPECS = {
|
|
'id': ('integer', True), 'revision': ('id_ref', True),
|
|
'discovery_paused': ('integer', True), 'dispatch_paused': ('integer', True),
|
|
'drain_state': ('text', True), 'actor': ('text', True),
|
|
'operation_id': ('text', False), 'created_at': ('text', True),
|
|
'updated_at': ('text', True),
|
|
}
|
|
|
|
RUNTIME_AUDIT_EVENT_COLUMN_SPECS = {
|
|
'id': ('id', True), 'operation_id': ('text', False),
|
|
'actor': ('text', True), 'action': ('text', True),
|
|
'target_kind': ('text', True), 'target_ref': ('text', True),
|
|
'result': ('text', True), 'safe_category': ('text', False),
|
|
'before_identity_json': ('text', False), 'after_identity_json': ('text', False),
|
|
'before_bytes': ('id_ref', False), 'after_bytes': ('id_ref', False),
|
|
'previous_event_id': ('id_ref', False), 'previous_event_sha256': ('text', False),
|
|
'event_sha256': ('text', True), 'created_at': ('text', True),
|
|
}
|
|
|
|
WORKER_PROGRESS_EVENT_COLUMN_SPECS = {
|
|
'id': ('id', True), 'reservation_id': ('id_ref', True),
|
|
'remote_device_id': ('id_ref', True), 'schema_version': ('integer', True),
|
|
'sequence': ('integer', True), 'event_type': ('text', True),
|
|
'phase': ('text', True), 'event_timestamp': ('text', True),
|
|
'phase_started_at': ('text', False), 'instance_id': ('text', True),
|
|
'slot_id': ('integer', True), 'source': ('text', True),
|
|
'event_json': ('text', True), 'event_sha256': ('text', True),
|
|
'received_at': ('text', True),
|
|
}
|
|
|
|
WORKER_DIAGNOSTIC_COLUMN_SPECS = {
|
|
'id': ('id', True), 'diagnostic_uid': ('text', True),
|
|
'reservation_id': ('id_ref', True), 'target_scan_id': ('id_ref', False),
|
|
'schema_version': ('integer', True), 'scan_event_id': ('text', False),
|
|
'slot_id': ('integer', True), 'attempt': ('integer', True),
|
|
'source': ('text', True), 'phase': ('text', True), 'kind': ('text', True),
|
|
'category': ('text', True), 'code': ('text', True), 'summary': ('text', True),
|
|
'retryable': ('integer', True), 'occurred_at': ('text', True),
|
|
'captured_at': ('text', True), 'envelope_json': ('text', True),
|
|
'envelope_sha256': ('text', True), 'body_payload_json': ('text', False),
|
|
'log_payload_json': ('text', False), 'received_at': ('text', True),
|
|
}
|
|
|
|
RUNTIME_TABLE_SPECS = {
|
|
'runtime_operations': RUNTIME_OPERATION_COLUMN_SPECS,
|
|
'runtime_operations_control': RUNTIME_OPERATIONS_CONTROL_COLUMN_SPECS,
|
|
'runtime_audit_events': RUNTIME_AUDIT_EVENT_COLUMN_SPECS,
|
|
'worker_progress_events': WORKER_PROGRESS_EVENT_COLUMN_SPECS,
|
|
'worker_diagnostics': WORKER_DIAGNOSTIC_COLUMN_SPECS,
|
|
'runs': RUNS_COLUMN_SPECS,
|
|
'source_cycles': SOURCE_CYCLES_COLUMN_SPECS,
|
|
'target_queue': TARGET_QUEUE_COLUMN_SPECS,
|
|
'discovery_retry_queue': DISCOVERY_RETRY_QUEUE_COLUMN_SPECS,
|
|
'target_queue_policy_events': TARGET_QUEUE_POLICY_EVENT_COLUMN_SPECS,
|
|
'target_scans': TARGET_SCANS_COLUMN_SPECS,
|
|
'findings': FINDINGS_COLUMN_SPECS,
|
|
'errors': ERRORS_COLUMN_SPECS,
|
|
'queue_snapshots': QUEUE_SNAPSHOTS_COLUMN_SPECS,
|
|
'config_snapshots': CONFIG_SNAPSHOTS_COLUMN_SPECS,
|
|
'package_repo_candidates': PACKAGE_REPO_CANDIDATES_COLUMN_SPECS,
|
|
'scan_publication_outbox': OUTBOX_COLUMN_SPECS,
|
|
'keycheck_results': KEYCHECK_COLUMN_SPECS,
|
|
'keycheck_event_map': KEYCHECK_EVENT_MAP_COLUMN_SPECS,
|
|
'finding_uid_map': FINDING_UID_MAP_COLUMN_SPECS,
|
|
'target_queue_reconciliation_cursors': CURSOR_COLUMN_SPECS,
|
|
'target_queue_reconciliation_issues': RECONCILIATION_ISSUE_COLUMN_SPECS,
|
|
'docker_content_blobs': DOCKER_CONTENT_BLOB_COLUMN_SPECS,
|
|
'docker_image_blob_coverage': DOCKER_IMAGE_BLOB_COVERAGE_COLUMN_SPECS,
|
|
'docker_adaptive_shadow_reports': DOCKER_ADAPTIVE_SHADOW_REPORT_COLUMN_SPECS,
|
|
'docker_depth_experiments': DOCKER_DEPTH_EXPERIMENT_COLUMN_SPECS,
|
|
'docker_depth_experiment_queries': DOCKER_DEPTH_EXPERIMENT_QUERY_COLUMN_SPECS,
|
|
'docker_discovery_passes': DOCKER_DISCOVERY_PASS_COLUMN_SPECS,
|
|
'docker_discovery_pages': DOCKER_DISCOVERY_PAGE_COLUMN_SPECS,
|
|
'docker_repository_query_provenance': DOCKER_REPOSITORY_QUERY_PROVENANCE_COLUMN_SPECS,
|
|
'docker_repository_query_observations': DOCKER_REPOSITORY_QUERY_OBSERVATION_COLUMN_SPECS,
|
|
'docker_image_manifests': DOCKER_IMAGE_MANIFEST_COLUMN_SPECS,
|
|
'docker_manifest_layers': DOCKER_MANIFEST_LAYER_COLUMN_SPECS,
|
|
'docker_depth_experiment_repositories': DOCKER_DEPTH_EXPERIMENT_REPOSITORY_COLUMN_SPECS,
|
|
'docker_depth_experiment_targets': DOCKER_DEPTH_EXPERIMENT_TARGET_COLUMN_SPECS,
|
|
'docker_depth_experiment_selections': DOCKER_DEPTH_EXPERIMENT_SELECTION_COLUMN_SPECS,
|
|
'docker_depth_experiment_candidate_skips': DOCKER_DEPTH_EXPERIMENT_SKIP_COLUMN_SPECS,
|
|
'docker_depth_resolver_attempt_refunds': DOCKER_DEPTH_RESOLVER_REFUND_COLUMN_SPECS,
|
|
'docker_depth_resolver_dispositions': DOCKER_DEPTH_RESOLVER_DISPOSITION_COLUMN_SPECS,
|
|
'docker_depth_experiment_scan_bindings': DOCKER_DEPTH_EXPERIMENT_SCAN_BINDING_COLUMN_SPECS,
|
|
'docker_finding_layer_attributions': DOCKER_FINDING_LAYER_ATTRIBUTION_COLUMN_SPECS,
|
|
}
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_TABLES = (
|
|
'docker_depth_experiments',
|
|
'docker_depth_experiment_queries',
|
|
'docker_discovery_passes',
|
|
'docker_discovery_pages',
|
|
'docker_repository_query_provenance',
|
|
'docker_repository_query_observations',
|
|
'docker_image_manifests',
|
|
'docker_manifest_layers',
|
|
'docker_depth_experiment_repositories',
|
|
'docker_depth_experiment_targets',
|
|
'docker_depth_experiment_selections',
|
|
'docker_depth_experiment_candidate_skips',
|
|
'docker_depth_resolver_attempt_refunds',
|
|
'docker_depth_resolver_dispositions',
|
|
'docker_depth_experiment_scan_bindings',
|
|
'docker_finding_layer_attributions',
|
|
)
|
|
PIPELINE_REQUIRED_COLUMNS.update({
|
|
table: set(RUNTIME_TABLE_SPECS[table]) for table in DOCKER_DEPTH_EXPERIMENT_TABLES
|
|
})
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS = (
|
|
('docker_depth_experiments', 'uq_docker_depth_experiments_key',
|
|
('experiment_key',), True, ''),
|
|
('docker_depth_experiments', 'idx_docker_depth_experiments_state',
|
|
('source', 'state', 'id'), False, ''),
|
|
('docker_depth_experiment_queries', 'uq_docker_depth_experiment_queries_ordinal',
|
|
('experiment_id', 'query_ordinal'), True, ''),
|
|
('docker_depth_experiment_queries', 'uq_docker_depth_experiment_queries_query',
|
|
('experiment_id', 'source', 'query'), True, ''),
|
|
('docker_discovery_passes', 'uq_docker_discovery_passes_token',
|
|
('pass_token',), True, ''),
|
|
('docker_discovery_passes', 'idx_docker_discovery_passes_policy',
|
|
('source', 'policy_sha256', 'state', 'id'), False, ''),
|
|
('docker_discovery_passes', 'idx_docker_discovery_passes_experiment',
|
|
('experiment_id', 'state', 'id'), False, ''),
|
|
('docker_discovery_pages', 'uq_docker_discovery_pages_position',
|
|
('pass_id', 'query_ordinal', 'page_number'), True, ''),
|
|
('docker_discovery_pages', 'idx_docker_discovery_pages_query',
|
|
('pass_id', 'query', 'query_complete', 'page_number'), False, ''),
|
|
('docker_discovery_pages', 'idx_docker_discovery_pages_retry',
|
|
('retry_work_id', 'id'), False, 'retry_work_id IS NOT NULL'),
|
|
('docker_repository_query_provenance', 'idx_docker_repository_provenance_queue',
|
|
('repository_queue_id', 'source', 'query'), False, ''),
|
|
('docker_repository_query_provenance', 'idx_docker_repository_provenance_eligible',
|
|
('source', 'query', 'fresh_coverage_eligible', 'best_search_rank', 'repository_queue_id'),
|
|
False, ''),
|
|
('docker_repository_query_observations', 'idx_docker_repository_observations_query',
|
|
('source', 'query', 'repository_queue_id', 'page_id'), False, ''),
|
|
('docker_image_manifests', 'uq_docker_image_manifests_queue',
|
|
('target_queue_id',), True, ''),
|
|
('docker_image_manifests', 'idx_docker_image_manifests_digest',
|
|
('manifest_digest', 'id'), False, ''),
|
|
('docker_manifest_layers', 'uq_docker_manifest_layers_base_position',
|
|
('manifest_id', 'position_from_base'), True, ''),
|
|
('docker_manifest_layers', 'uq_docker_manifest_layers_top_position',
|
|
('manifest_id', 'position_from_top'), True, ''),
|
|
('docker_manifest_layers', 'idx_docker_manifest_layers_digest',
|
|
('layer_digest', 'manifest_id', 'position_from_base'), False, ''),
|
|
('docker_depth_experiment_repositories', 'uq_docker_experiment_repositories_member',
|
|
('experiment_id', 'query_ordinal', 'repository_queue_id'), True, ''),
|
|
('docker_depth_experiment_repositories', 'uq_docker_experiment_repositories_rank',
|
|
('experiment_id', 'query_ordinal', 'repository_rank'), True, ''),
|
|
('docker_depth_experiment_repositories', 'idx_docker_experiment_repository_work',
|
|
('experiment_id', 'work_state', 'repository_rank', 'query_ordinal', 'id'), False, ''),
|
|
('docker_depth_experiment_repositories', 'idx_docker_experiment_repository_due',
|
|
('experiment_id', 'work_state', 'resolver_due_at', 'repository_rank',
|
|
'query_ordinal', 'id'), False, ''),
|
|
('docker_depth_experiment_targets', 'uq_docker_experiment_targets_queue',
|
|
('experiment_id', 'target_queue_id'), True, ''),
|
|
('docker_depth_experiment_targets', 'uq_docker_experiment_targets_counter',
|
|
('experiment_id', 'counter_ordinal'), True, ''),
|
|
('docker_depth_experiment_targets', 'idx_docker_experiment_targets_dispatch',
|
|
('experiment_id', 'state', 'dispatch_wave', 'dispatch_order', 'id'), False, ''),
|
|
('docker_depth_experiment_selections', 'uq_docker_experiment_selections_rank',
|
|
('experiment_repository_id', 'image_rank'), True, ''),
|
|
('docker_depth_experiment_selections', 'idx_docker_experiment_selections_query',
|
|
('experiment_id', 'query_ordinal', 'image_rank', 'experiment_repository_id', 'id'),
|
|
False, ''),
|
|
('docker_depth_experiment_selections', 'idx_docker_experiment_selections_target',
|
|
('experiment_target_id', 'id'), False, ''),
|
|
('docker_depth_experiment_candidate_skips', 'uq_docker_experiment_candidate_skip',
|
|
('experiment_repository_id', 'candidate_kind', 'candidate_identity_sha256'),
|
|
True, ''),
|
|
('docker_depth_experiment_candidate_skips', 'idx_docker_experiment_candidate_skips',
|
|
('experiment_id', 'experiment_repository_id', 'candidate_kind', 'id'),
|
|
False, ''),
|
|
('docker_depth_resolver_attempt_refunds', 'uq_docker_depth_resolver_attempt_refund',
|
|
('experiment_repository_id', 'recovery_kind'), True, ''),
|
|
('docker_depth_resolver_attempt_refunds', 'idx_docker_depth_resolver_attempt_refunds',
|
|
('experiment_id', 'id'), False, ''),
|
|
('docker_depth_resolver_dispositions', 'uq_docker_depth_resolver_disposition',
|
|
('experiment_repository_id', 'disposition_kind'), True, ''),
|
|
('docker_depth_resolver_dispositions', 'idx_docker_depth_resolver_dispositions',
|
|
('experiment_id', 'id'), False, ''),
|
|
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_reservation',
|
|
('reservation_id',), True, ''),
|
|
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_attempt',
|
|
('experiment_target_id', 'attempt'), True, ''),
|
|
('docker_depth_experiment_scan_bindings', 'uq_docker_experiment_bindings_scan',
|
|
('target_scan_id',), True, 'target_scan_id IS NOT NULL'),
|
|
('docker_depth_experiment_scan_bindings', 'idx_docker_experiment_bindings_target',
|
|
('experiment_target_id', 'state', 'id'), False, ''),
|
|
('docker_finding_layer_attributions', 'uq_docker_finding_layer_exact',
|
|
('finding_id', 'manifest_layer_id'), True, "attribution_state = 'exact'"),
|
|
('docker_finding_layer_attributions', 'uq_docker_finding_layer_unattributed',
|
|
('finding_id',), True, "attribution_state = 'unattributed'"),
|
|
('docker_finding_layer_attributions', 'idx_docker_finding_layer_binding',
|
|
('scan_binding_id', 'finding_id', 'id'), False, ''),
|
|
('target_queue_policy_events', 'idx_target_queue_policy_events_experiment',
|
|
('experiment_id', 'id'), False, 'experiment_id IS NOT NULL'),
|
|
)
|
|
|
|
RUNTIME_PRIMARY_KEYS = {
|
|
'runtime_operations': ['operation_id'],
|
|
'runtime_operations_control': ['id'],
|
|
'runtime_audit_events': ['id'],
|
|
'worker_progress_events': ['id'], 'worker_diagnostics': ['id'],
|
|
'runs': ['id'], 'source_cycles': ['id'], 'errors': ['id'],
|
|
'queue_snapshots': ['id'], 'config_snapshots': ['id'],
|
|
'package_repo_candidates': ['id'],
|
|
'target_queue': ['id'], 'target_scans': ['id'], 'findings': ['id'],
|
|
'discovery_retry_queue': ['id'],
|
|
'target_queue_policy_events': ['id'],
|
|
'scan_publication_outbox': ['id'], 'keycheck_results': ['id'],
|
|
'keycheck_event_map': ['event_id'], 'finding_uid_map': ['finding_uid'],
|
|
'target_queue_reconciliation_cursors': ['source_file'],
|
|
'target_queue_reconciliation_issues': ['id'],
|
|
'docker_content_blobs': ['digest', 'coverage_policy_sha256'],
|
|
'docker_image_blob_coverage': ['reservation_id', 'position'],
|
|
'docker_adaptive_shadow_reports': ['id'],
|
|
'docker_depth_experiments': ['id'],
|
|
'docker_depth_experiment_queries': ['id'],
|
|
'docker_discovery_passes': ['id'],
|
|
'docker_discovery_pages': ['id'],
|
|
'docker_repository_query_provenance': ['source', 'query', 'repository_queue_id'],
|
|
'docker_repository_query_observations': ['page_id', 'repository_queue_id'],
|
|
'docker_image_manifests': ['id'],
|
|
'docker_manifest_layers': ['id'],
|
|
'docker_depth_experiment_repositories': ['id'],
|
|
'docker_depth_experiment_targets': ['id'],
|
|
'docker_depth_experiment_selections': ['id'],
|
|
'docker_depth_experiment_candidate_skips': ['id'],
|
|
'docker_depth_resolver_attempt_refunds': ['id'],
|
|
'docker_depth_resolver_dispositions': ['id'],
|
|
'docker_depth_experiment_scan_bindings': ['id'],
|
|
'docker_finding_layer_attributions': ['id'],
|
|
}
|
|
|
|
RUNTIME_COLUMN_DEFAULTS = {
|
|
'runtime_operations': {
|
|
'target_ref': '', 'status': 'requested',
|
|
'expected_identity_json': '{}', 'agent_state': 'not_required',
|
|
},
|
|
'runtime_operations_control': {
|
|
'revision': '0', 'discovery_paused': '0', 'dispatch_paused': '0',
|
|
'drain_state': 'normal',
|
|
},
|
|
'runtime_audit_events': {'target_ref': ''},
|
|
'runs': {
|
|
'total_fetched': '0', 'total_queued_new': '0', 'total_scan_requested': '0',
|
|
'total_scanned': '0', 'total_clean': '0', 'total_found': '0',
|
|
'total_skipped': '0', 'total_errors': '0', 'total_findings': '0',
|
|
'total_verified_findings': '0', 'total_unique_secrets': '0',
|
|
'total_unique_findings': '0', 'total_staged': '0', 'total_quarantined': '0',
|
|
},
|
|
'source_cycles': {
|
|
'fetched_count': '0', 'queued_new_count': '0', 'queued_updated_count': '0',
|
|
'scan_requested_count': '0',
|
|
'scanned_count': '0', 'clean_count': '0', 'found_count': '0',
|
|
'skipped_count': '0', 'error_count': '0', 'findings_count': '0',
|
|
'verified_findings_count': '0', 'unique_secrets_count': '0',
|
|
'unique_findings_count': '0', 'targets_per_hour': '0', 'hit_rate': '0',
|
|
'verified_hit_rate': '0', 'error_rate': '0', 'staged_count': '0',
|
|
'ingested_count': '0', 'quarantined_count': '0',
|
|
},
|
|
'target_queue': {'status': 'pending', 'attempts': '0', 'resolver_attempts': '0'},
|
|
'discovery_retry_queue': {
|
|
'page_start': '1', 'page_end': '1', 'next_page': '1',
|
|
'status': 'pending', 'attempts': '0',
|
|
},
|
|
'target_scans': {
|
|
'queue_completion_applied': '0', 'findings_count': '0',
|
|
'verified_findings_count': '0', 'error_count': '0',
|
|
'compat_schema_version': '2', 'raw_result_storage': 'legacy',
|
|
},
|
|
'findings': {'verified': '0', 'required_context_missing': '0', 'raw_payload_omitted': '0'},
|
|
'scan_publication_outbox': {'status': 'pending', 'attempts': '0'},
|
|
'keycheck_results': {
|
|
'link_status': 'pending', 'link_attempts': '0', 'result_source': 'api_check',
|
|
},
|
|
'target_queue_reconciliation_cursors': {
|
|
'file_size': '0', 'file_mtime_ns': '0', 'byte_offset': '0', 'line_number': '0',
|
|
'discarding_oversized': '0', 'cumulative_rows': '0', 'cumulative_bytes': '0',
|
|
'cumulative_inserted': '0', 'cumulative_rejected': '0',
|
|
},
|
|
'docker_content_blobs': {'state': 'pending', 'attempts': '0'},
|
|
'docker_image_blob_coverage': {'selection_policy_sha256': ''},
|
|
'docker_adaptive_shadow_reports': {
|
|
'state': 'running', 'completed_pairs': '0', 'full_routed_count': '0',
|
|
'adaptive_routed_count': '0', 'routed_intersection_count': '0',
|
|
'full_detector_count': '0', 'adaptive_detector_count': '0',
|
|
'detector_intersection_count': '0', 'full_slot_ms': '0',
|
|
'adaptive_slot_ms': '0', 'omitted_descriptor_count': '0',
|
|
'failure_count': '0', 'privacy_violation_count': '0',
|
|
'safety_regression_count': '0', 'selection_metrics_json': '{}',
|
|
'sink_checkpoint_count': '0', 'routed_recall_ppm': '0',
|
|
'slot_ratio_ppm': '0',
|
|
'recall_threshold_ppm': '850000',
|
|
'slot_threshold_ppm': '400000',
|
|
'passed': '0',
|
|
},
|
|
'docker_depth_experiments': {
|
|
'state': 'collecting', 'target_count': '0', 'selection_count': '0',
|
|
'fence_generation': '0',
|
|
'collection_generation': 'docker-depth-provenance-v1',
|
|
},
|
|
'docker_depth_experiment_queries': {
|
|
'required_repository_count': '10', 'selected_repository_count': '0',
|
|
},
|
|
'docker_discovery_passes': {
|
|
'completed_query_count': '0', 'state': 'collecting',
|
|
'collection_generation': 'docker-depth-provenance-v1',
|
|
},
|
|
'docker_discovery_pages': {
|
|
'result_count': '0', 'admitted_count': '0', 'query_complete': '0',
|
|
'admission_kind': 'main',
|
|
},
|
|
'docker_repository_query_provenance': {
|
|
'observation_count': '1', 'fresh_observation_count': '0',
|
|
'fresh_complete_observation_count': '0', 'fresh_coverage_eligible': '0',
|
|
},
|
|
'docker_depth_experiment_repositories': {
|
|
'planned_is_deep_probe': '0', 'is_deep_probe': '0',
|
|
'work_state': 'pending', 'resolver_generation': '0',
|
|
'resolver_attempts': '0', 'candidate_distinct_graph_count': '0',
|
|
'selected_image_count': '0', 'replacement_count': '0',
|
|
},
|
|
'docker_depth_experiment_targets': {'state': 'pending', 'reservation_count': '0'},
|
|
'docker_depth_experiment_scan_bindings': {'state': 'reserved'},
|
|
}
|
|
|
|
GENERATED_ID_COLUMNS = {
|
|
'runtime_audit_events': 'id',
|
|
'worker_progress_events': 'id', 'worker_diagnostics': 'id',
|
|
'runs': 'id', 'source_cycles': 'id', 'target_queue': 'id',
|
|
'discovery_retry_queue': 'id',
|
|
'target_scans': 'id', 'findings': 'id', 'errors': 'id',
|
|
'queue_snapshots': 'id', 'config_snapshots': 'id',
|
|
'package_repo_candidates': 'id',
|
|
'scan_publication_outbox': 'id', 'keycheck_results': 'id',
|
|
'target_queue_reconciliation_issues': 'id',
|
|
'docker_adaptive_shadow_reports': 'id',
|
|
'target_queue_policy_events': 'id',
|
|
'docker_depth_experiments': 'id',
|
|
'docker_depth_experiment_queries': 'id',
|
|
'docker_discovery_passes': 'id',
|
|
'docker_discovery_pages': 'id',
|
|
'docker_image_manifests': 'id',
|
|
'docker_manifest_layers': 'id',
|
|
'docker_depth_experiment_repositories': 'id',
|
|
'docker_depth_experiment_targets': 'id',
|
|
'docker_depth_experiment_selections': 'id',
|
|
'docker_depth_experiment_candidate_skips': 'id',
|
|
'docker_depth_resolver_attempt_refunds': 'id',
|
|
'docker_depth_resolver_dispositions': 'id',
|
|
'docker_depth_experiment_scan_bindings': 'id',
|
|
'docker_finding_layer_attributions': 'id',
|
|
}
|
|
|
|
REQUIRED_FOREIGN_KEYS = {
|
|
'runtime_operations_control': [
|
|
(('operation_id',), 'runtime_operations', ('operation_id',)),
|
|
],
|
|
'runtime_audit_events': [
|
|
(('operation_id',), 'runtime_operations', ('operation_id',)),
|
|
(('previous_event_id',), 'runtime_audit_events', ('id',)),
|
|
],
|
|
'worker_progress_events': [
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('remote_device_id',), 'remote_worker_devices', ('id',)),
|
|
],
|
|
'worker_diagnostics': [
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
],
|
|
'source_cycles': [(('run_id',), 'runs', ('id',))],
|
|
'discovery_retry_queue': [
|
|
(('source_cycle_id',), 'source_cycles', ('id',)),
|
|
],
|
|
'target_scans': [
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
(('queue_id',), 'target_queue', ('id',)),
|
|
(('result_reservation_id',), 'result_reservations', ('id',)),
|
|
],
|
|
'findings': [
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
],
|
|
'errors': [
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
],
|
|
'queue_snapshots': [
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
],
|
|
'config_snapshots': [
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
],
|
|
'package_repo_candidates': [
|
|
(('last_run_id',), 'runs', ('id',)),
|
|
(('last_cycle_id',), 'source_cycles', ('id',)),
|
|
],
|
|
'target_queue': [
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
(('current_result_reservation_id',), 'result_reservations', ('id',)),
|
|
],
|
|
'target_queue_policy_events': [
|
|
(('queue_id',), 'target_queue', ('id',)),
|
|
(('reverses_event_id',), 'target_queue_policy_events', ('id',)),
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
],
|
|
'scan_publication_outbox': [(('target_scan_id',), 'target_scans', ('id',))],
|
|
'keycheck_results': [
|
|
(('finding_id',), 'findings', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('candidate_id',), 'keycheck_candidates', ('id',)),
|
|
(('credential_id',), 'keycheck_credentials', ('id',)),
|
|
],
|
|
'keycheck_event_map': [(('keycheck_result_id',), 'keycheck_results', ('id',))],
|
|
'finding_uid_map': [(('finding_id',), 'findings', ('id',))],
|
|
'result_reservations': [
|
|
(('queue_id',), 'target_queue', ('id',)),
|
|
(('run_id',), 'runs', ('id',)),
|
|
(('cycle_id',), 'source_cycles', ('id',)),
|
|
(('remote_device_id', 'remote_user_id'), 'remote_worker_devices', ('id', 'user_id')),
|
|
],
|
|
'remote_worker_devices': [(('user_id',), 'remote_worker_users', ('id',))],
|
|
'admission_intents': [
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('remote_device_id', 'remote_user_id'), 'remote_worker_devices', ('id', 'user_id')),
|
|
],
|
|
'docker_content_blobs': [
|
|
(('lease_reservation_id',), 'result_reservations', ('id',)),
|
|
(('covered_reservation_id',), 'result_reservations', ('id',)),
|
|
],
|
|
'docker_image_blob_coverage': [
|
|
(('queue_id',), 'target_queue', ('id',)),
|
|
(
|
|
('blob_digest', 'coverage_policy_sha256'),
|
|
'docker_content_blobs',
|
|
('digest', 'coverage_policy_sha256'),
|
|
),
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
],
|
|
'result_bundles': [
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
],
|
|
'scan_result_compat': [(('target_scan_id',), 'target_scans', ('id',))],
|
|
'finding_compat_payloads': [(('finding_id',), 'findings', ('id',))],
|
|
'projection_jobs': [
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
(('keycheck_result_id',), 'keycheck_results', ('id',)),
|
|
],
|
|
'projection_cursors': [
|
|
(('stream_name',), 'projection_streams', ('stream_name',)),
|
|
(('last_append_id',), 'projection_appends', ('id',)),
|
|
(('last_job_id',), 'projection_jobs', ('id',)),
|
|
],
|
|
'projection_appends': [
|
|
(('job_id',), 'projection_jobs', ('id',)),
|
|
(('stream_name',), 'projection_streams', ('stream_name',)),
|
|
],
|
|
'projection_rotations': [(('stream_name',), 'projection_streams', ('stream_name',))],
|
|
'keycheck_candidates': [
|
|
(('credential_id',), 'keycheck_credentials', ('id',)),
|
|
(('finding_id',), 'findings', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
(('keycheck_result_id',), 'keycheck_results', ('id',)),
|
|
],
|
|
'keycheck_current_state': [
|
|
(('credential_id',), 'keycheck_credentials', ('id',)),
|
|
(('last_result_id',), 'keycheck_results', ('id',)),
|
|
],
|
|
'pipeline_quarantine': [
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('projection_job_id',), 'projection_jobs', ('id',)),
|
|
(('keycheck_candidate_id',), 'keycheck_candidates', ('id',)),
|
|
],
|
|
'docker_depth_experiment_queries': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
],
|
|
'docker_discovery_passes': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
],
|
|
'docker_discovery_pages': [
|
|
(('pass_id',), 'docker_discovery_passes', ('id',)),
|
|
(('source_cycle_id',), 'source_cycles', ('id',)),
|
|
(('retry_work_id',), 'discovery_retry_queue', ('id',)),
|
|
],
|
|
'docker_repository_query_provenance': [
|
|
(('repository_queue_id',), 'target_queue', ('id',)),
|
|
(('first_cycle_id',), 'source_cycles', ('id',)),
|
|
(('last_cycle_id',), 'source_cycles', ('id',)),
|
|
(('first_page_id',), 'docker_discovery_pages', ('id',)),
|
|
(('last_page_id',), 'docker_discovery_pages', ('id',)),
|
|
],
|
|
'docker_repository_query_observations': [
|
|
(('page_id',), 'docker_discovery_pages', ('id',)),
|
|
(('repository_queue_id',), 'target_queue', ('id',)),
|
|
(
|
|
('source', 'query', 'repository_queue_id'),
|
|
'docker_repository_query_provenance',
|
|
('source', 'query', 'repository_queue_id'),
|
|
),
|
|
],
|
|
'docker_image_manifests': [
|
|
(('target_queue_id',), 'target_queue', ('id',)),
|
|
],
|
|
'docker_manifest_layers': [
|
|
(('manifest_id',), 'docker_image_manifests', ('id',)),
|
|
],
|
|
'docker_depth_experiment_repositories': [
|
|
(
|
|
('experiment_id', 'query_ordinal', 'source', 'query'),
|
|
'docker_depth_experiment_queries',
|
|
('experiment_id', 'query_ordinal', 'source', 'query'),
|
|
),
|
|
(('repository_queue_id',), 'target_queue', ('id',)),
|
|
(
|
|
('source', 'query', 'repository_queue_id'),
|
|
'docker_repository_query_provenance',
|
|
('source', 'query', 'repository_queue_id'),
|
|
),
|
|
(
|
|
('eligibility_page_id', 'repository_queue_id', 'source', 'query'),
|
|
'docker_repository_query_observations',
|
|
('page_id', 'repository_queue_id', 'source', 'query'),
|
|
),
|
|
(('replacement_repository_queue_id',), 'target_queue', ('id',)),
|
|
(
|
|
(
|
|
'replacement_eligibility_page_id',
|
|
'replacement_repository_queue_id',
|
|
'source',
|
|
'query',
|
|
),
|
|
'docker_repository_query_observations',
|
|
('page_id', 'repository_queue_id', 'source', 'query'),
|
|
),
|
|
],
|
|
'docker_depth_experiment_targets': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
(('target_queue_id',), 'target_queue', ('id',)),
|
|
(
|
|
('manifest_id', 'target_queue_id'),
|
|
'docker_image_manifests',
|
|
('id', 'target_queue_id'),
|
|
),
|
|
],
|
|
'docker_depth_experiment_selections': [
|
|
(
|
|
('experiment_id', 'query_ordinal'),
|
|
'docker_depth_experiment_queries',
|
|
('experiment_id', 'query_ordinal'),
|
|
),
|
|
(
|
|
('experiment_id', 'query_ordinal', 'experiment_repository_id'),
|
|
'docker_depth_experiment_repositories',
|
|
('experiment_id', 'query_ordinal', 'id'),
|
|
),
|
|
(
|
|
('experiment_id', 'experiment_target_id'),
|
|
'docker_depth_experiment_targets',
|
|
('experiment_id', 'id'),
|
|
),
|
|
],
|
|
'docker_depth_experiment_candidate_skips': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
|
|
(('repository_queue_id',), 'target_queue', ('id',)),
|
|
],
|
|
'docker_depth_resolver_attempt_refunds': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
|
|
(('repository_queue_id',), 'target_queue', ('id',)),
|
|
],
|
|
'docker_depth_resolver_dispositions': [
|
|
(('experiment_id',), 'docker_depth_experiments', ('id',)),
|
|
(('experiment_repository_id',), 'docker_depth_experiment_repositories', ('id',)),
|
|
(('prior_repository_queue_id',), 'target_queue', ('id',)),
|
|
(('replacement_repository_queue_id',), 'target_queue', ('id',)),
|
|
(('replacement_eligibility_page_id',), 'docker_discovery_pages', ('id',)),
|
|
],
|
|
'docker_depth_experiment_scan_bindings': [
|
|
(('experiment_target_id',), 'docker_depth_experiment_targets', ('id',)),
|
|
(('reservation_id',), 'result_reservations', ('id',)),
|
|
(('target_scan_id',), 'target_scans', ('id',)),
|
|
],
|
|
'docker_finding_layer_attributions': [
|
|
(('scan_binding_id',), 'docker_depth_experiment_scan_bindings', ('id',)),
|
|
(('finding_id',), 'findings', ('id',)),
|
|
(
|
|
(
|
|
'manifest_layer_id', 'position_from_base', 'position_from_top',
|
|
'reported_layer_digest',
|
|
),
|
|
'docker_manifest_layers',
|
|
('id', 'position_from_base', 'position_from_top', 'layer_digest'),
|
|
),
|
|
],
|
|
}
|
|
|
|
REQUIRED_FOREIGN_KEY_ACTIONS = {
|
|
(table, columns): ('NO ACTION', 'NO ACTION')
|
|
for table, foreign_keys in REQUIRED_FOREIGN_KEYS.items()
|
|
for columns, _, _ in foreign_keys
|
|
}
|
|
|
|
|
|
def _required_foreign_key_actions(table, columns):
|
|
return REQUIRED_FOREIGN_KEY_ACTIONS[(table, columns)]
|
|
|
|
|
|
def _foreign_key_actions_match(foreign_key, expected_actions):
|
|
update_action, delete_action = expected_actions
|
|
return (
|
|
str(foreign_key.get('update_action') or '').strip().upper() == update_action
|
|
and str(foreign_key.get('delete_action') or '').strip().upper() == delete_action
|
|
)
|
|
|
|
|
|
def _schema_type_matches(actual, expected, postgres):
|
|
value = re.sub(r'\s+', ' ', str(actual or '').strip().lower())
|
|
if expected in ('id', 'id_ref'):
|
|
return value == ('bigint' if postgres else 'integer')
|
|
if expected == 'integer':
|
|
return value == 'integer'
|
|
if expected == 'text':
|
|
return value == 'text'
|
|
if expected == 'real':
|
|
return value == 'real'
|
|
return value == expected
|
|
|
|
|
|
def _normalized_predicate(value):
|
|
text = str(value or '').lower().replace('"', '')
|
|
text = re.sub(r'::(?:text|character varying)', '', text)
|
|
return re.sub(r'[\s()]', '', text)
|
|
|
|
|
|
def _normalized_default(value):
|
|
text = str(value or '').strip().lower()
|
|
text = re.sub(r'::(?:text|character varying|integer|bigint|smallint)$', '', text)
|
|
while len(text) >= 2 and text[0] == '(' and text[-1] == ')':
|
|
text = text[1:-1].strip()
|
|
if len(text) >= 2 and text[0] == text[-1] == "'":
|
|
text = text[1:-1].replace("''", "'")
|
|
return text
|
|
|
|
|
|
def _sql_enum_check(column, values):
|
|
return '(' + ' OR '.join(f"{column} = '{value}'" for value in values) + ')'
|
|
|
|
|
|
def _sql_sha256_check(column, nullable=False):
|
|
remainder = column
|
|
for character in '0123456789abcdef':
|
|
remainder = f"replace({remainder}, '{character}', '')"
|
|
expression = (
|
|
f'(length({column}) = 64 AND {column} = lower({column}) '
|
|
f'AND length({remainder}) = 0)'
|
|
)
|
|
return f'({column} IS NULL OR {expression})' if nullable else expression
|
|
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_STATES = (
|
|
'collecting', 'planned', 'holding', 'resolving', 'active', 'draining',
|
|
'completed', 'released', 'held',
|
|
)
|
|
DOCKER_DISCOVERY_PASS_STATES = ('collecting', 'complete', 'held', 'failed')
|
|
DOCKER_EXPERIMENT_REPOSITORY_STATES = (
|
|
'pending', 'resolving', 'resolved', 'held', 'failed', 'skipped',
|
|
)
|
|
DOCKER_EXPERIMENT_TARGET_STATES = (
|
|
'pending', 'reserved', 'scanning', 'done', 'failed', 'held',
|
|
'quarantined', 'skipped',
|
|
)
|
|
DOCKER_EXPERIMENT_BINDING_STATES = (
|
|
'reserved', 'scanning', 'completed', 'failed', 'quarantined', 'released',
|
|
)
|
|
DOCKER_FINDING_UNATTRIBUTED_REASONS = (
|
|
'digest_absent', 'digest_invalid', 'digest_not_in_manifest',
|
|
'manifest_mismatch',
|
|
)
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS = {
|
|
'discovery_retry_queue': {
|
|
'discovery_retry_queue_lifecycle_check': (
|
|
"(status = 'leased' AND lease_owner IS NOT NULL AND lease_token IS NOT NULL "
|
|
'AND leased_at IS NOT NULL AND lease_expires_at IS NOT NULL AND held_at IS NULL) '
|
|
"OR (status = 'pending' AND lease_owner IS NULL AND lease_token IS NULL "
|
|
'AND leased_at IS NULL AND lease_expires_at IS NULL AND held_at IS NULL) '
|
|
"OR (status = 'held' AND lease_owner IS NULL AND lease_token IS NULL "
|
|
'AND leased_at IS NULL AND lease_expires_at IS NULL AND held_at IS NOT NULL)'
|
|
),
|
|
'discovery_retry_queue_sha256_check': (
|
|
f'{_sql_sha256_check("work_key")} '
|
|
f'AND {_sql_sha256_check("policy_sha256")}'
|
|
),
|
|
},
|
|
'target_queue_policy_events': {
|
|
'target_queue_policy_events_transition_check': (
|
|
"(action = 'cold' AND (prior_status = 'pending' OR prior_status = 'deferred') "
|
|
"AND next_status = 'cold' AND reverses_event_id IS NULL) OR "
|
|
"(action = 'reactivate' AND prior_status = 'cold' "
|
|
"AND (next_status = 'pending' OR next_status = 'deferred') "
|
|
'AND reverses_event_id IS NOT NULL)'
|
|
),
|
|
'target_queue_policy_events_hash_check': (
|
|
f'{_sql_sha256_check("config_sha256")} '
|
|
f'AND {_sql_sha256_check("policy_sha256")} '
|
|
f'AND {_sql_sha256_check("manifest_sha256")} '
|
|
f'AND {_sql_sha256_check("review_audit_sha256")}'
|
|
),
|
|
'target_queue_policy_events_experiment_ownership_check': (
|
|
"(experiment_id IS NULL "
|
|
"AND reason_code <> 'docker_depth_experiment_hold' "
|
|
"AND reason_code <> 'docker_depth_experiment_dynamic_hold' "
|
|
"AND reason_code <> 'docker_depth_experiment_reviewed_release') OR "
|
|
"(experiment_id IS NOT NULL AND ((action = 'cold' AND "
|
|
"(reason_code = 'docker_depth_experiment_hold' "
|
|
"OR reason_code = 'docker_depth_experiment_dynamic_hold')) "
|
|
"OR (action = 'reactivate' "
|
|
"AND reason_code = 'docker_depth_experiment_reviewed_release')))"
|
|
),
|
|
},
|
|
'docker_depth_experiments': {
|
|
'docker_depth_experiments_state_check': _sql_enum_check(
|
|
'state', DOCKER_DEPTH_EXPERIMENT_STATES,
|
|
),
|
|
'docker_depth_experiments_range_check': (
|
|
'query_count >= 1 AND query_count <= 1000 '
|
|
'AND repositories_per_query >= 1 AND repositories_per_query <= 39 '
|
|
'AND images_per_repository >= 1 AND images_per_repository <= 10 '
|
|
'AND target_limit >= 1 AND target_limit <= 2000 '
|
|
'AND target_count >= 0 AND target_count <= target_limit '
|
|
'AND selection_count >= 0 AND fence_generation >= 0'
|
|
),
|
|
'docker_depth_experiments_identity_check': (
|
|
'length(experiment_key) >= 1 AND length(experiment_key) <= 128 '
|
|
'AND length(source) >= 1 AND length(source) <= 64 '
|
|
f'AND {_sql_sha256_check("config_sha256")} '
|
|
f'AND {_sql_sha256_check("ordered_queries_sha256")} '
|
|
'AND length(selector_version) >= 1 AND length(selector_version) <= 128 '
|
|
f'AND {_sql_sha256_check("selector_sha256")} '
|
|
f'AND {_sql_sha256_check("provenance_policy_sha256")} '
|
|
f'AND {_sql_sha256_check("plan_sha256", nullable=True)} '
|
|
f'AND {_sql_sha256_check("hold_manifest_sha256", nullable=True)}'
|
|
),
|
|
'docker_depth_experiments_capacity_check': (
|
|
'(query_count * (repositories_per_query + images_per_repository - 1) '
|
|
"<= target_limit OR (selector_version = 'docker-rank1-breadth-v1' "
|
|
'AND query_count = 61 AND repositories_per_query = 39 '
|
|
'AND images_per_repository = 1 AND target_limit = 2000))'
|
|
),
|
|
'docker_depth_experiments_selection_check': _sql_sha256_check(
|
|
'selection_sha256', nullable=True,
|
|
),
|
|
'docker_depth_experiments_fence_check': (
|
|
'(fence_owner IS NULL AND fence_token IS NULL AND fence_expires_at IS NULL) '
|
|
'OR (fence_owner IS NOT NULL AND fence_token IS NOT NULL '
|
|
'AND fence_expires_at IS NOT NULL)'
|
|
),
|
|
},
|
|
'docker_depth_experiment_queries': {
|
|
'docker_depth_experiment_queries_range_check': (
|
|
'query_ordinal >= 0 AND required_repository_count >= 1 '
|
|
'AND required_repository_count <= 39'
|
|
),
|
|
'docker_depth_experiment_queries_identity_check': (
|
|
'length(source) >= 1 AND length(source) <= 64 '
|
|
'AND length(query) >= 1 AND length(query) <= 256 '
|
|
f'AND {_sql_sha256_check("query_sha256")}'
|
|
),
|
|
'docker_depth_experiment_queries_selection_check': (
|
|
'selected_repository_count >= 0 '
|
|
'AND selected_repository_count <= required_repository_count'
|
|
),
|
|
},
|
|
'docker_discovery_passes': {
|
|
'docker_discovery_passes_kind_check': _sql_enum_check(
|
|
'pass_kind', ('ordinary', 'deep'),
|
|
),
|
|
'docker_discovery_passes_state_check': _sql_enum_check(
|
|
'state', DOCKER_DISCOVERY_PASS_STATES,
|
|
),
|
|
'docker_discovery_passes_count_check': (
|
|
'expected_query_count >= 1 AND expected_query_count <= 1000 '
|
|
'AND completed_query_count >= 0 '
|
|
'AND completed_query_count <= expected_query_count'
|
|
),
|
|
'docker_discovery_passes_identity_check': (
|
|
'length(pass_token) >= 16 AND length(pass_token) <= 256 '
|
|
'AND length(source) >= 1 AND length(source) <= 64 '
|
|
f'AND {_sql_sha256_check("policy_sha256")} '
|
|
f'AND {_sql_sha256_check("ordered_queries_sha256")}'
|
|
),
|
|
'docker_discovery_passes_complete_check': (
|
|
"state <> 'complete' OR (completed_query_count = expected_query_count "
|
|
'AND completed_at IS NOT NULL)'
|
|
),
|
|
},
|
|
'docker_discovery_pages': {
|
|
'docker_discovery_pages_range_check': (
|
|
'query_ordinal >= 0 AND page_number >= 1 AND page_number <= 30 '
|
|
'AND result_count >= 0 AND admitted_count >= 0 '
|
|
'AND admitted_count <= result_count'
|
|
),
|
|
'docker_discovery_pages_boolean_check': '(query_complete = 0 OR query_complete = 1)',
|
|
'docker_discovery_pages_kind_check': _sql_enum_check(
|
|
'admission_kind', ('main', 'retry'),
|
|
),
|
|
'docker_discovery_pages_identity_check': (
|
|
'length(query) >= 1 AND length(query) <= 256 '
|
|
f'AND {_sql_sha256_check("page_sha256")}'
|
|
),
|
|
'docker_discovery_pages_evidence_check': (
|
|
"(admission_kind = 'main' AND retry_work_id IS NULL) "
|
|
"OR admission_kind = 'retry'"
|
|
),
|
|
'docker_discovery_pages_total_count_check': (
|
|
'total_count IS NULL OR (total_count >= 0 AND total_count >= result_count)'
|
|
),
|
|
},
|
|
'docker_repository_query_provenance': {
|
|
'docker_repository_query_provenance_kind_check': _sql_enum_check(
|
|
'provenance_kind', ('legacy_queue', 'fresh_page'),
|
|
),
|
|
'docker_repository_query_provenance_range_check': (
|
|
'(first_search_rank IS NULL OR first_search_rank >= 1) '
|
|
'AND (best_search_rank IS NULL OR best_search_rank >= 1) '
|
|
'AND (last_search_rank IS NULL OR last_search_rank >= 1) '
|
|
'AND observation_count >= 1 AND fresh_observation_count >= 0 '
|
|
'AND fresh_complete_observation_count >= 0'
|
|
),
|
|
'docker_repository_query_provenance_boolean_check': (
|
|
'(fresh_coverage_eligible = 0 OR fresh_coverage_eligible = 1)'
|
|
),
|
|
'docker_repository_query_provenance_identity_check': (
|
|
'length(source) >= 1 AND length(source) <= 64 '
|
|
'AND length(query) >= 1 AND length(query) <= 256 '
|
|
f'AND {_sql_sha256_check("first_policy_sha256", nullable=True)} '
|
|
f'AND {_sql_sha256_check("last_policy_sha256", nullable=True)}'
|
|
),
|
|
'docker_repository_query_provenance_counts_check': (
|
|
'fresh_complete_observation_count <= fresh_observation_count '
|
|
'AND fresh_observation_count <= observation_count'
|
|
),
|
|
'docker_repository_query_provenance_legacy_check': (
|
|
"provenance_kind <> 'legacy_queue' OR (fresh_observation_count = 0 "
|
|
'AND fresh_complete_observation_count = 0 AND fresh_coverage_eligible = 0 '
|
|
'AND first_page_id IS NULL AND last_page_id IS NULL '
|
|
'AND first_policy_sha256 IS NULL AND last_policy_sha256 IS NULL)'
|
|
),
|
|
'docker_repository_query_provenance_fresh_check': (
|
|
"provenance_kind <> 'fresh_page' OR (fresh_observation_count >= 1 "
|
|
'AND first_search_rank IS NOT NULL AND best_search_rank IS NOT NULL '
|
|
'AND last_search_rank IS NOT NULL AND first_page_id IS NOT NULL '
|
|
'AND last_page_id IS NOT NULL AND first_policy_sha256 IS NOT NULL '
|
|
'AND last_policy_sha256 IS NOT NULL)'
|
|
),
|
|
'docker_repository_query_provenance_eligibility_check': (
|
|
'(fresh_complete_observation_count = 0 AND fresh_coverage_eligible = 0) '
|
|
"OR (fresh_complete_observation_count >= 1 AND fresh_coverage_eligible = 1 "
|
|
"AND provenance_kind = 'fresh_page')"
|
|
),
|
|
},
|
|
'docker_repository_query_observations': {
|
|
'docker_repository_query_observations_rank_check': 'search_rank >= 1',
|
|
},
|
|
'docker_image_manifests': {
|
|
'docker_image_manifests_range_check': (
|
|
'layer_count >= 0 AND (manifest_size_bytes IS NULL OR manifest_size_bytes >= 0)'
|
|
),
|
|
'docker_image_manifests_identity_check': (
|
|
'length(source) >= 1 AND length(source) <= 64 '
|
|
'AND length(repository) >= 1 AND length(repository) <= 512 '
|
|
'AND length(manifest_digest) >= 1 AND length(manifest_digest) <= 256 '
|
|
'AND length(manifest_media_type) >= 1 AND length(manifest_media_type) <= 256 '
|
|
'AND (config_digest IS NULL OR (length(config_digest) >= 1 '
|
|
'AND length(config_digest) <= 256)) '
|
|
f'AND {_sql_sha256_check("graph_sha256")}'
|
|
),
|
|
},
|
|
'docker_manifest_layers': {
|
|
'docker_manifest_layers_range_check': (
|
|
'position_from_base >= 1 AND position_from_top >= 1 AND layer_size_bytes >= 0'
|
|
),
|
|
'docker_manifest_layers_identity_check': (
|
|
'length(layer_digest) >= 1 AND length(layer_digest) <= 256 '
|
|
'AND length(media_type) >= 1 AND length(media_type) <= 256 '
|
|
f'AND {_sql_sha256_check("descriptor_sha256")}'
|
|
),
|
|
},
|
|
'docker_depth_experiment_repositories': {
|
|
'docker_depth_experiment_repositories_range_check': (
|
|
'query_ordinal >= 0 AND repository_rank >= 1 AND repository_rank <= 39 '
|
|
'AND resolver_generation >= 0 AND resolver_attempts >= 0 '
|
|
'AND selected_image_count >= 0 AND selected_image_count <= 10'
|
|
),
|
|
'docker_depth_experiment_repositories_boolean_check': (
|
|
'(is_deep_probe = 0 OR is_deep_probe = 1)'
|
|
),
|
|
'docker_depth_experiment_repositories_state_check': _sql_enum_check(
|
|
'work_state', DOCKER_EXPERIMENT_REPOSITORY_STATES,
|
|
),
|
|
'docker_depth_experiment_repositories_fence_check': (
|
|
"(work_state = 'resolving' AND resolver_owner IS NOT NULL "
|
|
'AND resolver_token IS NOT NULL AND resolver_expires_at IS NOT NULL) '
|
|
"OR (work_state <> 'resolving' AND resolver_owner IS NULL "
|
|
'AND resolver_token IS NULL AND resolver_expires_at IS NULL)'
|
|
),
|
|
'docker_depth_experiment_repositories_resolver_check': (
|
|
'(planned_is_deep_probe = 0 OR planned_is_deep_probe = 1) '
|
|
'AND resolver_attempts >= 0 AND resolver_attempts <= 3 '
|
|
'AND candidate_distinct_graph_count >= 0 '
|
|
'AND candidate_distinct_graph_count <= 100 '
|
|
'AND selected_image_count <= candidate_distinct_graph_count '
|
|
'AND replacement_count >= 0 AND replacement_count <= 3000'
|
|
),
|
|
'docker_depth_experiment_repositories_replacement_check': (
|
|
'(replacement_count = 0 AND replacement_repository_queue_id IS NULL '
|
|
'AND replacement_eligibility_page_id IS NULL '
|
|
'AND replacement_evidence_sha256 IS NULL) OR '
|
|
'(replacement_count >= 1 AND replacement_repository_queue_id IS NOT NULL '
|
|
'AND replacement_eligibility_page_id IS NOT NULL '
|
|
f'AND {_sql_sha256_check("replacement_evidence_sha256")})'
|
|
),
|
|
},
|
|
'docker_depth_experiment_targets': {
|
|
'docker_depth_experiment_targets_range_check': (
|
|
'counter_ordinal >= 1 AND counter_ordinal <= 2000 '
|
|
'AND dispatch_wave >= 1 AND dispatch_wave <= 3 '
|
|
'AND dispatch_order >= 1 AND reservation_count >= 0'
|
|
),
|
|
'docker_depth_experiment_targets_state_check': _sql_enum_check(
|
|
'state', DOCKER_EXPERIMENT_TARGET_STATES,
|
|
),
|
|
},
|
|
'docker_depth_experiment_selections': {
|
|
'docker_depth_experiment_selections_range_check': (
|
|
'query_ordinal >= 0 AND image_rank >= 1 AND image_rank <= 10'
|
|
),
|
|
'docker_depth_experiment_selections_identity_check': (
|
|
'length(selection_reason) >= 1 AND length(selection_reason) <= 128 '
|
|
f'AND {_sql_sha256_check("selection_evidence_sha256")} '
|
|
f'AND {_sql_sha256_check("graph_sha256")}'
|
|
),
|
|
},
|
|
'docker_depth_experiment_candidate_skips': {
|
|
'docker_depth_experiment_candidate_skips_kind_check': _sql_enum_check(
|
|
'candidate_kind', ('image', 'repository'),
|
|
),
|
|
'docker_depth_experiment_candidate_skips_range_check': (
|
|
'candidate_ordinal >= 1 AND candidate_ordinal <= 3000'
|
|
),
|
|
'docker_depth_experiment_candidate_skips_identity_check': (
|
|
f'{_sql_sha256_check("candidate_identity_sha256")} '
|
|
f'AND {_sql_sha256_check("evidence_sha256")}'
|
|
),
|
|
},
|
|
'docker_depth_resolver_attempt_refunds': {
|
|
'docker_depth_resolver_attempt_refunds_kind_check': (
|
|
"recovery_kind = 'zero_graph_limit_v1'"
|
|
),
|
|
'docker_depth_resolver_attempt_refunds_state_check': (
|
|
"(prior_work_state = 'pending' OR prior_work_state = 'held') "
|
|
"AND next_work_state = 'pending'"
|
|
),
|
|
'docker_depth_resolver_attempt_refunds_range_check': (
|
|
'query_ordinal >= 0 AND repository_rank >= 1 '
|
|
'AND prior_resolver_attempts >= 2 AND refund_attempts = 2 '
|
|
'AND next_resolver_attempts = prior_resolver_attempts - refund_attempts '
|
|
'AND next_resolver_attempts >= 0 AND confirmed_bug_event_count = 2'
|
|
),
|
|
'docker_depth_resolver_attempt_refunds_hash_check': (
|
|
f'{_sql_sha256_check("manifest_sha256")} '
|
|
f'AND {_sql_sha256_check("entry_evidence_sha256")} '
|
|
f'AND {_sql_sha256_check("log_sha256")} '
|
|
f'AND {_sql_sha256_check("target_identity_sha256")} '
|
|
f'AND {_sql_sha256_check("prior_error_code_sha256")}'
|
|
),
|
|
},
|
|
'docker_depth_resolver_dispositions': {
|
|
'docker_depth_resolver_dispositions_kind_check': (
|
|
"disposition_kind = 'resolver_attempt_limit_replace_or_skip_v1'"
|
|
),
|
|
'docker_depth_resolver_dispositions_outcome_check': (
|
|
"outcome = 'replaced' OR outcome = 'skipped'"
|
|
),
|
|
'docker_depth_resolver_dispositions_state_check': (
|
|
"prior_work_state = 'held' AND ("
|
|
"(outcome = 'replaced' AND next_work_state = 'pending' "
|
|
"AND replacement_repository_queue_id IS NOT NULL "
|
|
"AND replacement_eligibility_page_id IS NOT NULL "
|
|
"AND replacement_best_search_rank IS NOT NULL "
|
|
"AND replacement_target_identity_sha256 IS NOT NULL "
|
|
"AND next_resolver_attempts = 0) OR "
|
|
"(outcome = 'skipped' AND next_work_state = 'skipped' "
|
|
"AND replacement_repository_queue_id IS NULL "
|
|
"AND replacement_eligibility_page_id IS NULL "
|
|
"AND replacement_best_search_rank IS NULL "
|
|
"AND replacement_target_identity_sha256 IS NULL "
|
|
"AND next_resolver_attempts = prior_resolver_attempts))"
|
|
),
|
|
'docker_depth_resolver_dispositions_range_check': (
|
|
'query_ordinal >= 0 AND repository_rank >= 1 '
|
|
'AND prior_resolver_attempts >= 3 AND next_resolver_attempts >= 0 '
|
|
'AND (replacement_best_search_rank IS NULL '
|
|
'OR replacement_best_search_rank >= 1)'
|
|
),
|
|
'docker_depth_resolver_dispositions_hash_check': (
|
|
f'{_sql_sha256_check("manifest_sha256")} '
|
|
f'AND {_sql_sha256_check("entry_evidence_sha256")} '
|
|
f'AND {_sql_sha256_check("candidate_snapshot_sha256")} '
|
|
f'AND {_sql_sha256_check("prior_target_identity_sha256")} '
|
|
f'AND {_sql_sha256_check("prior_error_code_sha256")} '
|
|
f'AND (replacement_target_identity_sha256 IS NULL '
|
|
f'OR {_sql_sha256_check("replacement_target_identity_sha256")})'
|
|
),
|
|
},
|
|
'docker_depth_experiment_scan_bindings': {
|
|
'docker_depth_experiment_scan_bindings_range_check': 'attempt >= 1',
|
|
'docker_depth_experiment_scan_bindings_state_check': _sql_enum_check(
|
|
'state', DOCKER_EXPERIMENT_BINDING_STATES,
|
|
),
|
|
},
|
|
'docker_finding_layer_attributions': {
|
|
'docker_finding_layer_attributions_state_check': _sql_enum_check(
|
|
'attribution_state', ('exact', 'unattributed'),
|
|
),
|
|
'docker_finding_layer_attributions_evidence_check': (
|
|
"(attribution_state = 'exact' AND manifest_layer_id IS NOT NULL "
|
|
'AND reported_layer_digest IS NOT NULL AND position_from_base IS NOT NULL '
|
|
'AND position_from_top IS NOT NULL AND position_from_base >= 1 '
|
|
"AND position_from_top >= 1) OR (attribution_state = 'unattributed' "
|
|
'AND manifest_layer_id IS NULL AND reported_layer_digest IS NULL '
|
|
'AND position_from_base IS NULL AND position_from_top IS NULL)'
|
|
),
|
|
'docker_finding_layer_attributions_reason_check': (
|
|
"(attribution_state = 'exact' AND unattributed_reason IS NULL) OR "
|
|
"(attribution_state = 'unattributed' AND unattributed_reason IS NOT NULL AND "
|
|
+ _sql_enum_check(
|
|
'unattributed_reason', DOCKER_FINDING_UNATTRIBUTED_REASONS,
|
|
)
|
|
+ ')'
|
|
),
|
|
},
|
|
}
|
|
|
|
DOCKER_DEPTH_SCHEMA_AUTHORITY_CHECKS = (
|
|
('discovery_retry_queue', 'discovery_retry_queue_sha256_check'),
|
|
('target_queue_policy_events', 'target_queue_policy_events_transition_check'),
|
|
('target_queue_policy_events', 'target_queue_policy_events_hash_check'),
|
|
(
|
|
'target_queue_policy_events',
|
|
'target_queue_policy_events_experiment_ownership_check',
|
|
),
|
|
('docker_depth_experiments', 'docker_depth_experiments_selection_check'),
|
|
('docker_discovery_pages', 'docker_discovery_pages_total_count_check'),
|
|
(
|
|
'docker_depth_experiment_repositories',
|
|
'docker_depth_experiment_repositories_resolver_check',
|
|
),
|
|
(
|
|
'docker_depth_experiment_repositories',
|
|
'docker_depth_experiment_repositories_replacement_check',
|
|
),
|
|
(
|
|
'docker_depth_experiment_candidate_skips',
|
|
'docker_depth_experiment_candidate_skips_kind_check',
|
|
),
|
|
(
|
|
'docker_depth_experiment_candidate_skips',
|
|
'docker_depth_experiment_candidate_skips_range_check',
|
|
),
|
|
(
|
|
'docker_depth_experiment_candidate_skips',
|
|
'docker_depth_experiment_candidate_skips_identity_check',
|
|
),
|
|
)
|
|
|
|
DOCKER_DEPTH_SCARCITY_COHORT_CHECKS = (
|
|
(
|
|
'docker_depth_experiment_queries',
|
|
'docker_depth_experiment_queries_selection_check',
|
|
),
|
|
)
|
|
|
|
|
|
_CHECK_TOKEN = re.compile(
|
|
r"'(?:''|[^'])*'|<>|>=|<=|!=|=|>|<|[A-Za-z_][A-Za-z0-9_$]*|"
|
|
r'\d+|[(),+*\-]'
|
|
)
|
|
|
|
|
|
def _normalized_check_expression(value):
|
|
text = str(value or '').strip().lower().replace('"', '')
|
|
if text.startswith('check'):
|
|
text = text[5:].strip()
|
|
text = re.sub(
|
|
r'::(?:text|character varying|character|varchar|integer|bigint|smallint|numeric)',
|
|
'',
|
|
text,
|
|
)
|
|
tokens = []
|
|
position = 0
|
|
while position < len(text):
|
|
if text[position].isspace():
|
|
position += 1
|
|
continue
|
|
match = _CHECK_TOKEN.match(text, position)
|
|
if not match:
|
|
return ('unparsed', re.sub(r'\s+', '', text.replace('!=', '<>')))
|
|
token = match.group(0)
|
|
tokens.append('<>' if token == '!=' else token)
|
|
position = match.end()
|
|
|
|
cursor = 0
|
|
|
|
def combine(operator, values):
|
|
flattened = []
|
|
for item in values:
|
|
if item and item[0] == operator:
|
|
flattened.extend(item[1])
|
|
else:
|
|
flattened.append(item)
|
|
return flattened[0] if len(flattened) == 1 else (operator, tuple(flattened))
|
|
|
|
def parse_primary():
|
|
nonlocal cursor
|
|
if cursor >= len(tokens):
|
|
raise ValueError('missing CHECK expression operand')
|
|
token = tokens[cursor]
|
|
if token == '(':
|
|
cursor += 1
|
|
expression = parse_or()
|
|
if cursor >= len(tokens) or tokens[cursor] != ')':
|
|
raise ValueError('unclosed CHECK expression group')
|
|
cursor += 1
|
|
return expression
|
|
cursor += 1
|
|
if re.fullmatch(r'[a-z_][a-z0-9_$]*', token) and (
|
|
cursor < len(tokens) and tokens[cursor] == '('
|
|
):
|
|
cursor += 1
|
|
arguments = []
|
|
if cursor < len(tokens) and tokens[cursor] != ')':
|
|
while True:
|
|
arguments.append(parse_or())
|
|
if cursor >= len(tokens) or tokens[cursor] != ',':
|
|
break
|
|
cursor += 1
|
|
if cursor >= len(tokens) or tokens[cursor] != ')':
|
|
raise ValueError('unclosed CHECK expression function')
|
|
cursor += 1
|
|
return ('call', token, tuple(arguments))
|
|
if token.startswith("'"):
|
|
return ('string', token[1:-1].replace("''", "'"))
|
|
if token.isdigit():
|
|
return ('number', int(token))
|
|
return ('name', token)
|
|
|
|
def parse_multiply():
|
|
nonlocal cursor
|
|
expression = parse_primary()
|
|
while cursor < len(tokens) and tokens[cursor] == '*':
|
|
operator = tokens[cursor]
|
|
cursor += 1
|
|
expression = (operator, expression, parse_primary())
|
|
return expression
|
|
|
|
def parse_add():
|
|
nonlocal cursor
|
|
expression = parse_multiply()
|
|
while cursor < len(tokens) and tokens[cursor] in ('+', '-'):
|
|
operator = tokens[cursor]
|
|
cursor += 1
|
|
expression = (operator, expression, parse_multiply())
|
|
return expression
|
|
|
|
def parse_comparison():
|
|
nonlocal cursor
|
|
expression = parse_add()
|
|
if cursor < len(tokens) and tokens[cursor] == 'is':
|
|
cursor += 1
|
|
negated = cursor < len(tokens) and tokens[cursor] == 'not'
|
|
if negated:
|
|
cursor += 1
|
|
if cursor >= len(tokens) or tokens[cursor] != 'null':
|
|
raise ValueError('unsupported CHECK IS expression')
|
|
cursor += 1
|
|
return ('is not null' if negated else 'is null', expression)
|
|
if cursor < len(tokens) and tokens[cursor] in ('=', '<>', '>=', '<=', '>', '<'):
|
|
operator = tokens[cursor]
|
|
cursor += 1
|
|
return (operator, expression, parse_add())
|
|
return expression
|
|
|
|
def parse_and():
|
|
nonlocal cursor
|
|
values = [parse_comparison()]
|
|
while cursor < len(tokens) and tokens[cursor] == 'and':
|
|
cursor += 1
|
|
values.append(parse_comparison())
|
|
return combine('and', values)
|
|
|
|
def parse_or():
|
|
nonlocal cursor
|
|
values = [parse_and()]
|
|
while cursor < len(tokens) and tokens[cursor] == 'or':
|
|
cursor += 1
|
|
values.append(parse_and())
|
|
return combine('or', values)
|
|
|
|
try:
|
|
normalized = parse_or()
|
|
if cursor != len(tokens):
|
|
raise ValueError('unsupported trailing CHECK expression')
|
|
return normalized
|
|
except ValueError:
|
|
return ('unparsed', re.sub(r'\s+', '', text.replace('!=', '<>')))
|
|
|
|
|
|
def _docker_depth_check_constraint_problems(conn):
|
|
problems = []
|
|
for table, expected_constraints in DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS.items():
|
|
if not conn.table_exists(table):
|
|
continue
|
|
constraints = conn.table_check_constraints(table)
|
|
for name, expected_expression in expected_constraints.items():
|
|
constraint = constraints.get(name)
|
|
if (
|
|
not constraint
|
|
or not constraint.get('valid', True)
|
|
or _normalized_check_expression(constraint.get('expression'))
|
|
!= _normalized_check_expression(expected_expression)
|
|
):
|
|
problems.append(f'check constraint {table}.{name}')
|
|
return problems
|
|
|
|
|
|
def _docker_depth_check_clause(table, name):
|
|
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
|
|
return f'CONSTRAINT {name} CHECK ({expression})'
|
|
|
|
|
|
def _index_usable(index):
|
|
return bool(index and index.get('valid', True) and index.get('ready', True) and index.get('live', True))
|
|
|
|
|
|
def _normalized_trigger_sql(value):
|
|
return re.sub(r'[\s"]', '', str(value or '').lower())
|
|
|
|
|
|
def _runtime_audit_trigger_problems(conn):
|
|
if not conn.table_exists('runtime_audit_events'):
|
|
return []
|
|
triggers = conn.table_triggers('runtime_audit_events')
|
|
if conn.is_postgres:
|
|
expected = {
|
|
'runtime_audit_events_reject_mutation': (
|
|
'before', 'update', 'delete', 'runtime_audit_events',
|
|
'executefunctionreject_runtime_audit_event_mutation()',
|
|
),
|
|
'runtime_audit_events_reject_truncate': (
|
|
'before', 'truncate', 'runtime_audit_events',
|
|
'executefunctionreject_runtime_audit_event_mutation()',
|
|
),
|
|
}
|
|
else:
|
|
expected = {
|
|
'runtime_audit_events_reject_update': (
|
|
'beforeupdateonruntime_audit_events',
|
|
"raise(abort,'runtime_audit_eventsisappend-only')",
|
|
),
|
|
'runtime_audit_events_reject_delete': (
|
|
'beforedeleteonruntime_audit_events',
|
|
"raise(abort,'runtime_audit_eventsisappend-only')",
|
|
),
|
|
}
|
|
problems = []
|
|
for name, fragments in expected.items():
|
|
trigger = triggers.get(name)
|
|
sql = _normalized_trigger_sql((trigger or {}).get('sql'))
|
|
if (
|
|
not trigger
|
|
or not trigger.get('enabled', True)
|
|
or any(fragment not in sql for fragment in fragments)
|
|
or (
|
|
conn.is_postgres
|
|
and "raiseexception'runtime_audit_eventsisappend-only'"
|
|
not in _normalized_trigger_sql(trigger.get('function_sql'))
|
|
)
|
|
):
|
|
problems.append(f'trigger {name}')
|
|
return problems
|
|
|
|
|
|
def _column_generates_id(details, postgres):
|
|
if not details or not details.get('primary_key') or details.get('generated'):
|
|
return False
|
|
if not postgres:
|
|
return str(details.get('type') or '').strip().lower() == 'integer'
|
|
identity = str(details.get('identity') or '')
|
|
sequence = str(details.get('sequence') or '')
|
|
default_sql = str(details.get('default') or '')
|
|
if identity in ('a', 'd'):
|
|
return bool(sequence)
|
|
return bool(sequence and re.search(r'\bnextval\s*\(', default_sql, re.IGNORECASE))
|
|
|
|
|
|
def utc_now_iso():
|
|
return datetime.now(timezone.utc).isoformat(timespec='seconds')
|
|
|
|
|
|
def _worker_contract_timestamp(value):
|
|
parsed = datetime.fromisoformat(str(value).replace('Z', '+00:00'))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc).isoformat(
|
|
timespec='seconds'
|
|
).replace('+00:00', 'Z')
|
|
|
|
|
|
def _canonical_runtime_json(value):
|
|
try:
|
|
return json.dumps(
|
|
value, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
allow_nan=False,
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('runtime authority identity must be canonical JSON data') from exc
|
|
|
|
|
|
def _runtime_operation_id(value):
|
|
if not isinstance(value, str) or len(value) != 36:
|
|
raise ValueError('runtime operation ID must be a canonical UUID')
|
|
try:
|
|
parsed = uuid.UUID(value)
|
|
except (AttributeError, TypeError, ValueError) as exc:
|
|
raise ValueError('runtime operation ID must be a canonical UUID') from exc
|
|
if parsed.int == 0 or str(parsed) != value:
|
|
raise ValueError('runtime operation ID must be a canonical UUID')
|
|
return value
|
|
|
|
|
|
def _runtime_actor(value):
|
|
if not isinstance(value, str) or not 1 <= len(value) <= 256:
|
|
raise ValueError('runtime operation actor is invalid')
|
|
if any(ord(character) < 32 or ord(character) == 127 for character in value):
|
|
raise ValueError('runtime operation actor is invalid')
|
|
return value
|
|
|
|
|
|
def _runtime_revision(value):
|
|
if (
|
|
isinstance(value, bool) or not isinstance(value, int)
|
|
or not 0 <= value <= RUNTIME_CONTROL_MAX_REVISION
|
|
):
|
|
raise ValueError('runtime control revision is invalid')
|
|
return value
|
|
|
|
|
|
def _runtime_control_identity(value):
|
|
if not isinstance(value, dict) or set(value) != {
|
|
'discovery_paused', 'dispatch_paused', 'drain_state', 'revision',
|
|
}:
|
|
raise RuntimeSafetySchemaError('runtime control identity shape is invalid')
|
|
if type(value['discovery_paused']) is not bool or type(value['dispatch_paused']) is not bool:
|
|
raise RuntimeSafetySchemaError('runtime control pause identity is invalid')
|
|
drain_state = value['drain_state']
|
|
if drain_state not in RUNTIME_CONTROL_DRAIN_STATES:
|
|
raise RuntimeSafetySchemaError('runtime control drain identity is invalid')
|
|
try:
|
|
revision = _runtime_revision(value['revision'])
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError('runtime control revision identity is invalid') from exc
|
|
return {
|
|
'discovery_paused': value['discovery_paused'],
|
|
'dispatch_paused': value['dispatch_paused'],
|
|
'drain_state': drain_state,
|
|
'revision': revision,
|
|
}
|
|
|
|
|
|
def _runtime_control_identity_json(value):
|
|
return _canonical_runtime_json(_runtime_control_identity(value))
|
|
|
|
|
|
def _stored_runtime_control_identity(value):
|
|
if not isinstance(value, str) or not value:
|
|
raise RuntimeSafetySchemaError('runtime control identity JSON is missing')
|
|
try:
|
|
parsed = json.loads(value)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime control identity JSON is invalid') from exc
|
|
canonical = _runtime_control_identity_json(parsed)
|
|
if not hmac.compare_digest(canonical, value):
|
|
raise RuntimeSafetySchemaError('runtime control identity JSON is not canonical')
|
|
return parsed
|
|
|
|
|
|
def _runtime_control_state_from_row(row):
|
|
if not row:
|
|
raise RuntimeSafetySchemaError('runtime operations control singleton is missing')
|
|
try:
|
|
revision = row['revision']
|
|
discovery_raw = row['discovery_paused']
|
|
dispatch_raw = row['dispatch_paused']
|
|
except (KeyError, TypeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime operations control state is invalid') from exc
|
|
if (
|
|
isinstance(revision, bool) or not isinstance(revision, int)
|
|
or revision < 0 or revision > RUNTIME_CONTROL_MAX_REVISION
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime operations control revision is invalid')
|
|
if (
|
|
isinstance(discovery_raw, bool) or not isinstance(discovery_raw, int)
|
|
or isinstance(dispatch_raw, bool) or not isinstance(dispatch_raw, int)
|
|
or discovery_raw not in (0, 1) or dispatch_raw not in (0, 1)
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime operations control pause state is invalid')
|
|
drain_state = str(row['drain_state'] or '')
|
|
if drain_state not in RUNTIME_CONTROL_DRAIN_STATES:
|
|
raise RuntimeSafetySchemaError('runtime operations control drain state is invalid')
|
|
actor = str(row['actor'] or '')
|
|
try:
|
|
_runtime_actor(actor)
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError('runtime operations control actor is invalid') from exc
|
|
operation_id = row['operation_id']
|
|
if operation_id is not None:
|
|
try:
|
|
operation_id = _runtime_operation_id(str(operation_id))
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError('runtime operations control operation ID is invalid') from exc
|
|
created_at = str(row['created_at'] or '')
|
|
updated_at = str(row['updated_at'] or '')
|
|
if not created_at or not updated_at:
|
|
raise RuntimeSafetySchemaError('runtime operations control timestamps are invalid')
|
|
discovery_paused = bool(discovery_raw)
|
|
dispatch_paused = bool(dispatch_raw)
|
|
draining = drain_state != 'normal'
|
|
return {
|
|
'revision': revision,
|
|
'discovery_paused': discovery_paused,
|
|
'dispatch_paused': dispatch_paused,
|
|
'drain_state': drain_state,
|
|
'effective_discovery_paused': discovery_paused or draining,
|
|
'effective_dispatch_paused': dispatch_paused or draining,
|
|
'actor': actor,
|
|
'operation_id': operation_id,
|
|
'created_at': created_at,
|
|
'updated_at': updated_at,
|
|
}
|
|
|
|
|
|
def _runtime_control_state_identity(state):
|
|
return {
|
|
'discovery_paused': state['discovery_paused'],
|
|
'dispatch_paused': state['dispatch_paused'],
|
|
'drain_state': state['drain_state'],
|
|
'revision': state['revision'],
|
|
}
|
|
|
|
|
|
def _runtime_control_transition(before, action):
|
|
before = _runtime_control_identity(before)
|
|
after = dict(before)
|
|
if before['revision'] >= RUNTIME_CONTROL_MAX_REVISION:
|
|
raise RuntimeControlTransitionError('runtime control revision cannot be advanced')
|
|
if action == 'control.discovery.pause':
|
|
if before['discovery_paused']:
|
|
raise RuntimeControlTransitionError(
|
|
'runtime discovery control is already in the requested state'
|
|
)
|
|
after['discovery_paused'] = True
|
|
elif action == 'control.discovery.resume':
|
|
if not before['discovery_paused']:
|
|
raise RuntimeControlTransitionError(
|
|
'runtime discovery control is already in the requested state'
|
|
)
|
|
after['discovery_paused'] = False
|
|
elif action == 'control.dispatch.pause':
|
|
if before['dispatch_paused']:
|
|
raise RuntimeControlTransitionError(
|
|
'runtime dispatch control is already in the requested state'
|
|
)
|
|
after['dispatch_paused'] = True
|
|
elif action == 'control.dispatch.resume':
|
|
if not before['dispatch_paused']:
|
|
raise RuntimeControlTransitionError(
|
|
'runtime dispatch control is already in the requested state'
|
|
)
|
|
after['dispatch_paused'] = False
|
|
elif action == 'control.drain.start':
|
|
if before['drain_state'] != 'normal':
|
|
raise RuntimeControlTransitionError('runtime drain is already active')
|
|
after['drain_state'] = 'draining'
|
|
elif action == 'control.drain.cancel':
|
|
if before['drain_state'] not in ('draining', 'drained'):
|
|
raise RuntimeControlTransitionError('runtime drain is not active')
|
|
after['drain_state'] = 'normal'
|
|
elif action == 'control.drain.complete':
|
|
if before['drain_state'] != 'draining':
|
|
raise RuntimeControlTransitionError('runtime drain is not awaiting completion')
|
|
after['drain_state'] = 'drained'
|
|
else:
|
|
raise ValueError('runtime control action is invalid')
|
|
after['revision'] = before['revision'] + 1
|
|
return after
|
|
|
|
|
|
def _runtime_audit_event_sha256(payload):
|
|
previous = payload.get('previous_event_sha256')
|
|
if previous is None:
|
|
previous_digest = b'\x00' * 32
|
|
elif re.fullmatch(r'[a-f0-9]{64}', str(previous or '')):
|
|
previous_digest = bytes.fromhex(str(previous))
|
|
else:
|
|
raise RuntimeSafetySchemaError('runtime audit parent identity is invalid')
|
|
encoded = _canonical_runtime_json(payload).encode('ascii')
|
|
return hashlib.sha256(
|
|
previous_digest + hashlib.sha256(encoded).digest()
|
|
).hexdigest()
|
|
|
|
|
|
def _runtime_sha256(value, field):
|
|
if not isinstance(value, str) or not re.fullmatch(r'[a-f0-9]{64}', value):
|
|
raise ValueError(f'runtime operation {field} is invalid')
|
|
return value
|
|
|
|
|
|
def _runtime_safe_code(value, field, *, required=False):
|
|
if value is None and not required:
|
|
return None
|
|
allowed = (
|
|
RUNTIME_OPERATION_SAFE_CATEGORIES
|
|
if field == 'safe category' else RUNTIME_OPERATION_SAFE_DETAILS
|
|
)
|
|
if not isinstance(value, str) or value not in allowed:
|
|
raise ValueError(f'runtime operation {field} is invalid')
|
|
return value
|
|
|
|
|
|
def _runtime_audit_row_hash(row):
|
|
event_id = row['id']
|
|
if type(event_id) is not int or event_id < 1:
|
|
raise ValueError('audit event ID')
|
|
operation_id = row['operation_id']
|
|
if operation_id is not None:
|
|
operation_id = str(operation_id)
|
|
actor = str(row['actor'] or '')
|
|
action = str(row['action'] or '')
|
|
target_kind = str(row['target_kind'] or '')
|
|
target_ref = str(row['target_ref'] or '')
|
|
result = str(row['result'] or '')
|
|
safe_category = row['safe_category']
|
|
if safe_category is not None:
|
|
safe_category = str(safe_category)
|
|
identities = []
|
|
for field in ('before_identity_json', 'after_identity_json'):
|
|
raw = row[field]
|
|
if raw is None:
|
|
identities.append(None)
|
|
continue
|
|
raw = str(raw)
|
|
identities.append(raw)
|
|
byte_counts = []
|
|
for field in ('before_bytes', 'after_bytes'):
|
|
value = row[field]
|
|
if value is not None and (
|
|
type(value) is not int
|
|
or value < 0
|
|
):
|
|
raise ValueError('audit byte count')
|
|
byte_counts.append(value)
|
|
previous_event_id = row['previous_event_id']
|
|
previous_event_sha256 = row['previous_event_sha256']
|
|
if previous_event_id is None:
|
|
if previous_event_sha256 is not None:
|
|
raise ValueError('audit parent')
|
|
else:
|
|
if (
|
|
type(previous_event_id) is not int
|
|
or previous_event_id < 1
|
|
or previous_event_id >= event_id
|
|
):
|
|
raise ValueError('audit parent')
|
|
previous_event_sha256 = _runtime_sha256(
|
|
str(previous_event_sha256 or ''), 'audit parent hash',
|
|
)
|
|
created_at = str(row['created_at'] or '')
|
|
event_sha256 = _runtime_sha256(
|
|
str(row['event_sha256'] or ''), 'audit event hash',
|
|
)
|
|
payload = {
|
|
'schema': 'runtime-audit-event-v1',
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': target_kind,
|
|
'target_ref': target_ref,
|
|
'result': result,
|
|
'safe_category': safe_category,
|
|
'before_identity_json': identities[0],
|
|
'after_identity_json': identities[1],
|
|
'before_bytes': byte_counts[0],
|
|
'after_bytes': byte_counts[1],
|
|
'previous_event_id': previous_event_id,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'created_at': created_at,
|
|
}
|
|
if not hmac.compare_digest(
|
|
_runtime_audit_event_sha256(payload), event_sha256,
|
|
):
|
|
raise ValueError('audit event hash')
|
|
return event_id, event_sha256
|
|
|
|
|
|
def _runtime_expected_identity(value, action):
|
|
fields = {
|
|
'active_config_sha256', 'active_secrets_sha256',
|
|
'candidate_config_sha256', 'candidate_secrets_sha256',
|
|
}
|
|
if not isinstance(value, dict) or set(value) != fields:
|
|
raise ValueError('runtime operation expected identity is invalid')
|
|
identity = {
|
|
'active_config_sha256': _runtime_sha256(
|
|
value['active_config_sha256'], 'active config hash',
|
|
),
|
|
'active_secrets_sha256': _runtime_sha256(
|
|
value['active_secrets_sha256'], 'active secrets hash',
|
|
),
|
|
'candidate_config_sha256': value['candidate_config_sha256'],
|
|
'candidate_secrets_sha256': value['candidate_secrets_sha256'],
|
|
}
|
|
required_candidates = {
|
|
'apply-config': (True, False),
|
|
'apply-secrets': (False, True),
|
|
'apply-both': (True, True),
|
|
'restart': (False, False),
|
|
}
|
|
if action not in required_candidates:
|
|
raise ValueError('runtime operation action is invalid')
|
|
for key, required in zip(
|
|
('candidate_config_sha256', 'candidate_secrets_sha256'),
|
|
required_candidates[action],
|
|
):
|
|
current = identity[key]
|
|
if required:
|
|
identity[key] = _runtime_sha256(current, key.replace('_', ' '))
|
|
elif current is not None:
|
|
raise ValueError('runtime operation expected identity is invalid')
|
|
return identity
|
|
|
|
|
|
def _runtime_resulting_identity(value):
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, dict) or set(value) != {
|
|
'active_config_sha256', 'active_secrets_sha256',
|
|
}:
|
|
raise ValueError('runtime operation resulting identity is invalid')
|
|
return {
|
|
'active_config_sha256': _runtime_sha256(
|
|
value['active_config_sha256'], 'resulting config hash',
|
|
),
|
|
'active_secrets_sha256': _runtime_sha256(
|
|
value['active_secrets_sha256'], 'resulting secrets hash',
|
|
),
|
|
}
|
|
|
|
|
|
def _runtime_success_identity(expected, action):
|
|
return {
|
|
'active_config_sha256': (
|
|
expected['candidate_config_sha256']
|
|
if action in ('apply-config', 'apply-both')
|
|
else expected['active_config_sha256']
|
|
),
|
|
'active_secrets_sha256': (
|
|
expected['candidate_secrets_sha256']
|
|
if action in ('apply-secrets', 'apply-both')
|
|
else expected['active_secrets_sha256']
|
|
),
|
|
}
|
|
|
|
|
|
def _runtime_source_id(value):
|
|
if (
|
|
type(value) is str
|
|
and value != 'all'
|
|
and re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:@-]{0,127}', value)
|
|
):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _runtime_source_expected_identity(value, action, target_ref):
|
|
source_action = RUNTIME_SOURCE_OPERATION_ACTIONS.get(action)
|
|
if source_action is None or _runtime_source_id(target_ref) is None or not isinstance(value, dict):
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
legacy_fields = {'source_id', 'source_action', 'interval_seconds'}
|
|
extended_fields = legacy_fields | {
|
|
'mode', 'restart_enabled', 'restart_delay_seconds',
|
|
}
|
|
if set(value) not in (legacy_fields, extended_fields):
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
if value.get('source_id') != target_ref or value.get('source_action') != source_action:
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
interval = value.get('interval_seconds')
|
|
if source_action == 'set-interval':
|
|
if type(interval) is not int or not 1 <= interval <= 365 * 24 * 60 * 60:
|
|
raise ValueError('runtime source operation interval is invalid')
|
|
elif interval is not None:
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
normalized = {
|
|
'source_id': target_ref,
|
|
'source_action': source_action,
|
|
'interval_seconds': interval,
|
|
}
|
|
if set(value) == legacy_fields:
|
|
if source_action not in ('start', 'stop', 'restart', 'pause', 'resume', 'set-interval'):
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
return normalized
|
|
mode = value.get('mode')
|
|
restart_enabled = value.get('restart_enabled')
|
|
restart_delay = value.get('restart_delay_seconds')
|
|
if source_action == 'set-mode':
|
|
if mode not in ('loop', 'once', 'repeat'):
|
|
raise ValueError('runtime source operation mode is invalid')
|
|
elif mode is not None:
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
if source_action == 'set-restart':
|
|
if type(restart_enabled) is not bool:
|
|
raise ValueError('runtime source operation restart setting is invalid')
|
|
elif restart_enabled is not None:
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
if source_action == 'set-restart-delay':
|
|
if (
|
|
type(restart_delay) is not int
|
|
or not 1 <= restart_delay <= 365 * 24 * 60 * 60
|
|
):
|
|
raise ValueError('runtime source operation restart delay is invalid')
|
|
elif restart_delay is not None:
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
if source_action == 'once' and any(
|
|
current is not None for current in (mode, restart_enabled, restart_delay)
|
|
):
|
|
raise ValueError('runtime source operation identity is invalid')
|
|
normalized.update({
|
|
'mode': mode,
|
|
'restart_enabled': restart_enabled,
|
|
'restart_delay_seconds': restart_delay,
|
|
})
|
|
return normalized
|
|
|
|
|
|
def _runtime_source_resulting_identity(value, action, target_ref):
|
|
source_action = RUNTIME_SOURCE_OPERATION_ACTIONS.get(action)
|
|
if (
|
|
source_action is None
|
|
or _runtime_source_id(target_ref) is None
|
|
or not isinstance(value, dict)
|
|
or set(value) != {'source_id', 'source_action', 'outcome'}
|
|
or value.get('source_id') != target_ref
|
|
or value.get('source_action') != source_action
|
|
or value.get('outcome') not in ('completed', 'dependency-blocked')
|
|
):
|
|
raise ValueError('runtime source operation result is invalid')
|
|
return {
|
|
'source_id': target_ref,
|
|
'source_action': source_action,
|
|
'outcome': value['outcome'],
|
|
}
|
|
|
|
|
|
def _stored_runtime_source_identity(value, action, target_ref, *, resulting=False):
|
|
if not isinstance(value, str) or not value:
|
|
raise RuntimeSafetySchemaError('runtime source operation identity is missing')
|
|
try:
|
|
parsed = json.loads(value)
|
|
normalized = (
|
|
_runtime_source_resulting_identity(parsed, action, target_ref)
|
|
if resulting else
|
|
_runtime_source_expected_identity(parsed, action, target_ref)
|
|
)
|
|
canonical = _canonical_runtime_json(normalized)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime source operation identity is invalid') from exc
|
|
if not hmac.compare_digest(canonical, value):
|
|
raise RuntimeSafetySchemaError('runtime source operation identity is not canonical')
|
|
return normalized
|
|
|
|
|
|
def _runtime_worker_admin_target(action, target_ref):
|
|
target_type = RUNTIME_WORKER_ADMIN_ACTION_TARGETS.get(action)
|
|
if target_type is None or not isinstance(target_ref, str):
|
|
raise ValueError('runtime worker admin operation target is invalid')
|
|
if target_type == 'deferred-queue':
|
|
if target_ref != 'deferred-queue':
|
|
raise ValueError('runtime worker admin operation target is invalid')
|
|
elif not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:@-]{0,127}', target_ref):
|
|
raise ValueError('runtime worker admin operation target is invalid')
|
|
return target_ref
|
|
|
|
|
|
def _runtime_worker_admin_expected_identity(value, action, target_ref):
|
|
target_ref = _runtime_worker_admin_target(action, target_ref)
|
|
if (
|
|
not isinstance(value, dict)
|
|
or set(value) != {'action', 'target_ref', 'request_sha256'}
|
|
or value.get('action') != action
|
|
or value.get('target_ref') != target_ref
|
|
):
|
|
raise ValueError('runtime worker admin operation identity is invalid')
|
|
return {
|
|
'action': action,
|
|
'target_ref': target_ref,
|
|
'request_sha256': _runtime_sha256(
|
|
value.get('request_sha256'), 'worker admin request hash',
|
|
),
|
|
}
|
|
|
|
|
|
def _runtime_worker_admin_resulting_identity(value, action, target_ref):
|
|
target_ref = _runtime_worker_admin_target(action, target_ref)
|
|
if (
|
|
not isinstance(value, dict)
|
|
or set(value) != {'action', 'target_ref', 'outcome', 'affected_count'}
|
|
or value.get('action') != action
|
|
or value.get('target_ref') != target_ref
|
|
or value.get('outcome') != 'completed'
|
|
or type(value.get('affected_count')) is not int
|
|
or not 0 <= value['affected_count'] <= 10000
|
|
):
|
|
raise ValueError('runtime worker admin operation result is invalid')
|
|
return {
|
|
'action': action,
|
|
'target_ref': target_ref,
|
|
'outcome': 'completed',
|
|
'affected_count': value['affected_count'],
|
|
}
|
|
|
|
|
|
def _stored_runtime_worker_admin_identity(
|
|
value, action, target_ref, *, resulting=False,
|
|
):
|
|
if not isinstance(value, str) or not value:
|
|
raise RuntimeSafetySchemaError('runtime worker admin operation identity is missing')
|
|
try:
|
|
parsed = json.loads(value)
|
|
normalized = (
|
|
_runtime_worker_admin_resulting_identity(parsed, action, target_ref)
|
|
if resulting else
|
|
_runtime_worker_admin_expected_identity(parsed, action, target_ref)
|
|
)
|
|
canonical = _canonical_runtime_json(normalized)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime worker admin operation identity is invalid') from exc
|
|
if not hmac.compare_digest(canonical, value):
|
|
raise RuntimeSafetySchemaError('runtime worker admin operation identity is not canonical')
|
|
return normalized
|
|
|
|
|
|
def _runtime_document_expected_identity(value, action, target_ref):
|
|
document = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
|
|
if (
|
|
document is None or target_ref != document
|
|
or not isinstance(value, dict)
|
|
or set(value) != {
|
|
'document', 'active_config_sha256', 'active_secrets_sha256',
|
|
'candidate_config_sha256', 'candidate_secrets_sha256',
|
|
'candidate_after_sha256',
|
|
'candidate_before_bytes', 'candidate_after_bytes',
|
|
'candidate_before_present',
|
|
}
|
|
or value.get('document') != document
|
|
):
|
|
raise ValueError('runtime document operation identity is invalid')
|
|
before_bytes = value.get('candidate_before_bytes')
|
|
after_bytes = value.get('candidate_after_bytes')
|
|
before_present = value.get('candidate_before_present')
|
|
if (
|
|
type(before_bytes) is not int or type(after_bytes) is not int
|
|
or type(before_present) is not bool
|
|
or not 0 <= before_bytes <= 4 * 1024 * 1024
|
|
or not 0 <= after_bytes <= 4 * 1024 * 1024
|
|
):
|
|
raise ValueError('runtime document operation byte counts are invalid')
|
|
return {
|
|
'document': document,
|
|
'active_config_sha256': _runtime_sha256(
|
|
value.get('active_config_sha256'), 'active config hash',
|
|
),
|
|
'active_secrets_sha256': _runtime_sha256(
|
|
value.get('active_secrets_sha256'), 'active secrets hash',
|
|
),
|
|
'candidate_config_sha256': _runtime_sha256(
|
|
value.get('candidate_config_sha256'), 'candidate config hash',
|
|
),
|
|
'candidate_secrets_sha256': _runtime_sha256(
|
|
value.get('candidate_secrets_sha256'), 'candidate secrets hash',
|
|
),
|
|
'candidate_after_sha256': _runtime_sha256(
|
|
value.get('candidate_after_sha256'), 'candidate after hash',
|
|
),
|
|
'candidate_before_bytes': before_bytes,
|
|
'candidate_after_bytes': after_bytes,
|
|
'candidate_before_present': before_present,
|
|
}
|
|
|
|
|
|
def _runtime_document_resulting_identity(value, action, target_ref):
|
|
document = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
|
|
if (
|
|
document is None or target_ref != document
|
|
or not isinstance(value, dict)
|
|
or set(value) != {'document', 'outcome', 'candidate_sha256', 'written'}
|
|
or value.get('document') != document
|
|
or value.get('outcome') != 'completed'
|
|
or type(value.get('written')) is not bool
|
|
):
|
|
raise ValueError('runtime document operation result is invalid')
|
|
return {
|
|
'document': document,
|
|
'outcome': 'completed',
|
|
'candidate_sha256': _runtime_sha256(
|
|
value.get('candidate_sha256'), 'candidate result hash',
|
|
),
|
|
'written': value['written'],
|
|
}
|
|
|
|
|
|
def _stored_runtime_document_identity(value, action, target_ref, *, resulting=False):
|
|
if not isinstance(value, str) or not value:
|
|
raise RuntimeSafetySchemaError('runtime document operation identity is missing')
|
|
try:
|
|
parsed = json.loads(value)
|
|
normalized = (
|
|
_runtime_document_resulting_identity(parsed, action, target_ref)
|
|
if resulting else
|
|
_runtime_document_expected_identity(parsed, action, target_ref)
|
|
)
|
|
canonical = _canonical_runtime_json(normalized)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime document operation identity is invalid') from exc
|
|
if not hmac.compare_digest(canonical, value):
|
|
raise RuntimeSafetySchemaError('runtime document operation identity is not canonical')
|
|
return normalized
|
|
|
|
|
|
def _runtime_managed_file_target(root_id, relative_path, target_ref):
|
|
if (
|
|
not isinstance(root_id, str)
|
|
or root_id != target_ref
|
|
or not re.fullmatch(r'[a-z][a-z0-9-]{0,63}', root_id)
|
|
or not isinstance(relative_path, str)
|
|
):
|
|
raise ValueError('runtime managed file operation target is invalid')
|
|
try:
|
|
encoded_path = relative_path.encode('utf-8', errors='strict')
|
|
except UnicodeEncodeError as exc:
|
|
raise ValueError('runtime managed file operation target is invalid') from exc
|
|
components = relative_path.split('/')
|
|
if (
|
|
not encoded_path or len(encoded_path) > 4096
|
|
or relative_path.startswith('/') or '\\' in relative_path or '\x00' in relative_path
|
|
or len(components) > 32
|
|
or any(
|
|
not component or component in ('.', '..')
|
|
or component.startswith('.truf-managed-file-')
|
|
or re.fullmatch(r'[A-Za-z]:.*', component)
|
|
or len(component.encode('utf-8')) > 255
|
|
for component in components
|
|
)
|
|
):
|
|
raise ValueError('runtime managed file operation target is invalid')
|
|
return root_id, relative_path
|
|
|
|
|
|
def _runtime_optional_sha256(value, label):
|
|
return None if value is None else _runtime_sha256(value, label)
|
|
|
|
|
|
def _runtime_optional_byte_count(value, label):
|
|
if value is None:
|
|
return None
|
|
if type(value) is not int or not 0 <= value <= 64 * 1024 * 1024:
|
|
raise ValueError(f'{label} is invalid')
|
|
return value
|
|
|
|
|
|
def _runtime_managed_file_expected_identity(value, action, target_ref):
|
|
if (
|
|
action not in RUNTIME_MANAGED_FILE_ACTIONS
|
|
or not isinstance(value, dict)
|
|
or set(value) != {
|
|
'root_id', 'relative_path', 'expected_sha256',
|
|
'proposed_sha256', 'proposed_byte_count',
|
|
}
|
|
):
|
|
raise ValueError('runtime managed file operation identity is invalid')
|
|
root_id, relative_path = _runtime_managed_file_target(
|
|
value.get('root_id'), value.get('relative_path'), target_ref,
|
|
)
|
|
expected_sha256 = _runtime_optional_sha256(
|
|
value.get('expected_sha256'), 'managed file expected hash',
|
|
)
|
|
proposed_sha256 = _runtime_optional_sha256(
|
|
value.get('proposed_sha256'), 'managed file proposed hash',
|
|
)
|
|
proposed_byte_count = _runtime_optional_byte_count(
|
|
value.get('proposed_byte_count'), 'managed file proposed byte count',
|
|
)
|
|
if (
|
|
action == 'files.create'
|
|
and (expected_sha256 is not None or proposed_sha256 is None or proposed_byte_count is None)
|
|
or action == 'files.replace'
|
|
and (expected_sha256 is None or proposed_sha256 is None or proposed_byte_count is None)
|
|
or action == 'files.delete'
|
|
and (expected_sha256 is None or proposed_sha256 is not None or proposed_byte_count is not None)
|
|
):
|
|
raise ValueError('runtime managed file operation identity is invalid')
|
|
return {
|
|
'root_id': root_id,
|
|
'relative_path': relative_path,
|
|
'expected_sha256': expected_sha256,
|
|
'proposed_sha256': proposed_sha256,
|
|
'proposed_byte_count': proposed_byte_count,
|
|
}
|
|
|
|
|
|
def _runtime_managed_file_resulting_identity(value, action, target_ref):
|
|
if (
|
|
action not in RUNTIME_MANAGED_FILE_ACTIONS
|
|
or not isinstance(value, dict)
|
|
or set(value) != {
|
|
'root_id', 'relative_path', 'outcome',
|
|
'before_sha256', 'before_byte_count',
|
|
'after_sha256', 'after_byte_count', 'written',
|
|
}
|
|
or value.get('outcome') != 'completed'
|
|
or type(value.get('written')) is not bool
|
|
):
|
|
raise ValueError('runtime managed file operation result is invalid')
|
|
root_id, relative_path = _runtime_managed_file_target(
|
|
value.get('root_id'), value.get('relative_path'), target_ref,
|
|
)
|
|
before_sha256 = _runtime_optional_sha256(
|
|
value.get('before_sha256'), 'managed file before hash',
|
|
)
|
|
after_sha256 = _runtime_optional_sha256(
|
|
value.get('after_sha256'), 'managed file after hash',
|
|
)
|
|
before_byte_count = _runtime_optional_byte_count(
|
|
value.get('before_byte_count'), 'managed file before byte count',
|
|
)
|
|
after_byte_count = _runtime_optional_byte_count(
|
|
value.get('after_byte_count'), 'managed file after byte count',
|
|
)
|
|
if (
|
|
action == 'files.create'
|
|
and (before_sha256 is not None or before_byte_count is not None
|
|
or after_sha256 is None or after_byte_count is None or not value['written'])
|
|
or action == 'files.replace'
|
|
and (before_sha256 is None or after_sha256 is None or after_byte_count is None)
|
|
or action == 'files.delete'
|
|
and (before_sha256 is None or after_sha256 is not None or after_byte_count is not None
|
|
or not value['written'])
|
|
):
|
|
raise ValueError('runtime managed file operation result is invalid')
|
|
return {
|
|
'root_id': root_id,
|
|
'relative_path': relative_path,
|
|
'outcome': 'completed',
|
|
'before_sha256': before_sha256,
|
|
'before_byte_count': before_byte_count,
|
|
'after_sha256': after_sha256,
|
|
'after_byte_count': after_byte_count,
|
|
'written': value['written'],
|
|
}
|
|
|
|
|
|
def _stored_runtime_managed_file_identity(
|
|
value, action, target_ref, *, resulting=False,
|
|
):
|
|
if not isinstance(value, str) or not value:
|
|
raise RuntimeSafetySchemaError('runtime managed file operation identity is missing')
|
|
try:
|
|
parsed = json.loads(value)
|
|
normalized = (
|
|
_runtime_managed_file_resulting_identity(parsed, action, target_ref)
|
|
if resulting else
|
|
_runtime_managed_file_expected_identity(parsed, action, target_ref)
|
|
)
|
|
canonical = _canonical_runtime_json(normalized)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file operation identity is invalid'
|
|
) from exc
|
|
if not hmac.compare_digest(canonical, value):
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file operation identity is not canonical'
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _runtime_result_json_is_too_deep(text):
|
|
depth = 0
|
|
in_string = False
|
|
escaped = False
|
|
for character in text:
|
|
if in_string:
|
|
if escaped:
|
|
escaped = False
|
|
elif character == '\\':
|
|
escaped = True
|
|
elif character == '"':
|
|
in_string = False
|
|
elif character == '"':
|
|
in_string = True
|
|
elif character in '[{':
|
|
depth += 1
|
|
if depth > RUNTIME_AGENT_RESULT_MAX_DEPTH:
|
|
return True
|
|
elif character in ']}' and depth:
|
|
depth -= 1
|
|
return False
|
|
|
|
|
|
def _runtime_result_envelope(payload, max_bytes):
|
|
raw = text = parsed = None
|
|
try:
|
|
if type(payload) is not bytes:
|
|
return None, None, 'runtime operation result must be exact bytes'
|
|
if (
|
|
isinstance(max_bytes, bool) or not isinstance(max_bytes, int)
|
|
or not 1 <= max_bytes <= RUNTIME_AGENT_RESULT_MAX_BYTES
|
|
):
|
|
return None, None, 'runtime operation result bound is invalid'
|
|
raw = payload
|
|
if not raw or len(raw) > max_bytes:
|
|
return None, None, 'runtime operation result exceeds its byte bound'
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
try:
|
|
text = raw.decode('utf-8', errors='strict')
|
|
except UnicodeDecodeError:
|
|
return None, None, 'runtime operation result is not valid UTF-8'
|
|
if _runtime_result_json_is_too_deep(text):
|
|
return None, None, 'runtime operation result is invalid JSON'
|
|
|
|
def reject_duplicates(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError('duplicate')
|
|
result[key] = value
|
|
return result
|
|
|
|
try:
|
|
parsed = json.loads(
|
|
text, object_pairs_hook=reject_duplicates,
|
|
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('constant')),
|
|
)
|
|
except (TypeError, ValueError, json.JSONDecodeError, RecursionError):
|
|
return None, None, 'runtime operation result is invalid JSON'
|
|
if not isinstance(parsed, dict) or set(parsed) != {
|
|
'schema', 'operation_id', 'action', 'result', 'safe_category',
|
|
'safe_detail', 'resulting_identity',
|
|
}:
|
|
return None, None, 'runtime operation result shape is invalid'
|
|
if parsed['schema'] != 1 or type(parsed['schema']) is not int:
|
|
return None, None, 'runtime operation result schema is invalid'
|
|
try:
|
|
operation_id = _runtime_operation_id(parsed['operation_id'])
|
|
action = str(parsed['action']) if isinstance(parsed['action'], str) else ''
|
|
if action not in RUNTIME_ASYNC_ACTIONS:
|
|
raise ValueError('runtime operation action is invalid')
|
|
result = str(parsed['result']) if isinstance(parsed['result'], str) else ''
|
|
if result not in RUNTIME_ASYNC_TERMINAL_RESULTS:
|
|
raise ValueError('runtime operation result state is invalid')
|
|
category = _runtime_safe_code(
|
|
parsed['safe_category'], 'safe category', required=result != 'succeeded',
|
|
)
|
|
detail = _runtime_safe_code(parsed['safe_detail'], 'safe detail')
|
|
if result == 'succeeded' and (category is not None or detail is not None):
|
|
raise ValueError('successful runtime operation result must not have an error')
|
|
identity = _runtime_resulting_identity(parsed['resulting_identity'])
|
|
if result in ('succeeded', 'rolled_back') and identity is None:
|
|
raise ValueError('runtime operation result identity is required')
|
|
except ValueError as exc:
|
|
return None, None, str(exc)
|
|
return {
|
|
'operation_id': operation_id,
|
|
'action': action,
|
|
'result': result,
|
|
'safe_category': category,
|
|
'safe_detail': detail,
|
|
'resulting_identity': identity,
|
|
}, digest, None
|
|
finally:
|
|
payload = raw = text = parsed = None
|
|
|
|
|
|
def fixed_lease_window(lease_seconds, now=None):
|
|
issued_at = now or datetime.now(timezone.utc)
|
|
if issued_at.tzinfo is None:
|
|
issued_at = issued_at.replace(tzinfo=timezone.utc)
|
|
issued_at = issued_at.astimezone(timezone.utc)
|
|
expires_at = issued_at + timedelta(seconds=max(60, int(lease_seconds)))
|
|
return (
|
|
issued_at.isoformat(timespec='seconds'),
|
|
expires_at.isoformat(timespec='seconds'),
|
|
)
|
|
|
|
|
|
def compact_delivered_scan_publications(conn, now=None):
|
|
"""Delivered publication state is represented by absence from the outbox."""
|
|
now_dt = now or datetime.now(timezone.utc)
|
|
if isinstance(now_dt, str):
|
|
now_dt = parse_time(now_dt) or datetime.now(timezone.utc)
|
|
now_text = now_dt.isoformat(timespec='seconds')
|
|
conn.execute("DELETE FROM scan_publication_outbox WHERE status = 'delivered'")
|
|
return now_text
|
|
|
|
|
|
def parse_time(value):
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace('Z', '+00:00'))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def json_dumps(value):
|
|
return json.dumps(value, ensure_ascii=False, default=str, sort_keys=True)
|
|
|
|
|
|
def canonical_git_scan_plan_bytes(plan):
|
|
try:
|
|
encoded = json.dumps(
|
|
plan, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('utf-8')
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('Git scan plan must be canonical JSON data') from exc
|
|
if len(encoded) > 16 * 1024:
|
|
raise ValueError('Git scan plan exceeds its byte bound')
|
|
return encoded
|
|
|
|
|
|
def canonical_remote_execution_snapshot_bytes(snapshot):
|
|
if not isinstance(snapshot, dict):
|
|
raise ValueError('remote execution snapshot must be a JSON object')
|
|
# Imported lazily to avoid scanner_db <-> scan_execution import recursion.
|
|
from scan_execution import normalize_remote_execution_snapshot
|
|
snapshot = normalize_remote_execution_snapshot(snapshot)
|
|
try:
|
|
encoded = json.dumps(
|
|
snapshot, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('utf-8')
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('remote execution snapshot must be canonical JSON data') from exc
|
|
if len(encoded) > 64 * 1024:
|
|
raise ValueError('remote execution snapshot exceeds its byte bound')
|
|
return encoded
|
|
|
|
|
|
def stored_remote_execution_snapshot(reservation):
|
|
stored_json = reservation['remote_execution_snapshot_json']
|
|
stored_sha256 = reservation['remote_execution_snapshot_sha256']
|
|
if stored_json is None and stored_sha256 is None:
|
|
return None
|
|
if not stored_json or not re.fullmatch(r'[a-f0-9]{64}', str(stored_sha256 or '')):
|
|
raise ScanEventConflictError('remote execution snapshot identity is incomplete')
|
|
try:
|
|
value = json.loads(str(stored_json))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ScanEventConflictError('remote execution snapshot JSON is invalid') from exc
|
|
encoded = canonical_remote_execution_snapshot_bytes(value)
|
|
if encoded.decode('ascii') != str(stored_json) or not hmac.compare_digest(
|
|
hashlib.sha256(encoded).hexdigest(), str(stored_sha256),
|
|
):
|
|
raise ScanEventConflictError('remote execution snapshot identity is invalid')
|
|
return value
|
|
|
|
|
|
def stored_git_scan_plan(reservation):
|
|
stored_json = reservation['git_scan_plan_json']
|
|
stored_sha256 = reservation['git_scan_plan_sha256']
|
|
if stored_json is None and stored_sha256 is None:
|
|
return None
|
|
if stored_json is None or stored_sha256 is None:
|
|
raise RuntimeSafetySchemaError('Git scan reservation plan identity is incomplete')
|
|
try:
|
|
stored_bytes = str(stored_json).encode('ascii')
|
|
plan = json.loads(stored_bytes.decode('ascii'))
|
|
except (UnicodeEncodeError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('stored Git scan plan JSON is invalid') from exc
|
|
if (
|
|
hashlib.sha256(stored_bytes).hexdigest() != str(stored_sha256)
|
|
or canonical_git_scan_plan_bytes(plan) != stored_bytes
|
|
):
|
|
raise RuntimeSafetySchemaError('stored Git scan plan identity is invalid')
|
|
return plan
|
|
|
|
|
|
def _optional_mapping_value(value, key):
|
|
try:
|
|
return value[key]
|
|
except (KeyError, IndexError):
|
|
return None
|
|
|
|
|
|
def remote_assignment_execution_plan(reservation, *, snapshot=None):
|
|
# Imported lazily to avoid scanner_db <-> scan_execution import recursion.
|
|
from scan_execution import ScanExecutionError
|
|
|
|
try:
|
|
if str(reservation['assignment_kind']) != 'remote':
|
|
raise ScanEventConflictError('execution plan requires a remote assignment')
|
|
snapshot = (
|
|
stored_remote_execution_snapshot(reservation)
|
|
if snapshot is None else snapshot
|
|
)
|
|
if snapshot is None:
|
|
raise ScanEventConflictError('remote assignment has no execution snapshot')
|
|
encoded = canonical_remote_execution_snapshot_bytes(snapshot)
|
|
snapshot = json.loads(encoded.decode('ascii'))
|
|
source = str(reservation['source'])
|
|
platform = str(reservation['platform'])
|
|
target = str(reservation['target'])
|
|
normalized_target = str(reservation['normalized_target'])
|
|
effective_sha256 = str(
|
|
reservation['remote_effective_config_sha256'] or ''
|
|
)
|
|
if (
|
|
snapshot['credential_ref']['source'] != source
|
|
or snapshot['execution']['source'] != platform
|
|
or snapshot['compatibility']['effective_config_sha256']
|
|
!= effective_sha256
|
|
or normalize_target(target, platform) != normalized_target
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote assignment execution context conflicts with its reservation'
|
|
)
|
|
kind = snapshot['planning']['kind']
|
|
git_plan = stored_git_scan_plan(reservation)
|
|
has_docker_plan = any(
|
|
_optional_mapping_value(reservation, name) is not None
|
|
for name in ('docker_layer_plan_json', 'docker_layer_plan_sha256')
|
|
)
|
|
if kind == 'exact_git_v1':
|
|
if source not in ('github', 'gitlab') or platform != source or has_docker_plan:
|
|
raise ScanEventConflictError(
|
|
'remote Git execution context conflicts with its reservation'
|
|
)
|
|
if git_plan is not None and normalize_target(
|
|
git_plan.get('repo_url'), platform,
|
|
) != normalized_target:
|
|
raise ScanEventConflictError(
|
|
'remote Git plan target conflicts with its reservation'
|
|
)
|
|
return {
|
|
'kind': kind,
|
|
'execution_target': target,
|
|
'bound_plan': git_plan,
|
|
}
|
|
if git_plan is not None or has_docker_plan:
|
|
raise ScanEventConflictError(
|
|
'direct execution context has an unexpected bound plan'
|
|
)
|
|
if kind == 'docker_direct_v1':
|
|
if source != 'dockerhub' or platform != 'docker':
|
|
raise ScanEventConflictError(
|
|
'remote Docker execution context conflicts with its reservation'
|
|
)
|
|
parsed = parse_dockerhub_digest_target(target)
|
|
if parsed['normalized_target'] != normalized_target:
|
|
raise ScanEventConflictError(
|
|
'remote Docker target conflicts with its reservation'
|
|
)
|
|
return {
|
|
'kind': kind,
|
|
'execution_target': parsed['image'],
|
|
'bound_plan': None,
|
|
}
|
|
if kind == 'huggingface_space_v1':
|
|
if source != 'huggingface' or platform != 'huggingface':
|
|
raise ScanEventConflictError(
|
|
'remote HuggingFace context conflicts with its reservation'
|
|
)
|
|
execution_target = normalize_huggingface_space_id(target)
|
|
if execution_target.lower() != normalized_target:
|
|
raise ScanEventConflictError(
|
|
'remote HuggingFace target conflicts with its reservation'
|
|
)
|
|
return {
|
|
'kind': kind,
|
|
'execution_target': execution_target,
|
|
'bound_plan': None,
|
|
}
|
|
raise ScanEventConflictError('remote assignment planning kind is unsupported')
|
|
except ScanEventConflictError:
|
|
raise
|
|
except (
|
|
KeyError, IndexError, TypeError, ValueError,
|
|
RuntimeSafetySchemaError, ScanExecutionError,
|
|
) as exc:
|
|
raise ScanEventConflictError(
|
|
'remote assignment execution plan is invalid'
|
|
) from exc
|
|
|
|
|
|
def validate_result_git_scan_plan(reservation, metadata):
|
|
plan = stored_git_scan_plan(reservation)
|
|
metadata_plan = metadata.get('git_scan_plan')
|
|
if plan is None:
|
|
if metadata_plan is not None:
|
|
raise ScanEventConflictError('result has an unbound Git scan plan')
|
|
return None
|
|
if not isinstance(metadata_plan, dict):
|
|
raise ScanEventConflictError('result is missing its bound Git scan plan')
|
|
if canonical_git_scan_plan_bytes(metadata_plan) != canonical_git_scan_plan_bytes(plan):
|
|
raise ScanEventConflictError('result Git scan plan conflicts with its reservation')
|
|
return plan
|
|
|
|
|
|
def validate_remote_result_execution_plan(reservation, result_metadata):
|
|
if not isinstance(result_metadata, dict):
|
|
raise ScanEventConflictError('remote bundle is missing result metadata')
|
|
plan = remote_assignment_execution_plan(reservation)
|
|
kind = plan['kind']
|
|
if kind == 'exact_git_v1':
|
|
if any(
|
|
result_metadata.get(name) is not None
|
|
for name in ('docker_layer_plan', 'docker_layer_execution')
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote Git result has unexpected Docker execution metadata'
|
|
)
|
|
validate_result_git_scan_plan(reservation, result_metadata)
|
|
return plan
|
|
if any(
|
|
result_metadata.get(name) is not None
|
|
for name in (
|
|
'git_scan_plan', 'git_scan_execution',
|
|
'docker_layer_plan', 'docker_layer_execution',
|
|
)
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote direct result has unexpected planning metadata'
|
|
)
|
|
return plan
|
|
|
|
|
|
def validate_git_resolution(resolved):
|
|
resolved = dict(resolved or {})
|
|
provider = str(resolved.get('provider') or '').strip().lower()
|
|
if provider not in ('github', 'gitlab'):
|
|
raise ValueError('Git scan provider must be github or gitlab')
|
|
branch = str(resolved.get('branch') or '')
|
|
ref = str(resolved.get('ref') or '')
|
|
if not branch or ref != f'refs/heads/{branch}' or len(ref) > 1024:
|
|
raise ValueError('Git scan resolution has an invalid branch ref')
|
|
if (
|
|
branch.startswith(('/', '.')) or branch.endswith(('/', '.', '.lock'))
|
|
or '..' in branch or '@{' in branch or '\\' in branch
|
|
or re.search(r'[\x00-\x20\x7f~^:?*\[]', branch)
|
|
or any(part in ('', '.', '..') or part.endswith('.lock') for part in branch.split('/'))
|
|
):
|
|
raise ValueError('Git scan resolution has an unsafe branch ref')
|
|
head_sha = str(resolved.get('head_sha') or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{40}|[a-f0-9]{64}', head_sha):
|
|
raise ValueError('Git scan resolution has an invalid head SHA')
|
|
repo_url = str(resolved.get('repo_url') or '').strip()
|
|
repo_path = str(resolved.get('repo_path') or '').strip()
|
|
expected_host = f'{provider}.com'
|
|
parsed = urlsplit(repo_url)
|
|
if (
|
|
parsed.scheme != 'https' or (parsed.hostname or '').lower() != expected_host
|
|
or parsed.username is not None or parsed.password is not None
|
|
or parsed.port is not None or parsed.query or parsed.fragment
|
|
or not repo_path or len(repo_path) > 1024
|
|
or parsed.path != f'/{repo_path}.git'
|
|
or repo_path.startswith('/') or repo_path.endswith('/') or '\\' in repo_path
|
|
or re.search(r'%(?:2f|5c)', repo_path, flags=re.IGNORECASE)
|
|
or any(part in ('', '.', '..') for part in repo_path.split('/'))
|
|
):
|
|
raise ValueError('Git scan resolution has an invalid canonical repository')
|
|
ref_source = str(resolved.get('ref_source') or '')
|
|
if ref_source not in ('explicit', 'provider_default'):
|
|
raise ValueError('Git scan resolution has an invalid ref source')
|
|
return {
|
|
'provider': provider,
|
|
'repo_url': repo_url,
|
|
'repo_path': repo_path,
|
|
'branch': branch,
|
|
'ref': ref,
|
|
'head_sha': head_sha,
|
|
'ref_source': ref_source,
|
|
}
|
|
|
|
|
|
def matching_git_coverage(reservation, metadata, queue_status, error_count):
|
|
plan = validate_result_git_scan_plan(reservation, metadata)
|
|
if plan is None:
|
|
return False, None, None
|
|
stored_sha256 = reservation['git_scan_plan_sha256']
|
|
execution = metadata.get('git_scan_execution')
|
|
if queue_status != 'done' or int(error_count or 0) != 0:
|
|
return False, None, None
|
|
if not isinstance(execution, dict):
|
|
raise ScanEventConflictError('successful Git result is missing execution evidence')
|
|
if execution.get('success') is not True or execution.get('pinned') is not True:
|
|
return False, None, None
|
|
if str(execution.get('plan_sha256') or '') != str(stored_sha256):
|
|
raise ScanEventConflictError('Git execution evidence has a conflicting plan hash')
|
|
if 'coverage_complete' in execution:
|
|
if execution['coverage_complete'] is not True:
|
|
return False, None, None
|
|
elif metadata.get('degraded') or metadata.get('warnings'):
|
|
return False, None, None
|
|
plan_mode = str(plan.get('mode') or '')
|
|
execution_mode = str(execution.get('mode') or '')
|
|
valid_execution = (
|
|
(plan_mode == 'baseline' and execution_mode == 'baseline')
|
|
or (plan_mode == 'delta' and execution_mode == 'delta')
|
|
or (
|
|
plan_mode == 'delta' and execution_mode == 'baseline_reset'
|
|
and execution.get('continuity_reset') is True
|
|
)
|
|
or (plan_mode == 'noop' and execution_mode == 'noop')
|
|
)
|
|
if not valid_execution:
|
|
return False, None, None
|
|
return True, str(plan['ref']), str(plan['head_sha'])
|
|
|
|
|
|
DOCKER_LAYER_PLAN_MAX_BYTES = 1024 * 1024
|
|
DOCKER_LAYER_MAX_DESCRIPTORS = 2048
|
|
# Exact duplicate-position attribution is all-or-nothing per result bundle.
|
|
DOCKER_DEPTH_MAX_ATTRIBUTION_ROWS_PER_BUNDLE = 100000
|
|
DOCKER_BLOB_LEASE_MARGIN_SEC = 300
|
|
DOCKER_LAYER_SUPPORTED_MEDIA_TYPES = frozenset((
|
|
'application/vnd.oci.image.layer.v1.tar',
|
|
'application/vnd.oci.image.layer.v1.tar+gzip',
|
|
'application/vnd.oci.image.layer.v1.tar+zstd',
|
|
'application/vnd.docker.image.rootfs.diff.tar',
|
|
'application/vnd.docker.image.rootfs.diff.tar.gzip',
|
|
))
|
|
DOCKER_CONFIG_MEDIA_TYPES = frozenset((
|
|
'application/vnd.oci.image.config.v1+json',
|
|
'application/vnd.docker.container.image.v1+json',
|
|
))
|
|
DOCKER_LAYER_LIMIT_KEYS = frozenset((
|
|
'config_max_bytes', 'layer_max_bytes', 'image_max_bytes', 'max_layers',
|
|
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
|
|
'blob_timeout_sec', 'filesystem_concurrency', 'blob_max_attempts',
|
|
))
|
|
DOCKER_ADAPTIVE_SELECTOR_VERSION = 'docker-adaptive-payload-v1'
|
|
DOCKER_ADAPTIVE_EXECUTION_VERSION = 'docker-layer-execution-v4'
|
|
DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION = 'docker-adaptive-shadow-v1'
|
|
DOCKER_ADAPTIVE_GATE_MIN_CONTROLS = 50
|
|
DOCKER_ADAPTIVE_GATE_MAX_CONTROLS = 100
|
|
DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM = 850000
|
|
DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM = 400000
|
|
DOCKER_ADAPTIVE_PAYLOAD_CLASSES = frozenset((
|
|
'config', 'copy_add', 'app_config_run', 'package_run', 'other_run',
|
|
'bulk_data', 'unknown',
|
|
))
|
|
DOCKER_ADAPTIVE_LAYER_CLASS_ORDER = (
|
|
'copy_add', 'app_config_run', 'package_run', 'unknown', 'other_run',
|
|
'bulk_data',
|
|
)
|
|
DOCKER_ADAPTIVE_CHECKPOINT_KEYS = frozenset(('max_blobs', 'max_bytes'))
|
|
DOCKER_ADAPTIVE_SELECTION_REASONS = frozenset((
|
|
'config_selected', 'already_covered', 'duplicate_digest',
|
|
'unsupported_media_type', 'config_too_large', 'layer_too_large',
|
|
'image_budget_exhausted', 'layer_limit_exhausted',
|
|
*(f'selected_{payload_class}' for payload_class in DOCKER_ADAPTIVE_LAYER_CLASS_ORDER),
|
|
))
|
|
DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS = (
|
|
'selected_config', 'selected_copy_add', 'selected_app_config_run',
|
|
'selected_package_run', 'selected_other_run', 'selected_bulk_data',
|
|
'selected_unknown', 'reuse_already_covered', 'reuse_duplicate_digest',
|
|
'omitted_unsupported_media_type', 'omitted_config_too_large',
|
|
'omitted_layer_too_large', 'omitted_image_budget_exhausted',
|
|
'omitted_layer_limit_exhausted', 'adaptive_checkpoints',
|
|
)
|
|
DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS = tuple(
|
|
name for name in DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS
|
|
if name.startswith('omitted_')
|
|
)
|
|
|
|
|
|
def validate_docker_adaptive_policy_hashes(
|
|
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
|
|
):
|
|
values = {
|
|
'scan_policy_sha256': str(scan_policy_sha256 or '').lower(),
|
|
'execution_policy_sha256': str(execution_policy_sha256 or '').lower(),
|
|
'selection_policy_sha256': str(selection_policy_sha256 or '').lower(),
|
|
}
|
|
for label, value in values.items():
|
|
if not re.fullmatch(r'[a-f0-9]{64}', value):
|
|
raise ValueError(f'Docker adaptive {label} must be a lowercase SHA-256')
|
|
return values
|
|
|
|
|
|
def docker_adaptive_shadow_gate_metrics(values):
|
|
names = (
|
|
'completed_pairs', 'full_routed_count', 'adaptive_routed_count',
|
|
'routed_intersection_count', 'full_detector_count',
|
|
'adaptive_detector_count', 'detector_intersection_count',
|
|
'full_slot_ms', 'adaptive_slot_ms', 'omitted_descriptor_count',
|
|
'failure_count', 'privacy_violation_count', 'safety_regression_count',
|
|
)
|
|
metrics = {}
|
|
for name in names:
|
|
raw = values.get(name, 0)
|
|
if isinstance(raw, bool):
|
|
raise ValueError(f'Docker adaptive shadow {name} must be an integer')
|
|
try:
|
|
value = int(raw)
|
|
except (TypeError, ValueError, OverflowError) as exc:
|
|
raise ValueError(f'Docker adaptive shadow {name} must be an integer') from exc
|
|
if value < 0 or value > 9223372036854775807:
|
|
raise ValueError(f'Docker adaptive shadow {name} is outside the supported range')
|
|
metrics[name] = value
|
|
if metrics['routed_intersection_count'] > min(
|
|
metrics['full_routed_count'], metrics['adaptive_routed_count'],
|
|
):
|
|
raise ValueError('Docker adaptive routed intersection exceeds its evidence sets')
|
|
if metrics['detector_intersection_count'] > min(
|
|
metrics['full_detector_count'], metrics['adaptive_detector_count'],
|
|
):
|
|
raise ValueError('Docker adaptive detector intersection exceeds its evidence sets')
|
|
full_routed = metrics['full_routed_count']
|
|
full_slot_ms = metrics['full_slot_ms']
|
|
metrics['routed_recall_ppm'] = (
|
|
metrics['routed_intersection_count'] * 1000000 // full_routed
|
|
if full_routed else 0
|
|
)
|
|
metrics['slot_ratio_ppm'] = (
|
|
(metrics['adaptive_slot_ms'] * 1000000 + full_slot_ms - 1) // full_slot_ms
|
|
if full_slot_ms else 1000001
|
|
)
|
|
return metrics
|
|
|
|
|
|
def validate_docker_adaptive_shadow_selection_metrics(value):
|
|
if isinstance(value, str):
|
|
try:
|
|
value = json.loads(value)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise ValueError('Docker adaptive shadow selection metrics are invalid JSON') from exc
|
|
if not isinstance(value, dict) or set(value) != set(DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS):
|
|
raise ValueError('Docker adaptive shadow selection metrics have invalid fields')
|
|
metrics = {}
|
|
for name in DOCKER_ADAPTIVE_SHADOW_SELECTION_METRIC_KEYS:
|
|
raw = value[name]
|
|
if isinstance(raw, bool):
|
|
raise ValueError(f'Docker adaptive shadow selection metric {name} must be an integer')
|
|
try:
|
|
number = int(raw)
|
|
except (TypeError, ValueError, OverflowError) as exc:
|
|
raise ValueError(
|
|
f'Docker adaptive shadow selection metric {name} must be an integer'
|
|
) from exc
|
|
if number < 0 or number > 9223372036854775807:
|
|
raise ValueError(
|
|
f'Docker adaptive shadow selection metric {name} is outside the supported range'
|
|
)
|
|
metrics[name] = number
|
|
return metrics
|
|
|
|
|
|
def docker_content_media_class(kind, media_type):
|
|
kind = str(kind or '').strip().lower()
|
|
media_type = str(media_type or '').strip().lower()
|
|
if kind == 'config' and media_type in DOCKER_CONFIG_MEDIA_TYPES:
|
|
return 'config-json'
|
|
if kind == 'layer' and media_type in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
|
|
if media_type.endswith('+zstd'):
|
|
return 'layer-zstd'
|
|
if media_type.endswith('+gzip') or media_type.endswith('.gzip'):
|
|
return 'layer-gzip'
|
|
return 'layer-tar'
|
|
return f'{kind}:{media_type}'
|
|
|
|
|
|
def _docker_sha256_digest(value, label='Docker content digest'):
|
|
digest = str(value or '').strip().lower()
|
|
if not re.fullmatch(r'sha256:[a-f0-9]{64}', digest):
|
|
raise ValueError(f'{label} is invalid')
|
|
return digest
|
|
|
|
|
|
def _dockerhub_image_parts(target):
|
|
parsed = parse_docker_target(target)
|
|
image = str(parsed['image']).lower()
|
|
image_name, manifest_digest = image.rsplit('@', 1)
|
|
manifest_digest = _docker_sha256_digest(manifest_digest, 'Docker manifest digest')
|
|
parts = image_name.split('/')
|
|
if len(parts) > 1 and ('.' in parts[0] or ':' in parts[0] or parts[0] == 'localhost'):
|
|
registry = parts.pop(0)
|
|
if registry not in ('docker.io', 'index.docker.io', 'registry-1.docker.io'):
|
|
raise ValueError('Docker layer scanning only supports Docker Hub targets')
|
|
if not parts or any(not part for part in parts):
|
|
raise ValueError('Docker Hub repository is invalid')
|
|
repository = '/'.join(parts)
|
|
registry_repository = repository if '/' in repository else f'library/{repository}'
|
|
return image, repository, registry_repository, manifest_digest
|
|
|
|
|
|
def _normalized_docker_descriptor(value, kind, position=None):
|
|
if not isinstance(value, dict) or set(value) != {'digest', 'size', 'media_type'}:
|
|
raise ValueError(f'Docker {kind} descriptor has an invalid shape')
|
|
digest = _docker_sha256_digest(value.get('digest'), f'Docker {kind} digest')
|
|
try:
|
|
size = int(value.get('size'))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError(f'Docker {kind} descriptor size is invalid') from exc
|
|
if size < 0 or size > 1024 * 1024 * 1024 * 1024:
|
|
raise ValueError(f'Docker {kind} descriptor size is outside its hard bound')
|
|
media_type = str(value.get('media_type') or '').strip().lower()
|
|
if not media_type or len(media_type) > 256 or any(ord(char) < 32 for char in media_type):
|
|
raise ValueError(f'Docker {kind} media type is invalid')
|
|
result = {
|
|
'digest': digest,
|
|
'size': size,
|
|
'media_type': media_type,
|
|
'kind': kind,
|
|
}
|
|
if position is not None:
|
|
result['position'] = int(position)
|
|
return result
|
|
|
|
|
|
def validate_docker_layer_resolution(resolved):
|
|
required = {
|
|
'version', 'image', 'repository', 'manifest_digest', 'platform_os',
|
|
'platform_arch', 'manifest_media_type', 'config', 'layers',
|
|
}
|
|
if not isinstance(resolved, dict) or set(resolved) != required or resolved.get('version') != 1:
|
|
raise ValueError('Docker layer resolution has an invalid shape')
|
|
image, _, registry_repository, target_digest = _dockerhub_image_parts(resolved.get('image'))
|
|
manifest_digest = _docker_sha256_digest(
|
|
resolved.get('manifest_digest'), 'Docker resolved manifest digest',
|
|
)
|
|
if manifest_digest != target_digest:
|
|
raise ValueError('Docker resolved manifest conflicts with the immutable image target')
|
|
repository = str(resolved.get('repository') or '').strip().lower()
|
|
if repository != registry_repository:
|
|
raise ValueError('Docker resolved repository conflicts with the image target')
|
|
platform_os = str(resolved.get('platform_os') or '').strip().lower()
|
|
platform_arch = str(resolved.get('platform_arch') or '').strip().lower()
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_os):
|
|
raise ValueError('Docker resolved platform OS is invalid')
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_arch):
|
|
raise ValueError('Docker resolved platform architecture is invalid')
|
|
manifest_media_type = str(resolved.get('manifest_media_type') or '').strip().lower()
|
|
if not manifest_media_type or len(manifest_media_type) > 256:
|
|
raise ValueError('Docker resolved manifest media type is invalid')
|
|
config = _normalized_docker_descriptor(resolved.get('config'), 'config', 0)
|
|
layers = resolved.get('layers')
|
|
if not isinstance(layers, list) or len(layers) > DOCKER_LAYER_MAX_DESCRIPTORS:
|
|
raise ValueError('Docker layer descriptor count exceeds its bound')
|
|
normalized_layers = [
|
|
_normalized_docker_descriptor(descriptor, 'layer', position)
|
|
for position, descriptor in enumerate(layers, 1)
|
|
]
|
|
return {
|
|
'version': 1,
|
|
'image': image,
|
|
'repository': repository,
|
|
'manifest_digest': manifest_digest,
|
|
'platform_os': platform_os,
|
|
'platform_arch': platform_arch,
|
|
'manifest_media_type': manifest_media_type,
|
|
'config': config,
|
|
'layers': normalized_layers,
|
|
}
|
|
|
|
|
|
def validate_docker_layer_limits(limits):
|
|
if not isinstance(limits, dict) or set(limits) != DOCKER_LAYER_LIMIT_KEYS:
|
|
raise ValueError('Docker layer limits have an invalid shape')
|
|
bounds = {
|
|
'config_max_bytes': (1024, 64 * 1024 * 1024),
|
|
'layer_max_bytes': (1024, 8 * 1024 * 1024 * 1024),
|
|
'image_max_bytes': (1024, 32 * 1024 * 1024 * 1024),
|
|
'max_layers': (1, 256),
|
|
'archive_max_size_bytes': (1024, 4 * 1024 * 1024 * 1024),
|
|
'archive_max_depth': (1, 16),
|
|
'archive_timeout_sec': (1, 600),
|
|
'blob_timeout_sec': (10, 3600),
|
|
'filesystem_concurrency': (1, 16),
|
|
'blob_max_attempts': (1, 10),
|
|
}
|
|
normalized = {}
|
|
for key, (minimum, maximum) in bounds.items():
|
|
try:
|
|
value = int(limits.get(key))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError(f'Docker layer limit {key} must be an integer') from exc
|
|
if not minimum <= value <= maximum:
|
|
raise ValueError(f'Docker layer limit {key} is outside its hard bound')
|
|
normalized[key] = value
|
|
return normalized
|
|
|
|
|
|
def validate_docker_adaptive_checkpoint(checkpoint):
|
|
if not isinstance(checkpoint, dict) or set(checkpoint) != DOCKER_ADAPTIVE_CHECKPOINT_KEYS:
|
|
raise ValueError('Docker adaptive checkpoint has an invalid shape')
|
|
try:
|
|
max_blobs = int(checkpoint.get('max_blobs'))
|
|
max_bytes = int(checkpoint.get('max_bytes'))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('Docker adaptive checkpoint values must be integers') from exc
|
|
if not 1 <= max_blobs <= 32:
|
|
raise ValueError('Docker adaptive checkpoint blob count is outside its hard bound')
|
|
if not 1024 <= max_bytes <= 32 * 1024 * 1024 * 1024:
|
|
raise ValueError('Docker adaptive checkpoint bytes are outside its hard bound')
|
|
return {'max_blobs': max_blobs, 'max_bytes': max_bytes}
|
|
|
|
|
|
def docker_layer_coverage_policy_sha256(scan_policy_sha256, limits):
|
|
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
|
|
raise ValueError('Docker layer scanner policy hash is invalid')
|
|
limits = validate_docker_layer_limits(limits)
|
|
payload = {
|
|
'limits': limits,
|
|
'scan_policy_sha256': scan_policy_sha256,
|
|
'validation_version': DOCKER_ADAPTIVE_EXECUTION_VERSION,
|
|
}
|
|
return hashlib.sha256(json.dumps(
|
|
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest()
|
|
|
|
|
|
def docker_layer_execution_policy_sha256(scan_policy_sha256, limits):
|
|
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
|
|
raise ValueError('Docker layer scanner policy hash is invalid')
|
|
limits = validate_docker_layer_limits(limits)
|
|
payload = {
|
|
'archive_semantics': {
|
|
key: limits[key] for key in (
|
|
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
|
|
)
|
|
},
|
|
'content_media_classes': sorted({
|
|
docker_content_media_class('config', media_type)
|
|
for media_type in DOCKER_CONFIG_MEDIA_TYPES
|
|
} | {
|
|
docker_content_media_class('layer', media_type)
|
|
for media_type in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES
|
|
}),
|
|
'scan_policy_sha256': scan_policy_sha256,
|
|
'version': DOCKER_ADAPTIVE_EXECUTION_VERSION,
|
|
}
|
|
return hashlib.sha256(json.dumps(
|
|
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest()
|
|
|
|
|
|
def docker_layer_selection_policy_sha256(limits):
|
|
limits = validate_docker_layer_limits(limits)
|
|
payload = {
|
|
'class_order': list(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER),
|
|
'limits': {
|
|
key: limits[key] for key in (
|
|
'config_max_bytes', 'layer_max_bytes', 'image_max_bytes', 'max_layers',
|
|
)
|
|
},
|
|
'supported_config_media_types': sorted(DOCKER_CONFIG_MEDIA_TYPES),
|
|
'supported_layer_media_types': sorted(DOCKER_LAYER_SUPPORTED_MEDIA_TYPES),
|
|
'tie_breaks': ['position_desc', 'compressed_size_asc', 'digest_asc'],
|
|
'version': DOCKER_ADAPTIVE_SELECTOR_VERSION,
|
|
}
|
|
return hashlib.sha256(json.dumps(
|
|
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest()
|
|
|
|
|
|
def canonical_docker_layer_plan_bytes(plan):
|
|
if not isinstance(plan, dict):
|
|
raise ValueError('Docker layer plan must be an object')
|
|
payload = json.dumps(
|
|
plan, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')
|
|
if len(payload) > DOCKER_LAYER_PLAN_MAX_BYTES:
|
|
raise ValueError('Docker layer plan exceeds its byte bound')
|
|
return payload
|
|
|
|
|
|
def docker_layer_canary_selected(manifest_digest, basis_points):
|
|
manifest_digest = _docker_sha256_digest(manifest_digest, 'Docker canary manifest digest')
|
|
basis_points = max(0, min(10000, int(basis_points or 0)))
|
|
bucket = int(hashlib.sha256(manifest_digest.encode('ascii')).hexdigest()[:8], 16) % 10000
|
|
return bucket < basis_points
|
|
|
|
|
|
def docker_adaptive_canary_selected(manifest_digest, selection_policy_sha256, basis_points):
|
|
manifest_digest = _docker_sha256_digest(
|
|
manifest_digest, 'Docker adaptive canary manifest digest',
|
|
)
|
|
selection_policy_sha256 = str(selection_policy_sha256 or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', selection_policy_sha256):
|
|
raise ValueError('Docker adaptive selector policy hash is invalid')
|
|
basis_points = max(0, min(10000, int(basis_points or 0)))
|
|
material = (
|
|
f'docker-adaptive-canary-v1:{manifest_digest}:{selection_policy_sha256}'
|
|
).encode('ascii')
|
|
bucket = int(hashlib.sha256(material).hexdigest()[:8], 16) % 10000
|
|
return bucket < basis_points
|
|
|
|
|
|
def validate_docker_layer_plan(plan):
|
|
common_required = {
|
|
'version', 'image', 'repository', 'manifest_digest', 'platform_os',
|
|
'platform_arch', 'manifest_media_type', 'limits',
|
|
'selection_policy_sha256', 'scan_policy_sha256', 'descriptors',
|
|
}
|
|
if not isinstance(plan, dict):
|
|
raise ValueError('Docker layer plan has an invalid shape')
|
|
version = plan.get('version')
|
|
required = common_required if version == 1 else common_required | {
|
|
'selector_version', 'execution_policy_sha256', 'checkpoint',
|
|
}
|
|
if version not in (1, 2) or set(plan) != required:
|
|
raise ValueError('Docker layer plan has an invalid shape')
|
|
image, _, repository, manifest_digest = _dockerhub_image_parts(plan.get('image'))
|
|
if str(plan.get('repository') or '') != repository:
|
|
raise ValueError('Docker layer plan repository conflicts with its image')
|
|
if _docker_sha256_digest(plan.get('manifest_digest')) != manifest_digest:
|
|
raise ValueError('Docker layer plan manifest conflicts with its image')
|
|
limits = validate_docker_layer_limits(plan.get('limits'))
|
|
selection_policy_sha256 = str(plan.get('selection_policy_sha256') or '').lower()
|
|
expected_selection_hash = (
|
|
hashlib.sha256(json.dumps(
|
|
limits, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')).hexdigest()
|
|
if version == 1 else docker_layer_selection_policy_sha256(limits)
|
|
)
|
|
scan_policy_sha256 = str(plan.get('scan_policy_sha256') or '').lower()
|
|
if selection_policy_sha256 != expected_selection_hash:
|
|
raise ValueError('Docker layer plan selection policy hash is invalid')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
|
|
raise ValueError('Docker layer plan scanner policy hash is invalid')
|
|
selector_version = None
|
|
execution_policy_sha256 = None
|
|
checkpoint = None
|
|
if version == 2:
|
|
selector_version = str(plan.get('selector_version') or '')
|
|
if selector_version != DOCKER_ADAPTIVE_SELECTOR_VERSION:
|
|
raise ValueError('Docker adaptive selector version is invalid')
|
|
execution_policy_sha256 = str(
|
|
plan.get('execution_policy_sha256') or ''
|
|
).lower()
|
|
if execution_policy_sha256 != docker_layer_execution_policy_sha256(
|
|
scan_policy_sha256, limits,
|
|
):
|
|
raise ValueError('Docker adaptive execution policy hash is invalid')
|
|
checkpoint = validate_docker_adaptive_checkpoint(plan.get('checkpoint'))
|
|
descriptors = plan.get('descriptors')
|
|
if not isinstance(descriptors, list) or not descriptors or len(descriptors) > DOCKER_LAYER_MAX_DESCRIPTORS + 1:
|
|
raise ValueError('Docker layer plan descriptor count is invalid')
|
|
positions = set()
|
|
normalized_descriptors = []
|
|
allowed_coverage = {
|
|
'selected', 'leased', 'shared_pending', 'covered', 'terminal_failed', 'skipped',
|
|
}
|
|
for descriptor in descriptors:
|
|
expected = {
|
|
'digest', 'size', 'media_type', 'kind', 'position', 'selected',
|
|
'selection_reason', 'coverage_state', 'lease_token', 'attempt',
|
|
'max_attempts',
|
|
}
|
|
if version == 2:
|
|
expected.add('payload_class')
|
|
if not isinstance(descriptor, dict) or set(descriptor) != expected:
|
|
raise ValueError('Docker layer plan descriptor has an invalid shape')
|
|
kind = str(descriptor.get('kind') or '')
|
|
position = int(descriptor.get('position'))
|
|
if kind not in ('config', 'layer') or position < 0 or position in positions:
|
|
raise ValueError('Docker layer plan descriptor identity is invalid')
|
|
if (position == 0) != (kind == 'config'):
|
|
raise ValueError('Docker layer plan configuration position is invalid')
|
|
positions.add(position)
|
|
normalized = _normalized_docker_descriptor({
|
|
'digest': descriptor.get('digest'),
|
|
'size': descriptor.get('size'),
|
|
'media_type': descriptor.get('media_type'),
|
|
}, kind, position)
|
|
selected = descriptor.get('selected')
|
|
coverage_state = str(descriptor.get('coverage_state') or '')
|
|
reason = str(descriptor.get('selection_reason') or '')
|
|
lease_token = descriptor.get('lease_token')
|
|
attempt = int(descriptor.get('attempt'))
|
|
max_attempts = int(descriptor.get('max_attempts'))
|
|
if max_attempts < 1 or max_attempts > 100 or attempt < 0 or attempt > max_attempts:
|
|
raise ValueError('Docker layer plan descriptor attempt bounds are invalid')
|
|
if not isinstance(selected, bool) or coverage_state not in allowed_coverage:
|
|
raise ValueError('Docker layer plan descriptor state is invalid')
|
|
if not reason or len(reason) > 64 or not re.fullmatch(r'[a-z0-9_]+', reason):
|
|
raise ValueError('Docker layer plan selection reason is invalid')
|
|
payload_class = None
|
|
if version == 2:
|
|
payload_class = str(descriptor.get('payload_class') or '')
|
|
if payload_class not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES:
|
|
raise ValueError('Docker layer plan payload class is invalid')
|
|
if (kind == 'config') != (payload_class == 'config'):
|
|
raise ValueError('Docker layer plan payload class conflicts with descriptor kind')
|
|
if reason not in DOCKER_ADAPTIVE_SELECTION_REASONS:
|
|
raise ValueError('Docker adaptive selection reason is invalid')
|
|
if coverage_state == 'leased':
|
|
if not isinstance(lease_token, str) or not 32 <= len(lease_token) <= 128:
|
|
raise ValueError('Docker layer plan lease token is invalid')
|
|
elif lease_token is not None:
|
|
raise ValueError('Docker layer plan has an unexpected lease token')
|
|
if selected != (coverage_state != 'skipped'):
|
|
raise ValueError('Docker layer plan selected state is inconsistent')
|
|
normalized.update({
|
|
'selected': selected,
|
|
'selection_reason': reason,
|
|
'coverage_state': coverage_state,
|
|
'lease_token': lease_token,
|
|
'attempt': attempt,
|
|
'max_attempts': max_attempts,
|
|
})
|
|
if version == 2:
|
|
normalized['payload_class'] = payload_class
|
|
normalized_descriptors.append(normalized)
|
|
if positions != set(range(len(descriptors))):
|
|
raise ValueError('Docker layer plan positions are not contiguous')
|
|
platform_os = str(plan.get('platform_os') or '').lower()
|
|
platform_arch = str(plan.get('platform_arch') or '').lower()
|
|
manifest_media_type = str(plan.get('manifest_media_type') or '').lower()
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_os):
|
|
raise ValueError('Docker layer plan platform OS is invalid')
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', platform_arch):
|
|
raise ValueError('Docker layer plan platform architecture is invalid')
|
|
if not manifest_media_type or len(manifest_media_type) > 256:
|
|
raise ValueError('Docker layer plan manifest media type is invalid')
|
|
normalized_descriptors.sort(key=lambda item: item['position'])
|
|
normalized_plan = {
|
|
'version': version,
|
|
'image': image,
|
|
'repository': repository,
|
|
'manifest_digest': manifest_digest,
|
|
'platform_os': platform_os,
|
|
'platform_arch': platform_arch,
|
|
'manifest_media_type': manifest_media_type,
|
|
'limits': limits,
|
|
'selection_policy_sha256': selection_policy_sha256,
|
|
'scan_policy_sha256': scan_policy_sha256,
|
|
'descriptors': normalized_descriptors,
|
|
}
|
|
if version == 2:
|
|
normalized_plan.update({
|
|
'selector_version': selector_version,
|
|
'execution_policy_sha256': execution_policy_sha256,
|
|
'checkpoint': checkpoint,
|
|
})
|
|
return normalized_plan
|
|
|
|
|
|
def docker_layer_plan_coverage_policy_sha256(plan):
|
|
if int(plan.get('version') or 0) == 2:
|
|
execution_policy_sha256 = str(
|
|
plan.get('execution_policy_sha256') or ''
|
|
).lower()
|
|
expected = docker_layer_execution_policy_sha256(
|
|
plan.get('scan_policy_sha256'), plan.get('limits'),
|
|
)
|
|
if execution_policy_sha256 != expected:
|
|
raise ValueError('Docker adaptive execution policy hash is invalid')
|
|
return execution_policy_sha256
|
|
return docker_layer_coverage_policy_sha256(
|
|
plan.get('scan_policy_sha256'), plan.get('limits'),
|
|
)
|
|
|
|
|
|
def select_docker_adaptive_payload(descriptors, payload_classes, limits, covered_digests=()):
|
|
limits = validate_docker_layer_limits(limits)
|
|
descriptors = list(descriptors or [])
|
|
payload_classes = list(payload_classes or [])
|
|
if (
|
|
not descriptors
|
|
or descriptors[0].get('kind') != 'config'
|
|
or len(payload_classes) != len(descriptors) - 1
|
|
or any(value not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES - {'config'} for value in payload_classes)
|
|
):
|
|
raise ValueError('Docker adaptive payload classes do not match its descriptors')
|
|
covered_digests = set(covered_digests or ())
|
|
entries = {}
|
|
config = descriptors[0]
|
|
if config['digest'] in covered_digests:
|
|
entries[config['position']] = (config, True, 'already_covered', 'config')
|
|
elif config['media_type'] not in DOCKER_CONFIG_MEDIA_TYPES:
|
|
entries[config['position']] = (config, False, 'unsupported_media_type', 'config')
|
|
elif config['size'] > limits['config_max_bytes']:
|
|
entries[config['position']] = (config, False, 'config_too_large', 'config')
|
|
else:
|
|
entries[config['position']] = (config, True, 'config_selected', 'config')
|
|
|
|
class_rank = {
|
|
payload_class: index
|
|
for index, payload_class in enumerate(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER)
|
|
}
|
|
occurrences = {}
|
|
for descriptor, payload_class in zip(descriptors[1:], payload_classes):
|
|
occurrences.setdefault(descriptor['digest'], []).append((descriptor, payload_class))
|
|
|
|
representatives = {}
|
|
for digest, values in occurrences.items():
|
|
representatives[digest] = min(
|
|
values,
|
|
key=lambda value: (
|
|
class_rank[value[1]], -value[0]['position'], value[0]['size'], digest,
|
|
),
|
|
)
|
|
candidates = []
|
|
decisions = {}
|
|
for digest, (descriptor, payload_class) in representatives.items():
|
|
if digest in covered_digests:
|
|
decisions[digest] = (True, 'already_covered')
|
|
elif descriptor['media_type'] not in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
|
|
decisions[digest] = (False, 'unsupported_media_type')
|
|
elif descriptor['size'] > limits['layer_max_bytes']:
|
|
decisions[digest] = (False, 'layer_too_large')
|
|
else:
|
|
candidates.append((descriptor, payload_class))
|
|
candidates.sort(key=lambda value: (
|
|
class_rank[value[1]], -value[0]['position'], value[0]['size'], value[0]['digest'],
|
|
))
|
|
all_fit = (
|
|
len(candidates) <= limits['max_layers']
|
|
and sum(descriptor['size'] for descriptor, _ in candidates) <= limits['image_max_bytes']
|
|
)
|
|
selected_count = 0
|
|
selected_bytes = 0
|
|
for descriptor, payload_class in candidates:
|
|
if all_fit or (
|
|
selected_count < limits['max_layers']
|
|
and selected_bytes + descriptor['size'] <= limits['image_max_bytes']
|
|
):
|
|
decisions[descriptor['digest']] = (True, f'selected_{payload_class}')
|
|
selected_count += 1
|
|
selected_bytes += descriptor['size']
|
|
elif selected_count >= limits['max_layers']:
|
|
decisions[descriptor['digest']] = (False, 'layer_limit_exhausted')
|
|
else:
|
|
decisions[descriptor['digest']] = (False, 'image_budget_exhausted')
|
|
|
|
for digest, values in occurrences.items():
|
|
representative, representative_class = representatives[digest]
|
|
selected, reason = decisions[digest]
|
|
entries[representative['position']] = (
|
|
representative, selected, reason, representative_class,
|
|
)
|
|
for descriptor, payload_class in values:
|
|
if descriptor['position'] == representative['position']:
|
|
continue
|
|
entries[descriptor['position']] = (
|
|
descriptor,
|
|
selected,
|
|
'duplicate_digest' if selected else reason,
|
|
payload_class,
|
|
)
|
|
return [entries[position] for position in sorted(entries)]
|
|
|
|
|
|
def stored_docker_layer_plan(reservation):
|
|
stored_json = reservation['docker_layer_plan_json']
|
|
stored_sha256 = reservation['docker_layer_plan_sha256']
|
|
if stored_json is None and stored_sha256 is None:
|
|
return None, None
|
|
if stored_json is None or stored_sha256 is None:
|
|
raise RuntimeSafetySchemaError('Docker layer reservation plan identity is incomplete')
|
|
try:
|
|
stored_bytes = str(stored_json).encode('ascii')
|
|
plan = json.loads(stored_bytes.decode('ascii'))
|
|
plan = validate_docker_layer_plan(plan)
|
|
except (UnicodeEncodeError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc:
|
|
raise RuntimeSafetySchemaError('stored Docker layer plan JSON is invalid') from exc
|
|
if (
|
|
canonical_docker_layer_plan_bytes(plan) != stored_bytes
|
|
or hashlib.sha256(stored_bytes).hexdigest() != str(stored_sha256)
|
|
):
|
|
raise RuntimeSafetySchemaError('stored Docker layer plan identity is invalid')
|
|
return plan, str(stored_sha256)
|
|
|
|
|
|
def validate_docker_layer_execution(execution, plan, plan_sha256):
|
|
if not isinstance(execution, dict) or set(execution) != {
|
|
'version', 'plan_sha256', 'blobs',
|
|
}:
|
|
raise ValueError('Docker layer execution has an invalid shape')
|
|
if (
|
|
execution.get('version') != plan['version']
|
|
or str(execution.get('plan_sha256') or '') != plan_sha256
|
|
):
|
|
raise ValueError('Docker layer execution plan identity is invalid')
|
|
records = execution.get('blobs')
|
|
if not isinstance(records, list) or len(records) > DOCKER_LAYER_MAX_DESCRIPTORS + 1:
|
|
raise ValueError('Docker layer execution record count is invalid')
|
|
leased = {}
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] == 'leased':
|
|
existing = leased.get(descriptor['digest'])
|
|
if existing and existing['lease_token'] != descriptor['lease_token']:
|
|
raise ValueError('Docker layer plan duplicates a digest with conflicting leases')
|
|
leased[descriptor['digest']] = descriptor
|
|
normalized = []
|
|
seen = set()
|
|
statuses = {'covered', 'retryable_failed', 'terminal_failed'}
|
|
for record in records:
|
|
expected = {
|
|
'digest', 'lease_token', 'status', 'verified_bytes', 'transfer_bytes',
|
|
'transfer_duration_ms', 'scan_duration_ms', 'finding_count', 'error_code',
|
|
}
|
|
if not isinstance(record, dict) or set(record) != expected:
|
|
raise ValueError('Docker layer execution record has an invalid shape')
|
|
digest = _docker_sha256_digest(record.get('digest'), 'execution blob')
|
|
descriptor = leased.get(digest)
|
|
if not descriptor or digest in seen:
|
|
raise ValueError('Docker layer execution has an unclaimed or duplicate blob')
|
|
seen.add(digest)
|
|
if str(record.get('lease_token') or '') != descriptor['lease_token']:
|
|
raise ValueError('Docker layer execution lease token is invalid')
|
|
status = str(record.get('status') or '')
|
|
if status not in statuses:
|
|
raise ValueError('Docker layer execution status is invalid')
|
|
numeric = {}
|
|
for key, upper in (
|
|
('verified_bytes', descriptor['size']),
|
|
('transfer_bytes', descriptor['size']),
|
|
('transfer_duration_ms', 24 * 60 * 60 * 1000),
|
|
('scan_duration_ms', 24 * 60 * 60 * 1000),
|
|
('finding_count', 10_000_000),
|
|
):
|
|
value = record.get(key)
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > upper:
|
|
raise ValueError(f'Docker layer execution {key} is invalid')
|
|
numeric[key] = value
|
|
error_code = record.get('error_code')
|
|
if status == 'covered':
|
|
if numeric['verified_bytes'] != descriptor['size'] or error_code is not None:
|
|
raise ValueError('successful Docker layer execution is incomplete')
|
|
elif (
|
|
not isinstance(error_code, str)
|
|
or not re.fullmatch(r'[a-z0-9_]{1,64}', error_code)
|
|
):
|
|
raise ValueError('failed Docker layer execution error code is invalid')
|
|
normalized.append({
|
|
'digest': digest,
|
|
'lease_token': descriptor['lease_token'],
|
|
'status': status,
|
|
**numeric,
|
|
'error_code': error_code,
|
|
})
|
|
if seen != set(leased):
|
|
raise ValueError('Docker layer execution does not account for every leased blob')
|
|
normalized.sort(key=lambda item: item['digest'])
|
|
return {
|
|
'version': plan['version'],
|
|
'plan_sha256': plan_sha256,
|
|
'blobs': normalized,
|
|
}
|
|
|
|
|
|
def docker_layer_canary_metadata_eligible(metadata, target):
|
|
if not isinstance(metadata, dict):
|
|
return False
|
|
raw_plan = metadata.get('docker_layer_plan')
|
|
if raw_plan is None:
|
|
scan_meta = metadata.get('scan_meta')
|
|
scan_meta = scan_meta if isinstance(scan_meta, dict) else {}
|
|
return bool(
|
|
scan_meta.get('command_timed_out') or metadata.get('error_class') == 'timeout'
|
|
)
|
|
try:
|
|
plan = validate_docker_layer_plan(raw_plan)
|
|
if plan['image'] != _dockerhub_image_parts(target)[0]:
|
|
return False
|
|
plan_sha256 = hashlib.sha256(canonical_docker_layer_plan_bytes(plan)).hexdigest()
|
|
execution = validate_docker_layer_execution(
|
|
metadata.get('docker_layer_execution'), plan, plan_sha256,
|
|
)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
if any(
|
|
descriptor['coverage_state'] in ('selected', 'shared_pending', 'retryable_failed')
|
|
for descriptor in plan['descriptors']
|
|
):
|
|
return True
|
|
return any(
|
|
record['status'] == 'retryable_failed' for record in execution['blobs']
|
|
)
|
|
|
|
|
|
def sqlite_lock_error(exc):
|
|
message = str(exc or '').lower()
|
|
return any(item in message for item in ('locked', 'busy', 'deadlock', 'lock timeout', 'could not serialize access'))
|
|
|
|
|
|
def sqlite_lock_retry_delay(attempt):
|
|
base = SQLITE_LOCK_RETRY_BASE_SEC * (2 ** min(int(attempt or 0), 5))
|
|
jitter = random.uniform(0, SQLITE_LOCK_RETRY_BASE_SEC)
|
|
return min(SQLITE_LOCK_RETRY_MAX_SEC, base + jitter)
|
|
|
|
|
|
def safe_json_loads(value):
|
|
if not value:
|
|
return None
|
|
if isinstance(value, (dict, list)):
|
|
return value
|
|
try:
|
|
return json.loads(str(value))
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return None
|
|
|
|
|
|
def sha256_text(value):
|
|
if value is None:
|
|
return ''
|
|
value = str(value)
|
|
if not value:
|
|
return ''
|
|
return hashlib.sha256(value.encode('utf-8', errors='replace')).hexdigest()
|
|
|
|
|
|
def _identity_mapping(value):
|
|
if hasattr(value, 'as_dict'):
|
|
value = value.as_dict()
|
|
value = dict(value or {})
|
|
required = ('pid', 'creation_time', 'executable')
|
|
if any(not value.get(key) for key in required):
|
|
raise ValueError('exact producer identity is incomplete')
|
|
return {
|
|
'pid': int(value['pid']),
|
|
'creation_time': str(value['creation_time']),
|
|
'executable': os.path.normcase(os.path.realpath(os.path.abspath(str(value['executable'])))),
|
|
}
|
|
|
|
|
|
def hash_file(path):
|
|
if not path or not os.path.exists(path):
|
|
return None
|
|
digest = hashlib.sha256()
|
|
with open(path, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def get_database_path(results_dir=None, explicit_path=None):
|
|
path = explicit_path or os.getenv('SCANNER_DB_PATH') or os.getenv('SCAN_DB_PATH')
|
|
if path:
|
|
if results_dir and not os.path.isabs(path):
|
|
return os.path.join(results_dir, path)
|
|
return path
|
|
return os.path.join(results_dir or os.getcwd(), DB_FILENAME)
|
|
|
|
|
|
def get_database_url(explicit_url=None):
|
|
return explicit_url or database_url_from_env()
|
|
|
|
|
|
def database_disabled():
|
|
return str(os.getenv('SCANNER_DB_DISABLED', '')).strip().lower() in ('1', 'true', 'yes', 'on')
|
|
|
|
|
|
def redact_config(value):
|
|
if isinstance(value, dict):
|
|
redacted = {}
|
|
for key, item in value.items():
|
|
key_text = str(key).lower()
|
|
if any(part in key_text for part in DATABASE_SECRET_KEY_PARTS):
|
|
redacted[key] = redact_database_url(item) if item else item
|
|
elif any(part in key_text for part in SECRET_KEY_PARTS):
|
|
redacted[key] = REDACTED if item else item
|
|
else:
|
|
redacted[key] = redact_config(item)
|
|
return redacted
|
|
if isinstance(value, list):
|
|
return [redact_config(item) for item in value]
|
|
return value
|
|
|
|
|
|
def redact_argv(argv):
|
|
sensitive_flags = {
|
|
'--token', '--docker-token', '--password', '--api-key', '--secret',
|
|
'--db-url', '--database-url', '--scanner-db-url',
|
|
}
|
|
output = []
|
|
hide_next = False
|
|
for value in argv or []:
|
|
text = str(value)
|
|
if hide_next:
|
|
output.append(REDACTED)
|
|
hide_next = False
|
|
continue
|
|
flag = text.split('=', 1)[0].lower()
|
|
if flag in sensitive_flags:
|
|
if '=' in text:
|
|
output.append(text.split('=', 1)[0] + '=' + REDACTED)
|
|
else:
|
|
output.append(text)
|
|
hide_next = True
|
|
continue
|
|
output.append(redact_database_url(text))
|
|
return output
|
|
|
|
|
|
def normalize_target(target, source):
|
|
target_text = str(target or '').strip()
|
|
source = 'docker' if source == 'dockerhub' else str(source or '').lower()
|
|
lowered = target_text.lower()
|
|
if source == 'postman':
|
|
return postman_target_identity(target)
|
|
if source == 'docker':
|
|
return docker_target_identity(target)
|
|
if source == 'github_archive':
|
|
try:
|
|
data = json.loads(target_text)
|
|
repo_url = str(data.get('url') or data.get('repo_url') or target_text).strip().lower()
|
|
branch = str(data.get('branch') or '').strip().lower()
|
|
if repo_url.endswith('.git'):
|
|
repo_url = repo_url[:-4]
|
|
repo_url = repo_url.rstrip('/')
|
|
if repo_url.startswith('http://'):
|
|
repo_url = 'https://' + repo_url[7:]
|
|
return f'{repo_url}#{branch}' if branch else repo_url
|
|
except Exception:
|
|
pass
|
|
if source in ('github', 'github_archive', 'gitlab', 'git'):
|
|
if lowered.endswith('.git'):
|
|
lowered = lowered[:-4]
|
|
lowered = lowered.rstrip('/')
|
|
if lowered.startswith('http://'):
|
|
lowered = 'https://' + lowered[7:]
|
|
return lowered
|
|
if source in ('npm', 'pypi'):
|
|
try:
|
|
data = json.loads(target_text)
|
|
name = data.get('name') or ''
|
|
version = data.get('version') or ''
|
|
return f'{source}:{name}@{version}'.lower()
|
|
except Exception:
|
|
return target_text.split('|', 1)[0].lower()
|
|
return lowered
|
|
|
|
|
|
def _admin_safe_target(target):
|
|
text = str(target or '').strip()
|
|
try:
|
|
parsed = urlsplit(text)
|
|
port = parsed.port
|
|
except ValueError:
|
|
return '[invalid target]'
|
|
if parsed.scheme and parsed.hostname:
|
|
host = parsed.hostname
|
|
if ':' in host and not host.startswith('['):
|
|
host = f'[{host}]'
|
|
authority = host + (f':{port}' if port is not None else '')
|
|
return f'{parsed.scheme.lower()}://{authority}{parsed.path}'[:2048]
|
|
if '@' in text:
|
|
text = text.rsplit('@', 1)[-1]
|
|
return text[:2048]
|
|
|
|
|
|
def _admin_assignment_outcome_sql(reservation='r'):
|
|
return f'''CASE {reservation}.remote_resolution_kind
|
|
WHEN 'bundle_accepted' THEN 'accepted'
|
|
WHEN 'prebundle_report' THEN 'prebundle_failed'
|
|
WHEN 'expired' THEN 'expired'
|
|
ELSE 'unfinished' END'''
|
|
|
|
|
|
def _admin_scan_outcome_sql(reservation='r', scan='s'):
|
|
return (
|
|
f"CASE WHEN {reservation}.remote_resolution_kind = 'bundle_accepted' "
|
|
f"THEN COALESCE({scan}.status, 'unavailable') ELSE 'unavailable' END"
|
|
)
|
|
|
|
|
|
def _validated_admin_worker_filters(value):
|
|
filters = dict(value or {})
|
|
allowed = {
|
|
'source', 'worker', 'assignment_outcome', 'scan_outcome', 'phase',
|
|
'category', 'code', 'retryable', 'since',
|
|
}
|
|
if set(filters) - allowed:
|
|
raise ValueError('worker administration filters are invalid')
|
|
patterns = {
|
|
'source': r'[a-z0-9][a-z0-9_.-]{0,63}',
|
|
'phase': r'[a-z][a-z0-9_]{0,63}',
|
|
'category': r'[a-z][a-z0-9_]{0,127}',
|
|
'code': r'[A-Za-z0-9][A-Za-z0-9._:-]{0,255}',
|
|
}
|
|
normalized = {}
|
|
for name, pattern in patterns.items():
|
|
if name not in filters:
|
|
continue
|
|
item = str(filters[name] or '')
|
|
if re.fullmatch(pattern, item) is None:
|
|
raise ValueError('worker administration filters are invalid')
|
|
normalized[name] = item
|
|
if 'worker' in filters:
|
|
item = str(filters['worker'] or '').strip()
|
|
if not 1 <= len(item) <= 128 or '\x00' in item:
|
|
raise ValueError('worker administration filters are invalid')
|
|
normalized['worker'] = item
|
|
if 'assignment_outcome' in filters:
|
|
item = str(filters['assignment_outcome'] or '')
|
|
if item not in {'accepted', 'prebundle_failed', 'expired', 'unfinished'}:
|
|
raise ValueError('worker administration filters are invalid')
|
|
normalized['assignment_outcome'] = item
|
|
if 'scan_outcome' in filters:
|
|
item = str(filters['scan_outcome'] or '')
|
|
if item not in {'clean', 'found', 'degraded', 'error', 'skipped', 'unavailable'}:
|
|
raise ValueError('worker administration filters are invalid')
|
|
normalized['scan_outcome'] = item
|
|
if 'retryable' in filters:
|
|
if type(filters['retryable']) is not bool:
|
|
raise ValueError('worker administration filters are invalid')
|
|
normalized['retryable'] = filters['retryable']
|
|
if 'since' in filters:
|
|
item = str(filters['since'] or '')
|
|
parse_time(item)
|
|
normalized['since'] = item
|
|
return normalized
|
|
|
|
|
|
def _admin_worker_filter_sql(filters, *, diagnostic_alias=None):
|
|
filters = _validated_admin_worker_filters(filters)
|
|
conditions = []
|
|
parameters = []
|
|
if 'source' in filters:
|
|
conditions.append('r.source = ?')
|
|
parameters.append(filters['source'])
|
|
if 'worker' in filters:
|
|
conditions.append('d.device_key = ?')
|
|
parameters.append(filters['worker'])
|
|
if 'assignment_outcome' in filters:
|
|
conditions.append(f'({_admin_assignment_outcome_sql()}) = ?')
|
|
parameters.append(filters['assignment_outcome'])
|
|
if 'scan_outcome' in filters:
|
|
conditions.append(f'({_admin_scan_outcome_sql()}) = ?')
|
|
parameters.append(filters['scan_outcome'])
|
|
if 'since' in filters:
|
|
timestamp_column = f'{diagnostic_alias}.occurred_at' if diagnostic_alias else 'r.remote_issued_at'
|
|
conditions.append(f'{timestamp_column} >= ?')
|
|
parameters.append(filters['since'])
|
|
|
|
diagnostic_conditions = []
|
|
if 'phase' in filters:
|
|
if diagnostic_alias:
|
|
conditions.append(f'{diagnostic_alias}.phase = ?')
|
|
parameters.append(filters['phase'])
|
|
else:
|
|
conditions.append('''(
|
|
p.phase = ? OR EXISTS (
|
|
SELECT 1 FROM worker_diagnostics fd
|
|
WHERE fd.reservation_id = r.id AND fd.phase = ?
|
|
)
|
|
)''')
|
|
parameters.extend((filters['phase'], filters['phase']))
|
|
for name in ('category', 'code'):
|
|
if name in filters:
|
|
diagnostic_conditions.append(f'fd.{name} = ?')
|
|
parameters.append(filters[name])
|
|
if 'retryable' in filters:
|
|
diagnostic_conditions.append('fd.retryable = ?')
|
|
parameters.append(1 if filters['retryable'] else 0)
|
|
if diagnostic_conditions:
|
|
if diagnostic_alias:
|
|
conditions.extend(
|
|
condition.replace('fd.', f'{diagnostic_alias}.')
|
|
for condition in diagnostic_conditions
|
|
)
|
|
else:
|
|
conditions.append('''EXISTS (
|
|
SELECT 1 FROM worker_diagnostics fd
|
|
WHERE fd.reservation_id = r.id AND %s
|
|
)''' % ' AND '.join(diagnostic_conditions))
|
|
return filters, conditions, parameters
|
|
|
|
|
|
def _validated_discovery_retry_source(value):
|
|
if not isinstance(value, str) or not re.fullmatch(r'[a-z][a-z0-9_-]{0,63}', value):
|
|
raise ValueError('discovery retry source is invalid')
|
|
return value
|
|
|
|
|
|
def _validated_discovery_retry_query(value):
|
|
if (
|
|
not isinstance(value, str)
|
|
or not value
|
|
or value != value.strip()
|
|
or len(value) > DISCOVERY_RETRY_MAX_QUERY_CHARS
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in value)
|
|
):
|
|
raise ValueError('discovery retry query is invalid')
|
|
return value
|
|
|
|
|
|
def _validated_discovery_retry_policy(value):
|
|
policy = str(value or '')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', policy):
|
|
raise ValueError('discovery retry policy hash is invalid')
|
|
return policy
|
|
|
|
|
|
def _validated_discovery_retry_pages(work_kind, page_start, page_end):
|
|
work_kind = str(work_kind or '')
|
|
if work_kind not in DISCOVERY_RETRY_WORK_KINDS:
|
|
raise ValueError('discovery retry work kind is invalid')
|
|
if isinstance(page_start, bool) or isinstance(page_end, bool):
|
|
raise ValueError('discovery retry page range is invalid')
|
|
try:
|
|
start = int(page_start)
|
|
end = int(page_end)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry page range is invalid') from None
|
|
if not 1 <= start <= end <= DISCOVERY_RETRY_MAX_PAGE:
|
|
raise ValueError('discovery retry page range is invalid')
|
|
if work_kind == 'query' and start != 1:
|
|
raise ValueError('query-level discovery retry work must start at page one')
|
|
if work_kind == 'page' and start != end:
|
|
raise ValueError('page-level discovery retry work must identify one page')
|
|
return work_kind, start, end
|
|
|
|
|
|
def _validated_discovery_retry_error_category(value, required=False):
|
|
category = str(value or '')
|
|
if not category and not required:
|
|
return None
|
|
if category not in DISCOVERY_RETRY_ERROR_CATEGORIES:
|
|
raise ValueError('discovery retry error category is invalid')
|
|
return category
|
|
|
|
|
|
def _validated_discovery_retry_fence(retry_id, lease_owner, lease_token):
|
|
if isinstance(retry_id, bool):
|
|
raise ValueError('discovery retry lease identity is invalid')
|
|
try:
|
|
retry_id = int(retry_id)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry lease identity is invalid') from None
|
|
owner = str(lease_owner or '')
|
|
token = str(lease_token or '')
|
|
if (
|
|
retry_id < 1
|
|
or not 1 <= len(owner) <= 128
|
|
or not 1 <= len(token) <= 128
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in owner + token)
|
|
):
|
|
raise ValueError('discovery retry lease identity is invalid')
|
|
return retry_id, owner, token
|
|
|
|
|
|
def _validated_discovery_retry_time(value, now=None):
|
|
if value is None:
|
|
return None
|
|
parsed = parse_time(value)
|
|
if not parsed:
|
|
raise ValueError('discovery retry time is invalid')
|
|
now = now or datetime.now(timezone.utc)
|
|
if parsed > now + timedelta(seconds=DISCOVERY_RETRY_MAX_TRUSTED_DELAY_SEC):
|
|
raise ValueError('discovery retry time exceeds the trusted delay bound')
|
|
return max(parsed, now).isoformat(timespec='seconds')
|
|
|
|
|
|
def _discovery_retry_allowlist(configured_query_policies):
|
|
if isinstance(configured_query_policies, dict):
|
|
entries = list(configured_query_policies.items())
|
|
else:
|
|
try:
|
|
entries = list(configured_query_policies or ())
|
|
except TypeError:
|
|
raise ValueError('discovery retry query policy allowlist is invalid') from None
|
|
if len(entries) > DISCOVERY_RETRY_MAX_ALLOWLIST:
|
|
raise ValueError('discovery retry query policy allowlist exceeds its bound')
|
|
allowed = {}
|
|
for entry in entries:
|
|
if not isinstance(entry, (list, tuple)) or len(entry) != 2:
|
|
raise ValueError('discovery retry query policy allowlist is invalid')
|
|
query = _validated_discovery_retry_query(entry[0])
|
|
policy = _validated_discovery_retry_policy(entry[1])
|
|
if query in allowed and allowed[query] != policy:
|
|
raise ValueError('discovery retry query policy allowlist conflicts')
|
|
allowed[query] = policy
|
|
return tuple(sorted(allowed.items()))
|
|
|
|
|
|
def discovery_retry_work_key(
|
|
source, query, policy_sha256, pass_kind, work_kind, page_start, page_end,
|
|
pass_id=None,
|
|
):
|
|
source = _validated_discovery_retry_source(source)
|
|
query = _validated_discovery_retry_query(query)
|
|
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
|
|
pass_kind = str(pass_kind or '')
|
|
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
|
|
raise ValueError('discovery retry pass kind is invalid')
|
|
work_kind, page_start, page_end = _validated_discovery_retry_pages(
|
|
work_kind, page_start, page_end,
|
|
)
|
|
identity = [
|
|
source, query, policy_sha256, pass_kind, work_kind, page_start, page_end,
|
|
]
|
|
if pass_id is not None:
|
|
if isinstance(pass_id, bool):
|
|
raise ValueError('discovery retry pass identity is invalid')
|
|
try:
|
|
pass_id = int(pass_id)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry pass identity is invalid') from None
|
|
if not 1 <= pass_id <= 0x7fffffffffffffff:
|
|
raise ValueError('discovery retry pass identity is invalid')
|
|
identity.append(pass_id)
|
|
payload = json.dumps(
|
|
identity,
|
|
ensure_ascii=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
if pass_id is None:
|
|
return digest
|
|
return f'{pass_id:016x}{digest[:48]}'
|
|
|
|
|
|
def _validated_docker_discovery_observation(value, source, query):
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, dict):
|
|
raise ValueError('DockerHub discovery observation metadata is invalid')
|
|
required = {
|
|
'cycle_id', 'query_ordinal', 'query_count', 'page_number', 'page_limit',
|
|
'per_page', 'total_count', 'policy_sha256', 'pass_kind',
|
|
'collection_generation', 'ordered_query_hash', 'query_complete',
|
|
}
|
|
if set(value) != required:
|
|
raise ValueError('DockerHub discovery observation metadata shape is invalid')
|
|
numeric = {}
|
|
for name in ('query_ordinal', 'query_count', 'page_number', 'page_limit', 'per_page'):
|
|
raw = value[name]
|
|
if isinstance(raw, bool):
|
|
raise ValueError('DockerHub discovery observation bounds are invalid')
|
|
try:
|
|
numeric[name] = int(raw)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('DockerHub discovery observation bounds are invalid') from None
|
|
if not (
|
|
1 <= numeric['query_count'] <= 1000
|
|
and 0 <= numeric['query_ordinal'] < numeric['query_count']
|
|
and 1 <= numeric['page_number'] <= DISCOVERY_RETRY_MAX_PAGE
|
|
and numeric['page_number'] <= numeric['page_limit'] <= DISCOVERY_RETRY_MAX_PAGE
|
|
and 1 <= numeric['per_page'] <= DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE
|
|
):
|
|
raise ValueError('DockerHub discovery observation bounds are invalid')
|
|
cycle_id = value['cycle_id']
|
|
if cycle_id is not None:
|
|
if isinstance(cycle_id, bool):
|
|
raise ValueError('DockerHub discovery source cycle identity is invalid')
|
|
try:
|
|
cycle_id = int(cycle_id)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('DockerHub discovery source cycle identity is invalid') from None
|
|
if cycle_id < 1:
|
|
raise ValueError('DockerHub discovery source cycle identity is invalid')
|
|
pass_kind = str(value['pass_kind'] or '')
|
|
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
|
|
raise ValueError('DockerHub discovery pass kind is invalid')
|
|
ordered_query_hash = str(value['ordered_query_hash'] or '')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', ordered_query_hash):
|
|
raise ValueError('DockerHub discovery ordered-query hash is invalid')
|
|
if not isinstance(value['query_complete'], bool):
|
|
raise ValueError('DockerHub discovery query completion evidence is invalid')
|
|
total_count = value['total_count']
|
|
if total_count is not None:
|
|
if isinstance(total_count, bool):
|
|
raise ValueError('DockerHub discovery total-count evidence is invalid')
|
|
try:
|
|
total_count = int(total_count)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('DockerHub discovery total-count evidence is invalid') from None
|
|
if not 0 <= total_count <= 0x7fffffffffffffff:
|
|
raise ValueError('DockerHub discovery total-count evidence is invalid')
|
|
if value['query_complete'] and total_count is None:
|
|
raise ValueError('DockerHub terminal discovery evidence lacks a total count')
|
|
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
|
|
collection_generation = str(value['collection_generation'] or '')
|
|
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
|
|
raise ValueError('DockerHub discovery collection generation is invalid')
|
|
return {
|
|
'cycle_id': cycle_id,
|
|
**numeric,
|
|
'policy_sha256': _validated_discovery_retry_policy(value['policy_sha256']),
|
|
'pass_kind': pass_kind,
|
|
'collection_generation': collection_generation,
|
|
'ordered_query_hash': ordered_query_hash,
|
|
'total_count': total_count,
|
|
'query_complete': value['query_complete'],
|
|
'source': source,
|
|
'query': query,
|
|
}
|
|
|
|
|
|
def _normalized_dockerhub_repositories(repositories):
|
|
try:
|
|
offered = list(repositories or ())
|
|
except TypeError:
|
|
raise ValueError('DockerHub discovery repositories must be iterable') from None
|
|
if len(offered) > DISCOVERY_RETRY_MAX_REPOSITORIES_PER_PAGE:
|
|
raise ValueError('DockerHub discovery page exceeds its repository bound')
|
|
normalized = []
|
|
observed = []
|
|
ordinals = {}
|
|
seen = set()
|
|
for ordinal, repository in enumerate(offered, 1):
|
|
value = repository.get('repo_name') if isinstance(repository, dict) else repository
|
|
if not isinstance(value, str) or not value or value != value.strip():
|
|
raise ValueError('DockerHub discovery repository is invalid')
|
|
try:
|
|
bare = validate_docker_image_reference(value, require_digest=False)
|
|
except (TypeError, ValueError):
|
|
raise ValueError('DockerHub discovery repository is invalid') from None
|
|
if '@' in bare or ':' in bare.rsplit('/', 1)[-1]:
|
|
raise ValueError('DockerHub discovery repository must be a bare repository anchor')
|
|
identity = docker_target_identity(bare)
|
|
if not identity:
|
|
raise ValueError('DockerHub discovery repository is invalid')
|
|
observed.append(identity)
|
|
if identity in seen:
|
|
continue
|
|
seen.add(identity)
|
|
normalized.append(identity)
|
|
ordinals[identity] = ordinal
|
|
return len(offered), normalized, ordinals, observed
|
|
|
|
|
|
def extract_package_metadata(target, result, source):
|
|
source = str(source or '').lower()
|
|
package = result.get('package') if isinstance(result, dict) else None
|
|
if not package and source in ('npm', 'pypi', 'package_git'):
|
|
try:
|
|
package = json.loads(str(target).strip())
|
|
except Exception:
|
|
package = {}
|
|
if not isinstance(package, dict):
|
|
package = {}
|
|
return {
|
|
'package_name': package.get('name'),
|
|
'package_version': package.get('version'),
|
|
'package_artifact': package.get('artifact') or package.get('tarball') or package.get('repo_url'),
|
|
'repo_url': package.get('repo_url'),
|
|
'repo_provider': package.get('provider'),
|
|
'package_date': package.get('date'),
|
|
'package_filename': package.get('filename'),
|
|
'package_type': package.get('packagetype') or package.get('type'),
|
|
'package_size': package.get('size'),
|
|
}
|
|
|
|
|
|
def extract_raw_secret(finding):
|
|
for key in ('RawV2', 'Raw'):
|
|
value = finding.get(key)
|
|
if value:
|
|
return str(value)
|
|
structured = finding.get('StructuredData')
|
|
if isinstance(structured, dict):
|
|
for value in structured.values():
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return ''
|
|
|
|
|
|
def extract_redacted_secret(finding):
|
|
value = finding.get('Redacted')
|
|
if value:
|
|
return str(value)
|
|
return '***REDACTED***' if extract_raw_secret(finding) else ''
|
|
|
|
|
|
def extract_finding_location(finding):
|
|
metadata = finding.get('SourceMetadata') or {}
|
|
data = metadata.get('Data') if isinstance(metadata, dict) else {}
|
|
source_type = ''
|
|
details = {}
|
|
if isinstance(data, dict):
|
|
for key, value in data.items():
|
|
if isinstance(value, dict):
|
|
source_type = key
|
|
details = value
|
|
break
|
|
if not isinstance(details, dict):
|
|
details = {}
|
|
return {
|
|
'source_metadata_type': source_type,
|
|
'file_path': details.get('file') or details.get('path') or details.get('File') or '',
|
|
'line_number': str(details.get('line') or details.get('Line') or ''),
|
|
'commit_hash': details.get('commit') or details.get('commitHash') or details.get('commit_hash') or '',
|
|
'source_timestamp': details.get('timestamp') or details.get('Timestamp') or '',
|
|
'source_metadata_json': json_dumps(metadata) if metadata else '',
|
|
}
|
|
|
|
|
|
def finding_identity(source, normalized_target, finding):
|
|
raw_secret = extract_raw_secret(finding)
|
|
secret_hash = sha256_text(raw_secret)
|
|
detector = str(finding.get('DetectorName') or finding.get('DetectorType') or '')
|
|
location = extract_finding_location(finding)
|
|
fallback_hash = sha256_text(json_dumps(finding)) if not secret_hash else ''
|
|
detector_secret_hash = sha256_text('|'.join([detector, secret_hash or fallback_hash]))
|
|
fingerprint = sha256_text('|'.join([
|
|
str(source or ''),
|
|
str(normalized_target or ''),
|
|
detector,
|
|
secret_hash or fallback_hash,
|
|
str(location.get('file_path') or ''),
|
|
str(location.get('line_number') or ''),
|
|
str(location.get('commit_hash') or ''),
|
|
str(bool(finding.get('Verified', False))),
|
|
]))
|
|
return raw_secret, secret_hash, detector_secret_hash, fingerprint, location
|
|
|
|
|
|
def scanner_context(finding):
|
|
context = finding.get('ScannerContext')
|
|
return context if isinstance(context, dict) else {}
|
|
|
|
|
|
def context_text(finding):
|
|
context = scanner_context(finding)
|
|
return str(context.get('nearby') or '')
|
|
|
|
|
|
def find_first(patterns, text):
|
|
for pattern in patterns:
|
|
match = re.search(pattern, text or '', re.IGNORECASE | re.MULTILINE)
|
|
if match:
|
|
return match.group(1)
|
|
return ''
|
|
|
|
|
|
def split_dockerhub_rawv2(rawv2):
|
|
if not rawv2 or ':' not in rawv2:
|
|
return '', ''
|
|
username, token = rawv2.split(':', 1)
|
|
return username, token
|
|
|
|
|
|
def split_azure_openai_rawv2(rawv2):
|
|
rawv2 = rawv2 or ''
|
|
match = re.match(r'^([a-f0-9]{32}):(.+\.openai\.azure\.com)$', rawv2, re.IGNORECASE)
|
|
if not match:
|
|
return '', ''
|
|
return match.group(1), match.group(2)
|
|
|
|
|
|
def split_azure_devops_rawv2(raw, rawv2):
|
|
raw = raw or ''
|
|
rawv2 = rawv2 or ''
|
|
if raw and rawv2.startswith(raw) and len(rawv2) > len(raw):
|
|
return rawv2[len(raw):]
|
|
return ''
|
|
|
|
|
|
def confidence_for(finding, complete=False, legacy=False, missing=False):
|
|
if finding.get('Verified'):
|
|
return 'verified'
|
|
if legacy:
|
|
return 'legacy_unverified'
|
|
if missing:
|
|
return 'token_only_missing_context'
|
|
if complete:
|
|
return 'structured_complete'
|
|
return 'unverified'
|
|
|
|
|
|
def enrich_finding(finding):
|
|
finding = sanitize_postman_finding(finding)
|
|
detector = str(finding.get('DetectorName') or '')
|
|
detector_key = detector.lower()
|
|
raw = str(finding.get('Raw') or '')
|
|
rawv2 = str(finding.get('RawV2') or '')
|
|
extra = finding.get('ExtraData') if isinstance(finding.get('ExtraData'), dict) else {}
|
|
analysis = finding.get('AnalysisInfo') if isinstance(finding.get('AnalysisInfo'), dict) else {}
|
|
text = context_text(finding)
|
|
|
|
out = {
|
|
'provider': '',
|
|
'credential_kind': detector,
|
|
'credential_confidence': confidence_for(finding),
|
|
'required_context_missing': 0,
|
|
'principal': '',
|
|
'username': '',
|
|
'email': '',
|
|
'project_id': '',
|
|
'tenant_id': '',
|
|
'organization': '',
|
|
'registry': '',
|
|
'endpoint': '',
|
|
'scope': '',
|
|
'resource': '',
|
|
'enrichment_json': '',
|
|
}
|
|
|
|
postman_context = finding.get('PostmanContext') if isinstance(finding.get('PostmanContext'), dict) else {}
|
|
if postman_context:
|
|
postman_kind = postman_context.get('credential_kind') or detector
|
|
is_adc = (
|
|
detector_key == 'gcpapplicationdefaultcredentials'
|
|
or str(postman_kind or '').lower() == 'application_default_credentials'
|
|
)
|
|
out.update({
|
|
'provider': postman_context.get('provider') or out['provider'],
|
|
'credential_kind': postman_kind,
|
|
'credential_confidence': postman_context.get('credential_confidence') or confidence_for(finding),
|
|
'endpoint': postman_context.get('endpoint') or postman_context.get('host') or '',
|
|
'resource': '' if is_adc else postman_context.get('json_path') or '',
|
|
'scope': postman_context.get('context_location') or '',
|
|
'username': postman_context.get('variable_name') or '',
|
|
'required_context_missing': 0,
|
|
'enrichment_json': json_dumps(postman_context),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'gcp':
|
|
data = safe_json_loads(rawv2)
|
|
out.update({'provider': 'gcp', 'credential_kind': 'service_account'})
|
|
if isinstance(data, dict):
|
|
out.update({
|
|
'project_id': data.get('project_id') or extra.get('project') or '',
|
|
'principal': data.get('client_email') or analysis.get('principal') or '',
|
|
'username': data.get('client_email') or '',
|
|
'resource': data.get('private_key_id') or '',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
'enrichment_json': json_dumps({
|
|
'type': data.get('type'),
|
|
'client_id': data.get('client_id'),
|
|
'private_key_id': data.get('private_key_id'),
|
|
'client_x509_cert_url': data.get('client_x509_cert_url'),
|
|
}),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'gcpapplicationdefaultcredentials':
|
|
data = safe_json_loads(text)
|
|
if not isinstance(data, dict):
|
|
data = {}
|
|
project_id = data.get('quota_project_id') or data.get('project_id') or extra.get('project') or ''
|
|
out.update({
|
|
'provider': 'gcp',
|
|
'credential_kind': 'application_default_credentials',
|
|
'principal': data.get('client_id') or '',
|
|
'project_id': project_id,
|
|
'resource': '',
|
|
'required_context_missing': 0 if data else 1,
|
|
'credential_confidence': confidence_for(finding, complete=bool(data), missing=not bool(data)),
|
|
'enrichment_json': json_dumps({
|
|
'client_id': data.get('client_id'),
|
|
'type': data.get('type'),
|
|
'project_id': data.get('project_id'),
|
|
'quota_project_id': data.get('quota_project_id'),
|
|
'has_client_secret': bool(data.get('client_secret')),
|
|
'has_refresh_token': bool(data.get('refresh_token')),
|
|
}),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'azurecontainerregistry':
|
|
data = safe_json_loads(rawv2)
|
|
registry = data.get('username') if isinstance(data, dict) else ''
|
|
out.update({
|
|
'provider': 'azure',
|
|
'credential_kind': 'azure_container_registry',
|
|
'registry': registry or find_first([r'([a-z0-9][a-z0-9-]{1,100}[a-z0-9])\.azurecr\.io'], text),
|
|
'endpoint': (registry + '.azurecr.io') if registry else '',
|
|
'username': registry or '',
|
|
'credential_confidence': confidence_for(finding, complete=bool(registry)),
|
|
'required_context_missing': 0 if registry else 1,
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'azureopenai':
|
|
_, endpoint = split_azure_openai_rawv2(rawv2)
|
|
endpoint = endpoint or find_first([r'([a-z0-9-]+\.openai\.azure\.com)'], text)
|
|
out.update({
|
|
'provider': 'azure',
|
|
'credential_kind': 'azure_openai_key',
|
|
'endpoint': endpoint,
|
|
'resource': endpoint.split('.')[0] if endpoint else '',
|
|
'credential_confidence': confidence_for(finding, complete=bool(endpoint), missing=not bool(endpoint)),
|
|
'required_context_missing': 0 if endpoint else 1,
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'azuredevopspersonalaccesstoken':
|
|
org = split_azure_devops_rawv2(raw, rawv2) or find_first([
|
|
r'https?://dev\.azure\.com/([A-Za-z0-9][A-Za-z0-9-]{1,80})',
|
|
r'https?://([A-Za-z0-9][A-Za-z0-9-]{1,80})\.visualstudio\.com',
|
|
], text)
|
|
out.update({
|
|
'provider': 'azure',
|
|
'credential_kind': 'azure_devops_pat',
|
|
'organization': org,
|
|
'endpoint': f'https://dev.azure.com/{org}' if org else '',
|
|
'credential_confidence': confidence_for(finding, complete=bool(org), missing=not bool(org)),
|
|
'required_context_missing': 0 if org else 1,
|
|
})
|
|
return out
|
|
|
|
if detector_key in ('googleai', 'googleaistudio') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() == 'googleaistudio'):
|
|
is_aistudio = detector_key == 'googleaistudio' or str(extra.get('name') or '').lower() == 'googleaistudio'
|
|
out.update({
|
|
'provider': 'google',
|
|
'credential_kind': 'google_ai_studio_api_key' if is_aistudio else 'google_ai_api_key',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key in ('qwendashscope', 'qwen_dashscope', 'qwen', 'dashscope') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() in ('qwendashscope', 'qwen_dashscope')):
|
|
endpoint = find_first([
|
|
r'((?:dashscope(?:-intl|-us)?|cn-hongkong\.dashscope)\.aliyuncs\.com)',
|
|
r'(cn-hongkong\.aliyuncs\.com)',
|
|
], text)
|
|
out.update({
|
|
'provider': 'alibaba',
|
|
'credential_kind': 'qwen_dashscope_api_key',
|
|
'endpoint': endpoint,
|
|
'resource': 'dashscope',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key in ('kimimoonshot', 'moonshotai', 'moonshot', 'kimi') or (detector_key == 'customregex' and str(extra.get('name') or '').lower() in ('kimimoonshot', 'moonshotai')):
|
|
endpoint = find_first([
|
|
r'(api\.moonshot\.ai)',
|
|
r'(api\.moonshot\.cn)',
|
|
r'(platform\.kimi\.(?:ai|com))',
|
|
], text)
|
|
out.update({
|
|
'provider': 'moonshot',
|
|
'credential_kind': 'kimi_moonshot_api_key',
|
|
'endpoint': endpoint,
|
|
'resource': 'kimi',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'zaiglm' or (detector_key == 'customregex' and str(extra.get('name') or '').lower() == 'zaiglm'):
|
|
endpoint = find_first([
|
|
r'(api\.z\.ai)',
|
|
r'(open\.bigmodel\.cn)',
|
|
r'(bigmodel\.cn)',
|
|
], text)
|
|
out.update({
|
|
'provider': 'zai',
|
|
'credential_kind': 'glm_api_key',
|
|
'endpoint': endpoint or 'api.z.ai',
|
|
'resource': 'glm',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'groq':
|
|
out.update({
|
|
'provider': 'groq',
|
|
'credential_kind': 'groq_api_key',
|
|
'endpoint': 'api.groq.com',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'replicate':
|
|
out.update({
|
|
'provider': 'replicate',
|
|
'credential_kind': 'replicate_api_token',
|
|
'endpoint': 'api.replicate.com',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'xai':
|
|
out.update({
|
|
'provider': 'xai',
|
|
'credential_kind': 'xai_api_key',
|
|
'endpoint': 'api.x.ai',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'huggingface':
|
|
out.update({
|
|
'provider': 'huggingface',
|
|
'credential_kind': 'huggingface_user_access_token',
|
|
'endpoint': 'huggingface.co',
|
|
'credential_confidence': confidence_for(finding, complete=True),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'dockerhub':
|
|
username, _ = split_dockerhub_rawv2(rawv2)
|
|
username = username or extra.get('hub_username') or analysis.get('username') or find_first([
|
|
r'(?i)(?:docker(?:hub)?[_-]?)?(?:user|username|usr|login|id)\s*[:=]\s*["\']?([a-zA-Z0-9][a-zA-Z0-9_.-]{2,60})',
|
|
r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})',
|
|
], text)
|
|
out.update({
|
|
'provider': 'dockerhub',
|
|
'credential_kind': 'dockerhub_pat',
|
|
'username': username,
|
|
'email': extra.get('hub_email') or (username if '@' in username else ''),
|
|
'scope': extra.get('hub_scope') or '',
|
|
'credential_confidence': confidence_for(finding, complete=bool(username), missing=not bool(username)),
|
|
'required_context_missing': 0 if username else 1,
|
|
'enrichment_json': json_dumps({'2fa_required': extra.get('2fa_required')}),
|
|
})
|
|
return out
|
|
|
|
if detector_key in ('github', 'githuboauth2'):
|
|
legacy = detector_key == 'githuboauth2' or (raw and not raw.startswith(('ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_')))
|
|
out.update({
|
|
'provider': 'github',
|
|
'credential_kind': 'github_oauth_app' if detector_key == 'githuboauth2' else 'github_token',
|
|
'principal': extra.get('username') or analysis.get('key') or '',
|
|
'scope': extra.get('scopes') or '',
|
|
'credential_confidence': confidence_for(finding, complete=not legacy, legacy=legacy),
|
|
})
|
|
return out
|
|
|
|
if detector_key == 'gitlab':
|
|
legacy = raw and not raw.startswith(('glpat-', 'gloas-', 'glcbt-', 'glimt-', 'glrt-', 'glft-', 'glsoat-'))
|
|
out.update({
|
|
'provider': 'gitlab',
|
|
'credential_kind': 'gitlab_token',
|
|
'principal': analysis.get('key') or '',
|
|
'endpoint': analysis.get('host') or '',
|
|
'credential_confidence': confidence_for(finding, complete=not legacy, legacy=legacy),
|
|
})
|
|
return out
|
|
|
|
return out
|
|
|
|
|
|
def first_error_summary(result):
|
|
for error in result.get('errors', []) or []:
|
|
for line in str(error).splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
payload = json.loads(line)
|
|
for key in ('error', 'msg', 'message'):
|
|
if payload.get(key):
|
|
return str(payload[key])[:500]
|
|
except Exception:
|
|
pass
|
|
return line[:500]
|
|
return ''
|
|
|
|
|
|
def categorize_error(error):
|
|
text = str(error or '').lower()
|
|
if 'secondary rate limit' in text or 'abuse' in text:
|
|
return 'secondary_rate_limit'
|
|
if 'rate limit' in text or 'too many requests' in text or ' 429' in text or '429 ' in text:
|
|
return 'rate_limit'
|
|
if 'auth_invalid' in text or 'authentication failed' in text or 'unauthorized' in text or '401' in text:
|
|
return 'auth_invalid'
|
|
if 'auth_forbidden' in text or 'forbidden' in text or 'permission denied' in text or 'access denied' in text or '403' in text:
|
|
return 'auth_forbidden'
|
|
if 'query_invalid' in text or 'validation failed' in text or '422' in text:
|
|
return 'query_invalid'
|
|
if 'not found' in text or '404' in text:
|
|
return 'not_found'
|
|
if any(item in text for item in ('server error', '500', '502', '503', '504')):
|
|
return 'server_error'
|
|
if any(item in text for item in ('no space left', 'not enough free space', 'disk')):
|
|
return 'disk_space'
|
|
if any(item in text for item in ('timed out', 'timeout', 'deadline exceeded')):
|
|
return 'timeout'
|
|
if any(item in text for item in ('extract', 'tar', 'zip', 'gzip', 'invalid header')):
|
|
return 'extract'
|
|
if any(item in text for item in ('download', 'artifact exceeds', 'fetch')):
|
|
return 'download'
|
|
if any(item in text for item in ('connection', 'dns', 'tls', 'ssl', 'proxy', 'network')):
|
|
return 'network'
|
|
if 'api' in text or 'http' in text:
|
|
return 'api'
|
|
if any(item in text for item in ('trufflehog', 'error processing image', 'failed to clone')):
|
|
return 'trufflehog'
|
|
return 'unknown'
|
|
|
|
|
|
def target_status(result):
|
|
if result.get('errors'):
|
|
return 'error'
|
|
if result.get('skipped'):
|
|
return 'skipped'
|
|
if result.get('findings'):
|
|
return 'found'
|
|
if result.get('degraded') or result.get('warnings'):
|
|
return 'degraded'
|
|
return 'clean'
|
|
|
|
|
|
def summarize_results(results):
|
|
summary = {
|
|
'scanned_count': len(results or []),
|
|
'clean_count': 0,
|
|
'found_count': 0,
|
|
'skipped_count': 0,
|
|
'error_count': 0,
|
|
'degraded_count': 0,
|
|
'findings_count': 0,
|
|
'verified_findings_count': 0,
|
|
'unique_secrets_count': 0,
|
|
'unique_findings_count': 0,
|
|
}
|
|
secret_hashes = set()
|
|
fingerprints = set()
|
|
for result in results or []:
|
|
status = target_status(result)
|
|
summary[f'{status}_count'] += 1
|
|
findings = result.get('findings') or []
|
|
summary['findings_count'] += len(findings)
|
|
summary['verified_findings_count'] += sum(1 for finding in findings if finding.get('Verified', False))
|
|
source = result.get('scan_type') or ''
|
|
normalized = normalize_target(result.get('target', ''), source)
|
|
for finding in findings:
|
|
_, secret_hash, _, fingerprint, _ = finding_identity(source, normalized, finding)
|
|
if secret_hash:
|
|
secret_hashes.add(secret_hash)
|
|
if fingerprint:
|
|
fingerprints.add(fingerprint)
|
|
summary['unique_secrets_count'] = len(secret_hashes)
|
|
summary['unique_findings_count'] = len(fingerprints)
|
|
return summary
|
|
|
|
|
|
def count_file_lines(path):
|
|
if not path or not os.path.exists(path):
|
|
return 0
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
return sum(1 for line in f if line.strip())
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
def queue_counts(todo_file=None, checked_file=None):
|
|
return {
|
|
'todo_count': count_file_lines(todo_file),
|
|
'checked_count': count_file_lines(checked_file),
|
|
'todo_file': todo_file,
|
|
'checked_file': checked_file,
|
|
}
|
|
|
|
|
|
class ScannerDB:
|
|
def __init__(self, results_dir=None, db_path=None, enabled=True, initialize=True, db_url=None):
|
|
explicit_url = get_database_url() if db_url is None else db_url
|
|
if is_postgres_url(db_path) and not explicit_url:
|
|
explicit_url = db_path
|
|
db_path = None
|
|
self.url = explicit_url if is_postgres_url(explicit_url) else None
|
|
self.postgres_required = bool(explicit_url)
|
|
self.path = None if self.url else get_database_path(results_dir, db_path)
|
|
self.db_display = redact_database_url(self.url) if self.url else self.path
|
|
self.conn = None
|
|
self._keycheck_result_columns = None
|
|
self._last_claim_expectation = None
|
|
self._result_spool_publisher_held = False
|
|
self._pipeline_advisory_held = set()
|
|
self._runtime_managed_file_execution_held = None
|
|
self._target_queue_counts_cache = {}
|
|
self._target_queue_counts_retry_after = {}
|
|
self.last_error = ''
|
|
if explicit_url and not self.url:
|
|
logger.error(f'Unsupported database URL scheme: {redact_database_url(explicit_url)}')
|
|
self.path = None
|
|
return
|
|
if not enabled or database_disabled():
|
|
self.path = None
|
|
self.url = None
|
|
self.db_display = None
|
|
return
|
|
try:
|
|
if not self.url:
|
|
parent = os.path.dirname(self.path)
|
|
if parent:
|
|
os.makedirs(parent, exist_ok=True)
|
|
self.conn = self._new_connection(timeout_sec=SQLITE_CONNECT_TIMEOUT_SEC)
|
|
if initialize and self.conn.is_sqlite:
|
|
self.conn.execute('PRAGMA journal_mode=WAL')
|
|
if initialize and self.conn.is_sqlite:
|
|
self.conn.execute('PRAGMA synchronous=NORMAL')
|
|
if initialize and self.conn.is_sqlite:
|
|
self.initialize_schema()
|
|
except Exception as e:
|
|
logger.error(f'Observability DB disabled after init failure: {e}')
|
|
self.conn = None
|
|
|
|
@property
|
|
def enabled(self):
|
|
return self.conn is not None
|
|
|
|
@classmethod
|
|
def host_agent_authority(cls):
|
|
database = cls(enabled=False)
|
|
database.postgres_required = True
|
|
database.db_display = 'fixed host-agent PostgreSQL authority'
|
|
connection = connect_host_agent_postgres()
|
|
try:
|
|
connection.execute("SET application_name = 'truf-host-agent'")
|
|
except BaseException:
|
|
connection.close()
|
|
raise
|
|
database.conn = connection
|
|
return database
|
|
|
|
def _new_connection(self, timeout_sec=None):
|
|
if self.url:
|
|
conn = connect_postgres(self.url)
|
|
conn.execute("SET application_name = 'truf-observability'")
|
|
return conn
|
|
conn = connect_sqlite(self.path, timeout_sec=max(1, int(timeout_sec or SQLITE_CONNECT_TIMEOUT_SEC)))
|
|
conn.execute(f'PRAGMA busy_timeout={SQLITE_BUSY_TIMEOUT_MS}')
|
|
conn.execute('PRAGMA foreign_keys=ON')
|
|
return conn
|
|
|
|
def _reset_connection(self):
|
|
held = self._runtime_managed_file_execution_held
|
|
self._runtime_managed_file_execution_held = None
|
|
connection = self.conn
|
|
self.conn = None
|
|
cancellation = None
|
|
if held is not None and held[2] is not None:
|
|
try:
|
|
held[2].release()
|
|
except RuntimeError:
|
|
pass
|
|
except BaseException as exc:
|
|
cancellation = exc
|
|
try:
|
|
if connection:
|
|
connection.close()
|
|
except BaseException as exc:
|
|
if not isinstance(exc, Exception) and cancellation is None:
|
|
cancellation = exc
|
|
self._result_spool_publisher_held = False
|
|
self._pipeline_advisory_held.clear()
|
|
if cancellation is not None:
|
|
raise cancellation
|
|
try:
|
|
self.conn = self._new_connection()
|
|
self._runtime_safety_schema_validated = False
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f'Observability DB reconnect failed: {e}')
|
|
self.conn = None
|
|
return False
|
|
|
|
def close(self):
|
|
held = self._runtime_managed_file_execution_held
|
|
self._runtime_managed_file_execution_held = None
|
|
connection = self.conn
|
|
self.conn = None
|
|
self._result_spool_publisher_held = False
|
|
self._pipeline_advisory_held.clear()
|
|
failure = None
|
|
if held is not None and held[2] is not None:
|
|
try:
|
|
held[2].release()
|
|
except RuntimeError:
|
|
pass
|
|
except BaseException as exc:
|
|
failure = exc
|
|
try:
|
|
if connection:
|
|
connection.close()
|
|
except BaseException as exc:
|
|
if (
|
|
failure is None
|
|
or isinstance(failure, Exception) and not isinstance(exc, Exception)
|
|
):
|
|
failure = exc
|
|
if failure is not None:
|
|
raise failure
|
|
|
|
def set_application_name(self, value):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return True
|
|
name = str(value or '').strip()
|
|
if not name or any(character in name for character in ('\x00', '\r', '\n')):
|
|
raise ValueError('PostgreSQL application name is invalid')
|
|
self.conn.execute("SELECT set_config('application_name', ?, false)", (name[:63],))
|
|
self.conn.commit()
|
|
return True
|
|
|
|
def acquire_runtime_managed_file_execution(self, operation_id):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if self._runtime_managed_file_execution_held is not None:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime managed file execution lock is already held'
|
|
)
|
|
lock_key = int.from_bytes(
|
|
hashlib.sha256(operation_id.encode('ascii')).digest()[:4],
|
|
'big', signed=True,
|
|
)
|
|
if self.conn.is_postgres:
|
|
self.conn.execute(
|
|
'SELECT pg_catalog.pg_advisory_lock(?, ?)',
|
|
(RUNTIME_MANAGED_FILE_ADVISORY_CLASS, lock_key),
|
|
)
|
|
self.conn.commit()
|
|
lock = None
|
|
else:
|
|
lock = _RUNTIME_MANAGED_FILE_SQLITE_LOCKS[
|
|
lock_key % len(_RUNTIME_MANAGED_FILE_SQLITE_LOCKS)
|
|
]
|
|
lock.acquire()
|
|
self._runtime_managed_file_execution_held = (
|
|
operation_id, lock_key, lock,
|
|
)
|
|
return True
|
|
|
|
def release_runtime_managed_file_execution(self, operation_id):
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
held = self._runtime_managed_file_execution_held
|
|
if held is None or held[0] != operation_id:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime managed file execution lock is not held'
|
|
)
|
|
self._runtime_managed_file_execution_held = None
|
|
if self.conn.is_postgres:
|
|
row = self.conn.execute(
|
|
'''SELECT pg_catalog.pg_advisory_unlock(?, ?) AS released''',
|
|
(RUNTIME_MANAGED_FILE_ADVISORY_CLASS, held[1]),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
if not row or not bool(row['released']):
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file execution lock release failed'
|
|
)
|
|
else:
|
|
held[2].release()
|
|
return True
|
|
|
|
def acquire_pipeline_lease(
|
|
self, worker_name, supervisor_instance_id, owner_identity,
|
|
lease_seconds=30, initial_state='starting',
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('pipeline singleton leases require PostgreSQL')
|
|
worker_name = str(worker_name or '')
|
|
advisory_object = PIPELINE_ADVISORY_OBJECTS.get(worker_name)
|
|
if advisory_object is None:
|
|
raise ValueError('unknown pipeline worker lease name')
|
|
if worker_name in self._pipeline_advisory_held:
|
|
raise RuntimeError(f'pipeline advisory lease is already held: {worker_name}')
|
|
identity = _identity_mapping(owner_identity)
|
|
token = secrets.token_urlsafe(32)
|
|
now = utc_now_iso()
|
|
expires = datetime.fromtimestamp(
|
|
time.time() + max(5, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
acquired = False
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
|
|
(PIPELINE_ADVISORY_CLASS, advisory_object),
|
|
).fetchone()
|
|
acquired = bool(row and row['acquired'])
|
|
if not acquired:
|
|
self.conn.commit()
|
|
return None
|
|
current = self.conn.execute(
|
|
'SELECT generation FROM pipeline_leases WHERE worker_name = ? FOR UPDATE',
|
|
(worker_name,),
|
|
).fetchone()
|
|
generation = int(current['generation'] or 0) + 1 if current else 1
|
|
self.conn.execute(
|
|
'''INSERT INTO pipeline_leases(
|
|
worker_name, generation, lease_token, supervisor_instance_id,
|
|
owner_pid, owner_creation_time, owner_executable, state,
|
|
acquired_at, heartbeat_at, lease_expires_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(worker_name) DO UPDATE SET
|
|
generation = excluded.generation,
|
|
lease_token = excluded.lease_token,
|
|
supervisor_instance_id = excluded.supervisor_instance_id,
|
|
owner_pid = excluded.owner_pid,
|
|
owner_creation_time = excluded.owner_creation_time,
|
|
owner_executable = excluded.owner_executable,
|
|
state = excluded.state,
|
|
acquired_at = excluded.acquired_at,
|
|
heartbeat_at = excluded.heartbeat_at,
|
|
lease_expires_at = excluded.lease_expires_at,
|
|
last_error = NULL,
|
|
updated_at = excluded.updated_at''',
|
|
(
|
|
worker_name, generation, token, str(supervisor_instance_id or ''),
|
|
identity['pid'], identity['creation_time'], identity['executable'],
|
|
initial_state, now, now, expires, now,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
self._pipeline_advisory_held.add(worker_name)
|
|
return {'worker_name': worker_name, 'generation': generation, 'lease_token': token}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
if acquired:
|
|
try:
|
|
self.conn.execute(
|
|
'SELECT pg_advisory_unlock(?, ?)',
|
|
(PIPELINE_ADVISORY_CLASS, advisory_object),
|
|
)
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def heartbeat_pipeline_lease(
|
|
self, worker_name, generation, lease_token, lease_seconds=30,
|
|
state='ready', error='',
|
|
):
|
|
if not self.conn or not self.conn.is_postgres or worker_name not in self._pipeline_advisory_held:
|
|
return False
|
|
now = utc_now_iso()
|
|
expires = datetime.fromtimestamp(
|
|
time.time() + max(5, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
cursor = self.conn.execute(
|
|
'''UPDATE pipeline_leases SET state = ?, heartbeat_at = ?, lease_expires_at = ?,
|
|
last_error = ?, updated_at = ?
|
|
WHERE worker_name = ? AND generation = ? AND lease_token = ?''',
|
|
(
|
|
state, now, expires, first_line(error, 1000) if error else None, now,
|
|
worker_name, int(generation), str(lease_token),
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
|
|
def release_pipeline_lease(self, worker_name, generation, lease_token, state='released', error=''):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return False
|
|
advisory_object = PIPELINE_ADVISORY_OBJECTS.get(worker_name)
|
|
if advisory_object is None or worker_name not in self._pipeline_advisory_held:
|
|
return False
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE pipeline_leases SET state = ?, heartbeat_at = ?, lease_expires_at = NULL,
|
|
lease_token = NULL, last_error = ?, updated_at = ?
|
|
WHERE worker_name = ? AND generation = ? AND lease_token = ?''',
|
|
(
|
|
state, now, first_line(error, 1000) if error else None, now,
|
|
worker_name, int(generation), str(lease_token),
|
|
),
|
|
)
|
|
unlocked = self.conn.execute(
|
|
'SELECT pg_advisory_unlock(?, ?) AS released',
|
|
(PIPELINE_ADVISORY_CLASS, advisory_object),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
if unlocked and unlocked['released']:
|
|
self._pipeline_advisory_held.discard(worker_name)
|
|
return int(cursor.rowcount or 0) == 1 and bool(unlocked and unlocked['released'])
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def pipeline_worker_health(self, worker_name='result_ingester', supervisor_instance_id=None):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return {'healthy': False, 'reason': 'PostgreSQL is unavailable'}
|
|
row = self.conn.execute(
|
|
'''SELECT worker_name, generation, state, lease_expires_at,
|
|
supervisor_instance_id, heartbeat_at, last_error
|
|
FROM pipeline_leases WHERE worker_name = ?''',
|
|
(worker_name,),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
healthy = bool(
|
|
row and row['state'] == 'ready' and row['lease_expires_at']
|
|
and str(row['lease_expires_at']) > utc_now_iso()
|
|
and (
|
|
not supervisor_instance_id
|
|
or str(row['supervisor_instance_id'] or '') == str(supervisor_instance_id)
|
|
)
|
|
)
|
|
return {
|
|
'healthy': healthy,
|
|
'reason': '' if healthy else str((row or {}).get('last_error') if isinstance(row, dict) else '') or 'worker lease is not ready',
|
|
**(dict(row) if row else {}),
|
|
}
|
|
|
|
def try_acquire_result_spool_publisher(self):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return True
|
|
if self._result_spool_publisher_held:
|
|
raise RuntimeError('result-spool publisher lease is already held by this database session')
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
|
|
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
self.last_error = ''
|
|
acquired = bool(row and row['acquired'])
|
|
self._result_spool_publisher_held = acquired
|
|
return acquired
|
|
except Exception:
|
|
connection = self.conn
|
|
self.conn = None
|
|
self._result_spool_publisher_held = False
|
|
try:
|
|
if connection:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
def result_spool_publisher_state(self):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return {
|
|
'status': 'held', 'authenticated': True,
|
|
'application_name': 'local', 'pid': os.getpid(),
|
|
'holder_identity': f'local:{os.getpid()}', 'state': 'active',
|
|
}
|
|
try:
|
|
rows = self.conn.execute(
|
|
'''SELECT a.pid, a.application_name, a.state,
|
|
a.backend_start::text AS backend_start,
|
|
a.backend_type,
|
|
COALESCE(a.client_addr::text, '') AS client_addr,
|
|
(a.usename = CURRENT_USER) AS same_user,
|
|
(a.datname = CURRENT_DATABASE()) AS same_database
|
|
FROM pg_catalog.pg_locks l
|
|
JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
|
|
WHERE l.locktype = 'advisory' AND l.classid = ? AND l.objid = ?
|
|
AND l.objsubid = 2 AND l.granted
|
|
ORDER BY a.pid''',
|
|
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
if not rows:
|
|
return {'status': 'free', 'authenticated': True}
|
|
if len(rows) != 1:
|
|
raise RuntimeError('result-spool advisory lock has ambiguous multiple holders')
|
|
row = rows[0]
|
|
application_name = str(row['application_name'] or '')
|
|
client_addr = str(row['client_addr'] or '')
|
|
try:
|
|
client_ip = ipaddress.ip_interface(client_addr).ip
|
|
except ValueError:
|
|
client_ip = None
|
|
authenticated = bool(
|
|
re.fullmatch(r'truf-source:[a-z0-9_]+', application_name)
|
|
and row['same_user']
|
|
and row['same_database']
|
|
and row['backend_type'] == 'client backend'
|
|
and client_ip in (ipaddress.ip_address('127.0.0.1'), ipaddress.ip_address('::1'))
|
|
and int(row['pid'] or 0) > 0
|
|
and row['backend_start']
|
|
)
|
|
if not authenticated:
|
|
raise RuntimeError('result-spool advisory lock holder is not an authenticated managed source')
|
|
return {
|
|
'status': 'held',
|
|
'authenticated': True,
|
|
'application_name': application_name,
|
|
'pid': int(row['pid']),
|
|
'state': str(row['state'] or ''),
|
|
'holder_identity': f"{int(row['pid'])}:{row['backend_start']}",
|
|
}
|
|
except Exception:
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
def release_result_spool_publisher(self):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return True
|
|
if not self._result_spool_publisher_held:
|
|
return False
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT pg_advisory_unlock(?, ?) AS released',
|
|
(RESULT_SPOOL_ADVISORY_CLASS, RESULT_SPOOL_ADVISORY_OBJECT),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
self.last_error = ''
|
|
released = bool(row and row['released'])
|
|
if released:
|
|
self._result_spool_publisher_held = False
|
|
return True
|
|
connection = self.conn
|
|
self.conn = None
|
|
self._result_spool_publisher_held = False
|
|
try:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
return False
|
|
except Exception:
|
|
connection = self.conn
|
|
self.conn = None
|
|
self._result_spool_publisher_held = False
|
|
try:
|
|
if connection:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
def result_spool_reservation_progress(self, reservations):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result-spool reservation progress requires PostgreSQL')
|
|
reservations = list(reservations or [])
|
|
if not reservations or len(reservations) > 10000:
|
|
raise RuntimeError('result-spool reservation progress input is outside its bound')
|
|
now = time.time()
|
|
claims = []
|
|
earliest_progress_at = None
|
|
for record in reservations:
|
|
try:
|
|
recover_after = float(record['recover_after'])
|
|
expires_at = float(record['expires_at'])
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise RuntimeError('result-spool reservation deadline metadata is invalid') from exc
|
|
if not math.isfinite(recover_after) or not math.isfinite(expires_at) or recover_after < expires_at:
|
|
raise RuntimeError('result-spool reservation deadlines are indeterminate')
|
|
if recover_after <= now:
|
|
raise RuntimeError('result-spool recoverable reservation remained in capacity accounting')
|
|
if recover_after - now > RESULT_SPOOL_PROGRESS_MAX_WAIT_SEC:
|
|
raise RuntimeError('result-spool reservation recovery deadline exceeds its progress bound')
|
|
deadline = min(value for value in (expires_at, recover_after) if value > now) if expires_at > now else recover_after
|
|
earliest_progress_at = deadline if earliest_progress_at is None else min(earliest_progress_at, deadline)
|
|
for claim in record.get('claims') or []:
|
|
if claim.get('queue_id') is None or not claim.get('lease_owner') or not claim.get('lease_token') or not claim.get('claim_batch'):
|
|
raise RuntimeError('result-spool reservation claim ownership is incomplete')
|
|
claims.append((record, claim))
|
|
rows_by_id = {}
|
|
ids = sorted({int(claim['queue_id']) for _, claim in claims})
|
|
for start in range(0, len(ids), 64):
|
|
batch = ids[start:start + 64]
|
|
placeholders = ','.join('?' for _ in batch)
|
|
rows = self.conn.execute(
|
|
f'''SELECT id, status, lease_owner, lease_token, claim_batch, lease_expires_at
|
|
FROM target_queue WHERE id IN ({placeholders})''',
|
|
batch,
|
|
).fetchall()
|
|
rows_by_id.update({int(row['id']): row for row in rows})
|
|
self.conn.commit()
|
|
counts = {'exact_live': 0, 'exact_expired': 0, 'stale_or_reassigned': 0, 'unbound': 0}
|
|
counts['unbound'] = sum(1 for record in reservations if not (record.get('claims') or []))
|
|
for _, claim in claims:
|
|
row = rows_by_id.get(int(claim['queue_id']))
|
|
exact = bool(
|
|
row
|
|
and row['status'] == 'in_progress'
|
|
and row['lease_owner'] == claim['lease_owner']
|
|
and row['lease_token'] == claim['lease_token']
|
|
and row['claim_batch'] == claim['claim_batch']
|
|
)
|
|
if not exact:
|
|
counts['stale_or_reassigned'] += 1
|
|
continue
|
|
lease = parse_time(row['lease_expires_at'])
|
|
if lease and lease.timestamp() > now:
|
|
counts['exact_live'] += 1
|
|
else:
|
|
counts['exact_expired'] += 1
|
|
return {
|
|
'safe_progress': True,
|
|
'counts': counts,
|
|
'earliest_progress_in_sec': max(0, int((earliest_progress_at or now) - now)),
|
|
}
|
|
|
|
def initialize_schema(self):
|
|
if not self.conn:
|
|
return
|
|
self.conn.executescript(SCHEMA_SQL)
|
|
self.conn.execute(
|
|
'''CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_credentials_provider_key
|
|
ON keycheck_credentials(service, provider_key_hash)'''
|
|
)
|
|
now = utc_now_iso()
|
|
_ensure_runtime_operations_authority(self.conn, now)
|
|
self.conn.execute(
|
|
'INSERT INTO pipeline_capacity(id, updated_at) VALUES (1, ?) ON CONFLICT(id) DO NOTHING',
|
|
(now,),
|
|
)
|
|
for stream_name, relative_path, rotation_bytes in (
|
|
('scan_results', 'scan_results.jsonl', 256 * 1024 * 1024),
|
|
('found_secrets', 'found_secrets.jsonl', 128 * 1024 * 1024),
|
|
('scan_errors', 'scan_errors.log', 32 * 1024 * 1024),
|
|
):
|
|
self.conn.execute(
|
|
'''INSERT INTO projection_streams(
|
|
stream_name, base_relative_path, current_generation,
|
|
rotation_bytes, max_generations, created_at, updated_at
|
|
) VALUES (?, ?, 0, ?, 16, ?, ?)
|
|
ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, relative_path, rotation_bytes, now, now),
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO projection_cursors(stream_name, generation, committed_offset, updated_at)
|
|
VALUES (?, 0, 0, ?) ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, now),
|
|
)
|
|
migration_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
|
|
for version in PIPELINE_MIGRATION_VERSIONS:
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_schema_migrations(version, applied_at, code_sha256)
|
|
VALUES (?, ?, ?) ON CONFLICT(version) DO NOTHING''',
|
|
(version, now, migration_code),
|
|
)
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('PRAGMA user_version=1')
|
|
self.conn.commit()
|
|
|
|
def _safe(self, label, func, default=None):
|
|
if not self.conn:
|
|
return default
|
|
last_error = None
|
|
for attempt in range(SQLITE_LOCK_RETRY_ATTEMPTS):
|
|
try:
|
|
result = func()
|
|
self.last_error = ''
|
|
if self.conn and self.conn.is_postgres:
|
|
try:
|
|
self.conn.commit()
|
|
except Exception:
|
|
pass
|
|
return result
|
|
except Exception as e:
|
|
last_error = e
|
|
self.last_error = str(e)
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
if isinstance(e, DiscoveryPausedError):
|
|
raise
|
|
if not sqlite_lock_error(e) or attempt == SQLITE_LOCK_RETRY_ATTEMPTS - 1:
|
|
if sqlite_lock_error(e):
|
|
self._reset_connection()
|
|
else:
|
|
logger.error(f'Observability DB write failed during {label}: {e}')
|
|
return default
|
|
break
|
|
if attempt and attempt % 4 == 0:
|
|
self._reset_connection()
|
|
if not self.conn:
|
|
return default
|
|
delay = sqlite_lock_retry_delay(attempt)
|
|
logger.warning(f'Observability DB locked during {label}; retrying in {delay:.2f}s ({attempt + 1}/{SQLITE_LOCK_RETRY_ATTEMPTS})')
|
|
time.sleep(delay)
|
|
logger.error(f'Observability DB write failed during {label}: {last_error}')
|
|
return default
|
|
|
|
def _finalize_stale_source_runs(self, source, timestamp):
|
|
source = str(source or '').strip()
|
|
if not source:
|
|
return 0, 0
|
|
reason = 'superseded by a new supervised source process'
|
|
totals = []
|
|
statements = (
|
|
(
|
|
'''UPDATE source_cycles SET ended_at = ?, status = 'interrupted',
|
|
message = COALESCE(NULLIF(message, ''), ?)
|
|
WHERE id IN (
|
|
SELECT id FROM source_cycles
|
|
WHERE source = ? AND status = 'running'
|
|
ORDER BY id LIMIT ?
|
|
)''',
|
|
(timestamp, reason, source, STALE_RUN_FINALIZE_BATCH_SIZE),
|
|
'source_cycles',
|
|
'source',
|
|
),
|
|
(
|
|
'''UPDATE runs SET ended_at = ?, status = 'interrupted',
|
|
error = COALESCE(NULLIF(error, ''), ?), updated_at = ?
|
|
WHERE id IN (
|
|
SELECT id FROM runs
|
|
WHERE selected_source = ? AND status = 'running'
|
|
ORDER BY id LIMIT ?
|
|
)''',
|
|
(timestamp, reason, timestamp, source, STALE_RUN_FINALIZE_BATCH_SIZE),
|
|
'runs',
|
|
'selected_source',
|
|
),
|
|
)
|
|
for statement, params, table, source_column in statements:
|
|
finalized = 0
|
|
for _ in range(STALE_RUN_FINALIZE_MAX_BATCHES):
|
|
cursor = self.conn.execute(statement, params)
|
|
changed = max(0, int(getattr(cursor, 'rowcount', 0) or 0))
|
|
finalized += changed
|
|
self.conn.commit()
|
|
if changed < STALE_RUN_FINALIZE_BATCH_SIZE:
|
|
break
|
|
remaining = self.conn.execute(
|
|
f'''SELECT 1 FROM {table}
|
|
WHERE {source_column} = ? AND status = 'running' LIMIT 1''',
|
|
(source,),
|
|
).fetchone()
|
|
if remaining:
|
|
raise RuntimeError(
|
|
f'bounded stale-run finalization limit reached for {table}:{source}; retry source start'
|
|
)
|
|
totals.append(finalized)
|
|
return tuple(totals)
|
|
|
|
def start_run(self, invocation_mode, argv=None, selected_source=None, selected_platform=None, config_path=None, config_hash=None, enabled_sources=None, global_config=None):
|
|
def op():
|
|
now = utc_now_iso()
|
|
if selected_source:
|
|
self._finalize_stale_source_runs(selected_source, now)
|
|
safe_argv = redact_argv(argv)
|
|
command_line = ' '.join(safe_argv)
|
|
run_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO runs (
|
|
started_at, status, invocation_mode, command_line, argv_json,
|
|
selected_source, selected_platform, config_path, config_hash,
|
|
enabled_sources_json, db_path, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
now, 'running', invocation_mode, command_line, json_dumps(safe_argv),
|
|
selected_source, selected_platform, config_path, config_hash,
|
|
json_dumps(enabled_sources or []), self.db_display, now, now,
|
|
),
|
|
)
|
|
if global_config is not None:
|
|
self.record_config_snapshot(run_id, None, 'global', None, global_config)
|
|
self.conn.commit()
|
|
return run_id
|
|
return self._safe('start_run', op)
|
|
|
|
def finish_run(self, run_id, status='completed', error=None):
|
|
if not run_id:
|
|
return
|
|
def op():
|
|
now = utc_now_iso()
|
|
row = self.conn.execute('SELECT started_at FROM runs WHERE id = ?', (run_id,)).fetchone()
|
|
duration = elapsed_seconds(row['started_at'], now) if row else None
|
|
totals = self.conn.execute(
|
|
'''SELECT
|
|
COALESCE(SUM(fetched_count), 0) AS fetched,
|
|
COALESCE(SUM(queued_new_count), 0) AS queued_new,
|
|
COALESCE(SUM(scan_requested_count), 0) AS scan_requested,
|
|
COALESCE(SUM(scanned_count), 0) AS scanned,
|
|
COALESCE(SUM(clean_count), 0) AS clean,
|
|
COALESCE(SUM(found_count), 0) AS found,
|
|
COALESCE(SUM(skipped_count), 0) AS skipped,
|
|
COALESCE(SUM(error_count), 0) AS errors,
|
|
COALESCE(SUM(findings_count), 0) AS findings,
|
|
COALESCE(SUM(verified_findings_count), 0) AS verified,
|
|
COALESCE(SUM(unique_secrets_count), 0) AS unique_secrets,
|
|
COALESCE(SUM(unique_findings_count), 0) AS unique_findings
|
|
FROM source_cycles WHERE run_id = ?''',
|
|
(run_id,),
|
|
).fetchone()
|
|
self.conn.execute(
|
|
'''UPDATE runs SET ended_at = ?, duration_sec = ?, status = ?, error = ?,
|
|
total_fetched = ?, total_queued_new = ?, total_scan_requested = ?, total_scanned = ?,
|
|
total_clean = ?, total_found = ?, total_skipped = ?, total_errors = ?,
|
|
total_findings = ?, total_verified_findings = ?, total_unique_secrets = ?,
|
|
total_unique_findings = ?, updated_at = ? WHERE id = ?''',
|
|
(
|
|
now, duration, status, error,
|
|
totals['fetched'], totals['queued_new'], totals['scan_requested'], totals['scanned'],
|
|
totals['clean'], totals['found'], totals['skipped'], totals['errors'],
|
|
totals['findings'], totals['verified'], totals['unique_secrets'], totals['unique_findings'], now, run_id,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
self._safe('finish_run', op)
|
|
|
|
def start_source_cycle(self, run_id, source, platform, mode, query, query_index=None, query_count=None, auth_name=None, source_config=None, queue_before=None):
|
|
def op():
|
|
now = utc_now_iso()
|
|
queue_data = queue_before or {}
|
|
cycle_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO source_cycles (
|
|
run_id, source, platform, mode, query, query_index, query_count, auth_name,
|
|
started_at, status, config_json, queue_todo_before, queue_checked_before,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
run_id, source, platform, mode, query, query_index, query_count, auth_name,
|
|
now, 'running', json_dumps(redact_config(source_config or {})),
|
|
queue_data.get('todo_count'), queue_data.get('checked_count'), now, now,
|
|
),
|
|
)
|
|
self.record_config_snapshot(run_id, cycle_id, 'source', source, source_config or {})
|
|
self.record_queue_snapshot(run_id, cycle_id, source, 'before', queue_data)
|
|
self.conn.commit()
|
|
return cycle_id
|
|
return self._safe('start_source_cycle', op)
|
|
|
|
def finish_source_cycle(self, cycle_id, status='completed', metrics=None, queue_after=None, message=None):
|
|
if not cycle_id:
|
|
return
|
|
def op():
|
|
now = utc_now_iso()
|
|
row = self.conn.execute('SELECT started_at, run_id, source FROM source_cycles WHERE id = ?', (cycle_id,)).fetchone()
|
|
duration = elapsed_seconds(row['started_at'], now) if row else None
|
|
metrics_data = metrics or {}
|
|
queue_data = queue_after or {}
|
|
if metrics_data.get('authoritative_async'):
|
|
staged = int(metrics_data.get('staged_count') or 0)
|
|
self.conn.execute(
|
|
'''UPDATE source_cycles SET ended_at = ?, duration_sec = ?, status = ?, message = ?,
|
|
fetched_count = ?, queued_new_count = ?, queued_updated_count = ?,
|
|
scan_requested_count = ?,
|
|
staged_count = staged_count + ?, queue_todo_after = ?,
|
|
queue_checked_after = ?, updated_at = ? WHERE id = ?''',
|
|
(
|
|
now, duration, status, message,
|
|
metrics_data.get('fetched_count', 0),
|
|
metrics_data.get('queued_new_count', 0),
|
|
metrics_data.get('queued_updated_count', 0),
|
|
metrics_data.get('scan_requested_count', 0), staged,
|
|
queue_data.get('todo_count'), queue_data.get('checked_count'), now, cycle_id,
|
|
),
|
|
)
|
|
if row:
|
|
self.conn.execute(
|
|
'UPDATE runs SET total_staged = total_staged + ?, updated_at = ? WHERE id = ?',
|
|
(staged, now, row['run_id']),
|
|
)
|
|
self.record_queue_snapshot(row['run_id'], cycle_id, row['source'], 'after', queue_data)
|
|
self.conn.commit()
|
|
return
|
|
scanned = int(metrics_data.get('scanned_count') or metrics_data.get('scanned') or 0)
|
|
found = int(metrics_data.get('found_count') or 0)
|
|
errors = int(metrics_data.get('error_count') or 0)
|
|
verified = int(metrics_data.get('verified_findings_count') or 0)
|
|
targets_per_hour = scanned / (duration / 3600) if duration and duration > 0 else 0
|
|
hit_rate = found / scanned if scanned else 0
|
|
verified_hit_rate = verified / scanned if scanned else 0
|
|
error_rate = errors / scanned if scanned else 0
|
|
self.conn.execute(
|
|
'''UPDATE source_cycles SET ended_at = ?, duration_sec = ?, status = ?, message = ?,
|
|
fetched_count = ?, queued_new_count = ?, queued_updated_count = ?,
|
|
scan_requested_count = ?, scanned_count = ?,
|
|
clean_count = ?, found_count = ?, skipped_count = ?, error_count = ?,
|
|
findings_count = ?, verified_findings_count = ?, unique_secrets_count = ?, unique_findings_count = ?,
|
|
queue_todo_after = ?, queue_checked_after = ?, targets_per_hour = ?, hit_rate = ?,
|
|
verified_hit_rate = ?, error_rate = ?, updated_at = ? WHERE id = ?''',
|
|
(
|
|
now, duration, status, message,
|
|
metrics_data.get('fetched_count', 0), metrics_data.get('queued_new_count', 0),
|
|
metrics_data.get('queued_updated_count', 0), metrics_data.get('scan_requested_count', 0), scanned,
|
|
metrics_data.get('clean_count', 0), found, metrics_data.get('skipped_count', 0), errors,
|
|
metrics_data.get('findings_count', 0), verified, metrics_data.get('unique_secrets_count', 0), metrics_data.get('unique_findings_count', 0),
|
|
queue_data.get('todo_count'), queue_data.get('checked_count'), targets_per_hour, hit_rate,
|
|
verified_hit_rate, error_rate, now, cycle_id,
|
|
),
|
|
)
|
|
if row:
|
|
self.record_queue_snapshot(row['run_id'], cycle_id, row['source'], 'after', queue_data)
|
|
self.conn.commit()
|
|
self._safe('finish_source_cycle', op)
|
|
|
|
def record_config_snapshot(self, run_id, cycle_id, scope, source, config):
|
|
if not self.conn or run_id is None:
|
|
return
|
|
self.conn.execute(
|
|
'INSERT INTO config_snapshots (run_id, cycle_id, scope, source, config_json, captured_at) VALUES (?, ?, ?, ?, ?, ?)',
|
|
(run_id, cycle_id, scope, source, json_dumps(redact_config(config or {})), utc_now_iso()),
|
|
)
|
|
|
|
def record_queue_snapshot(self, run_id, cycle_id, source, phase, counts):
|
|
if not self.conn or run_id is None:
|
|
return
|
|
counts = counts or {}
|
|
self.conn.execute(
|
|
'''INSERT INTO queue_snapshots (
|
|
run_id, cycle_id, source, phase, todo_count, checked_count, todo_file, checked_file, captured_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
run_id, cycle_id, source, phase, counts.get('todo_count'), counts.get('checked_count'),
|
|
counts.get('todo_file'), counts.get('checked_file'), utc_now_iso(),
|
|
),
|
|
)
|
|
|
|
def target_queue_available(self):
|
|
if not self.conn or not self.conn.table_exists('target_queue'):
|
|
return False
|
|
required = {
|
|
'id', 'source', 'platform', 'query', 'target', 'normalized_target', 'status', 'attempts',
|
|
'lease_owner', 'lease_token', 'claim_batch', 'leased_at', 'lease_expires_at', 'available_after', 'target_scan_id',
|
|
'last_error', 'created_at', 'updated_at', 'completed_at',
|
|
}
|
|
return required.issubset(set(self.conn.table_columns('target_queue')))
|
|
|
|
def runtime_safety_schema_available(self, commit=True):
|
|
if not self.conn:
|
|
self.last_error = 'database connection is unavailable'
|
|
return False
|
|
required = {table: set(specs) for table, specs in RUNTIME_TABLE_SPECS.items()}
|
|
try:
|
|
missing = []
|
|
for table, columns in required.items():
|
|
if not self.conn.table_exists(table):
|
|
missing.append(f'table {table}')
|
|
continue
|
|
absent = sorted(columns - set(self.conn.table_columns(table)))
|
|
if absent:
|
|
missing.append(f'{table} columns {", ".join(absent)}')
|
|
table_details = {}
|
|
for table, specs in RUNTIME_TABLE_SPECS.items():
|
|
if not self.conn.table_exists(table):
|
|
continue
|
|
details = self.conn.table_column_details(table)
|
|
table_details[table] = details
|
|
for name, (expected_type, expected_not_null) in specs.items():
|
|
actual = details.get(name)
|
|
if not actual:
|
|
continue
|
|
if not _schema_type_matches(actual['type'], expected_type, self.conn.is_postgres):
|
|
missing.append(f'{table}.{name} type {actual["type"] or "<empty>"}')
|
|
if bool(actual['not_null']) != bool(expected_not_null):
|
|
missing.append(f'{table}.{name} nullability')
|
|
primary_key = [name for name in specs if details.get(name, {}).get('primary_key')]
|
|
if primary_key != RUNTIME_PRIMARY_KEYS[table]:
|
|
missing.append(f'{table} primary key')
|
|
generated_id = GENERATED_ID_COLUMNS.get(table)
|
|
if generated_id and not _column_generates_id(details.get(generated_id), self.conn.is_postgres):
|
|
missing.append(f'{table}.{generated_id} generated ID identity/sequence')
|
|
for name, actual in details.items():
|
|
if name != generated_id and (actual.get('identity') or actual.get('generated')):
|
|
missing.append(f'{table}.{name} unexpected generated expression')
|
|
expected_defaults = RUNTIME_COLUMN_DEFAULTS.get(table, {})
|
|
for name in specs:
|
|
if name == generated_id:
|
|
continue
|
|
actual = details.get(name)
|
|
expected_present = name in expected_defaults
|
|
expected_default = expected_defaults.get(name, '')
|
|
if actual and (
|
|
bool(actual.get('has_default', bool(actual.get('default')))) != expected_present
|
|
or (
|
|
expected_present
|
|
and _normalized_default(actual['default']) != _normalized_default(expected_default)
|
|
)
|
|
):
|
|
missing.append(f'{table}.{name} default')
|
|
|
|
if self.conn.is_postgres:
|
|
primary_indexes = [
|
|
index for index in self.conn.table_indexes(table).values()
|
|
if index.get('primary')
|
|
]
|
|
if len(primary_indexes) != 1 or primary_indexes[0]['columns'] != RUNTIME_PRIMARY_KEYS[table] or not _index_usable(primary_indexes[0]):
|
|
missing.append(f'{table} primary-key index validity/readiness')
|
|
|
|
queue_indexes = self.conn.table_indexes('target_queue') if self.conn.table_exists('target_queue') else {}
|
|
if not any(
|
|
index['unique'] and index['columns'] == ['source', 'normalized_target']
|
|
and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in queue_indexes.values()
|
|
):
|
|
missing.append('unique target_queue source/normalized_target index')
|
|
|
|
retry_indexes = (
|
|
self.conn.table_indexes('discovery_retry_queue')
|
|
if self.conn.table_exists('discovery_retry_queue') else {}
|
|
)
|
|
work_key_index = retry_indexes.get('uq_discovery_retry_queue_work_key')
|
|
if (
|
|
not work_key_index
|
|
or not work_key_index['unique']
|
|
or work_key_index['columns'] != ['work_key']
|
|
or _normalized_predicate(work_key_index['predicate'])
|
|
or not _index_usable(work_key_index)
|
|
):
|
|
missing.append('unique discovery retry work-key index')
|
|
|
|
scan_indexes = self.conn.table_indexes('target_scans') if self.conn.table_exists('target_scans') else {}
|
|
event_index = scan_indexes.get('uq_target_scans_scan_event_id')
|
|
if (
|
|
not event_index
|
|
or not event_index['unique']
|
|
or event_index['columns'] != ['scan_event_id']
|
|
or _normalized_predicate(event_index['predicate']) != 'scan_event_idisnotnull'
|
|
or not _index_usable(event_index)
|
|
):
|
|
missing.append('unique partial scan-event index')
|
|
|
|
for table, name, columns in (
|
|
(
|
|
'worker_progress_events', 'uq_worker_progress_reservation_sequence',
|
|
['reservation_id', 'sequence'],
|
|
),
|
|
('worker_diagnostics', 'uq_worker_diagnostics_uid', ['diagnostic_uid']),
|
|
):
|
|
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
|
|
index = indexes.get(name)
|
|
if (
|
|
not index or not index['unique'] or index['columns'] != columns
|
|
or _normalized_predicate(index['predicate']) or not _index_usable(index)
|
|
):
|
|
missing.append(f'unique index {name}')
|
|
|
|
outbox_indexes = self.conn.table_indexes('scan_publication_outbox') if self.conn.table_exists('scan_publication_outbox') else {}
|
|
if not any(
|
|
index['unique'] and index['columns'] == ['target_scan_id'] and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in outbox_indexes.values()
|
|
):
|
|
missing.append('unique outbox target-scan index')
|
|
status_index = outbox_indexes.get('idx_scan_publication_outbox_status')
|
|
if not status_index or status_index['columns'] != ['status', 'available_after', 'id'] or status_index['unique'] or not _index_usable(status_index):
|
|
missing.append('outbox status index')
|
|
age_index = outbox_indexes.get('idx_scan_publication_outbox_age')
|
|
if not age_index or age_index['columns'] != ['status', 'created_at', 'id'] or age_index['unique'] or not _index_usable(age_index):
|
|
missing.append('outbox age index')
|
|
|
|
for table, column in (('keycheck_event_map', 'event_id'), ('finding_uid_map', 'finding_uid')):
|
|
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
|
|
if not any(
|
|
index['unique'] and index['columns'] == [column] and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in indexes.values()
|
|
):
|
|
missing.append(f'unique {table}.{column} index')
|
|
|
|
issue_indexes = self.conn.table_indexes('target_queue_reconciliation_issues') if self.conn.table_exists('target_queue_reconciliation_issues') else {}
|
|
if not any(
|
|
index['unique']
|
|
and index['columns'] == ['source_file', 'file_identity', 'line_number', 'byte_offset', 'reason']
|
|
and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in issue_indexes.values()
|
|
):
|
|
missing.append('unique reconciliation issue identity index')
|
|
|
|
package_indexes = self.conn.table_indexes('package_repo_candidates') if self.conn.table_exists('package_repo_candidates') else {}
|
|
if not any(
|
|
index['unique']
|
|
and index['columns'] == ['package_source', 'package_name', 'package_version', 'repo_url']
|
|
and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in package_indexes.values()
|
|
):
|
|
missing.append('unique package repository candidate identity index')
|
|
|
|
required_indexes = {
|
|
'runs': {
|
|
'idx_runs_started_at': ['started_at'],
|
|
'idx_runs_status': ['status'],
|
|
'idx_runs_selected_source_status': ['selected_source', 'status', 'id'],
|
|
},
|
|
'source_cycles': {
|
|
'idx_source_cycles_run_id': ['run_id'],
|
|
'idx_source_cycles_source_query': ['source', 'query'],
|
|
'idx_source_cycles_source_status': ['source', 'status', 'id'],
|
|
},
|
|
'target_queue': {
|
|
'idx_target_queue_source_status': ['source', 'status', 'updated_at'],
|
|
'idx_target_queue_observe_source_status': ['source', 'status'],
|
|
'idx_target_queue_lease': ['source', 'status', 'lease_expires_at'],
|
|
'idx_target_queue_platform_status': ['platform', 'status'],
|
|
'idx_target_queue_claim_batch': ['claim_batch', 'lease_owner'],
|
|
'idx_target_queue_claim': ['source', 'platform', 'status', 'available_after', 'lease_expires_at', 'id'],
|
|
'idx_target_queue_resolver_claim': ['source', 'platform', 'status', 'resolver_state', 'resolver_due_at', 'id'],
|
|
'idx_target_queue_source_platform_normalized': ['source', 'platform', 'normalized_target'],
|
|
'idx_target_queue_claim_pending': ['source', 'platform', 'id', 'attempts', 'available_after'],
|
|
'idx_target_queue_claim_deferred': ['source', 'platform', 'available_after', 'id', 'attempts'],
|
|
'idx_target_queue_claim_in_progress': ['source', 'platform', 'id', 'attempts', 'available_after', 'lease_expires_at', 'resolver_state'],
|
|
'idx_target_queue_active_lease_owner_token': ['lease_owner', 'lease_token'],
|
|
'idx_target_queue_exhausted_attempts': ['source', 'platform', 'status', 'attempts', 'id', 'lease_expires_at'],
|
|
'idx_target_queue_updated_rescan': ['source', 'platform', 'completed_at', 'remote_modified_at', 'scan_remote_modified_at', 'id'],
|
|
'idx_target_queue_cold': ['source', 'platform', 'query', 'id'],
|
|
},
|
|
'discovery_retry_queue': {
|
|
'idx_discovery_retry_queue_due': ['source', 'available_after', 'id'],
|
|
'idx_discovery_retry_queue_lease': ['lease_expires_at', 'id'],
|
|
'idx_discovery_retry_queue_policy': [
|
|
'source', 'query', 'policy_sha256', 'status', 'id',
|
|
],
|
|
},
|
|
'target_queue_policy_events': {
|
|
'idx_target_queue_policy_events_queue': ['queue_id', 'id'],
|
|
'idx_target_queue_policy_events_manifest': ['manifest_sha256', 'id'],
|
|
},
|
|
'target_scans': {
|
|
'idx_target_scans_cycle_id': ['cycle_id'],
|
|
'idx_target_scans_queue_id': ['queue_id'],
|
|
'idx_target_scans_source_status': ['source', 'status'],
|
|
'idx_target_scans_source_ended': ['source', 'ended_at'],
|
|
'idx_target_scans_source_ended_id': ['source', 'ended_at', 'id'],
|
|
'idx_target_scans_source_skip_ended': ['source', 'skipped_reason', 'ended_at'],
|
|
'idx_target_scans_cooldown_recent': ['source', 'ended_at', 'id'],
|
|
'idx_target_scans_normalized_target': ['normalized_target'],
|
|
'idx_target_scans_result_reservation': ['result_reservation_id', 'id'],
|
|
},
|
|
'keycheck_results': {
|
|
'idx_keycheck_results_checked': ['checked_at'],
|
|
'idx_keycheck_results_service_status': ['service', 'status_group', 'status'],
|
|
'idx_keycheck_results_finding': ['finding_id'],
|
|
'idx_keycheck_results_cycle': ['cycle_id'],
|
|
'idx_keycheck_results_source_query': ['source', 'query'],
|
|
'idx_keycheck_results_key_hash': ['key_hash'],
|
|
'idx_keycheck_results_secret_hash': ['secret_hash'],
|
|
'idx_keycheck_results_link_repair': ['link_status', 'id'],
|
|
},
|
|
'findings': {
|
|
'idx_findings_target_scan_id_id': ['target_scan_id', 'id'],
|
|
'idx_findings_cycle_id': ['cycle_id'],
|
|
'idx_findings_source': ['source'],
|
|
'idx_findings_secret_hash': ['secret_hash'],
|
|
'idx_findings_finding_uid': ['finding_uid'],
|
|
},
|
|
'errors': {
|
|
'idx_errors_cycle_id': ['cycle_id'],
|
|
'idx_errors_source_category': ['source', 'category'],
|
|
'idx_errors_target_scan_id': ['target_scan_id'],
|
|
},
|
|
'queue_snapshots': {
|
|
'idx_queue_snapshots_source_time': ['source', 'captured_at'],
|
|
},
|
|
'package_repo_candidates': {
|
|
'idx_package_repo_candidates_query': ['query'],
|
|
'idx_package_repo_candidates_repo': ['repo_url'],
|
|
'idx_package_repo_candidates_source_seen': ['package_source', 'last_seen_at'],
|
|
'idx_package_repo_candidates_query_seen': ['query', 'last_seen_at', 'id'],
|
|
'idx_package_repo_candidates_recent_lookup': ['last_seen_at', 'id', 'package_source', 'query', 'package_name'],
|
|
},
|
|
'target_queue_reconciliation_issues': {
|
|
'idx_reconciliation_issues_open': ['source_file', 'resolved_at', 'id'],
|
|
},
|
|
'docker_content_blobs': {
|
|
'idx_docker_content_blobs_reclaim': [
|
|
'state', 'available_after', 'lease_expires_at', 'digest',
|
|
],
|
|
'idx_docker_content_blobs_reservation': ['lease_reservation_id', 'digest'],
|
|
},
|
|
'docker_image_blob_coverage': {
|
|
'idx_docker_image_blob_coverage_manifest': [
|
|
'manifest_digest', 'coverage_state', 'position',
|
|
],
|
|
'idx_docker_image_blob_coverage_blob': [
|
|
'blob_digest', 'coverage_state', 'queue_id',
|
|
],
|
|
'idx_docker_image_blob_coverage_reservation': ['reservation_id', 'position'],
|
|
'idx_docker_image_blob_coverage_selection': [
|
|
'queue_id', 'manifest_digest', 'selection_policy_sha256',
|
|
'position', 'reservation_id',
|
|
],
|
|
},
|
|
'docker_adaptive_shadow_reports': {
|
|
'idx_docker_adaptive_shadow_reports_gate': [
|
|
'scan_policy_sha256', 'execution_policy_sha256',
|
|
'selection_policy_sha256', 'state', 'completed_at', 'id',
|
|
],
|
|
},
|
|
'runtime_operations': {
|
|
'idx_runtime_operations_status_updated': [
|
|
'status', 'updated_at', 'operation_id',
|
|
],
|
|
'idx_runtime_operations_agent_state_updated': [
|
|
'agent_state', 'updated_at', 'operation_id',
|
|
],
|
|
},
|
|
'runtime_audit_events': {
|
|
'idx_runtime_audit_events_created': ['created_at', 'id'],
|
|
'idx_runtime_audit_events_operation': ['operation_id', 'id'],
|
|
},
|
|
'worker_progress_events': {
|
|
'idx_worker_progress_reservation_received': [
|
|
'reservation_id', 'received_at', 'id',
|
|
],
|
|
'idx_worker_progress_device_received': [
|
|
'remote_device_id', 'received_at', 'id',
|
|
],
|
|
'idx_worker_progress_phase_received': ['phase', 'received_at', 'id'],
|
|
},
|
|
'result_reservations': {
|
|
'idx_result_reservations_remote_resolved': [
|
|
'remote_resolved_at', 'id',
|
|
],
|
|
},
|
|
'worker_diagnostics': {
|
|
'idx_worker_diagnostics_reservation_received': [
|
|
'reservation_id', 'received_at', 'id',
|
|
],
|
|
'idx_worker_diagnostics_scan_received': [
|
|
'target_scan_id', 'received_at', 'id',
|
|
],
|
|
'idx_worker_diagnostics_phase_received': ['phase', 'received_at', 'id'],
|
|
'idx_worker_diagnostics_category_code': [
|
|
'category', 'code', 'received_at', 'id',
|
|
],
|
|
'idx_worker_diagnostics_code_received': ['code', 'received_at', 'id'],
|
|
'idx_worker_diagnostics_kind_received': ['kind', 'received_at', 'id'],
|
|
'idx_worker_diagnostics_retryable_received': [
|
|
'retryable', 'received_at', 'id',
|
|
],
|
|
},
|
|
}
|
|
for table, expected_indexes in required_indexes.items():
|
|
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
|
|
for name, columns in expected_indexes.items():
|
|
index = indexes.get(name)
|
|
if not index or index['columns'] != columns or index['unique'] or not _index_usable(index):
|
|
missing.append(f'index {name}')
|
|
control = self.conn.execute(
|
|
'SELECT id FROM runtime_operations_control WHERE id = 1'
|
|
).fetchone() if self.conn.table_exists('runtime_operations_control') else None
|
|
if not control:
|
|
missing.append('runtime_operations_control singleton row')
|
|
missing.extend(_runtime_audit_trigger_problems(self.conn))
|
|
experiment_indexes = {}
|
|
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
|
|
indexes = experiment_indexes.setdefault(
|
|
table,
|
|
self.conn.table_indexes(table) if self.conn.table_exists(table) else {},
|
|
)
|
|
index = indexes.get(name)
|
|
if (
|
|
not index
|
|
or index['columns'] != list(columns)
|
|
or bool(index['unique']) != bool(unique)
|
|
or _normalized_predicate(index['predicate']) != _normalized_predicate(predicate)
|
|
or not _index_usable(index)
|
|
):
|
|
missing.append(f'index {name}')
|
|
queue_indexes = self.conn.table_indexes('target_queue') if self.conn.table_exists('target_queue') else {}
|
|
for name, predicate in (
|
|
('idx_target_queue_claim_pending', "status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
|
|
('idx_target_queue_claim_deferred', "status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
|
|
('idx_target_queue_claim_in_progress', "status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved')"),
|
|
('idx_target_queue_active_lease_owner_token', "status = 'in_progress'"),
|
|
('idx_target_queue_exhausted_attempts', "status = 'pending' OR status = 'deferred' OR status = 'in_progress'"),
|
|
('idx_target_queue_updated_rescan', "status = 'done' AND remote_modified_at IS NOT NULL"),
|
|
('idx_target_queue_cold', "status = 'cold'"),
|
|
):
|
|
if _normalized_predicate((queue_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(predicate):
|
|
missing.append(f'predicate {name}')
|
|
for name, predicate in (
|
|
('idx_discovery_retry_queue_due', DISCOVERY_RETRY_DUE_INDEX_PREDICATE),
|
|
('idx_discovery_retry_queue_lease', DISCOVERY_RETRY_LEASE_INDEX_PREDICATE),
|
|
):
|
|
if _normalized_predicate((retry_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(predicate):
|
|
missing.append(f'predicate {name}')
|
|
cooldown_index = scan_indexes.get('idx_target_scans_cooldown_recent')
|
|
if _normalized_predicate((cooldown_index or {}).get('predicate')) != _normalized_predicate(CI_COOLDOWN_INDEX_PREDICATE):
|
|
missing.append('predicate idx_target_scans_cooldown_recent')
|
|
for name in (
|
|
'idx_package_repo_candidates_query_seen',
|
|
'idx_package_repo_candidates_recent_lookup',
|
|
):
|
|
if _normalized_predicate((package_indexes.get(name) or {}).get('predicate')) != _normalized_predicate(PACKAGE_REPO_NONEMPTY_PREDICATE):
|
|
missing.append(f'predicate {name}')
|
|
missing.extend(_docker_depth_check_constraint_problems(self.conn))
|
|
for table, expected_keys in REQUIRED_FOREIGN_KEYS.items():
|
|
if not self.conn.table_exists(table):
|
|
continue
|
|
foreign_keys = self.conn.table_foreign_keys(table)
|
|
for columns, referenced_table, referenced_columns in expected_keys:
|
|
authority_matching = [
|
|
foreign_key for foreign_key in foreign_keys.values()
|
|
if tuple(foreign_key.get('columns') or ()) == columns
|
|
and foreign_key.get('referenced_table') == referenced_table
|
|
and tuple(foreign_key.get('referenced_columns') or ()) == referenced_columns
|
|
and foreign_key.get('referenced_schema') in (
|
|
self.conn.application_schema if self.conn.is_postgres else 'main',
|
|
)
|
|
]
|
|
expected_actions = _required_foreign_key_actions(table, columns)
|
|
matching = [
|
|
foreign_key for foreign_key in authority_matching
|
|
if _foreign_key_actions_match(foreign_key, expected_actions)
|
|
]
|
|
if len(matching) != 1 or not matching[0].get('valid', True):
|
|
detail = (
|
|
f'foreign key {table}({", ".join(columns)}) -> '
|
|
f'{referenced_table}({", ".join(referenced_columns)})'
|
|
)
|
|
if len(authority_matching) == 1 and not _foreign_key_actions_match(
|
|
authority_matching[0], expected_actions,
|
|
):
|
|
detail += (
|
|
f' actions ON UPDATE {expected_actions[0]} '
|
|
f'ON DELETE {expected_actions[1]}'
|
|
)
|
|
missing.append(detail)
|
|
if self.conn.is_postgres and commit:
|
|
self.conn.commit()
|
|
if missing:
|
|
self.last_error = 'runtime safety schema is incomplete: ' + '; '.join(missing)
|
|
return False
|
|
self.last_error = ''
|
|
return True
|
|
except Exception as exc:
|
|
self.last_error = f'unable to validate runtime safety schema: {exc}'
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
return False
|
|
|
|
def require_runtime_safety_schema(self, commit=True):
|
|
if not self.runtime_safety_schema_available(commit=commit):
|
|
detail = self.last_error or 'runtime safety schema is unavailable'
|
|
raise RuntimeSafetySchemaError(f'{detail}; run migrate_runtime_safety.py offline with --apply')
|
|
if not self.pipeline_schema_available(commit=commit):
|
|
detail = self.last_error or 'pipeline schema is unavailable'
|
|
raise RuntimeSafetySchemaError(f'{detail}; run migrate_runtime_safety.py offline with --apply')
|
|
return True
|
|
|
|
def _locked_runtime_control_state(self, shared=False):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
suffix = ''
|
|
if self.conn.is_postgres:
|
|
suffix = ' FOR SHARE' if shared else ' FOR UPDATE'
|
|
row = self.conn.execute(
|
|
'SELECT * FROM runtime_operations_control WHERE id = 1' + suffix
|
|
).fetchone()
|
|
return _runtime_control_state_from_row(row)
|
|
|
|
def _require_discovery_admission_locked(self):
|
|
state = self._locked_runtime_control_state(shared=True)
|
|
if state['effective_discovery_paused']:
|
|
raise DiscoveryPausedError(state)
|
|
return state
|
|
|
|
def runtime_control_state(self):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT * FROM runtime_operations_control WHERE id = 1'
|
|
).fetchone()
|
|
state = _runtime_control_state_from_row(row)
|
|
self.conn.commit()
|
|
return state
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _locked_runtime_operation(self, operation_id):
|
|
suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
return self.conn.execute(
|
|
'SELECT * FROM runtime_operations WHERE operation_id = ?' + suffix,
|
|
(operation_id,),
|
|
).fetchone()
|
|
|
|
@staticmethod
|
|
def _runtime_operation_state(row):
|
|
if not row:
|
|
return None
|
|
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
|
|
actor = _runtime_actor(str(row['actor'] or ''))
|
|
action = str(row['action'] or '')
|
|
target_kind = str(row['target_kind'] or '')
|
|
target_ref = str(row['target_ref'] or '')
|
|
status = str(row['status'] or '')
|
|
agent_state = str(row['agent_state'] or '')
|
|
if action in RUNTIME_ASYNC_ACTIONS:
|
|
if (
|
|
target_kind != RUNTIME_ASYNC_TARGET_KIND
|
|
or target_ref != RUNTIME_ASYNC_ACTION_TARGETS[action]
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime operation target is invalid')
|
|
try:
|
|
expected_raw = json.loads(str(row['expected_identity_json'] or ''))
|
|
expected = _runtime_expected_identity(expected_raw, action)
|
|
expected_json = _canonical_runtime_json(expected)
|
|
resulting_json = row['resulting_identity_json']
|
|
resulting = None
|
|
if resulting_json is not None:
|
|
resulting = _runtime_resulting_identity(json.loads(str(resulting_json)))
|
|
if _canonical_runtime_json(resulting) != str(resulting_json):
|
|
raise ValueError('noncanonical')
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeSafetySchemaError('runtime operation identity is invalid') from exc
|
|
if expected_json != str(row['expected_identity_json'] or ''):
|
|
raise RuntimeSafetySchemaError('runtime operation identity is not canonical')
|
|
elif action in RUNTIME_CONTROL_ACTIONS:
|
|
expected = _stored_runtime_control_identity(str(row['expected_identity_json'] or ''))
|
|
resulting = None
|
|
if row['resulting_identity_json'] is not None:
|
|
resulting = _stored_runtime_control_identity(
|
|
str(row['resulting_identity_json']),
|
|
)
|
|
elif action in RUNTIME_SOURCE_OPERATION_ACTIONS:
|
|
if target_kind != RUNTIME_SOURCE_TARGET_KIND or _runtime_source_id(target_ref) is None:
|
|
raise RuntimeSafetySchemaError('runtime source operation target is invalid')
|
|
expected = _stored_runtime_source_identity(
|
|
str(row['expected_identity_json'] or ''), action, target_ref,
|
|
)
|
|
resulting = None
|
|
if row['resulting_identity_json'] is not None:
|
|
resulting = _stored_runtime_source_identity(
|
|
str(row['resulting_identity_json']), action, target_ref,
|
|
resulting=True,
|
|
)
|
|
elif action in RUNTIME_WORKER_ADMIN_ACTION_TARGETS:
|
|
if target_kind != RUNTIME_WORKER_ADMIN_TARGET_KIND:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime worker admin operation target is invalid'
|
|
)
|
|
expected = _stored_runtime_worker_admin_identity(
|
|
str(row['expected_identity_json'] or ''), action, target_ref,
|
|
)
|
|
resulting = None
|
|
if row['resulting_identity_json'] is not None:
|
|
resulting = _stored_runtime_worker_admin_identity(
|
|
str(row['resulting_identity_json']), action, target_ref,
|
|
resulting=True,
|
|
)
|
|
elif action in RUNTIME_DOCUMENT_ACTION_TARGETS:
|
|
if target_kind != RUNTIME_DOCUMENT_TARGET_KIND:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime document operation target is invalid'
|
|
)
|
|
expected = _stored_runtime_document_identity(
|
|
str(row['expected_identity_json'] or ''), action, target_ref,
|
|
)
|
|
resulting = None
|
|
if row['resulting_identity_json'] is not None:
|
|
resulting = _stored_runtime_document_identity(
|
|
str(row['resulting_identity_json']), action, target_ref,
|
|
resulting=True,
|
|
)
|
|
elif action in RUNTIME_MANAGED_FILE_ACTIONS:
|
|
if target_kind != RUNTIME_MANAGED_FILE_TARGET_KIND:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file operation target is invalid'
|
|
)
|
|
expected = _stored_runtime_managed_file_identity(
|
|
str(row['expected_identity_json'] or ''), action, target_ref,
|
|
)
|
|
resulting = None
|
|
if row['resulting_identity_json'] is not None:
|
|
resulting = _stored_runtime_managed_file_identity(
|
|
str(row['resulting_identity_json']), action, target_ref,
|
|
resulting=True,
|
|
)
|
|
else:
|
|
raise RuntimeSafetySchemaError('runtime operation action is invalid')
|
|
if status not in (
|
|
'requested', 'running', 'succeeded', 'failed', 'rolled_back',
|
|
'failed_hold', 'canceled',
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime operation status is invalid')
|
|
if agent_state not in (
|
|
'not_required', 'pending', 'running', 'succeeded', 'failed',
|
|
'rolled_back', 'failed_hold',
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime operation agent state is invalid')
|
|
category = row['safe_category']
|
|
detail = row['safe_detail']
|
|
try:
|
|
category = _runtime_safe_code(category, 'safe category')
|
|
detail = _runtime_safe_code(detail, 'safe detail')
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError('runtime operation safe result is invalid') from exc
|
|
result_sha256 = row['agent_result_sha256']
|
|
if result_sha256 is not None:
|
|
try:
|
|
result_sha256 = _runtime_sha256(str(result_sha256), 'agent result hash')
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError('runtime operation result hash is invalid') from exc
|
|
return {
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': target_kind,
|
|
'target_ref': target_ref,
|
|
'status': status,
|
|
'safe_category': category,
|
|
'safe_detail': detail,
|
|
'expected_revision': row['expected_revision'],
|
|
'resulting_revision': row['resulting_revision'],
|
|
'expected_identity': expected,
|
|
'resulting_identity': resulting,
|
|
'agent_state': agent_state,
|
|
'agent_result_sha256': result_sha256,
|
|
'requested_at': str(row['requested_at'] or ''),
|
|
'started_at': str(row['started_at']) if row['started_at'] is not None else None,
|
|
'completed_at': str(row['completed_at']) if row['completed_at'] is not None else None,
|
|
'agent_reconciled_at': (
|
|
str(row['agent_reconciled_at'])
|
|
if row['agent_reconciled_at'] is not None else None
|
|
),
|
|
'updated_at': str(row['updated_at'] or ''),
|
|
}
|
|
|
|
def _append_runtime_audit_event_locked(
|
|
self, *, operation_id, actor, action, target_kind, target_ref,
|
|
result, safe_category=None, before_identity=None, after_identity=None,
|
|
before_bytes=None, after_bytes=None, created_at=None,
|
|
):
|
|
if result not in (
|
|
'accepted', 'succeeded', 'rejected', 'failed', 'rolled_back',
|
|
'canceled', 'failed_hold',
|
|
):
|
|
raise ValueError('runtime audit result is invalid')
|
|
before_json = (
|
|
_canonical_runtime_json(before_identity) if before_identity is not None else None
|
|
)
|
|
after_json = (
|
|
_canonical_runtime_json(after_identity) if after_identity is not None else None
|
|
)
|
|
for value in (before_bytes, after_bytes):
|
|
if value is not None and (
|
|
isinstance(value, bool) or not isinstance(value, int) or value < 0
|
|
):
|
|
raise ValueError('runtime audit byte count is invalid')
|
|
tail = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
ORDER BY id DESC LIMIT 1'''
|
|
).fetchone()
|
|
previous_event_id = int(tail['id']) if tail else None
|
|
previous_event_sha256 = str(tail['event_sha256']) if tail else None
|
|
now = created_at or utc_now_iso()
|
|
payload = {
|
|
'schema': 'runtime-audit-event-v1',
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': target_kind,
|
|
'target_ref': target_ref,
|
|
'result': result,
|
|
'safe_category': safe_category,
|
|
'before_identity_json': before_json,
|
|
'after_identity_json': after_json,
|
|
'before_bytes': before_bytes,
|
|
'after_bytes': after_bytes,
|
|
'previous_event_id': previous_event_id,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'created_at': now,
|
|
}
|
|
event_sha256 = _runtime_audit_event_sha256(payload)
|
|
event_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO runtime_audit_events(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
result, safe_category, before_identity_json,
|
|
after_identity_json, before_bytes, after_bytes,
|
|
previous_event_id, previous_event_sha256, event_sha256,
|
|
created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, target_kind, target_ref, result,
|
|
safe_category, before_json, after_json, before_bytes, after_bytes,
|
|
previous_event_id, previous_event_sha256, event_sha256, now,
|
|
),
|
|
)
|
|
if event_id is None:
|
|
raise RuntimeSafetySchemaError('runtime audit insertion failed')
|
|
return {
|
|
'audit_event_id': int(event_id),
|
|
'audit_event_sha256': event_sha256,
|
|
}
|
|
|
|
def runtime_operation(self, operation_id):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT * FROM runtime_operations WHERE operation_id = ?',
|
|
(operation_id,),
|
|
).fetchone()
|
|
result = self._runtime_operation_state(row)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def recent_runtime_operations(
|
|
self, limit=200, *, before_updated_at=None, before_operation_id=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
if type(limit) is not int or not 1 <= limit <= 500:
|
|
raise ValueError('runtime operation snapshot limit is out of range')
|
|
if (before_updated_at is None) is not (before_operation_id is None):
|
|
raise ValueError('runtime operation cursor is incomplete')
|
|
if before_updated_at is not None:
|
|
if type(before_updated_at) is not str or not before_updated_at:
|
|
raise ValueError('runtime operation cursor timestamp is invalid')
|
|
before_operation_id = _runtime_operation_id(before_operation_id)
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN')
|
|
else:
|
|
self.conn.execute(
|
|
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY'
|
|
)
|
|
rows = []
|
|
for status in (
|
|
'requested', 'running', 'succeeded', 'failed', 'rolled_back',
|
|
'failed_hold', 'canceled',
|
|
):
|
|
if before_updated_at is None:
|
|
sql = '''SELECT * FROM runtime_operations WHERE status = ?
|
|
ORDER BY updated_at DESC, operation_id DESC LIMIT ?'''
|
|
parameters = (status, limit)
|
|
else:
|
|
sql = '''SELECT * FROM runtime_operations WHERE status = ?
|
|
AND (
|
|
updated_at < ? OR (
|
|
updated_at = ? AND operation_id < ?
|
|
)
|
|
)
|
|
ORDER BY updated_at DESC, operation_id DESC LIMIT ?'''
|
|
parameters = (
|
|
status, before_updated_at, before_updated_at,
|
|
before_operation_id, limit,
|
|
)
|
|
rows.extend(self.conn.execute(sql, parameters).fetchall())
|
|
operations = {}
|
|
for row in rows:
|
|
operation = self._runtime_operation_state(row)
|
|
operations[operation['operation_id']] = operation
|
|
result = sorted(
|
|
operations.values(),
|
|
key=lambda item: (item['updated_at'], item['operation_id']),
|
|
reverse=True,
|
|
)[:limit]
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def pending_runtime_agent_operations(self, limit=32):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
if type(limit) is not int or not 1 <= limit <= 128:
|
|
raise ValueError('runtime agent operation limit is out of range')
|
|
try:
|
|
rows = self.conn.execute(
|
|
'''SELECT * FROM runtime_operations
|
|
WHERE target_kind = ?
|
|
AND status IN ('requested', 'running')
|
|
AND agent_state IN ('pending', 'running')
|
|
ORDER BY CASE status WHEN 'running' THEN 0 ELSE 1 END,
|
|
updated_at ASC, operation_id ASC
|
|
LIMIT ?''',
|
|
(RUNTIME_ASYNC_TARGET_KIND, limit),
|
|
).fetchall()
|
|
result = [self._runtime_operation_state(row) for row in rows]
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def runtime_audit_events(self, before_event_id=None, limit=50):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
if (
|
|
before_event_id is not None
|
|
and (
|
|
type(before_event_id) is not int
|
|
or not 1 <= before_event_id <= RUNTIME_CONTROL_MAX_REVISION
|
|
)
|
|
):
|
|
raise ValueError('runtime audit cursor is out of range')
|
|
if type(limit) is not int or not 1 <= limit <= 200:
|
|
raise ValueError('runtime audit page limit is out of range')
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN')
|
|
else:
|
|
self.conn.execute(
|
|
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY'
|
|
)
|
|
parameters = []
|
|
# The console exposes only events whose identities can be checked
|
|
# against a durable typed operation.
|
|
clauses = ['operation_id IS NOT NULL']
|
|
if before_event_id is not None:
|
|
clauses.append('id < ?')
|
|
parameters.append(before_event_id)
|
|
parameters.append(limit + 1)
|
|
rows = self.conn.execute(
|
|
'SELECT * FROM runtime_audit_events WHERE ' + ' AND '.join(clauses)
|
|
+ ' ORDER BY id DESC LIMIT ?',
|
|
tuple(parameters),
|
|
).fetchall()
|
|
page_rows = rows[:limit]
|
|
operation_ids = []
|
|
for row in page_rows:
|
|
try:
|
|
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime audit operation identity is invalid'
|
|
) from exc
|
|
if operation_id not in operation_ids:
|
|
operation_ids.append(operation_id)
|
|
operations = {}
|
|
if operation_ids:
|
|
placeholders = ','.join('?' for _value in operation_ids)
|
|
operation_rows = self.conn.execute(
|
|
'SELECT * FROM runtime_operations WHERE operation_id IN ('
|
|
+ placeholders + ')',
|
|
tuple(operation_ids),
|
|
).fetchall()
|
|
for row in operation_rows:
|
|
operation = self._runtime_operation_state(row)
|
|
operations[operation['operation_id']] = operation
|
|
|
|
events = []
|
|
allowed_results = {
|
|
'accepted', 'succeeded', 'rejected', 'failed', 'rolled_back',
|
|
'canceled', 'failed_hold',
|
|
}
|
|
for row in page_rows:
|
|
try:
|
|
event_id = row['id']
|
|
if type(event_id) is not int or event_id < 1:
|
|
raise ValueError('event ID')
|
|
operation_id = _runtime_operation_id(str(row['operation_id'] or ''))
|
|
operation = operations.get(operation_id)
|
|
if operation is None:
|
|
raise ValueError('operation')
|
|
actor = _runtime_actor(str(row['actor'] or ''))
|
|
action = str(row['action'] or '')
|
|
target_kind = str(row['target_kind'] or '')
|
|
target_ref = str(row['target_ref'] or '')
|
|
result = str(row['result'] or '')
|
|
if (
|
|
actor != operation['actor']
|
|
or action != operation['action']
|
|
or target_kind != operation['target_kind']
|
|
or target_ref != operation['target_ref']
|
|
or result not in allowed_results
|
|
):
|
|
raise ValueError('event identity')
|
|
safe_category = _runtime_safe_code(
|
|
row['safe_category'], 'safe category',
|
|
)
|
|
|
|
identities = []
|
|
identity_json = []
|
|
for field in ('before_identity_json', 'after_identity_json'):
|
|
raw = row[field]
|
|
if raw is None:
|
|
identities.append(None)
|
|
identity_json.append(None)
|
|
continue
|
|
raw = str(raw)
|
|
parsed = json.loads(raw)
|
|
if not isinstance(parsed, dict) or _canonical_runtime_json(parsed) != raw:
|
|
raise ValueError('event identity JSON')
|
|
identities.append(parsed)
|
|
identity_json.append(raw)
|
|
before_identity, after_identity = identities
|
|
if before_identity != operation['expected_identity'] or (
|
|
after_identity is not None
|
|
and after_identity != operation['resulting_identity']
|
|
):
|
|
raise ValueError('event operation identity')
|
|
|
|
byte_counts = []
|
|
for field in ('before_bytes', 'after_bytes'):
|
|
value = row[field]
|
|
if value is not None and (
|
|
type(value) is not int
|
|
or not 0 <= value <= RUNTIME_CONTROL_MAX_REVISION
|
|
):
|
|
raise ValueError('event byte count')
|
|
byte_counts.append(value)
|
|
before_bytes, after_bytes = byte_counts
|
|
|
|
previous_event_id = row['previous_event_id']
|
|
previous_event_sha256 = row['previous_event_sha256']
|
|
if previous_event_id is None:
|
|
if previous_event_sha256 is not None:
|
|
raise ValueError('event parent')
|
|
elif (
|
|
type(previous_event_id) is not int
|
|
or previous_event_id < 1
|
|
or previous_event_id >= event_id
|
|
):
|
|
raise ValueError('event parent')
|
|
else:
|
|
previous_event_sha256 = _runtime_sha256(
|
|
str(previous_event_sha256 or ''), 'audit parent hash',
|
|
)
|
|
event_sha256 = _runtime_sha256(
|
|
str(row['event_sha256'] or ''), 'audit event hash',
|
|
)
|
|
created_at = str(row['created_at'] or '')
|
|
if not 1 <= len(created_at) <= 64:
|
|
raise ValueError('event time')
|
|
payload = {
|
|
'schema': 'runtime-audit-event-v1',
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': target_kind,
|
|
'target_ref': target_ref,
|
|
'result': result,
|
|
'safe_category': safe_category,
|
|
'before_identity_json': identity_json[0],
|
|
'after_identity_json': identity_json[1],
|
|
'before_bytes': before_bytes,
|
|
'after_bytes': after_bytes,
|
|
'previous_event_id': previous_event_id,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'created_at': created_at,
|
|
}
|
|
if not hmac.compare_digest(
|
|
_runtime_audit_event_sha256(payload), event_sha256,
|
|
):
|
|
raise ValueError('event hash')
|
|
except (
|
|
KeyError, TypeError, ValueError, json.JSONDecodeError,
|
|
) as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime audit event is invalid'
|
|
) from exc
|
|
events.append({
|
|
'id': event_id,
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': target_kind,
|
|
'target_ref': target_ref,
|
|
'result': result,
|
|
'safe_category': safe_category,
|
|
'before_identity': before_identity,
|
|
'after_identity': after_identity,
|
|
'before_bytes': before_bytes,
|
|
'after_bytes': after_bytes,
|
|
'previous_event_id': previous_event_id,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'event_sha256': event_sha256,
|
|
'created_at': created_at,
|
|
})
|
|
next_before_event_id = (
|
|
events[-1]['id'] if len(rows) > limit and events else None
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'events': events,
|
|
'next_before_event_id': next_before_event_id,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def create_runtime_operation(self, *, operation_id, actor, action, expected_identity):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
if not isinstance(action, str) or action not in RUNTIME_ASYNC_ACTIONS:
|
|
raise ValueError('runtime operation action is invalid')
|
|
expected_identity = _runtime_expected_identity(expected_identity, action)
|
|
expected_json = _canonical_runtime_json(expected_identity)
|
|
target_ref = RUNTIME_ASYNC_ACTION_TARGETS[action]
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
existing_row = self._locked_runtime_operation(operation_id)
|
|
if existing_row:
|
|
existing = self._runtime_operation_state(existing_row)
|
|
if not (
|
|
existing['actor'] == actor
|
|
and existing['action'] == action
|
|
and existing['target_kind'] == RUNTIME_ASYNC_TARGET_KIND
|
|
and existing['target_ref'] == target_ref
|
|
and existing['expected_identity'] == expected_identity
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
accepted = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = 'accepted'
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(accepted) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime operation accepted audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**existing, 'replayed': True,
|
|
'audit_event_id': int(accepted[0]['id']),
|
|
'audit_event_sha256': str(accepted[0]['event_sha256']),
|
|
}
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_identity_json, agent_state,
|
|
requested_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'requested', ?, 'pending', ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_ASYNC_TARGET_KIND,
|
|
target_ref, expected_json, now, now,
|
|
),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_ASYNC_TARGET_KIND, target_ref=target_ref,
|
|
result='accepted', before_identity=expected_identity,
|
|
created_at=now,
|
|
)
|
|
row = self._locked_runtime_operation(operation_id)
|
|
result = self._runtime_operation_state(row)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime operation creation retry budget exhausted')
|
|
|
|
def mark_runtime_operation_running(self, operation_id):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if current['action'] not in RUNTIME_ASYNC_ACTIONS:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation does not use agent lifecycle'
|
|
)
|
|
if current['status'] == 'running':
|
|
self.conn.commit()
|
|
return {**current, 'replayed': True}
|
|
if current['status'] != 'requested':
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation cannot enter running state'
|
|
)
|
|
now = utc_now_iso()
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = 'running', agent_state = 'running',
|
|
started_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'requested'
|
|
AND agent_state = 'pending' ''',
|
|
(now, now, operation_id),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation running transition conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime operation transition retry budget exhausted')
|
|
|
|
def claim_runtime_operation_execution(self, *, operation_id, action, expected_identity):
|
|
"""Atomically bind a host-agent claim to persisted request identity."""
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if not isinstance(action, str) or action not in RUNTIME_ASYNC_ACTIONS:
|
|
raise ValueError('runtime operation action is invalid')
|
|
expected_identity = _runtime_expected_identity(expected_identity, action)
|
|
expected_json = _canonical_runtime_json(expected_identity)
|
|
target_ref = RUNTIME_ASYNC_ACTION_TARGETS[action]
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is unavailable'
|
|
)
|
|
current = self._runtime_operation_state(row)
|
|
if not (
|
|
current['action'] == action
|
|
and current['target_kind'] == RUNTIME_ASYNC_TARGET_KIND
|
|
and current['target_ref'] == target_ref
|
|
and current['expected_identity'] == expected_identity
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation identity does not match the persisted request'
|
|
)
|
|
if any((
|
|
current['expected_revision'] is not None,
|
|
current['resulting_identity'] is not None,
|
|
current['resulting_revision'] is not None,
|
|
current['completed_at'] is not None,
|
|
current['agent_reconciled_at'] is not None,
|
|
current['safe_category'] is not None,
|
|
current['safe_detail'] is not None,
|
|
current['agent_result_sha256'] is not None,
|
|
)):
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation cannot be claimed for execution'
|
|
)
|
|
requested = (
|
|
current['status'] == 'requested'
|
|
and current['agent_state'] == 'pending'
|
|
and current['started_at'] is None
|
|
)
|
|
running = (
|
|
current['status'] == 'running'
|
|
and current['agent_state'] == 'running'
|
|
and current['started_at'] is not None
|
|
)
|
|
if not requested and not running:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation cannot be claimed for execution'
|
|
)
|
|
|
|
audits = self.conn.execute(
|
|
'''SELECT * FROM runtime_audit_events
|
|
WHERE operation_id = ? ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(audits) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime operation accepted audit evidence is incomplete'
|
|
)
|
|
audit = audits[0]
|
|
try:
|
|
audit_id = int(audit['id'])
|
|
before_json = str(audit['before_identity_json'] or '')
|
|
previous_event_id = audit['previous_event_id']
|
|
previous_event_sha256 = audit['previous_event_sha256']
|
|
if previous_event_id is None:
|
|
if previous_event_sha256 is not None:
|
|
raise ValueError('audit parent')
|
|
else:
|
|
if (
|
|
type(previous_event_id) is not int
|
|
or previous_event_id < 1
|
|
or previous_event_id >= audit_id
|
|
):
|
|
raise ValueError('audit parent')
|
|
previous_event_sha256 = _runtime_sha256(
|
|
str(previous_event_sha256 or ''), 'audit parent hash',
|
|
)
|
|
predecessor = self.conn.execute(
|
|
'''SELECT * FROM runtime_audit_events
|
|
WHERE id < ? ORDER BY id DESC LIMIT 1''',
|
|
(audit_id,),
|
|
).fetchone()
|
|
if predecessor is None:
|
|
if previous_event_id is not None:
|
|
raise ValueError('audit parent')
|
|
else:
|
|
predecessor_id, predecessor_sha256 = (
|
|
_runtime_audit_row_hash(predecessor)
|
|
)
|
|
if not (
|
|
predecessor_id == previous_event_id
|
|
and hmac.compare_digest(
|
|
predecessor_sha256,
|
|
previous_event_sha256 or '',
|
|
)
|
|
):
|
|
raise ValueError('audit parent')
|
|
event_sha256 = _runtime_sha256(
|
|
str(audit['event_sha256'] or ''), 'audit event hash',
|
|
)
|
|
if not (
|
|
audit_id >= 1
|
|
and str(audit['operation_id'] or '') == operation_id
|
|
and str(audit['actor'] or '') == current['actor']
|
|
and str(audit['action'] or '') == action
|
|
and str(audit['target_kind'] or '') == RUNTIME_ASYNC_TARGET_KIND
|
|
and str(audit['target_ref'] or '') == target_ref
|
|
and str(audit['result'] or '') == 'accepted'
|
|
and audit['safe_category'] is None
|
|
and before_json == expected_json
|
|
and audit['after_identity_json'] is None
|
|
and audit['before_bytes'] is None
|
|
and audit['after_bytes'] is None
|
|
and str(audit['created_at'] or '') == current['requested_at']
|
|
):
|
|
raise ValueError('accepted audit')
|
|
payload = {
|
|
'schema': 'runtime-audit-event-v1',
|
|
'operation_id': operation_id,
|
|
'actor': current['actor'],
|
|
'action': action,
|
|
'target_kind': RUNTIME_ASYNC_TARGET_KIND,
|
|
'target_ref': target_ref,
|
|
'result': 'accepted',
|
|
'safe_category': None,
|
|
'before_identity_json': expected_json,
|
|
'after_identity_json': None,
|
|
'before_bytes': None,
|
|
'after_bytes': None,
|
|
'previous_event_id': previous_event_id,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'created_at': current['requested_at'],
|
|
}
|
|
if not hmac.compare_digest(
|
|
_runtime_audit_event_sha256(payload), event_sha256,
|
|
):
|
|
raise ValueError('audit hash')
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime operation accepted audit evidence is invalid'
|
|
) from exc
|
|
|
|
if running:
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': audit_id,
|
|
'audit_event_sha256': event_sha256,
|
|
}
|
|
now = utc_now_iso()
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = 'running', agent_state = 'running',
|
|
started_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND action = ?
|
|
AND target_kind = ? AND target_ref = ?
|
|
AND expected_identity_json = ?
|
|
AND status = 'requested' AND agent_state = 'pending' ''',
|
|
(
|
|
now, now, operation_id, action, RUNTIME_ASYNC_TARGET_KIND,
|
|
target_ref, expected_json,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation claim conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**result, 'replayed': False,
|
|
'audit_event_id': audit_id,
|
|
'audit_event_sha256': event_sha256,
|
|
}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime operation claim retry budget exhausted')
|
|
|
|
def create_runtime_source_operation(
|
|
self, *, operation_id, actor, source_id, source_action,
|
|
interval_seconds=None, mode=None, restart_enabled=None,
|
|
restart_delay_seconds=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
if _runtime_source_id(source_id) is None or source_action not in RUNTIME_SOURCE_ACTIONS:
|
|
raise ValueError('runtime source operation request is invalid')
|
|
action = f'supervisor.source.{source_action}'
|
|
identity = {
|
|
'source_id': source_id,
|
|
'source_action': source_action,
|
|
'interval_seconds': interval_seconds,
|
|
}
|
|
if source_action in ('once', 'set-mode', 'set-restart', 'set-restart-delay'):
|
|
identity.update({
|
|
'mode': mode,
|
|
'restart_enabled': restart_enabled,
|
|
'restart_delay_seconds': restart_delay_seconds,
|
|
})
|
|
elif any(
|
|
value is not None
|
|
for value in (mode, restart_enabled, restart_delay_seconds)
|
|
):
|
|
raise ValueError('runtime source operation request is invalid')
|
|
expected = _runtime_source_expected_identity(identity, action, source_id)
|
|
expected_json = _canonical_runtime_json(expected)
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
existing_row = self._locked_runtime_operation(operation_id)
|
|
if existing_row:
|
|
existing = self._runtime_operation_state(existing_row)
|
|
if not (
|
|
existing['actor'] == actor
|
|
and existing['action'] == action
|
|
and existing['target_kind'] == RUNTIME_SOURCE_TARGET_KIND
|
|
and existing['target_ref'] == source_id
|
|
and existing['expected_identity'] == expected
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
accepted = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = 'accepted'
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(accepted) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime source operation accepted audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**existing, 'replayed': True,
|
|
'audit_event_id': int(accepted[0]['id']),
|
|
'audit_event_sha256': str(accepted[0]['event_sha256']),
|
|
}
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_identity_json, agent_state,
|
|
requested_at, started_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_SOURCE_TARGET_KIND,
|
|
source_id, expected_json, now, now, now,
|
|
),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_SOURCE_TARGET_KIND, target_ref=source_id,
|
|
result='accepted', before_identity=expected, created_at=now,
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime source operation creation retry budget exhausted')
|
|
|
|
def complete_runtime_source_operation(
|
|
self, operation_id, *, succeeded, outcome=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if type(succeeded) is not bool:
|
|
raise ValueError('runtime source operation result is invalid')
|
|
if succeeded:
|
|
if outcome not in ('completed', 'dependency-blocked'):
|
|
raise ValueError('runtime source operation result is invalid')
|
|
elif outcome is not None:
|
|
raise ValueError('runtime source operation result is invalid')
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if current['action'] not in RUNTIME_SOURCE_OPERATION_ACTIONS:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is not a managed source action'
|
|
)
|
|
status = 'succeeded' if succeeded else 'failed'
|
|
safe_category = None if succeeded else 'supervisor_action_failed'
|
|
resulting = None
|
|
if succeeded:
|
|
resulting = _runtime_source_resulting_identity({
|
|
'source_id': current['target_ref'],
|
|
'source_action': RUNTIME_SOURCE_OPERATION_ACTIONS[current['action']],
|
|
'outcome': outcome,
|
|
}, current['action'], current['target_ref'])
|
|
if current['status'] in ('succeeded', 'failed'):
|
|
if (
|
|
current['status'] != status
|
|
or current['safe_category'] != safe_category
|
|
or current['resulting_identity'] != resulting
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime source operation already has another result'
|
|
)
|
|
events = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id, status),
|
|
).fetchall()
|
|
if len(events) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime source operation terminal audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': int(events[0]['id']),
|
|
'audit_event_sha256': str(events[0]['event_sha256']),
|
|
}
|
|
if current['status'] != 'running' or current['agent_state'] != 'not_required':
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime source operation cannot be completed'
|
|
)
|
|
now = utc_now_iso()
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=current['actor'],
|
|
action=current['action'], target_kind=RUNTIME_SOURCE_TARGET_KIND,
|
|
target_ref=current['target_ref'], result=status,
|
|
safe_category=safe_category,
|
|
before_identity=current['expected_identity'],
|
|
after_identity=resulting, created_at=now,
|
|
)
|
|
resulting_json = (
|
|
_canonical_runtime_json(resulting) if resulting is not None else None
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = ?, safe_category = ?, resulting_identity_json = ?,
|
|
completed_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'running'
|
|
AND agent_state = 'not_required' ''',
|
|
(
|
|
status, safe_category, resulting_json, now, now,
|
|
operation_id,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime source operation completion conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime source operation completion retry budget exhausted')
|
|
|
|
def create_runtime_worker_admin_operation(
|
|
self, *, operation_id, actor, action, target_ref, request_sha256,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
target_ref = _runtime_worker_admin_target(action, target_ref)
|
|
expected = _runtime_worker_admin_expected_identity({
|
|
'action': action,
|
|
'target_ref': target_ref,
|
|
'request_sha256': request_sha256,
|
|
}, action, target_ref)
|
|
expected_json = _canonical_runtime_json(expected)
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
existing_row = self._locked_runtime_operation(operation_id)
|
|
if existing_row:
|
|
existing = self._runtime_operation_state(existing_row)
|
|
if not (
|
|
existing['actor'] == actor
|
|
and existing['action'] == action
|
|
and existing['target_kind'] == RUNTIME_WORKER_ADMIN_TARGET_KIND
|
|
and existing['target_ref'] == target_ref
|
|
and existing['expected_identity'] == expected
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
accepted = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = 'accepted'
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(accepted) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime worker admin accepted audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**existing, 'replayed': True,
|
|
'audit_event_id': int(accepted[0]['id']),
|
|
'audit_event_sha256': str(accepted[0]['event_sha256']),
|
|
}
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_identity_json, agent_state,
|
|
requested_at, started_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_WORKER_ADMIN_TARGET_KIND,
|
|
target_ref, expected_json, now, now, now,
|
|
),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_WORKER_ADMIN_TARGET_KIND,
|
|
target_ref=target_ref, result='accepted',
|
|
before_identity=expected, created_at=now,
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime worker admin operation creation retry budget exhausted'
|
|
)
|
|
|
|
def complete_runtime_worker_admin_operation(
|
|
self, operation_id, *, succeeded, affected_count=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if type(succeeded) is not bool:
|
|
raise ValueError('runtime worker admin operation result is invalid')
|
|
if succeeded:
|
|
if type(affected_count) is not int or not 0 <= affected_count <= 10000:
|
|
raise ValueError('runtime worker admin operation result is invalid')
|
|
elif affected_count is not None:
|
|
raise ValueError('runtime worker admin operation result is invalid')
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if current['action'] not in RUNTIME_WORKER_ADMIN_ACTION_TARGETS:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is not a worker admin action'
|
|
)
|
|
status = 'succeeded' if succeeded else 'failed'
|
|
safe_category = None if succeeded else 'worker_admin_mutation_failed'
|
|
resulting = None
|
|
if succeeded:
|
|
resulting = _runtime_worker_admin_resulting_identity({
|
|
'action': current['action'],
|
|
'target_ref': current['target_ref'],
|
|
'outcome': 'completed',
|
|
'affected_count': affected_count,
|
|
}, current['action'], current['target_ref'])
|
|
if current['status'] in ('succeeded', 'failed'):
|
|
if (
|
|
current['status'] != status
|
|
or current['safe_category'] != safe_category
|
|
or current['resulting_identity'] != resulting
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime worker admin operation already has another result'
|
|
)
|
|
events = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id, status),
|
|
).fetchall()
|
|
if len(events) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime worker admin terminal audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': int(events[0]['id']),
|
|
'audit_event_sha256': str(events[0]['event_sha256']),
|
|
}
|
|
if current['status'] != 'running' or current['agent_state'] != 'not_required':
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime worker admin operation cannot be completed'
|
|
)
|
|
now = utc_now_iso()
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=current['actor'],
|
|
action=current['action'],
|
|
target_kind=RUNTIME_WORKER_ADMIN_TARGET_KIND,
|
|
target_ref=current['target_ref'], result=status,
|
|
safe_category=safe_category,
|
|
before_identity=current['expected_identity'],
|
|
after_identity=resulting, created_at=now,
|
|
)
|
|
resulting_json = (
|
|
_canonical_runtime_json(resulting) if resulting is not None else None
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = ?, safe_category = ?, resulting_identity_json = ?,
|
|
completed_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'running'
|
|
AND agent_state = 'not_required' ''',
|
|
(
|
|
status, safe_category, resulting_json, now, now,
|
|
operation_id,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime worker admin operation completion conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime worker admin operation completion retry budget exhausted'
|
|
)
|
|
|
|
def create_runtime_managed_file_operation(
|
|
self, *, operation_id, actor, action, root_id, relative_path,
|
|
expected_sha256, proposed_sha256, proposed_byte_count,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
if action not in RUNTIME_MANAGED_FILE_ACTIONS:
|
|
raise ValueError('runtime managed file operation action is invalid')
|
|
expected = _runtime_managed_file_expected_identity({
|
|
'root_id': root_id,
|
|
'relative_path': relative_path,
|
|
'expected_sha256': expected_sha256,
|
|
'proposed_sha256': proposed_sha256,
|
|
'proposed_byte_count': proposed_byte_count,
|
|
}, action, root_id)
|
|
expected_json = _canonical_runtime_json(expected)
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
existing_row = self._locked_runtime_operation(operation_id)
|
|
if existing_row:
|
|
existing = self._runtime_operation_state(existing_row)
|
|
if not (
|
|
existing['actor'] == actor
|
|
and existing['action'] == action
|
|
and existing['target_kind'] == RUNTIME_MANAGED_FILE_TARGET_KIND
|
|
and existing['target_ref'] == root_id
|
|
and existing['expected_identity'] == expected
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
accepted = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = 'accepted'
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(accepted) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file accepted audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**existing, 'replayed': True,
|
|
'audit_event_id': int(accepted[0]['id']),
|
|
'audit_event_sha256': str(accepted[0]['event_sha256']),
|
|
}
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_identity_json, agent_state,
|
|
requested_at, started_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_MANAGED_FILE_TARGET_KIND,
|
|
root_id, expected_json, now, now, now,
|
|
),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_MANAGED_FILE_TARGET_KIND,
|
|
target_ref=root_id, result='accepted',
|
|
before_identity=expected,
|
|
after_bytes=expected['proposed_byte_count'], created_at=now,
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file operation creation retry budget exhausted'
|
|
)
|
|
|
|
def complete_runtime_managed_file_operation(
|
|
self, operation_id, *, succeeded, before_sha256=None,
|
|
before_byte_count=None, after_sha256=None, after_byte_count=None,
|
|
written=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if type(succeeded) is not bool:
|
|
raise ValueError('runtime managed file operation result is invalid')
|
|
supplied = (
|
|
before_sha256, before_byte_count, after_sha256, after_byte_count, written,
|
|
)
|
|
if not succeeded and any(value is not None for value in supplied):
|
|
raise ValueError('runtime managed file operation result is invalid')
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if current['action'] not in RUNTIME_MANAGED_FILE_ACTIONS:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is not a managed file action'
|
|
)
|
|
status = 'succeeded' if succeeded else 'failed'
|
|
safe_category = None if succeeded else 'managed_file_mutation_failed'
|
|
resulting = None
|
|
if succeeded:
|
|
resulting = _runtime_managed_file_resulting_identity({
|
|
'root_id': current['target_ref'],
|
|
'relative_path': current['expected_identity']['relative_path'],
|
|
'outcome': 'completed',
|
|
'before_sha256': before_sha256,
|
|
'before_byte_count': before_byte_count,
|
|
'after_sha256': after_sha256,
|
|
'after_byte_count': after_byte_count,
|
|
'written': written,
|
|
}, current['action'], current['target_ref'])
|
|
expected = current['expected_identity']
|
|
if (
|
|
current['action'] == 'files.create'
|
|
and (
|
|
resulting['after_sha256'] != expected['proposed_sha256']
|
|
or resulting['after_byte_count'] != expected['proposed_byte_count']
|
|
)
|
|
or current['action'] == 'files.replace'
|
|
and (
|
|
resulting['before_sha256'] != expected['expected_sha256']
|
|
or resulting['after_sha256'] != expected['proposed_sha256']
|
|
or resulting['after_byte_count'] != expected['proposed_byte_count']
|
|
or not resulting['written'] and (
|
|
resulting['before_sha256'] != resulting['after_sha256']
|
|
or resulting['before_byte_count'] != resulting['after_byte_count']
|
|
)
|
|
)
|
|
or current['action'] == 'files.delete'
|
|
and resulting['before_sha256'] != expected['expected_sha256']
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime managed file result does not match its accepted identity'
|
|
)
|
|
if current['status'] in ('succeeded', 'failed'):
|
|
if (
|
|
current['status'] != status
|
|
or current['safe_category'] != safe_category
|
|
or current['resulting_identity'] != resulting
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime managed file operation already has another result'
|
|
)
|
|
events = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id, status),
|
|
).fetchall()
|
|
if len(events) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file terminal audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': int(events[0]['id']),
|
|
'audit_event_sha256': str(events[0]['event_sha256']),
|
|
}
|
|
if current['status'] != 'running' or current['agent_state'] != 'not_required':
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime managed file operation cannot be completed'
|
|
)
|
|
now = utc_now_iso()
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=current['actor'],
|
|
action=current['action'],
|
|
target_kind=RUNTIME_MANAGED_FILE_TARGET_KIND,
|
|
target_ref=current['target_ref'], result=status,
|
|
safe_category=safe_category,
|
|
before_identity=current['expected_identity'],
|
|
after_identity=resulting,
|
|
before_bytes=(
|
|
resulting['before_byte_count'] if resulting is not None else None
|
|
),
|
|
after_bytes=(
|
|
resulting['after_byte_count'] if resulting is not None else None
|
|
),
|
|
created_at=now,
|
|
)
|
|
resulting_json = (
|
|
_canonical_runtime_json(resulting) if resulting is not None else None
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = ?, safe_category = ?, resulting_identity_json = ?,
|
|
completed_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'running'
|
|
AND agent_state = 'not_required' ''',
|
|
(
|
|
status, safe_category, resulting_json, now, now,
|
|
operation_id,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime managed file operation completion conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime managed file operation completion retry budget exhausted'
|
|
)
|
|
|
|
def create_runtime_document_operation(
|
|
self, *, operation_id, actor, action, active_config_sha256,
|
|
active_secrets_sha256, candidate_config_sha256, candidate_secrets_sha256,
|
|
candidate_after_sha256, candidate_before_bytes, candidate_after_bytes,
|
|
candidate_before_present,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
target_ref = RUNTIME_DOCUMENT_ACTION_TARGETS.get(action)
|
|
if target_ref is None:
|
|
raise ValueError('runtime document operation action is invalid')
|
|
expected = _runtime_document_expected_identity({
|
|
'document': target_ref,
|
|
'active_config_sha256': active_config_sha256,
|
|
'active_secrets_sha256': active_secrets_sha256,
|
|
'candidate_config_sha256': candidate_config_sha256,
|
|
'candidate_secrets_sha256': candidate_secrets_sha256,
|
|
'candidate_after_sha256': candidate_after_sha256,
|
|
'candidate_before_bytes': candidate_before_bytes,
|
|
'candidate_after_bytes': candidate_after_bytes,
|
|
'candidate_before_present': candidate_before_present,
|
|
}, action, target_ref)
|
|
expected_json = _canonical_runtime_json(expected)
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
existing_row = self._locked_runtime_operation(operation_id)
|
|
if existing_row:
|
|
existing = self._runtime_operation_state(existing_row)
|
|
if not (
|
|
existing['actor'] == actor
|
|
and existing['action'] == action
|
|
and existing['target_kind'] == RUNTIME_DOCUMENT_TARGET_KIND
|
|
and existing['target_ref'] == target_ref
|
|
and existing['expected_identity'] == expected
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
accepted = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = 'accepted'
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(accepted) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime document accepted audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**existing, 'replayed': True,
|
|
'audit_event_id': int(accepted[0]['id']),
|
|
'audit_event_sha256': str(accepted[0]['event_sha256']),
|
|
}
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_identity_json, agent_state,
|
|
requested_at, started_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'running', ?, 'not_required', ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_DOCUMENT_TARGET_KIND,
|
|
target_ref, expected_json, now, now, now,
|
|
),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_DOCUMENT_TARGET_KIND,
|
|
target_ref=target_ref, result='accepted',
|
|
before_identity=expected,
|
|
before_bytes=expected['candidate_before_bytes'],
|
|
after_bytes=expected['candidate_after_bytes'], created_at=now,
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime document operation creation retry budget exhausted'
|
|
)
|
|
|
|
def complete_runtime_document_operation(
|
|
self, operation_id, *, succeeded, candidate_sha256=None, written=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
if type(succeeded) is not bool:
|
|
raise ValueError('runtime document operation result is invalid')
|
|
if succeeded:
|
|
if type(written) is not bool:
|
|
raise ValueError('runtime document operation result is invalid')
|
|
candidate_sha256 = _runtime_sha256(
|
|
candidate_sha256, 'candidate result hash',
|
|
)
|
|
elif candidate_sha256 is not None or written is not None:
|
|
raise ValueError('runtime document operation result is invalid')
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if current['action'] not in RUNTIME_DOCUMENT_ACTION_TARGETS:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is not a document action'
|
|
)
|
|
if succeeded and not hmac.compare_digest(
|
|
candidate_sha256,
|
|
current['expected_identity']['candidate_after_sha256'],
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime document result does not match its accepted candidate'
|
|
)
|
|
status = 'succeeded' if succeeded else 'failed'
|
|
safe_category = None if succeeded else 'runtime_document_save_failed'
|
|
resulting = None
|
|
if succeeded:
|
|
resulting = _runtime_document_resulting_identity({
|
|
'document': current['target_ref'],
|
|
'outcome': 'completed',
|
|
'candidate_sha256': candidate_sha256,
|
|
'written': written,
|
|
}, current['action'], current['target_ref'])
|
|
if current['status'] in ('succeeded', 'failed'):
|
|
if (
|
|
current['status'] != status
|
|
or current['safe_category'] != safe_category
|
|
or current['resulting_identity'] != resulting
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime document operation already has another result'
|
|
)
|
|
events = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id, status),
|
|
).fetchall()
|
|
if len(events) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime document terminal audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': int(events[0]['id']),
|
|
'audit_event_sha256': str(events[0]['event_sha256']),
|
|
}
|
|
if current['status'] != 'running' or current['agent_state'] != 'not_required':
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime document operation cannot be completed'
|
|
)
|
|
now = utc_now_iso()
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=current['actor'],
|
|
action=current['action'], target_kind=RUNTIME_DOCUMENT_TARGET_KIND,
|
|
target_ref=current['target_ref'], result=status,
|
|
safe_category=safe_category,
|
|
before_identity=current['expected_identity'],
|
|
after_identity=resulting,
|
|
before_bytes=current['expected_identity']['candidate_before_bytes'],
|
|
after_bytes=current['expected_identity']['candidate_after_bytes'],
|
|
created_at=now,
|
|
)
|
|
resulting_json = (
|
|
_canonical_runtime_json(resulting) if resulting is not None else None
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = ?, safe_category = ?, resulting_identity_json = ?,
|
|
completed_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'running'
|
|
AND agent_state = 'not_required' ''',
|
|
(
|
|
status, safe_category, resulting_json, now, now,
|
|
operation_id,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime document operation completion conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime document operation completion retry budget exhausted'
|
|
)
|
|
|
|
def reconcile_runtime_operation_result(
|
|
self, result_envelope, *, max_bytes=RUNTIME_AGENT_RESULT_MAX_BYTES,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
parsed, result_sha256, failure = _runtime_result_envelope(
|
|
result_envelope, max_bytes,
|
|
)
|
|
result_envelope = None
|
|
if failure:
|
|
raise ValueError(failure)
|
|
operation_id = parsed['operation_id']
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._locked_runtime_control_state()
|
|
row = self._locked_runtime_operation(operation_id)
|
|
if not row:
|
|
self.conn.commit()
|
|
return None
|
|
current = self._runtime_operation_state(row)
|
|
if (
|
|
current['action'] != parsed['action']
|
|
or current['action'] not in RUNTIME_ASYNC_ACTIONS
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation result does not match its request'
|
|
)
|
|
existing_digest = current['agent_result_sha256']
|
|
if existing_digest is not None:
|
|
if not hmac.compare_digest(existing_digest, result_sha256):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation already has another result'
|
|
)
|
|
if (
|
|
current['status'] != parsed['result']
|
|
or current['agent_state'] != parsed['result']
|
|
or current['safe_category'] != parsed['safe_category']
|
|
or current['safe_detail'] != parsed['safe_detail']
|
|
or current['resulting_identity'] != parsed['resulting_identity']
|
|
):
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime operation replay result is inconsistent'
|
|
)
|
|
terminal_events = self.conn.execute(
|
|
'''SELECT id, event_sha256 FROM runtime_audit_events
|
|
WHERE operation_id = ? AND result = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id, parsed['result']),
|
|
).fetchall()
|
|
if len(terminal_events) != 1:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime operation terminal audit evidence is incomplete'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
**current, 'replayed': True,
|
|
'audit_event_id': int(terminal_events[0]['id']),
|
|
'audit_event_sha256': str(terminal_events[0]['event_sha256']),
|
|
}
|
|
if current['status'] not in ('requested', 'running'):
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation is already terminal'
|
|
)
|
|
if parsed['result'] == 'rolled_back':
|
|
expected = current['expected_identity']
|
|
if parsed['resulting_identity'] != {
|
|
'active_config_sha256': expected['active_config_sha256'],
|
|
'active_secrets_sha256': expected['active_secrets_sha256'],
|
|
}:
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'rolled-back runtime operation identity is invalid'
|
|
)
|
|
elif parsed['result'] == 'succeeded':
|
|
if parsed['resulting_identity'] != _runtime_success_identity(
|
|
current['expected_identity'], current['action'],
|
|
):
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'successful runtime operation identity is invalid'
|
|
)
|
|
now = utc_now_iso()
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=current['actor'],
|
|
action=current['action'], target_kind=current['target_kind'],
|
|
target_ref=current['target_ref'], result=parsed['result'],
|
|
safe_category=parsed['safe_category'],
|
|
before_identity=current['expected_identity'],
|
|
after_identity=parsed['resulting_identity'], created_at=now,
|
|
)
|
|
resulting_json = (
|
|
_canonical_runtime_json(parsed['resulting_identity'])
|
|
if parsed['resulting_identity'] is not None else None
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = ?, safe_category = ?, safe_detail = ?,
|
|
resulting_identity_json = ?, agent_state = ?,
|
|
agent_result_sha256 = ?,
|
|
started_at = COALESCE(started_at, ?),
|
|
completed_at = ?, agent_reconciled_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status IN ('requested','running')
|
|
AND agent_result_sha256 IS NULL''',
|
|
(
|
|
parsed['result'], parsed['safe_category'], parsed['safe_detail'],
|
|
resulting_json, parsed['result'], result_sha256, now, now,
|
|
now, now, operation_id,
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeOperationTransitionError(
|
|
'runtime operation reconciliation conflicted'
|
|
)
|
|
result = self._runtime_operation_state(
|
|
self._locked_runtime_operation(operation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {**result, 'replayed': False, **audit}
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime operation reconciliation retry budget exhausted')
|
|
|
|
def _runtime_control_replay_locked(
|
|
self, *, operation_id, actor, action, target_ref, expected_revision,
|
|
):
|
|
operation = self.conn.execute(
|
|
'SELECT * FROM runtime_operations WHERE operation_id = ?',
|
|
(operation_id,),
|
|
).fetchone()
|
|
if not operation:
|
|
return None
|
|
stored_expected_revision = operation['expected_revision']
|
|
stored_resulting_revision = operation['resulting_revision']
|
|
revisions_valid = (
|
|
isinstance(stored_expected_revision, int)
|
|
and not isinstance(stored_expected_revision, bool)
|
|
and isinstance(stored_resulting_revision, int)
|
|
and not isinstance(stored_resulting_revision, bool)
|
|
)
|
|
immutable_request = (
|
|
str(operation['actor'] or '') == actor
|
|
and str(operation['action'] or '') == action
|
|
and str(operation['target_kind'] or '') == RUNTIME_CONTROL_TARGET_KIND
|
|
and str(operation['target_ref'] or '') == target_ref
|
|
)
|
|
if not immutable_request:
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
if not revisions_valid:
|
|
raise RuntimeSafetySchemaError('runtime control replay revisions are invalid')
|
|
if stored_expected_revision != expected_revision:
|
|
raise RuntimeOperationIdentityConflictError(
|
|
'runtime operation ID is already bound to another request'
|
|
)
|
|
if (
|
|
operation['status'] != 'succeeded'
|
|
or operation['agent_state'] != 'not_required'
|
|
or operation['safe_category'] is not None
|
|
or operation['safe_detail'] is not None
|
|
or operation['agent_result_sha256'] is not None
|
|
or not operation['requested_at']
|
|
or not operation['started_at']
|
|
or not operation['completed_at']
|
|
or not operation['updated_at']
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime control replay operation is incomplete')
|
|
before_json = str(operation['expected_identity_json'] or '')
|
|
after_json = str(operation['resulting_identity_json'] or '')
|
|
before = _stored_runtime_control_identity(before_json)
|
|
after = _stored_runtime_control_identity(after_json)
|
|
if (
|
|
before['revision'] != expected_revision
|
|
or stored_resulting_revision != after['revision']
|
|
or after['revision'] != before['revision'] + 1
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime control replay revisions are invalid')
|
|
try:
|
|
expected_after = _runtime_control_transition(before, action)
|
|
except (RuntimeControlTransitionError, ValueError) as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'runtime control replay transition is invalid'
|
|
) from exc
|
|
if after != expected_after:
|
|
raise RuntimeSafetySchemaError('runtime control replay transition does not match')
|
|
events = self.conn.execute(
|
|
'''SELECT * FROM runtime_audit_events WHERE operation_id = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(operation_id,),
|
|
).fetchall()
|
|
if len(events) != 1:
|
|
raise RuntimeSafetySchemaError('runtime control replay audit evidence is incomplete')
|
|
event = events[0]
|
|
if (
|
|
str(event['actor'] or '') != actor
|
|
or str(event['action'] or '') != action
|
|
or str(event['target_kind'] or '') != RUNTIME_CONTROL_TARGET_KIND
|
|
or str(event['target_ref'] or '') != target_ref
|
|
or event['result'] != 'succeeded'
|
|
or event['safe_category'] is not None
|
|
or event['before_identity_json'] != before_json
|
|
or event['after_identity_json'] != after_json
|
|
or event['before_bytes'] is not None
|
|
or event['after_bytes'] is not None
|
|
or not event['created_at']
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime control replay audit evidence does not match')
|
|
previous_event_id = event['previous_event_id']
|
|
previous_event_sha256 = event['previous_event_sha256']
|
|
if previous_event_id is None:
|
|
if previous_event_sha256 is not None:
|
|
raise RuntimeSafetySchemaError('runtime control replay audit parent is invalid')
|
|
else:
|
|
parent = self.conn.execute(
|
|
'SELECT event_sha256 FROM runtime_audit_events WHERE id = ?',
|
|
(int(previous_event_id),),
|
|
).fetchone()
|
|
if (
|
|
not parent
|
|
or not hmac.compare_digest(
|
|
str(parent['event_sha256'] or ''), str(previous_event_sha256 or ''),
|
|
)
|
|
):
|
|
raise RuntimeSafetySchemaError('runtime control replay audit parent is invalid')
|
|
payload = {
|
|
'schema': 'runtime-audit-event-v1',
|
|
'operation_id': operation_id,
|
|
'actor': actor,
|
|
'action': action,
|
|
'target_kind': RUNTIME_CONTROL_TARGET_KIND,
|
|
'target_ref': target_ref,
|
|
'result': 'succeeded',
|
|
'safe_category': None,
|
|
'before_identity_json': before_json,
|
|
'after_identity_json': after_json,
|
|
'before_bytes': None,
|
|
'after_bytes': None,
|
|
'previous_event_id': int(previous_event_id) if previous_event_id is not None else None,
|
|
'previous_event_sha256': previous_event_sha256,
|
|
'created_at': str(event['created_at']),
|
|
}
|
|
expected_sha256 = _runtime_audit_event_sha256(payload)
|
|
if not hmac.compare_digest(expected_sha256, str(event['event_sha256'] or '')):
|
|
raise RuntimeSafetySchemaError('runtime control replay audit hash is invalid')
|
|
return {
|
|
'operation_id': operation_id,
|
|
'action': action,
|
|
'replayed': True,
|
|
'before': before,
|
|
'after': after,
|
|
'audit_event_id': int(event['id']),
|
|
'audit_event_sha256': expected_sha256,
|
|
}
|
|
|
|
def _commit_runtime_control_transition_locked(
|
|
self, *, state, action, target_ref, actor, operation_id,
|
|
):
|
|
before = _runtime_control_state_identity(state)
|
|
after = _runtime_control_transition(before, action)
|
|
before_json = _runtime_control_identity_json(before)
|
|
after_json = _runtime_control_identity_json(after)
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_operations(
|
|
operation_id, actor, action, target_kind, target_ref,
|
|
status, expected_revision, expected_identity_json,
|
|
agent_state, requested_at, started_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, 'not_required', ?, ?, ?)''',
|
|
(
|
|
operation_id, actor, action, RUNTIME_CONTROL_TARGET_KIND,
|
|
target_ref, before['revision'], before_json, now, now, now,
|
|
),
|
|
)
|
|
updated = self.conn.execute(
|
|
'''UPDATE runtime_operations_control
|
|
SET revision = ?, discovery_paused = ?, dispatch_paused = ?,
|
|
drain_state = ?, actor = ?, operation_id = ?, updated_at = ?
|
|
WHERE id = 1 AND revision = ?''',
|
|
(
|
|
after['revision'], int(after['discovery_paused']),
|
|
int(after['dispatch_paused']), after['drain_state'], actor,
|
|
operation_id, now, before['revision'],
|
|
),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeControlRevisionConflictError(
|
|
before['revision'], self._locked_runtime_control_state(),
|
|
)
|
|
audit = self._append_runtime_audit_event_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_kind=RUNTIME_CONTROL_TARGET_KIND, target_ref=target_ref,
|
|
result='succeeded', before_identity=before,
|
|
after_identity=after, created_at=now,
|
|
)
|
|
completed = self.conn.execute(
|
|
'''UPDATE runtime_operations
|
|
SET status = 'succeeded', resulting_revision = ?,
|
|
resulting_identity_json = ?, completed_at = ?, updated_at = ?
|
|
WHERE operation_id = ? AND status = 'running' ''',
|
|
(after['revision'], after_json, now, now, operation_id),
|
|
)
|
|
if int(completed.rowcount or 0) != 1:
|
|
raise RuntimeSafetySchemaError('runtime control operation completion failed')
|
|
return {
|
|
'operation_id': operation_id,
|
|
'action': action,
|
|
'replayed': False,
|
|
'before': before,
|
|
'after': after,
|
|
**audit,
|
|
}
|
|
|
|
def _runtime_control_mutation(
|
|
self, *, action, target_ref, expected_revision, actor, operation_id,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
operation_id = _runtime_operation_id(operation_id)
|
|
actor = _runtime_actor(actor)
|
|
expected_revision = _runtime_revision(expected_revision)
|
|
if expected_revision > RUNTIME_CONTROL_MAX_EXPECTED_REVISION:
|
|
raise ValueError('runtime control revision cannot be advanced')
|
|
if action not in RUNTIME_CONTROL_ACTIONS:
|
|
raise ValueError('runtime control action is invalid')
|
|
if RUNTIME_CONTROL_ACTION_TARGETS[action] != target_ref:
|
|
raise ValueError('runtime control target is invalid')
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
state = self._locked_runtime_control_state()
|
|
replay = self._runtime_control_replay_locked(
|
|
operation_id=operation_id, actor=actor, action=action,
|
|
target_ref=target_ref, expected_revision=expected_revision,
|
|
)
|
|
if replay is not None:
|
|
self.conn.commit()
|
|
return replay
|
|
if state['revision'] != expected_revision:
|
|
raise RuntimeControlRevisionConflictError(expected_revision, state)
|
|
result = self._commit_runtime_control_transition_locked(
|
|
state=state, action=action, target_ref=target_ref,
|
|
actor=actor, operation_id=operation_id,
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime control mutation retry budget exhausted')
|
|
|
|
def set_runtime_discovery_paused(
|
|
self, paused, *, expected_revision, actor, operation_id,
|
|
):
|
|
if type(paused) is not bool:
|
|
raise ValueError('runtime discovery pause state must be a boolean')
|
|
return self._runtime_control_mutation(
|
|
action='control.discovery.pause' if paused else 'control.discovery.resume',
|
|
target_ref='discovery', expected_revision=expected_revision,
|
|
actor=actor, operation_id=operation_id,
|
|
)
|
|
|
|
def set_runtime_dispatch_paused(
|
|
self, paused, *, expected_revision, actor, operation_id,
|
|
):
|
|
if type(paused) is not bool:
|
|
raise ValueError('runtime dispatch pause state must be a boolean')
|
|
return self._runtime_control_mutation(
|
|
action='control.dispatch.pause' if paused else 'control.dispatch.resume',
|
|
target_ref='dispatch', expected_revision=expected_revision,
|
|
actor=actor, operation_id=operation_id,
|
|
)
|
|
|
|
def start_runtime_drain(self, *, expected_revision, actor, operation_id):
|
|
return self._runtime_control_mutation(
|
|
action='control.drain.start', target_ref='drain',
|
|
expected_revision=expected_revision, actor=actor,
|
|
operation_id=operation_id,
|
|
)
|
|
|
|
def cancel_runtime_drain(self, *, expected_revision, actor, operation_id):
|
|
return self._runtime_control_mutation(
|
|
action='control.drain.cancel', target_ref='drain',
|
|
expected_revision=expected_revision, actor=actor,
|
|
operation_id=operation_id,
|
|
)
|
|
|
|
def _runtime_drain_blockers_locked(self):
|
|
row = self.conn.execute(
|
|
'''SELECT
|
|
(SELECT COUNT(*) FROM result_reservations
|
|
WHERE assignment_kind = 'remote'
|
|
AND remote_resolved_at IS NULL) AS live_remote_assignments,
|
|
(SELECT COUNT(*) FROM result_bundles
|
|
WHERE state IN ('ready', 'ingesting')) AS precommit_result_bundles'''
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeSafetySchemaError('runtime drain progress is unavailable')
|
|
values = {}
|
|
for name in ('live_remote_assignments', 'precommit_result_bundles'):
|
|
value = row[name]
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
raise RuntimeSafetySchemaError('runtime drain progress is invalid')
|
|
values[name] = value
|
|
values['blocker_count'] = sum(values.values())
|
|
return values
|
|
|
|
def runtime_drain_progress(self):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
try:
|
|
state = self._locked_runtime_control_state(shared=True)
|
|
progress = self._runtime_drain_blockers_locked()
|
|
self.conn.commit()
|
|
return {**state, **progress}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def reconcile_runtime_drain(self):
|
|
if not self.conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
actor = 'system:drain-reconciler'
|
|
operation_id = str(uuid.uuid4())
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
for attempt in range(attempts):
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
state = self._locked_runtime_control_state()
|
|
if state['drain_state'] != 'draining':
|
|
self.conn.commit()
|
|
return None
|
|
blockers = self._runtime_drain_blockers_locked()
|
|
if blockers['blocker_count']:
|
|
self.conn.commit()
|
|
return None
|
|
result = self._commit_runtime_control_transition_locked(
|
|
state=state, action='control.drain.complete',
|
|
target_ref='drain', actor=actor, operation_id=operation_id,
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception as exc:
|
|
self.conn.rollback()
|
|
if (
|
|
self.conn.is_sqlite and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
raise
|
|
raise RuntimeSafetySchemaError('runtime drain reconciliation retry budget exhausted')
|
|
|
|
def final_cutover_status(self):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return None
|
|
row = self.conn.execute(
|
|
'SELECT marker, checked_at, evidence_sha256 FROM runtime_final_cutover WHERE id = 1'
|
|
).fetchone()
|
|
self.conn.commit()
|
|
if not row:
|
|
return None
|
|
result = dict(row)
|
|
if (
|
|
result['marker'] != FINAL_CUTOVER_MARKER
|
|
or not result['checked_at']
|
|
or not re.fullmatch(r'[a-f0-9]{64}', str(result['evidence_sha256'] or ''))
|
|
):
|
|
return None
|
|
return result
|
|
|
|
def require_final_cutover(self):
|
|
if not self.final_cutover_status():
|
|
raise RuntimeSafetySchemaError(
|
|
'final PostgreSQL v2 cutover marker is absent or invalid; '
|
|
'run migrate_runtime_safety.py offline with --apply --sources-stopped'
|
|
)
|
|
return True
|
|
|
|
def record_final_cutover(self, evidence):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeSafetySchemaError('final cutover authority can only be recorded in PostgreSQL')
|
|
encoded = json.dumps(
|
|
evidence, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
if len(encoded) > 1024 * 1024:
|
|
raise ValueError('final cutover evidence exceeds its byte bound')
|
|
digest = hashlib.sha256(encoded).hexdigest()
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO runtime_final_cutover(id, marker, checked_at, evidence_sha256)
|
|
VALUES (1, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET marker = excluded.marker,
|
|
checked_at = excluded.checked_at,
|
|
evidence_sha256 = excluded.evidence_sha256''',
|
|
(FINAL_CUTOVER_MARKER, now, digest),
|
|
)
|
|
self.conn.commit()
|
|
return {'marker': FINAL_CUTOVER_MARKER, 'checked_at': now, 'evidence_sha256': digest}
|
|
|
|
def revoke_final_cutover(self):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeSafetySchemaError('final cutover authority can only be revoked in PostgreSQL')
|
|
self.conn.execute('DELETE FROM runtime_final_cutover WHERE id = 1')
|
|
self.conn.commit()
|
|
return True
|
|
|
|
def pipeline_schema_available(self, commit=True):
|
|
if not self.conn:
|
|
self.last_error = 'database connection is unavailable'
|
|
return False
|
|
try:
|
|
problems = []
|
|
for table, columns in PIPELINE_REQUIRED_COLUMNS.items():
|
|
if not self.conn.table_exists(table):
|
|
problems.append(f'table {table}')
|
|
continue
|
|
missing = columns - set(self.conn.table_columns(table))
|
|
if missing:
|
|
problems.append(f'{table} columns {", ".join(sorted(missing))}')
|
|
capacity = self.conn.execute(
|
|
'SELECT id FROM pipeline_capacity WHERE id = 1'
|
|
).fetchone() if self.conn.table_exists('pipeline_capacity') else None
|
|
if not capacity:
|
|
problems.append('pipeline_capacity singleton row')
|
|
if self.conn.table_exists('runtime_schema_migrations'):
|
|
rows = self.conn.execute(
|
|
'SELECT version, code_sha256 FROM runtime_schema_migrations'
|
|
).fetchall()
|
|
applied = {str(row['version']) for row in rows}
|
|
missing_versions = set(PIPELINE_MIGRATION_VERSIONS) - applied
|
|
if missing_versions:
|
|
problems.append('migration versions ' + ', '.join(sorted(missing_versions)))
|
|
marker = next(
|
|
(
|
|
row for row in rows
|
|
if str(row['version']) == PIPELINE_MIGRATION_VERSIONS[-1]
|
|
),
|
|
None,
|
|
)
|
|
expected_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
|
|
if marker and str(marker['code_sha256'] or '') != expected_code:
|
|
problems.append('pipeline migration marker code identity')
|
|
if self.conn.table_exists('pipeline_quarantine'):
|
|
constraint = _pipeline_quarantine_review_status_constraint(self.conn)
|
|
if (
|
|
constraint.get('statuses') != PIPELINE_QUARANTINE_REVIEW_STATUSES
|
|
or not constraint.get('valid')
|
|
):
|
|
problems.append('pipeline quarantine review-status constraint')
|
|
problems.extend(_docker_depth_check_constraint_problems(self.conn))
|
|
required_indexes = {
|
|
'target_queue': (
|
|
'idx_target_queue_current_reservation', 'idx_target_queue_claimable_v2',
|
|
'idx_target_queue_cold',
|
|
),
|
|
'target_queue_policy_events': (
|
|
'idx_target_queue_policy_events_queue',
|
|
'idx_target_queue_policy_events_manifest',
|
|
),
|
|
'target_scans': (
|
|
'idx_target_scans_event_hash', 'idx_target_scans_queue_id',
|
|
'idx_target_scans_result_reservation',
|
|
),
|
|
'findings': ('idx_findings_target_scan_id_id',),
|
|
'result_reservations': (
|
|
'uq_result_reservation_queue_lease', 'idx_result_reservations_recovery',
|
|
'idx_result_reservations_remote_active_user',
|
|
'idx_result_reservations_remote_expiry',
|
|
'idx_result_reservations_remote_device_history',
|
|
'uq_result_reservations_remote_receipt',
|
|
),
|
|
'remote_worker_users': ('uq_remote_worker_users_key',),
|
|
'remote_worker_devices': (
|
|
'uq_remote_worker_devices_key', 'uq_remote_worker_devices_token',
|
|
),
|
|
'admission_intents': ('idx_admission_intents_remote_device',),
|
|
'result_bundles': ('idx_result_bundles_ready', 'idx_result_bundles_ingest_lease'),
|
|
'projection_jobs': ('idx_projection_jobs_claim', 'idx_projection_jobs_lease'),
|
|
'keycheck_candidates': (
|
|
'idx_keycheck_candidates_pending', 'idx_keycheck_candidates_deferred',
|
|
'idx_keycheck_candidates_lease',
|
|
),
|
|
'keycheck_credentials': ('uq_keycheck_credentials_provider_key',),
|
|
'pipeline_artifacts': ('idx_pipeline_artifacts_owner',),
|
|
'docker_content_blobs': (
|
|
'idx_docker_content_blobs_reclaim',
|
|
'idx_docker_content_blobs_reservation',
|
|
),
|
|
'docker_image_blob_coverage': (
|
|
'idx_docker_image_blob_coverage_manifest',
|
|
'idx_docker_image_blob_coverage_blob',
|
|
'idx_docker_image_blob_coverage_reservation',
|
|
'idx_docker_image_blob_coverage_selection',
|
|
),
|
|
'docker_adaptive_shadow_reports': (
|
|
'idx_docker_adaptive_shadow_reports_gate',
|
|
),
|
|
'discovery_retry_queue': (
|
|
'uq_discovery_retry_queue_work_key',
|
|
'idx_discovery_retry_queue_due',
|
|
'idx_discovery_retry_queue_lease',
|
|
'idx_discovery_retry_queue_policy',
|
|
),
|
|
'runtime_operations': (
|
|
'idx_runtime_operations_status_updated',
|
|
'idx_runtime_operations_agent_state_updated',
|
|
),
|
|
'runtime_audit_events': (
|
|
'idx_runtime_audit_events_created',
|
|
'idx_runtime_audit_events_operation',
|
|
),
|
|
'worker_progress_events': (
|
|
'uq_worker_progress_reservation_sequence',
|
|
'idx_worker_progress_reservation_received',
|
|
'idx_worker_progress_device_received',
|
|
'idx_worker_progress_phase_received',
|
|
),
|
|
'worker_diagnostics': (
|
|
'uq_worker_diagnostics_uid',
|
|
'idx_worker_diagnostics_reservation_received',
|
|
'idx_worker_diagnostics_scan_received',
|
|
'idx_worker_diagnostics_phase_received',
|
|
'idx_worker_diagnostics_category_code',
|
|
'idx_worker_diagnostics_code_received',
|
|
'idx_worker_diagnostics_kind_received',
|
|
'idx_worker_diagnostics_retryable_received',
|
|
),
|
|
}
|
|
for table, names in required_indexes.items():
|
|
indexes = self.conn.table_indexes(table) if self.conn.table_exists(table) else {}
|
|
for name in names:
|
|
if not _index_usable(indexes.get(name)):
|
|
problems.append(f'index {name}')
|
|
experiment_indexes = {}
|
|
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
|
|
indexes = experiment_indexes.setdefault(
|
|
table,
|
|
self.conn.table_indexes(table) if self.conn.table_exists(table) else {},
|
|
)
|
|
index = indexes.get(name)
|
|
if (
|
|
not index
|
|
or index['columns'] != list(columns)
|
|
or bool(index['unique']) != bool(unique)
|
|
or _normalized_predicate(index['predicate']) != _normalized_predicate(predicate)
|
|
or not _index_usable(index)
|
|
):
|
|
problems.append(f'index {name}')
|
|
credential_index = (
|
|
self.conn.table_indexes('keycheck_credentials').get('uq_keycheck_credentials_provider_key')
|
|
if self.conn.table_exists('keycheck_credentials') else None
|
|
)
|
|
if not credential_index or not credential_index['unique'] or credential_index['columns'] != [
|
|
'service', 'provider_key_hash'
|
|
]:
|
|
problems.append('unique provider-canonical keycheck credential index')
|
|
if self.conn.is_postgres and commit:
|
|
self.conn.commit()
|
|
if problems:
|
|
self.last_error = 'pipeline schema is incomplete: ' + '; '.join(problems)
|
|
return False
|
|
self.last_error = ''
|
|
return True
|
|
except Exception as exc:
|
|
self.last_error = f'unable to validate pipeline schema: {exc}'
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
return False
|
|
|
|
def provider_routing_finding_rows(self, secret_hash, limit, max_json_chars):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
secret_hash = str(secret_hash or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', secret_hash):
|
|
raise ValueError('provider routing lookup requires one SHA-256 digest')
|
|
limit = min(1025, max(1, int(limit or 0)))
|
|
max_json_chars = min(1024 * 1024, max(1024, int(max_json_chars or 0)))
|
|
return self.conn.execute(
|
|
'''SELECT
|
|
detector_name,
|
|
SUBSTR(COALESCE(raw_finding_json, ''), 1, ?) AS raw_finding_json,
|
|
CASE WHEN LENGTH(COALESCE(raw_finding_json, '')) > ? THEN 1 ELSE 0 END AS truncated
|
|
FROM findings
|
|
WHERE secret_hash = ?
|
|
ORDER BY id DESC
|
|
LIMIT ?''',
|
|
(max_json_chars, max_json_chars, secret_hash, limit),
|
|
).fetchall()
|
|
|
|
def ensure_scan_publication_outbox(self):
|
|
if not self.conn or not self.conn.table_exists('scan_publication_outbox'):
|
|
self.last_error = 'scan_publication_outbox is unavailable'
|
|
return False
|
|
required = {
|
|
'id', 'target_scan_id', 'payload_json', 'status', 'attempts', 'last_error',
|
|
'lease_owner', 'lease_expires_at', 'available_after', 'created_at', 'delivered_at', 'updated_at',
|
|
}
|
|
missing = required - set(self.conn.table_columns('scan_publication_outbox'))
|
|
if missing:
|
|
self.last_error = f'scan_publication_outbox is missing columns: {", ".join(sorted(missing))}'
|
|
return False
|
|
self.last_error = ''
|
|
return True
|
|
|
|
def scan_publication_backlog_health(self, max_items, max_bytes, max_age_sec, additional_items=0):
|
|
if not self.conn or not self.conn.table_exists('scan_publication_outbox'):
|
|
return {'healthy': False, 'accepting': False, 'reason': 'publication outbox is unavailable'}
|
|
|
|
def op():
|
|
now = datetime.now(timezone.utc)
|
|
payload_size = (
|
|
"OCTET_LENGTH(COALESCE(s.raw_result_json, ''))"
|
|
if self.conn.is_postgres
|
|
else "LENGTH(CAST(COALESCE(s.raw_result_json, '') AS BLOB))"
|
|
)
|
|
row = self.conn.execute(
|
|
f'''SELECT COUNT(*) AS item_count,
|
|
COALESCE(SUM({payload_size}), 0) AS payload_bytes,
|
|
MIN(o.created_at) AS oldest_at
|
|
FROM scan_publication_outbox o
|
|
LEFT JOIN target_scans s ON s.id = o.target_scan_id
|
|
WHERE o.status IN ('pending', 'delivering', 'dead')'''
|
|
).fetchone()
|
|
item_count = int(row['item_count'] or 0)
|
|
payload_bytes = int(row['payload_bytes'] or 0)
|
|
oldest = parse_time(row['oldest_at'])
|
|
oldest_age_sec = max(0, int((now - oldest).total_seconds())) if oldest else 0
|
|
item_limit = max(1, int(max_items))
|
|
byte_limit = max(1, int(max_bytes))
|
|
age_limit = max(1, int(max_age_sec))
|
|
reasons = []
|
|
if item_count > item_limit:
|
|
reasons.append(f'items={item_count}>{item_limit}')
|
|
if payload_bytes > byte_limit:
|
|
reasons.append(f'bytes={payload_bytes}>{byte_limit}')
|
|
if oldest_age_sec > age_limit:
|
|
reasons.append(f'oldest_age_sec={oldest_age_sec}>{age_limit}')
|
|
healthy = not reasons
|
|
accepting = healthy and item_count + max(0, int(additional_items or 0)) <= item_limit
|
|
if not accepting and not reasons:
|
|
reasons.append(f'items={item_count}+{max(0, int(additional_items or 0))}>{item_limit}')
|
|
return {
|
|
'healthy': healthy,
|
|
'accepting': accepting,
|
|
'reason': '; '.join(reasons),
|
|
'items': item_count,
|
|
'bytes': payload_bytes,
|
|
'oldest_age_sec': oldest_age_sec,
|
|
'max_items': item_limit,
|
|
'max_bytes': byte_limit,
|
|
'max_age_sec': age_limit,
|
|
}
|
|
return self._safe('scan_publication_backlog_health', op, {
|
|
'healthy': False, 'accepting': False, 'reason': self.last_error or 'publication backlog query failed',
|
|
})
|
|
|
|
def claim_scan_publications(self, lease_owner, limit=100, lease_seconds=300, max_attempts=None):
|
|
if not self.conn or not self.conn.table_exists('scan_publication_outbox') or not lease_owner:
|
|
return []
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
compact_delivered_scan_publications(self.conn, now)
|
|
lease_until = datetime.fromtimestamp(time.time() + max(60, int(lease_seconds)), timezone.utc).isoformat(timespec='seconds')
|
|
self.conn.execute(
|
|
'''UPDATE scan_publication_outbox SET status = 'pending', payload_json = '',
|
|
lease_owner = NULL, lease_expires_at = NULL, available_after = COALESCE(available_after, ?), updated_at = ?
|
|
WHERE status = 'dead' ''',
|
|
(now, now),
|
|
)
|
|
self.conn.execute("UPDATE scan_publication_outbox SET payload_json = '' WHERE payload_json != ''")
|
|
self.conn.execute(
|
|
'''UPDATE scan_publication_outbox SET status = 'pending', lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
|
|
WHERE status = 'delivering' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?''',
|
|
(now, now),
|
|
)
|
|
limit_value = max(1, int(limit or 100))
|
|
if self.conn.is_postgres:
|
|
rows = self.conn.execute(
|
|
'''WITH picked AS (
|
|
SELECT id FROM scan_publication_outbox
|
|
WHERE status = 'pending' AND (available_after IS NULL OR available_after <= ?)
|
|
ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE scan_publication_outbox o SET status = 'delivering', lease_owner = ?,
|
|
lease_expires_at = ?, attempts = CASE WHEN COALESCE(attempts, 0) < 1000000
|
|
THEN COALESCE(attempts, 0) + 1 ELSE COALESCE(attempts, 0) END, updated_at = ?
|
|
FROM picked WHERE o.id = picked.id''',
|
|
(now, limit_value, lease_owner, lease_until, now),
|
|
)
|
|
else:
|
|
rows = self.conn.execute(
|
|
'''SELECT id FROM scan_publication_outbox
|
|
WHERE status = 'pending' AND (available_after IS NULL OR available_after <= ?)
|
|
ORDER BY id LIMIT ?''',
|
|
(now, limit_value),
|
|
).fetchall()
|
|
for row in rows:
|
|
self.conn.execute(
|
|
'''UPDATE scan_publication_outbox SET status = 'delivering', lease_owner = ?,
|
|
lease_expires_at = ?, attempts = CASE WHEN COALESCE(attempts, 0) < 1000000
|
|
THEN COALESCE(attempts, 0) + 1 ELSE COALESCE(attempts, 0) END,
|
|
updated_at = ? WHERE id = ?''',
|
|
(lease_owner, lease_until, now, row['id']),
|
|
)
|
|
rows = self.conn.execute(
|
|
'''SELECT o.id, o.target_scan_id, COALESCE(s.raw_result_json, '') AS payload_json, o.attempts
|
|
FROM scan_publication_outbox o
|
|
JOIN target_scans s ON s.id = o.target_scan_id
|
|
WHERE o.status = 'delivering' AND o.lease_owner = ?
|
|
ORDER BY o.id LIMIT ?''',
|
|
(lease_owner, limit_value),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return rows
|
|
return self._safe('claim_scan_publications', op, [])
|
|
|
|
def renew_scan_publication(self, outbox_id, lease_owner, lease_seconds=300):
|
|
if not self.conn or outbox_id is None or not lease_owner:
|
|
return False
|
|
try:
|
|
now = utc_now_iso()
|
|
lease_until = datetime.fromtimestamp(
|
|
time.time() + max(60, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
cur = self.conn.execute(
|
|
'''UPDATE scan_publication_outbox SET lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
|
|
(lease_until, now, outbox_id, lease_owner),
|
|
)
|
|
renewed = int(getattr(cur, 'rowcount', 0) or 0) == 1
|
|
self.conn.commit()
|
|
self.last_error = ''
|
|
return renewed
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
logger.error(f'Observability DB write failed during renew_scan_publication: {exc}')
|
|
return False
|
|
|
|
def finish_scan_publication(self, outbox_id, lease_owner, delivered, error=''):
|
|
if not self.conn:
|
|
return False
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
if delivered:
|
|
cur = self.conn.execute(
|
|
'''DELETE FROM scan_publication_outbox
|
|
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
|
|
(outbox_id, lease_owner),
|
|
)
|
|
else:
|
|
claimed = self.conn.execute(
|
|
'''SELECT attempts FROM scan_publication_outbox
|
|
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
|
|
(outbox_id, lease_owner),
|
|
).fetchone()
|
|
if not claimed:
|
|
self.conn.rollback()
|
|
return False
|
|
attempts = max(1, int(claimed['attempts'] or 1))
|
|
base = max(1, env_int('SCAN_OUTBOX_RETRY_BASE_SEC', 30))
|
|
maximum = max(base, env_int('SCAN_OUTBOX_RETRY_MAX_SEC', 3600))
|
|
delay = min(maximum, base * (2 ** min(attempts - 1, 20)))
|
|
available_after = datetime.fromtimestamp(
|
|
time.time() + delay, timezone.utc
|
|
).isoformat(timespec='seconds')
|
|
cur = self.conn.execute(
|
|
'''UPDATE scan_publication_outbox SET status = 'pending', last_error = ?, payload_json = '',
|
|
lease_owner = NULL, lease_expires_at = NULL, available_after = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'delivering' AND lease_owner = ?''',
|
|
(first_line(error, 500), available_after, now, outbox_id, lease_owner),
|
|
)
|
|
self.conn.commit()
|
|
return int(getattr(cur, 'rowcount', 0) or 0) == 1
|
|
return self._safe('finish_scan_publication', op, False)
|
|
|
|
def sync_target_queue_from_files(self, source, platform, todo_targets=None, checked_targets=None, query=None):
|
|
if not self.conn:
|
|
return False
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
for target in checked_targets or []:
|
|
normalized = normalize_target(target, platform)
|
|
self.conn.execute(
|
|
f'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status, created_at, updated_at, completed_at
|
|
) VALUES (?, ?, ?, ?, ?, 'done', ?, ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO UPDATE SET
|
|
status = CASE WHEN target_queue.status IN ('failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status ELSE 'done' END,
|
|
target = excluded.target,
|
|
platform = excluded.platform,
|
|
completed_at = CASE
|
|
WHEN target_queue.status IN ('deferred', 'in_progress', 'cold') THEN target_queue.completed_at
|
|
ELSE COALESCE(target_queue.completed_at, excluded.completed_at)
|
|
END,
|
|
lease_owner = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_owner ELSE NULL END,
|
|
lease_token = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_token ELSE NULL END,
|
|
claim_batch = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.claim_batch ELSE NULL END,
|
|
leased_at = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.leased_at ELSE NULL END,
|
|
lease_expires_at = CASE WHEN target_queue.status = 'in_progress' THEN target_queue.lease_expires_at ELSE NULL END,
|
|
available_after = CASE WHEN target_queue.status IN ('failed', 'deferred', 'in_progress', 'cold') THEN target_queue.available_after ELSE NULL END,
|
|
updated_at = excluded.updated_at
|
|
WHERE target_queue.status NOT IN ('done', 'failed', 'deferred', 'in_progress', 'cold')
|
|
OR target_queue.target <> excluded.target
|
|
OR target_queue.platform <> excluded.platform''',
|
|
(source, platform, query, target, normalized, now, now, now),
|
|
)
|
|
for target in todo_targets or []:
|
|
normalized = normalize_target(target, platform)
|
|
self.conn.execute(
|
|
f'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO UPDATE SET
|
|
target = excluded.target,
|
|
platform = excluded.platform,
|
|
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN excluded.query ELSE COALESCE(target_queue.query, excluded.query) END,
|
|
status = CASE
|
|
WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 'pending'
|
|
WHEN target_queue.status IN ('done', 'failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status
|
|
WHEN target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL THEN 'deferred'
|
|
ELSE 'pending'
|
|
END,
|
|
available_after = CASE
|
|
WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL
|
|
WHEN target_queue.status IN ('pending', 'deferred', 'cold') THEN target_queue.available_after
|
|
ELSE NULL
|
|
END,
|
|
attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN 0 ELSE target_queue.attempts END,
|
|
last_error = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.last_error END,
|
|
completed_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.completed_at END,
|
|
resolver_state = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_state END,
|
|
resolver_due_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_due_at END,
|
|
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN 0 ELSE target_queue.resolver_attempts END,
|
|
resolver_token = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_token END,
|
|
updated_at = excluded.updated_at
|
|
WHERE ({ADMIN_DISCARDED_QUEUE_SQL})
|
|
OR target_queue.target <> excluded.target
|
|
OR target_queue.platform <> excluded.platform
|
|
OR (target_queue.query IS NULL AND excluded.query IS NOT NULL)
|
|
OR (target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL)''',
|
|
(source, platform, query, target, normalized, now, now),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
return self._safe('sync_target_queue_from_files', op, False)
|
|
|
|
def _docker_discovery_pass_locked(
|
|
self, source, query, observation, now, pass_id=None,
|
|
):
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
params = (
|
|
source,
|
|
observation['pass_kind'],
|
|
observation['collection_generation'],
|
|
observation['policy_sha256'],
|
|
observation['ordered_query_hash'],
|
|
observation['query_count'],
|
|
)
|
|
if pass_id is not None:
|
|
rows = self.conn.execute(
|
|
f'''SELECT * FROM docker_discovery_passes
|
|
WHERE id = ? AND source = ? AND pass_kind = ?
|
|
AND collection_generation = ? AND policy_sha256 = ?
|
|
AND ordered_queries_sha256 = ?
|
|
AND expected_query_count = ?
|
|
AND state IN ('collecting','complete'){lock_suffix}''',
|
|
(pass_id, *params),
|
|
).fetchall()
|
|
if len(rows) != 1:
|
|
raise RuntimeError('DockerHub discovery retry pass identity changed')
|
|
candidates = rows
|
|
else:
|
|
cycle_predicate = (
|
|
'page.source_cycle_id IS NULL'
|
|
if observation['cycle_id'] is None
|
|
else 'page.source_cycle_id = ?'
|
|
)
|
|
cycle_params = (
|
|
() if observation['cycle_id'] is None
|
|
else (observation['cycle_id'],)
|
|
)
|
|
candidates = self.conn.execute(
|
|
f'''SELECT * FROM docker_discovery_passes
|
|
WHERE source = ? AND pass_kind = ? AND collection_generation = ?
|
|
AND policy_sha256 = ?
|
|
AND ordered_queries_sha256 = ? AND expected_query_count = ?
|
|
AND (
|
|
state = 'collecting' OR EXISTS (
|
|
SELECT 1 FROM docker_discovery_pages page
|
|
WHERE page.pass_id = docker_discovery_passes.id
|
|
AND page.query_ordinal = ? AND page.page_number = ?
|
|
AND {cycle_predicate}
|
|
)
|
|
)
|
|
ORDER BY id{lock_suffix}''',
|
|
(
|
|
*params, observation['query_ordinal'],
|
|
observation['page_number'], *cycle_params,
|
|
),
|
|
).fetchall()
|
|
|
|
empty_candidate = None
|
|
selected = None
|
|
for candidate in candidates:
|
|
pages = self.conn.execute(
|
|
'''SELECT query, page_number, source_cycle_id
|
|
FROM docker_discovery_pages
|
|
WHERE pass_id = ? AND query_ordinal = ? ORDER BY page_number''',
|
|
(candidate['id'], observation['query_ordinal']),
|
|
).fetchall()
|
|
if any(str(page['query']) != query for page in pages):
|
|
raise RuntimeError('DockerHub discovery pass query identity conflicts')
|
|
exact_page = [
|
|
page for page in pages
|
|
if int(page['page_number']) == observation['page_number']
|
|
]
|
|
if pass_id is not None:
|
|
if str(candidate['state']) == 'complete' and not exact_page:
|
|
raise RuntimeError('DockerHub discovery complete retry pass cannot admit another page')
|
|
return candidate
|
|
same_cycle = any(
|
|
(
|
|
page['source_cycle_id'] is None
|
|
and observation['cycle_id'] is None
|
|
) or (
|
|
page['source_cycle_id'] is not None
|
|
and observation['cycle_id'] is not None
|
|
and int(page['source_cycle_id']) == observation['cycle_id']
|
|
)
|
|
for page in pages
|
|
)
|
|
if exact_page and (same_cycle or observation['cycle_id'] is None):
|
|
selected = candidate
|
|
break
|
|
if same_cycle:
|
|
if str(candidate['state']) == 'complete':
|
|
raise RuntimeError('DockerHub discovery complete pass cannot admit another page')
|
|
selected = candidate
|
|
break
|
|
if not pages and str(candidate['state']) == 'collecting' and empty_candidate is None:
|
|
empty_candidate = candidate
|
|
selected = selected or empty_candidate
|
|
if selected is not None:
|
|
return selected
|
|
if pass_id is not None:
|
|
raise RuntimeError('DockerHub discovery retry pass is unavailable')
|
|
|
|
token = secrets.token_urlsafe(32)
|
|
row = self.conn.execute(
|
|
'''INSERT INTO docker_discovery_passes(
|
|
experiment_id, pass_token, source, pass_kind,
|
|
collection_generation, policy_sha256,
|
|
ordered_queries_sha256, expected_query_count,
|
|
completed_query_count, state, started_at, completed_at,
|
|
created_at, updated_at
|
|
) VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, 0, 'collecting', ?, NULL, ?, ?)
|
|
RETURNING *''',
|
|
(
|
|
token, source, observation['pass_kind'],
|
|
observation['collection_generation'], observation['policy_sha256'],
|
|
observation['ordered_query_hash'], observation['query_count'],
|
|
now, now, now,
|
|
),
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeError('DockerHub discovery pass could not be opened')
|
|
return row
|
|
|
|
def persist_dockerhub_discovery_page(
|
|
self, source, query, repositories, retry_id=None, lease_owner=None,
|
|
lease_token=None, next_page=None, complete=False, observation=None,
|
|
experiment_authority=None, final_cutover=False,
|
|
):
|
|
"""Durably admit one normalized page without changing existing target state."""
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
source = _validated_discovery_retry_source(source)
|
|
query = _validated_discovery_retry_query(query)
|
|
attempted_count, normalized, ordinals, observed = (
|
|
_normalized_dockerhub_repositories(repositories)
|
|
)
|
|
observation = _validated_docker_discovery_observation(
|
|
observation, source, query,
|
|
)
|
|
if observation is not None:
|
|
if observation['total_count'] is None:
|
|
raise ValueError('DockerHub page admission lacks total-count evidence')
|
|
absolute_start = (
|
|
observation['page_number'] - 1
|
|
) * observation['per_page']
|
|
absolute_bound = absolute_start + attempted_count
|
|
terminal_by_count = observation['total_count'] <= (
|
|
observation['page_number'] * observation['per_page']
|
|
)
|
|
if (
|
|
attempted_count > observation['per_page']
|
|
or observation['total_count'] < absolute_bound
|
|
or (attempted_count == 0 and observation['total_count'] > absolute_start)
|
|
or (
|
|
observation['query_complete']
|
|
and not terminal_by_count
|
|
and observation['page_number'] != observation['page_limit']
|
|
)
|
|
or (
|
|
not observation['query_complete']
|
|
and (
|
|
terminal_by_count
|
|
or observation['page_number'] == observation['page_limit']
|
|
)
|
|
)
|
|
):
|
|
raise ValueError('DockerHub page cardinality or continuation evidence conflicts')
|
|
if not isinstance(complete, bool):
|
|
raise ValueError('discovery retry completion flag is invalid')
|
|
retry_requested = any(
|
|
value is not None for value in (retry_id, lease_owner, lease_token, next_page)
|
|
) or complete
|
|
fence = None
|
|
if retry_requested:
|
|
fence = _validated_discovery_retry_fence(retry_id, lease_owner, lease_token)
|
|
if complete and next_page is not None:
|
|
raise ValueError('completed discovery retry work cannot retain a next page')
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._require_discovery_admission_locked()
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
page_experiment = None
|
|
page_authority = None
|
|
if self.conn.is_postgres and source == 'dockerhub':
|
|
page_experiment, page_authority, authority_reason = (
|
|
self._locked_docker_depth_page_authority(
|
|
source, query, observation, experiment_authority,
|
|
final_cutover=final_cutover, now=now,
|
|
)
|
|
)
|
|
if authority_reason:
|
|
self.conn.commit()
|
|
raise ScanEventConflictError(
|
|
f'Docker depth page authority drifted: {authority_reason}'
|
|
)
|
|
retry_row = None
|
|
pass_row = None
|
|
if fence:
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
retry_row = self.conn.execute(
|
|
f'''SELECT work_key, policy_sha256, pass_kind, work_kind,
|
|
page_start, page_end, next_page, source_cycle_id
|
|
FROM discovery_retry_queue
|
|
WHERE id = ? AND source = ? AND query = ? AND status = 'leased'
|
|
AND lease_owner = ? AND lease_token = ?
|
|
AND lease_expires_at > ?{lock_suffix}''',
|
|
(fence[0], source, query, fence[1], fence[2], now),
|
|
).fetchone()
|
|
if not retry_row:
|
|
raise DiscoveryRetryLeaseError('discovery retry page admission lost its lease fence')
|
|
if next_page is not None:
|
|
if isinstance(next_page, bool):
|
|
raise ValueError('discovery retry next page is invalid')
|
|
try:
|
|
next_page = int(next_page)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry next page is invalid') from None
|
|
if not (
|
|
int(retry_row['next_page']) <= next_page <= int(retry_row['page_end'])
|
|
and 1 <= next_page <= DISCOVERY_RETRY_MAX_PAGE
|
|
):
|
|
raise ValueError('discovery retry next page is outside its durable range')
|
|
|
|
retry_identity = (
|
|
source, query, retry_row['policy_sha256'], retry_row['pass_kind'],
|
|
retry_row['work_kind'], retry_row['page_start'], retry_row['page_end'],
|
|
)
|
|
legacy_work_key = discovery_retry_work_key(*retry_identity)
|
|
if str(retry_row['work_key']) == legacy_work_key:
|
|
observation = None
|
|
else:
|
|
try:
|
|
pass_id = int(str(retry_row['work_key'])[:16], 16)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise RuntimeError('DockerHub discovery retry pass identity is invalid') from None
|
|
if str(retry_row['work_key']) != discovery_retry_work_key(
|
|
*retry_identity, pass_id=pass_id,
|
|
):
|
|
raise RuntimeError('DockerHub discovery retry pass identity conflicts')
|
|
if observation is None:
|
|
raise RuntimeError('DockerHub discovery retry provenance is unavailable')
|
|
retry_cycle_id = (
|
|
int(retry_row['source_cycle_id'])
|
|
if retry_row['source_cycle_id'] is not None else None
|
|
)
|
|
if (
|
|
observation['policy_sha256'] != str(retry_row['policy_sha256'])
|
|
or observation['pass_kind'] != str(retry_row['pass_kind'])
|
|
or observation['page_number'] != int(retry_row['next_page'])
|
|
or observation['cycle_id'] != retry_cycle_id
|
|
):
|
|
raise RuntimeError('DockerHub discovery retry provenance changed')
|
|
pass_row = self._docker_discovery_pass_locked(
|
|
source, query, observation, now, pass_id=pass_id,
|
|
)
|
|
elif observation is not None:
|
|
pass_row = self._docker_discovery_pass_locked(
|
|
source, query, observation, now,
|
|
)
|
|
|
|
preexisting = set()
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
for start in range(0, len(normalized), KNOWN_TARGET_LOOKUP_BATCH_SIZE):
|
|
values = normalized[start:start + KNOWN_TARGET_LOOKUP_BATCH_SIZE]
|
|
rows = self.conn.execute(
|
|
'''SELECT normalized_target FROM target_queue
|
|
WHERE source = ? AND platform = 'docker'
|
|
AND normalized_target IN ({}){}'''.format(
|
|
','.join('?' for _ in values), lock_suffix,
|
|
),
|
|
(source, *values),
|
|
).fetchall()
|
|
preexisting.update(
|
|
str(row['normalized_target']) for row in rows if row['normalized_target']
|
|
)
|
|
|
|
resolver_due = (now_dt + timedelta(
|
|
seconds=max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)),
|
|
)).isoformat(timespec='seconds')
|
|
inserted_count = 0
|
|
queue_ids = {}
|
|
new_queue_ids = set()
|
|
for repository in normalized:
|
|
inserted = self.conn.execute(
|
|
'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status,
|
|
available_after, last_error, resolver_state, resolver_due_at,
|
|
created_at, updated_at
|
|
) VALUES (?, 'docker', ?, ?, ?, 'deferred', ?,
|
|
'Docker tag resolution unresolved', 'pending', ?, ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO NOTHING
|
|
RETURNING id''',
|
|
(
|
|
source, query, repository, repository, resolver_due,
|
|
resolver_due, now, now,
|
|
),
|
|
).fetchone()
|
|
inserted_count += int(bool(inserted))
|
|
if inserted:
|
|
queue_ids[repository] = int(inserted['id'])
|
|
new_queue_ids.add(int(inserted['id']))
|
|
else:
|
|
queue_row = self.conn.execute(
|
|
f'''SELECT id, platform, status, last_error FROM target_queue
|
|
WHERE source = ? AND normalized_target = ?{lock_suffix}''',
|
|
(source, repository),
|
|
).fetchone()
|
|
if not queue_row or str(queue_row['platform']) != 'docker':
|
|
raise RuntimeError('DockerHub repository queue identity conflicts')
|
|
queue_id = int(queue_row['id'])
|
|
queue_ids[repository] = queue_id
|
|
if (
|
|
str(queue_row['status']) == 'quarantined'
|
|
and str(queue_row['last_error'] or '')
|
|
== ADMIN_DISCARDED_QUEUE_REASON
|
|
):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET query = ?, target = ?,
|
|
status = 'deferred', attempts = 0,
|
|
available_after = ?,
|
|
last_error = 'Docker tag resolution unresolved',
|
|
completed_at = NULL, lease_owner = NULL,
|
|
lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL,
|
|
resolver_state = 'pending', resolver_due_at = ?,
|
|
resolver_attempts = 0, resolver_token = NULL,
|
|
claim_event_id = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'quarantined'
|
|
AND last_error = ?''',
|
|
(
|
|
query, repository, resolver_due, resolver_due, now,
|
|
queue_id, ADMIN_DISCARDED_QUEUE_REASON,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError(
|
|
'discarded DockerHub target reactivation authority changed'
|
|
)
|
|
inserted_count += 1
|
|
new_queue_ids.add(queue_id)
|
|
preexisting.discard(repository)
|
|
|
|
page_id = None
|
|
page_inserted_count = 0
|
|
observation_inserted_count = 0
|
|
completed_query_count = 0
|
|
pass_complete = False
|
|
query_complete = False
|
|
if pass_row is not None:
|
|
page_payload = json.dumps(
|
|
{
|
|
'per_page': observation['per_page'],
|
|
'page_number': observation['page_number'],
|
|
'page_limit': observation['page_limit'],
|
|
'query': query,
|
|
'query_ordinal': observation['query_ordinal'],
|
|
'repositories': observed,
|
|
'total_count': observation['total_count'],
|
|
'collection_generation': observation['collection_generation'],
|
|
'schema': 'docker-discovery-page-v2',
|
|
'source': source,
|
|
},
|
|
ensure_ascii=True,
|
|
sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')
|
|
page_sha256 = hashlib.sha256(page_payload).hexdigest()
|
|
admission_kind = 'retry' if fence else 'main'
|
|
inserted_page = self.conn.execute(
|
|
'''INSERT INTO docker_discovery_pages(
|
|
pass_id, source_cycle_id, retry_work_id, query,
|
|
query_ordinal, page_number, result_count, total_count,
|
|
admitted_count,
|
|
query_complete, admission_kind, page_sha256,
|
|
observed_at, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(pass_id, query_ordinal, page_number) DO NOTHING
|
|
RETURNING *''',
|
|
(
|
|
pass_row['id'], observation['cycle_id'],
|
|
fence[0] if fence else None, query,
|
|
observation['query_ordinal'], observation['page_number'],
|
|
attempted_count, observation['total_count'], len(normalized),
|
|
int(observation['query_complete']),
|
|
admission_kind, page_sha256, now, now,
|
|
),
|
|
).fetchone()
|
|
page_inserted_count = int(bool(inserted_page))
|
|
page_row = inserted_page or self.conn.execute(
|
|
f'''SELECT * FROM docker_discovery_pages
|
|
WHERE pass_id = ? AND query_ordinal = ? AND page_number = ?{lock_suffix}''',
|
|
(
|
|
pass_row['id'], observation['query_ordinal'],
|
|
observation['page_number'],
|
|
),
|
|
).fetchone()
|
|
if not page_row or (
|
|
str(page_row['query']) != query
|
|
or int(page_row['result_count']) != attempted_count
|
|
or int(page_row['total_count']) != observation['total_count']
|
|
or int(page_row['admitted_count']) != len(normalized)
|
|
or str(page_row['admission_kind']) != admission_kind
|
|
or str(page_row['page_sha256']) != page_sha256
|
|
or (
|
|
(page_row['retry_work_id'] is None) != (fence is None)
|
|
)
|
|
or (
|
|
fence is not None
|
|
and int(page_row['retry_work_id']) != fence[0]
|
|
)
|
|
or (
|
|
(page_row['source_cycle_id'] is None) !=
|
|
(observation['cycle_id'] is None)
|
|
)
|
|
or (
|
|
page_row['source_cycle_id'] is not None
|
|
and int(page_row['source_cycle_id']) != observation['cycle_id']
|
|
)
|
|
):
|
|
raise RuntimeError('DockerHub discovery page evidence conflicts')
|
|
page_id = int(page_row['id'])
|
|
page_observed_at = str(page_row['observed_at'])
|
|
if observation['query_complete'] and not int(page_row['query_complete']):
|
|
self.conn.execute(
|
|
'''UPDATE docker_discovery_pages SET query_complete = 1
|
|
WHERE id = ? AND query_complete = 0''',
|
|
(page_id,),
|
|
)
|
|
|
|
for repository in normalized:
|
|
queue_id = queue_ids[repository]
|
|
search_rank = (
|
|
(observation['page_number'] - 1) * observation['per_page']
|
|
+ ordinals[repository]
|
|
)
|
|
existing_observation = self.conn.execute(
|
|
'''SELECT search_rank FROM docker_repository_query_observations
|
|
WHERE page_id = ? AND repository_queue_id = ?''',
|
|
(page_id, queue_id),
|
|
).fetchone()
|
|
if existing_observation:
|
|
if int(existing_observation['search_rank']) != search_rank:
|
|
raise RuntimeError('DockerHub repository observation rank conflicts')
|
|
continue
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_repository_query_provenance(
|
|
source, query, repository_queue_id, provenance_kind,
|
|
first_observed_at, last_observed_at, first_search_rank,
|
|
best_search_rank, last_search_rank, first_cycle_id,
|
|
last_cycle_id, first_page_id, last_page_id,
|
|
first_policy_sha256, last_policy_sha256,
|
|
observation_count, fresh_observation_count,
|
|
fresh_complete_observation_count, fresh_coverage_eligible,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, ?, 'fresh_page', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
1, 1, 0, 0, ?, ?)
|
|
ON CONFLICT(source, query, repository_queue_id) DO UPDATE SET
|
|
provenance_kind = 'fresh_page',
|
|
last_observed_at = excluded.last_observed_at,
|
|
first_search_rank = COALESCE(
|
|
docker_repository_query_provenance.first_search_rank,
|
|
excluded.first_search_rank
|
|
),
|
|
best_search_rank = CASE
|
|
WHEN docker_repository_query_provenance.best_search_rank IS NULL
|
|
OR excluded.best_search_rank < docker_repository_query_provenance.best_search_rank
|
|
THEN excluded.best_search_rank
|
|
ELSE docker_repository_query_provenance.best_search_rank
|
|
END,
|
|
last_search_rank = excluded.last_search_rank,
|
|
first_cycle_id = COALESCE(
|
|
docker_repository_query_provenance.first_cycle_id,
|
|
excluded.first_cycle_id
|
|
),
|
|
last_cycle_id = COALESCE(
|
|
excluded.last_cycle_id,
|
|
docker_repository_query_provenance.last_cycle_id
|
|
),
|
|
first_page_id = COALESCE(
|
|
docker_repository_query_provenance.first_page_id,
|
|
excluded.first_page_id
|
|
),
|
|
last_page_id = excluded.last_page_id,
|
|
first_policy_sha256 = COALESCE(
|
|
docker_repository_query_provenance.first_policy_sha256,
|
|
excluded.first_policy_sha256
|
|
),
|
|
last_policy_sha256 = excluded.last_policy_sha256,
|
|
observation_count = docker_repository_query_provenance.observation_count + 1,
|
|
fresh_observation_count = docker_repository_query_provenance.fresh_observation_count + 1,
|
|
updated_at = excluded.updated_at''',
|
|
(
|
|
source, query, queue_id, page_observed_at, page_observed_at,
|
|
search_rank, search_rank, search_rank,
|
|
observation['cycle_id'], observation['cycle_id'], page_id, page_id,
|
|
observation['policy_sha256'], observation['policy_sha256'], now, now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_repository_query_observations(
|
|
page_id, repository_queue_id, source, query,
|
|
search_rank, observed_at
|
|
) VALUES (?, ?, ?, ?, ?, ?)''',
|
|
(page_id, queue_id, source, query, search_rank, page_observed_at),
|
|
)
|
|
observation_inserted_count += 1
|
|
|
|
pass_pages = self.conn.execute(
|
|
'''SELECT query, query_ordinal, page_number, query_complete
|
|
FROM docker_discovery_pages WHERE pass_id = ?
|
|
ORDER BY query_ordinal, page_number''',
|
|
(pass_row['id'],),
|
|
).fetchall()
|
|
expected_query_count = int(pass_row['expected_query_count'])
|
|
pages_by_query = {}
|
|
names_by_query = {}
|
|
terminals_by_query = {}
|
|
for pass_page in pass_pages:
|
|
ordinal = int(pass_page['query_ordinal'])
|
|
if not 0 <= ordinal < expected_query_count:
|
|
raise RuntimeError('DockerHub discovery pass ordinal is outside its bound')
|
|
names_by_query.setdefault(ordinal, set()).add(str(pass_page['query']))
|
|
pages_by_query.setdefault(ordinal, set()).add(int(pass_page['page_number']))
|
|
if int(pass_page['query_complete']):
|
|
terminals_by_query.setdefault(ordinal, set()).add(
|
|
int(pass_page['page_number'])
|
|
)
|
|
if any(len(names) != 1 for names in names_by_query.values()):
|
|
raise RuntimeError('DockerHub discovery pass query identity conflicts')
|
|
complete_ordinals = set()
|
|
for ordinal, terminals in terminals_by_query.items():
|
|
pages = pages_by_query.get(ordinal, set())
|
|
if any(len({page for page in pages if page <= end}) == end for end in terminals):
|
|
complete_ordinals.add(ordinal)
|
|
completed_query_count = len(complete_ordinals)
|
|
next_pass_state = (
|
|
'complete'
|
|
if completed_query_count == expected_query_count
|
|
else 'collecting'
|
|
)
|
|
previous_pass_state = str(pass_row['state'])
|
|
if previous_pass_state == 'complete' and next_pass_state != 'complete':
|
|
raise RuntimeError('DockerHub discovery complete pass lost query evidence')
|
|
became_complete = (
|
|
previous_pass_state == 'collecting' and next_pass_state == 'complete'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_discovery_passes
|
|
SET completed_query_count = ?, state = ?,
|
|
completed_at = CASE WHEN ? = 'complete'
|
|
THEN COALESCE(completed_at, ?) ELSE NULL END,
|
|
updated_at = ?
|
|
WHERE id = ? AND state IN ('collecting','complete')''',
|
|
(
|
|
completed_query_count, next_pass_state, next_pass_state,
|
|
now, now, pass_row['id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError('DockerHub discovery pass completion changed')
|
|
if became_complete and str(pass_row['pass_kind']) == 'deep':
|
|
completed_rows = self.conn.execute(
|
|
'''SELECT observation.source, observation.query,
|
|
observation.repository_queue_id,
|
|
COUNT(*) AS observation_count
|
|
FROM docker_repository_query_observations observation
|
|
JOIN docker_discovery_pages page ON page.id = observation.page_id
|
|
WHERE page.pass_id = ?
|
|
GROUP BY observation.source, observation.query,
|
|
observation.repository_queue_id''',
|
|
(pass_row['id'],),
|
|
).fetchall()
|
|
for completed_row in completed_rows:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_repository_query_provenance
|
|
SET fresh_complete_observation_count =
|
|
fresh_complete_observation_count + ?,
|
|
fresh_coverage_eligible = 1,
|
|
updated_at = ?
|
|
WHERE source = ? AND query = ?
|
|
AND repository_queue_id = ?''',
|
|
(
|
|
int(completed_row['observation_count']), now,
|
|
completed_row['source'], completed_row['query'],
|
|
completed_row['repository_queue_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError('DockerHub discovery coverage evidence changed')
|
|
query_complete = observation['query_ordinal'] in complete_ordinals
|
|
pass_complete = next_pass_state == 'complete'
|
|
|
|
dynamic_hold_count = self._hold_new_docker_depth_repositories_locked(
|
|
source, query, new_queue_ids, observation, now,
|
|
experiment=page_experiment, authority=page_authority,
|
|
)
|
|
retry_progress_count = 0
|
|
retry_completed_count = 0
|
|
if fence and complete:
|
|
if page_id is None:
|
|
cursor = self.conn.execute(
|
|
'''DELETE FROM discovery_retry_queue
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(fence[0], fence[1], fence[2], now),
|
|
)
|
|
else:
|
|
# Keep the inactive work row because admitted pages reference its identity.
|
|
cursor = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET status = 'held',
|
|
available_after = NULL, last_error_category = NULL,
|
|
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, held_at = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(now, now, fence[0], fence[1], fence[2], now),
|
|
)
|
|
retry_completed_count = int(cursor.rowcount or 0)
|
|
if retry_completed_count != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry completion lost its lease fence')
|
|
elif fence and next_page is not None:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET next_page = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(next_page, now, fence[0], fence[1], fence[2], now),
|
|
)
|
|
retry_progress_count = int(cursor.rowcount or 0)
|
|
if retry_progress_count != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry progress lost its lease fence')
|
|
self.conn.commit()
|
|
return {
|
|
'attempted_count': attempted_count,
|
|
'normalized_count': len(normalized),
|
|
'duplicate_count': attempted_count - len(normalized),
|
|
'preexisting_count': len(preexisting),
|
|
'inserted_count': inserted_count,
|
|
'dynamic_hold_count': dynamic_hold_count,
|
|
'retry_progress_count': retry_progress_count,
|
|
'retry_completed_count': retry_completed_count,
|
|
'normalized_repositories': frozenset(normalized),
|
|
'preexisting_repositories': frozenset(preexisting),
|
|
'pass_id': int(pass_row['id']) if pass_row is not None else None,
|
|
'page_id': page_id,
|
|
'page_inserted_count': page_inserted_count,
|
|
'observation_inserted_count': observation_inserted_count,
|
|
'completed_query_count': completed_query_count,
|
|
'query_complete': query_complete,
|
|
'pass_complete': pass_complete,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def dockerhub_discovery_generation_complete(
|
|
self, source, collection_generation, ordered_query_hash,
|
|
expected_query_count, policy_sha256,
|
|
):
|
|
"""Return durable authority for the generation's first complete deep pass."""
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
|
|
source = _validated_discovery_retry_source(source)
|
|
collection_generation = str(collection_generation or '')
|
|
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
|
|
raise ValueError('DockerHub collection generation authority conflicts')
|
|
ordered_query_hash = str(ordered_query_hash or '')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', ordered_query_hash):
|
|
raise ValueError('DockerHub ordered-query generation authority is invalid')
|
|
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
|
|
if isinstance(expected_query_count, bool):
|
|
raise ValueError('DockerHub generation query count is invalid')
|
|
try:
|
|
expected_query_count = int(expected_query_count)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('DockerHub generation query count is invalid') from None
|
|
if not 1 <= expected_query_count <= 1000:
|
|
raise ValueError('DockerHub generation query count is invalid')
|
|
row = self.conn.execute(
|
|
'''SELECT 1 FROM docker_discovery_passes
|
|
WHERE source = ? AND pass_kind = 'deep'
|
|
AND collection_generation = ? AND policy_sha256 = ?
|
|
AND ordered_queries_sha256 = ? AND expected_query_count = ?
|
|
AND completed_query_count = expected_query_count
|
|
AND state = 'complete' AND completed_at IS NOT NULL
|
|
AND (
|
|
SELECT COUNT(DISTINCT terminal.query_ordinal)
|
|
FROM docker_discovery_pages terminal
|
|
WHERE terminal.pass_id = docker_discovery_passes.id
|
|
AND terminal.query_complete = 1
|
|
AND terminal.total_count IS NOT NULL
|
|
AND terminal.query_ordinal >= 0
|
|
AND terminal.query_ordinal < expected_query_count
|
|
AND terminal.total_count >= terminal.result_count
|
|
AND (
|
|
SELECT COUNT(DISTINCT page.page_number)
|
|
FROM docker_discovery_pages page
|
|
WHERE page.pass_id = terminal.pass_id
|
|
AND page.query_ordinal = terminal.query_ordinal
|
|
AND page.page_number <= terminal.page_number
|
|
) = terminal.page_number
|
|
) = expected_query_count
|
|
ORDER BY id DESC LIMIT 1''',
|
|
(
|
|
source, collection_generation, policy_sha256,
|
|
ordered_query_hash, expected_query_count,
|
|
),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return bool(row)
|
|
|
|
def dockerhub_discovery_coverage_summary(
|
|
self, source, ordered_queries, ordered_query_hash,
|
|
configured_query_policies, required_repository_count=10,
|
|
collection_generation=None,
|
|
):
|
|
"""Return a bounded, read-only gate over complete deep-pass evidence."""
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
source = _validated_discovery_retry_source(source)
|
|
from docker_depth_experiment import DOCKER_DEPTH_COLLECTION_GENERATION
|
|
collection_generation = str(
|
|
collection_generation or DOCKER_DEPTH_COLLECTION_GENERATION
|
|
)
|
|
if collection_generation != DOCKER_DEPTH_COLLECTION_GENERATION:
|
|
raise ValueError('DockerHub collection generation authority conflicts')
|
|
if isinstance(ordered_queries, (str, bytes)):
|
|
raise ValueError('DockerHub discovery ordered queries are invalid')
|
|
try:
|
|
queries = tuple(ordered_queries)
|
|
except TypeError:
|
|
raise ValueError('DockerHub discovery ordered queries are invalid') from None
|
|
if (
|
|
not 1 <= len(queries) <= DISCOVERY_RETRY_MAX_ALLOWLIST
|
|
or len(set(queries)) != len(queries)
|
|
):
|
|
raise ValueError('DockerHub discovery ordered queries are invalid')
|
|
queries = tuple(_validated_discovery_retry_query(query) for query in queries)
|
|
ordered_query_hash = str(ordered_query_hash or '')
|
|
expected_hash = hashlib.sha256(json.dumps(
|
|
list(queries), ensure_ascii=True, allow_nan=False, sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')).hexdigest()
|
|
if ordered_query_hash != expected_hash:
|
|
raise ValueError('DockerHub discovery ordered-query hash conflicts')
|
|
if isinstance(required_repository_count, bool):
|
|
raise ValueError('DockerHub discovery repository requirement is invalid')
|
|
try:
|
|
required_repository_count = int(required_repository_count)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('DockerHub discovery repository requirement is invalid') from None
|
|
if not 1 <= required_repository_count <= 39:
|
|
raise ValueError('DockerHub discovery repository requirement is invalid')
|
|
if not isinstance(configured_query_policies, dict):
|
|
raise ValueError('DockerHub discovery query policies are invalid')
|
|
policy_values = {
|
|
query: (
|
|
value.get('policy_sha256') if isinstance(value, dict) else value
|
|
)
|
|
for query, value in configured_query_policies.items()
|
|
}
|
|
policies = dict(_discovery_retry_allowlist(policy_values))
|
|
if set(policies) != set(queries):
|
|
raise ValueError('DockerHub discovery query policies do not match ordered queries')
|
|
|
|
query_summaries = []
|
|
try:
|
|
for query_ordinal, query in enumerate(queries):
|
|
rows = self.conn.execute(
|
|
'''SELECT provenance.repository_queue_id,
|
|
MIN(observation.search_rank) AS best_search_rank
|
|
FROM docker_repository_query_provenance provenance
|
|
JOIN docker_repository_query_observations observation
|
|
ON observation.source = provenance.source
|
|
AND observation.query = provenance.query
|
|
AND observation.repository_queue_id = provenance.repository_queue_id
|
|
JOIN docker_discovery_pages page ON page.id = observation.page_id
|
|
JOIN docker_discovery_passes discovery_pass
|
|
ON discovery_pass.id = page.pass_id
|
|
JOIN target_queue queue
|
|
ON queue.id = provenance.repository_queue_id
|
|
WHERE provenance.source = ? AND provenance.query = ?
|
|
AND provenance.provenance_kind = 'fresh_page'
|
|
AND provenance.fresh_coverage_eligible = 1
|
|
AND provenance.fresh_complete_observation_count > 0
|
|
AND page.query_ordinal = ? AND page.query = ?
|
|
AND discovery_pass.source = ?
|
|
AND discovery_pass.pass_kind = 'deep'
|
|
AND discovery_pass.collection_generation = ?
|
|
AND discovery_pass.policy_sha256 = ?
|
|
AND discovery_pass.ordered_queries_sha256 = ?
|
|
AND discovery_pass.expected_query_count = ?
|
|
AND discovery_pass.state = 'complete'
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND queue.status IN ('pending','deferred')
|
|
AND queue.target_scan_id IS NULL
|
|
AND queue.target NOT LIKE '%@%'
|
|
AND queue.normalized_target NOT LIKE '%@%'
|
|
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
|
|
AND queue.claim_batch IS NULL AND queue.leased_at IS NULL
|
|
AND queue.lease_expires_at IS NULL
|
|
AND queue.current_result_reservation_id IS NULL
|
|
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
|
|
AND COALESCE(queue.resolver_state, '') <> 'resolving'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_scans scan
|
|
WHERE scan.queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events policy_event
|
|
WHERE policy_event.queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations reservation
|
|
WHERE reservation.queue_id = queue.id
|
|
AND reservation.state IN (
|
|
'scanning','ready','ingesting','db_committed'
|
|
)
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM result_reservations reservation
|
|
JOIN pipeline_quarantine quarantine
|
|
ON quarantine.reservation_id = reservation.id
|
|
WHERE reservation.queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_image_manifests manifest
|
|
WHERE manifest.target_queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM docker_image_blob_coverage coverage
|
|
JOIN docker_content_blobs blob
|
|
ON blob.digest = coverage.blob_digest
|
|
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
|
|
WHERE coverage.queue_id = queue.id
|
|
AND blob.state IN ('leased','submitted')
|
|
)
|
|
GROUP BY provenance.repository_queue_id
|
|
ORDER BY MIN(observation.search_rank), provenance.repository_queue_id
|
|
LIMIT ?''',
|
|
(
|
|
source, query, query_ordinal, query, source,
|
|
collection_generation, policies[query],
|
|
ordered_query_hash, len(queries), source,
|
|
required_repository_count,
|
|
),
|
|
).fetchall()
|
|
eligible_count = len(rows)
|
|
query_summaries.append({
|
|
'query_ordinal': query_ordinal,
|
|
'query': query,
|
|
'policy_sha256': policies[query],
|
|
'eligible_repository_count': eligible_count,
|
|
'required_repository_count': required_repository_count,
|
|
'covered': eligible_count >= required_repository_count,
|
|
})
|
|
covered_query_count = sum(
|
|
int(item['covered']) for item in query_summaries
|
|
)
|
|
ready = covered_query_count == len(queries)
|
|
self.conn.commit()
|
|
return {
|
|
'source': source,
|
|
'ordered_query_hash': ordered_query_hash,
|
|
'query_count': len(queries),
|
|
'required_repository_count': required_repository_count,
|
|
'covered_query_count': covered_query_count,
|
|
'minimum_eligible_repository_count': min(
|
|
item['eligible_repository_count'] for item in query_summaries
|
|
),
|
|
'planning_allowed': ready,
|
|
'state': 'coverage_complete' if ready else 'collecting',
|
|
'counts_capped_at_requirement': True,
|
|
'queries': query_summaries,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def enqueue_discovery_retry(
|
|
self, source, query, policy_sha256, pass_kind, work_kind,
|
|
page_start=1, page_end=None, available_after=None, error_category=None,
|
|
observation=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
source = _validated_discovery_retry_source(source)
|
|
query = _validated_discovery_retry_query(query)
|
|
policy_sha256 = _validated_discovery_retry_policy(policy_sha256)
|
|
pass_kind = str(pass_kind or '')
|
|
if pass_kind not in DISCOVERY_RETRY_PASS_KINDS:
|
|
raise ValueError('discovery retry pass kind is invalid')
|
|
if page_end is None:
|
|
page_end = DISCOVERY_RETRY_MAX_PAGE if work_kind == 'query' else page_start
|
|
work_kind, page_start, page_end = _validated_discovery_retry_pages(
|
|
work_kind, page_start, page_end,
|
|
)
|
|
observation = _validated_docker_discovery_observation(
|
|
observation, source, query,
|
|
)
|
|
if observation is not None and (
|
|
observation['policy_sha256'] != policy_sha256
|
|
or observation['pass_kind'] != pass_kind
|
|
or observation['page_number'] != page_start
|
|
):
|
|
raise ValueError('discovery retry observation identity conflicts')
|
|
error_category = _validated_discovery_retry_error_category(error_category)
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
available_after = _validated_discovery_retry_time(available_after, now_dt)
|
|
pass_row = None
|
|
source_cycle_id = observation['cycle_id'] if observation is not None else None
|
|
if source_cycle_id is not None:
|
|
cycle_row = self.conn.execute(
|
|
'''SELECT id FROM source_cycles
|
|
WHERE id = ? AND source = ? AND query = ?''',
|
|
(source_cycle_id, source, query),
|
|
).fetchone()
|
|
if not cycle_row:
|
|
raise RuntimeError('discovery retry source cycle identity conflicts')
|
|
if observation is not None:
|
|
pass_row = self._docker_discovery_pass_locked(
|
|
source, query, observation, now,
|
|
)
|
|
work_key = discovery_retry_work_key(
|
|
source, query, policy_sha256, pass_kind, work_kind,
|
|
page_start, page_end,
|
|
pass_id=pass_row['id'] if pass_row is not None else None,
|
|
)
|
|
inserted = self.conn.execute(
|
|
'''INSERT INTO discovery_retry_queue (
|
|
work_key, source, query, source_cycle_id, policy_sha256,
|
|
pass_kind, work_kind,
|
|
page_start, page_end, next_page, status, attempts,
|
|
available_after, last_error_category, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?)
|
|
ON CONFLICT(work_key) DO NOTHING
|
|
RETURNING id, status, attempts, next_page''',
|
|
(
|
|
work_key, source, query, source_cycle_id, policy_sha256,
|
|
pass_kind, work_kind,
|
|
page_start, page_end, page_start, available_after, error_category,
|
|
now, now,
|
|
),
|
|
).fetchone()
|
|
inserted_count = int(bool(inserted))
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
row = inserted or self.conn.execute(
|
|
f'''SELECT id, work_key, source, query, policy_sha256, pass_kind,
|
|
work_kind, page_start, page_end, next_page, status, attempts,
|
|
available_after, source_cycle_id
|
|
FROM discovery_retry_queue WHERE work_key = ?{lock_suffix}''',
|
|
(work_key,),
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeError('discovery retry enqueue lost its durable row')
|
|
if not inserted and (
|
|
str(row['source']) != source
|
|
or str(row['query']) != query
|
|
or str(row['policy_sha256']) != policy_sha256
|
|
or str(row['pass_kind']) != pass_kind
|
|
or str(row['work_kind']) != work_kind
|
|
or int(row['page_start']) != page_start
|
|
or int(row['page_end']) != page_end
|
|
):
|
|
raise RuntimeError('discovery retry work-key identity conflict')
|
|
if not inserted and row['status'] == 'held':
|
|
self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET status = 'pending', next_page = page_start,
|
|
available_after = ?, last_error_category = ?, held_at = NULL,
|
|
updated_at = ? WHERE id = ? AND status = 'held' ''',
|
|
(available_after, error_category, now, row['id']),
|
|
)
|
|
elif not inserted and row['status'] == 'pending':
|
|
self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET
|
|
available_after = CASE
|
|
WHEN available_after IS NULL OR CAST(? AS TEXT) IS NULL THEN NULL
|
|
WHEN available_after <= ? THEN available_after ELSE ? END,
|
|
last_error_category = COALESCE(?, last_error_category),
|
|
updated_at = ?
|
|
WHERE id = ? AND status = 'pending' ''',
|
|
(
|
|
available_after, available_after, available_after,
|
|
error_category, now, row['id'],
|
|
),
|
|
)
|
|
row = self.conn.execute(
|
|
'''SELECT id, status, attempts, page_start, page_end, next_page,
|
|
source_cycle_id
|
|
FROM discovery_retry_queue WHERE work_key = ?''',
|
|
(work_key,),
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeError('discovery retry enqueue could not verify durability')
|
|
self.conn.commit()
|
|
return {
|
|
'id': int(row['id']),
|
|
'work_key': work_key,
|
|
'inserted_count': inserted_count,
|
|
'coalesced_count': 1 - inserted_count,
|
|
'status': str(row['status']),
|
|
'attempts': int(row['attempts']),
|
|
'page_start': int(row['page_start']),
|
|
'page_end': int(row['page_end']),
|
|
'next_page': int(row['next_page']),
|
|
'source_cycle_id': (
|
|
int(row['source_cycle_id'])
|
|
if row['source_cycle_id'] is not None else None
|
|
),
|
|
'pass_id': int(pass_row['id']) if pass_row is not None else None,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def claim_discovery_retries(
|
|
self, source, configured_query_policies, lease_owner, limit=1,
|
|
lease_seconds=300,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
source = _validated_discovery_retry_source(source)
|
|
allowed = _discovery_retry_allowlist(configured_query_policies)
|
|
owner = str(lease_owner or '')
|
|
if (
|
|
not 1 <= len(owner) <= 128
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in owner)
|
|
):
|
|
raise ValueError('discovery retry lease owner is invalid')
|
|
if isinstance(limit, bool) or isinstance(lease_seconds, bool):
|
|
raise ValueError('discovery retry claim bounds are invalid')
|
|
try:
|
|
limit = int(limit)
|
|
lease_seconds = int(lease_seconds)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry claim bounds are invalid') from None
|
|
if not 1 <= limit <= DISCOVERY_RETRY_MAX_CLAIM or not 1 <= lease_seconds <= 86400:
|
|
raise ValueError('discovery retry claim bounds are invalid')
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
state = self._locked_runtime_control_state(shared=True)
|
|
if state['effective_discovery_paused']:
|
|
self.conn.commit()
|
|
return []
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
lease_expires_at = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
|
|
timespec='seconds',
|
|
)
|
|
pair_sql = ' OR '.join(
|
|
'(query = ? AND policy_sha256 = ?)' for _ in allowed
|
|
) or '0 = 1'
|
|
pair_params = tuple(value for pair in allowed for value in pair)
|
|
query_sql = ','.join('?' for _ in allowed)
|
|
query_params = tuple(query for query, _ in allowed)
|
|
category_sql = (
|
|
f"CASE WHEN query IN ({query_sql}) THEN 'policy_mismatch' ELSE 'query_removed' END"
|
|
if allowed else "'query_removed'"
|
|
)
|
|
self.conn.execute(
|
|
f'''UPDATE discovery_retry_queue SET status = 'held',
|
|
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, available_after = NULL,
|
|
held_at = ?, last_error_category = {category_sql}, updated_at = ?
|
|
WHERE source = ?
|
|
AND (status = 'pending' OR (status = 'leased' AND lease_expires_at <= ?))
|
|
AND NOT ({pair_sql})''',
|
|
(now, *query_params, now, source, now, *pair_params),
|
|
)
|
|
if not allowed:
|
|
self.conn.commit()
|
|
return []
|
|
lock_suffix = ' FOR UPDATE SKIP LOCKED' if self.conn.is_postgres else ''
|
|
rows = self.conn.execute(
|
|
f'''SELECT id, work_key, source, query, policy_sha256, pass_kind,
|
|
work_kind, page_start, page_end, next_page, attempts,
|
|
last_error_category, source_cycle_id
|
|
FROM discovery_retry_queue
|
|
WHERE source = ? AND ({pair_sql})
|
|
AND (
|
|
(status = 'pending' AND (available_after IS NULL OR available_after <= ?))
|
|
OR (status = 'leased' AND lease_expires_at IS NOT NULL
|
|
AND lease_expires_at <= ?)
|
|
)
|
|
ORDER BY COALESCE(available_after, lease_expires_at, created_at), id
|
|
LIMIT ?{lock_suffix}''',
|
|
(source, *pair_params, now, now, limit),
|
|
).fetchall()
|
|
claimed = []
|
|
for row in rows:
|
|
token = secrets.token_urlsafe(32)
|
|
updated = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET status = 'leased',
|
|
lease_owner = ?, lease_token = ?, leased_at = ?,
|
|
lease_expires_at = ?, held_at = NULL,
|
|
attempts = CASE WHEN attempts < ? THEN attempts + 1 ELSE attempts END,
|
|
updated_at = ?
|
|
WHERE id = ? AND source = ? AND (
|
|
(status = 'pending' AND (available_after IS NULL OR available_after <= ?))
|
|
OR (status = 'leased' AND lease_expires_at IS NOT NULL
|
|
AND lease_expires_at <= ?)
|
|
) RETURNING attempts''',
|
|
(
|
|
owner, token, now, lease_expires_at, DISCOVERY_RETRY_MAX_ATTEMPTS,
|
|
now, row['id'], source, now, now,
|
|
),
|
|
).fetchone()
|
|
if not updated:
|
|
raise DiscoveryRetryLeaseError('discovery retry claim lost its selected row')
|
|
claimed.append({
|
|
'id': int(row['id']),
|
|
'work_key': str(row['work_key']),
|
|
'source': str(row['source']),
|
|
'query': str(row['query']),
|
|
'policy_sha256': str(row['policy_sha256']),
|
|
'pass_kind': str(row['pass_kind']),
|
|
'work_kind': str(row['work_kind']),
|
|
'page_start': int(row['page_start']),
|
|
'page_end': int(row['page_end']),
|
|
'next_page': int(row['next_page']),
|
|
'attempts': int(updated['attempts']),
|
|
'lease_owner': owner,
|
|
'lease_token': token,
|
|
'lease_expires_at': lease_expires_at,
|
|
'last_error_category': row['last_error_category'],
|
|
'source_cycle_id': (
|
|
int(row['source_cycle_id'])
|
|
if row['source_cycle_id'] is not None else None
|
|
),
|
|
})
|
|
self.conn.commit()
|
|
return claimed
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def finish_discovery_retry(self, retry_id, lease_owner, lease_token):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
|
|
retry_id, lease_owner, lease_token,
|
|
)
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''DELETE FROM discovery_retry_queue
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(retry_id, lease_owner, lease_token, now),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry completion lost its lease fence')
|
|
self.conn.commit()
|
|
return {'id': retry_id, 'deleted_count': 1}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def renew_discovery_retry_lease(
|
|
self, retry_id, lease_owner, lease_token, lease_seconds=300,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
|
|
retry_id, lease_owner, lease_token,
|
|
)
|
|
if isinstance(lease_seconds, bool):
|
|
raise ValueError('discovery retry lease duration is invalid')
|
|
try:
|
|
lease_seconds = int(lease_seconds)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry lease duration is invalid') from None
|
|
if not 1 <= lease_seconds <= 86400:
|
|
raise ValueError('discovery retry lease duration is invalid')
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
lease_expires_at = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
|
|
timespec='seconds',
|
|
)
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(
|
|
lease_expires_at, now, retry_id, lease_owner, lease_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry renewal lost its lease fence')
|
|
self.conn.commit()
|
|
return {
|
|
'id': retry_id,
|
|
'status': 'leased',
|
|
'lease_expires_at': lease_expires_at,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def update_discovery_retry(
|
|
self, retry_id, lease_owner, lease_token, error_category,
|
|
retry_at=None, refund_attempt=False, next_page=None,
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
|
|
retry_id, lease_owner, lease_token,
|
|
)
|
|
error_category = _validated_discovery_retry_error_category(
|
|
error_category, required=True,
|
|
)
|
|
if not isinstance(refund_attempt, bool):
|
|
raise ValueError('discovery retry refund flag is invalid')
|
|
if refund_attempt and retry_at is None:
|
|
raise ValueError('discovery retry attempt refund requires a trusted retry time')
|
|
if next_page is not None:
|
|
if isinstance(next_page, bool):
|
|
raise ValueError('discovery retry next page is invalid')
|
|
try:
|
|
next_page = int(next_page)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('discovery retry next page is invalid') from None
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
row = self.conn.execute(
|
|
f'''SELECT attempts, page_start, page_end, next_page
|
|
FROM discovery_retry_queue
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?{lock_suffix}''',
|
|
(retry_id, lease_owner, lease_token, now),
|
|
).fetchone()
|
|
if not row:
|
|
raise DiscoveryRetryLeaseError('discovery retry update lost its lease fence')
|
|
if next_page is None:
|
|
next_page = int(row['next_page'])
|
|
elif not (
|
|
int(row['next_page']) <= next_page <= int(row['page_end'])
|
|
and 1 <= next_page <= DISCOVERY_RETRY_MAX_PAGE
|
|
):
|
|
raise ValueError('discovery retry next page is outside its durable range')
|
|
retry_due = _validated_discovery_retry_time(retry_at, now_dt)
|
|
attempts = int(row['attempts'])
|
|
if retry_due is None:
|
|
exponent = min(20, max(0, attempts - 1))
|
|
delay = min(
|
|
DISCOVERY_RETRY_MAX_DELAY_SEC,
|
|
DISCOVERY_RETRY_BASE_DELAY_SEC * (2 ** exponent),
|
|
)
|
|
retry_due = (now_dt + timedelta(seconds=delay)).isoformat(timespec='seconds')
|
|
next_attempts = max(0, attempts - 1) if refund_attempt else attempts
|
|
cursor = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET status = 'pending', attempts = ?,
|
|
next_page = ?, available_after = ?, last_error_category = ?,
|
|
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, held_at = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(
|
|
next_attempts, next_page, retry_due, error_category, now,
|
|
retry_id, lease_owner, lease_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry update lost its lease fence')
|
|
self.conn.commit()
|
|
return {
|
|
'id': retry_id,
|
|
'status': 'pending',
|
|
'attempts': next_attempts,
|
|
'next_page': next_page,
|
|
'available_after': retry_due,
|
|
'last_error_category': error_category,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def hold_discovery_retry(
|
|
self, retry_id, lease_owner, lease_token, error_category='policy_mismatch',
|
|
):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
retry_id, lease_owner, lease_token = _validated_discovery_retry_fence(
|
|
retry_id, lease_owner, lease_token,
|
|
)
|
|
error_category = _validated_discovery_retry_error_category(
|
|
error_category, required=True,
|
|
)
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE discovery_retry_queue SET status = 'held',
|
|
available_after = NULL, last_error_category = ?, held_at = ?,
|
|
lease_owner = NULL, lease_token = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(
|
|
error_category, now, now, retry_id, lease_owner, lease_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise DiscoveryRetryLeaseError('discovery retry hold lost its lease fence')
|
|
self.conn.commit()
|
|
return {
|
|
'id': retry_id,
|
|
'status': 'held',
|
|
'last_error_category': error_category,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def enqueue_targets(
|
|
self, source, platform, query, targets, requeue_done=False,
|
|
unresolved_targets=None, *, discovery_admission=False,
|
|
):
|
|
targets = list(targets or [])
|
|
unresolved_targets = list(unresolved_targets or [])
|
|
if not isinstance(discovery_admission, bool):
|
|
raise ValueError('discovery admission flag must be boolean')
|
|
if not self.conn or (not targets and not unresolved_targets):
|
|
return 0
|
|
|
|
def op():
|
|
if discovery_admission:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
self._require_discovery_admission_locked()
|
|
now = utc_now_iso()
|
|
count = 0
|
|
for target in targets or []:
|
|
normalized = normalize_target(target, platform)
|
|
if requeue_done:
|
|
reset_status = f"target_queue.status = 'done' OR ({ADMIN_DISCARDED_QUEUE_SQL})"
|
|
status_expr = f"CASE WHEN {reset_status} THEN 'pending' ELSE target_queue.status END"
|
|
available_after_expr = f"CASE WHEN {reset_status} THEN NULL ELSE target_queue.available_after END"
|
|
attempts_expr = f"CASE WHEN {reset_status} THEN 0 ELSE target_queue.attempts END"
|
|
completed_at_expr = f"CASE WHEN {reset_status} THEN NULL ELSE target_queue.completed_at END"
|
|
conflict_status_where = reset_status
|
|
else:
|
|
status_expr = f"CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 'pending' WHEN target_queue.status IN ('done', 'failed', 'deferred', 'in_progress', 'cold') THEN target_queue.status WHEN target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL THEN 'deferred' ELSE 'pending' END"
|
|
available_after_expr = f"CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL WHEN target_queue.status IN ('pending', 'deferred', 'cold') THEN target_queue.available_after ELSE NULL END"
|
|
attempts_expr = f'CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN 0 ELSE target_queue.attempts END'
|
|
completed_at_expr = f'CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL} THEN NULL ELSE target_queue.completed_at END'
|
|
conflict_status_where = f"target_queue.status = 'pending' AND target_queue.available_after IS NOT NULL OR ({ADMIN_DISCARDED_QUEUE_SQL})"
|
|
reset_for_discovery = f"({ADMIN_DISCARDED_QUEUE_SQL}) OR (target_queue.status = 'done' AND {str(bool(requeue_done)).upper()})"
|
|
self.conn.execute(
|
|
f'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO UPDATE SET
|
|
target = excluded.target,
|
|
platform = excluded.platform,
|
|
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN excluded.query ELSE COALESCE(target_queue.query, excluded.query) END,
|
|
status = {status_expr},
|
|
lease_owner = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_owner END,
|
|
lease_token = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_token END,
|
|
claim_batch = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.claim_batch END,
|
|
leased_at = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.leased_at END,
|
|
lease_expires_at = CASE WHEN {reset_for_discovery} THEN NULL ELSE target_queue.lease_expires_at END,
|
|
available_after = {available_after_expr},
|
|
attempts = {attempts_expr},
|
|
completed_at = {completed_at_expr},
|
|
last_error = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.last_error END,
|
|
resolver_state = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_state END,
|
|
resolver_due_at = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_due_at END,
|
|
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN 0 ELSE target_queue.resolver_attempts END,
|
|
resolver_token = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN NULL ELSE target_queue.resolver_token END,
|
|
updated_at = excluded.updated_at
|
|
WHERE ({conflict_status_where})
|
|
OR target_queue.target <> excluded.target
|
|
OR target_queue.platform <> excluded.platform
|
|
OR (target_queue.query IS NULL AND excluded.query IS NOT NULL)''',
|
|
(source, platform, query, target, normalized, now, now),
|
|
)
|
|
count += 1
|
|
for target in unresolved_targets:
|
|
normalized = normalize_target(target, platform)
|
|
resolver_due = datetime.fromtimestamp(
|
|
time.time() + max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)), timezone.utc
|
|
).isoformat(timespec='seconds')
|
|
self.conn.execute(
|
|
f'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status,
|
|
available_after, last_error, resolver_state, resolver_due_at, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'deferred', ?, 'Docker tag resolution unresolved', 'pending', ?, ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO UPDATE SET
|
|
status = 'deferred', available_after = excluded.available_after,
|
|
query = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN excluded.query ELSE target_queue.query END,
|
|
target = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN excluded.target ELSE target_queue.target END,
|
|
resolver_state = 'pending', resolver_due_at = excluded.resolver_due_at,
|
|
resolver_attempts = CASE WHEN {ADMIN_DISCARDED_QUEUE_SQL}
|
|
THEN 0 ELSE target_queue.resolver_attempts END,
|
|
resolver_token = NULL, completed_at = NULL,
|
|
last_error = excluded.last_error, updated_at = excluded.updated_at
|
|
WHERE target_queue.source = excluded.source
|
|
AND target_queue.platform = excluded.platform
|
|
AND excluded.platform = 'docker'
|
|
AND (
|
|
({ADMIN_DISCARDED_QUEUE_SQL})
|
|
OR (target_queue.resolver_state IS NULL AND target_queue.status IN ('pending', 'deferred'))
|
|
OR (target_queue.resolver_state = 'resolving' AND target_queue.status = 'deferred'
|
|
AND target_queue.resolver_due_at IS NOT NULL AND target_queue.resolver_due_at <= ?)
|
|
)''',
|
|
(source, platform, query, target, normalized, resolver_due, resolver_due, now, now, now),
|
|
)
|
|
count += 1
|
|
self.conn.commit()
|
|
return count
|
|
return self._safe('enqueue_targets', op, 0)
|
|
|
|
def observe_discovered_targets(
|
|
self, source, platform, query, discoveries, rescan_limit=0, cooldown_seconds=0,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('updated-target observation requires PostgreSQL')
|
|
|
|
normalized_records = {}
|
|
attempted_count = 0
|
|
for discovery in discoveries or []:
|
|
attempted_count += 1
|
|
if isinstance(discovery, dict):
|
|
target = str(discovery.get('target') or '').strip()
|
|
remote_value = discovery.get('remote_modified_at')
|
|
else:
|
|
target = str(discovery or '').strip()
|
|
remote_value = None
|
|
if not target:
|
|
continue
|
|
if source == 'huggingface':
|
|
try:
|
|
target = normalize_huggingface_space_id(target)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
normalized = normalize_target(target, platform)
|
|
remote_time = parse_time(remote_value)
|
|
remote_text = remote_time.isoformat(timespec='seconds') if remote_time else None
|
|
current = normalized_records.get(normalized)
|
|
if current is None:
|
|
normalized_records[normalized] = {
|
|
'target': target,
|
|
'remote_modified_at': remote_text,
|
|
}
|
|
elif remote_text and (
|
|
not current['remote_modified_at']
|
|
or remote_text > current['remote_modified_at']
|
|
):
|
|
current['target'] = target
|
|
current['remote_modified_at'] = remote_text
|
|
|
|
if not normalized_records:
|
|
return {
|
|
'attempted_count': attempted_count,
|
|
'queued_new_count': 0,
|
|
'queued_updated_count': 0,
|
|
}
|
|
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
cooldown_cutoff = (
|
|
now_dt - timedelta(seconds=max(0, int(cooldown_seconds or 0)))
|
|
).isoformat(timespec='seconds')
|
|
observed_existing_ids = []
|
|
queued_new = 0
|
|
queued_updated = 0
|
|
try:
|
|
self._require_discovery_admission_locked()
|
|
for normalized in sorted(normalized_records):
|
|
record = normalized_records[normalized]
|
|
inserted = self.conn.execute(
|
|
'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status,
|
|
remote_modified_at, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO NOTHING
|
|
RETURNING id''',
|
|
(
|
|
source, platform, query, record['target'], normalized,
|
|
record['remote_modified_at'], now, now,
|
|
),
|
|
).fetchone()
|
|
if inserted:
|
|
queued_new += 1
|
|
continue
|
|
|
|
row = self.conn.execute(
|
|
'''SELECT id, target, platform, query, remote_modified_at,
|
|
status, last_error
|
|
FROM target_queue
|
|
WHERE source = ? AND normalized_target = ?
|
|
FOR UPDATE''',
|
|
(source, normalized),
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeError('discovered target disappeared during observation')
|
|
remote_text = record['remote_modified_at']
|
|
if (
|
|
str(row['status']) == 'quarantined'
|
|
and str(row['last_error'] or '') == ADMIN_DISCARDED_QUEUE_REASON
|
|
):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET target = ?, platform = ?, query = ?,
|
|
status = 'pending', attempts = 0, available_after = NULL,
|
|
last_error = NULL, completed_at = NULL,
|
|
lease_owner = NULL, lease_token = NULL,
|
|
claim_batch = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, resolver_state = NULL,
|
|
resolver_due_at = NULL, resolver_attempts = 0,
|
|
resolver_token = NULL, remote_modified_at = ?,
|
|
updated_at = ?
|
|
WHERE id = ? AND status = 'quarantined' AND last_error = ?''',
|
|
(
|
|
record['target'], platform, query, remote_text, now,
|
|
row['id'], ADMIN_DISCARDED_QUEUE_REASON,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError('discarded target reactivation authority changed')
|
|
queued_updated += 1
|
|
continue
|
|
remote_advanced = bool(
|
|
remote_text
|
|
and (
|
|
not row['remote_modified_at']
|
|
or remote_text > str(row['remote_modified_at'])
|
|
)
|
|
)
|
|
if (
|
|
remote_advanced
|
|
or str(row['target']) != record['target']
|
|
or str(row['platform']) != str(platform)
|
|
or (row['query'] is None and query is not None)
|
|
):
|
|
next_remote = remote_text if remote_advanced else row['remote_modified_at']
|
|
self.conn.execute(
|
|
'''UPDATE target_queue SET
|
|
target = ?, platform = ?, query = COALESCE(query, ?),
|
|
remote_modified_at = ?,
|
|
updated_at = ?
|
|
WHERE id = ?''',
|
|
(
|
|
record['target'], platform, query, next_remote, now, row['id'],
|
|
),
|
|
)
|
|
if remote_text:
|
|
observed_existing_ids.append(int(row['id']))
|
|
|
|
limit = max(0, int(rescan_limit or 0))
|
|
if limit and observed_existing_ids:
|
|
placeholders = ','.join('?' for _ in observed_existing_ids)
|
|
eligible = self.conn.execute(
|
|
f'''SELECT id FROM target_queue
|
|
WHERE id IN ({placeholders})
|
|
AND source = ? AND platform = ? AND status = 'done'
|
|
AND remote_modified_at IS NOT NULL
|
|
AND completed_at IS NOT NULL AND completed_at <= ?
|
|
AND (
|
|
(scan_remote_modified_at IS NULL AND remote_modified_at > completed_at)
|
|
OR (scan_remote_modified_at IS NOT NULL AND remote_modified_at > scan_remote_modified_at)
|
|
)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND resolver_token IS NULL
|
|
AND lease_owner IS NULL AND lease_token IS NULL
|
|
AND claim_batch IS NULL AND leased_at IS NULL
|
|
AND lease_expires_at IS NULL
|
|
AND current_result_reservation_id IS NULL
|
|
AND claim_event_id IS NULL
|
|
ORDER BY remote_modified_at DESC, id
|
|
LIMIT ? FOR UPDATE SKIP LOCKED''',
|
|
(*observed_existing_ids, source, platform, cooldown_cutoff, limit),
|
|
).fetchall()
|
|
eligible_ids = [int(row['id']) for row in eligible]
|
|
if eligible_ids:
|
|
eligible_placeholders = ','.join('?' for _ in eligible_ids)
|
|
promoted = self.conn.execute(
|
|
f'''UPDATE target_queue SET
|
|
status = 'pending', attempts = 0, available_after = NULL,
|
|
last_error = NULL, completed_at = NULL,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL,
|
|
current_result_reservation_id = NULL, claim_event_id = NULL,
|
|
updated_at = ?
|
|
WHERE id IN ({eligible_placeholders})
|
|
AND status = 'done' AND current_result_reservation_id IS NULL''',
|
|
(now, *eligible_ids),
|
|
)
|
|
promoted_count = int(promoted.rowcount or 0)
|
|
if promoted_count != len(eligible_ids):
|
|
raise RuntimeError('updated-target promotion authority changed')
|
|
queued_updated += promoted_count
|
|
self.conn.commit()
|
|
return {
|
|
'attempted_count': attempted_count,
|
|
'queued_new_count': queued_new,
|
|
'queued_updated_count': queued_updated,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _admission_intent_sha256(values):
|
|
encoded = json.dumps(
|
|
values, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
def provision_remote_worker_device(
|
|
self, user_key, device_key, token_sha256, active_assignment_cap,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker provisioning requires PostgreSQL')
|
|
user_key = str(user_key or '').strip()
|
|
device_key = str(device_key or '').strip()
|
|
token_sha256 = str(token_sha256 or '').strip().lower()
|
|
cap = int(active_assignment_cap)
|
|
if not 1 <= len(user_key) <= 128 or not 1 <= len(device_key) <= 128:
|
|
raise ValueError('remote worker user and device keys must be 1..128 characters')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
|
|
raise ValueError('remote worker token digest must be lowercase SHA-256')
|
|
if not 0 <= cap <= 10000:
|
|
raise ValueError('remote worker active assignment cap is out of range')
|
|
now = utc_now_iso()
|
|
try:
|
|
self.conn.execute(
|
|
'''INSERT INTO remote_worker_users(
|
|
user_key, active_assignment_cap, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?)
|
|
ON CONFLICT(user_key) DO UPDATE SET
|
|
active_assignment_cap = excluded.active_assignment_cap,
|
|
updated_at = excluded.updated_at''',
|
|
(user_key, cap, now, now),
|
|
)
|
|
user = self.conn.execute(
|
|
'SELECT * FROM remote_worker_users WHERE user_key = ? FOR UPDATE',
|
|
(user_key,),
|
|
).fetchone()
|
|
self.conn.execute(
|
|
'''INSERT INTO remote_worker_devices(
|
|
user_id, device_key, token_sha256, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT(device_key) DO UPDATE SET
|
|
token_sha256 = excluded.token_sha256,
|
|
updated_at = excluded.updated_at''',
|
|
(user['id'], device_key, token_sha256, now, now),
|
|
)
|
|
device = self.conn.execute(
|
|
'''SELECT id, user_id, device_key, revoked_at, created_at, updated_at
|
|
FROM remote_worker_devices WHERE device_key = ?''',
|
|
(device_key,),
|
|
).fetchone()
|
|
if int(device['user_id']) != int(user['id']):
|
|
raise ScanEventConflictError(
|
|
'remote worker device key already belongs to another user'
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'user_id': int(user['id']), 'user_key': user_key,
|
|
'device_id': int(device['id']), 'device_key': device_key,
|
|
'active_assignment_cap': cap,
|
|
'revoked': device['revoked_at'] is not None,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def create_remote_worker_user(self, user_key, active_assignment_cap):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker user creation requires PostgreSQL')
|
|
user_key = str(user_key or '').strip()
|
|
cap = int(active_assignment_cap)
|
|
if not 1 <= len(user_key) <= 128:
|
|
raise ValueError('remote worker user key must be 1..128 characters')
|
|
if not 0 <= cap <= 10000:
|
|
raise ValueError('remote worker active assignment cap is out of range')
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''INSERT INTO remote_worker_users(
|
|
user_key, active_assignment_cap, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING''',
|
|
(user_key, cap, now, now),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT id, user_key, active_assignment_cap, disabled_at
|
|
FROM remote_worker_users WHERE user_key = ?''',
|
|
(user_key,),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return {
|
|
'user_id': int(row['id']), 'user_key': str(row['user_key']),
|
|
'active_assignment_cap': int(row['active_assignment_cap']),
|
|
'disabled': row['disabled_at'] is not None,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def set_remote_worker_user_cap(self, user_key, active_assignment_cap):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker user update requires PostgreSQL')
|
|
user_key = str(user_key or '').strip()
|
|
cap = int(active_assignment_cap)
|
|
if not 1 <= len(user_key) <= 128:
|
|
raise ValueError('remote worker user key must be 1..128 characters')
|
|
if not 0 <= cap <= 10000:
|
|
raise ValueError('remote worker active assignment cap is out of range')
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE remote_worker_users SET active_assignment_cap = ?, updated_at = ?
|
|
WHERE user_key = ?''',
|
|
(cap, now, user_key),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def set_remote_worker_user_disabled(self, user_key, disabled=True):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker user update requires PostgreSQL')
|
|
user_key = str(user_key or '').strip()
|
|
if not 1 <= len(user_key) <= 128:
|
|
raise ValueError('remote worker user key must be 1..128 characters')
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE remote_worker_users SET disabled_at = ?, updated_at = ?
|
|
WHERE user_key = ?''',
|
|
(now if disabled else None, now, user_key),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def issue_remote_worker_device(
|
|
self, user_key, device_key, token_sha256, *, rotate=False,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker device issuance requires PostgreSQL')
|
|
user_key = str(user_key or '').strip()
|
|
device_key = str(device_key or '').strip()
|
|
token_sha256 = str(token_sha256 or '').strip().lower()
|
|
if not 1 <= len(user_key) <= 128 or not 1 <= len(device_key) <= 128:
|
|
raise ValueError('remote worker user and device keys must be 1..128 characters')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
|
|
raise ValueError('remote worker token digest must be lowercase SHA-256')
|
|
now = utc_now_iso()
|
|
try:
|
|
user = self.conn.execute(
|
|
'''SELECT id, user_key FROM remote_worker_users
|
|
WHERE user_key = ? FOR SHARE''',
|
|
(user_key,),
|
|
).fetchone()
|
|
if not user:
|
|
self.conn.rollback()
|
|
return None
|
|
if rotate:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE remote_worker_devices SET token_sha256 = ?, updated_at = ?
|
|
WHERE user_id = ? AND device_key = ?''',
|
|
(token_sha256, now, user['id'], device_key),
|
|
)
|
|
else:
|
|
cursor = self.conn.execute(
|
|
'''INSERT INTO remote_worker_devices(
|
|
user_id, device_key, token_sha256, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING''',
|
|
(user['id'], device_key, token_sha256, now, now),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return None
|
|
device = self.conn.execute(
|
|
'''SELECT id, user_id, device_key, revoked_at
|
|
FROM remote_worker_devices WHERE user_id = ? AND device_key = ?''',
|
|
(user['id'], device_key),
|
|
).fetchone()
|
|
if not device:
|
|
raise ScanEventConflictError('remote worker device issuance was not durable')
|
|
self.conn.commit()
|
|
return {
|
|
'user_id': int(user['id']), 'user_key': str(user['user_key']),
|
|
'device_id': int(device['id']), 'device_key': str(device['device_key']),
|
|
'revoked': device['revoked_at'] is not None,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def admin_remote_worker_snapshot(self, limit=200, filters=None):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker admin snapshot requires PostgreSQL')
|
|
limit = int(limit)
|
|
if not 1 <= limit <= 500:
|
|
raise ValueError('remote worker admin snapshot limit is out of range')
|
|
filters, filter_conditions, filter_parameters = _admin_worker_filter_sql(
|
|
filters,
|
|
)
|
|
assignment_outcome = _admin_assignment_outcome_sql()
|
|
scan_outcome = _admin_scan_outcome_sql()
|
|
assignment_where = ' AND '.join((
|
|
"r.assignment_kind = 'remote'", *filter_conditions,
|
|
))
|
|
try:
|
|
users = self.conn.execute(
|
|
'''SELECT user_key, active_assignment_cap,
|
|
(disabled_at IS NOT NULL) AS disabled
|
|
FROM remote_worker_users ORDER BY id DESC LIMIT ?''',
|
|
(limit,),
|
|
).fetchall()
|
|
workers = self.conn.execute(
|
|
'''SELECT d.device_key, u.user_key, u.active_assignment_cap,
|
|
d.last_contact_at,
|
|
(d.revoked_at IS NOT NULL) AS revoked,
|
|
activity.current_phases,
|
|
activity.latest_progress_age_seconds,
|
|
activity.known_reasons,
|
|
activity.pending_local_recovery,
|
|
activity.active_package_identity,
|
|
COUNT(r.id) FILTER (
|
|
WHERE r.remote_resolution_kind IS NULL
|
|
) AS unfinished_count,
|
|
COUNT(r.id) FILTER (
|
|
WHERE r.remote_resolution_kind IS NULL
|
|
) AS active_slot_count,
|
|
COUNT(r.id) FILTER (
|
|
WHERE r.remote_resolution_kind = 'bundle_accepted'
|
|
) AS completed_count,
|
|
COUNT(r.id) FILTER (
|
|
WHERE r.remote_resolution_kind = 'prebundle_report'
|
|
) AS failed_count,
|
|
COUNT(r.id) FILTER (
|
|
WHERE r.remote_resolution_kind = 'expired'
|
|
) AS expired_count
|
|
FROM remote_worker_devices d
|
|
JOIN remote_worker_users u ON u.id = d.user_id
|
|
LEFT JOIN result_reservations r
|
|
ON r.remote_device_id = d.id AND r.assignment_kind = 'remote'
|
|
LEFT JOIN LATERAL (
|
|
SELECT STRING_AGG(
|
|
DISTINCT COALESCE(active.phase, 'legacy/unavailable'),
|
|
', ' ORDER BY COALESCE(active.phase, 'legacy/unavailable')
|
|
) AS current_phases,
|
|
MIN(active.progress_age_seconds)
|
|
AS latest_progress_age_seconds,
|
|
STRING_AGG(
|
|
DISTINCT active.known_reason, ', '
|
|
ORDER BY active.known_reason
|
|
) FILTER (WHERE active.known_reason IS NOT NULL)
|
|
AS known_reasons,
|
|
COALESCE(BOOL_OR(
|
|
active.pending_local_recovery = 'true'
|
|
), FALSE) AS pending_local_recovery,
|
|
STRING_AGG(
|
|
DISTINCT active.package_identity, ', '
|
|
ORDER BY active.package_identity
|
|
) FILTER (WHERE active.package_identity IS NOT NULL)
|
|
AS active_package_identity
|
|
FROM (
|
|
SELECT ar.id, lp.phase,
|
|
CASE WHEN lp.event_timestamp IS NOT NULL
|
|
THEN GREATEST(0, EXTRACT(EPOCH FROM (
|
|
CURRENT_TIMESTAMP
|
|
- lp.event_timestamp::timestamptz
|
|
))::BIGINT) END AS progress_age_seconds,
|
|
lp.known_reason, lp.pending_local_recovery,
|
|
(
|
|
ar.remote_execution_snapshot_json::jsonb #>>
|
|
'{compatibility,platform_tag}'
|
|
) || ':' || LEFT((
|
|
ar.remote_execution_snapshot_json::jsonb #>>
|
|
'{compatibility,code_manifest_sha256}'
|
|
), 12) AS package_identity
|
|
FROM result_reservations ar
|
|
LEFT JOIN LATERAL (
|
|
SELECT e.phase, e.event_timestamp,
|
|
e.event_json::jsonb #>> '{progress,reason}'
|
|
AS known_reason,
|
|
e.event_json::jsonb #>>
|
|
'{progress,pending_local_recovery}'
|
|
AS pending_local_recovery
|
|
FROM worker_progress_events e
|
|
WHERE e.reservation_id = ar.id
|
|
ORDER BY e.sequence DESC, e.id DESC LIMIT 1
|
|
) lp ON TRUE
|
|
WHERE ar.remote_device_id = d.id
|
|
AND ar.assignment_kind = 'remote'
|
|
AND ar.remote_resolution_kind IS NULL
|
|
) active
|
|
) activity ON TRUE
|
|
GROUP BY d.id, d.device_key, u.user_key,
|
|
u.active_assignment_cap, d.last_contact_at, d.revoked_at,
|
|
activity.current_phases,
|
|
activity.latest_progress_age_seconds,
|
|
activity.known_reasons,
|
|
activity.pending_local_recovery,
|
|
activity.active_package_identity
|
|
ORDER BY d.id DESC LIMIT ?''',
|
|
(limit,),
|
|
).fetchall()
|
|
assignments = self.conn.execute(
|
|
f'''SELECT r.id AS reservation_id, q.id AS queue_id,
|
|
u.user_key, u.active_assignment_cap, d.device_key, r.source,
|
|
r.normalized_target AS target,
|
|
r.remote_issued_at AS issued_at,
|
|
r.remote_expires_at AS assignment_deadline_at,
|
|
r.remote_result_upload_body_timeout_seconds,
|
|
r.remote_resolved_at AS finished_at,
|
|
CASE WHEN r.remote_resolved_at IS NOT NULL
|
|
AND r.remote_issued_at IS NOT NULL
|
|
THEN GREATEST(0, EXTRACT(EPOCH FROM (
|
|
r.remote_resolved_at::timestamptz
|
|
- r.remote_issued_at::timestamptz
|
|
))::BIGINT)
|
|
ELSE NULL END AS duration_seconds,
|
|
{assignment_outcome} AS assignment_outcome,
|
|
{scan_outcome} AS scan_outcome,
|
|
COALESCE(
|
|
r.remote_resolution_kind = 'bundle_accepted', FALSE
|
|
) AS accepted,
|
|
COALESCE(
|
|
b.committed_at IS NOT NULL OR b.state IN (
|
|
'db_committed', 'acknowledged'
|
|
), FALSE
|
|
) AS ingested,
|
|
r.last_error_code AS assignment_code,
|
|
s.id AS target_scan_id, s.error_count AS scan_error_count,
|
|
s.first_error_summary, s.skipped_reason,
|
|
src.metadata_json::jsonb #>> '{{warnings,0}}'
|
|
AS scan_warning_summary,
|
|
src.metadata_json::jsonb #>> '{{warning_classes,0}}'
|
|
AS scan_warning_class,
|
|
COALESCE(dg.diagnostic_count, 0) AS diagnostic_count,
|
|
dg.diagnostic_categories, dg.diagnostic_codes,
|
|
dg.primary_diagnostic,
|
|
p.phase AS active_phase, p.phase_started_at,
|
|
p.event_timestamp AS last_progress_at,
|
|
p.received_at AS last_progress_received_at,
|
|
p.slot_id,
|
|
p.scan_deadline_at,
|
|
p.known_reason, p.pending_local_recovery,
|
|
CASE WHEN p.phase_started_at IS NOT NULL
|
|
THEN GREATEST(0, EXTRACT(EPOCH FROM (
|
|
COALESCE(
|
|
r.remote_resolved_at::timestamptz,
|
|
CURRENT_TIMESTAMP
|
|
) - p.phase_started_at::timestamptz
|
|
))::BIGINT) END AS phase_age_seconds,
|
|
CASE WHEN p.event_timestamp IS NOT NULL
|
|
THEN GREATEST(0, EXTRACT(EPOCH FROM (
|
|
COALESCE(
|
|
r.remote_resolved_at::timestamptz,
|
|
CURRENT_TIMESTAMP
|
|
) - p.event_timestamp::timestamptz
|
|
))::BIGINT) END AS last_progress_age_seconds,
|
|
CASE WHEN r.remote_expires_at IS NOT NULL
|
|
THEN EXTRACT(EPOCH FROM (
|
|
r.remote_expires_at::timestamptz - COALESCE(
|
|
r.remote_resolved_at::timestamptz,
|
|
CURRENT_TIMESTAMP
|
|
)
|
|
))::BIGINT END AS assignment_remaining_seconds,
|
|
CASE WHEN p.scan_deadline_at IS NOT NULL
|
|
THEN EXTRACT(EPOCH FROM (
|
|
p.scan_deadline_at::timestamptz - COALESCE(
|
|
r.remote_resolved_at::timestamptz,
|
|
CURRENT_TIMESTAMP
|
|
)
|
|
))::BIGINT END AS scan_remaining_seconds,
|
|
b.state AS ingestion_state,
|
|
pj.status AS projection_state,
|
|
r.remote_diagnostic_projection_version AS diagnostic_projection_version,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,protocol_version}}' AS protocol_version,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,bundle_format_version}}' AS bundle_format_version,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,platform_tag}}' AS platform_tag,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,code_manifest_sha256}}' AS code_manifest_sha256,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,detector_policy_sha256}}' AS detector_policy_sha256,
|
|
r.remote_execution_snapshot_json::jsonb #>>
|
|
'{{compatibility,effective_config_sha256}}' AS effective_config_sha256
|
|
FROM result_reservations r
|
|
JOIN remote_worker_users u ON u.id = r.remote_user_id
|
|
JOIN remote_worker_devices d ON d.id = r.remote_device_id
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN result_bundles b ON b.reservation_id = r.id
|
|
LEFT JOIN target_scans s ON s.id = b.target_scan_id
|
|
LEFT JOIN scan_result_compat src ON src.target_scan_id = s.id
|
|
LEFT JOIN projection_jobs pj
|
|
ON pj.target_scan_id = s.id AND pj.job_kind = 'scan_event'
|
|
LEFT JOIN LATERAL (
|
|
SELECT e.phase, e.phase_started_at, e.event_timestamp,
|
|
e.received_at, e.slot_id,
|
|
NULLIF(e.event_json::jsonb ->> 'scan_deadline_at', '')
|
|
AS scan_deadline_at,
|
|
e.event_json::jsonb #>> '{{progress,reason}}'
|
|
AS known_reason,
|
|
e.event_json::jsonb #>>
|
|
'{{progress,pending_local_recovery}}'
|
|
AS pending_local_recovery
|
|
FROM worker_progress_events e
|
|
WHERE e.reservation_id = r.id
|
|
ORDER BY e.sequence DESC, e.id DESC LIMIT 1
|
|
) p ON TRUE
|
|
LEFT JOIN LATERAL (
|
|
SELECT COUNT(*) AS diagnostic_count,
|
|
STRING_AGG(DISTINCT wd.category, ', ' ORDER BY wd.category)
|
|
AS diagnostic_categories,
|
|
STRING_AGG(DISTINCT wd.code, ', ' ORDER BY wd.code)
|
|
AS diagnostic_codes,
|
|
(ARRAY_AGG(
|
|
wd.category || '/' || wd.code
|
|
ORDER BY CASE wd.category
|
|
WHEN 'internal' THEN 0
|
|
WHEN 'storage' THEN 1
|
|
WHEN 'scanner' THEN 2
|
|
WHEN 'timeout' THEN 3
|
|
ELSE 4 END,
|
|
wd.occurred_at DESC, wd.id DESC
|
|
))[1] AS primary_diagnostic
|
|
FROM worker_diagnostics wd
|
|
WHERE wd.reservation_id = r.id
|
|
) dg ON TRUE
|
|
WHERE {assignment_where}
|
|
ORDER BY r.remote_issued_at DESC, r.id DESC LIMIT ?''',
|
|
(*filter_parameters, limit),
|
|
).fetchall()
|
|
deferred = self.conn.execute(
|
|
'''SELECT id AS queue_id, source, normalized_target AS target,
|
|
available_after
|
|
FROM target_queue WHERE status = 'deferred'
|
|
ORDER BY updated_at DESC, id DESC LIMIT ?''',
|
|
(limit,),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
assignment_rows = [dict(row) for row in assignments]
|
|
deferred_rows = [dict(row) for row in deferred]
|
|
for row in (*assignment_rows, *deferred_rows):
|
|
row['target'] = _admin_safe_target(row.get('target'))
|
|
return {
|
|
'filters': filters,
|
|
'users': [dict(row) for row in users],
|
|
'workers': [dict(row) for row in workers],
|
|
'assignments': assignment_rows,
|
|
'deferred_queue': deferred_rows,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def admin_worker_diagnostic_groups(
|
|
self, limit=200, filters=None, *, occurrence_offset=0,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker diagnostic administration requires PostgreSQL')
|
|
limit = int(limit)
|
|
occurrence_offset = int(occurrence_offset)
|
|
if not 1 <= limit <= 500 or occurrence_offset < 0:
|
|
raise ValueError('worker diagnostic administration limit is out of range')
|
|
filters, conditions, parameters = _admin_worker_filter_sql(
|
|
filters, diagnostic_alias='wd',
|
|
)
|
|
where = ' AND '.join(("r.assignment_kind = 'remote'", *conditions))
|
|
try:
|
|
rows = self.conn.execute(
|
|
f'''WITH filtered AS (
|
|
SELECT wd.id, wd.diagnostic_uid, wd.reservation_id,
|
|
wd.target_scan_id, wd.source, wd.phase, wd.kind,
|
|
wd.category, wd.code, wd.summary, wd.retryable,
|
|
wd.occurred_at, wd.received_at, wd.envelope_json,
|
|
u.user_key, d.device_key,
|
|
{_admin_assignment_outcome_sql()} AS assignment_outcome,
|
|
{_admin_scan_outcome_sql()} AS scan_outcome,
|
|
COALESCE(
|
|
NULLIF(
|
|
wd.envelope_json::jsonb #>>
|
|
'{{exception,fingerprint}}',
|
|
''
|
|
),
|
|
'taxonomy-v1:' || wd.kind || ':'
|
|
|| wd.category || ':' || wd.code
|
|
) AS fingerprint
|
|
FROM worker_diagnostics wd
|
|
JOIN result_reservations r ON r.id = wd.reservation_id
|
|
JOIN remote_worker_users u ON u.id = r.remote_user_id
|
|
JOIN remote_worker_devices d ON d.id = r.remote_device_id
|
|
LEFT JOIN result_bundles b ON b.reservation_id = r.id
|
|
LEFT JOIN target_scans s ON s.id = b.target_scan_id
|
|
WHERE {where}
|
|
), group_stats AS (
|
|
SELECT fingerprint, COUNT(*) AS fingerprint_count,
|
|
COUNT(DISTINCT reservation_id)
|
|
AS affected_assignment_count
|
|
FROM filtered GROUP BY fingerprint
|
|
), counted AS (
|
|
SELECT filtered.*, group_stats.fingerprint_count,
|
|
group_stats.affected_assignment_count,
|
|
COUNT(*) OVER() AS matched_occurrence_count
|
|
FROM filtered
|
|
JOIN group_stats USING (fingerprint)
|
|
)
|
|
SELECT * FROM counted
|
|
ORDER BY occurred_at DESC, id DESC
|
|
LIMIT ? OFFSET ?''',
|
|
(*parameters, limit + 1, occurrence_offset),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
has_next = len(rows) > limit
|
|
page_rows = rows[:limit]
|
|
matched_occurrence_count = (
|
|
int(page_rows[0]['matched_occurrence_count']) if page_rows else 0
|
|
)
|
|
grouped = {}
|
|
for raw in page_rows:
|
|
row = dict(raw)
|
|
row.pop('matched_occurrence_count', None)
|
|
fingerprint = str(row.pop('fingerprint'))
|
|
fingerprint_count = int(row.pop('fingerprint_count'))
|
|
affected_assignment_count = int(
|
|
row.pop('affected_assignment_count')
|
|
)
|
|
try:
|
|
envelope = json.loads(str(row.pop('envelope_json')))
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'durable worker diagnostic JSON is invalid'
|
|
) from exc
|
|
occurrence = {
|
|
'diagnostic_uid': str(row['diagnostic_uid']),
|
|
'reservation_id': int(row['reservation_id']),
|
|
'target_scan_id': (
|
|
int(row['target_scan_id'])
|
|
if row['target_scan_id'] is not None else None
|
|
),
|
|
'source': str(row['source']),
|
|
'worker': str(row['device_key']),
|
|
'user': str(row['user_key']),
|
|
'assignment_outcome': str(row['assignment_outcome']),
|
|
'scan_outcome': str(row['scan_outcome']),
|
|
'phase': str(row['phase']),
|
|
'kind': str(row['kind']),
|
|
'category': str(row['category']),
|
|
'code': str(row['code']),
|
|
'summary': str(row['summary']),
|
|
'retryable': bool(row['retryable']),
|
|
'occurred_at': str(row['occurred_at']),
|
|
'received_at': str(row['received_at']),
|
|
}
|
|
group = grouped.setdefault(fingerprint, {
|
|
'fingerprint': fingerprint,
|
|
'count': fingerprint_count,
|
|
'affected_assignment_count': affected_assignment_count,
|
|
'page_occurrence_count': 0,
|
|
'affected_assignments': [],
|
|
'occurrences': [],
|
|
})
|
|
group['page_occurrence_count'] += 1
|
|
group['occurrences'].append(occurrence)
|
|
if occurrence['reservation_id'] not in group['affected_assignments']:
|
|
group['affected_assignments'].append(occurrence['reservation_id'])
|
|
return {
|
|
'filters': filters,
|
|
'matched_occurrence_count': matched_occurrence_count,
|
|
'occurrence_limit': limit,
|
|
'occurrence_offset': occurrence_offset,
|
|
'page_occurrence_count': len(page_rows),
|
|
'has_previous': occurrence_offset > 0,
|
|
'has_next': has_next,
|
|
'previous_occurrence_offset': (
|
|
max(0, occurrence_offset - limit)
|
|
if occurrence_offset > 0 else None
|
|
),
|
|
'next_occurrence_offset': (
|
|
occurrence_offset + limit if has_next else None
|
|
),
|
|
'truncated': has_next,
|
|
'groups': list(grouped.values()),
|
|
}
|
|
|
|
def admin_worker_duration_metrics(self, limit=200, filters=None, *, offset=0):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker duration metrics require PostgreSQL')
|
|
limit = int(limit)
|
|
offset = int(offset)
|
|
if not 1 <= limit <= 500 or offset < 0:
|
|
raise ValueError('worker duration metrics limit is out of range')
|
|
filters = _validated_admin_worker_filters(filters)
|
|
conditions = ["r.assignment_kind = 'remote'"]
|
|
parameters = []
|
|
if 'source' in filters:
|
|
conditions.append('r.source = ?')
|
|
parameters.append(filters['source'])
|
|
if 'worker' in filters:
|
|
conditions.append('d.device_key = ?')
|
|
parameters.append(filters['worker'])
|
|
if 'assignment_outcome' in filters:
|
|
conditions.append(f'({_admin_assignment_outcome_sql()}) = ?')
|
|
parameters.append(filters['assignment_outcome'])
|
|
if 'scan_outcome' in filters:
|
|
conditions.append(f'({_admin_scan_outcome_sql()}) = ?')
|
|
parameters.append(filters['scan_outcome'])
|
|
diagnostic_conditions = []
|
|
for name in ('category', 'code'):
|
|
if name in filters:
|
|
diagnostic_conditions.append(f'fd.{name} = ?')
|
|
parameters.append(filters[name])
|
|
if 'retryable' in filters:
|
|
diagnostic_conditions.append('fd.retryable = ?')
|
|
parameters.append(1 if filters['retryable'] else 0)
|
|
if diagnostic_conditions:
|
|
conditions.append('''EXISTS (
|
|
SELECT 1 FROM worker_diagnostics fd
|
|
WHERE fd.reservation_id = r.id AND %s
|
|
)''' % ' AND '.join(diagnostic_conditions))
|
|
sample_conditions = ['duration_seconds >= 0']
|
|
sample_parameters = []
|
|
if 'phase' in filters:
|
|
sample_conditions.append('phase = ?')
|
|
sample_parameters.append(filters['phase'])
|
|
if 'since' in filters:
|
|
conditions.append(
|
|
'(r.remote_resolved_at IS NULL OR r.remote_resolved_at >= ?)'
|
|
)
|
|
parameters.append(filters['since'])
|
|
sample_conditions.append('completed_at >= ?')
|
|
sample_parameters.append(filters['since'])
|
|
where = ' AND '.join(conditions)
|
|
sample_where = ' AND '.join(sample_conditions)
|
|
try:
|
|
rows = self.conn.execute(
|
|
f'''WITH reservation_scope AS (
|
|
SELECT r.id, r.source, r.remote_issued_at,
|
|
r.remote_resolved_at,
|
|
CASE WHEN r.remote_resolution_kind = 'bundle_accepted'
|
|
THEN COALESCE(s.status, 'unavailable')
|
|
ELSE {_admin_assignment_outcome_sql()} END AS outcome
|
|
FROM result_reservations r
|
|
JOIN remote_worker_devices d ON d.id = r.remote_device_id
|
|
LEFT JOIN result_bundles b ON b.reservation_id = r.id
|
|
LEFT JOIN target_scans s ON s.id = b.target_scan_id
|
|
WHERE {where}
|
|
), ordered_events AS (
|
|
SELECT e.reservation_id, e.phase, e.event_timestamp,
|
|
e.phase_started_at,
|
|
LAG(e.phase) OVER (
|
|
PARTITION BY e.reservation_id
|
|
ORDER BY e.sequence, e.id
|
|
) AS prior_phase,
|
|
e.sequence, e.id
|
|
FROM worker_progress_events e
|
|
JOIN reservation_scope rs ON rs.id = e.reservation_id
|
|
), transitions AS (
|
|
SELECT reservation_id, phase,
|
|
COALESCE(
|
|
NULLIF(phase_started_at, ''), event_timestamp
|
|
) AS phase_started_at,
|
|
sequence, id
|
|
FROM ordered_events
|
|
WHERE prior_phase IS DISTINCT FROM phase
|
|
), phase_edges AS (
|
|
SELECT t.reservation_id, t.phase, t.phase_started_at,
|
|
LEAD(t.phase_started_at) OVER (
|
|
PARTITION BY t.reservation_id
|
|
ORDER BY t.sequence, t.id
|
|
) AS next_phase_started_at
|
|
FROM transitions t
|
|
), samples AS (
|
|
SELECT rs.source, pe.phase, rs.outcome,
|
|
COALESCE(
|
|
pe.next_phase_started_at, rs.remote_resolved_at
|
|
) AS completed_at,
|
|
EXTRACT(EPOCH FROM (
|
|
COALESCE(
|
|
pe.next_phase_started_at, rs.remote_resolved_at
|
|
)::timestamptz
|
|
- pe.phase_started_at::timestamptz
|
|
))::DOUBLE PRECISION AS duration_seconds
|
|
FROM phase_edges pe
|
|
JOIN reservation_scope rs ON rs.id = pe.reservation_id
|
|
WHERE pe.next_phase_started_at IS NOT NULL
|
|
OR rs.remote_resolved_at IS NOT NULL
|
|
UNION ALL
|
|
SELECT rs.source, 'end_to_end' AS phase, rs.outcome,
|
|
rs.remote_resolved_at AS completed_at,
|
|
EXTRACT(EPOCH FROM (
|
|
rs.remote_resolved_at::timestamptz
|
|
- rs.remote_issued_at::timestamptz
|
|
))::DOUBLE PRECISION AS duration_seconds
|
|
FROM reservation_scope rs
|
|
WHERE rs.remote_resolved_at IS NOT NULL
|
|
AND rs.remote_issued_at IS NOT NULL
|
|
)
|
|
, grouped AS (
|
|
SELECT source, phase, outcome, COUNT(*) AS sample_count,
|
|
PERCENTILE_CONT(0.50) WITHIN GROUP (
|
|
ORDER BY duration_seconds
|
|
) AS p50_seconds,
|
|
PERCENTILE_CONT(0.95) WITHIN GROUP (
|
|
ORDER BY duration_seconds
|
|
) AS p95_seconds,
|
|
PERCENTILE_CONT(0.99) WITHIN GROUP (
|
|
ORDER BY duration_seconds
|
|
) AS p99_seconds
|
|
FROM samples WHERE {sample_where}
|
|
GROUP BY source, phase, outcome
|
|
)
|
|
SELECT grouped.*, COUNT(*) OVER() AS total_group_count
|
|
FROM grouped
|
|
ORDER BY source, phase, outcome LIMIT ? OFFSET ?''',
|
|
(*parameters, *sample_parameters, limit + 1, offset),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
has_next = len(rows) > limit
|
|
page_rows = rows[:limit]
|
|
total_group_count = (
|
|
int(page_rows[0]['total_group_count']) if page_rows else 0
|
|
)
|
|
metrics = []
|
|
for raw in page_rows:
|
|
row = dict(raw)
|
|
count = int(row['sample_count'])
|
|
metrics.append({
|
|
'source': str(row['source']),
|
|
'phase': str(row['phase']),
|
|
'outcome': str(row['outcome']),
|
|
'sample_count': count,
|
|
'sufficient': count >= 5,
|
|
'minimum_sample_count': 5,
|
|
'p50_seconds': float(row['p50_seconds']),
|
|
'p95_seconds': float(row['p95_seconds']),
|
|
'p99_seconds': float(row['p99_seconds']),
|
|
})
|
|
return {
|
|
'filters': filters,
|
|
'metrics': metrics,
|
|
'total_group_count': total_group_count,
|
|
'metric_limit': limit,
|
|
'metric_offset': offset,
|
|
'page_group_count': len(metrics),
|
|
'has_previous': offset > 0,
|
|
'has_next': has_next,
|
|
'previous_metric_offset': max(0, offset - limit) if offset > 0 else None,
|
|
'next_metric_offset': offset + limit if has_next else None,
|
|
'truncated': has_next,
|
|
}
|
|
|
|
def admin_worker_assignment_detail(
|
|
self, reservation_id, *, event_limit=500, diagnostic_limit=500,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker assignment detail requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
event_limit = int(event_limit)
|
|
diagnostic_limit = int(diagnostic_limit)
|
|
if (
|
|
reservation_id <= 0
|
|
or not 1 <= event_limit <= 2000
|
|
or not 1 <= diagnostic_limit <= 2000
|
|
):
|
|
raise ValueError('worker assignment detail request is out of range')
|
|
try:
|
|
raw_assignment = self.conn.execute(
|
|
f'''SELECT r.*, q.id AS queue_id, q.status AS queue_status,
|
|
q.completed_at AS queue_settled_at,
|
|
u.user_key, u.active_assignment_cap, d.device_key,
|
|
d.last_contact_at,
|
|
b.state AS bundle_state, b.ready_at AS bundle_ready_at,
|
|
b.committed_at AS bundle_committed_at,
|
|
b.acknowledged_at AS bundle_acknowledged_at,
|
|
b.actual_bytes AS bundle_bytes,
|
|
b.finding_count AS bundle_finding_count,
|
|
b.error_count AS bundle_error_count,
|
|
s.id AS target_scan_id, s.status AS scan_status,
|
|
s.started_at AS scan_started_at,
|
|
s.ended_at AS scan_ended_at,
|
|
s.duration_sec AS scan_duration_seconds,
|
|
s.findings_count, s.verified_findings_count,
|
|
s.error_count AS scan_error_count, s.skipped_reason,
|
|
s.first_error_summary, s.queue_completion_applied,
|
|
s.queue_completion_disposition,
|
|
src.metadata_json::jsonb #>> '{{warnings,0}}'
|
|
AS scan_warning_summary,
|
|
src.metadata_json::jsonb #>> '{{warning_classes,0}}'
|
|
AS scan_warning_class,
|
|
pj.status AS projection_status,
|
|
pj.completed_at AS projection_completed_at,
|
|
CURRENT_TIMESTAMP AS authority_at,
|
|
{ _admin_assignment_outcome_sql() } AS assignment_outcome,
|
|
{ _admin_scan_outcome_sql() } AS scan_outcome
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
JOIN remote_worker_users u ON u.id = r.remote_user_id
|
|
JOIN remote_worker_devices d ON d.id = r.remote_device_id
|
|
LEFT JOIN result_bundles b ON b.reservation_id = r.id
|
|
LEFT JOIN target_scans s ON s.id = b.target_scan_id
|
|
LEFT JOIN scan_result_compat src ON src.target_scan_id = s.id
|
|
LEFT JOIN LATERAL (
|
|
SELECT j.status, j.completed_at
|
|
FROM projection_jobs j
|
|
WHERE j.target_scan_id = s.id
|
|
AND j.job_kind = 'scan_event'
|
|
ORDER BY j.id DESC LIMIT 1
|
|
) pj ON TRUE
|
|
WHERE r.id = ? AND r.assignment_kind = 'remote' ''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not raw_assignment:
|
|
self.conn.commit()
|
|
return None
|
|
event_rows = self.conn.execute(
|
|
'''SELECT * FROM (
|
|
SELECT e.*, COUNT(*) OVER() AS total_event_count
|
|
FROM worker_progress_events e
|
|
WHERE e.reservation_id = ?
|
|
ORDER BY e.sequence DESC, e.id DESC LIMIT ?
|
|
) recent
|
|
ORDER BY sequence, id''',
|
|
(reservation_id, event_limit + 1),
|
|
).fetchall()
|
|
diagnostic_rows = self.conn.execute(
|
|
'''SELECT * FROM worker_diagnostics
|
|
WHERE reservation_id = ?
|
|
ORDER BY occurred_at, id LIMIT ?''',
|
|
(reservation_id, diagnostic_limit + 1),
|
|
).fetchall()
|
|
target_scan_id = raw_assignment['target_scan_id']
|
|
error_rows = []
|
|
if target_scan_id is not None:
|
|
error_rows = self.conn.execute(
|
|
'''SELECT id, category, summary, raw_error, created_at
|
|
FROM errors WHERE target_scan_id = ?
|
|
ORDER BY id LIMIT ?''',
|
|
(int(target_scan_id), diagnostic_limit + 1),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
assignment_row = dict(raw_assignment)
|
|
total_event_count = (
|
|
int(event_rows[0]['total_event_count']) if event_rows else 0
|
|
)
|
|
events_truncated = total_event_count > event_limit
|
|
diagnostics_truncated = len(diagnostic_rows) > diagnostic_limit
|
|
legacy_errors_truncated = len(error_rows) > diagnostic_limit
|
|
events = []
|
|
for raw in event_rows[-event_limit:]:
|
|
row = dict(raw)
|
|
row.pop('total_event_count', None)
|
|
events.append(self._worker_observability_row(
|
|
row, 'event_json', 'event',
|
|
))
|
|
diagnostics = []
|
|
for raw in diagnostic_rows[:diagnostic_limit]:
|
|
row = self._worker_observability_row(
|
|
raw, 'envelope_json', 'diagnostic',
|
|
)
|
|
row['canonical_envelope_json'] = str(row['envelope_json'])
|
|
diagnostics.append(row)
|
|
legacy_errors = [dict(row) for row in error_rows[:diagnostic_limit]]
|
|
|
|
def parsed_json(name):
|
|
raw = assignment_row.get(name)
|
|
if not raw:
|
|
return None
|
|
try:
|
|
value = json.loads(str(raw))
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'durable remote assignment JSON is invalid'
|
|
) from exc
|
|
return value
|
|
|
|
resolution = parsed_json('remote_resolution_json')
|
|
execution_snapshot = parsed_json('remote_execution_snapshot_json')
|
|
timeline = []
|
|
|
|
def add_timeline(timestamp, kind, label, **values):
|
|
if timestamp:
|
|
timeline.append({
|
|
'timestamp': str(timestamp), 'kind': kind, 'label': label,
|
|
**values,
|
|
})
|
|
|
|
add_timeline(assignment_row.get('remote_issued_at'), 'assignment', 'issued')
|
|
for row in events:
|
|
event = row['event']
|
|
add_timeline(
|
|
event.get('timestamp') or row.get('event_timestamp'),
|
|
'phase', str(event.get('phase') or row.get('phase') or ''),
|
|
sequence=int(row['sequence']),
|
|
received_at=str(row['received_at']),
|
|
)
|
|
add_timeline(assignment_row.get('bundle_ready_at'), 'transport', 'bundle received')
|
|
add_timeline(assignment_row.get('remote_resolved_at'), 'receipt', str(
|
|
assignment_row.get('remote_resolution_kind') or 'terminal receipt'
|
|
))
|
|
add_timeline(assignment_row.get('bundle_committed_at'), 'ingestion', 'bundle ingested')
|
|
add_timeline(assignment_row.get('queue_settled_at'), 'settlement', 'queue settled')
|
|
add_timeline(
|
|
assignment_row.get('projection_completed_at'),
|
|
'projection', 'projection completed',
|
|
)
|
|
timeline.sort(key=lambda item: (
|
|
parse_time(item['timestamp']), item['kind'], item['label'],
|
|
))
|
|
|
|
durations = []
|
|
|
|
def duration(label, started, ended, outcome, *, complete=True, authority=None):
|
|
if not started or not ended:
|
|
return
|
|
seconds = max(0.0, (parse_time(ended) - parse_time(started)).total_seconds())
|
|
durations.append({
|
|
'phase': label, 'duration_seconds': seconds, 'outcome': outcome,
|
|
'complete': bool(complete), 'ended_at': str(ended),
|
|
'authority': authority or ('transition' if complete else 'database_current_time'),
|
|
})
|
|
|
|
current_authority = (
|
|
assignment_row.get('remote_resolved_at')
|
|
or assignment_row.get('authority_at')
|
|
)
|
|
duration(
|
|
'end_to_end', assignment_row.get('remote_issued_at'),
|
|
current_authority,
|
|
assignment_row['assignment_outcome'],
|
|
complete=assignment_row.get('remote_resolved_at') is not None,
|
|
authority=(
|
|
'assignment_resolution'
|
|
if assignment_row.get('remote_resolved_at') else 'database_current_time'
|
|
),
|
|
)
|
|
if assignment_row.get('scan_duration_seconds') is not None:
|
|
durations.append({
|
|
'phase': 'scan_total',
|
|
'duration_seconds': float(assignment_row['scan_duration_seconds']),
|
|
'outcome': assignment_row['scan_outcome'],
|
|
'complete': True,
|
|
'ended_at': assignment_row.get('scan_ended_at'),
|
|
'authority': 'target_scan',
|
|
})
|
|
transitions = []
|
|
for row in events:
|
|
event = row['event']
|
|
phase = str(event.get('phase') or row.get('phase') or '')
|
|
if not transitions or transitions[-1][0] != phase:
|
|
transitions.append((
|
|
phase,
|
|
str(event.get('phase_started_at') or event.get('timestamp') or ''),
|
|
))
|
|
for index, (phase, started) in enumerate(transitions):
|
|
final = index == len(transitions) - 1
|
|
ended = current_authority if final else transitions[index + 1][1]
|
|
duration(
|
|
phase, started, ended, assignment_row['scan_outcome'],
|
|
complete=(not final or assignment_row.get('remote_resolved_at') is not None),
|
|
authority=(
|
|
'assignment_resolution' if final and assignment_row.get('remote_resolved_at')
|
|
else 'database_current_time' if final else 'phase_transition'
|
|
),
|
|
)
|
|
|
|
latest_event = events[-1]['event'] if events else None
|
|
latest_progress_at = (
|
|
latest_event.get('timestamp') if latest_event else None
|
|
)
|
|
phase_started_at = (
|
|
latest_event.get('phase_started_at') if latest_event else None
|
|
)
|
|
phase_authority = current_authority if latest_event else None
|
|
phase_age_seconds = (
|
|
max(0.0, (
|
|
parse_time(phase_authority) - parse_time(phase_started_at)
|
|
).total_seconds())
|
|
if phase_started_at and phase_authority else None
|
|
)
|
|
last_progress_age_seconds = (
|
|
max(0.0, (
|
|
parse_time(phase_authority) - parse_time(latest_progress_at)
|
|
).total_seconds())
|
|
if latest_progress_at and phase_authority else None
|
|
)
|
|
|
|
assignment = {
|
|
'reservation_id': int(assignment_row['id']),
|
|
'queue_id': int(assignment_row['queue_id']),
|
|
'source': str(assignment_row['source']),
|
|
'target': _admin_safe_target(assignment_row.get('normalized_target')),
|
|
'user': str(assignment_row['user_key']),
|
|
'worker': str(assignment_row['device_key']),
|
|
'active_assignment_cap': int(assignment_row['active_assignment_cap']),
|
|
'assignment_outcome': str(assignment_row['assignment_outcome']),
|
|
'scan_outcome': str(assignment_row['scan_outcome']),
|
|
'issued_at': assignment_row.get('remote_issued_at'),
|
|
'resolved_at': assignment_row.get('remote_resolved_at'),
|
|
'assignment_code': assignment_row.get('last_error_code'),
|
|
'assignment_detail': assignment_row.get('last_error_detail'),
|
|
}
|
|
resolution_deadlines = (
|
|
resolution.get('deadlines') if isinstance(resolution, dict) else None
|
|
)
|
|
receipt_scan_deadline = (
|
|
resolution_deadlines.get('scan_deadline_at')
|
|
if isinstance(resolution_deadlines, dict) else None
|
|
)
|
|
deadlines = {
|
|
'assignment_deadline_at': assignment_row.get('remote_expires_at'),
|
|
'scan_deadline_at': (
|
|
receipt_scan_deadline
|
|
or (events[-1]['event'].get('scan_deadline_at') if events else None)
|
|
),
|
|
'upload_timeout_seconds': assignment_row.get(
|
|
'remote_result_upload_body_timeout_seconds'
|
|
),
|
|
'upload_timeout_availability': (
|
|
'persisted at assignment issuance'
|
|
if assignment_row.get(
|
|
'remote_result_upload_body_timeout_seconds'
|
|
) is not None else 'legacy/unavailable'
|
|
),
|
|
}
|
|
compatibility = (execution_snapshot or {}).get('compatibility') or {}
|
|
package = {
|
|
key: compatibility.get(key) for key in (
|
|
'protocol_version', 'bundle_format_version', 'platform_tag',
|
|
'code_manifest_sha256', 'detector_policy_sha256',
|
|
'effective_config_sha256',
|
|
)
|
|
}
|
|
scan = {
|
|
'available': assignment_row.get('target_scan_id') is not None,
|
|
'target_scan_id': assignment_row.get('target_scan_id'),
|
|
'status': assignment_row.get('scan_status'),
|
|
'started_at': assignment_row.get('scan_started_at'),
|
|
'ended_at': assignment_row.get('scan_ended_at'),
|
|
'duration_seconds': assignment_row.get('scan_duration_seconds'),
|
|
'findings_count': assignment_row.get('findings_count'),
|
|
'verified_findings_count': assignment_row.get('verified_findings_count'),
|
|
'error_count': assignment_row.get('scan_error_count'),
|
|
'skipped_reason': assignment_row.get('skipped_reason'),
|
|
'first_error_summary': assignment_row.get('first_error_summary'),
|
|
'warning_summary': assignment_row.get('scan_warning_summary'),
|
|
'warning_class': assignment_row.get('scan_warning_class'),
|
|
}
|
|
projection_version = assignment_row.get('remote_diagnostic_projection_version')
|
|
protocol2 = str(package.get('protocol_version') or '') == '2'
|
|
diagnostic_availability = (
|
|
'current' if diagnostics or projection_version == 1 or protocol2
|
|
else 'legacy/unavailable'
|
|
)
|
|
return {
|
|
'schema': 1,
|
|
'assignment': assignment,
|
|
'deadlines': deadlines,
|
|
'package': package,
|
|
'transport': {
|
|
'resolution': resolution,
|
|
'receipt_id': assignment_row.get('remote_receipt_id'),
|
|
'bundle_state': assignment_row.get('bundle_state'),
|
|
'bundle_ready_at': assignment_row.get('bundle_ready_at'),
|
|
'bundle_committed_at': assignment_row.get('bundle_committed_at'),
|
|
'bundle_acknowledged_at': assignment_row.get('bundle_acknowledged_at'),
|
|
'queue_status': assignment_row.get('queue_status'),
|
|
'queue_settled_at': assignment_row.get('queue_settled_at'),
|
|
'projection_status': assignment_row.get('projection_status'),
|
|
'projection_completed_at': assignment_row.get('projection_completed_at'),
|
|
},
|
|
'scan': scan,
|
|
'timeline': timeline,
|
|
'durations': durations,
|
|
'progress': {
|
|
'available': bool(events),
|
|
'availability': (
|
|
'current' if events
|
|
else 'unavailable/no persisted progress' if protocol2
|
|
else 'legacy/unavailable'
|
|
),
|
|
'truncated': events_truncated,
|
|
'total_event_count': total_event_count,
|
|
'omitted_older_event_count': max(0, total_event_count - len(events)),
|
|
'current_phase': (
|
|
str(latest_event.get('phase')) if latest_event else None
|
|
),
|
|
'phase_started_at': phase_started_at,
|
|
'phase_age_seconds': phase_age_seconds,
|
|
'last_progress_at': latest_progress_at,
|
|
'last_progress_age_seconds': last_progress_age_seconds,
|
|
'age_authority': (
|
|
'assignment_resolution'
|
|
if assignment_row.get('remote_resolved_at') else 'database_current_time'
|
|
) if latest_event else None,
|
|
'events': events,
|
|
},
|
|
'diagnostics': {
|
|
'availability': diagnostic_availability,
|
|
'projection_version': projection_version,
|
|
'declared_count': assignment_row.get('remote_diagnostic_count'),
|
|
'truncated': diagnostics_truncated,
|
|
'items': diagnostics,
|
|
},
|
|
'legacy_evidence': {
|
|
'available': bool(legacy_errors) or bool(
|
|
assignment_row.get('first_error_summary')
|
|
),
|
|
'explicitly_not_an_envelope': True,
|
|
'truncated': legacy_errors_truncated,
|
|
'errors': legacy_errors,
|
|
'first_error_summary': assignment_row.get('first_error_summary'),
|
|
},
|
|
}
|
|
|
|
def admin_worker_diagnostic_envelope(self, reservation_id, diagnostic_uid):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker diagnostic download requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
diagnostic_uid = str(diagnostic_uid or '')
|
|
if reservation_id <= 0 or re.fullmatch(r'[a-f0-9]{64}', diagnostic_uid) is None:
|
|
raise ValueError('worker diagnostic download identity is invalid')
|
|
row = self.conn.execute(
|
|
'''SELECT envelope_json, envelope_sha256 FROM worker_diagnostics
|
|
WHERE reservation_id = ? AND diagnostic_uid = ?''',
|
|
(reservation_id, diagnostic_uid),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
if not row:
|
|
return None
|
|
return {
|
|
'canonical_json': str(row['envelope_json']),
|
|
'sha256': str(row['envelope_sha256']),
|
|
}
|
|
|
|
def admin_requeue_deferred_targets(self, queue_ids, *, max_items=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('admin queue requeue requires PostgreSQL')
|
|
maximum = min(500, max(1, int(max_items)))
|
|
ids = []
|
|
for value in queue_ids or ():
|
|
if isinstance(value, bool):
|
|
raise ValueError('admin queue IDs must be positive integers')
|
|
queue_id = int(value)
|
|
if queue_id <= 0 or queue_id > 9223372036854775807:
|
|
raise ValueError('admin queue IDs must be positive integers')
|
|
ids.append(queue_id)
|
|
if not ids or len(ids) > maximum or len(ids) != len(set(ids)):
|
|
raise ValueError('admin queue ID selection exceeds its bound')
|
|
now = utc_now_iso()
|
|
try:
|
|
placeholders = ','.join('?' for _ in ids)
|
|
cursor = self.conn.execute(
|
|
f'''UPDATE target_queue SET status = 'pending', attempts = 0,
|
|
available_after = NULL, lease_owner = NULL, lease_token = NULL,
|
|
claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
last_error = NULL, completed_at = NULL, updated_at = ?
|
|
WHERE id IN ({placeholders}) AND status = 'deferred' ''',
|
|
(now, *ids),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def admin_discard_queued_source(self, source):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('admin source queue discard requires PostgreSQL')
|
|
source = str(source or '').strip().lower()
|
|
if re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', source) is None:
|
|
raise ValueError('admin queue source is invalid')
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'quarantined',
|
|
completed_at = ?, available_after = NULL,
|
|
lease_owner = NULL, lease_token = NULL,
|
|
claim_batch = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, resolver_state = 'resolved',
|
|
resolver_due_at = NULL, resolver_attempts = 0,
|
|
resolver_token = NULL, claim_event_id = NULL,
|
|
last_error = ?,
|
|
updated_at = ?
|
|
WHERE source = ?
|
|
AND status IN ('pending', 'deferred', 'cold')
|
|
AND current_result_reservation_id IS NULL
|
|
AND target_scan_id IS NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations reservation
|
|
WHERE reservation.queue_id = target_queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_scans scan
|
|
WHERE scan.queue_id = target_queue.id
|
|
)''',
|
|
(now, ADMIN_DISCARDED_QUEUE_REASON, now, source),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def authenticate_remote_worker(self, token_sha256):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker authentication requires PostgreSQL')
|
|
token_sha256 = str(token_sha256 or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
|
|
return None
|
|
now = utc_now_iso()
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT d.id AS device_id, d.user_id, d.device_key,
|
|
u.user_key, u.active_assignment_cap
|
|
FROM remote_worker_devices d
|
|
JOIN remote_worker_users u ON u.id = d.user_id
|
|
WHERE d.token_sha256 = ? AND d.revoked_at IS NULL
|
|
AND u.disabled_at IS NULL FOR UPDATE OF d''',
|
|
(token_sha256,),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return None
|
|
self.conn.execute(
|
|
'UPDATE remote_worker_devices SET last_contact_at = ?, updated_at = ? WHERE id = ?',
|
|
(now, now, row['device_id']),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'device_id': int(row['device_id']), 'user_id': int(row['user_id']),
|
|
'device_key': str(row['device_key']), 'user_key': str(row['user_key']),
|
|
'active_assignment_cap': int(row['active_assignment_cap']),
|
|
'token_sha256': token_sha256,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def set_remote_worker_device_revoked(self, device_key, revoked=True):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker revocation requires PostgreSQL')
|
|
device_key = str(device_key or '').strip()
|
|
if not 1 <= len(device_key) <= 128:
|
|
raise ValueError('remote worker device key must be 1..128 characters')
|
|
now = utc_now_iso()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE remote_worker_devices SET revoked_at = ?, updated_at = ?
|
|
WHERE device_key = ?''',
|
|
(now if revoked else None, now, device_key),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
|
|
@staticmethod
|
|
def _remote_credential_mapping(device_id, token_sha256):
|
|
device_id = int(device_id or 0)
|
|
token_sha256 = str(token_sha256 or '').strip().lower()
|
|
if device_id <= 0 or not re.fullmatch(r'[a-f0-9]{64}', token_sha256):
|
|
raise ValueError('remote worker credential identity is invalid')
|
|
return device_id, token_sha256
|
|
|
|
def _lock_active_remote_credential(self, device_id, token_sha256, user_id=None):
|
|
device_id, token_sha256 = self._remote_credential_mapping(
|
|
device_id, token_sha256,
|
|
)
|
|
row = self.conn.execute(
|
|
'''SELECT d.id AS device_id, d.user_id
|
|
FROM remote_worker_devices d
|
|
JOIN remote_worker_users u ON u.id = d.user_id
|
|
WHERE d.id = ? AND d.token_sha256 = ? AND d.revoked_at IS NULL
|
|
AND u.disabled_at IS NULL
|
|
FOR SHARE OF d, u''',
|
|
(device_id, token_sha256),
|
|
).fetchone()
|
|
if row and user_id is not None and int(row['user_id']) != int(user_id):
|
|
return None
|
|
return row
|
|
|
|
@staticmethod
|
|
def _worker_observability_row(row, json_column, value_key, replayed=False):
|
|
result = dict(row)
|
|
try:
|
|
result[value_key] = json.loads(str(result[json_column]))
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
f'durable worker {value_key} JSON is invalid'
|
|
) from exc
|
|
result['replayed'] = bool(replayed)
|
|
return result
|
|
|
|
def _record_worker_progress_event(self, reservation_id, device_id, event, received_at=None):
|
|
reservation_id = int(reservation_id)
|
|
device_id = int(device_id)
|
|
event, event_json, event_sha256 = _canonical_worker_contract('progress', event)
|
|
schema_version = event.get('schema', event.get('schema_version'))
|
|
sequence = event.get('sequence')
|
|
slot_id = event.get('slot_id')
|
|
if (
|
|
isinstance(schema_version, bool) or not isinstance(schema_version, int)
|
|
or schema_version <= 0
|
|
or isinstance(sequence, bool) or not isinstance(sequence, int) or sequence <= 0
|
|
or isinstance(slot_id, bool) or not isinstance(slot_id, int) or slot_id < 0
|
|
):
|
|
raise ValueError('worker progress numeric metadata is invalid')
|
|
if int(event.get('reservation_id') or 0) != reservation_id:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress reservation identity does not match the request'
|
|
)
|
|
values = {
|
|
'event_type': str(event.get('type') or '').strip(),
|
|
'phase': str(event.get('phase') or '').strip(),
|
|
'event_timestamp': str(event.get('timestamp') or '').strip(),
|
|
'phase_started_at': str(event.get('phase_started_at') or '').strip() or None,
|
|
'instance_id': str(event.get('instance_id') or '').strip(),
|
|
'source': str(event.get('source') or '').strip(),
|
|
}
|
|
if any(not values[name] for name in (
|
|
'event_type', 'phase', 'event_timestamp', 'instance_id', 'source',
|
|
)):
|
|
raise ValueError('worker progress required metadata is incomplete')
|
|
if len(event_json.encode('utf-8')) > 8 * 1024:
|
|
raise ValueError('worker progress event exceeds its canonical JSON bound')
|
|
lock = ' FOR UPDATE OF r, q' if self.conn.is_postgres else ''
|
|
reservation = self.conn.execute(
|
|
'''SELECT r.id, r.remote_device_id, r.source, r.state,
|
|
r.claim_lease_token, r.scan_event_id, r.remote_expires_at,
|
|
r.remote_issued_at, r.remote_resolved_at,
|
|
r.remote_resolution_kind, q.status AS queue_status,
|
|
q.lease_token AS queue_lease_token,
|
|
q.current_result_reservation_id, q.claim_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? AND r.assignment_kind = 'remote' ''' + lock,
|
|
(reservation_id,),
|
|
).fetchone()
|
|
now = str(received_at or utc_now_iso())
|
|
if not reservation or (
|
|
int(reservation['remote_device_id'] or 0) != device_id
|
|
or str(reservation['source']) != values['source']
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress does not match the owned reservation'
|
|
)
|
|
existing = self.conn.execute(
|
|
'''SELECT * FROM worker_progress_events
|
|
WHERE reservation_id = ? AND sequence = ?''',
|
|
(reservation_id, sequence),
|
|
).fetchone()
|
|
if existing:
|
|
if (
|
|
int(existing['remote_device_id']) != device_id
|
|
or str(existing['event_sha256']) != event_sha256
|
|
or str(existing['event_json']) != event_json
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress sequence already has different canonical content'
|
|
)
|
|
return self._worker_observability_row(
|
|
existing, 'event_json', 'event', replayed=True,
|
|
)
|
|
try:
|
|
event_time = datetime.fromisoformat(
|
|
str(event['timestamp']).replace('Z', '+00:00')
|
|
).astimezone(timezone.utc)
|
|
received_time = datetime.fromisoformat(
|
|
now.replace('Z', '+00:00')
|
|
).astimezone(timezone.utc)
|
|
issued_time = datetime.fromisoformat(
|
|
str(reservation['remote_issued_at']).replace('Z', '+00:00')
|
|
).astimezone(timezone.utc)
|
|
except (AttributeError, TypeError, ValueError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress authority timestamps are invalid'
|
|
) from exc
|
|
tolerance = timedelta(seconds=REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS)
|
|
if (
|
|
event_time < issued_time - tolerance
|
|
or event_time > received_time + tolerance
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress timestamp is outside its authority window'
|
|
)
|
|
resolved = reservation['remote_resolution_kind'] is not None
|
|
if resolved:
|
|
try:
|
|
resolved_time = datetime.fromisoformat(
|
|
str(reservation['remote_resolved_at']).replace('Z', '+00:00')
|
|
).astimezone(timezone.utc)
|
|
except (AttributeError, TypeError, ValueError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'resolved worker progress authority timestamp is invalid'
|
|
) from exc
|
|
if received_time > resolved_time + timedelta(
|
|
seconds=REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS
|
|
):
|
|
raise WorkerProgressInactiveError(
|
|
'resolved worker progress grace window elapsed'
|
|
)
|
|
if event_time > resolved_time + tolerance:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress timestamp is after terminal resolution'
|
|
)
|
|
if event_time < resolved_time - timedelta(
|
|
seconds=(
|
|
REMOTE_PROGRESS_RESOLUTION_GRACE_SECONDS
|
|
+ REMOTE_PROGRESS_CLOCK_TOLERANCE_SECONDS
|
|
)
|
|
):
|
|
raise WorkerProgressInactiveError(
|
|
'worker progress timestamp is too old for terminal grace'
|
|
)
|
|
if not resolved and (
|
|
str(reservation['state']) != 'scanning'
|
|
or not reservation['remote_expires_at']
|
|
or str(reservation['remote_expires_at']) <= now
|
|
or str(reservation['queue_status']) != 'in_progress'
|
|
or str(reservation['queue_lease_token'] or '')
|
|
!= str(reservation['claim_lease_token'] or '')
|
|
or int(reservation['current_result_reservation_id'] or 0) != reservation_id
|
|
or str(reservation['claim_event_id'] or '')
|
|
!= str(reservation['scan_event_id'] or '')
|
|
):
|
|
raise WorkerProgressInactiveError(
|
|
'worker progress requires the owned current unresolved reservation'
|
|
)
|
|
latest = self.conn.execute(
|
|
'''SELECT sequence, event_timestamp FROM worker_progress_events
|
|
WHERE reservation_id = ? ORDER BY sequence DESC LIMIT 1''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if latest and sequence <= int(latest['sequence']):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress sequence must strictly increase'
|
|
)
|
|
if latest:
|
|
try:
|
|
latest_time = datetime.fromisoformat(
|
|
str(latest['event_timestamp']).replace('Z', '+00:00')
|
|
).astimezone(timezone.utc)
|
|
except (AttributeError, TypeError, ValueError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'durable worker progress timestamp is invalid'
|
|
) from exc
|
|
if event_time < latest_time - tolerance:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress timestamp moves backward beyond tolerance'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''INSERT INTO worker_progress_events(
|
|
reservation_id, remote_device_id, schema_version, sequence,
|
|
event_type, phase, event_timestamp, phase_started_at, instance_id,
|
|
slot_id, source, event_json, event_sha256, received_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(reservation_id, sequence) DO NOTHING''',
|
|
(
|
|
reservation_id, device_id, schema_version, sequence,
|
|
values['event_type'], values['phase'], values['event_timestamp'],
|
|
values['phase_started_at'], values['instance_id'], slot_id,
|
|
values['source'], event_json, event_sha256, now,
|
|
),
|
|
)
|
|
stored = self.conn.execute(
|
|
'''SELECT * FROM worker_progress_events
|
|
WHERE reservation_id = ? AND sequence = ?''',
|
|
(reservation_id, sequence),
|
|
).fetchone()
|
|
if not stored or (
|
|
int(cursor.rowcount or 0) != 1
|
|
and (
|
|
int(stored['remote_device_id']) != device_id
|
|
or str(stored['event_sha256']) != event_sha256
|
|
or str(stored['event_json']) != event_json
|
|
)
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker progress insert conflicted with different canonical content'
|
|
)
|
|
return self._worker_observability_row(stored, 'event_json', 'event')
|
|
|
|
def record_worker_progress_event(self, reservation_id, device_id, event):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker progress persistence requires PostgreSQL')
|
|
try:
|
|
result = self._record_worker_progress_event(
|
|
reservation_id, device_id, event,
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def record_remote_worker_progress_event(
|
|
self, reservation_id, device_id, token_sha256, event,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote worker progress persistence requires PostgreSQL')
|
|
try:
|
|
if not self._lock_active_remote_credential(device_id, token_sha256):
|
|
raise WorkerObservabilityConflictError(
|
|
'remote worker credential changed before progress acceptance'
|
|
)
|
|
result = self._record_worker_progress_event(
|
|
reservation_id, device_id, event,
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _record_worker_diagnostic(
|
|
self, reservation_id, diagnostic, target_scan_id=None, received_at=None,
|
|
):
|
|
reservation_id = int(reservation_id)
|
|
target_scan_id = int(target_scan_id) if target_scan_id is not None else None
|
|
diagnostic, envelope_json, envelope_sha256 = _canonical_worker_contract(
|
|
'diagnostic', diagnostic,
|
|
)
|
|
diagnostic_uid = str(
|
|
diagnostic.get('diagnostic_uid')
|
|
or diagnostic.get('uid')
|
|
or diagnostic.get('id')
|
|
or ''
|
|
).strip()
|
|
schema_version = diagnostic.get('schema', diagnostic.get('schema_version'))
|
|
slot_id = diagnostic.get('slot_id')
|
|
attempt = diagnostic.get('attempt')
|
|
retryable = diagnostic.get('retryable')
|
|
scan_event_id = diagnostic.get('scan_event_id')
|
|
if not diagnostic_uid or len(diagnostic_uid) > 256 or not diagnostic_uid.isascii():
|
|
raise ValueError('worker diagnostic UID is invalid')
|
|
if (
|
|
isinstance(schema_version, bool) or not isinstance(schema_version, int)
|
|
or schema_version <= 0
|
|
or isinstance(slot_id, bool) or not isinstance(slot_id, int) or slot_id < 0
|
|
or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt <= 0
|
|
or (
|
|
scan_event_id is not None
|
|
and (
|
|
not isinstance(scan_event_id, str)
|
|
or re.fullmatch(r'[a-f0-9]{32,64}', scan_event_id) is None
|
|
)
|
|
)
|
|
or not isinstance(retryable, bool)
|
|
):
|
|
raise ValueError('worker diagnostic identity or numeric metadata is invalid')
|
|
if int(diagnostic.get('reservation_id') or 0) != reservation_id:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic reservation identity does not match the request'
|
|
)
|
|
assignment_outcome = diagnostic.get('assignment_outcome')
|
|
scan_outcome = diagnostic.get('scan_outcome')
|
|
if target_scan_id is not None:
|
|
if (
|
|
assignment_outcome != 'accepted'
|
|
or scan_outcome not in {
|
|
'clean', 'found', 'degraded', 'error', 'skipped',
|
|
}
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'accepted bundle diagnostic outcomes are invalid'
|
|
)
|
|
elif assignment_outcome == 'accepted':
|
|
raise WorkerObservabilityConflictError(
|
|
'accepted diagnostic requires an authoritative target scan'
|
|
)
|
|
elif assignment_outcome in {'prebundle_failed', 'expired'} and (
|
|
scan_outcome != 'unavailable'
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'terminal diagnostic scan outcome must be unavailable'
|
|
)
|
|
values = {
|
|
'source': str(diagnostic.get('source') or '').strip(),
|
|
'phase': str(diagnostic.get('phase') or '').strip(),
|
|
'kind': str(diagnostic.get('kind') or '').strip(),
|
|
'category': str(diagnostic.get('category') or '').strip(),
|
|
'code': str(diagnostic.get('code') or '').strip(),
|
|
'summary': str(diagnostic.get('summary') or '').strip(),
|
|
'occurred_at': str(diagnostic.get('occurred_at') or '').strip(),
|
|
'captured_at': str(diagnostic.get('captured_at') or '').strip(),
|
|
}
|
|
if any(not value for value in values.values()):
|
|
raise ValueError('worker diagnostic required metadata is incomplete')
|
|
if len(envelope_json.encode('utf-8')) > 64 * 1024:
|
|
raise ValueError('worker diagnostic exceeds its canonical JSON bound')
|
|
reservation = self.conn.execute(
|
|
'''SELECT id, source, scan_event_id FROM result_reservations
|
|
WHERE id = ?''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not reservation or str(reservation['source']) != values['source']:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic does not match its reservation'
|
|
)
|
|
if (
|
|
scan_event_id is not None
|
|
and str(reservation['scan_event_id'] or '') != scan_event_id
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic scan event does not match its remote reservation'
|
|
)
|
|
if target_scan_id is not None:
|
|
scan = self.conn.execute(
|
|
'''SELECT id, result_reservation_id, scan_event_id FROM target_scans
|
|
WHERE id = ?''',
|
|
(target_scan_id,),
|
|
).fetchone()
|
|
if not scan or int(scan['result_reservation_id'] or 0) != reservation_id:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic target scan does not belong to its reservation'
|
|
)
|
|
if scan_event_id is not None and str(scan['scan_event_id'] or '') != scan_event_id:
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic scan event does not match its target scan'
|
|
)
|
|
http = diagnostic.get('http') or {}
|
|
process = diagnostic.get('process') or {}
|
|
body = diagnostic.get('body', diagnostic.get('raw_body'))
|
|
if body is None and isinstance(http, dict):
|
|
body = http.get('body')
|
|
logs = diagnostic.get('logs', diagnostic.get('log'))
|
|
if logs is None and isinstance(process, dict):
|
|
logs = {
|
|
name: process.get(name) for name in ('stdout', 'stderr')
|
|
if process.get(name) is not None
|
|
} or None
|
|
body_json = (
|
|
json.dumps(body, ensure_ascii=True, sort_keys=True, separators=(',', ':'))
|
|
if body is not None else None
|
|
)
|
|
log_json = (
|
|
json.dumps(logs, ensure_ascii=True, sort_keys=True, separators=(',', ':'))
|
|
if logs is not None else None
|
|
)
|
|
existing = self.conn.execute(
|
|
'SELECT * FROM worker_diagnostics WHERE diagnostic_uid = ?',
|
|
(diagnostic_uid,),
|
|
).fetchone()
|
|
if existing:
|
|
if (
|
|
int(existing['reservation_id']) != reservation_id
|
|
or (
|
|
int(existing['target_scan_id'])
|
|
if existing['target_scan_id'] is not None else None
|
|
) != target_scan_id
|
|
or str(existing['envelope_sha256']) != envelope_sha256
|
|
or str(existing['envelope_json']) != envelope_json
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic UID already has different canonical content or authority'
|
|
)
|
|
return self._worker_observability_row(
|
|
existing, 'envelope_json', 'diagnostic', replayed=True,
|
|
)
|
|
now = str(received_at or utc_now_iso())
|
|
cursor = self.conn.execute(
|
|
'''INSERT INTO worker_diagnostics(
|
|
diagnostic_uid, reservation_id, target_scan_id, schema_version,
|
|
scan_event_id, slot_id, attempt, source, phase, kind, category,
|
|
code, summary, retryable, occurred_at, captured_at, envelope_json,
|
|
envelope_sha256, body_payload_json, log_payload_json, received_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(diagnostic_uid) DO NOTHING''',
|
|
(
|
|
diagnostic_uid, reservation_id, target_scan_id, schema_version,
|
|
scan_event_id, slot_id, attempt, values['source'],
|
|
values['phase'], values['kind'], values['category'], values['code'],
|
|
values['summary'], int(retryable), values['occurred_at'],
|
|
values['captured_at'], envelope_json, envelope_sha256,
|
|
body_json, log_json, now,
|
|
),
|
|
)
|
|
stored = self.conn.execute(
|
|
'SELECT * FROM worker_diagnostics WHERE diagnostic_uid = ?',
|
|
(diagnostic_uid,),
|
|
).fetchone()
|
|
if not stored or (
|
|
int(cursor.rowcount or 0) != 1
|
|
and (
|
|
int(stored['reservation_id']) != reservation_id
|
|
or (
|
|
int(stored['target_scan_id'])
|
|
if stored['target_scan_id'] is not None else None
|
|
) != target_scan_id
|
|
or str(stored['envelope_sha256']) != envelope_sha256
|
|
or str(stored['envelope_json']) != envelope_json
|
|
)
|
|
):
|
|
raise WorkerObservabilityConflictError(
|
|
'worker diagnostic insert conflicted with different canonical content'
|
|
)
|
|
return self._worker_observability_row(
|
|
stored, 'envelope_json', 'diagnostic',
|
|
)
|
|
|
|
def record_worker_diagnostic(
|
|
self, reservation_id, diagnostic, target_scan_id=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('worker diagnostic persistence requires PostgreSQL')
|
|
try:
|
|
result = self._record_worker_diagnostic(
|
|
reservation_id, diagnostic, target_scan_id=target_scan_id,
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def worker_progress_events(self, reservation_id, limit=1000):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
limit = int(limit)
|
|
if not 1 <= limit <= 10000:
|
|
raise ValueError('worker progress query limit is out of range')
|
|
rows = self.conn.execute(
|
|
'''SELECT e.*, r.remote_user_id, r.assignment_kind,
|
|
r.remote_issued_at, r.remote_expires_at,
|
|
r.remote_resolution_kind, q.id AS queue_id,
|
|
q.status AS queue_status, s.id AS target_scan_id,
|
|
s.status AS scan_status, s.started_at AS scan_started_at,
|
|
s.ended_at AS scan_ended_at
|
|
FROM worker_progress_events e
|
|
JOIN result_reservations r ON r.id = e.reservation_id
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN target_scans s ON s.result_reservation_id = r.id
|
|
WHERE e.reservation_id = ?
|
|
ORDER BY e.sequence, e.id LIMIT ?''',
|
|
(int(reservation_id), limit),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return [
|
|
self._worker_observability_row(row, 'event_json', 'event')
|
|
for row in rows
|
|
]
|
|
|
|
def worker_diagnostics(self, reservation_id, limit=1000):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
limit = int(limit)
|
|
if not 1 <= limit <= 10000:
|
|
raise ValueError('worker diagnostic query limit is out of range')
|
|
rows = self.conn.execute(
|
|
'''SELECT d.*, r.remote_device_id, r.remote_user_id, r.assignment_kind,
|
|
r.remote_issued_at, r.remote_expires_at,
|
|
r.remote_resolution_kind, q.id AS queue_id,
|
|
q.status AS queue_status, s.status AS scan_status,
|
|
s.started_at AS scan_started_at, s.ended_at AS scan_ended_at
|
|
FROM worker_diagnostics d
|
|
JOIN result_reservations r ON r.id = d.reservation_id
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN target_scans s ON s.id = d.target_scan_id
|
|
WHERE d.reservation_id = ?
|
|
ORDER BY d.occurred_at, d.id LIMIT ?''',
|
|
(int(reservation_id), limit),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return [
|
|
self._worker_observability_row(row, 'envelope_json', 'diagnostic')
|
|
for row in rows
|
|
]
|
|
|
|
@staticmethod
|
|
def _remote_assignment_mapping(value):
|
|
if value is None:
|
|
return None
|
|
value = dict(value)
|
|
mapped = {
|
|
'user_id': int(value.get('user_id') or 0),
|
|
'device_id': int(value.get('device_id') or 0),
|
|
'effective_config_sha256': str(value.get('effective_config_sha256') or '').lower(),
|
|
'client_compat_sha256': str(value.get('client_compat_sha256') or '').lower(),
|
|
'token_sha256': str(value.get('token_sha256') or '').lower(),
|
|
'result_upload_body_timeout_seconds': value.get(
|
|
'result_upload_body_timeout_seconds'
|
|
),
|
|
}
|
|
if mapped['user_id'] <= 0 or mapped['device_id'] <= 0:
|
|
raise ValueError('remote assignment requires positive user and device identities')
|
|
for key in ('effective_config_sha256', 'client_compat_sha256', 'token_sha256'):
|
|
if not re.fullmatch(r'[a-f0-9]{64}', mapped[key]):
|
|
raise ValueError(f'remote assignment {key} must be lowercase SHA-256')
|
|
timeout = mapped['result_upload_body_timeout_seconds']
|
|
if (
|
|
isinstance(timeout, bool) or not isinstance(timeout, int)
|
|
or not 30 <= timeout <= 86400
|
|
):
|
|
raise ValueError('remote assignment upload body timeout is invalid')
|
|
snapshot_bytes = canonical_remote_execution_snapshot_bytes(
|
|
value.get('execution_snapshot'),
|
|
)
|
|
snapshot = json.loads(snapshot_bytes.decode('ascii'))
|
|
if (
|
|
snapshot['compatibility']['effective_config_sha256']
|
|
!= mapped['effective_config_sha256']
|
|
):
|
|
raise ValueError(
|
|
'remote assignment snapshot effective configuration is invalid'
|
|
)
|
|
mapped.update({
|
|
'execution_snapshot_json': snapshot_bytes.decode('ascii'),
|
|
'execution_snapshot_sha256': hashlib.sha256(snapshot_bytes).hexdigest(),
|
|
})
|
|
return mapped
|
|
|
|
def ensure_admission_intent(
|
|
self, reservation_token, values, remote_user_id=None, remote_device_id=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('v2 admission intent requires PostgreSQL')
|
|
token = str(reservation_token or '')
|
|
if not token:
|
|
raise ValueError('admission intent reservation token is required')
|
|
digest = self._admission_intent_sha256(values)
|
|
now = utc_now_iso()
|
|
try:
|
|
self.conn.execute(
|
|
'''INSERT INTO admission_intents(
|
|
reservation_token, intent_sha256, state, remote_user_id,
|
|
remote_device_id, created_at, updated_at
|
|
) VALUES (?, ?, 'pending', ?, ?, ?, ?)
|
|
ON CONFLICT(reservation_token) DO NOTHING''',
|
|
(token, digest, remote_user_id, remote_device_id, now, now),
|
|
)
|
|
row = self.conn.execute(
|
|
'''SELECT intent_sha256, state, remote_user_id, remote_device_id
|
|
FROM admission_intents WHERE reservation_token = ?''',
|
|
(token,),
|
|
).fetchone()
|
|
if not row or row['intent_sha256'] != digest or (
|
|
str(row['remote_user_id'] or '') != str(remote_user_id or '')
|
|
or str(row['remote_device_id'] or '') != str(remote_device_id or '')
|
|
):
|
|
raise ScanEventConflictError('admission intent token resolves to conflicting identity')
|
|
self.conn.commit()
|
|
return dict(row)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def admission_intent_resolution(self, reservation_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('admission intent resolution requires PostgreSQL')
|
|
row = self.conn.execute(
|
|
'''SELECT state, resolution_detail FROM admission_intents
|
|
WHERE reservation_token = ?''',
|
|
(str(reservation_token),),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
if not row or str(row['state']) != 'aborted':
|
|
return None
|
|
return str(row['resolution_detail'] or '') or None
|
|
|
|
def reserve_and_claim_target(
|
|
self, source, platform, producer_identity, supervisor_instance_id,
|
|
declared_bundle_bytes, projection_bytes, candidate_items, candidate_bytes,
|
|
lease_seconds=3600, max_attempts=3, capacity_limits=None,
|
|
reservation_token=None, bundle_id=None, scan_event_id=None, run_id=None,
|
|
cycle_id=None, claim_order='oldest', docker_depth_authority=None,
|
|
final_cutover=False, remote_assignment=None, reserved_bundle_bytes=None,
|
|
remote_max_active=50,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('v2 capacity reservation and target claims require PostgreSQL')
|
|
experiment_authority = None
|
|
experiment_claim_states = ('active',)
|
|
if isinstance(docker_depth_authority, dict) and docker_depth_authority.get(
|
|
'enabled'
|
|
) is False:
|
|
self._hold_disabled_docker_depth_experiment(docker_depth_authority)
|
|
if isinstance(docker_depth_authority, dict) and docker_depth_authority.get(
|
|
'enabled'
|
|
) is True:
|
|
if source != 'dockerhub' or platform != 'docker':
|
|
raise ValueError(
|
|
'Docker depth experiment authority is valid only for DockerHub Docker claims'
|
|
)
|
|
try:
|
|
experiment_authority = self._normalize_docker_depth_resolver_authority(
|
|
docker_depth_authority
|
|
)
|
|
from docker_depth_experiment import (
|
|
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
|
|
)
|
|
if (
|
|
experiment_authority['selector_version']
|
|
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
):
|
|
experiment_claim_states = ('resolving', 'active')
|
|
except (TypeError, ValueError):
|
|
return None
|
|
identity = _identity_mapping(producer_identity)
|
|
remote = self._remote_assignment_mapping(remote_assignment)
|
|
remote_planning_kind = ''
|
|
if remote:
|
|
remote_snapshot = json.loads(remote['execution_snapshot_json'])
|
|
remote_planning_kind = str(remote_snapshot['planning']['kind'])
|
|
if remote_planning_kind == 'docker_direct_v1' and (
|
|
source != 'dockerhub' or platform != 'docker'
|
|
):
|
|
raise ValueError(
|
|
'Docker direct assignment requires the DockerHub Docker queue'
|
|
)
|
|
if remote_planning_kind == 'huggingface_space_v1' and (
|
|
source != 'huggingface' or platform != 'huggingface'
|
|
):
|
|
raise ValueError(
|
|
'HuggingFace direct assignment requires the HuggingFace queue'
|
|
)
|
|
declared_bundle_bytes = max(1, int(declared_bundle_bytes))
|
|
reserved_bundle_bytes = (
|
|
max(1, int(reserved_bundle_bytes))
|
|
if remote and reserved_bundle_bytes is not None
|
|
else declared_bundle_bytes
|
|
)
|
|
remote_max_active = int(remote_max_active)
|
|
if remote and not 1 <= remote_max_active <= 10000:
|
|
raise ValueError('remote active assignment limit is out of range')
|
|
projection_bytes = max(1, int(projection_bytes))
|
|
candidate_items = max(0, int(candidate_items))
|
|
candidate_bytes = max(0, int(candidate_bytes))
|
|
limits = {
|
|
'bundle_items': 10000,
|
|
'bundle_bytes': 3 * 1024 * 1024 * 1024,
|
|
'projection_items': 10000,
|
|
'projection_bytes': 2 * 1024 * 1024 * 1024,
|
|
'projection_headroom_bytes': 0,
|
|
'keycheck_items': 100000,
|
|
'keycheck_bytes': 512 * 1024 * 1024,
|
|
'quarantine_items': 10000,
|
|
'quarantine_bytes': 1024 * 1024 * 1024,
|
|
}
|
|
limits.update({key: int(value) for key, value in (capacity_limits or {}).items() if key in limits})
|
|
if not reservation_token or not bundle_id or not scan_event_id:
|
|
raise ValueError('caller must preassign reservation_token, bundle_id, and scan_event_id')
|
|
reservation_token = str(reservation_token)
|
|
bundle_id = str(bundle_id).lower()
|
|
scan_event_id = str(scan_event_id).lower()
|
|
if not re.fullmatch(r'[a-f0-9]{32,64}', bundle_id) or not re.fullmatch(r'[a-f0-9]{32,64}', scan_event_id):
|
|
raise ValueError('bundle and scan event IDs must be lowercase hexadecimal identities')
|
|
claim_order = str(claim_order or 'oldest').strip().lower()
|
|
if claim_order not in ('oldest', 'newest', 'balanced'):
|
|
raise ValueError('target claim order must be oldest, newest, or balanced')
|
|
newest_first = claim_order == 'newest' or (
|
|
claim_order == 'balanced' and int(scan_event_id[-1], 16) % 2 == 0
|
|
)
|
|
claim_direction = 'DESC' if newest_first else 'ASC'
|
|
now = utc_now_iso()
|
|
owner = (
|
|
f'remote:{remote["device_id"]}' if remote
|
|
else f'{source}:{identity["pid"]}:{cycle_id or "cycle"}'
|
|
)
|
|
claim_token = secrets.token_urlsafe(32)
|
|
claim_batch = reservation_token
|
|
ready_relative_path = f'ready/{bundle_id[:2]}/{bundle_id}.trb'
|
|
intent_values = {
|
|
'bundle_id': bundle_id,
|
|
'scan_event_id': scan_event_id,
|
|
'source': str(source),
|
|
'platform': str(platform),
|
|
'producer_instance_id': str(supervisor_instance_id or ''),
|
|
'producer_pid': identity['pid'],
|
|
'producer_creation_time': identity['creation_time'],
|
|
'producer_executable': identity['executable'],
|
|
'declared_bundle_bytes': declared_bundle_bytes,
|
|
'reserved_bundle_bytes': reserved_bundle_bytes,
|
|
'reserved_projection_items': 1,
|
|
'reserved_projection_bytes': projection_bytes,
|
|
'reserved_candidate_items': candidate_items,
|
|
'reserved_candidate_bytes': candidate_bytes,
|
|
'run_id': run_id,
|
|
'cycle_id': cycle_id,
|
|
'assignment_kind': 'remote' if remote else 'local',
|
|
'remote_user_id': remote['user_id'] if remote else None,
|
|
'remote_device_id': remote['device_id'] if remote else None,
|
|
'remote_effective_config_sha256': (
|
|
remote['effective_config_sha256'] if remote else None
|
|
),
|
|
'remote_client_compat_sha256': remote['client_compat_sha256'] if remote else None,
|
|
'remote_result_upload_body_timeout_seconds': (
|
|
remote['result_upload_body_timeout_seconds'] if remote else None
|
|
),
|
|
'remote_execution_snapshot_json': (
|
|
remote['execution_snapshot_json'] if remote else None
|
|
),
|
|
'remote_execution_snapshot_sha256': (
|
|
remote['execution_snapshot_sha256'] if remote else None
|
|
),
|
|
}
|
|
self.ensure_admission_intent(
|
|
reservation_token, intent_values,
|
|
remote_user_id=remote['user_id'] if remote else None,
|
|
remote_device_id=remote['device_id'] if remote else None,
|
|
)
|
|
try:
|
|
intent = self.conn.execute(
|
|
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
|
|
(reservation_token,),
|
|
).fetchone()
|
|
if not intent or intent['intent_sha256'] != self._admission_intent_sha256(intent_values):
|
|
raise ScanEventConflictError('admission intent changed before reservation')
|
|
if remote and (
|
|
int(intent['remote_user_id'] or 0) != remote['user_id']
|
|
or int(intent['remote_device_id'] or 0) != remote['device_id']
|
|
):
|
|
raise ScanEventConflictError('remote admission intent changed owner identity')
|
|
if intent['state'] == 'aborted':
|
|
self.conn.commit()
|
|
return None
|
|
if remote and intent['state'] == 'pending':
|
|
control = self._locked_runtime_control_state(shared=True)
|
|
if control['effective_dispatch_paused']:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'dispatch_gate_closed',
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ? AND state = 'pending' ''',
|
|
(now, now, reservation_token),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'dispatch gate lost the pending admission intent fence'
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
experiment = None
|
|
if experiment_authority:
|
|
experiment, authority_reason = (
|
|
self._locked_docker_depth_experiment_authority(
|
|
experiment_authority, final_cutover=final_cutover, now=now,
|
|
)
|
|
)
|
|
if not experiment:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = ?, resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(
|
|
authority_reason or 'experiment_authority_unavailable',
|
|
now, now, reservation_token,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
if str(experiment['state']) == 'released':
|
|
experiment = None
|
|
experiment_authority = None
|
|
remote_user = None
|
|
if remote:
|
|
remote_user = self.conn.execute(
|
|
'''SELECT * FROM remote_worker_users
|
|
WHERE id = ? AND disabled_at IS NULL FOR UPDATE''',
|
|
(remote['user_id'],),
|
|
).fetchone()
|
|
credential = self._lock_active_remote_credential(
|
|
remote['device_id'], remote['token_sha256'],
|
|
user_id=remote['user_id'],
|
|
)
|
|
if not remote_user or not credential:
|
|
raise ScanEventConflictError(
|
|
'remote worker credential is disabled, revoked, or rotated'
|
|
)
|
|
existing = self.conn.execute(
|
|
'''SELECT r.*, q.attempts,
|
|
binding.id AS experiment_binding_id,
|
|
binding.experiment_target_id,
|
|
binding.attempt AS experiment_attempt,
|
|
target.experiment_id,
|
|
target.dispatch_wave, target.dispatch_order,
|
|
experiment.experiment_key AS bound_experiment_key
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.reservation_id = r.id
|
|
LEFT JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
LEFT JOIN docker_depth_experiments experiment
|
|
ON experiment.id = target.experiment_id
|
|
WHERE r.reservation_token = ?''',
|
|
(reservation_token,),
|
|
).fetchone()
|
|
if existing:
|
|
expected = {
|
|
'bundle_id': bundle_id,
|
|
'scan_event_id': scan_event_id,
|
|
'source': str(source),
|
|
'platform': str(platform),
|
|
'producer_instance_id': str(supervisor_instance_id or ''),
|
|
'producer_pid': identity['pid'],
|
|
'producer_creation_time': identity['creation_time'],
|
|
'producer_executable': identity['executable'],
|
|
'declared_bundle_bytes': declared_bundle_bytes,
|
|
'reserved_bundle_bytes': reserved_bundle_bytes,
|
|
'reserved_projection_items': 1,
|
|
'reserved_projection_bytes': projection_bytes,
|
|
'reserved_candidate_items': candidate_items,
|
|
'reserved_candidate_bytes': candidate_bytes,
|
|
'run_id': run_id,
|
|
'cycle_id': cycle_id,
|
|
'assignment_kind': 'remote' if remote else 'local',
|
|
'remote_user_id': remote['user_id'] if remote else None,
|
|
'remote_device_id': remote['device_id'] if remote else None,
|
|
'remote_effective_config_sha256': (
|
|
remote['effective_config_sha256'] if remote else None
|
|
),
|
|
'remote_client_compat_sha256': (
|
|
remote['client_compat_sha256'] if remote else None
|
|
),
|
|
'remote_execution_snapshot_json': (
|
|
remote['execution_snapshot_json'] if remote else None
|
|
),
|
|
'remote_execution_snapshot_sha256': (
|
|
remote['execution_snapshot_sha256'] if remote else None
|
|
),
|
|
}
|
|
if any(str(existing[key]) != str(value) for key, value in expected.items()):
|
|
raise ScanEventConflictError('reservation token resolves to conflicting claim identity')
|
|
if experiment_authority and (
|
|
existing['experiment_binding_id'] is None
|
|
or str(existing['bound_experiment_key'])
|
|
!= experiment_authority['experiment_key']
|
|
):
|
|
raise ScanEventConflictError(
|
|
'reservation token is not bound to the active Docker depth experiment'
|
|
)
|
|
if remote:
|
|
remote_assignment_execution_plan(existing)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'committed', reservation_id = ?,
|
|
resolution_detail = 'existing_reservation', resolved_at = COALESCE(resolved_at, ?),
|
|
updated_at = ? WHERE reservation_token = ?''',
|
|
(existing['id'], now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return self._result_reservation_claim(existing)
|
|
if experiment_authority:
|
|
if (
|
|
str(experiment['state']) == 'held'
|
|
and str(experiment['hold_reason_code'] or '')
|
|
== 'pipeline_capacity_conflict'
|
|
):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'active', hold_reason_code = NULL,
|
|
held_at = NULL, updated_at = ?
|
|
WHERE id = ? AND state = 'held'
|
|
AND hold_reason_code = 'pipeline_capacity_conflict' ''',
|
|
(now, experiment['id']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth capacity backpressure recovery lost its fence'
|
|
)
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ? FOR UPDATE',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
if str(experiment['state']) not in experiment_claim_states:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = ?, resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(
|
|
f"experiment_{experiment['state']}", now, now,
|
|
reservation_token,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
health = self.conn.execute(
|
|
'''SELECT state, lease_expires_at, supervisor_instance_id
|
|
FROM pipeline_leases WHERE worker_name = 'result_ingester' '''
|
|
).fetchone()
|
|
if not (
|
|
health and health['state'] == 'ready'
|
|
and str(health['lease_expires_at'] or '') > now
|
|
and str(health['supervisor_instance_id'] or '') == str(supervisor_instance_id or '')
|
|
):
|
|
if experiment:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'ingester_lease_conflict', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'ingester_not_ready', resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
maximum_attempts = max(0, int(max_attempts or 0))
|
|
if experiment:
|
|
row, selection_reason = self._claim_docker_depth_experiment_target_locked(
|
|
experiment, experiment_authority, now, maximum_attempts,
|
|
)
|
|
if selection_reason:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = ?, resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(selection_reason, now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
if row:
|
|
active = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE id = ? FOR SHARE''',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
if active and str(active['state']) not in experiment_claim_states:
|
|
active = None
|
|
final_reason = self._docker_depth_authority_drift_reason(
|
|
active, experiment_authority,
|
|
) if active else None
|
|
if not active:
|
|
self.conn.rollback()
|
|
return None
|
|
if final_reason:
|
|
self._hold_docker_depth_experiment_locked(
|
|
active, final_reason, now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = ?, resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(final_reason, now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
experiment = active
|
|
else:
|
|
row = self.conn.execute(
|
|
f'''SELECT id, query, target, normalized_target, attempts
|
|
FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND status = 'pending'
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id {claim_direction}
|
|
LIMIT 1 FOR UPDATE SKIP LOCKED''',
|
|
(source, platform, now, maximum_attempts, maximum_attempts),
|
|
).fetchone()
|
|
if not row:
|
|
row = self.conn.execute(
|
|
f'''SELECT id, query, target, normalized_target, attempts
|
|
FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND status = 'deferred' AND available_after IS NOT NULL
|
|
AND available_after <= ?
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY available_after {claim_direction}, id {claim_direction}
|
|
LIMIT 1 FOR UPDATE SKIP LOCKED''',
|
|
(source, platform, now, maximum_attempts, maximum_attempts),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'no_claimable_target', resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
if experiment:
|
|
self.refresh_docker_depth_experiment_state(
|
|
experiment_authority, final_cutover=final_cutover,
|
|
)
|
|
return None
|
|
if remote:
|
|
remote_assignment_execution_plan({
|
|
'assignment_kind': 'remote',
|
|
'source': source,
|
|
'platform': platform,
|
|
'target': row['target'],
|
|
'normalized_target': row['normalized_target'],
|
|
'remote_effective_config_sha256': remote[
|
|
'effective_config_sha256'
|
|
],
|
|
'remote_execution_snapshot_json': remote[
|
|
'execution_snapshot_json'
|
|
],
|
|
'remote_execution_snapshot_sha256': remote[
|
|
'execution_snapshot_sha256'
|
|
],
|
|
'git_scan_plan_json': None,
|
|
'git_scan_plan_sha256': None,
|
|
'docker_layer_plan_json': None,
|
|
'docker_layer_plan_sha256': None,
|
|
})
|
|
# Global accounting is always acquired after the exact workload object.
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if not capacity:
|
|
raise RuntimeSafetySchemaError('pipeline capacity singleton is unavailable')
|
|
if remote:
|
|
active = self.conn.execute(
|
|
'''SELECT COUNT(*) AS user_value,
|
|
(SELECT COUNT(*) FROM result_reservations
|
|
WHERE assignment_kind = 'remote'
|
|
AND remote_resolved_at IS NULL) AS global_value
|
|
FROM result_reservations
|
|
WHERE assignment_kind = 'remote' AND remote_user_id = ?
|
|
AND remote_resolved_at IS NULL''',
|
|
(remote['user_id'],),
|
|
).fetchone()
|
|
quota_reason = None
|
|
if int(active['user_value'] or 0) >= int(
|
|
remote_user['active_assignment_cap'] or 0
|
|
):
|
|
quota_reason = 'remote_user_quota_closed'
|
|
elif int(active['global_value'] or 0) >= remote_max_active:
|
|
quota_reason = 'remote_global_quota_closed'
|
|
if quota_reason:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = ?, resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(quota_reason, now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
if (
|
|
int(capacity['quarantine_items'] or 0) >= max(0, limits['quarantine_items'])
|
|
or int(capacity['quarantine_bytes'] or 0) >= max(0, limits['quarantine_bytes'])
|
|
):
|
|
if experiment:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'quarantine_capacity_conflict', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'quarantine_capacity_conflict',
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'quarantine_admission_closed',
|
|
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
|
|
(now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
requested = {
|
|
'bundle_items': 1,
|
|
'bundle_bytes': reserved_bundle_bytes,
|
|
'projection_items': 1,
|
|
'projection_bytes': projection_bytes,
|
|
'keycheck_items': candidate_items,
|
|
'keycheck_bytes': candidate_bytes,
|
|
}
|
|
projection_ceiling = max(
|
|
0,
|
|
limits['projection_bytes']
|
|
- max(0, limits['projection_headroom_bytes']),
|
|
)
|
|
capacity_closed = any(
|
|
int(capacity[key] or 0) + requested[key] > max(0, limits[key])
|
|
for key in requested if key != 'projection_bytes'
|
|
) or (
|
|
int(capacity['projection_bytes'] or 0)
|
|
+ requested['projection_bytes'] > projection_ceiling
|
|
)
|
|
if capacity_closed:
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'pipeline_capacity_closed',
|
|
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
|
|
(now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
# Ownership starts only after every contended admission lock is held.
|
|
now, lease_until = fixed_lease_window(lease_seconds)
|
|
reservation_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO result_reservations(
|
|
reservation_token, bundle_id, scan_event_id, queue_id, run_id, cycle_id,
|
|
source, platform, query, target, normalized_target,
|
|
claim_lease_owner, claim_lease_token, claim_batch,
|
|
producer_instance_id, producer_pid, producer_creation_time, producer_executable,
|
|
assignment_kind, remote_user_id, remote_device_id, remote_issued_at,
|
|
remote_expires_at, remote_result_upload_body_timeout_seconds,
|
|
remote_effective_config_sha256,
|
|
remote_client_compat_sha256, remote_execution_snapshot_json,
|
|
remote_execution_snapshot_sha256,
|
|
declared_bundle_bytes, reserved_bundle_bytes,
|
|
reserved_projection_items, reserved_projection_bytes,
|
|
reserved_candidate_items, reserved_candidate_bytes, ready_relative_path,
|
|
state, producer_lease_expires_at, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
?, ?, 1, ?, ?, ?, ?,
|
|
'scanning', ?, ?, ?)''',
|
|
(
|
|
reservation_token, bundle_id, scan_event_id, row['id'], run_id, cycle_id,
|
|
source, platform, row['query'], row['target'], row['normalized_target'],
|
|
owner, claim_token, claim_batch, str(supervisor_instance_id or ''),
|
|
identity['pid'], identity['creation_time'], identity['executable'],
|
|
'remote' if remote else 'local', remote['user_id'] if remote else None,
|
|
remote['device_id'] if remote else None, now if remote else None,
|
|
lease_until if remote else None,
|
|
remote['result_upload_body_timeout_seconds'] if remote else None,
|
|
remote['effective_config_sha256'] if remote else None,
|
|
remote['client_compat_sha256'] if remote else None,
|
|
remote['execution_snapshot_json'] if remote else None,
|
|
remote['execution_snapshot_sha256'] if remote else None,
|
|
declared_bundle_bytes, reserved_bundle_bytes, projection_bytes,
|
|
candidate_items, candidate_bytes,
|
|
ready_relative_path, lease_until, now, now,
|
|
),
|
|
)
|
|
partial_token = hashlib.sha256(reservation_token.encode('utf-8')).hexdigest()[:24]
|
|
for artifact_kind, relative_path, artifact_bytes in (
|
|
(
|
|
'bundle_partial',
|
|
f'tmp/{bundle_id[:2]}/{bundle_id}.{partial_token}.partial',
|
|
declared_bundle_bytes,
|
|
),
|
|
('bundle_ready', ready_relative_path, 0),
|
|
):
|
|
self.conn.execute(
|
|
'''INSERT INTO pipeline_artifacts(
|
|
subsystem, artifact_kind, owner_id, owner_key, relative_path,
|
|
byte_count, state, created_at, updated_at
|
|
) VALUES ('result_bundle', ?, ?, '', ?, ?, 'expected', ?, ?)''',
|
|
(
|
|
artifact_kind, reservation_id, relative_path,
|
|
artifact_bytes, now, now,
|
|
),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'in_progress', lease_owner = ?, lease_token = ?,
|
|
claim_batch = ?, leased_at = ?, lease_expires_at = ?, attempts = COALESCE(attempts, 0) + 1,
|
|
current_result_reservation_id = ?, claim_event_id = ?,
|
|
scan_remote_modified_at = remote_modified_at, updated_at = ?
|
|
WHERE id = ? AND current_result_reservation_id IS NULL
|
|
AND status IN ('pending','deferred')''',
|
|
(
|
|
owner, claim_token, claim_batch, now, lease_until, reservation_id,
|
|
scan_event_id, now, row['id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError('target ownership changed during capacity reservation')
|
|
experiment_binding_id = None
|
|
experiment_attempt = None
|
|
if experiment:
|
|
experiment_attempt = int(row['reservation_count']) + 1
|
|
experiment_binding_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO docker_depth_experiment_scan_bindings(
|
|
experiment_target_id, reservation_id, attempt, state,
|
|
bound_at, created_at
|
|
) VALUES (?, ?, ?, 'reserved', ?, ?)''',
|
|
(
|
|
row['experiment_target_id'], reservation_id,
|
|
experiment_attempt, now, now,
|
|
),
|
|
)
|
|
target_cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_targets
|
|
SET state = 'reserved', reservation_count = reservation_count + 1,
|
|
terminal_at = NULL, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
|
|
AND manifest_id = ? AND state = 'pending'
|
|
AND reservation_count = ?''',
|
|
(
|
|
now, row['experiment_target_id'], experiment['id'], row['id'],
|
|
row['manifest_id'], row['reservation_count'],
|
|
),
|
|
)
|
|
if not experiment_binding_id or int(target_cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding lost its exact target reservation fence'
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
bundle_items = bundle_items + 1,
|
|
bundle_bytes = bundle_bytes + ?,
|
|
projection_items = projection_items + 1,
|
|
projection_bytes = projection_bytes + ?,
|
|
keycheck_items = keycheck_items + ?,
|
|
keycheck_bytes = keycheck_bytes + ?,
|
|
updated_at = ?
|
|
WHERE id = 1''',
|
|
(
|
|
reserved_bundle_bytes, projection_bytes,
|
|
candidate_items, candidate_bytes, now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'committed', reservation_id = ?,
|
|
resolution_detail = 'reservation_committed', resolved_at = ?, updated_at = ?
|
|
WHERE reservation_token = ?''',
|
|
(reservation_id, now, now, reservation_token),
|
|
)
|
|
self.conn.commit()
|
|
created = self.conn.execute(
|
|
'''SELECT r.*, q.attempts,
|
|
binding.id AS experiment_binding_id,
|
|
binding.experiment_target_id,
|
|
binding.attempt AS experiment_attempt,
|
|
target.experiment_id,
|
|
target.dispatch_wave, target.dispatch_order,
|
|
experiment.experiment_key AS bound_experiment_key
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.reservation_id = r.id
|
|
LEFT JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
LEFT JOIN docker_depth_experiments experiment
|
|
ON experiment.id = target.experiment_id
|
|
WHERE r.id = ?''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return self._result_reservation_claim(created)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _result_reservation_claim(row):
|
|
claim = {
|
|
'reservation_id': int(row['id']),
|
|
'reservation_token': str(row['reservation_token']),
|
|
'bundle_id': str(row['bundle_id']),
|
|
'scan_event_id': str(row['scan_event_id']),
|
|
'queue_id': int(row['queue_id']),
|
|
'claim_lease_owner': str(row['claim_lease_owner']),
|
|
'claim_lease_token': str(row['claim_lease_token']),
|
|
'claim_batch': row['claim_batch'],
|
|
'attempts': int(row['attempts'] or 0),
|
|
'declared_bundle_bytes': int(row['declared_bundle_bytes']),
|
|
'ready_relative_path': str(row['ready_relative_path']),
|
|
'source': str(row['source']),
|
|
'platform': str(row['platform']),
|
|
'query': row['query'],
|
|
'target': str(row['target']),
|
|
'normalized_target': str(row['normalized_target']),
|
|
'run_id': row['run_id'],
|
|
'cycle_id': row['cycle_id'],
|
|
'producer_instance_id': str(row['producer_instance_id']),
|
|
'producer_pid': int(row['producer_pid']),
|
|
'producer_creation_time': str(row['producer_creation_time']),
|
|
'producer_executable': str(row['producer_executable']),
|
|
'assignment_kind': str(row['assignment_kind']),
|
|
}
|
|
if claim['assignment_kind'] == 'remote':
|
|
claim.update({
|
|
'remote_user_id': int(row['remote_user_id']),
|
|
'remote_device_id': int(row['remote_device_id']),
|
|
'remote_issued_at': str(row['remote_issued_at']),
|
|
'remote_expires_at': str(row['remote_expires_at']),
|
|
'remote_result_upload_body_timeout_seconds': (
|
|
int(row['remote_result_upload_body_timeout_seconds'])
|
|
if row['remote_result_upload_body_timeout_seconds'] is not None
|
|
else None
|
|
),
|
|
'remote_effective_config_sha256': str(row['remote_effective_config_sha256']),
|
|
'remote_client_compat_sha256': str(row['remote_client_compat_sha256']),
|
|
})
|
|
if 'experiment_binding_id' in row.keys() and row['experiment_binding_id'] is not None:
|
|
claim.update({
|
|
'experiment_binding_id': int(row['experiment_binding_id']),
|
|
'experiment_target_id': int(row['experiment_target_id']),
|
|
'experiment_attempt': int(row['experiment_attempt']),
|
|
'experiment_id': int(row['experiment_id']),
|
|
'experiment_key': str(row['bound_experiment_key']),
|
|
'dispatch_wave': int(row['dispatch_wave']),
|
|
'dispatch_order': int(row['dispatch_order']),
|
|
})
|
|
return claim
|
|
|
|
def docker_layer_timeout_canary_eligible(self, reservation_id, claim_lease_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker layer canary eligibility requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
claim_lease_token = str(claim_lease_token or '')
|
|
if not claim_lease_token:
|
|
raise ValueError('Docker layer canary eligibility requires a claim lease token')
|
|
now = utc_now_iso()
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT q.target, src.metadata_json
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN scan_result_compat src ON src.target_scan_id = q.target_scan_id
|
|
WHERE r.id = ? AND r.state = 'scanning'
|
|
AND r.source = 'dockerhub' AND r.platform = 'docker'
|
|
AND r.claim_lease_token = ?
|
|
AND r.producer_lease_expires_at > ?
|
|
AND q.status = 'in_progress'
|
|
AND q.lease_token = ?
|
|
AND q.lease_expires_at > ?
|
|
AND q.current_result_reservation_id = r.id
|
|
AND q.claim_event_id = r.scan_event_id''',
|
|
(reservation_id, claim_lease_token, now, claim_lease_token, now),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
if not row:
|
|
raise ScanEventConflictError('Docker layer canary eligibility fence changed')
|
|
text = str(row['metadata_json'] or '')
|
|
if not text or len(text.encode('utf-8')) > DOCKER_LAYER_PLAN_MAX_BYTES:
|
|
return False
|
|
try:
|
|
metadata = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return False
|
|
return docker_layer_canary_metadata_eligible(metadata, row['target'])
|
|
|
|
def start_docker_adaptive_shadow_report(
|
|
self, scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
|
|
cohort_size, lease_owner, lease_seconds=3600,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
|
|
policies = validate_docker_adaptive_policy_hashes(
|
|
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
|
|
)
|
|
if isinstance(cohort_size, bool):
|
|
raise ValueError('Docker adaptive shadow cohort size must be an integer')
|
|
cohort_size = int(cohort_size)
|
|
if not DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS:
|
|
raise ValueError('Docker adaptive shadow cohort size must be between 50 and 100')
|
|
lease_owner = str(lease_owner or '').strip()
|
|
if not lease_owner or len(lease_owner) > 256 or not lease_owner.isascii():
|
|
raise ValueError('Docker adaptive shadow lease owner is invalid')
|
|
if isinstance(lease_seconds, bool):
|
|
raise ValueError('Docker adaptive shadow lease duration must be an integer')
|
|
lease_seconds = int(lease_seconds)
|
|
if not 60 <= lease_seconds <= 86400:
|
|
raise ValueError('Docker adaptive shadow lease duration is outside the supported range')
|
|
report_token = secrets.token_hex(32)
|
|
lease_token = secrets.token_urlsafe(32)
|
|
now = utc_now_iso()
|
|
lease_expires_at = datetime.fromtimestamp(
|
|
time.time() + lease_seconds, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
lock_identity = ':'.join(policies.values())
|
|
try:
|
|
self.conn.execute(
|
|
'SELECT pg_catalog.pg_advisory_xact_lock('
|
|
'pg_catalog.hashtextextended(?, 1095982177))',
|
|
(lock_identity,),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE docker_adaptive_shadow_reports SET
|
|
state = 'failed', failure_count = failure_count + 1,
|
|
passed = 0, completed_at = ?, updated_at = ?
|
|
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
|
|
AND selection_policy_sha256 = ? AND state = 'running'
|
|
AND lease_expires_at <= ?''',
|
|
(
|
|
now, now, policies['scan_policy_sha256'],
|
|
policies['execution_policy_sha256'],
|
|
policies['selection_policy_sha256'], now,
|
|
),
|
|
)
|
|
active = self.conn.execute(
|
|
'''SELECT id FROM docker_adaptive_shadow_reports
|
|
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
|
|
AND selection_policy_sha256 = ? AND state = 'running'
|
|
LIMIT 1 FOR UPDATE''',
|
|
(
|
|
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
|
|
policies['selection_policy_sha256'],
|
|
),
|
|
).fetchone()
|
|
if active:
|
|
raise ScanEventConflictError('Docker adaptive shadow report is already running')
|
|
row = self.conn.execute(
|
|
'''INSERT INTO docker_adaptive_shadow_reports(
|
|
report_token, evaluator_version, state, scan_policy_sha256,
|
|
execution_policy_sha256, selection_policy_sha256, cohort_size,
|
|
lease_owner, lease_token, lease_expires_at, started_at,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, 'running', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
RETURNING id, report_token, evaluator_version, state,
|
|
scan_policy_sha256, execution_policy_sha256,
|
|
selection_policy_sha256, cohort_size, lease_owner,
|
|
lease_token, lease_expires_at, started_at''',
|
|
(
|
|
report_token, DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION,
|
|
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
|
|
policies['selection_policy_sha256'], cohort_size, lease_owner,
|
|
lease_token, lease_expires_at, now, now, now,
|
|
),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def docker_adaptive_shadow_controls(self, scan_policy_sha256, cohort_size, selection_salt):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow controls require PostgreSQL')
|
|
scan_policy_sha256 = str(scan_policy_sha256 or '').lower()
|
|
selection_salt = str(selection_salt or '').lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
|
|
raise ValueError('Docker adaptive shadow scan policy must be a lowercase SHA-256')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', selection_salt):
|
|
raise ValueError('Docker adaptive shadow selection salt must be a lowercase SHA-256')
|
|
if isinstance(cohort_size, bool):
|
|
raise ValueError('Docker adaptive shadow cohort size must be an integer')
|
|
cohort_size = int(cohort_size)
|
|
if not DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS:
|
|
raise ValueError('Docker adaptive shadow cohort size must be between 50 and 100')
|
|
try:
|
|
rows = self.conn.execute(
|
|
'''WITH eligible AS (
|
|
SELECT ts.id AS target_scan_id, ts.normalized_target,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY ts.normalized_target
|
|
ORDER BY ts.ended_at DESC, ts.id DESC
|
|
) AS target_rank
|
|
FROM target_scans ts
|
|
JOIN target_queue q
|
|
ON q.id = ts.queue_id AND q.target_scan_id = ts.id
|
|
JOIN result_reservations r
|
|
ON r.id = ts.result_reservation_id AND r.queue_id = q.id
|
|
JOIN result_bundles b
|
|
ON b.reservation_id = r.id AND b.target_scan_id = ts.id
|
|
JOIN scan_result_compat src ON src.target_scan_id = ts.id
|
|
WHERE ts.source = 'dockerhub' AND ts.scan_type = 'docker'
|
|
AND ts.status IN ('clean','found')
|
|
AND COALESCE(ts.error_count, 0) = 0
|
|
AND ts.skipped_reason IS NULL AND ts.ended_at IS NOT NULL
|
|
AND ts.queue_completion_applied = 1
|
|
AND ts.queue_completion_disposition = 'applied'
|
|
AND ts.raw_result_storage = 'normalized_v2'
|
|
AND q.source = 'dockerhub' AND q.platform = 'docker'
|
|
AND q.status = 'done'
|
|
AND r.source = 'dockerhub' AND r.platform = 'docker'
|
|
AND r.state = 'acknowledged' AND b.state = 'acknowledged'
|
|
AND r.docker_layer_plan_json IS NULL
|
|
AND r.docker_layer_plan_sha256 IS NULL
|
|
AND ts.normalized_target ~ '@sha256:[a-f0-9]{64}$'
|
|
AND (ts.scan_options_json::jsonb ->> 'scan_policy_sha256') = ?
|
|
AND (src.metadata_json::jsonb #>>
|
|
'{scan_meta,docker_scan_assignment,effective_mode}') = 'full'
|
|
)
|
|
SELECT target_scan_id, normalized_target
|
|
FROM eligible WHERE target_rank = 1
|
|
ORDER BY md5(? || ':' || normalized_target), target_scan_id
|
|
LIMIT ?''',
|
|
(scan_policy_sha256, selection_salt, cohort_size),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
if len(rows) != cohort_size:
|
|
raise RuntimeError('Docker adaptive shadow exact-policy control cohort is incomplete')
|
|
controls = []
|
|
for row in rows:
|
|
normalized_target = str(row['normalized_target'] or '')
|
|
try:
|
|
parsed = parse_docker_target(normalized_target)
|
|
except (TypeError, ValueError) as exc:
|
|
raise RuntimeError('Docker adaptive shadow control target is invalid') from exc
|
|
if not re.search(r'@sha256:[a-f0-9]{64}$', str(parsed.get('image') or '')):
|
|
raise RuntimeError('Docker adaptive shadow control target is not immutable')
|
|
controls.append({
|
|
'target_scan_id': int(row['target_scan_id']),
|
|
'normalized_target': normalized_target,
|
|
})
|
|
return controls
|
|
|
|
def docker_adaptive_shadow_control_identities(self, target_scan_id):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow control identities require PostgreSQL')
|
|
target_scan_id = int(target_scan_id)
|
|
if target_scan_id <= 0:
|
|
raise ValueError('Docker adaptive shadow control scan identity is invalid')
|
|
try:
|
|
routed_rows = self.conn.execute(
|
|
'''SELECT c.routed_service, kc.provider_key_hash
|
|
FROM keycheck_candidates c
|
|
JOIN keycheck_credentials kc ON kc.id = c.credential_id
|
|
WHERE c.target_scan_id = ?''',
|
|
(target_scan_id,),
|
|
).fetchall()
|
|
detector_rows = self.conn.execute(
|
|
'''SELECT detector_secret_hash FROM findings
|
|
WHERE target_scan_id = ?''',
|
|
(target_scan_id,),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
routed = set()
|
|
for row in routed_rows:
|
|
service = str(row['routed_service'] or '')
|
|
provider_key_hash = str(row['provider_key_hash'] or '').lower()
|
|
if (
|
|
not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,63}', service)
|
|
or not re.fullmatch(r'[a-f0-9]{64}', provider_key_hash)
|
|
):
|
|
raise RuntimeError('Docker adaptive shadow routed identity snapshot is incomplete')
|
|
routed.add((service, provider_key_hash))
|
|
detectors = set()
|
|
for row in detector_rows:
|
|
detector_secret_hash = str(row['detector_secret_hash'] or '').lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', detector_secret_hash):
|
|
raise RuntimeError('Docker adaptive shadow detector identity snapshot is incomplete')
|
|
detectors.add(detector_secret_hash)
|
|
return frozenset(routed), frozenset(detectors)
|
|
|
|
def finish_docker_adaptive_shadow_report(
|
|
self, report_token, lease_owner, lease_token, **values,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
|
|
report_token = str(report_token or '')
|
|
lease_owner = str(lease_owner or '')
|
|
lease_token = str(lease_token or '')
|
|
if not report_token or not lease_owner or not lease_token:
|
|
raise ValueError('Docker adaptive shadow report fence is required')
|
|
metrics = docker_adaptive_shadow_gate_metrics(values)
|
|
selection_metrics = validate_docker_adaptive_shadow_selection_metrics(
|
|
values.get('selection_metrics')
|
|
)
|
|
if sum(
|
|
selection_metrics[name] for name in DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS
|
|
) != metrics['omitted_descriptor_count']:
|
|
raise ValueError('Docker adaptive shadow omission metrics do not reconcile')
|
|
now = utc_now_iso()
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM docker_adaptive_shadow_reports
|
|
WHERE report_token = ? FOR UPDATE''',
|
|
(report_token,),
|
|
).fetchone()
|
|
if not row:
|
|
raise ScanEventConflictError('Docker adaptive shadow report is absent')
|
|
cohort_size = int(row['cohort_size'])
|
|
if metrics['completed_pairs'] > cohort_size:
|
|
raise ValueError('Docker adaptive completed pairs exceed the cohort size')
|
|
if (
|
|
str(row['state']) != 'running'
|
|
or str(row['lease_owner']) != lease_owner
|
|
or str(row['lease_token']) != lease_token
|
|
or str(row['lease_expires_at']) <= now
|
|
):
|
|
raise ScanEventConflictError('Docker adaptive shadow report fence changed')
|
|
if int(row['sink_checkpoint_count']) != metrics['completed_pairs'] * 2:
|
|
raise ValueError('Docker adaptive shadow sink checkpoints do not reconcile')
|
|
passed = int(
|
|
metrics['completed_pairs'] == cohort_size
|
|
and DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS
|
|
and metrics['full_routed_count'] > 0
|
|
and metrics['full_slot_ms'] > 0
|
|
and metrics['adaptive_slot_ms'] > 0
|
|
and metrics['routed_recall_ppm'] >= DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
|
|
and metrics['slot_ratio_ppm'] <= DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
|
|
and metrics['failure_count'] == 0
|
|
and metrics['privacy_violation_count'] == 0
|
|
and metrics['safety_regression_count'] == 0
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_adaptive_shadow_reports SET
|
|
state = 'completed', completed_pairs = ?,
|
|
full_routed_count = ?, adaptive_routed_count = ?,
|
|
routed_intersection_count = ?, full_detector_count = ?,
|
|
adaptive_detector_count = ?, detector_intersection_count = ?,
|
|
full_slot_ms = ?, adaptive_slot_ms = ?,
|
|
omitted_descriptor_count = ?, failure_count = ?,
|
|
privacy_violation_count = ?, safety_regression_count = ?,
|
|
selection_metrics_json = ?,
|
|
routed_recall_ppm = ?, slot_ratio_ppm = ?, passed = ?,
|
|
completed_at = ?, updated_at = ?
|
|
WHERE id = ? AND state = 'running' AND lease_owner = ?
|
|
AND lease_token = ? AND lease_expires_at > ?''',
|
|
(
|
|
metrics['completed_pairs'], metrics['full_routed_count'],
|
|
metrics['adaptive_routed_count'], metrics['routed_intersection_count'],
|
|
metrics['full_detector_count'], metrics['adaptive_detector_count'],
|
|
metrics['detector_intersection_count'], metrics['full_slot_ms'],
|
|
metrics['adaptive_slot_ms'], metrics['omitted_descriptor_count'],
|
|
metrics['failure_count'], metrics['privacy_violation_count'],
|
|
metrics['safety_regression_count'], json_dumps(selection_metrics),
|
|
metrics['routed_recall_ppm'],
|
|
metrics['slot_ratio_ppm'], passed, now, now, int(row['id']),
|
|
lease_owner, lease_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker adaptive shadow report completion lost its fence')
|
|
self.conn.commit()
|
|
return {
|
|
'report_id': int(row['id']),
|
|
'passed': bool(passed),
|
|
'cohort_size': cohort_size,
|
|
**metrics,
|
|
'selection_metrics': selection_metrics,
|
|
'completed_at': now,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def checkpoint_docker_adaptive_shadow_report(
|
|
self, report_token, lease_owner, lease_token, lease_seconds=3600,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
|
|
if isinstance(lease_seconds, bool):
|
|
raise ValueError('Docker adaptive shadow lease duration must be an integer')
|
|
lease_seconds = int(lease_seconds)
|
|
if not 60 <= lease_seconds <= 86400:
|
|
raise ValueError('Docker adaptive shadow lease duration is outside the supported range')
|
|
now = utc_now_iso()
|
|
lease_expires_at = datetime.fromtimestamp(
|
|
time.time() + lease_seconds, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
try:
|
|
row = self.conn.execute(
|
|
'''UPDATE docker_adaptive_shadow_reports SET
|
|
sink_checkpoint_count = sink_checkpoint_count + 1,
|
|
lease_expires_at = ?, updated_at = ?
|
|
WHERE report_token = ? AND state = 'running'
|
|
AND lease_owner = ? AND lease_token = ? AND lease_expires_at > ?
|
|
RETURNING id, sink_checkpoint_count, lease_expires_at''',
|
|
(
|
|
lease_expires_at, now, str(report_token or ''),
|
|
str(lease_owner or ''), str(lease_token or ''), now,
|
|
),
|
|
).fetchone()
|
|
if not row:
|
|
raise ScanEventConflictError('Docker adaptive shadow checkpoint lost its fence')
|
|
self.conn.commit()
|
|
return {
|
|
'report_id': int(row['id']),
|
|
'sink_checkpoint_count': int(row['sink_checkpoint_count']),
|
|
'lease_expires_at': row['lease_expires_at'],
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def fail_docker_adaptive_shadow_report(
|
|
self, report_token, lease_owner, lease_token,
|
|
privacy_violation_count=0, safety_regression_count=0,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive shadow reports require PostgreSQL')
|
|
privacy_violation_count = int(privacy_violation_count)
|
|
safety_regression_count = int(safety_regression_count)
|
|
if privacy_violation_count < 0 or safety_regression_count < 0:
|
|
raise ValueError('Docker adaptive shadow failure counters must be non-negative')
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_adaptive_shadow_reports SET
|
|
state = 'failed', failure_count = failure_count + 1,
|
|
privacy_violation_count = ?, safety_regression_count = ?,
|
|
passed = 0, completed_at = ?, updated_at = ?
|
|
WHERE report_token = ? AND state = 'running'
|
|
AND lease_owner = ? AND lease_token = ? AND lease_expires_at > ?''',
|
|
(
|
|
privacy_violation_count, safety_regression_count, now, now,
|
|
str(report_token or ''), str(lease_owner or ''),
|
|
str(lease_token or ''), now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker adaptive shadow report failure lost its fence')
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def docker_adaptive_gate_report(
|
|
self, reservation_id, claim_lease_token, scan_policy_sha256,
|
|
execution_policy_sha256, selection_policy_sha256, max_age_sec=604800,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker adaptive gate lookup requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
claim_lease_token = str(claim_lease_token or '')
|
|
if not claim_lease_token:
|
|
raise ValueError('Docker adaptive gate lookup requires a claim lease token')
|
|
policies = validate_docker_adaptive_policy_hashes(
|
|
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
|
|
)
|
|
if isinstance(max_age_sec, bool):
|
|
raise ValueError('Docker adaptive gate freshness must be an integer')
|
|
max_age_sec = int(max_age_sec)
|
|
if not 60 <= max_age_sec <= 2592000:
|
|
raise ValueError('Docker adaptive gate freshness is outside the supported range')
|
|
now = utc_now_iso()
|
|
cutoff = (datetime.now(timezone.utc) - timedelta(seconds=max_age_sec)).isoformat(
|
|
timespec='seconds',
|
|
)
|
|
try:
|
|
claim = self.conn.execute(
|
|
'''SELECT r.id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? AND r.state = 'scanning'
|
|
AND r.source = 'dockerhub' AND r.platform = 'docker'
|
|
AND r.claim_lease_token = ? AND r.producer_lease_expires_at > ?
|
|
AND q.status = 'in_progress' AND q.lease_token = ?
|
|
AND q.lease_expires_at > ?
|
|
AND q.current_result_reservation_id = r.id
|
|
AND q.claim_event_id = r.scan_event_id
|
|
FOR UPDATE OF r, q''',
|
|
(reservation_id, claim_lease_token, now, claim_lease_token, now),
|
|
).fetchone()
|
|
if not claim:
|
|
raise ScanEventConflictError('Docker adaptive gate claim fence changed')
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM docker_adaptive_shadow_reports
|
|
WHERE scan_policy_sha256 = ? AND execution_policy_sha256 = ?
|
|
AND selection_policy_sha256 = ?
|
|
ORDER BY id DESC LIMIT 1''',
|
|
(
|
|
policies['scan_policy_sha256'], policies['execution_policy_sha256'],
|
|
policies['selection_policy_sha256'],
|
|
),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
if not row:
|
|
return {'passed': False, 'reason': 'missing'}
|
|
result = {
|
|
'report_id': int(row['id']),
|
|
'passed': False,
|
|
'reason': 'failed',
|
|
'completed_at': row['completed_at'],
|
|
}
|
|
if str(row['state']) != 'completed':
|
|
result['reason'] = str(row['state'])
|
|
return result
|
|
if not row['completed_at'] or str(row['completed_at']) < cutoff:
|
|
result['reason'] = 'stale'
|
|
return result
|
|
metrics = docker_adaptive_shadow_gate_metrics(dict(row))
|
|
try:
|
|
selection_metrics = validate_docker_adaptive_shadow_selection_metrics(
|
|
row['selection_metrics_json']
|
|
)
|
|
except ValueError:
|
|
result['reason'] = 'invalid_evidence'
|
|
return result
|
|
cohort_size = int(row['cohort_size'])
|
|
expected_pass = bool(
|
|
str(row['evaluator_version']) == DOCKER_ADAPTIVE_SHADOW_EVALUATOR_VERSION
|
|
and DOCKER_ADAPTIVE_GATE_MIN_CONTROLS <= cohort_size <= DOCKER_ADAPTIVE_GATE_MAX_CONTROLS
|
|
and metrics['completed_pairs'] == cohort_size
|
|
and int(row['sink_checkpoint_count']) == cohort_size * 2
|
|
and sum(
|
|
selection_metrics[name]
|
|
for name in DOCKER_ADAPTIVE_SHADOW_OMISSION_METRIC_KEYS
|
|
) == metrics['omitted_descriptor_count']
|
|
and metrics['full_routed_count'] > 0
|
|
and metrics['full_slot_ms'] > 0
|
|
and metrics['adaptive_slot_ms'] > 0
|
|
and metrics['routed_recall_ppm'] >= DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
|
|
and metrics['slot_ratio_ppm'] <= DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
|
|
and metrics['failure_count'] == 0
|
|
and metrics['privacy_violation_count'] == 0
|
|
and metrics['safety_regression_count'] == 0
|
|
and int(row['recall_threshold_ppm']) == DOCKER_ADAPTIVE_GATE_ROUTED_RECALL_PPM
|
|
and int(row['slot_threshold_ppm']) == DOCKER_ADAPTIVE_GATE_SLOT_RATIO_PPM
|
|
and int(row['routed_recall_ppm']) == metrics['routed_recall_ppm']
|
|
and int(row['slot_ratio_ppm']) == metrics['slot_ratio_ppm']
|
|
)
|
|
if bool(row['passed']) != expected_pass or not expected_pass:
|
|
result['reason'] = 'thresholds'
|
|
return result
|
|
result.update({
|
|
'passed': True,
|
|
'reason': 'passed',
|
|
'cohort_size': cohort_size,
|
|
'completed_pairs': metrics['completed_pairs'],
|
|
'routed_recall_ppm': metrics['routed_recall_ppm'],
|
|
'slot_ratio_ppm': metrics['slot_ratio_ppm'],
|
|
})
|
|
return result
|
|
|
|
def _alias_compatible_legacy_docker_coverage(
|
|
self, descriptor, coverage_policy_sha256, scan_policy_sha256, limits, now,
|
|
):
|
|
candidates = self.conn.execute(
|
|
'''SELECT blob.*, reservation.state AS reservation_state,
|
|
reservation.scan_event_id AS reservation_scan_event_id,
|
|
reservation.docker_layer_plan_json,
|
|
reservation.docker_layer_plan_sha256
|
|
FROM docker_content_blobs blob
|
|
JOIN result_reservations reservation
|
|
ON reservation.id = blob.covered_reservation_id
|
|
WHERE blob.digest = ? AND blob.coverage_policy_sha256 <> ?
|
|
AND blob.state = 'covered'
|
|
AND blob.covered_policy_sha256 = blob.coverage_policy_sha256
|
|
ORDER BY blob.covered_reservation_id, blob.coverage_policy_sha256
|
|
FOR UPDATE OF blob''',
|
|
(descriptor['digest'], coverage_policy_sha256),
|
|
).fetchall()
|
|
archive_keys = (
|
|
'archive_max_size_bytes', 'archive_max_depth', 'archive_timeout_sec',
|
|
)
|
|
for candidate in candidates:
|
|
try:
|
|
plan, plan_sha256 = stored_docker_layer_plan(candidate)
|
|
except RuntimeSafetySchemaError:
|
|
continue
|
|
if (
|
|
not plan
|
|
or plan['version'] != 1
|
|
or str(candidate['reservation_state']) not in ('db_committed', 'acknowledged')
|
|
or not candidate['covered_reservation_id']
|
|
or not candidate['covered_scan_event_id']
|
|
or str(candidate['covered_scan_event_id'])
|
|
!= str(candidate['reservation_scan_event_id'])
|
|
or not candidate['covered_at']
|
|
or int(candidate['verified_bytes'] or -1) != descriptor['size']
|
|
or str(candidate['descriptor_kind']) != descriptor['kind']
|
|
or int(candidate['declared_bytes']) != descriptor['size']
|
|
or docker_content_media_class(
|
|
candidate['descriptor_kind'], candidate['media_type'],
|
|
) != docker_content_media_class(descriptor['kind'], descriptor['media_type'])
|
|
or plan['scan_policy_sha256'] != scan_policy_sha256
|
|
or docker_layer_plan_coverage_policy_sha256(plan)
|
|
!= str(candidate['coverage_policy_sha256'])
|
|
or any(plan['limits'][key] != limits[key] for key in archive_keys)
|
|
):
|
|
continue
|
|
matching_descriptors = [
|
|
item for item in plan['descriptors']
|
|
if item['digest'] == descriptor['digest']
|
|
and item['kind'] == descriptor['kind']
|
|
and item['size'] == descriptor['size']
|
|
and docker_content_media_class(item['kind'], item['media_type'])
|
|
== docker_content_media_class(descriptor['kind'], descriptor['media_type'])
|
|
and item['selected']
|
|
]
|
|
if not matching_descriptors:
|
|
continue
|
|
evidence = self.conn.execute(
|
|
'''SELECT 1 AS present FROM docker_image_blob_coverage
|
|
WHERE reservation_id = ? AND blob_digest = ?
|
|
AND coverage_policy_sha256 = ? AND plan_sha256 = ?
|
|
AND descriptor_kind = ? AND selected = 1
|
|
AND coverage_state = 'covered' AND covered_at IS NOT NULL
|
|
LIMIT 1''',
|
|
(
|
|
candidate['covered_reservation_id'], descriptor['digest'],
|
|
candidate['coverage_policy_sha256'], plan_sha256, descriptor['kind'],
|
|
),
|
|
).fetchone()
|
|
if not evidence:
|
|
continue
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET
|
|
state = 'covered', attempts = 0, max_attempts = ?,
|
|
available_after = NULL, lease_reservation_id = NULL,
|
|
lease_token = NULL, lease_plan_sha256 = NULL, lease_expires_at = NULL,
|
|
covered_reservation_id = ?, covered_scan_event_id = ?,
|
|
covered_policy_sha256 = ?, verified_bytes = ?, covered_at = ?,
|
|
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ?
|
|
AND state = 'pending' AND attempts = 0
|
|
AND lease_reservation_id IS NULL AND covered_reservation_id IS NULL''',
|
|
(
|
|
limits['blob_max_attempts'], candidate['covered_reservation_id'],
|
|
candidate['covered_scan_event_id'], coverage_policy_sha256,
|
|
descriptor['size'], candidate['covered_at'], now,
|
|
descriptor['digest'], coverage_policy_sha256,
|
|
),
|
|
)
|
|
return int(cursor.rowcount or 0) == 1
|
|
return False
|
|
|
|
def bind_docker_layer_plan(
|
|
self, reservation_id, claim_lease_token, resolved, limits,
|
|
scan_policy_sha256, blob_lease_seconds=1800, *, payload_classes=None,
|
|
checkpoint=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker layer plan binding requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
claim_lease_token = str(claim_lease_token or '')
|
|
if not claim_lease_token:
|
|
raise ValueError('Docker layer plan binding requires a claim lease token')
|
|
resolved = validate_docker_layer_resolution(resolved)
|
|
limits = validate_docker_layer_limits(limits)
|
|
scan_policy_sha256 = str(scan_policy_sha256 or '').strip().lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', scan_policy_sha256):
|
|
raise ValueError('Docker layer scanner policy hash is invalid')
|
|
descriptors = [resolved['config']] + list(resolved['layers'])
|
|
adaptive = payload_classes is not None or checkpoint is not None
|
|
if adaptive:
|
|
payload_classes = list(payload_classes or [])
|
|
checkpoint = validate_docker_adaptive_checkpoint(checkpoint)
|
|
if (
|
|
len(payload_classes) != len(resolved['layers'])
|
|
or any(
|
|
value not in DOCKER_ADAPTIVE_PAYLOAD_CLASSES - {'config'}
|
|
for value in payload_classes
|
|
)
|
|
):
|
|
raise ValueError('Docker adaptive payload classes do not match its resolution')
|
|
selection_policy_sha256 = docker_layer_selection_policy_sha256(limits)
|
|
coverage_policy_sha256 = docker_layer_execution_policy_sha256(
|
|
scan_policy_sha256, limits,
|
|
)
|
|
else:
|
|
policy_bytes = json.dumps(
|
|
limits, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
).encode('ascii')
|
|
selection_policy_sha256 = hashlib.sha256(policy_bytes).hexdigest()
|
|
coverage_policy_sha256 = docker_layer_coverage_policy_sha256(
|
|
scan_policy_sha256, limits,
|
|
)
|
|
blob_lease_seconds = max(
|
|
limits['blob_timeout_sec'] + DOCKER_BLOB_LEASE_MARGIN_SEC,
|
|
int(blob_lease_seconds or 1800),
|
|
)
|
|
blob_lease_seconds = max(60, min(7200, blob_lease_seconds))
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not row:
|
|
raise ScanEventConflictError('Docker layer reservation is absent')
|
|
now, lease_until = fixed_lease_window(blob_lease_seconds)
|
|
if (
|
|
str(row['state']) != 'scanning'
|
|
or str(row['claim_lease_token']) != claim_lease_token
|
|
or str(row['queue_status']) != 'in_progress'
|
|
or str(row['queue_lease_token']) != claim_lease_token
|
|
or int(row['queue_reservation_id'] or 0) != reservation_id
|
|
or str(row['queue_event_id']) != str(row['scan_event_id'])
|
|
or str(row['queue_lease_expires_at'] or '') <= now
|
|
or str(row['producer_lease_expires_at'] or '') <= now
|
|
):
|
|
raise ScanEventConflictError('Docker layer reservation is no longer actively fenced')
|
|
if str(row['assignment_kind']) == 'remote':
|
|
remote_expires_at = str(row['remote_expires_at'] or '')
|
|
if remote_expires_at <= now:
|
|
raise ScanEventConflictError('remote Docker layer reservation has expired')
|
|
lease_until = remote_expires_at
|
|
if str(row['platform']).lower() != 'docker' or str(row['source']).lower() != 'dockerhub':
|
|
raise ScanEventConflictError('Docker layer plan conflicts with the reservation source')
|
|
if docker_target_identity(row['target']) != resolved['image']:
|
|
raise ScanEventConflictError('Docker layer resolution conflicts with the reservation target')
|
|
|
|
stored_json = row['docker_layer_plan_json']
|
|
stored_sha256 = row['docker_layer_plan_sha256']
|
|
if stored_json is not None or stored_sha256 is not None:
|
|
stored_plan, stored_sha256 = stored_docker_layer_plan(row)
|
|
stored_resolution = [
|
|
{
|
|
'digest': descriptor['digest'],
|
|
'size': descriptor['size'],
|
|
'media_type': descriptor['media_type'],
|
|
'kind': descriptor['kind'],
|
|
'position': descriptor['position'],
|
|
}
|
|
for descriptor in stored_plan['descriptors']
|
|
]
|
|
if (
|
|
stored_plan['image'] != resolved['image']
|
|
or stored_plan['repository'] != resolved['repository']
|
|
or stored_plan['manifest_digest'] != resolved['manifest_digest']
|
|
or stored_plan['platform_os'] != resolved['platform_os']
|
|
or stored_plan['platform_arch'] != resolved['platform_arch']
|
|
or stored_plan['manifest_media_type'] != resolved['manifest_media_type']
|
|
or str(stored_plan.get('scan_policy_sha256') or '') != scan_policy_sha256
|
|
or docker_layer_plan_coverage_policy_sha256(stored_plan) != coverage_policy_sha256
|
|
or stored_plan.get('limits') != limits
|
|
or stored_plan.get('selection_policy_sha256') != selection_policy_sha256
|
|
or stored_resolution != descriptors
|
|
or (stored_plan['version'] == 2) != adaptive
|
|
or (
|
|
adaptive
|
|
and (
|
|
stored_plan.get('checkpoint') != checkpoint
|
|
or [
|
|
descriptor['payload_class']
|
|
for descriptor in stored_plan['descriptors'][1:]
|
|
] != payload_classes
|
|
)
|
|
)
|
|
):
|
|
raise ScanEventConflictError('Docker layer plan replay conflicts with its reservation')
|
|
self.conn.commit()
|
|
return stored_plan
|
|
|
|
for digest in sorted({descriptor['digest'] for descriptor in descriptors}):
|
|
self.conn.execute(
|
|
'SELECT pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtextextended(?, 1764436591))',
|
|
(digest,),
|
|
)
|
|
blob_rows = {}
|
|
for descriptor in descriptors:
|
|
inserted = self.conn.execute(
|
|
'''INSERT INTO docker_content_blobs(
|
|
digest, coverage_policy_sha256, descriptor_kind,
|
|
declared_bytes, media_type, state,
|
|
attempts, max_attempts, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?)
|
|
ON CONFLICT(digest, coverage_policy_sha256) DO NOTHING''',
|
|
(
|
|
descriptor['digest'], coverage_policy_sha256, descriptor['kind'],
|
|
descriptor['size'], descriptor['media_type'],
|
|
limits['blob_max_attempts'], now, now,
|
|
),
|
|
)
|
|
blob = self.conn.execute(
|
|
'''SELECT * FROM docker_content_blobs
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
|
|
(descriptor['digest'], coverage_policy_sha256),
|
|
).fetchone()
|
|
digest_metadata = self.conn.execute(
|
|
'''SELECT descriptor_kind, declared_bytes, media_type
|
|
FROM docker_content_blobs WHERE digest = ?''',
|
|
(descriptor['digest'],),
|
|
).fetchall()
|
|
conflicting_metadata = any(
|
|
str(metadata['descriptor_kind']) != descriptor['kind']
|
|
or int(metadata['declared_bytes']) != descriptor['size']
|
|
or docker_content_media_class(
|
|
metadata['descriptor_kind'], metadata['media_type'],
|
|
) != docker_content_media_class(
|
|
descriptor['kind'], descriptor['media_type'],
|
|
)
|
|
for metadata in digest_metadata
|
|
)
|
|
if not blob or conflicting_metadata or (
|
|
str(blob['descriptor_kind']) != descriptor['kind']
|
|
or int(blob['declared_bytes']) != descriptor['size']
|
|
or docker_content_media_class(
|
|
blob['descriptor_kind'], blob['media_type'],
|
|
) != docker_content_media_class(
|
|
descriptor['kind'], descriptor['media_type'],
|
|
)
|
|
or (
|
|
not adaptive
|
|
and int(blob['max_attempts']) != limits['blob_max_attempts']
|
|
)
|
|
):
|
|
raise ScanEventConflictError('Docker content digest resolves to conflicting metadata')
|
|
if adaptive and int(inserted.rowcount or 0) == 1:
|
|
self._alias_compatible_legacy_docker_coverage(
|
|
descriptor, coverage_policy_sha256, scan_policy_sha256, limits, now,
|
|
)
|
|
blob = self.conn.execute(
|
|
'''SELECT * FROM docker_content_blobs
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
|
|
(descriptor['digest'], coverage_policy_sha256),
|
|
).fetchone()
|
|
blob_rows[descriptor['digest']] = blob
|
|
|
|
first_selection_rows = self.conn.execute(
|
|
'''SELECT coverage.position, coverage.blob_digest, coverage.selected,
|
|
coverage.selection_reason
|
|
FROM docker_image_blob_coverage coverage
|
|
JOIN (
|
|
SELECT position, MIN(reservation_id) AS reservation_id
|
|
FROM docker_image_blob_coverage
|
|
WHERE queue_id = ? AND manifest_digest = ?
|
|
AND (
|
|
(? = 1 AND selection_policy_sha256 = ?)
|
|
OR (? = 0 AND coverage_policy_sha256 = ?)
|
|
)
|
|
GROUP BY position
|
|
) first_position
|
|
ON first_position.position = coverage.position
|
|
AND first_position.reservation_id = coverage.reservation_id
|
|
WHERE coverage.queue_id = ? AND coverage.manifest_digest = ?
|
|
AND (
|
|
(? = 1 AND coverage.selection_policy_sha256 = ?)
|
|
OR (? = 0 AND coverage.coverage_policy_sha256 = ?)
|
|
)
|
|
ORDER BY coverage.position''',
|
|
(
|
|
row['queue_id'], resolved['manifest_digest'],
|
|
1 if adaptive else 0, selection_policy_sha256,
|
|
1 if adaptive else 0, coverage_policy_sha256,
|
|
row['queue_id'], resolved['manifest_digest'],
|
|
1 if adaptive else 0, selection_policy_sha256,
|
|
1 if adaptive else 0, coverage_policy_sha256,
|
|
),
|
|
).fetchall()
|
|
first_selection = {
|
|
int(selection['position']): selection for selection in first_selection_rows
|
|
}
|
|
if first_selection:
|
|
if len(first_selection) != len(descriptors):
|
|
raise ScanEventConflictError('Docker image selection baseline is incomplete')
|
|
for descriptor in descriptors:
|
|
selection = first_selection.get(descriptor['position'])
|
|
if not selection or str(selection['blob_digest']) != descriptor['digest']:
|
|
raise ScanEventConflictError('Docker image selection baseline changed')
|
|
entries = []
|
|
if adaptive:
|
|
classes_by_position = {
|
|
0: 'config',
|
|
**{
|
|
position: payload_class
|
|
for position, payload_class in enumerate(payload_classes, start=1)
|
|
},
|
|
}
|
|
if first_selection:
|
|
entries = [
|
|
(
|
|
descriptor,
|
|
bool(first_selection[descriptor['position']]['selected']),
|
|
str(first_selection[descriptor['position']]['selection_reason']),
|
|
classes_by_position[descriptor['position']],
|
|
)
|
|
for descriptor in descriptors
|
|
]
|
|
else:
|
|
covered_digests = {
|
|
digest for digest, blob in blob_rows.items()
|
|
if str(blob['state']) == 'covered'
|
|
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
|
|
}
|
|
entries = select_docker_adaptive_payload(
|
|
descriptors, payload_classes, limits, covered_digests,
|
|
)
|
|
class_rank = {
|
|
payload_class: index
|
|
for index, payload_class in enumerate(DOCKER_ADAPTIVE_LAYER_CLASS_ORDER)
|
|
}
|
|
entries.sort(key=lambda entry: (
|
|
0 if entry[0]['kind'] == 'config' else 1 + class_rank[entry[3]],
|
|
-entry[0]['position'], entry[0]['size'], entry[0]['digest'],
|
|
))
|
|
else:
|
|
selected_layer_bytes = 0
|
|
selected_layer_count = 0
|
|
selected_layer_digests = set()
|
|
config = descriptors[0]
|
|
if first_selection:
|
|
selection = first_selection[config['position']]
|
|
entries.append((
|
|
config, bool(selection['selected']),
|
|
str(selection['selection_reason']), None,
|
|
))
|
|
elif config['media_type'] not in DOCKER_CONFIG_MEDIA_TYPES:
|
|
entries.append((config, False, 'unsupported_media_type', None))
|
|
elif config['size'] > limits['config_max_bytes']:
|
|
entries.append((config, False, 'config_too_large', None))
|
|
else:
|
|
entries.append((config, True, 'config_selected', None))
|
|
|
|
for descriptor in reversed(descriptors[1:]):
|
|
blob = blob_rows[descriptor['digest']]
|
|
if first_selection:
|
|
selection = first_selection[descriptor['position']]
|
|
entries.append((
|
|
descriptor, bool(selection['selected']),
|
|
str(selection['selection_reason']), None,
|
|
))
|
|
continue
|
|
globally_covered = (
|
|
str(blob['state']) == 'covered'
|
|
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
|
|
)
|
|
if globally_covered:
|
|
entries.append((descriptor, True, 'globally_covered', None))
|
|
elif descriptor['digest'] in selected_layer_digests:
|
|
entries.append((descriptor, True, 'duplicate_selected_content', None))
|
|
elif descriptor['media_type'] not in DOCKER_LAYER_SUPPORTED_MEDIA_TYPES:
|
|
entries.append((descriptor, False, 'unsupported_media_type', None))
|
|
elif descriptor['size'] > limits['layer_max_bytes']:
|
|
entries.append((descriptor, False, 'layer_too_large', None))
|
|
elif selected_layer_count >= limits['max_layers']:
|
|
entries.append((descriptor, False, 'layer_limit_exhausted', None))
|
|
elif selected_layer_bytes + descriptor['size'] > limits['image_max_bytes']:
|
|
entries.append((descriptor, False, 'image_budget_exhausted', None))
|
|
else:
|
|
entries.append((descriptor, True, 'layer_selected', None))
|
|
selected_layer_digests.add(descriptor['digest'])
|
|
selected_layer_count += 1
|
|
selected_layer_bytes += descriptor['size']
|
|
planned = []
|
|
pending_leases = []
|
|
lease_tokens_by_digest = {}
|
|
leased_bytes = 0
|
|
checkpoint_max_blobs = checkpoint['max_blobs'] if adaptive else 1
|
|
checkpoint_max_bytes = checkpoint['max_bytes'] if adaptive else None
|
|
for descriptor, selected, reason, payload_class in entries:
|
|
blob = blob_rows[descriptor['digest']]
|
|
state = str(blob['state'])
|
|
effective_max_attempts = min(
|
|
int(blob['max_attempts']), limits['blob_max_attempts'],
|
|
)
|
|
active_other = (
|
|
state in ('leased', 'submitted')
|
|
and str(blob['lease_expires_at'] or '') > now
|
|
and int(blob['lease_reservation_id'] or 0) != reservation_id
|
|
)
|
|
same_policy_covered = (
|
|
state == 'covered'
|
|
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
|
|
)
|
|
lease_token = None
|
|
coverage_state = 'skipped'
|
|
if selected and same_policy_covered:
|
|
coverage_state = 'covered'
|
|
if not adaptive:
|
|
reason = 'globally_covered'
|
|
elif selected and active_other:
|
|
coverage_state = 'shared_pending'
|
|
if not adaptive:
|
|
reason = 'shared_active'
|
|
elif selected and (
|
|
state == 'failed'
|
|
or int(blob['attempts'] or 0) >= effective_max_attempts
|
|
):
|
|
coverage_state = 'terminal_failed'
|
|
if not adaptive:
|
|
reason = 'content_attempts_exhausted'
|
|
elif selected and str(blob['available_after'] or '') > now:
|
|
coverage_state = 'selected'
|
|
if not adaptive:
|
|
reason = 'content_retry_wait'
|
|
elif selected:
|
|
lease_token = lease_tokens_by_digest.get(descriptor['digest'])
|
|
can_lease = bool(lease_token) or (
|
|
len(lease_tokens_by_digest) < checkpoint_max_blobs
|
|
and (
|
|
checkpoint_max_bytes is None
|
|
or leased_bytes + descriptor['size'] <= checkpoint_max_bytes
|
|
or not lease_tokens_by_digest
|
|
)
|
|
)
|
|
if can_lease:
|
|
coverage_state = 'leased'
|
|
else:
|
|
coverage_state = 'selected'
|
|
if not adaptive:
|
|
reason = 'content_checkpoint_pending'
|
|
if can_lease and not lease_token:
|
|
lease_token = secrets.token_urlsafe(32)
|
|
lease_tokens_by_digest[descriptor['digest']] = lease_token
|
|
pending_leases.append((descriptor['digest'], lease_token))
|
|
leased_bytes += descriptor['size']
|
|
planned_descriptor = {
|
|
'digest': descriptor['digest'],
|
|
'size': descriptor['size'],
|
|
'media_type': descriptor['media_type'],
|
|
'kind': descriptor['kind'],
|
|
'position': descriptor['position'],
|
|
'selected': bool(selected),
|
|
'selection_reason': reason,
|
|
'coverage_state': coverage_state,
|
|
'lease_token': lease_token,
|
|
'attempt': (
|
|
int(blob['attempts'] or 0) + 1
|
|
if coverage_state == 'leased'
|
|
else min(int(blob['attempts'] or 0), effective_max_attempts)
|
|
),
|
|
'max_attempts': effective_max_attempts,
|
|
}
|
|
if adaptive:
|
|
planned_descriptor['payload_class'] = payload_class
|
|
planned.append(planned_descriptor)
|
|
|
|
planned.sort(key=lambda item: item['position'])
|
|
|
|
plan = {
|
|
'version': 2 if adaptive else 1,
|
|
'image': resolved['image'],
|
|
'repository': resolved['repository'],
|
|
'manifest_digest': resolved['manifest_digest'],
|
|
'platform_os': resolved['platform_os'],
|
|
'platform_arch': resolved['platform_arch'],
|
|
'manifest_media_type': resolved['manifest_media_type'],
|
|
'limits': limits,
|
|
'selection_policy_sha256': selection_policy_sha256,
|
|
'scan_policy_sha256': scan_policy_sha256,
|
|
'descriptors': planned,
|
|
}
|
|
if adaptive:
|
|
plan.update({
|
|
'selector_version': DOCKER_ADAPTIVE_SELECTOR_VERSION,
|
|
'execution_policy_sha256': coverage_policy_sha256,
|
|
'checkpoint': checkpoint,
|
|
})
|
|
plan_bytes = canonical_docker_layer_plan_bytes(plan)
|
|
plan_sha256 = hashlib.sha256(plan_bytes).hexdigest()
|
|
|
|
for digest, lease_token in pending_leases:
|
|
blob = blob_rows[digest]
|
|
effective_max = min(int(blob['max_attempts']), limits['blob_max_attempts'])
|
|
if (
|
|
str(blob['state']) in ('leased', 'submitted')
|
|
and blob['lease_reservation_id'] is not None
|
|
and int(blob['lease_reservation_id']) != reservation_id
|
|
):
|
|
self.conn.execute(
|
|
'''UPDATE docker_image_blob_coverage
|
|
SET coverage_state = 'retryable_failed', covered_at = NULL,
|
|
last_error_code = 'content_lease_reclaimed', updated_at = ?
|
|
WHERE reservation_id = ? AND plan_sha256 = ?
|
|
AND blob_digest = ? AND coverage_policy_sha256 = ?
|
|
AND coverage_state IN ('leased','shared_pending')''',
|
|
(
|
|
now, blob['lease_reservation_id'], blob['lease_plan_sha256'],
|
|
digest, coverage_policy_sha256,
|
|
),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET
|
|
state = 'leased', attempts = attempts + 1, max_attempts = ?,
|
|
available_after = NULL, lease_reservation_id = ?, lease_token = ?,
|
|
lease_plan_sha256 = ?, lease_expires_at = ?,
|
|
covered_reservation_id = NULL, covered_scan_event_id = NULL,
|
|
covered_policy_sha256 = NULL, verified_bytes = NULL, covered_at = NULL,
|
|
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ?
|
|
AND attempts < max_attempts
|
|
AND (
|
|
state = 'pending'
|
|
OR (
|
|
state = 'leased'
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
|
|
)
|
|
)''',
|
|
(
|
|
effective_max, reservation_id, lease_token, plan_sha256,
|
|
lease_until, now, digest, coverage_policy_sha256, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker content lease changed during plan binding')
|
|
|
|
for descriptor in planned:
|
|
conflicting = self.conn.execute(
|
|
'''SELECT 1 AS present FROM docker_image_blob_coverage
|
|
WHERE queue_id = ? AND position = ? AND (
|
|
manifest_digest <> ? OR blob_digest <> ? OR descriptor_kind <> ?
|
|
) LIMIT 1 FOR UPDATE''',
|
|
(
|
|
row['queue_id'], descriptor['position'], resolved['manifest_digest'],
|
|
descriptor['digest'], descriptor['kind'],
|
|
),
|
|
).fetchone()
|
|
if conflicting:
|
|
raise ScanEventConflictError('Docker image coverage position changed for an immutable target')
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_image_blob_coverage(
|
|
queue_id, manifest_digest, position, blob_digest,
|
|
coverage_policy_sha256, selection_policy_sha256,
|
|
descriptor_kind, plan_sha256,
|
|
reservation_id, selected, selection_reason,
|
|
coverage_state, covered_at, last_error_code, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
ON CONFLICT(reservation_id, position) DO UPDATE SET
|
|
selected = excluded.selected,
|
|
selection_reason = excluded.selection_reason,
|
|
coverage_state = excluded.coverage_state,
|
|
covered_at = excluded.covered_at,
|
|
last_error_code = NULL,
|
|
updated_at = excluded.updated_at''',
|
|
(
|
|
row['queue_id'], resolved['manifest_digest'], descriptor['position'],
|
|
descriptor['digest'], coverage_policy_sha256,
|
|
selection_policy_sha256, descriptor['kind'], plan_sha256, reservation_id,
|
|
1 if descriptor['selected'] else 0, descriptor['selection_reason'],
|
|
descriptor['coverage_state'],
|
|
now if descriptor['coverage_state'] == 'covered' else None,
|
|
now, now,
|
|
),
|
|
)
|
|
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations
|
|
SET docker_layer_plan_json = ?, docker_layer_plan_sha256 = ?,
|
|
producer_lease_expires_at = CASE
|
|
WHEN assignment_kind = 'remote' THEN ?
|
|
WHEN producer_lease_expires_at < ? THEN ?
|
|
ELSE producer_lease_expires_at
|
|
END,
|
|
updated_at = ?
|
|
WHERE id = ? AND state = 'scanning'
|
|
AND claim_lease_token = ?
|
|
AND docker_layer_plan_json IS NULL AND docker_layer_plan_sha256 IS NULL''',
|
|
(
|
|
plan_bytes.decode('ascii'), plan_sha256,
|
|
lease_until, lease_until, lease_until,
|
|
now, reservation_id, claim_lease_token,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker layer plan changed during binding')
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET lease_expires_at = CASE
|
|
WHEN ? = 'remote' THEN ?
|
|
WHEN lease_expires_at < ? THEN ? ELSE lease_expires_at END,
|
|
updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
str(row['assignment_kind']), lease_until, lease_until, lease_until,
|
|
now, row['queue_id'], claim_lease_token,
|
|
reservation_id, row['scan_event_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker parent lease changed during plan binding')
|
|
self.conn.commit()
|
|
return plan
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _submit_docker_blob_leases_locked(self, reservation, now):
|
|
plan, plan_sha256 = stored_docker_layer_plan(reservation)
|
|
if plan is None:
|
|
return 0
|
|
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
|
|
submitted = 0
|
|
seen = set()
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] != 'leased' or descriptor['digest'] in seen:
|
|
continue
|
|
seen.add(descriptor['digest'])
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET state = 'submitted',
|
|
updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ?
|
|
AND state IN ('leased','submitted')
|
|
AND lease_reservation_id = ? AND lease_token = ?
|
|
AND lease_plan_sha256 = ?
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
|
|
(
|
|
now, descriptor['digest'], coverage_policy_sha256,
|
|
int(reservation['id']), descriptor['lease_token'], plan_sha256, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker content lease changed before durable bundle submission'
|
|
)
|
|
submitted += 1
|
|
return submitted
|
|
|
|
def _release_docker_blob_leases_locked(
|
|
self, reservation, reason_code, reason_detail, now, *, refund_attempt=False,
|
|
):
|
|
plan, plan_sha256 = stored_docker_layer_plan(reservation)
|
|
if plan is None:
|
|
return 0
|
|
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
|
|
retry_at = (
|
|
datetime.now(timezone.utc) + timedelta(seconds=3600)
|
|
).isoformat(timespec='seconds')
|
|
released = 0
|
|
states = {}
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] != 'leased':
|
|
continue
|
|
digest = descriptor['digest']
|
|
if digest not in states:
|
|
blob = self.conn.execute(
|
|
'''SELECT * FROM docker_content_blobs
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
|
|
(digest, coverage_policy_sha256),
|
|
).fetchone()
|
|
if not blob:
|
|
raise ScanEventConflictError(
|
|
'Docker content policy is absent during reservation release'
|
|
)
|
|
owns_lease = (
|
|
str(blob['state']) in ('leased', 'submitted')
|
|
and int(blob['lease_reservation_id'] or 0) == int(reservation['id'])
|
|
and str(blob['lease_token'] or '') == descriptor['lease_token']
|
|
and str(blob['lease_plan_sha256'] or '') == plan_sha256
|
|
)
|
|
if owns_lease:
|
|
attempts = max(
|
|
0,
|
|
int(blob['attempts'] or 0) - (1 if refund_attempt else 0),
|
|
)
|
|
terminal = attempts >= int(blob['max_attempts'])
|
|
state = 'failed' if terminal else 'pending'
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET state = ?, attempts = ?,
|
|
available_after = ?, lease_reservation_id = NULL,
|
|
lease_token = NULL, lease_plan_sha256 = NULL,
|
|
lease_expires_at = NULL, last_error_code = ?,
|
|
last_error_detail = ?, updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ?
|
|
AND state IN ('leased','submitted')
|
|
AND lease_reservation_id = ? AND lease_token = ?
|
|
AND lease_plan_sha256 = ?''',
|
|
(
|
|
state, attempts, None if terminal else retry_at,
|
|
str(reason_code)[:64], first_line(reason_detail, 1000), now,
|
|
digest, coverage_policy_sha256, int(reservation['id']),
|
|
descriptor['lease_token'], plan_sha256,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker content lease changed during reservation release'
|
|
)
|
|
states[digest] = {
|
|
'coverage_state': 'terminal_failed' if terminal else 'retryable_failed',
|
|
'covered_at': None,
|
|
'error_code': str(reason_code)[:64],
|
|
}
|
|
released += 1
|
|
elif (
|
|
blob['lease_reservation_id'] is not None
|
|
and int(blob['lease_reservation_id']) == int(reservation['id'])
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker content lease identity changed within its reservation'
|
|
)
|
|
elif (
|
|
str(blob['state']) == 'covered'
|
|
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
|
|
):
|
|
states[digest] = {
|
|
'coverage_state': 'covered',
|
|
'covered_at': blob['covered_at'],
|
|
'error_code': None,
|
|
}
|
|
elif (
|
|
str(blob['state']) == 'failed'
|
|
or int(blob['attempts'] or 0) >= int(blob['max_attempts'])
|
|
):
|
|
states[digest] = {
|
|
'coverage_state': 'terminal_failed',
|
|
'covered_at': None,
|
|
'error_code': str(blob['last_error_code'] or reason_code)[:64],
|
|
}
|
|
else:
|
|
states[digest] = {
|
|
'coverage_state': 'retryable_failed',
|
|
'covered_at': None,
|
|
'error_code': str(reason_code)[:64],
|
|
}
|
|
disposition = states[digest]
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
|
|
last_error_code = ?, covered_at = ?, updated_at = ?
|
|
WHERE queue_id = ? AND position = ? AND reservation_id = ?
|
|
AND plan_sha256 = ? AND blob_digest = ?
|
|
AND coverage_policy_sha256 = ?
|
|
AND coverage_state = 'leased' ''',
|
|
(
|
|
disposition['coverage_state'], disposition['error_code'],
|
|
disposition['covered_at'], now,
|
|
int(reservation['queue_id']), descriptor['position'],
|
|
int(reservation['id']), plan_sha256, digest,
|
|
coverage_policy_sha256,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
existing = self.conn.execute(
|
|
'''SELECT coverage_state FROM docker_image_blob_coverage
|
|
WHERE reservation_id = ? AND position = ? AND plan_sha256 = ?
|
|
AND blob_digest = ? AND coverage_policy_sha256 = ?''',
|
|
(
|
|
int(reservation['id']), descriptor['position'], plan_sha256,
|
|
digest, coverage_policy_sha256,
|
|
),
|
|
).fetchone()
|
|
if not existing or str(existing['coverage_state']) != disposition['coverage_state']:
|
|
raise ScanEventConflictError(
|
|
'Docker image coverage changed during reservation release'
|
|
)
|
|
return released
|
|
|
|
def _apply_docker_layer_execution_locked(
|
|
self, reservation, metadata, queue_status, now,
|
|
):
|
|
plan, plan_sha256 = stored_docker_layer_plan(reservation)
|
|
metadata_plan = metadata.get('docker_layer_plan')
|
|
metadata_execution = metadata.get('docker_layer_execution')
|
|
if plan is None:
|
|
if metadata_plan is not None or metadata_execution is not None:
|
|
raise ScanEventConflictError(
|
|
'unbound reservation contains Docker layer execution metadata'
|
|
)
|
|
return None
|
|
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
|
|
try:
|
|
normalized_metadata_plan = validate_docker_layer_plan(metadata_plan)
|
|
metadata_plan_bytes = canonical_docker_layer_plan_bytes(normalized_metadata_plan)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ScanEventConflictError('Docker layer bundle plan is invalid') from exc
|
|
stored_bytes = canonical_docker_layer_plan_bytes(plan)
|
|
if (
|
|
metadata_plan_bytes != stored_bytes
|
|
or hashlib.sha256(metadata_plan_bytes).hexdigest() != plan_sha256
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker layer bundle plan does not match its reservation'
|
|
)
|
|
try:
|
|
execution = validate_docker_layer_execution(
|
|
metadata_execution, plan, plan_sha256,
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ScanEventConflictError('Docker layer bundle execution is invalid') from exc
|
|
|
|
retry_at = (
|
|
datetime.now(timezone.utc) + timedelta(seconds=3600)
|
|
).isoformat(timespec='seconds')
|
|
plan_by_digest = {}
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] == 'leased':
|
|
plan_by_digest.setdefault(descriptor['digest'], descriptor)
|
|
for record in execution['blobs']:
|
|
descriptor = plan_by_digest[record['digest']]
|
|
blob = self.conn.execute(
|
|
'''SELECT * FROM docker_content_blobs
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
|
|
(record['digest'], coverage_policy_sha256),
|
|
).fetchone()
|
|
if not blob or (
|
|
str(blob['state']) != 'submitted'
|
|
or int(blob['lease_reservation_id'] or 0) != int(reservation['id'])
|
|
or str(blob['lease_token'] or '') != record['lease_token']
|
|
or str(blob['lease_plan_sha256'] or '') != plan_sha256
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker layer execution does not own its submitted content lease'
|
|
)
|
|
terminal = (
|
|
record['status'] == 'terminal_failed'
|
|
or (
|
|
record['status'] == 'retryable_failed'
|
|
and int(blob['attempts']) >= int(blob['max_attempts'])
|
|
)
|
|
)
|
|
if record['status'] == 'covered':
|
|
blob_state = 'covered'
|
|
coverage_state = 'covered'
|
|
available_after = None
|
|
covered_reservation_id = int(reservation['id'])
|
|
covered_scan_event_id = str(reservation['scan_event_id'])
|
|
covered_policy_sha256 = coverage_policy_sha256
|
|
covered_at = now
|
|
error_code = None
|
|
else:
|
|
blob_state = 'failed' if terminal else 'pending'
|
|
coverage_state = 'terminal_failed' if terminal else 'retryable_failed'
|
|
available_after = None if terminal else retry_at
|
|
covered_reservation_id = None
|
|
covered_scan_event_id = None
|
|
covered_policy_sha256 = None
|
|
covered_at = None
|
|
error_code = record['error_code']
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET state = ?, available_after = ?,
|
|
lease_reservation_id = NULL, lease_token = NULL,
|
|
lease_plan_sha256 = NULL, lease_expires_at = NULL,
|
|
covered_reservation_id = ?, covered_scan_event_id = ?,
|
|
covered_policy_sha256 = ?, verified_bytes = ?, covered_at = ?,
|
|
last_error_code = ?, last_error_detail = NULL, updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ?
|
|
AND state = 'submitted'
|
|
AND lease_reservation_id = ? AND lease_token = ?
|
|
AND lease_plan_sha256 = ?
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
|
|
(
|
|
blob_state, available_after, covered_reservation_id,
|
|
covered_scan_event_id, covered_policy_sha256,
|
|
record['verified_bytes'], covered_at, error_code, now,
|
|
record['digest'], coverage_policy_sha256,
|
|
int(reservation['id']), record['lease_token'], plan_sha256, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker content lease changed during result ingestion'
|
|
)
|
|
expected_positions = sum(
|
|
1 for item in plan['descriptors']
|
|
if item['coverage_state'] == 'leased' and item['digest'] == record['digest']
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
|
|
covered_at = ?, last_error_code = ?, updated_at = ?
|
|
WHERE queue_id = ? AND reservation_id = ? AND plan_sha256 = ?
|
|
AND blob_digest = ? AND coverage_policy_sha256 = ?
|
|
AND coverage_state = 'leased' ''',
|
|
(
|
|
coverage_state, covered_at, error_code, now,
|
|
int(reservation['queue_id']), int(reservation['id']),
|
|
plan_sha256, record['digest'], coverage_policy_sha256,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != expected_positions:
|
|
raise ScanEventConflictError(
|
|
'Docker image coverage changed during result ingestion'
|
|
)
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] != 'shared_pending':
|
|
continue
|
|
blob = self.conn.execute(
|
|
'''SELECT * FROM docker_content_blobs
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? FOR UPDATE''',
|
|
(descriptor['digest'], coverage_policy_sha256),
|
|
).fetchone()
|
|
reconciled = None
|
|
if (
|
|
blob and str(blob['state']) == 'covered'
|
|
and str(blob['covered_policy_sha256'] or '') == coverage_policy_sha256
|
|
):
|
|
reconciled = 'covered'
|
|
elif blob and (
|
|
str(blob['state']) == 'failed'
|
|
or int(blob['attempts'] or 0) >= int(blob['max_attempts'])
|
|
):
|
|
reconciled = 'terminal_failed'
|
|
if reconciled:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_image_blob_coverage SET coverage_state = ?,
|
|
covered_at = ?, last_error_code = ?, updated_at = ?
|
|
WHERE queue_id = ? AND position = ? AND reservation_id = ?
|
|
AND plan_sha256 = ? AND blob_digest = ?
|
|
AND coverage_policy_sha256 = ?
|
|
AND coverage_state = 'shared_pending' ''',
|
|
(
|
|
reconciled, now if reconciled == 'covered' else None,
|
|
None if reconciled == 'covered' else 'content_attempts_exhausted',
|
|
now, int(reservation['queue_id']), descriptor['position'],
|
|
int(reservation['id']), plan_sha256, descriptor['digest'],
|
|
coverage_policy_sha256,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'shared Docker image coverage changed during result ingestion'
|
|
)
|
|
|
|
coverage_rows = self.conn.execute(
|
|
'''SELECT position, selected, coverage_state, blob_digest
|
|
FROM docker_image_blob_coverage
|
|
WHERE queue_id = ? AND reservation_id = ? AND plan_sha256 = ?
|
|
AND coverage_policy_sha256 = ?
|
|
ORDER BY position''',
|
|
(
|
|
int(reservation['queue_id']), int(reservation['id']), plan_sha256,
|
|
coverage_policy_sha256,
|
|
),
|
|
).fetchall()
|
|
if len(coverage_rows) != len(plan['descriptors']):
|
|
raise ScanEventConflictError('Docker image coverage row count is incomplete')
|
|
states = [str(row['coverage_state']) for row in coverage_rows]
|
|
if any(state == 'leased' for state in states):
|
|
raise ScanEventConflictError('Docker image coverage retained a consumed lease')
|
|
if any(state in ('selected', 'shared_pending', 'retryable_failed') for state in states):
|
|
image_state = 'retryable'
|
|
expected_queue_status = 'deferred'
|
|
elif any(state == 'terminal_failed' for state in states):
|
|
image_state = 'terminal_incomplete'
|
|
expected_queue_status = 'failed'
|
|
elif any(state == 'skipped' for state in states):
|
|
image_state = 'partial'
|
|
expected_queue_status = 'done'
|
|
else:
|
|
image_state = 'complete'
|
|
expected_queue_status = 'done'
|
|
frozen_shared_pending = any(
|
|
descriptor['coverage_state'] == 'shared_pending'
|
|
for descriptor in plan['descriptors']
|
|
)
|
|
reconciled_shared_completion = (
|
|
queue_status == 'deferred'
|
|
and expected_queue_status in ('done', 'failed')
|
|
and frozen_shared_pending
|
|
)
|
|
if queue_status != expected_queue_status and not reconciled_shared_completion:
|
|
raise DockerCoverageDispositionConflictError(
|
|
'Docker image queue disposition conflicts with content coverage'
|
|
)
|
|
available_after = metadata.get('available_after')
|
|
reset_attempts = metadata.get('reset_attempts')
|
|
if queue_status == 'deferred':
|
|
deferred_at = parse_time(available_after)
|
|
authoritative_now = parse_time(now)
|
|
expected_reset = (
|
|
any(
|
|
descriptor['coverage_state'] in ('selected', 'shared_pending')
|
|
for descriptor in plan['descriptors']
|
|
)
|
|
and not any(
|
|
record['status'] in ('retryable_failed', 'terminal_failed')
|
|
for record in execution['blobs']
|
|
)
|
|
)
|
|
if (
|
|
reset_attempts is not expected_reset
|
|
or deferred_at is None
|
|
or authoritative_now is None
|
|
or deferred_at <= authoritative_now
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker retry disposition lacks an authoritative future deadline and reset'
|
|
)
|
|
elif available_after is not None or reset_attempts is not False:
|
|
raise ScanEventConflictError(
|
|
'terminal Docker image disposition contains retry scheduling metadata'
|
|
)
|
|
counts = {
|
|
state: sum(1 for item in states if item == state)
|
|
for state in (
|
|
'covered', 'shared_pending', 'retryable_failed',
|
|
'terminal_failed', 'skipped',
|
|
)
|
|
}
|
|
return {'state': image_state, **counts}
|
|
|
|
def bind_git_scan_plan(
|
|
self, reservation_id, claim_lease_token, resolved, baseline_depth,
|
|
remote_credential=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('exact Git scan plan binding requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
claim_lease_token = str(claim_lease_token or '')
|
|
if not claim_lease_token:
|
|
raise ValueError('Git scan plan binding requires a claim lease token')
|
|
resolved = validate_git_resolution(resolved)
|
|
try:
|
|
baseline_depth = int(baseline_depth)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('Git baseline depth must be an integer') from exc
|
|
if not 1 <= baseline_depth <= 1000000:
|
|
raise ValueError('Git baseline depth must be between 1 and 1000000')
|
|
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id,
|
|
q.covered_ref AS queue_covered_ref,
|
|
q.covered_head AS queue_covered_head
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not row:
|
|
raise ScanEventConflictError('Git scan reservation is absent')
|
|
if str(row['assignment_kind']) == 'remote':
|
|
credential = dict(remote_credential or {})
|
|
if not self._lock_active_remote_credential(
|
|
credential.get('device_id'), credential.get('token_sha256'),
|
|
user_id=row['remote_user_id'],
|
|
) or int(credential.get('device_id') or 0) != int(
|
|
row['remote_device_id'] or 0
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote worker credential changed before Git plan binding'
|
|
)
|
|
now = utc_now_iso()
|
|
if (
|
|
str(row['state']) != 'scanning'
|
|
or str(row['claim_lease_token']) != claim_lease_token
|
|
or str(row['queue_status']) != 'in_progress'
|
|
or str(row['queue_lease_token']) != claim_lease_token
|
|
or int(row['queue_reservation_id'] or 0) != reservation_id
|
|
or str(row['queue_event_id']) != str(row['scan_event_id'])
|
|
or str(row['queue_lease_expires_at'] or '') <= now
|
|
or str(row['producer_lease_expires_at'] or '') <= now
|
|
or (
|
|
str(row['assignment_kind']) == 'remote'
|
|
and (
|
|
row['remote_resolution_kind'] is not None
|
|
or str(row['remote_expires_at'] or '') <= now
|
|
)
|
|
)
|
|
):
|
|
raise ScanEventConflictError('Git scan reservation is no longer actively fenced')
|
|
if str(row['platform']).lower() != resolved['provider']:
|
|
raise ScanEventConflictError('Git scan resolution provider conflicts with the reservation')
|
|
|
|
covered_ref = str(row['queue_covered_ref'] or '')
|
|
covered_head = str(row['queue_covered_head'] or '').lower()
|
|
if bool(covered_ref) != bool(covered_head):
|
|
raise RuntimeSafetySchemaError('Git coverage ref and head must be stored together')
|
|
if covered_head and not re.fullmatch(r'[a-f0-9]{40}|[a-f0-9]{64}', covered_head):
|
|
raise RuntimeSafetySchemaError('stored Git covered head is invalid')
|
|
if covered_ref == resolved['ref'] and covered_head == resolved['head_sha']:
|
|
mode = 'noop'
|
|
base_sha = covered_head
|
|
elif covered_ref == resolved['ref'] and covered_head:
|
|
mode = 'delta'
|
|
base_sha = covered_head
|
|
else:
|
|
mode = 'baseline'
|
|
base_sha = None
|
|
plan = {
|
|
'version': 1,
|
|
**resolved,
|
|
'base_sha': base_sha,
|
|
'mode': mode,
|
|
'baseline_depth': baseline_depth,
|
|
}
|
|
plan_bytes = canonical_git_scan_plan_bytes(plan)
|
|
plan_json = plan_bytes.decode('ascii')
|
|
plan_sha256 = hashlib.sha256(plan_bytes).hexdigest()
|
|
stored_json = row['git_scan_plan_json']
|
|
stored_sha256 = row['git_scan_plan_sha256']
|
|
if stored_json is not None or stored_sha256 is not None:
|
|
if str(stored_json or '') != plan_json or str(stored_sha256 or '') != plan_sha256:
|
|
raise ScanEventConflictError('Git scan reservation already has a conflicting plan')
|
|
self.conn.commit()
|
|
return plan
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations
|
|
SET git_scan_plan_json = ?, git_scan_plan_sha256 = ?, updated_at = ?
|
|
WHERE id = ? AND state = 'scanning' AND claim_lease_token = ?
|
|
AND git_scan_plan_json IS NULL AND git_scan_plan_sha256 IS NULL''',
|
|
(plan_json, plan_sha256, now, reservation_id, claim_lease_token),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Git scan plan binding lost its reservation fence')
|
|
self.conn.commit()
|
|
return plan
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def remote_bound_git_scan_plan(
|
|
self, reservation_id, device_id, claim_lease_token, token_sha256,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote Git scan plan recovery requires PostgreSQL')
|
|
reservation_id = int(reservation_id)
|
|
device_id = int(device_id)
|
|
claim_lease_token = str(claim_lease_token or '')
|
|
if reservation_id <= 0 or device_id <= 0 or not claim_lease_token:
|
|
raise ValueError('remote Git scan plan recovery identity is invalid')
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not row or (
|
|
str(row['assignment_kind']) != 'remote'
|
|
or int(row['remote_device_id'] or 0) != device_id
|
|
or str(row['state']) != 'scanning'
|
|
or row['remote_resolution_kind'] is not None
|
|
or str(row['claim_lease_token']) != claim_lease_token
|
|
or str(row['queue_status']) != 'in_progress'
|
|
or str(row['queue_lease_token']) != claim_lease_token
|
|
or int(row['queue_reservation_id'] or 0) != reservation_id
|
|
or str(row['queue_event_id']) != str(row['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError('remote Git scan reservation is no longer actively fenced')
|
|
if not self._lock_active_remote_credential(
|
|
device_id, token_sha256, user_id=row['remote_user_id'],
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote worker credential changed before Git plan recovery'
|
|
)
|
|
now = utc_now_iso()
|
|
if (
|
|
str(row['remote_expires_at'] or '') <= now
|
|
or str(row['queue_lease_expires_at'] or '') <= now
|
|
or str(row['producer_lease_expires_at'] or '') <= now
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote Git scan reservation is no longer actively fenced'
|
|
)
|
|
plan = stored_git_scan_plan(row)
|
|
self.conn.commit()
|
|
return plan
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def result_reservation_by_token(self, reservation_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result reservation recovery requires PostgreSQL')
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE reservation_token = ?',
|
|
(str(reservation_token),),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def recover_result_reservation_claim(self, reservation_token, expected):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result reservation recovery requires PostgreSQL')
|
|
token = str(reservation_token)
|
|
expected = dict(expected or {})
|
|
digest = self._admission_intent_sha256(expected)
|
|
now = utc_now_iso()
|
|
try:
|
|
intent = self.conn.execute(
|
|
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
|
|
(token,),
|
|
).fetchone()
|
|
if not intent:
|
|
raise RuntimeError('admission intent is absent during exact recovery')
|
|
if intent['intent_sha256'] != digest:
|
|
raise ScanEventConflictError(
|
|
'admission intent token resolves to conflicting recovery identity'
|
|
)
|
|
if intent['state'] == 'pending':
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'serialized_recovery_observed_no_commit',
|
|
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
|
|
(now, now, token),
|
|
)
|
|
self.conn.commit()
|
|
return None
|
|
if intent['state'] == 'aborted':
|
|
self.conn.commit()
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT r.*, q.attempts,
|
|
q.id AS bound_queue_id,
|
|
q.source AS bound_queue_source,
|
|
q.platform AS bound_queue_platform,
|
|
q.query AS bound_queue_query,
|
|
q.target AS bound_queue_target,
|
|
q.normalized_target AS bound_queue_normalized_target,
|
|
binding.id AS experiment_binding_id,
|
|
binding.experiment_target_id,
|
|
binding.attempt AS experiment_attempt,
|
|
target.experiment_id,
|
|
target.dispatch_wave, target.dispatch_order,
|
|
experiment.experiment_key AS bound_experiment_key
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.reservation_id = r.id
|
|
LEFT JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
LEFT JOIN docker_depth_experiments experiment
|
|
ON experiment.id = target.experiment_id
|
|
WHERE r.reservation_token = ? FOR UPDATE OF r, q''',
|
|
(token,),
|
|
).fetchone()
|
|
if not row or int(intent['reservation_id'] or 0) != int(row['id']):
|
|
raise RuntimeError('committed admission intent has no exact reservation')
|
|
for key, value in expected.items():
|
|
if str(row[key]) != str(value):
|
|
raise ScanEventConflictError(
|
|
'reservation token resolves to conflicting recovered claim identity'
|
|
)
|
|
queue_identity = {
|
|
'id': row['bound_queue_id'],
|
|
'source': row['bound_queue_source'],
|
|
'platform': row['bound_queue_platform'],
|
|
'query': row['bound_queue_query'],
|
|
'target': row['bound_queue_target'],
|
|
'normalized_target': row['bound_queue_normalized_target'],
|
|
}
|
|
self._validate_locked_reservation_queue_identity(row, queue_identity)
|
|
if str(row['assignment_kind']) == 'remote':
|
|
remote_assignment_execution_plan(row)
|
|
self.conn.commit()
|
|
return self._result_reservation_claim(row)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def reconcile_remote_assignment_request(
|
|
self, reservation_token, device_id, token_sha256,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote assignment request reconciliation requires PostgreSQL')
|
|
token = str(reservation_token or '')
|
|
device_id, token_sha256 = self._remote_credential_mapping(
|
|
device_id, token_sha256,
|
|
)
|
|
if not re.fullmatch(r'[a-f0-9]{32,64}', token):
|
|
raise ValueError('remote assignment request identity is invalid')
|
|
now = utc_now_iso()
|
|
try:
|
|
intent = self.conn.execute(
|
|
'SELECT * FROM admission_intents WHERE reservation_token = ? FOR UPDATE',
|
|
(token,),
|
|
).fetchone()
|
|
if not intent:
|
|
self.conn.commit()
|
|
return None
|
|
if int(intent['remote_device_id'] or 0) != device_id or not intent['remote_user_id']:
|
|
raise ScanEventConflictError(
|
|
'remote assignment request belongs to another device'
|
|
)
|
|
if not self._lock_active_remote_credential(
|
|
device_id, token_sha256, user_id=int(intent['remote_user_id']),
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote worker credential changed before request reconciliation'
|
|
)
|
|
if intent['state'] == 'pending':
|
|
self.conn.execute(
|
|
'''UPDATE admission_intents SET state = 'aborted',
|
|
resolution_detail = 'serialized_remote_recovery_observed_no_commit',
|
|
resolved_at = ?, updated_at = ? WHERE reservation_token = ?''',
|
|
(now, now, token),
|
|
)
|
|
self.conn.commit()
|
|
return {'state': 'aborted'}
|
|
if intent['state'] == 'aborted':
|
|
self.conn.commit()
|
|
return {'state': 'aborted'}
|
|
if intent['state'] != 'committed':
|
|
raise ScanEventConflictError('remote assignment request state is invalid')
|
|
row = self.conn.execute(
|
|
'''SELECT r.*, q.attempts,
|
|
q.id AS bound_queue_id,
|
|
q.source AS bound_queue_source,
|
|
q.platform AS bound_queue_platform,
|
|
q.query AS bound_queue_query,
|
|
q.target AS bound_queue_target,
|
|
q.normalized_target AS bound_queue_normalized_target
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.reservation_token = ? FOR UPDATE OF r, q''',
|
|
(token,),
|
|
).fetchone()
|
|
if (
|
|
not row or int(intent['reservation_id'] or 0) != int(row['id'])
|
|
or str(row['assignment_kind']) != 'remote'
|
|
or int(row['remote_user_id'] or 0) != int(intent['remote_user_id'])
|
|
or int(row['remote_device_id'] or 0) != device_id
|
|
):
|
|
raise ScanEventConflictError(
|
|
'committed remote assignment request lost its reservation identity'
|
|
)
|
|
queue_identity = {
|
|
'id': row['bound_queue_id'],
|
|
'source': row['bound_queue_source'],
|
|
'platform': row['bound_queue_platform'],
|
|
'query': row['bound_queue_query'],
|
|
'target': row['bound_queue_target'],
|
|
'normalized_target': row['bound_queue_normalized_target'],
|
|
}
|
|
self._validate_locked_reservation_queue_identity(row, queue_identity)
|
|
snapshot = stored_remote_execution_snapshot(row)
|
|
execution_plan = remote_assignment_execution_plan(row, snapshot=snapshot)
|
|
receipt = (
|
|
self._remote_resolution_from_row(row)
|
|
if row['remote_resolution_json'] else None
|
|
)
|
|
result = {
|
|
'state': 'committed',
|
|
'claim': self._result_reservation_claim(row),
|
|
'execution_snapshot': snapshot,
|
|
'execution_snapshot_sha256': str(
|
|
row['remote_execution_snapshot_sha256']
|
|
),
|
|
'execution_plan': execution_plan,
|
|
'git_plan': (
|
|
execution_plan['bound_plan']
|
|
if execution_plan['kind'] == 'exact_git_v1' else None
|
|
),
|
|
'receipt': receipt,
|
|
}
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def renew_result_claim(self, reservation_id, lease_token, lease_seconds=3600):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return False
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
reservation = self.conn.execute(
|
|
'''SELECT r.*, q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not reservation or (
|
|
str(reservation['state']) != 'scanning'
|
|
or str(reservation['assignment_kind']) == 'remote'
|
|
or str(reservation['claim_lease_token']) != str(lease_token)
|
|
or str(reservation['producer_lease_expires_at'] or '') <= now
|
|
or str(reservation['queue_status']) != 'in_progress'
|
|
or str(reservation['queue_lease_token']) != str(lease_token)
|
|
or int(reservation['queue_reservation_id'] or 0) != int(reservation_id)
|
|
or str(reservation['queue_event_id'] or '') != str(reservation['scan_event_id'])
|
|
or str(reservation['queue_lease_expires_at'] or '') <= now
|
|
):
|
|
self.conn.rollback()
|
|
return False
|
|
plan, plan_sha256 = stored_docker_layer_plan(reservation)
|
|
effective_seconds = max(60, int(lease_seconds))
|
|
if plan is not None:
|
|
effective_seconds = max(
|
|
effective_seconds,
|
|
plan['limits']['blob_timeout_sec'] + DOCKER_BLOB_LEASE_MARGIN_SEC,
|
|
)
|
|
lease_until = datetime.fromtimestamp(
|
|
time.time() + effective_seconds, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
if plan is not None:
|
|
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
|
|
seen = set()
|
|
for descriptor in plan['descriptors']:
|
|
if descriptor['coverage_state'] != 'leased' or descriptor['digest'] in seen:
|
|
continue
|
|
seen.add(descriptor['digest'])
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_content_blobs SET lease_expires_at = ?, updated_at = ?
|
|
WHERE digest = ? AND coverage_policy_sha256 = ? AND state = 'leased'
|
|
AND lease_reservation_id = ? AND lease_token = ?
|
|
AND lease_plan_sha256 = ?
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at > ?''',
|
|
(
|
|
lease_until, now, descriptor['digest'],
|
|
coverage_policy_sha256, int(reservation_id),
|
|
descriptor['lease_token'], plan_sha256, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET producer_lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ? AND claim_lease_token = ? AND state = 'scanning'
|
|
AND producer_lease_expires_at > ?''',
|
|
(lease_until, now, int(reservation_id), str(lease_token), now),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?
|
|
AND lease_expires_at > ?''',
|
|
(
|
|
lease_until, now, reservation['queue_id'], str(lease_token),
|
|
int(reservation_id), reservation['scan_event_id'], now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _validate_result_bundle_reservation_identity(
|
|
reservation, values, *, require_header=False,
|
|
):
|
|
values = dict(values or {})
|
|
ready_relative = str(reservation['ready_relative_path']).replace('\\', '/')
|
|
|
|
def integer_matches(value, expected):
|
|
if isinstance(value, bool) or value is None:
|
|
return False
|
|
try:
|
|
return int(value) == int(expected)
|
|
except (TypeError, ValueError, OverflowError):
|
|
return False
|
|
|
|
if (
|
|
('reservation_id' in values and not integer_matches(
|
|
values.get('reservation_id'), reservation['id'],
|
|
))
|
|
or str(values.get('bundle_id') or '') != str(reservation['bundle_id'])
|
|
or str(values.get('scan_event_id') or '')
|
|
!= str(reservation['scan_event_id'])
|
|
or str(values.get('relative_path') or '').replace('\\', '/')
|
|
!= ready_relative
|
|
):
|
|
raise ValueError('result bundle identity does not match its reservation')
|
|
|
|
header = values.get('header')
|
|
result_metadata = values.get('result_metadata')
|
|
if not require_header and header is None and result_metadata is None:
|
|
return
|
|
if not isinstance(header, dict) or not isinstance(result_metadata, dict):
|
|
raise ValueError('result bundle identity metadata is incomplete')
|
|
|
|
string_header = {
|
|
'reservation_token': reservation['reservation_token'],
|
|
'bundle_id': reservation['bundle_id'],
|
|
'scan_event_id': reservation['scan_event_id'],
|
|
'claim_lease_token': reservation['claim_lease_token'],
|
|
'source': reservation['source'],
|
|
'platform': reservation['platform'],
|
|
'query': reservation['query'],
|
|
'target': reservation['target'],
|
|
'normalized_target': reservation['normalized_target'],
|
|
'producer_instance_id': reservation['producer_instance_id'],
|
|
'producer_creation_time': reservation['producer_creation_time'],
|
|
}
|
|
if any(
|
|
str(header.get(name) or '') != str(expected or '')
|
|
for name, expected in string_header.items()
|
|
) or (
|
|
str(header.get('ready_relative_path') or '').replace('\\', '/')
|
|
!= ready_relative
|
|
) or (
|
|
str(header.get('producer_executable') or '').replace('\\', '/')
|
|
!= str(reservation['producer_executable'] or '').replace('\\', '/')
|
|
):
|
|
raise ValueError('result bundle header does not match its reservation')
|
|
|
|
integer_header = {
|
|
'format_version': 2,
|
|
'reservation_id': reservation['id'],
|
|
'queue_id': reservation['queue_id'],
|
|
'declared_bytes': reservation['declared_bundle_bytes'],
|
|
'producer_pid': reservation['producer_pid'],
|
|
}
|
|
if any(
|
|
not integer_matches(header.get(name), expected)
|
|
for name, expected in integer_header.items()
|
|
):
|
|
raise ValueError('result bundle header does not match its reservation')
|
|
for name in ('run_id', 'cycle_id'):
|
|
actual = header.get(name)
|
|
expected = reservation[name]
|
|
if actual is None or expected is None:
|
|
matches = actual is None and expected is None
|
|
else:
|
|
matches = integer_matches(actual, expected)
|
|
if not matches:
|
|
raise ValueError('result bundle execution identity does not match its reservation')
|
|
|
|
required_metadata = {
|
|
'scan_event_id': reservation['scan_event_id'],
|
|
'target': reservation['target'],
|
|
'scan_type': reservation['platform'],
|
|
}
|
|
if any(
|
|
str(result_metadata.get(name) or '') != str(expected or '')
|
|
for name, expected in required_metadata.items()
|
|
):
|
|
raise ValueError('result bundle scan identity does not match its reservation')
|
|
for name, expected in (
|
|
('bundle_id', reservation['bundle_id']),
|
|
('source', reservation['source']),
|
|
('platform', reservation['platform']),
|
|
('query', reservation['query']),
|
|
('normalized_target', reservation['normalized_target']),
|
|
):
|
|
if (
|
|
name in result_metadata
|
|
and result_metadata[name] is not None
|
|
and str(result_metadata[name]) != str(expected)
|
|
):
|
|
raise ValueError('result bundle scan identity does not match its reservation')
|
|
for name, expected in (
|
|
('reservation_id', reservation['id']),
|
|
('result_reservation_id', reservation['id']),
|
|
('queue_id', reservation['queue_id']),
|
|
):
|
|
if name in result_metadata and not integer_matches(
|
|
result_metadata.get(name), expected):
|
|
raise ValueError('result bundle scan identity does not match its reservation')
|
|
if normalize_target(
|
|
result_metadata.get('target'), reservation['platform'],
|
|
) != str(reservation['normalized_target']):
|
|
raise ValueError('result bundle target identity does not match its reservation')
|
|
|
|
scan_event_hash = str(values.get('scan_event_hash') or '').lower()
|
|
count_values = {
|
|
name: values.get(name) for name in (
|
|
'actual_bytes', 'frame_count', 'finding_count', 'error_count',
|
|
'candidate_count',
|
|
)
|
|
}
|
|
diagnostic_count = values.get('diagnostic_count', 0)
|
|
if (
|
|
not re.fullmatch(r'[a-f0-9]{64}', scan_event_hash)
|
|
or not integer_matches(values.get('reservation_id'), reservation['id'])
|
|
):
|
|
raise ValueError('result bundle metadata is outside its reservation bounds')
|
|
try:
|
|
if any(
|
|
value is None or isinstance(value, bool)
|
|
for value in count_values.values()
|
|
):
|
|
raise ValueError
|
|
counts = {
|
|
name: int(value) for name, value in count_values.items()
|
|
}
|
|
if isinstance(diagnostic_count, bool):
|
|
raise ValueError
|
|
diagnostic_count = int(diagnostic_count or 0)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError(
|
|
'result bundle metadata is outside its reservation bounds'
|
|
) from None
|
|
if (
|
|
counts['actual_bytes'] <= 0
|
|
or counts['actual_bytes'] > int(reservation['declared_bundle_bytes'])
|
|
or counts['frame_count'] < 3
|
|
or counts['frame_count'] != (
|
|
counts['finding_count'] + counts['error_count']
|
|
+ counts['candidate_count'] + diagnostic_count + 3
|
|
)
|
|
or diagnostic_count < 0
|
|
or any(counts[name] < 0 for name in (
|
|
'finding_count', 'error_count', 'candidate_count',
|
|
))
|
|
):
|
|
raise ValueError('result bundle metadata is outside its reservation bounds')
|
|
|
|
@staticmethod
|
|
def _validate_locked_reservation_queue_identity(reservation, queue):
|
|
if not queue or (
|
|
int(queue['id']) != int(reservation['queue_id'])
|
|
or str(queue['source']) != str(reservation['source'])
|
|
or str(queue['platform']) != str(reservation['platform'])
|
|
or str(queue['query'] or '') != str(reservation['query'] or '')
|
|
or str(queue['target']) != str(reservation['target'])
|
|
or str(queue['normalized_target'])
|
|
!= str(reservation['normalized_target'])
|
|
or normalize_target(queue['target'], queue['platform'])
|
|
!= str(queue['normalized_target'])
|
|
):
|
|
raise ScanEventConflictError(
|
|
'result reservation lost its exact target queue identity'
|
|
)
|
|
|
|
@staticmethod
|
|
def _validate_result_ingestion_arguments(
|
|
current_reservation, current_bundle, supplied_reservation, supplied_bundle,
|
|
):
|
|
supplied_reservation = dict(supplied_reservation or {})
|
|
supplied_bundle = dict(supplied_bundle or {})
|
|
reservation_fields = (
|
|
'bundle_id', 'scan_event_id', 'queue_id', 'source', 'platform',
|
|
'query', 'target', 'normalized_target', 'claim_lease_token',
|
|
'run_id', 'cycle_id',
|
|
)
|
|
supplied_reservation_id = supplied_reservation.get(
|
|
'id', supplied_reservation.get('reservation_id'),
|
|
)
|
|
try:
|
|
reservation_id_matches = (
|
|
not isinstance(supplied_reservation_id, bool)
|
|
and int(supplied_reservation_id) == int(current_reservation['id'])
|
|
)
|
|
except (TypeError, ValueError, OverflowError):
|
|
reservation_id_matches = False
|
|
if not reservation_id_matches or any(
|
|
str(supplied_reservation.get(name) or '')
|
|
!= str(current_reservation[name] or '')
|
|
for name in reservation_fields
|
|
):
|
|
raise ScanEventConflictError(
|
|
'result ingestion reservation argument lost its exact identity'
|
|
)
|
|
bundle_fields = (
|
|
'reservation_id', 'bundle_id', 'scan_event_id', 'scan_event_hash',
|
|
'relative_path', 'actual_bytes', 'frame_count', 'finding_count',
|
|
'error_count', 'candidate_count',
|
|
)
|
|
if any(
|
|
str(supplied_bundle.get(name) or '').replace('\\', '/')
|
|
!= str(current_bundle[name] or '').replace('\\', '/')
|
|
for name in bundle_fields
|
|
):
|
|
raise ScanEventConflictError(
|
|
'result ingestion bundle argument lost its exact identity'
|
|
)
|
|
|
|
def mark_result_bundle_ready(
|
|
self, reservation_id, metadata, remote_acceptance=None,
|
|
bundle_capacity_bytes=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result bundle readiness requires PostgreSQL')
|
|
values = metadata.as_dict() if hasattr(metadata, 'as_dict') else dict(metadata or {})
|
|
remote = dict(remote_acceptance or {}) if remote_acceptance is not None else None
|
|
diagnostic_projection = None
|
|
if remote is not None:
|
|
remote = {
|
|
'device_id': int(remote.get('device_id') or 0),
|
|
'payload_sha256': str(remote.get('payload_sha256') or '').lower(),
|
|
'token_sha256': str(remote.get('token_sha256') or '').lower(),
|
|
}
|
|
if remote['device_id'] <= 0 or not re.fullmatch(
|
|
r'[a-f0-9]{64}', remote['payload_sha256'],
|
|
) or not re.fullmatch(r'[a-f0-9]{64}', remote['token_sha256']):
|
|
raise ValueError('remote bundle acceptance identity is invalid')
|
|
diagnostic_projection = {
|
|
'version': values.get('effective_diagnostic_projection_version'),
|
|
'count': values.get('effective_diagnostic_count'),
|
|
'uids_sha256': str(
|
|
values.get('effective_diagnostic_uids_sha256') or ''
|
|
),
|
|
}
|
|
if (
|
|
diagnostic_projection['version'] != 1
|
|
or isinstance(diagnostic_projection['count'], bool)
|
|
or not isinstance(diagnostic_projection['count'], int)
|
|
or diagnostic_projection['count'] < 0
|
|
or not re.fullmatch(
|
|
r'[a-f0-9]{64}', diagnostic_projection['uids_sha256'],
|
|
)
|
|
):
|
|
raise ValueError(
|
|
'remote diagnostic projection authority is invalid'
|
|
)
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
reservation = self.conn.execute(
|
|
'''SELECT r.*, q.id AS bound_queue_id,
|
|
q.source AS bound_queue_source,
|
|
q.platform AS bound_queue_platform,
|
|
q.query AS bound_queue_query,
|
|
q.target AS bound_queue_target,
|
|
q.normalized_target AS bound_queue_normalized_target,
|
|
q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not reservation:
|
|
self.conn.rollback()
|
|
return False
|
|
if remote is not None:
|
|
if (
|
|
str(reservation['assignment_kind']) != 'remote'
|
|
or int(reservation['remote_device_id'] or 0) != remote['device_id']
|
|
):
|
|
raise ScanEventConflictError('remote bundle owner identity conflicts')
|
|
if not self._lock_active_remote_credential(
|
|
remote['device_id'], remote['token_sha256'],
|
|
user_id=reservation['remote_user_id'],
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote worker credential changed before bundle acceptance'
|
|
)
|
|
if reservation['remote_resolution_kind'] is not None:
|
|
if (
|
|
str(reservation['remote_resolution_kind']) != 'bundle_accepted'
|
|
or str(reservation['remote_payload_sha256'] or '')
|
|
!= remote['payload_sha256']
|
|
):
|
|
raise ScanEventConflictError('remote assignment already has a conflicting resolution')
|
|
receipt = self._remote_resolution_from_row(reservation)
|
|
receipt_diagnostics = receipt.get('diagnostics') or {}
|
|
if receipt_diagnostics.get('projection_version') is not None and (
|
|
int(reservation['remote_diagnostic_projection_version'] or -1)
|
|
!= int(receipt_diagnostics['projection_version'])
|
|
or int(reservation['remote_diagnostic_count'] or 0)
|
|
!= int(receipt_diagnostics.get('count') or 0)
|
|
or str(reservation['remote_diagnostic_uids_sha256'] or '')
|
|
!= str(receipt_diagnostics.get('ordered_uid_set_sha256') or '')
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote diagnostic projection receipt fence is invalid'
|
|
)
|
|
self.conn.commit()
|
|
return receipt
|
|
elif str(reservation['assignment_kind']) == 'remote':
|
|
raise ScanEventConflictError('remote bundle requires authenticated acceptance')
|
|
now = utc_now_iso()
|
|
queue_identity = {
|
|
'id': reservation['bound_queue_id'],
|
|
'source': reservation['bound_queue_source'],
|
|
'platform': reservation['bound_queue_platform'],
|
|
'query': reservation['bound_queue_query'],
|
|
'target': reservation['bound_queue_target'],
|
|
'normalized_target': reservation['bound_queue_normalized_target'],
|
|
}
|
|
self._validate_locked_reservation_queue_identity(
|
|
reservation, queue_identity,
|
|
)
|
|
self._validate_result_bundle_reservation_identity(
|
|
reservation, values, require_header='header' in values,
|
|
)
|
|
if remote is not None:
|
|
result_metadata = values.get('result_metadata')
|
|
validate_remote_result_execution_plan(
|
|
reservation, result_metadata,
|
|
)
|
|
actual_bytes = int(values['actual_bytes'])
|
|
reserved_bytes = int(reservation['reserved_bundle_bytes'])
|
|
if actual_bytes > reserved_bytes:
|
|
if bundle_capacity_bytes is None:
|
|
raise ValueError(
|
|
'remote bundle expansion requires a bundle capacity limit'
|
|
)
|
|
bundle_capacity_bytes = max(0, int(bundle_capacity_bytes))
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
additional_bytes = actual_bytes - reserved_bytes
|
|
if (
|
|
not capacity
|
|
or int(capacity['bundle_bytes']) + additional_bytes
|
|
> bundle_capacity_bytes
|
|
):
|
|
self.conn.rollback()
|
|
raise PipelineCapacityUnavailable(
|
|
'remote bundle expansion exceeds available capacity'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET reserved_bundle_bytes = ?,
|
|
updated_at = ? WHERE id = ? AND state = 'scanning'
|
|
AND reserved_bundle_bytes = ?''',
|
|
(actual_bytes, now, reservation_id, reserved_bytes),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'remote bundle capacity expansion lost its reservation fence'
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET bundle_bytes = bundle_bytes + ?,
|
|
updated_at = ? WHERE id = 1''',
|
|
(additional_bytes, now),
|
|
)
|
|
active_fence = (
|
|
str(reservation['state']) == 'scanning'
|
|
and str(reservation['queue_status']) == 'in_progress'
|
|
and str(reservation['queue_lease_token'] or '') == str(reservation['claim_lease_token'])
|
|
and int(reservation['queue_reservation_id'] or 0) == int(reservation_id)
|
|
and str(reservation['queue_event_id'] or '') == str(reservation['scan_event_id'])
|
|
and str(reservation['queue_lease_expires_at'] or '') > now
|
|
and str(reservation['producer_lease_expires_at'] or '') > now
|
|
and (
|
|
remote is None or str(reservation['remote_expires_at'] or '') > now
|
|
)
|
|
)
|
|
existing = self.conn.execute(
|
|
'SELECT scan_event_hash FROM result_bundles WHERE reservation_id = ?',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
receipt = None
|
|
if remote is not None:
|
|
receipt = {
|
|
'receipt_id': secrets.token_hex(32),
|
|
'resolution': 'bundle_accepted',
|
|
'payload_sha256': remote['payload_sha256'],
|
|
'reservation_id': int(reservation_id),
|
|
'bundle_id': str(reservation['bundle_id']),
|
|
'scan_event_id': str(reservation['scan_event_id']),
|
|
'accepted_at': now,
|
|
}
|
|
receipt.update(self._remote_assignment_observability_locked(
|
|
reservation,
|
|
diagnostic_count=int(
|
|
values.get(
|
|
'effective_diagnostic_count',
|
|
values.get('diagnostic_count') or 0,
|
|
)
|
|
),
|
|
))
|
|
receipt['diagnostics'].update({
|
|
'projection_version': diagnostic_projection['version'],
|
|
'ordered_uid_set_sha256': diagnostic_projection['uids_sha256'],
|
|
})
|
|
receipt_json = json.dumps(
|
|
receipt, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
)
|
|
artifact_sha256 = remote['payload_sha256']
|
|
else:
|
|
receipt_json = None
|
|
artifact_sha256 = values.get('scan_event_hash')
|
|
if existing:
|
|
if str(existing['scan_event_hash']) != str(values.get('scan_event_hash')):
|
|
raise ScanEventConflictError('ready bundle replay hash conflicts with its reservation')
|
|
if reservation['state'] == 'scanning':
|
|
if not active_fence:
|
|
self.conn.rollback()
|
|
return False
|
|
self._submit_docker_blob_leases_locked(reservation, now)
|
|
elif reservation['state'] not in ('ready', 'ingesting', 'db_committed', 'acknowledged'):
|
|
self.conn.rollback()
|
|
return False
|
|
if reservation['state'] in ('scanning', 'ready', 'ingesting', 'db_committed'):
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
|
|
AND owner_id = ?''',
|
|
(now, now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
|
|
byte_count = ?, deleted_at = NULL, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
|
|
AND owner_id = ?''',
|
|
(
|
|
artifact_sha256, int(values.get('actual_bytes') or 0),
|
|
now, reservation_id,
|
|
),
|
|
)
|
|
if remote is not None:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET
|
|
remote_resolution_kind = 'bundle_accepted',
|
|
remote_payload_sha256 = ?, remote_receipt_id = ?,
|
|
remote_resolution_json = ?, remote_resolved_at = ?,
|
|
remote_diagnostic_projection_version = ?,
|
|
remote_diagnostic_count = ?,
|
|
remote_diagnostic_uids_sha256 = ?, updated_at = ?
|
|
WHERE id = ? AND assignment_kind = 'remote'
|
|
AND remote_device_id = ? AND remote_resolution_kind IS NULL''',
|
|
(
|
|
remote['payload_sha256'], receipt['receipt_id'], receipt_json,
|
|
now, diagnostic_projection['version'],
|
|
diagnostic_projection['count'],
|
|
diagnostic_projection['uids_sha256'],
|
|
now, reservation_id, remote['device_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('remote bundle receipt lost its reservation fence')
|
|
if reservation['state'] in ('scanning', 'ready'):
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'scanning', 'scanning', now,
|
|
)
|
|
self.conn.commit()
|
|
return receipt if remote is not None else True
|
|
if reservation['state'] != 'scanning':
|
|
self.conn.rollback()
|
|
return False
|
|
if not active_fence:
|
|
self.conn.rollback()
|
|
return False
|
|
self._submit_docker_blob_leases_locked(reservation, now)
|
|
self.conn.execute(
|
|
'''INSERT INTO result_bundles(
|
|
reservation_id, bundle_id, scan_event_id, scan_event_hash, format_version,
|
|
relative_path, actual_bytes, frame_count, finding_count, error_count,
|
|
candidate_count, state, ready_at, updated_at
|
|
) VALUES (?, ?, ?, ?, 2, ?, ?, ?, ?, ?, ?, 'ready', ?, ?)''',
|
|
(
|
|
reservation_id, values['bundle_id'], values['scan_event_id'],
|
|
values['scan_event_hash'], values['relative_path'], values['actual_bytes'],
|
|
values['frame_count'], values['finding_count'], values['error_count'],
|
|
values['candidate_count'], now, now,
|
|
),
|
|
)
|
|
if remote is None:
|
|
self.conn.execute(
|
|
"UPDATE result_reservations SET state = 'ready', updated_at = ? WHERE id = ?",
|
|
(now, reservation_id),
|
|
)
|
|
else:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'ready',
|
|
remote_resolution_kind = 'bundle_accepted',
|
|
remote_payload_sha256 = ?, remote_receipt_id = ?,
|
|
remote_resolution_json = ?, remote_resolved_at = ?,
|
|
remote_diagnostic_projection_version = ?,
|
|
remote_diagnostic_count = ?,
|
|
remote_diagnostic_uids_sha256 = ?, updated_at = ?
|
|
WHERE id = ? AND state = 'scanning' AND assignment_kind = 'remote'
|
|
AND remote_device_id = ? AND remote_resolution_kind IS NULL''',
|
|
(
|
|
remote['payload_sha256'], receipt['receipt_id'], receipt_json,
|
|
now, diagnostic_projection['version'],
|
|
diagnostic_projection['count'],
|
|
diagnostic_projection['uids_sha256'],
|
|
now, reservation_id, remote['device_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('remote bundle receipt lost its reservation fence')
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'scanning', 'scanning', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
|
|
AND owner_id = ?''',
|
|
(now, now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
|
|
byte_count = ?, deleted_at = NULL, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
|
|
AND owner_id = ?''',
|
|
(
|
|
artifact_sha256, int(values['actual_bytes']), now, reservation_id,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return receipt if remote is not None else True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def recover_expired_result_bundle_ready(self, reservation_id, metadata):
|
|
"""Recover a validated bundle after its exact producer lease expired."""
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('expired result bundle recovery requires PostgreSQL')
|
|
values = metadata.as_dict() if hasattr(metadata, 'as_dict') else dict(metadata or {})
|
|
header = values.get('header')
|
|
if not isinstance(header, dict):
|
|
raise ValueError('expired ready bundle recovery requires its validated header')
|
|
try:
|
|
now = utc_now_iso()
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
reservation = self.conn.execute(
|
|
'''SELECT r.*, q.id AS bound_queue_id,
|
|
q.source AS bound_queue_source,
|
|
q.platform AS bound_queue_platform,
|
|
q.query AS bound_queue_query,
|
|
q.target AS bound_queue_target,
|
|
q.normalized_target AS bound_queue_normalized_target,
|
|
q.status AS queue_status, q.lease_token AS queue_lease_token,
|
|
q.lease_expires_at AS queue_lease_expires_at,
|
|
q.current_result_reservation_id AS queue_reservation_id,
|
|
q.claim_event_id AS queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not reservation:
|
|
self.conn.rollback()
|
|
return False
|
|
if str(reservation['assignment_kind']) == 'remote':
|
|
self.conn.rollback()
|
|
return False
|
|
queue_identity = {
|
|
'id': reservation['bound_queue_id'],
|
|
'source': reservation['bound_queue_source'],
|
|
'platform': reservation['bound_queue_platform'],
|
|
'query': reservation['bound_queue_query'],
|
|
'target': reservation['bound_queue_target'],
|
|
'normalized_target': reservation['bound_queue_normalized_target'],
|
|
}
|
|
self._validate_locked_reservation_queue_identity(
|
|
reservation, queue_identity,
|
|
)
|
|
self._validate_result_bundle_reservation_identity(
|
|
reservation, values, require_header=True,
|
|
)
|
|
ready_relative = str(reservation['ready_relative_path']).replace('\\', '/')
|
|
scan_event_hash = str(values.get('scan_event_hash') or '').lower()
|
|
counts = {
|
|
name: int(values.get(name) if values.get(name) is not None else -1)
|
|
for name in (
|
|
'actual_bytes', 'frame_count', 'finding_count', 'error_count',
|
|
'candidate_count',
|
|
)
|
|
}
|
|
diagnostic_count = int(values.get('diagnostic_count') or 0)
|
|
if (
|
|
not re.fullmatch(r'[a-f0-9]{64}', scan_event_hash)
|
|
or counts['actual_bytes'] <= 0
|
|
or counts['actual_bytes'] > int(reservation['declared_bundle_bytes'])
|
|
or counts['frame_count'] < 3
|
|
or counts['frame_count'] != (
|
|
counts['finding_count'] + counts['error_count']
|
|
+ counts['candidate_count'] + diagnostic_count + 3
|
|
)
|
|
or diagnostic_count < 0
|
|
or any(counts[name] < 0 for name in ('finding_count', 'error_count', 'candidate_count'))
|
|
):
|
|
raise ValueError('expired ready bundle metadata is outside its reservation bounds')
|
|
exact_fence = (
|
|
str(reservation['state']) == 'scanning'
|
|
and str(reservation['queue_status']) == 'in_progress'
|
|
and str(reservation['queue_lease_token'] or '') == str(reservation['claim_lease_token'])
|
|
and int(reservation['queue_reservation_id'] or 0) == int(reservation_id)
|
|
and str(reservation['queue_event_id'] or '') == str(reservation['scan_event_id'])
|
|
and bool(reservation['queue_lease_expires_at'])
|
|
and str(reservation['queue_lease_expires_at']) <= now
|
|
and bool(reservation['producer_lease_expires_at'])
|
|
and str(reservation['producer_lease_expires_at']) <= now
|
|
)
|
|
if not exact_fence:
|
|
self.conn.rollback()
|
|
return False
|
|
if exact_process_identity_state(
|
|
reservation['producer_pid'], reservation['producer_creation_time'],
|
|
reservation['producer_executable'],
|
|
) not in ('dead', 'reused'):
|
|
self.conn.rollback()
|
|
return False
|
|
if reservation['docker_layer_plan_json'] or reservation['docker_layer_plan_sha256']:
|
|
self.conn.rollback()
|
|
return False
|
|
if self.conn.execute(
|
|
'''SELECT 1 AS present FROM docker_content_blobs
|
|
WHERE lease_reservation_id = ? AND state IN ('leased','submitted') LIMIT 1''',
|
|
(int(reservation_id),),
|
|
).fetchone():
|
|
self.conn.rollback()
|
|
return False
|
|
if self.conn.execute(
|
|
'SELECT 1 AS present FROM result_bundles WHERE reservation_id = ?',
|
|
(int(reservation_id),),
|
|
).fetchone():
|
|
self.conn.rollback()
|
|
return False
|
|
if self.conn.execute(
|
|
'''SELECT 1 AS present FROM pipeline_quarantine
|
|
WHERE reservation_id = ? AND review_status = 'pending' LIMIT 1''',
|
|
(int(reservation_id),),
|
|
).fetchone():
|
|
self.conn.rollback()
|
|
return False
|
|
self._docker_depth_binding_for_reservation_locked(reservation)
|
|
self.conn.execute(
|
|
'''INSERT INTO result_bundles(
|
|
reservation_id, bundle_id, scan_event_id, scan_event_hash, format_version,
|
|
relative_path, actual_bytes, frame_count, finding_count, error_count,
|
|
candidate_count, state, ready_at, updated_at
|
|
) VALUES (?, ?, ?, ?, 2, ?, ?, ?, ?, ?, ?, 'ready', ?, ?)''',
|
|
(
|
|
int(reservation_id), values['bundle_id'], values['scan_event_id'],
|
|
scan_event_hash, ready_relative, counts['actual_bytes'], counts['frame_count'],
|
|
counts['finding_count'], counts['error_count'], counts['candidate_count'],
|
|
now, now,
|
|
),
|
|
)
|
|
cursor = self.conn.execute(
|
|
"UPDATE result_reservations SET state = 'ready', updated_at = ? WHERE id = ? AND state = 'scanning'",
|
|
(now, int(reservation_id)),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('expired ready bundle reservation changed during recovery')
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'scanning', 'scanning', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_partial'
|
|
AND owner_id = ?''',
|
|
(now, now, int(reservation_id)),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'present', payload_sha256 = ?,
|
|
byte_count = ?, deleted_at = NULL, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_ready'
|
|
AND owner_id = ?''',
|
|
(scan_event_hash, counts['actual_bytes'], now, int(reservation_id)),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def active_result_reservations(self, after_id=0, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result reservation recovery requires PostgreSQL')
|
|
rows = self.conn.execute(
|
|
'''SELECT * FROM result_reservations
|
|
WHERE id > ? AND state IN ('scanning','ready','ingesting','db_committed')
|
|
AND (cleanup_available_after IS NULL OR cleanup_available_after <= ?)
|
|
ORDER BY id LIMIT ?''',
|
|
(
|
|
max(0, int(after_id)), utc_now_iso(),
|
|
min(1000, max(1, int(limit))),
|
|
),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return [dict(row) for row in rows]
|
|
|
|
def defer_result_reservation_cleanup(self, reservation_id, error):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return False
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
row = self.conn.execute(
|
|
'SELECT cleanup_attempts, state FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return False
|
|
attempts = int(row['cleanup_attempts'] or 0) + 1
|
|
delay = min(300, 2 ** min(attempts, 8))
|
|
available = datetime.fromtimestamp(
|
|
time.time() + delay, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
self.conn.execute(
|
|
'''UPDATE result_reservations SET cleanup_attempts = ?,
|
|
cleanup_available_after = ?, last_error_code = 'storage_cleanup_deferred',
|
|
last_error_detail = ?, updated_at = ? WHERE id = ?''',
|
|
(attempts, available, first_line(error, 1000), now, int(reservation_id)),
|
|
)
|
|
bundle = self.conn.execute(
|
|
'SELECT target_scan_id, state FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if bundle:
|
|
next_state = 'db_committed' if bundle['target_scan_id'] is not None else 'ready'
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = ?, available_after = ?,
|
|
ingest_lease_generation = NULL, ingest_lease_token = NULL,
|
|
ingest_lease_expires_at = NULL, updated_at = ?
|
|
WHERE reservation_id = ?''',
|
|
(next_state, available, now, int(reservation_id)),
|
|
)
|
|
self.conn.execute(
|
|
'UPDATE result_reservations SET state = ? WHERE id = ?',
|
|
(next_state, int(reservation_id)),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def result_bundle_for_reservation(self, reservation_id):
|
|
if not self.conn:
|
|
return None
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ?',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def pending_result_bundle_quarantine(self, reservation_id):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM pipeline_quarantine
|
|
WHERE subsystem = 'result_ingester' AND object_type = 'result_bundle'
|
|
AND reservation_id = ? AND review_status = 'pending' ''',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def claim_ready_result_bundle(self, generation, lease_token, lease_seconds=60):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('result bundle claims require PostgreSQL')
|
|
now = utc_now_iso()
|
|
expires = datetime.fromtimestamp(
|
|
time.time() + max(10, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
ingest_token = secrets.token_urlsafe(32)
|
|
try:
|
|
owner = self.conn.execute(
|
|
'''SELECT 1 AS valid FROM pipeline_leases
|
|
WHERE worker_name = 'result_ingester' AND generation = ?
|
|
AND lease_token = ? AND state IN ('recovering','ready')
|
|
AND lease_expires_at > ? FOR SHARE''',
|
|
(int(generation), str(lease_token), now),
|
|
).fetchone()
|
|
if not owner:
|
|
self.conn.rollback()
|
|
raise RuntimeError('result ingester singleton fence is not valid')
|
|
row = self.conn.execute(
|
|
'''SELECT reservation_id FROM result_bundles
|
|
WHERE (
|
|
state IN ('ready','db_committed')
|
|
OR (state = 'ingesting' AND (
|
|
ingest_lease_generation IS NULL OR ingest_lease_generation <> ?
|
|
OR (ingest_lease_expires_at IS NOT NULL AND ingest_lease_expires_at <= ?)
|
|
))
|
|
) AND (available_after IS NULL OR available_after <= ?)
|
|
ORDER BY CASE WHEN state = 'db_committed' THEN 0 ELSE 1 END,
|
|
ready_at, reservation_id
|
|
LIMIT 1 FOR UPDATE SKIP LOCKED''',
|
|
(int(generation), now, now),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return None
|
|
bundle = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(row['reservation_id'],),
|
|
).fetchone()
|
|
next_state = 'db_committed' if bundle['state'] == 'db_committed' else 'ingesting'
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = ?, ingest_attempts = ingest_attempts + 1,
|
|
ingest_lease_generation = ?, ingest_lease_token = ?,
|
|
ingest_lease_expires_at = ?, updated_at = ?
|
|
WHERE reservation_id = ?''',
|
|
(
|
|
next_state, int(generation), ingest_token, expires, now,
|
|
row['reservation_id'],
|
|
),
|
|
)
|
|
if next_state == 'ingesting':
|
|
self.conn.execute(
|
|
"UPDATE result_reservations SET state = 'ingesting', updated_at = ? WHERE id = ?",
|
|
(now, row['reservation_id']),
|
|
)
|
|
reservation = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ?',
|
|
(row['reservation_id'],),
|
|
).fetchone()
|
|
updated = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ?',
|
|
(row['reservation_id'],),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return {
|
|
'reservation': dict(reservation),
|
|
'bundle': dict(updated),
|
|
'ingest_lease_token': ingest_token,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def confirm_scan_event(self, event_id, event_hash):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
row = self.conn.execute(
|
|
'''SELECT id, scan_event_id, scan_event_hash, result_reservation_id,
|
|
raw_result_storage, compat_schema_version
|
|
FROM target_scans WHERE scan_event_id = ?''',
|
|
(str(event_id),),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
if not row:
|
|
return None
|
|
if str(row['scan_event_hash'] or '') != str(event_hash or ''):
|
|
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
|
|
return dict(row)
|
|
|
|
def acknowledge_removed_bundle(self, reservation_id, event_id, event_hash):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('bundle acknowledgement requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
reservation = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
bundle = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
event = self.conn.execute(
|
|
'SELECT id, scan_event_hash FROM target_scans WHERE scan_event_id = ?',
|
|
(str(event_id),),
|
|
).fetchone()
|
|
if not reservation or not bundle or not event or str(event['scan_event_hash']) != str(event_hash):
|
|
self.conn.rollback()
|
|
return False
|
|
if str(bundle['scan_event_hash']) != str(event_hash):
|
|
raise ScanEventConflictError('bundle acknowledgement hash conflicts with committed event')
|
|
if not reservation['bundle_credit_released']:
|
|
if int(capacity['bundle_items']) < 1 or int(capacity['bundle_bytes']) < int(reservation['reserved_bundle_bytes']):
|
|
raise RuntimeError('bundle capacity accounting would become negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET bundle_items = bundle_items - 1,
|
|
bundle_bytes = bundle_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(reservation['reserved_bundle_bytes'], now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'acknowledged', bundle_credit_released = 1,
|
|
released_at = COALESCE(released_at, ?), updated_at = ? WHERE id = ?''',
|
|
(now, now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = 'acknowledged', acknowledged_at = COALESCE(acknowledged_at, ?),
|
|
ingest_lease_token = NULL, ingest_lease_expires_at = NULL, updated_at = ?
|
|
WHERE reservation_id = ?''',
|
|
(now, now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND owner_id = ?
|
|
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
|
|
(now, now, reservation_id),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _remote_resolution_from_row(row):
|
|
raw = str(row['remote_resolution_json'] or '')
|
|
try:
|
|
value = json.loads(raw)
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise ScanEventConflictError('remote assignment durable receipt is invalid') from exc
|
|
if not isinstance(value, dict) or str(value.get('receipt_id') or '') != str(
|
|
row['remote_receipt_id'] or ''
|
|
):
|
|
raise ScanEventConflictError('remote assignment durable receipt identity is invalid')
|
|
return value
|
|
|
|
def _remote_assignment_observability_locked(
|
|
self, row, *, diagnostic_count=None,
|
|
):
|
|
def row_value(name):
|
|
return row.get(name) if hasattr(row, 'get') else row[name]
|
|
|
|
latest = self.conn.execute(
|
|
'''SELECT sequence, slot_id, phase, event_timestamp, phase_started_at,
|
|
event_json, received_at
|
|
FROM worker_progress_events
|
|
WHERE reservation_id = ? ORDER BY sequence DESC, id DESC LIMIT 1''',
|
|
(int(row['id']),),
|
|
).fetchone()
|
|
latest_value = None
|
|
known_reason = None
|
|
scan_deadline_at = None
|
|
if latest:
|
|
try:
|
|
event = json.loads(str(latest['event_json']))
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise WorkerObservabilityConflictError(
|
|
'durable worker progress JSON is invalid'
|
|
) from exc
|
|
latest_value = {
|
|
'sequence': int(latest['sequence']),
|
|
'slot_id': int(latest['slot_id']),
|
|
'phase': str(latest['phase']),
|
|
'event_timestamp': str(latest['event_timestamp']),
|
|
'phase_started_at': str(latest['phase_started_at'] or ''),
|
|
'received_at': str(latest['received_at']),
|
|
'progress': dict(event.get('progress') or {}),
|
|
}
|
|
scan_deadline_at = event.get('scan_deadline_at')
|
|
if latest_value['phase'] in {'idle', 'backoff'}:
|
|
reason = latest_value['progress'].get('reason')
|
|
known_reason = str(reason) if reason is not None else None
|
|
if diagnostic_count is None:
|
|
count_row = self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM worker_diagnostics
|
|
WHERE reservation_id = ?''',
|
|
(int(row['id']),),
|
|
).fetchone()
|
|
diagnostic_count = int(count_row['count'] or 0) if count_row else 0
|
|
accepted_count = row_value('remote_diagnostic_count')
|
|
if accepted_count is not None:
|
|
diagnostic_count = max(
|
|
diagnostic_count, int(accepted_count),
|
|
)
|
|
timeout_seconds = None
|
|
snapshot_raw = row_value('remote_execution_snapshot_json')
|
|
if snapshot_raw:
|
|
try:
|
|
snapshot = json.loads(str(snapshot_raw))
|
|
timeout = ((snapshot.get('execution') or {}).get('scan_kwargs') or {}).get(
|
|
'timeout_sec'
|
|
)
|
|
if type(timeout) in (int, float) and not isinstance(timeout, bool):
|
|
timeout_seconds = int(timeout)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
timeout_seconds = None
|
|
return {
|
|
'deadlines': {
|
|
'assignment_issued_at': (
|
|
str(row_value('remote_issued_at'))
|
|
if row_value('remote_issued_at') else None
|
|
),
|
|
'assignment_deadline_at': (
|
|
str(row_value('remote_expires_at'))
|
|
if row_value('remote_expires_at') else None
|
|
),
|
|
'scan_deadline_at': scan_deadline_at,
|
|
'target_scan_timeout_seconds': timeout_seconds,
|
|
'result_upload_body_timeout_seconds': (
|
|
int(row_value('remote_result_upload_body_timeout_seconds'))
|
|
if row_value('remote_result_upload_body_timeout_seconds') is not None
|
|
else None
|
|
),
|
|
'result_upload_body_timeout_availability': (
|
|
'persisted'
|
|
if row_value('remote_result_upload_body_timeout_seconds') is not None
|
|
else 'legacy/unavailable'
|
|
),
|
|
'immutable': True,
|
|
},
|
|
'latest_progress': {
|
|
'available': latest_value is not None,
|
|
'event': latest_value,
|
|
},
|
|
'known_reason': known_reason,
|
|
'diagnostics': {
|
|
'available': int(diagnostic_count) > 0,
|
|
'count': int(diagnostic_count),
|
|
'projection_version': row_value(
|
|
'remote_diagnostic_projection_version'
|
|
),
|
|
'ordered_uid_set_sha256': row_value(
|
|
'remote_diagnostic_uids_sha256'
|
|
),
|
|
},
|
|
}
|
|
|
|
def remote_assignment_status(self, reservation_id, device_id, token_sha256):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote assignment status requires PostgreSQL')
|
|
try:
|
|
credential = self._lock_active_remote_credential(
|
|
device_id, token_sha256,
|
|
)
|
|
if not credential:
|
|
self.conn.rollback()
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT id, bundle_id, scan_event_id, state, remote_device_id,
|
|
remote_issued_at, remote_expires_at,
|
|
remote_result_upload_body_timeout_seconds,
|
|
remote_execution_snapshot_json,
|
|
remote_diagnostic_projection_version,
|
|
remote_diagnostic_count,
|
|
remote_diagnostic_uids_sha256,
|
|
remote_resolution_kind, remote_receipt_id,
|
|
remote_resolution_json, remote_payload_sha256, remote_resolved_at
|
|
FROM result_reservations
|
|
WHERE id = ? AND assignment_kind = 'remote' AND remote_device_id = ?''',
|
|
(int(reservation_id), int(device_id)),
|
|
).fetchone()
|
|
observability = (
|
|
self._remote_assignment_observability_locked(row) if row else None
|
|
)
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
if not row:
|
|
return None
|
|
if row['remote_resolution_kind'] is not None:
|
|
result = dict(self._remote_resolution_from_row(row))
|
|
result.setdefault('deadlines', observability['deadlines'])
|
|
for name in ('latest_progress', 'known_reason', 'diagnostics'):
|
|
result[name] = observability[name]
|
|
return result
|
|
return {
|
|
'reservation_id': int(row['id']),
|
|
'bundle_id': str(row['bundle_id']),
|
|
'scan_event_id': str(row['scan_event_id']),
|
|
'state': str(row['state']),
|
|
'expires_at': str(row['remote_expires_at']),
|
|
**observability,
|
|
}
|
|
|
|
def remote_assignment_transport(self, reservation_id, device_id, token_sha256):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote assignment transport requires PostgreSQL')
|
|
try:
|
|
credential = self._lock_active_remote_credential(
|
|
device_id, token_sha256,
|
|
)
|
|
if not credential:
|
|
self.conn.rollback()
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT id, reservation_token, bundle_id, scan_event_id, queue_id,
|
|
claim_lease_token, declared_bundle_bytes, ready_relative_path,
|
|
source, platform, query, target, normalized_target, run_id, cycle_id,
|
|
producer_instance_id, producer_pid, producer_creation_time,
|
|
producer_executable, state, remote_device_id, remote_expires_at,
|
|
remote_resolution_kind, remote_payload_sha256, remote_receipt_id,
|
|
remote_resolution_json
|
|
FROM result_reservations
|
|
WHERE id = ? AND assignment_kind = 'remote' AND remote_device_id = ?''',
|
|
(int(reservation_id), int(device_id)),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
if not row:
|
|
return None
|
|
result = dict(row)
|
|
result['reservation_id'] = int(result['id'])
|
|
result['declared_bytes'] = int(result['declared_bundle_bytes'])
|
|
result['ready_path'] = str(result['ready_relative_path'])
|
|
if result['remote_resolution_kind'] is not None:
|
|
result['receipt'] = self._remote_resolution_from_row(row)
|
|
return result
|
|
|
|
def _resolve_remote_scanning_locked(
|
|
self, row, resolution_kind, resolution_payload_sha256, detail, now,
|
|
receipt_fields=None,
|
|
):
|
|
reservation_id = int(row['id'])
|
|
queue = self.conn.execute(
|
|
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
|
|
(int(row['queue_id']),),
|
|
).fetchone()
|
|
if not queue or (
|
|
str(queue['status']) != 'in_progress'
|
|
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
|
|
or int(queue['current_result_reservation_id'] or 0) != reservation_id
|
|
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError('remote assignment lost its exact queue fence')
|
|
error_code = (
|
|
'remote_assignment_expired'
|
|
if resolution_kind == 'expired'
|
|
else 'remote_prebundle_infrastructure_failure'
|
|
)
|
|
self._release_docker_blob_leases_locked(
|
|
row, error_code, detail, now, refund_attempt=True,
|
|
)
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
deductions = (
|
|
int(row['reserved_bundle_bytes']), int(row['reserved_projection_items']),
|
|
int(row['reserved_projection_bytes']), int(row['reserved_candidate_items']),
|
|
int(row['reserved_candidate_bytes']),
|
|
)
|
|
if not capacity or (
|
|
int(capacity['bundle_items']) < 1
|
|
or int(capacity['bundle_bytes']) < deductions[0]
|
|
or int(capacity['projection_items']) < deductions[1]
|
|
or int(capacity['projection_bytes']) < deductions[2]
|
|
or int(capacity['keycheck_items']) < deductions[3]
|
|
or int(capacity['keycheck_bytes']) < deductions[4]
|
|
):
|
|
raise RuntimeError('remote assignment resolution would make capacity accounting negative')
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending',
|
|
attempts = CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL, available_after = NULL,
|
|
current_result_reservation_id = NULL, claim_event_id = NULL,
|
|
last_error = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
first_line(detail, 500), now, row['queue_id'], row['claim_lease_token'],
|
|
reservation_id, row['scan_event_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('remote assignment queue changed during resolution')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
bundle_items = bundle_items - 1, bundle_bytes = bundle_bytes - ?,
|
|
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
|
|
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
|
|
updated_at = ? WHERE id = 1''',
|
|
(*deductions, now),
|
|
)
|
|
receipt = {
|
|
'receipt_id': secrets.token_hex(32),
|
|
'resolution': resolution_kind,
|
|
'reservation_id': reservation_id,
|
|
'bundle_id': str(row['bundle_id']),
|
|
'scan_event_id': str(row['scan_event_id']),
|
|
'resolved_at': now,
|
|
}
|
|
receipt.update(self._remote_assignment_observability_locked(row))
|
|
if resolution_payload_sha256:
|
|
receipt['payload_sha256'] = resolution_payload_sha256
|
|
if receipt_fields:
|
|
receipt.update(dict(receipt_fields))
|
|
receipt_json = json.dumps(
|
|
receipt, ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'refunded', bundle_credit_released = 1,
|
|
last_error_code = ?, last_error_detail = ?, refunded_at = ?,
|
|
released_at = ?, remote_resolution_kind = ?, remote_payload_sha256 = ?,
|
|
remote_receipt_id = ?, remote_resolution_json = ?, remote_resolved_at = ?,
|
|
updated_at = ? WHERE id = ? AND state = 'scanning'
|
|
AND assignment_kind = 'remote' AND remote_resolution_kind IS NULL''',
|
|
(
|
|
error_code, first_line(detail, 1000), now, now, resolution_kind,
|
|
resolution_payload_sha256, receipt['receipt_id'], receipt_json,
|
|
now, now, reservation_id,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('remote assignment changed during resolution')
|
|
self._transition_docker_depth_binding_locked(row, 'released', 'pending', now)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET cleanup_attempts = 0,
|
|
cleanup_available_after = NULL, cleanup_last_error = NULL,
|
|
updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND owner_id = ?
|
|
AND artifact_kind IN ('bundle_partial','bundle_ready')
|
|
AND state IN ('expected','present')''',
|
|
(now, reservation_id),
|
|
)
|
|
return receipt
|
|
|
|
def report_remote_prebundle_failure(
|
|
self, reservation_id, device_id, token_sha256, report,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote pre-bundle reporting requires PostgreSQL')
|
|
report = dict(report or {})
|
|
failure_code = str(report.get('failure_code') or '').strip().lower()
|
|
if failure_code not in {
|
|
'client_process_failed', 'client_storage_failed', 'client_cancelled',
|
|
}:
|
|
raise ValueError('remote pre-bundle failure code is not permitted')
|
|
if set(report) not in (
|
|
{'failure_code', 'detail'},
|
|
{'failure_code', 'detail', 'diagnostics'},
|
|
) or not isinstance(report.get('detail'), str) or len(report['detail']) > 1000:
|
|
raise ValueError('remote pre-bundle report shape is invalid')
|
|
detail = first_line(report.get('detail') or failure_code, 1000)
|
|
normalized_report = {
|
|
'failure_code': failure_code,
|
|
'detail': report['detail'],
|
|
}
|
|
diagnostics = report.get('diagnostics')
|
|
if diagnostics is not None:
|
|
if not isinstance(diagnostics, list) or len(diagnostics) > 32:
|
|
raise ValueError('remote pre-bundle diagnostics are invalid')
|
|
normalized_report['diagnostics'] = []
|
|
diagnostic_uids = set()
|
|
for diagnostic in diagnostics:
|
|
normalized = _canonical_worker_contract(
|
|
'diagnostic', diagnostic,
|
|
)[0]
|
|
if (
|
|
normalized.get('assignment_outcome') != 'prebundle_failed'
|
|
or normalized.get('scan_outcome') != 'unavailable'
|
|
):
|
|
raise ValueError(
|
|
'remote pre-bundle diagnostic outcomes are invalid'
|
|
)
|
|
uid = normalized['diagnostic_uid']
|
|
if uid in diagnostic_uids:
|
|
raise ValueError(
|
|
'remote pre-bundle diagnostic UID is duplicated'
|
|
)
|
|
diagnostic_uids.add(uid)
|
|
normalized_report['diagnostics'].append(normalized)
|
|
report_json = json.dumps(
|
|
normalized_report,
|
|
ensure_ascii=True, sort_keys=True, separators=(',', ':'),
|
|
)
|
|
if len(report_json.encode('ascii')) > 16 * 1024:
|
|
raise ValueError('remote pre-bundle report exceeds its byte bound')
|
|
report_sha256 = hashlib.sha256(report_json.encode('utf-8')).hexdigest()
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not row or str(row['assignment_kind']) != 'remote' or int(
|
|
row['remote_device_id'] or 0
|
|
) != int(device_id):
|
|
self.conn.rollback()
|
|
return None
|
|
if not self._lock_active_remote_credential(
|
|
device_id, token_sha256, user_id=row['remote_user_id'],
|
|
):
|
|
raise ScanEventConflictError(
|
|
'remote worker credential changed before terminal report'
|
|
)
|
|
if row['remote_resolution_kind'] is not None:
|
|
if (
|
|
str(row['remote_resolution_kind']) != 'prebundle_report'
|
|
or str(row['remote_payload_sha256'] or '') != report_sha256
|
|
):
|
|
raise ScanEventConflictError('remote assignment already has a conflicting resolution')
|
|
receipt = self._remote_resolution_from_row(row)
|
|
self.conn.commit()
|
|
return receipt
|
|
now = utc_now_iso()
|
|
if (
|
|
str(row['state']) != 'scanning'
|
|
or not row['remote_expires_at']
|
|
or str(row['remote_expires_at']) <= now
|
|
):
|
|
self.conn.rollback()
|
|
return None
|
|
for diagnostic in normalized_report.get('diagnostics') or ():
|
|
self._record_worker_diagnostic(
|
|
int(reservation_id), diagnostic, received_at=now,
|
|
)
|
|
receipt = self._resolve_remote_scanning_locked(
|
|
row, 'prebundle_report', report_sha256, detail, now,
|
|
receipt_fields={'failure_code': failure_code},
|
|
)
|
|
self.conn.commit()
|
|
return receipt
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def expire_remote_assignment(self, reservation_id, now=None):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote assignment expiry requires PostgreSQL')
|
|
now = str(now or utc_now_iso())
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not row or (
|
|
str(row['assignment_kind']) != 'remote'
|
|
or str(row['state']) != 'scanning'
|
|
or row['remote_resolution_kind'] is not None
|
|
or not row['remote_expires_at']
|
|
or str(row['remote_expires_at']) > now
|
|
):
|
|
self.conn.rollback()
|
|
return None
|
|
contracts = importlib.import_module('worker_contracts')
|
|
observability = self._remote_assignment_observability_locked(row)
|
|
latest = observability['latest_progress']['event']
|
|
phase_name = (
|
|
latest['phase'] if latest is not None
|
|
else contracts.WorkerPhase.ASSIGNED.value
|
|
)
|
|
envelope = contracts.build_diagnostic_envelope(
|
|
occurrence_id=(
|
|
f'reservation:{int(reservation_id)}:'
|
|
f'{row["remote_expires_at"]}:expiry'
|
|
),
|
|
reservation_id=int(reservation_id),
|
|
scan_event_id=None,
|
|
slot_id=(latest['slot_id'] if latest is not None else 0),
|
|
source=str(row['source']),
|
|
phase=contracts.WorkerPhase(phase_name),
|
|
kind=contracts.DiagnosticKind.ASSIGNMENT,
|
|
category=contracts.DiagnosticCategory.ASSIGNMENT_EXPIRED,
|
|
code='assignment.deadline_expired',
|
|
summary=(
|
|
f'Assignment deadline expired after phase {phase_name}'
|
|
if latest is not None else 'Assignment deadline expired without progress'
|
|
),
|
|
retryable=True,
|
|
attempt=1,
|
|
assignment_outcome=contracts.AssignmentOutcome.EXPIRED,
|
|
scan_outcome=contracts.ScanOutcome.UNAVAILABLE,
|
|
occurred_at=_worker_contract_timestamp(row['remote_expires_at']),
|
|
captured_at=_worker_contract_timestamp(now),
|
|
)
|
|
diagnostic = json.loads(
|
|
contracts.encode_diagnostic_envelope(envelope).decode('ascii')
|
|
)
|
|
self._record_worker_diagnostic(
|
|
int(reservation_id), diagnostic, received_at=now,
|
|
)
|
|
receipt = self._resolve_remote_scanning_locked(
|
|
row, 'expired', None, 'remote assignment deadline expired', now,
|
|
)
|
|
self.conn.commit()
|
|
return receipt
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def reap_expired_remote_assignments(self, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('remote assignment reaper requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
rows = self.conn.execute(
|
|
'''SELECT id FROM result_reservations
|
|
WHERE assignment_kind = 'remote' AND state = 'scanning'
|
|
AND remote_resolution_kind IS NULL AND remote_expires_at <= ?
|
|
ORDER BY remote_expires_at, id LIMIT ?''',
|
|
(now, min(1000, max(1, int(limit)))),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
receipts = []
|
|
for row in rows:
|
|
try:
|
|
receipt = self.expire_remote_assignment(row['id'], now=now)
|
|
except Exception:
|
|
logger.error(
|
|
'remote assignment expiry failed for reservation_id=%s',
|
|
int(row['id']),
|
|
)
|
|
continue
|
|
if receipt:
|
|
receipts.append(receipt)
|
|
return receipts
|
|
|
|
def refund_uncommitted_reservation(
|
|
self, reservation_id, exact_producer_identity, reason,
|
|
partial_absence_confirmed=False,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('reservation refunds require PostgreSQL')
|
|
if partial_absence_confirmed is not True:
|
|
raise RuntimeError(
|
|
'reservation refund requires tri-state-confirmed deterministic partial absence'
|
|
)
|
|
identity = _identity_mapping(exact_producer_identity)
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not row or row['state'] != 'scanning':
|
|
self.conn.rollback()
|
|
return False
|
|
if str(row['assignment_kind']) != 'local':
|
|
self.conn.rollback()
|
|
return False
|
|
if (
|
|
int(row['producer_pid']) != identity['pid']
|
|
or str(row['producer_creation_time']) != identity['creation_time']
|
|
or os.path.normcase(os.path.realpath(str(row['producer_executable']))) != identity['executable']
|
|
):
|
|
self.conn.rollback()
|
|
return False
|
|
queue = self.conn.execute(
|
|
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
|
|
(int(row['queue_id']),),
|
|
).fetchone()
|
|
if not queue or (
|
|
str(queue['status']) != 'in_progress'
|
|
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
|
|
or int(queue['current_result_reservation_id'] or 0) != int(row['id'])
|
|
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
|
|
):
|
|
self.conn.rollback()
|
|
return False
|
|
self._release_docker_blob_leases_locked(
|
|
row,
|
|
'producer_failed_before_handoff',
|
|
reason,
|
|
now,
|
|
refund_attempt=True,
|
|
)
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending',
|
|
attempts = CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL, available_after = NULL,
|
|
current_result_reservation_id = NULL, claim_event_id = NULL,
|
|
last_error = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
first_line(reason, 500), now, row['queue_id'], row['claim_lease_token'],
|
|
row['id'], row['scan_event_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
deductions = (
|
|
int(row['reserved_bundle_bytes']), int(row['reserved_projection_items']),
|
|
int(row['reserved_projection_bytes']), int(row['reserved_candidate_items']),
|
|
int(row['reserved_candidate_bytes']),
|
|
)
|
|
if (
|
|
int(capacity['bundle_items']) < 1
|
|
or int(capacity['bundle_bytes']) < deductions[0]
|
|
or int(capacity['projection_items']) < deductions[1]
|
|
or int(capacity['projection_bytes']) < deductions[2]
|
|
or int(capacity['keycheck_items']) < deductions[3]
|
|
or int(capacity['keycheck_bytes']) < deductions[4]
|
|
):
|
|
raise RuntimeError('reservation refund would make capacity accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
bundle_items = bundle_items - 1, bundle_bytes = bundle_bytes - ?,
|
|
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
|
|
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
|
|
updated_at = ? WHERE id = 1''',
|
|
(*deductions, now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'refunded', bundle_credit_released = 1,
|
|
last_error_code = 'producer_failed_before_handoff', last_error_detail = ?,
|
|
refunded_at = ?, released_at = ?, updated_at = ? WHERE id = ?''',
|
|
(first_line(reason, 1000), now, now, now, row['id']),
|
|
)
|
|
self._transition_docker_depth_binding_locked(
|
|
row, 'released', 'pending', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND owner_id = ?
|
|
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
|
|
(now, now, row['id']),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _finalize_result_bundle_quarantine_locked(
|
|
self, reservation, reason_code, reason_detail, source_relative_path,
|
|
payload_sha256, byte_count, now,
|
|
):
|
|
reservation_id = int(reservation['id'])
|
|
queue = self.conn.execute(
|
|
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
|
|
(int(reservation['queue_id']),),
|
|
).fetchone()
|
|
if str(reservation['state']) == 'quarantined':
|
|
if not queue or (
|
|
str(queue['status']) != 'quarantined'
|
|
or int(queue['current_result_reservation_id'] or 0) != reservation_id
|
|
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
|
|
or not reservation['bundle_credit_released']
|
|
or str(reservation['last_error_code'] or '') != str(reason_code)
|
|
or str(reservation['last_error_detail'] or '') != first_line(reason_detail, 2000)
|
|
):
|
|
raise ScanEventConflictError('quarantined bundle lost its exact queue fence')
|
|
bundle = self.conn.execute(
|
|
'SELECT state FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if bundle and str(bundle['state']) != 'quarantined':
|
|
raise ScanEventConflictError('quarantined bundle row lost its terminal state')
|
|
if source_relative_path:
|
|
artifact = self.conn.execute(
|
|
'''SELECT state, payload_sha256, byte_count FROM pipeline_artifacts
|
|
WHERE subsystem = 'result_bundle'
|
|
AND artifact_kind = 'bundle_quarantine' AND owner_id = ?
|
|
AND relative_path = ? FOR UPDATE''',
|
|
(reservation_id, self._artifact_relative_path(source_relative_path)),
|
|
).fetchone()
|
|
if not artifact or (
|
|
str(artifact['state']) != 'quarantined'
|
|
or str(artifact['payload_sha256'] or '') != str(payload_sha256 or '')
|
|
or int(artifact['byte_count'] or 0) != max(0, int(byte_count))
|
|
):
|
|
raise ScanEventConflictError('quarantined bundle artifact evidence changed')
|
|
return
|
|
else:
|
|
if not queue or (
|
|
str(queue['status']) != 'in_progress'
|
|
or str(queue['lease_token'] or '') != str(reservation['claim_lease_token'])
|
|
or int(queue['current_result_reservation_id'] or 0) != reservation_id
|
|
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError('bundle quarantine lost its exact queue fence')
|
|
self._release_docker_blob_leases_locked(
|
|
reservation,
|
|
'bundle_quarantined',
|
|
reason_detail or reason_code,
|
|
now,
|
|
refund_attempt=False,
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'quarantined', completed_at = ?, last_error = ?,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
now, first_line(reason_detail or reason_code, 500), now,
|
|
reservation['queue_id'], reservation['claim_lease_token'],
|
|
reservation_id, reservation['scan_event_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('bundle quarantine queue transition lost its fence')
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'quarantined', bundle_credit_released = 1,
|
|
last_error_code = ?, last_error_detail = ?, released_at = ?, updated_at = ?
|
|
WHERE id = ? AND state = ?''',
|
|
(
|
|
str(reason_code), first_line(reason_detail, 2000), now, now,
|
|
reservation_id, reservation['state'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('bundle quarantine reservation transition lost its fence')
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'quarantined', 'quarantined', now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = 'quarantined', ingest_lease_token = NULL,
|
|
ingest_lease_expires_at = NULL, updated_at = ? WHERE reservation_id = ?''',
|
|
(now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND owner_id = ?
|
|
AND artifact_kind IN ('bundle_partial','bundle_ready')''',
|
|
(now, now, reservation_id),
|
|
)
|
|
if source_relative_path:
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'quarantined', payload_sha256 = ?,
|
|
byte_count = ?, deleted_at = NULL, updated_at = ?
|
|
WHERE subsystem = 'result_bundle' AND artifact_kind = 'bundle_quarantine'
|
|
AND owner_id = ? AND relative_path = ?''',
|
|
(
|
|
str(payload_sha256 or ''), max(0, int(byte_count)), now,
|
|
reservation_id, self._artifact_relative_path(source_relative_path),
|
|
),
|
|
)
|
|
|
|
def quarantine_result_bundle(
|
|
self, reservation_id, reason_code, reason_detail='', source_relative_path='',
|
|
payload_sha256='', byte_count=0,
|
|
quarantine_max_items=10000, quarantine_max_bytes=1024 * 1024 * 1024,
|
|
physical_confirmed=False,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('bundle quarantine requires PostgreSQL')
|
|
reason_code = str(reason_code or '').strip()
|
|
if not reason_code or len(reason_code) > 64 or not re.fullmatch(r'[a-z0-9_]+', reason_code):
|
|
raise ValueError('bundle quarantine reason code is invalid')
|
|
reason_detail = first_line(reason_detail, 2000)
|
|
source_relative_path = (
|
|
self._artifact_relative_path(source_relative_path) if source_relative_path else ''
|
|
)
|
|
payload_sha256 = str(payload_sha256 or '').strip().lower()
|
|
if payload_sha256 and not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
|
|
raise ValueError('bundle quarantine payload hash is invalid')
|
|
byte_count = max(0, int(byte_count))
|
|
now = utc_now_iso()
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
raise ValueError('result reservation is absent')
|
|
existing = self.conn.execute(
|
|
'''SELECT * FROM pipeline_quarantine
|
|
WHERE subsystem = 'result_ingester' AND object_type = 'result_bundle'
|
|
AND object_id = ? AND review_status = 'pending' ''',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if existing:
|
|
if (
|
|
str(existing['source_relative_path'] or '') != str(source_relative_path or '')
|
|
or str(existing['payload_sha256'] or '') != str(payload_sha256 or '')
|
|
or int(existing['byte_count']) != byte_count
|
|
or str(existing['reason_code']) != reason_code
|
|
or str(existing['reason_detail'] or '') != reason_detail
|
|
or int(existing['reservation_id'] or 0) != int(reservation_id)
|
|
or str(existing['event_id'] or '') != str(row['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError('bundle quarantine preparation conflicts with existing evidence')
|
|
if physical_confirmed:
|
|
self._finalize_result_bundle_quarantine_locked(
|
|
row, reason_code, reason_detail, source_relative_path,
|
|
payload_sha256, byte_count, now,
|
|
)
|
|
self.conn.commit()
|
|
return int(existing['id'])
|
|
if str(row['state']) not in ('scanning', 'ready', 'ingesting'):
|
|
raise ScanEventConflictError('bundle reservation is not exactly quarantineable')
|
|
queue = self.conn.execute(
|
|
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
|
|
(int(row['queue_id']),),
|
|
).fetchone()
|
|
if not queue or (
|
|
str(queue['status']) != 'in_progress'
|
|
or str(queue['lease_token'] or '') != str(row['claim_lease_token'])
|
|
or int(queue['current_result_reservation_id'] or 0) != int(row['id'])
|
|
or str(queue['claim_event_id'] or '') != str(row['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError('bundle quarantine does not own the target queue fence')
|
|
bundle = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(int(reservation_id),),
|
|
).fetchone()
|
|
if str(row['state']) in ('ready', 'ingesting') and (
|
|
not bundle
|
|
or str(bundle['bundle_id']) != str(row['bundle_id'])
|
|
or str(bundle['scan_event_id']) != str(row['scan_event_id'])
|
|
or str(bundle['relative_path']).replace('\\', '/')
|
|
!= str(row['ready_relative_path']).replace('\\', '/')
|
|
or str(bundle['state']) not in ('ready', 'ingesting')
|
|
):
|
|
raise ScanEventConflictError('bundle quarantine row identity is not exact')
|
|
if str(row['state']) == 'scanning' and bundle is not None:
|
|
raise ScanEventConflictError('scanning quarantine has an unexpected ready bundle row')
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
projection_items = 0 if row['projection_credit_transferred'] else int(row['reserved_projection_items'])
|
|
projection_bytes = 0 if row['projection_credit_transferred'] else int(row['reserved_projection_bytes'])
|
|
candidate_items = 0 if row['candidate_credit_transferred'] else int(row['reserved_candidate_items'])
|
|
candidate_bytes = 0 if row['candidate_credit_transferred'] else int(row['reserved_candidate_bytes'])
|
|
bundle_items = 0 if row['bundle_credit_released'] else 1
|
|
bundle_bytes = 0 if row['bundle_credit_released'] else int(row['reserved_bundle_bytes'])
|
|
if (
|
|
int(capacity['bundle_items']) < bundle_items
|
|
or int(capacity['bundle_bytes']) < bundle_bytes
|
|
or int(capacity['projection_items']) < projection_items
|
|
or int(capacity['projection_bytes']) < projection_bytes
|
|
or int(capacity['keycheck_items']) < candidate_items
|
|
or int(capacity['keycheck_bytes']) < candidate_bytes
|
|
):
|
|
raise RuntimeError('bundle quarantine would make capacity accounting negative')
|
|
quarantine_capacity_items = bundle_items + projection_items + candidate_items
|
|
quarantine_capacity_bytes = bundle_bytes + projection_bytes + candidate_bytes
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
bundle_items = bundle_items - ?, bundle_bytes = bundle_bytes - ?,
|
|
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
|
|
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
|
|
quarantine_items = quarantine_items + ?,
|
|
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
bundle_items, bundle_bytes, projection_items, projection_bytes,
|
|
candidate_items, candidate_bytes, quarantine_capacity_items,
|
|
quarantine_capacity_bytes, now,
|
|
),
|
|
)
|
|
quarantine_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_quarantine(
|
|
subsystem, object_type, object_id, reservation_id, event_id,
|
|
payload_sha256, reason_code, reason_detail, source_relative_path,
|
|
byte_count, capacity_items, capacity_bytes, detected_at
|
|
) VALUES ('result_ingester', 'result_bundle', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
reservation_id, reservation_id, row['scan_event_id'], payload_sha256,
|
|
reason_code, reason_detail, source_relative_path,
|
|
byte_count, quarantine_capacity_items,
|
|
quarantine_capacity_bytes, now,
|
|
),
|
|
)
|
|
if source_relative_path:
|
|
self.conn.execute(
|
|
'''INSERT INTO pipeline_artifacts(
|
|
subsystem, artifact_kind, owner_id, owner_key, relative_path,
|
|
payload_sha256, byte_count, state, created_at, updated_at
|
|
) VALUES ('result_bundle','bundle_quarantine',?,'',?,?,?,
|
|
'expected',?,?)
|
|
ON CONFLICT(subsystem, artifact_kind, owner_id, owner_key)
|
|
DO UPDATE SET relative_path = excluded.relative_path,
|
|
payload_sha256 = excluded.payload_sha256,
|
|
byte_count = excluded.byte_count,
|
|
deleted_at = NULL, updated_at = excluded.updated_at''',
|
|
(
|
|
reservation_id, source_relative_path,
|
|
payload_sha256, byte_count, now, now,
|
|
),
|
|
)
|
|
if physical_confirmed:
|
|
self._finalize_result_bundle_quarantine_locked(
|
|
row, reason_code, reason_detail, source_relative_path,
|
|
payload_sha256, byte_count, now,
|
|
)
|
|
self.conn.commit()
|
|
return int(quarantine_id)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def pipeline_capacity_snapshot(self):
|
|
if not self.conn:
|
|
return {}
|
|
row = self.conn.execute('SELECT * FROM pipeline_capacity WHERE id = 1').fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return dict(row) if row else {}
|
|
|
|
@staticmethod
|
|
def _artifact_relative_path(relative_path):
|
|
value = str(relative_path or '').replace('\\', '/').strip('/')
|
|
if not value or value.startswith('/') or any(part in ('', '.', '..') for part in value.split('/')):
|
|
raise ValueError('pipeline artifact path is not a canonical relative path')
|
|
return value
|
|
|
|
def register_pipeline_artifact(
|
|
self, subsystem, artifact_kind, owner_id, owner_key, relative_path,
|
|
*, state='expected', payload_sha256='', byte_count=0,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('pipeline artifact registration requires PostgreSQL')
|
|
if state not in ('expected', 'present', 'quarantined'):
|
|
raise ValueError('pipeline artifact registration state is invalid')
|
|
relative_path = self._artifact_relative_path(relative_path)
|
|
payload_sha256 = str(payload_sha256 or '').lower()
|
|
if payload_sha256 and not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
|
|
raise ValueError('pipeline artifact payload identity is invalid')
|
|
now = utc_now_iso()
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM pipeline_artifacts
|
|
WHERE subsystem = ? AND artifact_kind = ?
|
|
AND owner_id = ? AND owner_key = ? FOR UPDATE''',
|
|
(str(subsystem), str(artifact_kind), int(owner_id), str(owner_key or '')),
|
|
).fetchone()
|
|
if row and str(row['relative_path']) != relative_path:
|
|
raise ScanEventConflictError('pipeline artifact owner resolves to a conflicting path')
|
|
if row and row['payload_sha256'] and payload_sha256 and row['payload_sha256'] != payload_sha256:
|
|
raise ScanEventConflictError('pipeline artifact owner resolves to conflicting bytes')
|
|
if row:
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET payload_sha256 = ?, byte_count = ?,
|
|
state = ?, deleted_at = NULL, updated_at = ? WHERE id = ?''',
|
|
(
|
|
payload_sha256 or row['payload_sha256'], max(0, int(byte_count)),
|
|
state, now, row['id'],
|
|
),
|
|
)
|
|
artifact_id = int(row['id'])
|
|
else:
|
|
artifact_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_artifacts(
|
|
subsystem, artifact_kind, owner_id, owner_key, relative_path,
|
|
payload_sha256, byte_count, state, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
str(subsystem), str(artifact_kind), int(owner_id), str(owner_key or ''),
|
|
relative_path, payload_sha256, max(0, int(byte_count)), state, now, now,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return int(artifact_id)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def mark_pipeline_artifact_deleted(self, artifact_id):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('pipeline artifact deletion requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, cleanup_attempts = 0, cleanup_available_after = NULL,
|
|
cleanup_last_error = NULL, updated_at = ?
|
|
WHERE id = ? AND state != 'deleted' ''',
|
|
(now, now, int(artifact_id)),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
|
|
def defer_pipeline_artifact_cleanup(self, artifact_id, error):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return False
|
|
now = utc_now_iso()
|
|
try:
|
|
row = self.conn.execute(
|
|
'SELECT cleanup_attempts FROM pipeline_artifacts WHERE id = ? FOR UPDATE',
|
|
(int(artifact_id),),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return False
|
|
attempts = int(row['cleanup_attempts'] or 0) + 1
|
|
delay = min(300, 2 ** min(attempts, 8))
|
|
available = datetime.fromtimestamp(
|
|
time.time() + delay, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET cleanup_attempts = ?,
|
|
cleanup_available_after = ?, cleanup_last_error = ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(attempts, available, first_line(error, 1000), now, int(artifact_id)),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def projection_terminal_temp_artifacts(self, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return []
|
|
rows = self.conn.execute(
|
|
'''SELECT a.* FROM pipeline_artifacts a
|
|
JOIN projection_jobs j ON j.id = a.owner_id
|
|
WHERE a.subsystem = 'jsonl_projector'
|
|
AND a.artifact_kind IN ('prepared_stream','projection_tail_temp')
|
|
AND a.state IN ('expected','present')
|
|
AND j.status IN ('completed','quarantined')
|
|
ORDER BY a.id LIMIT ?''',
|
|
(min(1000, max(1, int(limit))),),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return [dict(row) for row in rows]
|
|
|
|
def bundle_terminal_temp_artifacts(self, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return []
|
|
rows = self.conn.execute(
|
|
'''SELECT a.* FROM pipeline_artifacts a
|
|
JOIN result_reservations r ON r.id = a.owner_id
|
|
WHERE a.subsystem = 'result_bundle'
|
|
AND a.artifact_kind IN ('bundle_partial','bundle_ready')
|
|
AND a.state IN ('expected','present')
|
|
AND r.state IN ('acknowledged','refunded','quarantined')
|
|
AND (a.cleanup_available_after IS NULL OR a.cleanup_available_after <= ?)
|
|
ORDER BY a.id LIMIT ?''',
|
|
(utc_now_iso(), min(1000, max(1, int(limit)))),
|
|
).fetchall()
|
|
self.conn.commit()
|
|
return [dict(row) for row in rows]
|
|
|
|
def register_projection_tail_quarantine(
|
|
self, job_id, append_id, stream_name, relative_path, payload_sha256,
|
|
byte_count, max_items, max_bytes,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection tail registration requires PostgreSQL')
|
|
relative_path = self._artifact_relative_path(relative_path)
|
|
payload_sha256 = str(payload_sha256 or '').lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', payload_sha256):
|
|
raise ValueError('projection tail payload identity is invalid')
|
|
owner_key = f'{int(append_id)}:{stream_name}:{payload_sha256}'
|
|
now = utc_now_iso()
|
|
try:
|
|
artifact = self.conn.execute(
|
|
'''SELECT * FROM pipeline_artifacts
|
|
WHERE subsystem = 'jsonl_projector' AND artifact_kind = 'partial_tail'
|
|
AND owner_id = ? AND owner_key = ? FOR UPDATE''',
|
|
(int(job_id), owner_key),
|
|
).fetchone()
|
|
if artifact and (
|
|
artifact['relative_path'] != relative_path
|
|
or artifact['payload_sha256'] != payload_sha256
|
|
or int(artifact['byte_count']) != int(byte_count)
|
|
):
|
|
raise ScanEventConflictError('projection tail artifact identity conflicts')
|
|
if not artifact:
|
|
artifact_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_artifacts(
|
|
subsystem, artifact_kind, owner_id, owner_key, relative_path,
|
|
payload_sha256, byte_count, state, created_at, updated_at
|
|
) VALUES ('jsonl_projector','partial_tail',?,?,?,?,?,'expected',?,?)''',
|
|
(
|
|
int(job_id), owner_key, relative_path, payload_sha256,
|
|
max(0, int(byte_count)), now, now,
|
|
),
|
|
)
|
|
else:
|
|
artifact_id = int(artifact['id'])
|
|
quarantine = self.conn.execute(
|
|
'''SELECT id FROM pipeline_quarantine
|
|
WHERE subsystem = 'jsonl_projector' AND object_type = 'projection_tail'
|
|
AND object_id = ? AND review_status = 'pending' FOR UPDATE''',
|
|
(artifact_id,),
|
|
).fetchone()
|
|
if not quarantine:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items + 1,
|
|
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(int(byte_count), now),
|
|
)
|
|
quarantine_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_quarantine(
|
|
subsystem, object_type, object_id, projection_job_id,
|
|
payload_sha256, reason_code, reason_detail, source_relative_path,
|
|
byte_count, capacity_items, capacity_bytes, detected_at
|
|
) VALUES ('jsonl_projector','projection_tail',?,?,?,?,?,?,?,?,?,?)''',
|
|
(
|
|
artifact_id, int(job_id), payload_sha256, 'partial_projection_tail',
|
|
f'append={int(append_id)} stream={stream_name}', relative_path,
|
|
int(byte_count), 1, int(byte_count), now,
|
|
),
|
|
)
|
|
else:
|
|
quarantine_id = int(quarantine['id'])
|
|
self.conn.commit()
|
|
return {'artifact_id': int(artifact_id), 'quarantine_id': int(quarantine_id)}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def confirm_projection_tail_artifact(self, artifact_id, payload_sha256, byte_count):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection tail confirmation requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'quarantined', payload_sha256 = ?,
|
|
byte_count = ?, updated_at = ?
|
|
WHERE id = ? AND subsystem = 'jsonl_projector'
|
|
AND artifact_kind = 'partial_tail'
|
|
AND payload_sha256 = ? AND byte_count = ?
|
|
AND state IN ('expected','quarantined')''',
|
|
(
|
|
str(payload_sha256), int(byte_count), now, int(artifact_id),
|
|
str(payload_sha256), int(byte_count),
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return int(cursor.rowcount or 0) == 1
|
|
|
|
def review_pipeline_quarantine(
|
|
self, quarantine_id, expected_reason_code, expected_payload_sha256,
|
|
action, audit_sha256,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('pipeline quarantine review requires PostgreSQL')
|
|
action = str(action or '').lower()
|
|
if action not in ('discard', 'rescan', 'retry'):
|
|
raise ValueError('quarantine review action must be discard, rescan, or retry')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', str(audit_sha256 or '')):
|
|
raise ValueError('quarantine review audit identity is invalid')
|
|
now = utc_now_iso()
|
|
try:
|
|
preview = self.conn.execute(
|
|
'''SELECT object_type, reservation_id FROM pipeline_quarantine
|
|
WHERE id = ?''',
|
|
(int(quarantine_id),),
|
|
).fetchone()
|
|
experiment = None
|
|
reservation = None
|
|
if (
|
|
preview and str(preview['object_type']) == 'result_bundle'
|
|
and preview['reservation_id'] is not None
|
|
):
|
|
experiment = self._lock_docker_depth_experiment_for_reservation(
|
|
preview['reservation_id']
|
|
)
|
|
reservation = self.conn.execute(
|
|
'SELECT * FROM result_reservations WHERE id = ? FOR UPDATE',
|
|
(preview['reservation_id'],),
|
|
).fetchone()
|
|
row = self.conn.execute(
|
|
'SELECT * FROM pipeline_quarantine WHERE id = ? FOR UPDATE',
|
|
(int(quarantine_id),),
|
|
).fetchone()
|
|
if not row:
|
|
raise ValueError('pipeline quarantine row is absent')
|
|
if row['review_status'] != 'pending':
|
|
if row['review_audit_sha256'] == audit_sha256:
|
|
self.conn.commit()
|
|
return {'reviewed': True, 'duplicate': True, 'status': row['review_status']}
|
|
raise ScanEventConflictError('pipeline quarantine row was already reviewed differently')
|
|
if (
|
|
str(row['reason_code']) != str(expected_reason_code)
|
|
or str(row['payload_sha256'] or '') != str(expected_payload_sha256 or '')
|
|
):
|
|
raise ScanEventConflictError('pipeline quarantine review evidence does not match')
|
|
if row['object_type'] == 'result_bundle':
|
|
if not preview or (
|
|
str(preview['object_type']) != str(row['object_type'])
|
|
or int(preview['reservation_id'] or 0)
|
|
!= int(row['reservation_id'] or 0)
|
|
):
|
|
raise ScanEventConflictError(
|
|
'pipeline quarantine review identity changed before locking'
|
|
)
|
|
bundle = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(row['reservation_id'],),
|
|
).fetchone()
|
|
if not reservation or reservation['state'] != 'quarantined':
|
|
raise RuntimeError('bundle quarantine review requires terminal quarantined reservation state')
|
|
if bundle and bundle['state'] != 'quarantined':
|
|
raise RuntimeError('bundle quarantine review requires terminal quarantined bundle state')
|
|
elif row['object_type'] == 'projection_job':
|
|
job_state = self.conn.execute(
|
|
'SELECT status FROM projection_jobs WHERE id = ? FOR UPDATE',
|
|
(row['projection_job_id'],),
|
|
).fetchone()
|
|
if not job_state or job_state['status'] != 'quarantined':
|
|
raise RuntimeError('projection review requires terminal quarantined job state')
|
|
elif row['object_type'] == 'keycheck_candidate':
|
|
candidate_state = self.conn.execute(
|
|
'SELECT * FROM keycheck_candidates WHERE id = ? FOR UPDATE',
|
|
(row['keycheck_candidate_id'],),
|
|
).fetchone()
|
|
if not candidate_state or candidate_state['state'] != 'quarantined':
|
|
raise RuntimeError('keycheck review requires terminal quarantined candidate state')
|
|
if action in ('discard', 'rescan') and row['source_relative_path']:
|
|
artifact = None
|
|
if row['object_type'] == 'projection_tail':
|
|
artifact = self.conn.execute(
|
|
'SELECT state FROM pipeline_artifacts WHERE id = ? FOR UPDATE',
|
|
(row['object_id'],),
|
|
).fetchone()
|
|
elif row['object_type'] == 'result_bundle':
|
|
artifact = self.conn.execute(
|
|
'''SELECT state FROM pipeline_artifacts
|
|
WHERE subsystem = 'result_bundle'
|
|
AND artifact_kind = 'bundle_quarantine' AND owner_id = ? FOR UPDATE''',
|
|
(row['reservation_id'],),
|
|
).fetchone()
|
|
if not artifact or artifact['state'] != 'deleted':
|
|
raise RuntimeError(
|
|
'physical quarantine artifact must be absent before capacity credit release'
|
|
)
|
|
if action == 'retry':
|
|
if not row['capacity_credit_applied']:
|
|
raise RuntimeError('quarantine retry requires exact applied capacity credit')
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['quarantine_items']) < int(row['capacity_items'])
|
|
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
|
|
):
|
|
raise RuntimeError('quarantine retry would make capacity accounting negative')
|
|
if row['object_type'] == 'projection_job':
|
|
job = self.conn.execute(
|
|
'SELECT * FROM projection_jobs WHERE id = ? FOR UPDATE',
|
|
(row['projection_job_id'],),
|
|
).fetchone()
|
|
if not job or job['status'] != 'quarantined' or not job['capacity_released']:
|
|
raise RuntimeError('projection quarantine retry state is not exact')
|
|
prepared = self.conn.execute(
|
|
"SELECT 1 FROM projection_appends WHERE job_id = ? AND state = 'prepared' LIMIT 1",
|
|
(job['id'],),
|
|
).fetchone()
|
|
if prepared:
|
|
raise RuntimeError('projection quarantine retry still has a prepared append')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
projection_items = projection_items + ?,
|
|
projection_bytes = projection_bytes + ?,
|
|
quarantine_items = quarantine_items - ?,
|
|
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
job['capacity_items'], job['capacity_bytes'],
|
|
row['capacity_items'], row['capacity_bytes'], now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE projection_jobs SET status = 'pending', capacity_released = 0,
|
|
available_after = NULL, lease_generation = NULL, lease_token = NULL,
|
|
lease_expires_at = NULL, last_error_code = NULL,
|
|
last_error_detail = NULL, completed_at = NULL, updated_at = ?
|
|
WHERE id = ?''',
|
|
(now, job['id']),
|
|
)
|
|
elif row['object_type'] == 'keycheck_candidate':
|
|
candidate = candidate_state
|
|
active = self.conn.execute(
|
|
'''SELECT 1 FROM keycheck_candidates
|
|
WHERE credential_id = ? AND id <> ?
|
|
AND state IN ('pending','deferred','leased') LIMIT 1 FOR UPDATE''',
|
|
(candidate['credential_id'], candidate['id']),
|
|
).fetchone()
|
|
if active:
|
|
raise RuntimeError('keycheck quarantine retry conflicts with another active credential candidate')
|
|
candidate_items = 1
|
|
candidate_bytes = int(candidate['capacity_bytes'])
|
|
if (
|
|
int(row['capacity_items']) < candidate_items
|
|
or int(row['capacity_bytes']) < candidate_bytes
|
|
):
|
|
raise RuntimeError('keycheck quarantine retry has insufficient capacity credit')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
keycheck_items = keycheck_items + ?,
|
|
keycheck_bytes = keycheck_bytes + ?,
|
|
quarantine_items = quarantine_items - ?,
|
|
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
candidate_items, candidate_bytes,
|
|
row['capacity_items'], row['capacity_bytes'], now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET state = 'pending', attempts = 0,
|
|
available_after = NULL, capacity_released = 0,
|
|
result_projection_credit_transferred = 0,
|
|
result_projection_reserved_bytes = 0,
|
|
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
|
|
last_error = NULL, completed_at = NULL, updated_at = ?
|
|
WHERE id = ?''',
|
|
(now, candidate['id']),
|
|
)
|
|
elif row['object_type'] == 'result_bundle':
|
|
if reservation['docker_layer_plan_json'] is not None:
|
|
raise ValueError(
|
|
'Docker layer bundle quarantine requires a fresh parent claim'
|
|
)
|
|
if not reservation['bundle_credit_released']:
|
|
raise RuntimeError('bundle quarantine retry requires released bundle capacity')
|
|
bundle_items = 1
|
|
bundle_bytes = int(reservation['reserved_bundle_bytes'])
|
|
projection_items = (
|
|
0 if reservation['projection_credit_transferred']
|
|
else int(reservation['reserved_projection_items'])
|
|
)
|
|
projection_bytes = (
|
|
0 if reservation['projection_credit_transferred']
|
|
else int(reservation['reserved_projection_bytes'])
|
|
)
|
|
candidate_items = (
|
|
0 if reservation['candidate_credit_transferred']
|
|
else int(reservation['reserved_candidate_items'])
|
|
)
|
|
candidate_bytes = (
|
|
0 if reservation['candidate_credit_transferred']
|
|
else int(reservation['reserved_candidate_bytes'])
|
|
)
|
|
expected_items = bundle_items + projection_items + candidate_items
|
|
expected_bytes = bundle_bytes + projection_bytes + candidate_bytes
|
|
if (
|
|
int(row['capacity_items']) != expected_items
|
|
or int(row['capacity_bytes']) != expected_bytes
|
|
):
|
|
raise RuntimeError('bundle quarantine retry capacity evidence is inconsistent')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
bundle_items = bundle_items + ?, bundle_bytes = bundle_bytes + ?,
|
|
projection_items = projection_items + ?, projection_bytes = projection_bytes + ?,
|
|
keycheck_items = keycheck_items + ?, keycheck_bytes = keycheck_bytes + ?,
|
|
quarantine_items = quarantine_items - ?,
|
|
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
bundle_items, bundle_bytes, projection_items, projection_bytes,
|
|
candidate_items, candidate_bytes,
|
|
row['capacity_items'], row['capacity_bytes'], now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'scanning',
|
|
bundle_credit_released = 0, released_at = NULL,
|
|
cleanup_attempts = 0, cleanup_available_after = NULL,
|
|
last_error_code = NULL, last_error_detail = NULL, updated_at = ?
|
|
WHERE id = ?''',
|
|
(now, reservation['id']),
|
|
)
|
|
if bundle:
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = 'ready',
|
|
relative_path = ?, available_after = NULL,
|
|
ingest_lease_generation = NULL, ingest_lease_token = NULL,
|
|
ingest_lease_expires_at = NULL, updated_at = ?
|
|
WHERE reservation_id = ?''',
|
|
(reservation['ready_relative_path'], now, reservation['id']),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'in_progress', completed_at = NULL,
|
|
last_error = NULL, lease_owner = ?, lease_token = ?,
|
|
claim_batch = ?, leased_at = ?, lease_expires_at = ?,
|
|
claim_event_id = ?, updated_at = ?
|
|
WHERE id = ? AND current_result_reservation_id = ?''',
|
|
(
|
|
reservation['claim_lease_owner'], reservation['claim_lease_token'],
|
|
reservation['claim_batch'], now, reservation['producer_lease_expires_at'],
|
|
reservation['scan_event_id'], now,
|
|
reservation['queue_id'], reservation['id'],
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'deleted', byte_count = 0,
|
|
deleted_at = ?, updated_at = ?
|
|
WHERE subsystem = 'result_bundle'
|
|
AND artifact_kind = 'bundle_quarantine' AND owner_id = ?''',
|
|
(now, now, reservation['id']),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifacts SET state = 'present',
|
|
relative_path = ?, payload_sha256 = ?, byte_count = ?,
|
|
deleted_at = NULL, updated_at = ?
|
|
WHERE subsystem = 'result_bundle'
|
|
AND artifact_kind = 'bundle_ready' AND owner_id = ?''',
|
|
(
|
|
reservation['ready_relative_path'], row['payload_sha256'] or '',
|
|
int(row['byte_count']), now, reservation['id'],
|
|
),
|
|
)
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'reserved', 'reserved', now,
|
|
)
|
|
else:
|
|
raise ValueError('quarantine object type does not support deterministic retry')
|
|
review_status = 'approved_retry'
|
|
elif action == 'rescan':
|
|
if (
|
|
row['object_type'] != 'result_bundle'
|
|
or reservation['docker_layer_plan_json'] is None
|
|
or str(reservation['source']) != 'dockerhub'
|
|
or str(reservation['platform']) != 'docker'
|
|
):
|
|
raise ValueError('quarantine rescan requires a Docker layer result bundle')
|
|
if not row['capacity_credit_applied']:
|
|
raise RuntimeError('quarantine rescan requires exact applied capacity credit')
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['quarantine_items']) < int(row['capacity_items'])
|
|
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
|
|
):
|
|
raise RuntimeError('quarantine rescan would make capacity accounting negative')
|
|
queue = self.conn.execute(
|
|
'SELECT * FROM target_queue WHERE id = ? FOR UPDATE',
|
|
(reservation['queue_id'],),
|
|
).fetchone()
|
|
if not queue or (
|
|
str(queue['source']) != 'dockerhub'
|
|
or str(queue['platform']) != 'docker'
|
|
or str(queue['status']) != 'quarantined'
|
|
or int(queue['current_result_reservation_id'] or 0) != int(reservation['id'])
|
|
or str(queue['claim_event_id'] or '') != str(reservation['scan_event_id'])
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker quarantine rescan lost its exact queue fence'
|
|
)
|
|
active_blob = self.conn.execute(
|
|
'''SELECT 1 FROM docker_content_blobs
|
|
WHERE lease_reservation_id = ? LIMIT 1 FOR UPDATE''',
|
|
(reservation['id'],),
|
|
).fetchone()
|
|
if active_blob:
|
|
raise RuntimeError('Docker quarantine rescan retains active content work')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items - ?,
|
|
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(row['capacity_items'], row['capacity_bytes'], now),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending', attempts = 0,
|
|
available_after = NULL, completed_at = NULL, last_error = NULL,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL,
|
|
current_result_reservation_id = NULL, claim_event_id = NULL,
|
|
updated_at = ?
|
|
WHERE id = ? AND status = 'quarantined'
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
now, reservation['queue_id'], reservation['id'],
|
|
reservation['scan_event_id'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker quarantine rescan queue transition lost its fence'
|
|
)
|
|
self._transition_docker_depth_binding_locked(
|
|
reservation, 'quarantined', 'pending', now,
|
|
)
|
|
review_status = 'approved_rescan'
|
|
else:
|
|
if row['capacity_credit_applied']:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['quarantine_items']) < int(row['capacity_items'])
|
|
or int(capacity['quarantine_bytes']) < int(row['capacity_bytes'])
|
|
):
|
|
raise RuntimeError('quarantine discard would make capacity accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET quarantine_items = quarantine_items - ?,
|
|
quarantine_bytes = quarantine_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(row['capacity_items'], row['capacity_bytes'], now),
|
|
)
|
|
review_status = 'discarded'
|
|
if experiment and action in ('retry', 'rescan'):
|
|
self._resume_docker_depth_after_quarantine_review_locked(
|
|
experiment, now,
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_quarantine SET review_status = ?, resolved_at = ?,
|
|
review_audit_sha256 = ? WHERE id = ?''',
|
|
(review_status, now, audit_sha256, row['id']),
|
|
)
|
|
self.conn.commit()
|
|
return {'reviewed': True, 'duplicate': False, 'status': review_status}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _resume_docker_depth_after_quarantine_review_locked(self, experiment, now):
|
|
if not experiment:
|
|
return
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ? FOR UPDATE',
|
|
(int(experiment['id']),),
|
|
).fetchone()
|
|
if not experiment:
|
|
raise ScanEventConflictError(
|
|
'Docker depth quarantine review lost its experiment authority'
|
|
)
|
|
state = str(experiment['state'])
|
|
if state in ('completed', 'released'):
|
|
raise ScanEventConflictError(
|
|
'Docker depth terminal experiment cannot reopen quarantined work'
|
|
)
|
|
if state == 'held':
|
|
if str(experiment['hold_reason_code'] or '') != 'scan_target_held':
|
|
return
|
|
remaining = self.conn.execute(
|
|
'''SELECT 1 FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ? AND state IN ('held','quarantined')
|
|
LIMIT 1 FOR UPDATE''',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
if remaining:
|
|
return
|
|
elif state != 'draining':
|
|
return
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'active', hold_reason_code = NULL, held_at = NULL,
|
|
draining_at = NULL, updated_at = ?
|
|
WHERE id = ? AND state = ?''',
|
|
(now, experiment['id'], state),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth quarantine review lost its resume fence'
|
|
)
|
|
|
|
def pipeline_quarantine_for_review(self, quarantine_id):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('pipeline quarantine review lookup requires PostgreSQL')
|
|
row = self.conn.execute(
|
|
'SELECT * FROM pipeline_quarantine WHERE id = ?', (int(quarantine_id),),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def _hold_docker_depth_experiment_locked(self, experiment, reason, now=None):
|
|
now = str(now or utc_now_iso())
|
|
reason = first_line(reason, 128)
|
|
if not experiment:
|
|
return {'status': 'unavailable', 'committed': False, 'reason': reason}
|
|
experiment_id = int(experiment['id'])
|
|
stable_reason = str(experiment['hold_reason_code'] or reason)
|
|
if str(experiment['state']) != 'released':
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = CASE WHEN work_state = 'resolving' THEN 'pending'
|
|
ELSE work_state END,
|
|
resolver_owner = NULL, resolver_token = NULL,
|
|
resolver_expires_at = NULL, resolver_due_at = NULL,
|
|
updated_at = ?
|
|
WHERE experiment_id = ? AND work_state = 'resolving' ''',
|
|
(now, experiment_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'held', hold_reason_code = COALESCE(hold_reason_code, ?),
|
|
held_at = COALESCE(held_at, ?), updated_at = ?
|
|
WHERE id = ? AND state <> 'released' ''',
|
|
(reason, now, now, experiment_id),
|
|
)
|
|
return {
|
|
'status': 'held', 'committed': True, 'reason': stable_reason,
|
|
'experiment_id': experiment_id,
|
|
}
|
|
|
|
@staticmethod
|
|
def _docker_depth_authority_drift_reason(experiment, authority):
|
|
if not experiment:
|
|
return 'experiment_authority_absent'
|
|
comparisons = (
|
|
('experiment_key', 'experiment_key', 'experiment_identity_drift'),
|
|
('source', 'source', 'experiment_identity_drift'),
|
|
('collection_generation', 'collection_generation', 'collection_generation_drift'),
|
|
('config_sha256', 'config_sha256', 'config_hash_drift'),
|
|
('ordered_queries_sha256', 'ordered_queries_sha256', 'ordered_query_drift'),
|
|
('selector_version', 'selector_version', 'selector_drift'),
|
|
('selector_sha256', 'selector_sha256', 'selector_drift'),
|
|
('provenance_policy_sha256', 'provenance_policy_sha256', 'provenance_policy_drift'),
|
|
)
|
|
for column, key, reason in comparisons:
|
|
if str(experiment[column]) != str(authority[key]):
|
|
return reason
|
|
for column, key in (
|
|
('query_count', 'query_count'),
|
|
('repositories_per_query', 'repositories_per_query'),
|
|
('images_per_repository', 'images_per_repository'),
|
|
('target_limit', 'target_limit'),
|
|
):
|
|
if int(experiment[column]) != int(authority[key]):
|
|
return 'capacity_limit_drift'
|
|
return None
|
|
|
|
@staticmethod
|
|
def _docker_depth_candidate_skip_evidence(
|
|
experiment_id, experiment_repository_id, repository_queue_id,
|
|
candidate_kind, candidate_ordinal, candidate_identity, reason,
|
|
):
|
|
candidate_identity_sha256 = hashlib.sha256(json.dumps(
|
|
candidate_identity, ensure_ascii=True, allow_nan=False,
|
|
sort_keys=True, separators=(',', ':'),
|
|
).encode('utf-8')).hexdigest()
|
|
evidence = {
|
|
'schema': 1,
|
|
'type': 'docker-depth-candidate-skip-v1',
|
|
'experiment_id': int(experiment_id),
|
|
'experiment_repository_id': int(experiment_repository_id),
|
|
'repository_queue_id': int(repository_queue_id),
|
|
'candidate_kind': str(candidate_kind),
|
|
'candidate_ordinal': int(candidate_ordinal),
|
|
'candidate_identity_sha256': candidate_identity_sha256,
|
|
'reason_code': str(reason),
|
|
}
|
|
evidence_sha256 = hashlib.sha256(json.dumps(
|
|
evidence, ensure_ascii=True, allow_nan=False, sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')).hexdigest()
|
|
return candidate_identity_sha256, evidence_sha256
|
|
|
|
def _docker_depth_terminal_repository_evidence_locked(
|
|
self, experiment, member,
|
|
):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
|
|
DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
|
|
)
|
|
|
|
state = str(member['work_state'])
|
|
resolver_fields = (
|
|
'resolver_owner', 'resolver_token', 'resolver_expires_at',
|
|
'resolver_due_at',
|
|
)
|
|
if state == 'resolved':
|
|
if (
|
|
int(member['selected_image_count']) < 1
|
|
or not member['resolved_at']
|
|
or member['last_error_code'] is not None
|
|
or any(member[name] is not None for name in resolver_fields)
|
|
):
|
|
return 'repository_state_drift', None
|
|
return None, None
|
|
if state != 'skipped':
|
|
return None, None
|
|
current_queue_id = int(
|
|
member['replacement_repository_queue_id']
|
|
or member['repository_queue_id']
|
|
)
|
|
skip_reason = str(member['last_error_code'] or '')
|
|
if (
|
|
int(member['selected_image_count']) != 0
|
|
or skip_reason not in (
|
|
DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
|
|
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
|
|
)
|
|
or not member['resolved_at']
|
|
or any(member[name] is not None for name in resolver_fields)
|
|
):
|
|
return 'repository_skip_evidence_drift', None
|
|
if self.conn.execute(
|
|
'''SELECT 1 FROM docker_depth_experiment_selections
|
|
WHERE experiment_repository_id = ? LIMIT 1''',
|
|
(member['id'],),
|
|
).fetchone():
|
|
return 'repository_skip_evidence_drift', None
|
|
rows = self.conn.execute(
|
|
'''SELECT candidate_ordinal, candidate_identity_sha256,
|
|
evidence_sha256
|
|
FROM docker_depth_experiment_candidate_skips
|
|
WHERE experiment_id = ? AND experiment_repository_id = ?
|
|
AND repository_queue_id = ? AND candidate_kind = 'repository'
|
|
AND reason_code = ?
|
|
ORDER BY id LIMIT 2''',
|
|
(
|
|
experiment['id'], member['id'], current_queue_id,
|
|
skip_reason,
|
|
),
|
|
).fetchall()
|
|
expected_ordinal = int(member['replacement_count']) + 1
|
|
identity_sha256, evidence_sha256 = self._docker_depth_candidate_skip_evidence(
|
|
experiment['id'], member['id'], current_queue_id, 'repository',
|
|
expected_ordinal, {'repository_queue_id': current_queue_id},
|
|
skip_reason,
|
|
)
|
|
if (
|
|
len(rows) != 1
|
|
or int(rows[0]['candidate_ordinal']) != expected_ordinal
|
|
or str(rows[0]['candidate_identity_sha256']) != identity_sha256
|
|
or str(rows[0]['evidence_sha256']) != evidence_sha256
|
|
):
|
|
return 'repository_skip_evidence_drift', None
|
|
return None, evidence_sha256
|
|
|
|
def _docker_depth_runtime_selection_sha256_locked(self, experiment, authority):
|
|
from docker_depth_experiment import DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
|
|
rank1_breadth = (
|
|
authority.get('selector_version') == DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
)
|
|
query_rows = self.conn.execute(
|
|
'''SELECT query_ordinal, required_repository_count,
|
|
selected_repository_count
|
|
FROM docker_depth_experiment_queries
|
|
WHERE experiment_id = ? ORDER BY query_ordinal''',
|
|
(experiment['id'],),
|
|
).fetchall()
|
|
if len(query_rows) != authority['query_count']:
|
|
raise ScanEventConflictError('Docker depth runtime query selection is incomplete')
|
|
selected_counts = []
|
|
for query_ordinal, row in enumerate(query_rows):
|
|
selected_count = int(row['selected_repository_count'])
|
|
if (
|
|
int(row['query_ordinal']) != query_ordinal
|
|
or int(row['required_repository_count'])
|
|
!= authority['repositories_per_query']
|
|
or not 0 <= selected_count <= authority['repositories_per_query']
|
|
):
|
|
raise ScanEventConflictError('Docker depth runtime query selection is invalid')
|
|
selected_counts.append(selected_count)
|
|
rows = self.conn.execute(
|
|
'''SELECT id, query_ordinal, repository_rank, repository_queue_id,
|
|
replacement_repository_queue_id,
|
|
replacement_eligibility_page_id, replacement_count,
|
|
replacement_evidence_sha256, is_deep_probe,
|
|
work_state, selected_image_count, last_error_code,
|
|
resolved_at, resolver_owner, resolver_token,
|
|
resolver_expires_at, resolver_due_at
|
|
FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ?
|
|
ORDER BY query_ordinal, repository_rank, id''',
|
|
(experiment['id'],),
|
|
).fetchall()
|
|
expected = sum(selected_counts)
|
|
if len(rows) != expected:
|
|
raise ScanEventConflictError('Docker depth runtime selection is incomplete')
|
|
document = {
|
|
'schema': 2,
|
|
'type': 'docker-depth-runtime-selection-v2',
|
|
'experiment_id': int(experiment['id']),
|
|
'plan_sha256': str(experiment['plan_sha256'] or ''),
|
|
'collection_generation': authority['collection_generation'],
|
|
'queries': [],
|
|
}
|
|
for query_ordinal in range(authority['query_count']):
|
|
members = [
|
|
row for row in rows if int(row['query_ordinal']) == query_ordinal
|
|
]
|
|
terminal_evidence = {}
|
|
for row in members:
|
|
reason, evidence_sha256 = (
|
|
self._docker_depth_terminal_repository_evidence_locked(
|
|
experiment, row,
|
|
)
|
|
)
|
|
if reason:
|
|
raise ScanEventConflictError(
|
|
'Docker depth runtime repository evidence is invalid'
|
|
)
|
|
terminal_evidence[int(row['id'])] = evidence_sha256
|
|
image_bearing = sum(
|
|
1 for row in members
|
|
if str(row['work_state']) == 'resolved'
|
|
and int(row['selected_image_count']) >= 1
|
|
)
|
|
expected_deep_probe_count = 0 if rank1_breadth else (1 if image_bearing else 0)
|
|
if (
|
|
len(members) != selected_counts[query_ordinal]
|
|
or [int(row['repository_rank']) for row in members]
|
|
!= list(range(1, selected_counts[query_ordinal] + 1))
|
|
or sum(int(row['is_deep_probe']) for row in members)
|
|
!= expected_deep_probe_count
|
|
or any(
|
|
str(row['work_state']) not in ('resolved', 'skipped')
|
|
for row in members
|
|
)
|
|
):
|
|
raise ScanEventConflictError('Docker depth runtime selection is invalid')
|
|
document['queries'].append({
|
|
'query_ordinal': query_ordinal,
|
|
'selected_repository_count': selected_counts[query_ordinal],
|
|
'repositories': [{
|
|
'member_id': int(row['id']),
|
|
'repository_rank': int(row['repository_rank']),
|
|
'planned_repository_queue_id': int(row['repository_queue_id']),
|
|
'selected_repository_queue_id': int(
|
|
row['replacement_repository_queue_id']
|
|
or row['repository_queue_id']
|
|
),
|
|
'replacement_eligibility_page_id': (
|
|
int(row['replacement_eligibility_page_id'])
|
|
if row['replacement_eligibility_page_id'] is not None else None
|
|
),
|
|
'replacement_count': int(row['replacement_count']),
|
|
'replacement_evidence_sha256': str(
|
|
row['replacement_evidence_sha256'] or ''
|
|
),
|
|
'is_deep_probe': bool(row['is_deep_probe']),
|
|
'work_state': str(row['work_state']),
|
|
'selected_image_count': int(row['selected_image_count']),
|
|
'terminal_reason': (
|
|
str(row['last_error_code'])
|
|
if str(row['work_state']) == 'skipped' else None
|
|
),
|
|
'skip_evidence_sha256': terminal_evidence[int(row['id'])],
|
|
} for row in members],
|
|
})
|
|
payload = json.dumps(
|
|
document, ensure_ascii=True, allow_nan=False, sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
def _freeze_docker_depth_runtime_selection_locked(
|
|
self, experiment, authority, now,
|
|
):
|
|
incomplete = self.conn.execute(
|
|
'''SELECT 1 FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ?
|
|
AND work_state NOT IN ('resolved','skipped') LIMIT 1''',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
if incomplete:
|
|
return experiment, None
|
|
rows = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ? ORDER BY id''',
|
|
(experiment['id'],),
|
|
).fetchall()
|
|
for row in rows:
|
|
reason, _evidence_sha256 = (
|
|
self._docker_depth_terminal_repository_evidence_locked(
|
|
experiment, row,
|
|
)
|
|
)
|
|
if reason:
|
|
return experiment, reason
|
|
selection_sha256 = self._docker_depth_runtime_selection_sha256_locked(
|
|
experiment, authority,
|
|
)
|
|
stored = str(experiment['selection_sha256'] or '')
|
|
if stored and stored != selection_sha256:
|
|
return experiment, 'selection_hash_drift'
|
|
if not stored:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET selection_sha256 = ?, updated_at = ?
|
|
WHERE id = ? AND selection_sha256 IS NULL
|
|
AND state = 'resolving' ''',
|
|
(selection_sha256, now, experiment['id']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
return experiment, 'selection_hash_drift'
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
return experiment, None
|
|
|
|
def _docker_depth_hold_event_drift_reason_locked(self, experiment, authority):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
DOCKER_DEPTH_HOLD_REASON,
|
|
DOCKER_DEPTH_RELEASE_REASON,
|
|
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
|
|
)
|
|
|
|
experiment_id = int(experiment['id'])
|
|
hold_sha256 = str(experiment['hold_manifest_sha256'] or '')
|
|
rows = self.conn.execute(
|
|
'''SELECT event.*, queue.status AS queue_status,
|
|
queue.source AS queue_source,
|
|
queue.platform AS queue_platform,
|
|
queue.query AS queue_query,
|
|
queue.updated_at AS queue_updated_at
|
|
FROM target_queue_policy_events event
|
|
JOIN target_queue queue ON queue.id = event.queue_id
|
|
WHERE event.experiment_id = ?
|
|
ORDER BY event.id LIMIT ?''',
|
|
(experiment_id, DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS + 1),
|
|
).fetchall()
|
|
if len(rows) > DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS:
|
|
return 'target_history_drift'
|
|
reversed_ids = {
|
|
int(row['reverses_event_id']) for row in rows
|
|
if row['reverses_event_id'] is not None
|
|
}
|
|
for row in rows:
|
|
if (
|
|
int(row['queue_id']) < 1
|
|
or str(row['source']) != authority['source']
|
|
or str(row['platform']) != 'docker'
|
|
or not str(row['query'])
|
|
or str(row['queue_source']) != str(row['source'])
|
|
or str(row['queue_platform']) != str(row['platform'])
|
|
or str(row['queue_query']) != str(row['query'])
|
|
or str(row['config_sha256']) != authority['config_sha256']
|
|
or str(row['policy_sha256'])
|
|
!= authority['provenance_policy_sha256']
|
|
):
|
|
return 'target_history_drift'
|
|
if row['action'] == 'cold':
|
|
entry = {
|
|
'queue_id': int(row['queue_id']),
|
|
'source': str(row['source']),
|
|
'platform': str(row['platform']),
|
|
'query': str(row['query']),
|
|
'prior_status': str(row['prior_status']),
|
|
'prior_updated_at': str(row['prior_updated_at']),
|
|
}
|
|
if (
|
|
str(row['manifest_sha256']) != hold_sha256
|
|
or str(row['reason_code']) not in (
|
|
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
)
|
|
or str(row['prior_status']) not in ('pending', 'deferred')
|
|
or str(row['next_status']) != 'cold'
|
|
or int(row['id']) in reversed_ids
|
|
or str(row['queue_status']) != 'cold'
|
|
or str(row['queue_updated_at']) != str(row['created_at'])
|
|
):
|
|
return 'target_history_drift'
|
|
elif row['action'] == 'reactivate':
|
|
# Reactivation and the completed -> released transition share one
|
|
# experiment-row lock, so it is never valid in a pre-release state.
|
|
if str(row['reason_code']) != DOCKER_DEPTH_RELEASE_REASON:
|
|
return 'target_history_drift'
|
|
return 'target_history_drift'
|
|
else:
|
|
return 'target_history_drift'
|
|
if str(row['review_audit_sha256']) != self._target_queue_policy_audit_sha256(
|
|
str(row['action']), str(row['manifest_sha256']), entry,
|
|
experiment_id,
|
|
):
|
|
return 'target_history_drift'
|
|
return None
|
|
|
|
def _docker_depth_persisted_drift_reason_locked(self, experiment, authority, now):
|
|
from docker_depth_experiment import (
|
|
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
|
|
_cohort_plan_document,
|
|
canonical_docker_depth_plan_hash,
|
|
)
|
|
|
|
experiment_id = int(experiment['id'])
|
|
plan_sha256 = str(experiment['plan_sha256'] or '')
|
|
hold_sha256 = str(experiment['hold_manifest_sha256'] or '')
|
|
if any(
|
|
experiment[name] is not None
|
|
for name in ('fence_owner', 'fence_token', 'fence_expires_at')
|
|
):
|
|
return 'experiment_fence_conflict'
|
|
if not re.fullmatch(r'[a-f0-9]{64}', plan_sha256):
|
|
return 'plan_hash_drift'
|
|
if not re.fullmatch(r'[a-f0-9]{64}', hold_sha256):
|
|
return 'hold_hash_drift'
|
|
|
|
query_rows = self.conn.execute(
|
|
'''SELECT query_ordinal, source, query, query_sha256,
|
|
required_repository_count, selected_repository_count
|
|
FROM docker_depth_experiment_queries
|
|
WHERE experiment_id = ? ORDER BY query_ordinal LIMIT ?''',
|
|
(experiment_id, authority['query_count'] + 1),
|
|
).fetchall()
|
|
if len(query_rows) != authority['query_count']:
|
|
return 'ordered_query_drift'
|
|
selected_counts = []
|
|
for ordinal, row in enumerate(query_rows):
|
|
query = authority['queries'][ordinal]
|
|
selected_count = int(row['selected_repository_count'])
|
|
if (
|
|
int(row['query_ordinal']) != ordinal
|
|
or str(row['source']) != authority['source']
|
|
or str(row['query']) != query
|
|
or str(row['query_sha256']) != hashlib.sha256(
|
|
json.dumps(
|
|
query, ensure_ascii=True, allow_nan=False, sort_keys=True,
|
|
separators=(',', ':'),
|
|
).encode('utf-8')
|
|
).hexdigest()
|
|
or int(row['required_repository_count'])
|
|
!= authority['repositories_per_query']
|
|
or not 0 <= selected_count <= authority['repositories_per_query']
|
|
):
|
|
return 'ordered_query_drift'
|
|
selected_counts.append(selected_count)
|
|
|
|
expected_repository_count = sum(selected_counts)
|
|
repository_rows = self.conn.execute(
|
|
'''SELECT id, query_ordinal, source, query, repository_queue_id,
|
|
eligibility_page_id, repository_rank, planned_is_deep_probe,
|
|
is_deep_probe, work_state, resolver_owner, resolver_token,
|
|
resolver_expires_at, resolver_due_at,
|
|
candidate_distinct_graph_count,
|
|
selected_image_count, replacement_repository_queue_id,
|
|
replacement_eligibility_page_id, replacement_count,
|
|
replacement_evidence_sha256, last_error_code, resolved_at
|
|
FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ?
|
|
ORDER BY query_ordinal, repository_rank, id LIMIT ?''',
|
|
(experiment_id, expected_repository_count + 1),
|
|
).fetchall()
|
|
if len(repository_rows) != expected_repository_count:
|
|
return 'repository_count_drift'
|
|
if str(experiment['state']) in ('active', 'draining', 'completed') and any(
|
|
str(row['work_state']) not in ('resolved', 'skipped')
|
|
for row in repository_rows
|
|
):
|
|
return 'repository_state_drift'
|
|
repositories_by_query = {
|
|
ordinal: [] for ordinal in range(authority['query_count'])
|
|
}
|
|
for row in repository_rows:
|
|
ordinal = int(row['query_ordinal'])
|
|
if ordinal not in repositories_by_query:
|
|
return 'plan_hash_drift'
|
|
repositories_by_query[ordinal].append(row)
|
|
replacement_count = int(row['replacement_count'])
|
|
replacement_present = (
|
|
row['replacement_repository_queue_id'] is not None
|
|
and row['replacement_eligibility_page_id'] is not None
|
|
and bool(re.fullmatch(
|
|
r'[a-f0-9]{64}', str(row['replacement_evidence_sha256'] or '')
|
|
))
|
|
)
|
|
if (
|
|
replacement_count < 0
|
|
or (replacement_count == 0 and (
|
|
row['replacement_repository_queue_id'] is not None
|
|
or row['replacement_eligibility_page_id'] is not None
|
|
or row['replacement_evidence_sha256'] is not None
|
|
))
|
|
or (replacement_count > 0 and not replacement_present)
|
|
):
|
|
return 'selection_hash_drift'
|
|
if replacement_present and not self.conn.execute(
|
|
'''SELECT 1
|
|
FROM docker_repository_query_observations observation
|
|
JOIN docker_discovery_pages page ON page.id = observation.page_id
|
|
JOIN docker_discovery_passes discovery_pass
|
|
ON discovery_pass.id = page.pass_id
|
|
JOIN target_queue queue
|
|
ON queue.id = observation.repository_queue_id
|
|
WHERE observation.page_id = ?
|
|
AND observation.repository_queue_id = ?
|
|
AND observation.source = ? AND observation.query = ?
|
|
AND page.query_ordinal = ? AND page.query = ?
|
|
AND discovery_pass.source = ?
|
|
AND discovery_pass.pass_kind = 'deep'
|
|
AND discovery_pass.collection_generation = ?
|
|
AND discovery_pass.policy_sha256 = ?
|
|
AND discovery_pass.ordered_queries_sha256 = ?
|
|
AND discovery_pass.expected_query_count = ?
|
|
AND discovery_pass.state = 'complete'
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND queue.target NOT LIKE '%@%'
|
|
AND queue.normalized_target NOT LIKE '%@%'
|
|
LIMIT 1''',
|
|
(
|
|
row['replacement_eligibility_page_id'],
|
|
row['replacement_repository_queue_id'],
|
|
authority['source'], authority['queries'][ordinal], ordinal,
|
|
authority['queries'][ordinal], authority['source'],
|
|
authority['collection_generation'],
|
|
authority['provenance_policy_sha256'],
|
|
authority['ordered_queries_sha256'], authority['query_count'],
|
|
authority['source'],
|
|
),
|
|
).fetchone():
|
|
return 'selection_hash_drift'
|
|
resolving = str(row['work_state']) == 'resolving'
|
|
if resolving and str(experiment['state']) != 'resolving':
|
|
return 'stale_resolver_fence'
|
|
if resolving and (
|
|
not row['resolver_owner']
|
|
or not row['resolver_token']
|
|
or not row['resolver_expires_at']
|
|
or str(row['resolver_expires_at']) <= now
|
|
):
|
|
return 'stale_resolver_fence'
|
|
if not resolving and any(
|
|
row[name] is not None
|
|
for name in ('resolver_owner', 'resolver_token', 'resolver_expires_at')
|
|
):
|
|
return 'stale_resolver_fence'
|
|
terminal_reason, _evidence_sha256 = (
|
|
self._docker_depth_terminal_repository_evidence_locked(
|
|
experiment, row,
|
|
)
|
|
)
|
|
if terminal_reason:
|
|
return terminal_reason
|
|
|
|
planned_queries = []
|
|
for ordinal, query_row in enumerate(query_rows):
|
|
members = repositories_by_query[ordinal]
|
|
selected_count = selected_counts[ordinal]
|
|
if (
|
|
[int(row['repository_rank']) for row in members]
|
|
!= list(range(1, selected_count + 1))
|
|
or any(
|
|
str(row['source']) != authority['source']
|
|
or str(row['query']) != authority['queries'][ordinal]
|
|
or int(row['eligibility_page_id']) < 1
|
|
for row in members
|
|
)
|
|
or sum(int(row['planned_is_deep_probe']) for row in members)
|
|
!= (1 if selected_count else 0)
|
|
):
|
|
return 'plan_hash_drift'
|
|
all_terminal = all(
|
|
str(row['work_state']) in ('resolved', 'skipped') for row in members
|
|
)
|
|
image_bearing = any(
|
|
str(row['work_state']) == 'resolved'
|
|
and int(row['selected_image_count']) >= 1
|
|
for row in members
|
|
)
|
|
if (
|
|
str(experiment['selector_version'])
|
|
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
and all_terminal
|
|
):
|
|
expected_deep_probe_count = 0
|
|
else:
|
|
expected_deep_probe_count = (
|
|
(1 if image_bearing else 0)
|
|
if all_terminal else (1 if selected_count else 0)
|
|
)
|
|
if sum(int(row['is_deep_probe']) for row in members) != (
|
|
expected_deep_probe_count
|
|
):
|
|
return 'selection_mutation'
|
|
planned_queries.append({
|
|
'query_ordinal': ordinal,
|
|
'query': authority['queries'][ordinal],
|
|
'query_sha256': str(query_row['query_sha256']),
|
|
'selected_repository_count': selected_count,
|
|
'repositories': [{
|
|
'repository_queue_id': int(row['repository_queue_id']),
|
|
'eligibility_page_id': int(row['eligibility_page_id']),
|
|
'repository_rank': int(row['repository_rank']),
|
|
'is_deep_probe': bool(row['planned_is_deep_probe']),
|
|
} for row in members],
|
|
})
|
|
if canonical_docker_depth_plan_hash(
|
|
_cohort_plan_document(authority, planned_queries)
|
|
) != plan_sha256:
|
|
return 'plan_hash_drift'
|
|
selection_sha256 = str(experiment['selection_sha256'] or '')
|
|
if selection_sha256:
|
|
if (
|
|
not re.fullmatch(r'[a-f0-9]{64}', selection_sha256)
|
|
or self._docker_depth_runtime_selection_sha256_locked(
|
|
experiment, authority,
|
|
) != selection_sha256
|
|
):
|
|
return 'selection_hash_drift'
|
|
elif str(experiment['state']) in ('active', 'draining'):
|
|
return 'selection_hash_drift'
|
|
|
|
target_rows = self.conn.execute(
|
|
'''SELECT target.id AS target_id, target.target_queue_id,
|
|
target.manifest_id, target.counter_ordinal,
|
|
target.state AS target_state, target.dispatch_wave,
|
|
target.dispatch_order, target.reservation_count,
|
|
queue.source AS queue_source, queue.platform AS queue_platform,
|
|
queue.target AS queue_target,
|
|
queue.normalized_target AS queue_normalized_target,
|
|
queue.status AS queue_status, queue.lease_owner,
|
|
queue.lease_token, queue.claim_batch, queue.leased_at,
|
|
queue.lease_expires_at, queue.current_result_reservation_id,
|
|
queue.claim_event_id, queue.resolver_token,
|
|
manifest.repository, manifest.manifest_digest,
|
|
manifest.graph_sha256 AS manifest_graph_sha256
|
|
FROM docker_depth_experiment_targets target
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
JOIN docker_image_manifests manifest
|
|
ON manifest.id = target.manifest_id
|
|
AND manifest.target_queue_id = target.target_queue_id
|
|
WHERE target.experiment_id = ?
|
|
ORDER BY target.id LIMIT ?''',
|
|
(experiment_id, authority['target_limit'] + 1),
|
|
).fetchall()
|
|
if (
|
|
len(target_rows) != int(experiment['target_count'])
|
|
or len(target_rows) > authority['target_limit']
|
|
):
|
|
return 'target_counter_drift'
|
|
target_by_id = {int(row['target_id']): row for row in target_rows}
|
|
if len(target_by_id) != len(target_rows):
|
|
return 'target_counter_drift'
|
|
|
|
selection_rows = self.conn.execute(
|
|
'''SELECT selection.id, selection.query_ordinal,
|
|
selection.experiment_repository_id,
|
|
selection.experiment_target_id, selection.image_rank,
|
|
selection.selection_evidence_sha256,
|
|
selection.graph_sha256,
|
|
member.query_ordinal AS member_query_ordinal,
|
|
member.is_deep_probe
|
|
FROM docker_depth_experiment_selections selection
|
|
JOIN docker_depth_experiment_repositories member
|
|
ON member.id = selection.experiment_repository_id
|
|
AND member.experiment_id = selection.experiment_id
|
|
WHERE selection.experiment_id = ?
|
|
ORDER BY selection.id LIMIT ?''',
|
|
(experiment_id, authority['theoretical_max_targets'] + 1),
|
|
).fetchall()
|
|
if (
|
|
len(selection_rows) != int(experiment['selection_count'])
|
|
or len(selection_rows) > authority['theoretical_max_targets']
|
|
):
|
|
return 'selection_counter_drift'
|
|
selected_by_member = {}
|
|
selected_ranks_by_member = {}
|
|
dispatch_by_target = {}
|
|
selected_target_ids = set()
|
|
for selection in selection_rows:
|
|
target_id = int(selection['experiment_target_id'])
|
|
target = target_by_id.get(target_id)
|
|
rank = int(selection['image_rank'])
|
|
if (
|
|
not target
|
|
or int(selection['query_ordinal'])
|
|
!= int(selection['member_query_ordinal'])
|
|
or (rank > 1 and not bool(selection['is_deep_probe']))
|
|
or str(selection['graph_sha256'])
|
|
!= str(target['manifest_graph_sha256'])
|
|
or not re.fullmatch(
|
|
r'[a-f0-9]{64}', str(selection['selection_evidence_sha256'] or '')
|
|
)
|
|
):
|
|
return 'selection_mutation'
|
|
member_id = int(selection['experiment_repository_id'])
|
|
selected_by_member[member_id] = selected_by_member.get(member_id, 0) + 1
|
|
selected_ranks_by_member.setdefault(member_id, []).append(rank)
|
|
position = self._docker_depth_dispatch_position(
|
|
int(selection['query_ordinal']),
|
|
next(
|
|
int(row['repository_rank']) for row in repository_rows
|
|
if int(row['id']) == member_id
|
|
),
|
|
rank, authority['query_count'],
|
|
)
|
|
dispatch_by_target[target_id] = min(
|
|
dispatch_by_target.get(target_id, position), position,
|
|
)
|
|
selected_target_ids.add(target_id)
|
|
if selected_target_ids != set(target_by_id):
|
|
return 'selection_mutation'
|
|
for member in repository_rows:
|
|
member_id = int(member['id'])
|
|
selected_count = selected_by_member.get(member_id, 0)
|
|
if (
|
|
selected_count != int(member['selected_image_count'])
|
|
or sorted(selected_ranks_by_member.get(member_id, ()))
|
|
!= list(range(1, selected_count + 1))
|
|
):
|
|
return 'selection_counter_drift'
|
|
|
|
for target_id, target in target_by_id.items():
|
|
expected_target = f"{target['repository']}@{target['manifest_digest']}"
|
|
if (
|
|
(int(target['dispatch_wave']), int(target['dispatch_order']))
|
|
!= dispatch_by_target[target_id]
|
|
or str(target['queue_source']) != authority['source']
|
|
or str(target['queue_platform']) != 'docker'
|
|
or str(target['queue_target']) != expected_target
|
|
or str(target['queue_normalized_target']) != expected_target
|
|
):
|
|
return 'selection_mutation'
|
|
target_state = str(target['target_state'])
|
|
queue_status = str(target['queue_status'])
|
|
if target_state in ('held', 'quarantined'):
|
|
return 'scan_target_held'
|
|
if target_state == 'pending' and queue_status not in ('pending', 'deferred'):
|
|
return 'stale_scan_fence'
|
|
if target_state in ('reserved', 'scanning') and queue_status != 'in_progress':
|
|
return 'stale_scan_fence'
|
|
if target_state == 'done' and queue_status != 'done':
|
|
return 'stale_scan_fence'
|
|
if target_state == 'failed' and queue_status != 'failed':
|
|
return 'stale_scan_fence'
|
|
if target_state == 'quarantined' and queue_status != 'quarantined':
|
|
return 'stale_scan_fence'
|
|
if target_state == 'pending' and any(
|
|
target[name] is not None for name in (
|
|
'lease_owner', 'lease_token', 'claim_batch', 'leased_at',
|
|
'lease_expires_at', 'current_result_reservation_id',
|
|
'claim_event_id', 'resolver_token',
|
|
)
|
|
):
|
|
return 'stale_scan_fence'
|
|
|
|
binding_rows = self.conn.execute(
|
|
'''SELECT binding.id, binding.experiment_target_id,
|
|
binding.reservation_id, binding.target_scan_id,
|
|
binding.attempt, binding.state AS binding_state,
|
|
target.target_queue_id,
|
|
reservation.queue_id AS reservation_queue_id,
|
|
reservation.state AS reservation_state,
|
|
reservation.claim_lease_token,
|
|
reservation.scan_event_id,
|
|
reservation.producer_lease_expires_at,
|
|
scan.queue_id AS scan_queue_id,
|
|
scan.result_reservation_id AS scan_reservation_id
|
|
FROM docker_depth_experiment_scan_bindings binding
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
JOIN result_reservations reservation
|
|
ON reservation.id = binding.reservation_id
|
|
LEFT JOIN target_scans scan ON scan.id = binding.target_scan_id
|
|
WHERE target.experiment_id = ?
|
|
ORDER BY binding.experiment_target_id, binding.attempt
|
|
LIMIT ?''',
|
|
(experiment_id, authority['target_limit'] * 10 + 1),
|
|
).fetchall()
|
|
if len(binding_rows) > authority['target_limit'] * 10:
|
|
return 'binding_count_drift'
|
|
binding_count = {}
|
|
active_by_target = {}
|
|
latest_by_target = {}
|
|
for binding in binding_rows:
|
|
target_id = int(binding['experiment_target_id'])
|
|
state = str(binding['binding_state'])
|
|
reservation_state = str(binding['reservation_state'])
|
|
if (
|
|
target_id not in target_by_id
|
|
or int(binding['target_queue_id'])
|
|
!= int(binding['reservation_queue_id'])
|
|
or int(binding['attempt']) < 1
|
|
):
|
|
return 'reservation_binding_drift'
|
|
has_scan = binding['target_scan_id'] is not None
|
|
if has_scan and (
|
|
int(binding['scan_queue_id'] or 0) != int(binding['target_queue_id'])
|
|
or int(binding['scan_reservation_id'] or 0)
|
|
!= int(binding['reservation_id'])
|
|
):
|
|
return 'reservation_binding_drift'
|
|
if state == 'reserved' and (reservation_state != 'scanning' or has_scan):
|
|
return 'reservation_binding_drift'
|
|
if state == 'scanning' and (
|
|
reservation_state not in ('ready', 'ingesting') or has_scan
|
|
):
|
|
return 'reservation_binding_drift'
|
|
if state in ('completed', 'failed') and (
|
|
reservation_state not in ('db_committed', 'acknowledged') or not has_scan
|
|
):
|
|
return 'reservation_binding_drift'
|
|
if state == 'released' and (reservation_state != 'refunded' or has_scan):
|
|
return 'reservation_binding_drift'
|
|
if state == 'quarantined' and reservation_state != 'quarantined':
|
|
return 'reservation_binding_drift'
|
|
if state in ('reserved', 'scanning'):
|
|
if (
|
|
state == 'reserved'
|
|
and str(binding['producer_lease_expires_at'] or '') <= now
|
|
):
|
|
return 'stale_scan_fence'
|
|
active_by_target.setdefault(target_id, []).append(binding)
|
|
binding_count[target_id] = binding_count.get(target_id, 0) + 1
|
|
latest_by_target[target_id] = binding
|
|
for target_id, target in target_by_id.items():
|
|
if int(target['reservation_count']) != binding_count.get(target_id, 0):
|
|
return 'binding_count_drift'
|
|
active = active_by_target.get(target_id, [])
|
|
target_state = str(target['target_state'])
|
|
latest = latest_by_target.get(target_id)
|
|
if target_state in ('done', 'failed') and (
|
|
latest is None
|
|
or str(latest['binding_state'])
|
|
!= ('completed' if target_state == 'done' else 'failed')
|
|
):
|
|
return 'reservation_binding_drift'
|
|
if target_state in ('reserved', 'scanning'):
|
|
if len(active) != 1:
|
|
return 'stale_scan_fence'
|
|
binding = active[0]
|
|
if (
|
|
int(target['current_result_reservation_id'] or 0)
|
|
!= int(binding['reservation_id'])
|
|
or str(target['lease_token'] or '')
|
|
!= str(binding['claim_lease_token'])
|
|
or str(target['claim_event_id'] or '')
|
|
!= str(binding['scan_event_id'])
|
|
):
|
|
return 'stale_scan_fence'
|
|
elif active:
|
|
return 'stale_scan_fence'
|
|
|
|
unbound = self.conn.execute(
|
|
'''SELECT
|
|
EXISTS(
|
|
SELECT 1 FROM result_reservations reservation
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.target_queue_id = reservation.queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.experiment_target_id = target.id
|
|
AND binding.reservation_id = reservation.id
|
|
WHERE target.experiment_id = ? AND binding.id IS NULL
|
|
) AS reservation_missing,
|
|
EXISTS(
|
|
SELECT 1 FROM target_scans scan
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.target_queue_id = scan.queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.experiment_target_id = target.id
|
|
AND binding.target_scan_id = scan.id
|
|
WHERE target.experiment_id = ? AND binding.id IS NULL
|
|
) AS scan_missing''',
|
|
(experiment_id, experiment_id),
|
|
).fetchone()
|
|
if bool(unbound['reservation_missing']) or bool(unbound['scan_missing']):
|
|
return 'historical_scan_conflict'
|
|
|
|
return self._docker_depth_hold_event_drift_reason_locked(
|
|
experiment, authority,
|
|
)
|
|
|
|
def _locked_docker_depth_experiment_authority(
|
|
self, authority, *, final_cutover, now=None, lock=True,
|
|
):
|
|
now = str(now or utc_now_iso())
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE experiment_key = ? AND source = ?''' + (
|
|
' FOR UPDATE' if lock else ''
|
|
),
|
|
(authority['experiment_key'], authority['source']),
|
|
).fetchone()
|
|
if not experiment:
|
|
return None, 'experiment_authority_absent'
|
|
reason = self._docker_depth_authority_drift_reason(experiment, authority)
|
|
marker = self.conn.execute(
|
|
'''SELECT marker, checked_at, evidence_sha256
|
|
FROM runtime_final_cutover WHERE id = 1'''
|
|
).fetchone()
|
|
if reason is None and (
|
|
final_cutover is not True
|
|
or not marker
|
|
or str(marker['marker']) != FINAL_CUTOVER_MARKER
|
|
or not marker['checked_at']
|
|
or not re.fullmatch(r'[a-f0-9]{64}', str(marker['evidence_sha256'] or ''))
|
|
):
|
|
reason = 'final_cutover_unavailable'
|
|
if reason is None and str(experiment['state']) in (
|
|
'holding', 'resolving', 'active', 'draining', 'completed', 'held',
|
|
):
|
|
reason = self._docker_depth_persisted_drift_reason_locked(
|
|
experiment, authority, now,
|
|
)
|
|
if reason:
|
|
self._hold_docker_depth_experiment_locked(experiment, reason, now)
|
|
return None, reason
|
|
return experiment, None
|
|
|
|
def _docker_depth_incomplete_membership_count_locked(
|
|
self, experiment_id, *, activation=False,
|
|
):
|
|
activation_clause = (
|
|
"OR target.state <> 'pending' OR queue.status NOT IN ('pending','deferred') "
|
|
"OR queue.lease_owner IS NOT NULL OR queue.lease_token IS NOT NULL "
|
|
"OR queue.claim_batch IS NOT NULL OR queue.leased_at IS NOT NULL "
|
|
"OR queue.lease_expires_at IS NOT NULL "
|
|
"OR queue.current_result_reservation_id IS NOT NULL "
|
|
"OR queue.claim_event_id IS NOT NULL "
|
|
"OR EXISTS (SELECT 1 FROM target_scans scan "
|
|
" WHERE scan.queue_id = queue.id) "
|
|
"OR EXISTS (SELECT 1 FROM result_reservations reservation "
|
|
" WHERE reservation.queue_id = queue.id) "
|
|
"OR EXISTS (SELECT 1 FROM target_queue_policy_events event "
|
|
" WHERE event.queue_id = queue.id) "
|
|
"OR EXISTS (SELECT 1 FROM docker_image_blob_coverage coverage "
|
|
" JOIN docker_content_blobs blob "
|
|
" ON blob.digest = coverage.blob_digest "
|
|
" AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256 "
|
|
" WHERE coverage.queue_id = queue.id "
|
|
" AND blob.state IN ('leased','submitted'))"
|
|
if activation else ''
|
|
)
|
|
return int(self.conn.execute(
|
|
f'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_repositories member
|
|
LEFT JOIN docker_depth_experiment_selections selection
|
|
ON selection.experiment_repository_id = member.id
|
|
AND selection.image_rank = 1
|
|
LEFT JOIN docker_depth_experiment_targets target
|
|
ON target.id = selection.experiment_target_id
|
|
AND target.experiment_id = member.experiment_id
|
|
LEFT JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
WHERE member.experiment_id = ?
|
|
AND member.work_state <> 'skipped'
|
|
AND (member.selected_image_count < 1 OR selection.id IS NULL
|
|
OR target.id IS NULL OR queue.id IS NULL {activation_clause})''',
|
|
(experiment_id,),
|
|
).fetchone()['count'])
|
|
|
|
def _advance_docker_depth_experiment_state_locked(self, experiment, now=None):
|
|
now = str(now or utc_now_iso())
|
|
experiment_id = int(experiment['id'])
|
|
state = str(experiment['state'])
|
|
unresolved = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ?
|
|
AND work_state NOT IN ('resolved','skipped')''',
|
|
(experiment_id,),
|
|
).fetchone()['count'])
|
|
if state == 'resolving' and unresolved == 0:
|
|
from docker_depth_experiment import (
|
|
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
|
|
)
|
|
incremental_scan_profile = (
|
|
str(experiment['selector_version'])
|
|
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
)
|
|
if self._docker_depth_incomplete_membership_count_locked(
|
|
experiment_id, activation=not incremental_scan_profile,
|
|
):
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'cohort_membership_incomplete', now,
|
|
)
|
|
return self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
experiment, selection_reason = (
|
|
self._freeze_docker_depth_runtime_selection_locked(
|
|
experiment, authority={
|
|
'query_count': int(experiment['query_count']),
|
|
'repositories_per_query': int(experiment['repositories_per_query']),
|
|
'collection_generation': str(experiment['collection_generation']),
|
|
'selector_version': str(experiment['selector_version']),
|
|
},
|
|
now=now,
|
|
)
|
|
)
|
|
if selection_reason:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, selection_reason, now,
|
|
)
|
|
return self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'active', activated_at = COALESCE(activated_at, ?),
|
|
updated_at = ?
|
|
WHERE id = ? AND state = 'resolving' ''',
|
|
(now, now, experiment_id),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth activation lost its compare-and-swap fence'
|
|
)
|
|
state = 'active'
|
|
if state == 'active':
|
|
nonterminal = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ? AND state IN ('pending','reserved','scanning')''',
|
|
(experiment_id,),
|
|
).fetchone()['count'])
|
|
if nonterminal == 0:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'draining', draining_at = COALESCE(draining_at, ?),
|
|
updated_at = ?
|
|
WHERE id = ? AND state = 'active' ''',
|
|
(now, now, experiment_id),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth draining transition lost its compare-and-swap fence'
|
|
)
|
|
state = 'draining'
|
|
if state == 'draining' and unresolved == 0:
|
|
if self._docker_depth_incomplete_membership_count_locked(experiment_id):
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'cohort_membership_incomplete', now,
|
|
)
|
|
return self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
fences = self.conn.execute(
|
|
'''SELECT
|
|
(SELECT COUNT(*) FROM docker_depth_experiment_targets target
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
WHERE target.experiment_id = ?
|
|
AND (target.state NOT IN ('done','failed','skipped')
|
|
OR queue.status NOT IN ('done','failed')
|
|
OR queue.lease_owner IS NOT NULL
|
|
OR queue.lease_token IS NOT NULL
|
|
OR queue.claim_batch IS NOT NULL
|
|
OR queue.leased_at IS NOT NULL
|
|
OR queue.lease_expires_at IS NOT NULL
|
|
OR queue.current_result_reservation_id IS NOT NULL
|
|
OR queue.claim_event_id IS NOT NULL)) AS queue_fences,
|
|
(SELECT COUNT(*) FROM docker_depth_experiment_scan_bindings binding
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
JOIN result_reservations reservation
|
|
ON reservation.id = binding.reservation_id
|
|
WHERE target.experiment_id = ?
|
|
AND (binding.state IN ('reserved','scanning')
|
|
OR reservation.state IN
|
|
('scanning','ready','ingesting','db_committed'))) AS reservation_fences,
|
|
(SELECT COUNT(*) FROM docker_image_blob_coverage coverage
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.target_queue_id = coverage.queue_id
|
|
JOIN docker_content_blobs blob
|
|
ON blob.digest = coverage.blob_digest
|
|
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
|
|
WHERE target.experiment_id = ?
|
|
AND blob.state IN ('leased','submitted')) AS blob_fences''',
|
|
(experiment_id, experiment_id, experiment_id),
|
|
).fetchone()
|
|
if not any(int(fences[name]) for name in (
|
|
'queue_fences', 'reservation_fences', 'blob_fences',
|
|
)):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'completed', completed_at = COALESCE(completed_at, ?),
|
|
updated_at = ?
|
|
WHERE id = ? AND state = 'draining'
|
|
AND fence_owner IS NULL AND fence_token IS NULL
|
|
AND fence_expires_at IS NULL''',
|
|
(now, now, experiment_id),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth completion lost its compare-and-swap fence'
|
|
)
|
|
state = 'completed'
|
|
if state == str(experiment['state']):
|
|
return experiment
|
|
return self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
|
|
def _hold_disabled_docker_depth_experiment(self, authority):
|
|
key = str(authority.get('experiment_key') or '') if isinstance(
|
|
authority, dict
|
|
) else ''
|
|
if not self.conn or not self.conn.is_postgres or not key:
|
|
return False
|
|
try:
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE experiment_key = ? AND source = 'dockerhub' FOR UPDATE''',
|
|
(key,),
|
|
).fetchone()
|
|
if not experiment or str(experiment['state']) not in (
|
|
'holding', 'resolving', 'active', 'draining',
|
|
):
|
|
self.conn.rollback()
|
|
return False
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'experiment_disabled', utc_now_iso(),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
return False
|
|
|
|
def refresh_docker_depth_experiment_state(self, authority, *, final_cutover=False):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return {'status': 'unavailable', 'committed': False}
|
|
if isinstance(authority, dict) and authority.get('enabled') is False:
|
|
held = self._hold_disabled_docker_depth_experiment(authority)
|
|
return {
|
|
'status': 'held' if held else 'disabled',
|
|
'committed': held,
|
|
'reason': 'experiment_disabled' if held else None,
|
|
}
|
|
if not isinstance(authority, dict) or authority.get('enabled') is not True:
|
|
return {'status': 'disabled', 'committed': False}
|
|
try:
|
|
normalized = self._normalize_docker_depth_resolver_authority(authority)
|
|
except (TypeError, ValueError):
|
|
return {
|
|
'status': 'invalid', 'committed': False,
|
|
'reason': 'authority_payload_invalid',
|
|
}
|
|
try:
|
|
now = utc_now_iso()
|
|
experiment, reason = self._locked_docker_depth_experiment_authority(
|
|
normalized, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not experiment:
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'held' if reason != 'experiment_authority_absent' else 'unavailable',
|
|
'committed': reason != 'experiment_authority_absent',
|
|
'reason': reason,
|
|
}
|
|
experiment = self._advance_docker_depth_experiment_state_locked(
|
|
experiment, now,
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'status': str(experiment['state']), 'committed': True,
|
|
'experiment_id': int(experiment['id']),
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _docker_depth_candidate_selection_drift_reason(self, candidate, authority):
|
|
from docker_depth_experiment import (
|
|
canonical_docker_depth_selection_evidence_hash,
|
|
canonical_docker_descriptor_hash,
|
|
canonical_docker_layer_graph_hash,
|
|
)
|
|
|
|
layers = self.conn.execute(
|
|
'''SELECT position_from_base, position_from_top, layer_digest,
|
|
media_type, layer_size_bytes, descriptor_sha256
|
|
FROM docker_manifest_layers WHERE manifest_id = ?
|
|
ORDER BY position_from_base LIMIT 10001''',
|
|
(candidate['manifest_id'],),
|
|
).fetchall()
|
|
if (
|
|
len(layers) != int(candidate['layer_count'])
|
|
or len(layers) > 10000
|
|
):
|
|
return 'selection_mutation'
|
|
normalized_layers = []
|
|
graph = []
|
|
for position, layer in enumerate(layers, 1):
|
|
digest = str(layer['layer_digest'])
|
|
media_type = str(layer['media_type'])
|
|
size_bytes = int(layer['layer_size_bytes'])
|
|
if (
|
|
int(layer['position_from_base']) != position
|
|
or int(layer['position_from_top']) != len(layers) - position + 1
|
|
or str(layer['descriptor_sha256'])
|
|
!= canonical_docker_descriptor_hash(digest, media_type, size_bytes)
|
|
):
|
|
return 'selection_mutation'
|
|
graph.append(digest)
|
|
normalized_layers.append({
|
|
'digest': digest,
|
|
'media_type': media_type,
|
|
'size_bytes': size_bytes,
|
|
'descriptor_sha256': str(layer['descriptor_sha256']),
|
|
'position_from_base': position,
|
|
'position_from_top': len(layers) - position + 1,
|
|
})
|
|
graph_sha256 = canonical_docker_layer_graph_hash(tuple(graph))
|
|
if graph_sha256 != str(candidate['graph_sha256']):
|
|
return 'selection_mutation'
|
|
selections = self.conn.execute(
|
|
'''SELECT selection.image_rank, selection.selection_reason,
|
|
selection.selection_evidence_sha256,
|
|
selection.graph_sha256 AS selection_graph_sha256,
|
|
member.candidate_distinct_graph_count
|
|
FROM docker_depth_experiment_selections selection
|
|
JOIN docker_depth_experiment_repositories member
|
|
ON member.id = selection.experiment_repository_id
|
|
AND member.experiment_id = selection.experiment_id
|
|
WHERE selection.experiment_target_id = ?
|
|
ORDER BY selection.id LIMIT ?''',
|
|
(candidate['experiment_target_id'], authority['query_count'] + 1),
|
|
).fetchall()
|
|
if not selections or len(selections) > authority['query_count']:
|
|
return 'selection_mutation'
|
|
for selection in selections:
|
|
evidence = {
|
|
'schema': 1,
|
|
'type': 'docker-depth-selection-evidence-v1',
|
|
'selector_version': authority['selector_version'],
|
|
'selector_sha256': authority['selector_sha256'],
|
|
'candidate_distinct_graph_count': int(
|
|
selection['candidate_distinct_graph_count']
|
|
),
|
|
'image_rank': int(selection['image_rank']),
|
|
'selection_reason': str(selection['selection_reason']),
|
|
'target': str(candidate['target']),
|
|
'repository': str(candidate['repository']),
|
|
'manifest_digest': str(candidate['manifest_digest']),
|
|
'manifest_media_type': str(candidate['manifest_media_type']),
|
|
'manifest_size_bytes': int(candidate['manifest_size_bytes']),
|
|
'config_digest': str(candidate['config_digest']),
|
|
'graph_sha256': graph_sha256,
|
|
'layers': normalized_layers,
|
|
}
|
|
if (
|
|
str(selection['selection_graph_sha256']) != graph_sha256
|
|
or str(selection['selection_evidence_sha256'])
|
|
!= canonical_docker_depth_selection_evidence_hash(evidence)
|
|
):
|
|
return 'selection_mutation'
|
|
return None
|
|
|
|
def _terminalize_exhausted_docker_depth_targets_locked(
|
|
self, experiment, authority, now, max_attempts,
|
|
):
|
|
if max_attempts <= 0:
|
|
return 0
|
|
rows = self.conn.execute(
|
|
'''SELECT target.id AS experiment_target_id,
|
|
target.target_queue_id, target.manifest_id,
|
|
target.reservation_count,
|
|
queue.attempts, queue.target_scan_id AS queue_target_scan_id,
|
|
binding.id AS binding_id,
|
|
binding.reservation_id AS binding_reservation_id,
|
|
binding.target_scan_id AS binding_target_scan_id,
|
|
binding.attempt AS binding_attempt,
|
|
binding.state AS binding_state,
|
|
reservation.id AS reservation_id,
|
|
reservation.queue_id AS reservation_queue_id,
|
|
reservation.state AS reservation_state,
|
|
scan.id AS scan_id, scan.queue_id AS scan_queue_id,
|
|
scan.result_reservation_id AS scan_reservation_id
|
|
FROM docker_depth_experiment_targets target
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
LEFT JOIN docker_depth_experiment_scan_bindings binding
|
|
ON binding.experiment_target_id = target.id
|
|
AND binding.attempt = target.reservation_count
|
|
LEFT JOIN result_reservations reservation
|
|
ON reservation.id = binding.reservation_id
|
|
LEFT JOIN target_scans scan ON scan.id = binding.target_scan_id
|
|
WHERE target.experiment_id = ? AND target.state = 'pending'
|
|
AND target.reservation_count > 0
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND queue.status = 'deferred'
|
|
AND COALESCE(queue.attempts, 0) >= ?
|
|
ORDER BY target.id
|
|
FOR UPDATE OF target, queue''',
|
|
(experiment['id'], authority['source'], max_attempts),
|
|
).fetchall()
|
|
for row in rows:
|
|
if (
|
|
row['binding_id'] is None
|
|
or int(row['binding_attempt'] or 0)
|
|
!= int(row['reservation_count'])
|
|
or str(row['binding_state'] or '') != 'failed'
|
|
or row['binding_target_scan_id'] is None
|
|
or int(row['reservation_id'] or 0)
|
|
!= int(row['binding_reservation_id'] or 0)
|
|
or int(row['reservation_queue_id'] or 0)
|
|
!= int(row['target_queue_id'])
|
|
or str(row['reservation_state'] or '')
|
|
not in ('db_committed', 'acknowledged')
|
|
or int(row['scan_id'] or 0)
|
|
!= int(row['binding_target_scan_id'])
|
|
or int(row['scan_queue_id'] or 0)
|
|
!= int(row['target_queue_id'])
|
|
or int(row['scan_reservation_id'] or 0)
|
|
!= int(row['reservation_id'] or 0)
|
|
or int(row['queue_target_scan_id'] or 0)
|
|
!= int(row['scan_id'] or 0)
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth exhausted retry recovery conflicts with scan history'
|
|
)
|
|
queue_cursor = self.conn.execute(
|
|
'''UPDATE target_queue
|
|
SET status = 'failed', available_after = NULL,
|
|
completed_at = COALESCE(completed_at, ?),
|
|
last_error = COALESCE(
|
|
last_error, 'target retry attempts exhausted'
|
|
), updated_at = ?
|
|
WHERE id = ? AND source = ? AND platform = 'docker'
|
|
AND status = 'deferred' AND COALESCE(attempts, 0) >= ?
|
|
AND target_scan_id = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND lease_owner IS NULL AND lease_token IS NULL
|
|
AND claim_batch IS NULL AND leased_at IS NULL
|
|
AND lease_expires_at IS NULL AND claim_event_id IS NULL
|
|
AND resolver_token IS NULL
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')''',
|
|
(
|
|
now, now, row['target_queue_id'], authority['source'],
|
|
max_attempts, row['scan_id'],
|
|
),
|
|
)
|
|
target_cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_targets
|
|
SET state = 'failed', terminal_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
|
|
AND manifest_id = ? AND state = 'pending'
|
|
AND reservation_count = ? AND terminal_at IS NULL''',
|
|
(
|
|
now, now, row['experiment_target_id'], experiment['id'],
|
|
row['target_queue_id'], row['manifest_id'],
|
|
row['reservation_count'],
|
|
),
|
|
)
|
|
if (
|
|
int(queue_cursor.rowcount or 0) != 1
|
|
or int(target_cursor.rowcount or 0) != 1
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth exhausted retry recovery lost its exact fence'
|
|
)
|
|
return len(rows)
|
|
|
|
def _claim_docker_depth_experiment_target_locked(
|
|
self, experiment, authority, now, max_attempts,
|
|
):
|
|
self._terminalize_exhausted_docker_depth_targets_locked(
|
|
experiment, authority, now, max_attempts,
|
|
)
|
|
row = self.conn.execute(
|
|
'''SELECT queue.id, queue.query, queue.target,
|
|
queue.normalized_target, queue.attempts,
|
|
target.id AS experiment_target_id,
|
|
target.experiment_id, target.manifest_id,
|
|
target.dispatch_wave, target.dispatch_order,
|
|
target.reservation_count,
|
|
manifest.repository, manifest.manifest_digest,
|
|
manifest.manifest_media_type, manifest.manifest_size_bytes,
|
|
manifest.config_digest, manifest.graph_sha256,
|
|
manifest.layer_count
|
|
FROM docker_depth_experiment_targets target
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
JOIN docker_image_manifests manifest
|
|
ON manifest.id = target.manifest_id
|
|
AND manifest.target_queue_id = queue.id
|
|
WHERE target.experiment_id = ? AND target.state = 'pending'
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND queue.status IN ('pending','deferred')
|
|
AND (queue.status = 'pending'
|
|
OR (queue.available_after IS NOT NULL
|
|
AND queue.available_after <= ?))
|
|
AND (? = 0 OR COALESCE(queue.attempts, 0) < ?)
|
|
AND queue.current_result_reservation_id IS NULL
|
|
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
|
|
AND queue.claim_batch IS NULL AND queue.leased_at IS NULL
|
|
AND queue.lease_expires_at IS NULL
|
|
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
|
|
AND (queue.resolver_state IS NULL OR queue.resolver_state = 'resolved')
|
|
AND EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_selections selection
|
|
WHERE selection.experiment_id = target.experiment_id
|
|
AND selection.experiment_target_id = target.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets prior
|
|
WHERE prior.experiment_id = target.experiment_id
|
|
AND prior.dispatch_wave < target.dispatch_wave
|
|
AND (
|
|
prior.state NOT IN ('done','failed')
|
|
OR NOT EXISTS (
|
|
SELECT 1
|
|
FROM docker_depth_experiment_scan_bindings binding
|
|
WHERE binding.experiment_target_id = prior.id
|
|
AND binding.attempt = prior.reservation_count
|
|
AND (
|
|
(prior.state = 'done'
|
|
AND binding.state = 'completed')
|
|
OR (prior.state = 'failed'
|
|
AND binding.state = 'failed')
|
|
)
|
|
)
|
|
)
|
|
)
|
|
ORDER BY target.dispatch_order, target.id
|
|
LIMIT 1 FOR UPDATE OF target, queue SKIP LOCKED''',
|
|
(
|
|
experiment['id'], authority['source'], now,
|
|
max_attempts, max_attempts,
|
|
),
|
|
).fetchone()
|
|
if not row:
|
|
return None, None
|
|
reason = self._docker_depth_candidate_selection_drift_reason(row, authority)
|
|
if reason:
|
|
self._hold_docker_depth_experiment_locked(experiment, reason, now)
|
|
return None, reason
|
|
return row, None
|
|
|
|
def _docker_depth_experiment_schema_installed(self):
|
|
cached = getattr(
|
|
self, '_docker_depth_experiment_schema_installed_cache', None,
|
|
)
|
|
if cached is None:
|
|
cached = bool(self.conn.execute(
|
|
'''SELECT 1 AS present FROM runtime_schema_migrations
|
|
WHERE version = ?''',
|
|
(DOCKER_DEPTH_EXPERIMENT_MIGRATION,),
|
|
).fetchone())
|
|
self._docker_depth_experiment_schema_installed_cache = cached
|
|
return cached
|
|
|
|
def _docker_depth_binding_for_reservation_locked(self, reservation):
|
|
if not self._docker_depth_experiment_schema_installed():
|
|
return None
|
|
reservation_id = int(reservation['id'])
|
|
binding = self.conn.execute(
|
|
'''SELECT binding.*, target.experiment_id,
|
|
target.target_queue_id, target.manifest_id,
|
|
target.state AS target_state,
|
|
target.reservation_count,
|
|
queue.source AS bound_source,
|
|
queue.platform AS bound_platform,
|
|
queue.query AS bound_query,
|
|
queue.target AS bound_target,
|
|
queue.normalized_target AS bound_normalized_target,
|
|
manifest.source AS manifest_source,
|
|
manifest.manifest_digest, manifest.layer_count
|
|
FROM docker_depth_experiment_scan_bindings binding
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
JOIN docker_image_manifests manifest
|
|
ON manifest.id = target.manifest_id
|
|
AND manifest.target_queue_id = target.target_queue_id
|
|
WHERE binding.reservation_id = ?
|
|
FOR UPDATE OF binding, target, queue, manifest''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if binding:
|
|
if (
|
|
int(binding['target_queue_id']) != int(reservation['queue_id'])
|
|
or str(binding['bound_source']) != str(reservation['source'])
|
|
or str(binding['bound_platform']) != str(reservation['platform'])
|
|
or str(binding['bound_query'] or '') != str(reservation['query'] or '')
|
|
or str(binding['bound_target']) != str(reservation['target'])
|
|
or str(binding['bound_normalized_target'])
|
|
!= str(reservation['normalized_target'])
|
|
or str(binding['bound_source']) != 'dockerhub'
|
|
or str(binding['bound_platform']) != 'docker'
|
|
or str(binding['manifest_source']) != str(binding['bound_source'])
|
|
or int(binding['attempt']) != int(binding['reservation_count'])
|
|
or str(binding['bound_at']) != str(reservation['created_at'])
|
|
or str(binding['created_at']) != str(reservation['created_at'])
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding lost its exact reservation target identity'
|
|
)
|
|
try:
|
|
bound_image = parse_docker_target(binding['bound_target'])['image']
|
|
except (TypeError, ValueError):
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding has an invalid immutable target identity'
|
|
) from None
|
|
if bound_image.rsplit('@', 1)[-1] != str(binding['manifest_digest']):
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding manifest identity conflicts with its target'
|
|
)
|
|
return binding
|
|
experiment_target = self.conn.execute(
|
|
'''SELECT id FROM docker_depth_experiment_targets
|
|
WHERE target_queue_id = ? LIMIT 1 FOR UPDATE''',
|
|
(int(reservation['queue_id']),),
|
|
).fetchone()
|
|
if experiment_target:
|
|
raise ScanEventConflictError(
|
|
'Docker depth target reservation is missing its atomic binding'
|
|
)
|
|
return None
|
|
|
|
def _lock_docker_depth_experiment_for_reservation(self, reservation_id):
|
|
if not self._docker_depth_experiment_schema_installed():
|
|
return None
|
|
identities = self.conn.execute(
|
|
'''SELECT DISTINCT identity.experiment_id
|
|
FROM (
|
|
SELECT target.experiment_id
|
|
FROM docker_depth_experiment_scan_bindings binding
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.id = binding.experiment_target_id
|
|
WHERE binding.reservation_id = ?
|
|
UNION
|
|
SELECT target.experiment_id
|
|
FROM result_reservations reservation
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.target_queue_id = reservation.queue_id
|
|
WHERE reservation.id = ?
|
|
) identity
|
|
ORDER BY identity.experiment_id LIMIT 2''',
|
|
(int(reservation_id), int(reservation_id)),
|
|
).fetchall()
|
|
if not identities:
|
|
return None
|
|
if len(identities) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth reservation resolves to conflicting experiment authority'
|
|
)
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE id = ? FOR UPDATE''',
|
|
(int(identities[0]['experiment_id']),),
|
|
).fetchone()
|
|
if not experiment:
|
|
raise ScanEventConflictError(
|
|
'Docker depth reservation lost its experiment authority'
|
|
)
|
|
return experiment
|
|
|
|
def _transition_docker_depth_binding_locked(
|
|
self, reservation, binding_state, target_state, now, *, target_scan_id=None,
|
|
):
|
|
self._lock_docker_depth_experiment_for_reservation(reservation['id'])
|
|
binding = self._docker_depth_binding_for_reservation_locked(reservation)
|
|
if not binding:
|
|
return None
|
|
binding_state = str(binding_state)
|
|
target_state = str(target_state)
|
|
allowed = {
|
|
'reserved': ('reserved', 'quarantined'),
|
|
'scanning': ('reserved', 'scanning'),
|
|
'completed': ('reserved', 'scanning', 'completed'),
|
|
'failed': ('reserved', 'scanning', 'failed'),
|
|
'quarantined': ('reserved', 'scanning', 'quarantined'),
|
|
'released': ('reserved', 'released'),
|
|
}
|
|
allowed_targets = {
|
|
'reserved': ('reserved', 'quarantined'),
|
|
'scanning': ('reserved', 'scanning'),
|
|
'done': ('reserved', 'scanning', 'done'),
|
|
'failed': ('reserved', 'scanning', 'failed'),
|
|
'quarantined': ('reserved', 'scanning', 'quarantined'),
|
|
'pending': ('reserved', 'scanning', 'pending', 'quarantined'),
|
|
}
|
|
if str(binding['state']) not in allowed[binding_state]:
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding state conflicts with its reservation transition'
|
|
)
|
|
if str(binding['target_state']) not in allowed_targets[target_state]:
|
|
raise ScanEventConflictError(
|
|
'Docker depth target state conflicts with its reservation transition'
|
|
)
|
|
if binding['target_scan_id'] is not None and int(
|
|
binding['target_scan_id']
|
|
) != int(target_scan_id or 0):
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding scan identity changed across replay'
|
|
)
|
|
completed_at = now if binding_state in (
|
|
'completed', 'failed', 'quarantined', 'released',
|
|
) else None
|
|
binding_cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_scan_bindings
|
|
SET state = ?, target_scan_id = COALESCE(target_scan_id, ?),
|
|
scan_bound_at = COALESCE(scan_bound_at, ?),
|
|
completed_at = CASE WHEN ? = 0 THEN NULL
|
|
ELSE COALESCE(completed_at, ?) END
|
|
WHERE id = ? AND experiment_target_id = ?
|
|
AND reservation_id = ? AND attempt = ? AND state = ?''',
|
|
(
|
|
binding_state, target_scan_id,
|
|
now if target_scan_id is not None else None,
|
|
1 if completed_at is not None else 0, completed_at,
|
|
binding['id'], binding['experiment_target_id'],
|
|
reservation['id'], binding['attempt'], binding['state'],
|
|
),
|
|
)
|
|
if int(binding_cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding transition lost its exact reservation identity'
|
|
)
|
|
terminal_at = now if target_state in (
|
|
'done', 'failed', 'quarantined', 'held', 'skipped',
|
|
) else None
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_targets
|
|
SET state = ?, terminal_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ? AND target_queue_id = ?
|
|
AND manifest_id = ? AND reservation_count = ? AND state = ?''',
|
|
(
|
|
target_state, terminal_at, now, binding['experiment_target_id'],
|
|
binding['experiment_id'], reservation['queue_id'],
|
|
binding['manifest_id'], binding['reservation_count'],
|
|
binding['target_state'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth target state lost its reservation binding identity'
|
|
)
|
|
return {
|
|
'binding_id': int(binding['id']),
|
|
'experiment_id': int(binding['experiment_id']),
|
|
'experiment_target_id': int(binding['experiment_target_id']),
|
|
'manifest_id': int(binding['manifest_id']),
|
|
'attempt': int(binding['attempt']),
|
|
}
|
|
|
|
@staticmethod
|
|
def _normalize_docker_depth_resolver_authority(authority):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_COLLECTION_GENERATION,
|
|
DOCKER_DEPTH_QUERY_COUNT,
|
|
canonical_ordered_query_hash,
|
|
canonical_selector_hash,
|
|
reviewed_docker_experiment_profile,
|
|
)
|
|
|
|
if not isinstance(authority, dict) or authority.get('enabled') is not True:
|
|
raise ValueError('Docker depth resolver authority is disabled or invalid')
|
|
queries = authority.get('queries')
|
|
if isinstance(queries, (str, bytes)):
|
|
raise ValueError('Docker depth resolver query authority is invalid')
|
|
try:
|
|
queries = tuple(queries)
|
|
except TypeError:
|
|
raise ValueError('Docker depth resolver query authority is invalid') from None
|
|
if (
|
|
len(queries) != DOCKER_DEPTH_QUERY_COUNT
|
|
or len(set(queries)) != len(queries)
|
|
or any(not isinstance(query, str) or not query or query != query.strip()
|
|
for query in queries)
|
|
):
|
|
raise ValueError('Docker depth resolver query authority is invalid')
|
|
selector_version = str(authority.get('selector_version') or '')
|
|
profile = reviewed_docker_experiment_profile(selector_version)
|
|
integer_values = {
|
|
'query_count': profile['query_count'],
|
|
'repositories_per_query': profile['repositories_per_query'],
|
|
'images_per_repository': profile['images_per_repository'],
|
|
'target_limit': profile['target_limit'],
|
|
'theoretical_max_targets': profile['theoretical_max_targets'],
|
|
}
|
|
for key, expected in integer_values.items():
|
|
value = authority.get(key)
|
|
if isinstance(value, bool) or value != expected:
|
|
raise ValueError(f'Docker depth resolver {key} authority conflicts')
|
|
hashes = (
|
|
'config_sha256', 'ordered_queries_sha256', 'selector_sha256',
|
|
'provenance_policy_sha256',
|
|
)
|
|
if any(not re.fullmatch(r'[a-f0-9]{64}', str(authority.get(key) or '')) for key in hashes):
|
|
raise ValueError('Docker depth resolver hash authority is invalid')
|
|
if authority['ordered_queries_sha256'] != canonical_ordered_query_hash(queries):
|
|
raise ValueError('Docker depth resolver ordered-query authority conflicts')
|
|
if (
|
|
authority.get('selector_version') != selector_version
|
|
or authority['selector_sha256'] != canonical_selector_hash(selector_version)
|
|
):
|
|
raise ValueError('Docker depth resolver selector authority conflicts')
|
|
experiment_key = str(authority.get('experiment_key') or '')
|
|
if not re.fullmatch(r'[a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?', experiment_key):
|
|
raise ValueError('Docker depth resolver experiment key is invalid')
|
|
if authority.get('source') != 'dockerhub':
|
|
raise ValueError('Docker depth resolver source authority conflicts')
|
|
if authority.get('collection_generation') != DOCKER_DEPTH_COLLECTION_GENERATION:
|
|
raise ValueError('Docker depth resolver collection generation conflicts')
|
|
return {
|
|
**integer_values,
|
|
'enabled': True,
|
|
'experiment_key': experiment_key,
|
|
'source': 'dockerhub',
|
|
'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION,
|
|
'queries': queries,
|
|
'config_sha256': str(authority['config_sha256']),
|
|
'ordered_queries_sha256': str(authority['ordered_queries_sha256']),
|
|
'selector_version': selector_version,
|
|
'selector_sha256': str(authority['selector_sha256']),
|
|
'provenance_policy_sha256': str(authority['provenance_policy_sha256']),
|
|
}
|
|
|
|
@staticmethod
|
|
def _docker_depth_experiment_matches_authority(row, authority):
|
|
expected = {
|
|
'experiment_key': authority['experiment_key'],
|
|
'source': authority['source'],
|
|
'collection_generation': authority['collection_generation'],
|
|
'config_sha256': authority['config_sha256'],
|
|
'ordered_queries_sha256': authority['ordered_queries_sha256'],
|
|
'selector_version': authority['selector_version'],
|
|
'selector_sha256': authority['selector_sha256'],
|
|
'provenance_policy_sha256': authority['provenance_policy_sha256'],
|
|
'query_count': authority['query_count'],
|
|
'repositories_per_query': authority['repositories_per_query'],
|
|
'images_per_repository': authority['images_per_repository'],
|
|
'target_limit': authority['target_limit'],
|
|
}
|
|
return bool(row) and all(
|
|
(int(row[key]) if isinstance(value, int) else str(row[key])) == value
|
|
for key, value in expected.items()
|
|
)
|
|
|
|
def _terminalize_docker_depth_attempt_limit_locked(
|
|
self, experiment, member, authority, now, *, held_recovery=False,
|
|
):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
|
|
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
|
|
)
|
|
|
|
attempts = int(member['resolver_attempts'])
|
|
selected_count = int(member['selected_image_count'])
|
|
if attempts < DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit terminalization is premature'
|
|
)
|
|
stored_selection_count = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_selections
|
|
WHERE experiment_repository_id = ?''',
|
|
(member['id'],),
|
|
).fetchone()['count'])
|
|
if stored_selection_count != selected_count:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit selection evidence drifted'
|
|
)
|
|
current_queue_id = int(
|
|
member['replacement_repository_queue_id']
|
|
or member['repository_queue_id']
|
|
)
|
|
self._record_docker_depth_candidate_skip_locked(
|
|
experiment, member, current_queue_id, 'repository',
|
|
int(member['replacement_count']) + 1,
|
|
{'repository_queue_id': current_queue_id},
|
|
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON, now,
|
|
)
|
|
next_state = 'skipped' if selected_count == 0 else 'resolved'
|
|
next_error = (
|
|
DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON
|
|
if next_state == 'skipped' else None
|
|
)
|
|
if held_recovery:
|
|
if (
|
|
str(experiment['state']) != 'held'
|
|
or str(experiment['hold_reason_code'] or '')
|
|
!= 'resolver_attempt_limit'
|
|
or str(member['work_state']) != 'held'
|
|
or str(member['last_error_code'] or '')
|
|
!= 'resolver_attempt_limit'
|
|
or any(member[name] is not None for name in (
|
|
'resolver_owner', 'resolver_token', 'resolver_expires_at',
|
|
'resolver_due_at',
|
|
))
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth held attempt-limit evidence drifted'
|
|
)
|
|
held_count = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ? AND work_state = 'held' ''',
|
|
(experiment['id'],),
|
|
).fetchone()['count'])
|
|
if held_count != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit hold is ambiguous'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = ?, resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = NULL, last_error_code = ?,
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ? AND work_state = 'held'
|
|
AND last_error_code = 'resolver_attempt_limit'
|
|
AND resolver_attempts >= ?
|
|
AND resolver_owner IS NULL AND resolver_token IS NULL
|
|
AND resolver_expires_at IS NULL AND resolver_due_at IS NULL''',
|
|
(
|
|
next_state, next_error, now, now, member['id'],
|
|
experiment['id'], DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit recovery lost its member fence'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'resolving', hold_reason_code = NULL,
|
|
held_at = NULL, updated_at = ?
|
|
WHERE id = ? AND state = 'held'
|
|
AND hold_reason_code = 'resolver_attempt_limit'
|
|
AND fence_owner IS NULL AND fence_token IS NULL
|
|
AND fence_expires_at IS NULL''',
|
|
(now, experiment['id']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit recovery lost its experiment fence'
|
|
)
|
|
else:
|
|
if (
|
|
str(experiment['state']) != 'resolving'
|
|
or str(member['work_state']) != 'resolving'
|
|
or not member['resolver_owner']
|
|
or not member['resolver_token']
|
|
or str(member['resolver_expires_at'] or '') <= now
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit lease evidence drifted'
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = ?, resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = NULL, last_error_code = ?,
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ? AND work_state = 'resolving'
|
|
AND resolver_generation = ? AND resolver_owner = ?
|
|
AND resolver_token = ? AND resolver_expires_at > ?
|
|
AND resolver_attempts >= ?''',
|
|
(
|
|
next_state, next_error, now, now, member['id'],
|
|
experiment['id'], member['resolver_generation'],
|
|
member['resolver_owner'], member['resolver_token'], now,
|
|
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth attempt-limit terminalization lost its lease fence'
|
|
)
|
|
|
|
if selected_count == 0:
|
|
self._finalize_docker_depth_query_breadth_locked(
|
|
experiment, member, now,
|
|
)
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
experiment, selection_reason = (
|
|
self._freeze_docker_depth_runtime_selection_locked(
|
|
experiment, authority, now,
|
|
)
|
|
)
|
|
if selection_reason:
|
|
result = self._hold_docker_depth_experiment_locked(
|
|
experiment, selection_reason, now,
|
|
)
|
|
result['experiment'] = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
return result
|
|
drift_reason = self._docker_depth_persisted_drift_reason_locked(
|
|
experiment, authority, now,
|
|
)
|
|
if drift_reason:
|
|
result = self._hold_docker_depth_experiment_locked(
|
|
experiment, drift_reason, now,
|
|
)
|
|
result['experiment'] = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
return result
|
|
experiment = self._advance_docker_depth_experiment_state_locked(
|
|
experiment, now,
|
|
)
|
|
return {
|
|
'status': 'skipped' if next_state == 'skipped' else 'resolved',
|
|
'committed': True,
|
|
'reason': DOCKER_DEPTH_REMOTE_UNAVAILABLE_SKIP_REASON,
|
|
'experiment_state': str(experiment['state']),
|
|
'experiment': experiment,
|
|
}
|
|
|
|
def claim_docker_depth_experiment_resolutions(
|
|
self, source, limit, lease_owner, lease_seconds=300, *, authority=None,
|
|
final_cutover=False,
|
|
):
|
|
if (
|
|
not self.conn
|
|
or not self.conn.is_postgres
|
|
or source != 'dockerhub'
|
|
or not lease_owner
|
|
):
|
|
return []
|
|
if isinstance(authority, dict) and authority.get('enabled') is False:
|
|
self._hold_disabled_docker_depth_experiment(authority)
|
|
return []
|
|
if not isinstance(authority, dict) or authority.get('enabled') is not True:
|
|
return []
|
|
try:
|
|
normalized = self._normalize_docker_depth_resolver_authority(authority)
|
|
except (TypeError, ValueError):
|
|
return []
|
|
from docker_depth_experiment import (
|
|
DOCKER_RANK1_BREADTH_SELECTOR_VERSION,
|
|
_require_released_policy_history,
|
|
)
|
|
if normalized['selector_version'] == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
|
|
original_queue_policy_clause = "queue.status IN ('pending','deferred')"
|
|
else:
|
|
original_queue_policy_clause = '''queue.status IN ('pending','deferred')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id
|
|
)'''
|
|
|
|
def op():
|
|
state = self._locked_runtime_control_state(shared=True)
|
|
if state['effective_discovery_paused']:
|
|
self.conn.commit()
|
|
return []
|
|
now = utc_now_iso()
|
|
lease_until = datetime.fromtimestamp(
|
|
time.time() + max(60, min(3600, int(lease_seconds or 300))),
|
|
timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
experiment, _reason = self._locked_docker_depth_experiment_authority(
|
|
normalized, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not experiment:
|
|
self.conn.commit()
|
|
return []
|
|
if (
|
|
str(experiment['state']) == 'held'
|
|
and str(experiment['hold_reason_code'] or '')
|
|
== 'resolver_attempt_limit'
|
|
):
|
|
held_members = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ? AND work_state = 'held'
|
|
ORDER BY id LIMIT 2 FOR UPDATE''',
|
|
(experiment['id'],),
|
|
).fetchall()
|
|
if len(held_members) != 1:
|
|
self.conn.commit()
|
|
return []
|
|
terminal = self._terminalize_docker_depth_attempt_limit_locked(
|
|
experiment, held_members[0], normalized, now,
|
|
held_recovery=True,
|
|
)
|
|
experiment = terminal['experiment']
|
|
if str(experiment['state']) != 'resolving':
|
|
self.conn.commit()
|
|
return []
|
|
if (
|
|
str(experiment['state']) == 'held'
|
|
and str(experiment['hold_reason_code'] or '')
|
|
== 'stale_resolver_fence'
|
|
):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'resolving', hold_reason_code = NULL,
|
|
held_at = NULL, updated_at = ?
|
|
WHERE id = ? AND state = 'held'
|
|
AND hold_reason_code = 'stale_resolver_fence'
|
|
AND fence_owner IS NULL AND fence_token IS NULL
|
|
AND fence_expires_at IS NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_repositories member
|
|
WHERE member.experiment_id = ?
|
|
AND (member.work_state = 'resolving'
|
|
OR member.resolver_owner IS NOT NULL
|
|
OR member.resolver_token IS NOT NULL
|
|
OR member.resolver_expires_at IS NOT NULL)
|
|
)''',
|
|
(now, experiment['id'], experiment['id']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.commit()
|
|
return []
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
if str(experiment['state']) == 'holding':
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET state = 'resolving', updated_at = ?
|
|
WHERE id = ? AND state = 'holding'
|
|
AND plan_sha256 = ? AND hold_manifest_sha256 = ?''',
|
|
(
|
|
now, experiment['id'], experiment['plan_sha256'],
|
|
experiment['hold_manifest_sha256'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'activation_fence_conflict', now,
|
|
)
|
|
self.conn.commit()
|
|
return []
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
experiment = self._advance_docker_depth_experiment_state_locked(
|
|
experiment, now,
|
|
)
|
|
if str(experiment['state']) != 'resolving':
|
|
self.conn.commit()
|
|
return []
|
|
|
|
output = []
|
|
maximum = max(1, min(100, int(limit or 1)))
|
|
if normalized['selector_version'] == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
|
|
maximum = 1
|
|
for _ in range(maximum):
|
|
row = self.conn.execute(
|
|
f'''SELECT member.*, queue.id AS resolution_repository_queue_id,
|
|
queue.target
|
|
FROM docker_depth_experiment_repositories member
|
|
JOIN target_queue queue ON queue.id = COALESCE(
|
|
member.replacement_repository_queue_id,
|
|
member.repository_queue_id
|
|
)
|
|
WHERE member.experiment_id = ?
|
|
AND member.selected_image_count = 0
|
|
AND (
|
|
(member.work_state = 'pending'
|
|
AND (member.resolver_due_at IS NULL OR member.resolver_due_at <= ?))
|
|
OR (member.work_state = 'resolving'
|
|
AND member.resolver_expires_at <= ?)
|
|
)
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND (
|
|
({original_queue_policy_clause})
|
|
OR (member.replacement_repository_queue_id IS NOT NULL
|
|
AND queue.status = 'cold'
|
|
AND (SELECT COUNT(*) FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id) = 1
|
|
AND EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id
|
|
AND event.action = 'cold'
|
|
AND event.experiment_id = member.experiment_id
|
|
AND event.reason_code IN (
|
|
'docker_depth_experiment_hold',
|
|
'docker_depth_experiment_dynamic_hold'
|
|
)
|
|
AND event.config_sha256 = ?
|
|
AND event.policy_sha256 = ?
|
|
AND event.manifest_sha256 = ?
|
|
)))
|
|
AND queue.target_scan_id IS NULL
|
|
AND queue.target NOT LIKE '%@%' AND queue.normalized_target NOT LIKE '%@%'
|
|
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
|
|
AND queue.claim_batch IS NULL AND queue.current_result_reservation_id IS NULL
|
|
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_scans scan
|
|
WHERE scan.queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations reservation
|
|
WHERE reservation.queue_id = queue.id
|
|
)
|
|
ORDER BY member.repository_rank, member.query_ordinal, member.id
|
|
LIMIT 1 FOR UPDATE OF member SKIP LOCKED''',
|
|
(
|
|
experiment['id'], now, now, source,
|
|
experiment['config_sha256'],
|
|
experiment['provenance_policy_sha256'],
|
|
experiment['hold_manifest_sha256'],
|
|
),
|
|
).fetchone()
|
|
stage = 'breadth'
|
|
if not row:
|
|
unfinished_breadth = self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ? AND selected_image_count = 0
|
|
AND work_state IN ('pending','resolving')''',
|
|
(experiment['id'],),
|
|
).fetchone()['count']
|
|
if int(unfinished_breadth):
|
|
break
|
|
experiment, selection_reason = (
|
|
self._freeze_docker_depth_runtime_selection_locked(
|
|
experiment, normalized, now,
|
|
)
|
|
)
|
|
if selection_reason:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, selection_reason, now,
|
|
)
|
|
break
|
|
row = self.conn.execute(
|
|
'''SELECT member.*, queue.id AS resolution_repository_queue_id,
|
|
queue.target
|
|
FROM docker_depth_experiment_repositories member
|
|
JOIN target_queue queue ON queue.id = COALESCE(
|
|
member.replacement_repository_queue_id,
|
|
member.repository_queue_id
|
|
)
|
|
WHERE member.experiment_id = ? AND member.is_deep_probe = 1
|
|
AND member.selected_image_count >= 1
|
|
AND member.selected_image_count < member.candidate_distinct_graph_count
|
|
AND member.selected_image_count < ?
|
|
AND (
|
|
(member.work_state = 'pending'
|
|
AND (member.resolver_due_at IS NULL OR member.resolver_due_at <= ?))
|
|
OR (member.work_state = 'resolving'
|
|
AND member.resolver_expires_at <= ?)
|
|
)
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND (
|
|
(queue.status IN ('pending','deferred') AND NOT EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id
|
|
))
|
|
OR (member.replacement_repository_queue_id IS NOT NULL
|
|
AND queue.status = 'cold'
|
|
AND (SELECT COUNT(*) FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id) = 1
|
|
AND EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events event
|
|
WHERE event.queue_id = queue.id
|
|
AND event.action = 'cold'
|
|
AND event.experiment_id = member.experiment_id
|
|
AND event.reason_code IN (
|
|
'docker_depth_experiment_hold',
|
|
'docker_depth_experiment_dynamic_hold'
|
|
)
|
|
AND event.config_sha256 = ?
|
|
AND event.policy_sha256 = ?
|
|
AND event.manifest_sha256 = ?
|
|
)))
|
|
AND queue.target_scan_id IS NULL
|
|
AND queue.lease_owner IS NULL AND queue.lease_token IS NULL
|
|
AND queue.claim_batch IS NULL
|
|
AND queue.current_result_reservation_id IS NULL
|
|
AND queue.claim_event_id IS NULL AND queue.resolver_token IS NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_scans scan
|
|
WHERE scan.queue_id = queue.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations reservation
|
|
WHERE reservation.queue_id = queue.id
|
|
)
|
|
ORDER BY member.query_ordinal, member.repository_rank, member.id
|
|
LIMIT 1 FOR UPDATE OF member SKIP LOCKED''',
|
|
(
|
|
experiment['id'], normalized['images_per_repository'],
|
|
now, now, source, experiment['config_sha256'],
|
|
experiment['provenance_policy_sha256'],
|
|
experiment['hold_manifest_sha256'],
|
|
),
|
|
).fetchone()
|
|
stage = 'deep'
|
|
if not row:
|
|
break
|
|
if (
|
|
normalized['selector_version']
|
|
== DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
and row['replacement_repository_queue_id'] is None
|
|
):
|
|
try:
|
|
_require_released_policy_history(
|
|
self.conn, [int(row['resolution_repository_queue_id'])],
|
|
)
|
|
except RuntimeError:
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'target_history_drift', now,
|
|
)
|
|
self.conn.commit()
|
|
return []
|
|
token = secrets.token_urlsafe(32)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'resolving',
|
|
resolver_generation = resolver_generation + 1,
|
|
resolver_owner = ?, resolver_token = ?, resolver_expires_at = ?,
|
|
resolver_due_at = NULL, resolver_attempts = resolver_attempts + 1,
|
|
updated_at = ?
|
|
WHERE id = ?''',
|
|
(lease_owner, token, lease_until, now, row['id']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker depth resolver claim lost its row fence')
|
|
output.append({
|
|
'id': int(row['id']),
|
|
'experiment_id': int(experiment['id']),
|
|
'repository_queue_id': int(row['resolution_repository_queue_id']),
|
|
'query_ordinal': int(row['query_ordinal']),
|
|
'query': str(row['query']),
|
|
'repository_rank': int(row['repository_rank']),
|
|
'target': str(row['target']),
|
|
'stage': stage,
|
|
'selection_limit': (
|
|
1 if stage == 'breadth'
|
|
else normalized['images_per_repository']
|
|
),
|
|
'resolver_owner': str(lease_owner),
|
|
'resolver_token': token,
|
|
'resolver_generation': int(row['resolver_generation']) + 1,
|
|
'resolver_attempts': int(row['resolver_attempts']) + 1,
|
|
'resolver_expires_at': lease_until,
|
|
'config_sha256': normalized['config_sha256'],
|
|
'ordered_queries_sha256': normalized['ordered_queries_sha256'],
|
|
'selector_sha256': normalized['selector_sha256'],
|
|
})
|
|
self.conn.commit()
|
|
return output
|
|
|
|
return self._safe('claim_docker_depth_experiment_resolutions', op, [])
|
|
|
|
def renew_docker_depth_experiment_resolution(
|
|
self, source, repository_id, resolver_generation, resolver_token, *,
|
|
resolver_owner, lease_seconds=300, authority=None, final_cutover=False,
|
|
):
|
|
if (
|
|
not self.conn
|
|
or not self.conn.is_postgres
|
|
or source != 'dockerhub'
|
|
or not resolver_owner
|
|
or not resolver_token
|
|
):
|
|
return {'status': 'unavailable', 'renewed': False}
|
|
try:
|
|
normalized = self._normalize_docker_depth_resolver_authority(authority)
|
|
repository_id = int(repository_id)
|
|
resolver_generation = int(resolver_generation)
|
|
lease_seconds = int(lease_seconds)
|
|
if (
|
|
repository_id < 1
|
|
or resolver_generation < 1
|
|
or not 60 <= lease_seconds <= 3600
|
|
):
|
|
raise ValueError('Docker depth resolver renewal identity is invalid')
|
|
except (TypeError, ValueError, OverflowError):
|
|
return {'status': 'invalid', 'renewed': False}
|
|
now_dt = datetime.now(timezone.utc)
|
|
now = now_dt.isoformat(timespec='seconds')
|
|
lease_until = (now_dt + timedelta(seconds=lease_seconds)).isoformat(
|
|
timespec='seconds',
|
|
)
|
|
try:
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
|
|
(normalized['experiment_key'], source),
|
|
).fetchone()
|
|
member = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_repositories
|
|
WHERE id = ? AND experiment_id = ? FOR UPDATE''',
|
|
(repository_id, experiment['id'] if experiment else 0),
|
|
).fetchone()
|
|
if not (
|
|
experiment
|
|
and str(experiment['state']) == 'resolving'
|
|
and member
|
|
and str(member['work_state']) == 'resolving'
|
|
and int(member['resolver_generation']) == resolver_generation
|
|
and str(member['resolver_owner']) == str(resolver_owner)
|
|
and str(member['resolver_token']) == str(resolver_token)
|
|
and str(member['resolver_expires_at'] or '') > now
|
|
):
|
|
self.conn.rollback()
|
|
return {'status': 'stale', 'renewed': False}
|
|
checked, reason = self._locked_docker_depth_experiment_authority(
|
|
normalized, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not checked:
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'held', 'renewed': False, 'reason': reason,
|
|
}
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET resolver_expires_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ?
|
|
AND work_state = 'resolving'
|
|
AND resolver_generation = ? AND resolver_owner = ?
|
|
AND resolver_token = ? AND resolver_expires_at > ?''',
|
|
(
|
|
lease_until, now, repository_id, experiment['id'],
|
|
resolver_generation, resolver_owner, resolver_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return {'status': 'stale', 'renewed': False}
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'renewed', 'renewed': True,
|
|
'resolver_expires_at': lease_until,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def reserve_docker_depth_experiment_target_capacity_locked(
|
|
self, experiment_id, target_queue_id, manifest_id,
|
|
dispatch_wave, dispatch_order, now=None,
|
|
):
|
|
"""Reserve one deduplicated target slot inside the caller's transaction."""
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('Docker depth target capacity requires PostgreSQL')
|
|
values = (
|
|
experiment_id, target_queue_id, manifest_id, dispatch_wave, dispatch_order,
|
|
)
|
|
if any(isinstance(value, bool) for value in values):
|
|
raise ValueError('Docker depth target capacity identity is invalid')
|
|
try:
|
|
experiment_id, target_queue_id, manifest_id, dispatch_wave, dispatch_order = (
|
|
int(value) for value in values
|
|
)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('Docker depth target capacity identity is invalid') from None
|
|
if (
|
|
min(experiment_id, target_queue_id, manifest_id, dispatch_order) < 1
|
|
or dispatch_wave not in (1, 2, 3)
|
|
):
|
|
raise ValueError('Docker depth target capacity identity is invalid')
|
|
experiment = self.conn.execute(
|
|
'''SELECT id, state, target_count, target_limit
|
|
FROM docker_depth_experiments WHERE id = ? FOR UPDATE''',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
if not experiment or experiment['state'] not in ('resolving', 'active'):
|
|
raise ScanEventConflictError('Docker depth target capacity has no active authority')
|
|
actual_count = self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ?''',
|
|
(experiment_id,),
|
|
).fetchone()['count']
|
|
target_count = int(experiment['target_count'])
|
|
if int(actual_count) != target_count:
|
|
raise ScanEventConflictError('Docker depth target capacity counter drifted')
|
|
existing = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ? AND target_queue_id = ? FOR UPDATE''',
|
|
(experiment_id, target_queue_id),
|
|
).fetchone()
|
|
if existing:
|
|
if int(existing['manifest_id']) != manifest_id:
|
|
raise ScanEventConflictError('Docker depth target manifest identity conflicts')
|
|
if (dispatch_wave, dispatch_order) < (
|
|
int(existing['dispatch_wave']), int(existing['dispatch_order']),
|
|
):
|
|
existing = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_targets
|
|
SET dispatch_wave = ?, dispatch_order = ?, updated_at = ?
|
|
WHERE id = ? RETURNING *''',
|
|
(dispatch_wave, dispatch_order, str(now or utc_now_iso()), existing['id']),
|
|
).fetchone()
|
|
return {'target': dict(existing), 'inserted': False}
|
|
if target_count >= int(experiment['target_limit']):
|
|
raise ScanEventConflictError('Docker depth unique target capacity is exhausted')
|
|
now = str(now or utc_now_iso())
|
|
target = self.conn.execute(
|
|
'''INSERT INTO docker_depth_experiment_targets(
|
|
experiment_id, target_queue_id, manifest_id, counter_ordinal,
|
|
state, dispatch_wave, dispatch_order, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, 'pending', ?, ?, ?, ?) RETURNING *''',
|
|
(
|
|
experiment_id, target_queue_id, manifest_id, target_count + 1,
|
|
dispatch_wave, dispatch_order, now, now,
|
|
),
|
|
).fetchone()
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET target_count = target_count + 1, updated_at = ?
|
|
WHERE id = ? AND target_count = ? AND target_count < target_limit''',
|
|
(now, experiment_id, target_count),
|
|
)
|
|
if not target or int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker depth target capacity lost its serialized fence')
|
|
return {'target': dict(target), 'inserted': True}
|
|
|
|
@staticmethod
|
|
def _normalize_docker_depth_resolution_outcome(outcome, repository, authority, stage):
|
|
from docker_depth_experiment import (
|
|
canonical_docker_depth_selection_evidence_hash,
|
|
canonical_docker_descriptor_hash,
|
|
canonical_docker_layer_graph_hash,
|
|
)
|
|
|
|
def value(name, default=None):
|
|
if isinstance(outcome, dict):
|
|
return outcome.get(name, default)
|
|
return getattr(outcome, name, default)
|
|
|
|
if (
|
|
not outcome
|
|
or value('selector_version') != authority['selector_version']
|
|
or value('selector_hash') != authority['selector_sha256']
|
|
or value('fresh_graph_evidence') is not True
|
|
or value('cache_bypassed') is not True
|
|
):
|
|
raise ScanEventConflictError('Docker depth resolver outcome lacks fresh selector evidence')
|
|
candidate_count = value('candidate_distinct_graph_count')
|
|
if (
|
|
isinstance(candidate_count, bool)
|
|
or not isinstance(candidate_count, int)
|
|
or not 0 <= candidate_count <= 100
|
|
):
|
|
raise ScanEventConflictError('Docker depth candidate graph count is invalid')
|
|
selected_records = value('selection_records', ())
|
|
if isinstance(selected_records, (str, bytes)):
|
|
raise ScanEventConflictError('Docker depth selection evidence is invalid')
|
|
try:
|
|
selected_records = list(selected_records)
|
|
except TypeError:
|
|
raise ScanEventConflictError('Docker depth selection evidence is invalid') from None
|
|
candidate_records = value('candidate_records', selected_records)
|
|
if not candidate_records:
|
|
candidate_records = selected_records
|
|
if isinstance(candidate_records, (str, bytes)):
|
|
raise ScanEventConflictError('Docker depth candidate evidence is invalid')
|
|
try:
|
|
candidate_records = list(candidate_records)
|
|
except TypeError:
|
|
raise ScanEventConflictError('Docker depth candidate evidence is invalid') from None
|
|
selected_maximum = 1 if stage == 'breadth' else authority['images_per_repository']
|
|
if (
|
|
len(selected_records) > selected_maximum
|
|
or len(candidate_records) > 100
|
|
or candidate_count < len(candidate_records)
|
|
or candidate_records[:len(selected_records)] != selected_records
|
|
):
|
|
raise ScanEventConflictError('Docker depth selection count conflicts with graph evidence')
|
|
tags = value('tags', ())
|
|
if list(tags or ()) != [
|
|
record.get('target') for record in selected_records
|
|
if isinstance(record, dict)
|
|
]:
|
|
raise ScanEventConflictError('Docker depth selected targets conflict with selection evidence')
|
|
|
|
normalized_records = []
|
|
seen_targets = set()
|
|
seen_graphs = set()
|
|
for expected_rank, raw in enumerate(candidate_records, 1):
|
|
if not isinstance(raw, dict):
|
|
raise ScanEventConflictError('Docker depth selection record is invalid')
|
|
rank = raw.get('image_rank', raw.get('rank'))
|
|
reason = str(raw.get('selection_reason', raw.get('reason')) or '')
|
|
if (
|
|
isinstance(rank, bool)
|
|
or rank != expected_rank
|
|
or not reason
|
|
or len(reason) > 128
|
|
):
|
|
raise ScanEventConflictError('Docker depth selection rank or reason is invalid')
|
|
target = str(raw.get('target') or '').strip()
|
|
try:
|
|
parsed = parse_docker_target(target)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ScanEventConflictError('Docker depth immutable target is invalid') from exc
|
|
image = str(parsed['image']).lower()
|
|
if '@' not in image or parsed['target'] != target:
|
|
raise ScanEventConflictError('Docker depth immutable target is not canonical')
|
|
target_repository, target_digest = image.rsplit('@', 1)
|
|
manifest_digest = str(raw.get('manifest_digest') or '').lower()
|
|
if (
|
|
target_repository != repository
|
|
or str(raw.get('repository') or '').lower() != repository
|
|
or manifest_digest != target_digest
|
|
or not re.fullmatch(r'sha256:[a-f0-9]{64}', manifest_digest)
|
|
):
|
|
raise ScanEventConflictError('Docker depth repository or manifest identity conflicts')
|
|
manifest_media_type = str(raw.get('manifest_media_type') or '')
|
|
config_digest = str(raw.get('config_digest') or '').lower()
|
|
manifest_size_bytes = raw.get('manifest_size_bytes')
|
|
if (
|
|
not manifest_media_type
|
|
or manifest_media_type != manifest_media_type.strip().lower()
|
|
or len(manifest_media_type) > 256
|
|
or not re.fullmatch(r'sha256:[a-f0-9]{64}', config_digest)
|
|
or isinstance(manifest_size_bytes, bool)
|
|
or not isinstance(manifest_size_bytes, int)
|
|
or not 0 <= manifest_size_bytes <= 128 * 1024 * 1024
|
|
):
|
|
raise ScanEventConflictError('Docker depth manifest metadata is invalid')
|
|
layer_metadata = raw.get('layer_metadata')
|
|
if isinstance(layer_metadata, (str, bytes)):
|
|
raise ScanEventConflictError('Docker depth layer descriptors are invalid')
|
|
try:
|
|
layer_metadata = list(layer_metadata)
|
|
except TypeError:
|
|
raise ScanEventConflictError('Docker depth layer descriptors are invalid') from None
|
|
if not layer_metadata or len(layer_metadata) > 10000:
|
|
raise ScanEventConflictError('Docker depth layer descriptor count is invalid')
|
|
layers = []
|
|
layer_count = len(layer_metadata)
|
|
for position, descriptor in enumerate(layer_metadata, 1):
|
|
if not isinstance(descriptor, dict):
|
|
raise ScanEventConflictError('Docker depth layer descriptor is invalid')
|
|
digest = str(descriptor.get('digest') or '').lower()
|
|
media_type = str(descriptor.get('media_type') or '')
|
|
size_bytes = descriptor.get('size_bytes')
|
|
descriptor_sha256 = str(descriptor.get('descriptor_sha256') or '')
|
|
if (
|
|
not re.fullmatch(r'sha256:[a-f0-9]{64}', digest)
|
|
or not media_type
|
|
or media_type != media_type.strip().lower()
|
|
or len(media_type) > 256
|
|
or isinstance(size_bytes, bool)
|
|
or not isinstance(size_bytes, int)
|
|
or not 0 <= size_bytes <= 1024 * 1024 * 1024 * 1024
|
|
or descriptor.get('position_from_base') != position
|
|
or descriptor.get('position_from_top') != layer_count - position + 1
|
|
or descriptor_sha256
|
|
!= canonical_docker_descriptor_hash(digest, media_type, size_bytes)
|
|
):
|
|
raise ScanEventConflictError('Docker depth layer descriptor evidence conflicts')
|
|
layers.append({
|
|
'digest': digest,
|
|
'media_type': media_type,
|
|
'size_bytes': size_bytes,
|
|
'descriptor_sha256': descriptor_sha256,
|
|
'position_from_base': position,
|
|
'position_from_top': layer_count - position + 1,
|
|
})
|
|
graph = tuple(layer['digest'] for layer in layers)
|
|
raw_graph = tuple(raw.get('graph', raw.get('layers', ())) or ())
|
|
graph_sha256 = canonical_docker_layer_graph_hash(graph)
|
|
if (
|
|
raw_graph != graph
|
|
or raw.get('layer_count') != layer_count
|
|
or raw.get('graph_sha256', raw.get('graph_hash')) != graph_sha256
|
|
):
|
|
raise ScanEventConflictError('Docker depth ordered layer graph evidence conflicts')
|
|
normalized_target = normalize_target(target, 'docker')
|
|
if normalized_target != image or normalized_target in seen_targets or graph in seen_graphs:
|
|
raise ScanEventConflictError('Docker depth duplicate target or graph consumed a rank')
|
|
seen_targets.add(normalized_target)
|
|
seen_graphs.add(graph)
|
|
evidence = {
|
|
'schema': 1,
|
|
'type': 'docker-depth-selection-evidence-v1',
|
|
'selector_version': authority['selector_version'],
|
|
'selector_sha256': authority['selector_sha256'],
|
|
'candidate_distinct_graph_count': candidate_count,
|
|
'image_rank': rank,
|
|
'selection_reason': reason,
|
|
'target': target,
|
|
'repository': repository,
|
|
'manifest_digest': manifest_digest,
|
|
'manifest_media_type': manifest_media_type,
|
|
'manifest_size_bytes': manifest_size_bytes,
|
|
'config_digest': config_digest,
|
|
'graph_sha256': graph_sha256,
|
|
'layers': layers,
|
|
}
|
|
selection_evidence_sha256 = canonical_docker_depth_selection_evidence_hash(evidence)
|
|
if str(raw.get('selection_evidence_sha256') or '') != selection_evidence_sha256:
|
|
raise ScanEventConflictError('Docker depth selector evidence hash conflicts')
|
|
normalized_records.append({
|
|
**evidence,
|
|
'normalized_target': normalized_target,
|
|
'selection_evidence_sha256': selection_evidence_sha256,
|
|
})
|
|
return candidate_count, normalized_records
|
|
|
|
def _docker_depth_exact_reactivation_locked(self, experiment_id, queue):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
DOCKER_DEPTH_HOLD_REASON,
|
|
DOCKER_DEPTH_RELEASE_REASON,
|
|
)
|
|
|
|
experiment = self.conn.execute(
|
|
'''SELECT config_sha256, provenance_policy_sha256,
|
|
hold_manifest_sha256
|
|
FROM docker_depth_experiments WHERE id = ?''',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
events = self.conn.execute(
|
|
'''SELECT * FROM target_queue_policy_events
|
|
WHERE queue_id = ? ORDER BY id LIMIT 3 FOR UPDATE''',
|
|
(queue['id'],),
|
|
).fetchall()
|
|
if not experiment or len(events) != 2:
|
|
return False
|
|
cold, reactivation = events
|
|
if (
|
|
cold['action'] != 'cold'
|
|
or reactivation['action'] != 'reactivate'
|
|
or int(reactivation['reverses_event_id'] or 0) != int(cold['id'])
|
|
or any(
|
|
event['experiment_id'] is None
|
|
or int(event['experiment_id']) != int(experiment_id)
|
|
or int(event['queue_id']) != int(queue['id'])
|
|
or str(event['source']) != str(queue['source'])
|
|
or str(event['platform']) != str(queue['platform'])
|
|
or str(event['query']) != str(queue['query'])
|
|
or str(event['config_sha256']) != str(experiment['config_sha256'])
|
|
or str(event['policy_sha256'])
|
|
!= str(experiment['provenance_policy_sha256'])
|
|
for event in events
|
|
)
|
|
or str(cold['manifest_sha256'])
|
|
!= str(experiment['hold_manifest_sha256'])
|
|
or str(cold['reason_code']) not in (
|
|
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
)
|
|
or str(reactivation['reason_code']) != DOCKER_DEPTH_RELEASE_REASON
|
|
or str(cold['next_status']) != 'cold'
|
|
or str(reactivation['prior_status']) != 'cold'
|
|
or str(reactivation['next_status']) != str(queue['status'])
|
|
or str(cold['prior_status']) != str(queue['status'])
|
|
):
|
|
return False
|
|
cold_entry = {
|
|
'queue_id': int(queue['id']),
|
|
'source': str(queue['source']),
|
|
'platform': str(queue['platform']),
|
|
'query': str(queue['query']),
|
|
'prior_status': str(cold['prior_status']),
|
|
'prior_updated_at': str(cold['prior_updated_at']),
|
|
}
|
|
reactivation_entry = {
|
|
'queue_id': int(queue['id']),
|
|
'source': str(queue['source']),
|
|
'platform': str(queue['platform']),
|
|
'query': str(queue['query']),
|
|
'cold_event_id': int(cold['id']),
|
|
'restore_status': str(reactivation['next_status']),
|
|
'prior_updated_at': str(reactivation['prior_updated_at']),
|
|
}
|
|
return (
|
|
str(cold['review_audit_sha256'])
|
|
== self._target_queue_policy_audit_sha256(
|
|
'cold', cold['manifest_sha256'], cold_entry, experiment_id,
|
|
)
|
|
and str(reactivation['review_audit_sha256'])
|
|
== self._target_queue_policy_audit_sha256(
|
|
'reactivate', reactivation['manifest_sha256'],
|
|
reactivation_entry, experiment_id,
|
|
)
|
|
)
|
|
|
|
def _record_docker_depth_candidate_skip_locked(
|
|
self, experiment, member, repository_queue_id, candidate_kind,
|
|
candidate_ordinal, candidate_identity, reason, now,
|
|
):
|
|
candidate_identity_sha256, evidence_sha256 = (
|
|
self._docker_depth_candidate_skip_evidence(
|
|
experiment['id'], member['id'], repository_queue_id,
|
|
candidate_kind, candidate_ordinal, candidate_identity, reason,
|
|
)
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_depth_experiment_candidate_skips(
|
|
experiment_id, experiment_repository_id, repository_queue_id,
|
|
candidate_kind, candidate_ordinal, reason_code,
|
|
candidate_identity_sha256, evidence_sha256, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(
|
|
experiment_repository_id, candidate_kind,
|
|
candidate_identity_sha256
|
|
) DO NOTHING''',
|
|
(
|
|
experiment['id'], member['id'], repository_queue_id,
|
|
candidate_kind, candidate_ordinal, reason,
|
|
candidate_identity_sha256, evidence_sha256, now,
|
|
),
|
|
)
|
|
|
|
def _docker_depth_immutable_candidate_conflict_locked(
|
|
self, experiment_id, source, record,
|
|
):
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM target_queue
|
|
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
|
|
(source, record['normalized_target']),
|
|
).fetchone()
|
|
if not row:
|
|
return None, None
|
|
experiment_target = self.conn.execute(
|
|
'''SELECT id FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ? AND target_queue_id = ?''',
|
|
(experiment_id, row['id']),
|
|
).fetchone()
|
|
historical = self.conn.execute(
|
|
'''SELECT
|
|
EXISTS(SELECT 1 FROM target_scans WHERE queue_id = ?) AS scanned,
|
|
EXISTS(SELECT 1 FROM result_reservations WHERE queue_id = ?) AS reserved,
|
|
EXISTS(SELECT 1 FROM target_queue_policy_events WHERE queue_id = ?) AS policy,
|
|
EXISTS(
|
|
SELECT 1 FROM result_reservations reservation
|
|
JOIN pipeline_quarantine quarantine
|
|
ON quarantine.reservation_id = reservation.id
|
|
WHERE reservation.queue_id = ?
|
|
) AS quarantined,
|
|
EXISTS(
|
|
SELECT 1 FROM docker_image_blob_coverage coverage
|
|
JOIN docker_content_blobs blob
|
|
ON blob.digest = coverage.blob_digest
|
|
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
|
|
WHERE coverage.queue_id = ?
|
|
AND blob.state IN ('leased','submitted')
|
|
) AS blob_fenced''',
|
|
(row['id'], row['id'], row['id'], row['id'], row['id']),
|
|
).fetchone()
|
|
if row['status'] in ('done', 'failed') or row['completed_at'] is not None or bool(
|
|
historical['scanned']
|
|
):
|
|
return 'immutable_target_terminal', int(row['id'])
|
|
if row['status'] == 'quarantined' or bool(historical['quarantined']):
|
|
return 'immutable_target_quarantined', int(row['id'])
|
|
if row['status'] == 'cold' or bool(historical['policy']):
|
|
return 'immutable_target_independently_cold', int(row['id'])
|
|
fenced = (
|
|
row['status'] == 'in_progress'
|
|
or bool(historical['reserved'])
|
|
or bool(historical['blob_fenced'])
|
|
or any(row[name] is not None for name in (
|
|
'lease_owner', 'lease_token', 'claim_batch', 'leased_at',
|
|
'lease_expires_at', 'current_result_reservation_id',
|
|
'claim_event_id', 'resolver_token',
|
|
))
|
|
)
|
|
if fenced:
|
|
return 'immutable_target_fenced', int(row['id'])
|
|
if experiment_target and row['status'] in ('pending', 'deferred'):
|
|
return None, int(row['id'])
|
|
return 'immutable_target_existing', int(row['id'])
|
|
|
|
def _docker_depth_repository_candidate_conflict(
|
|
self, experiment, queue_id, *, lock,
|
|
):
|
|
suffix = ' FOR UPDATE' if lock else ''
|
|
queue = self.conn.execute(
|
|
f'SELECT * FROM target_queue WHERE id = ?{suffix}',
|
|
(queue_id,),
|
|
).fetchone()
|
|
if not queue:
|
|
return 'repository_candidate_absent'
|
|
fences = any(queue[name] is not None for name in (
|
|
'target_scan_id', 'lease_owner', 'lease_token', 'claim_batch',
|
|
'leased_at', 'lease_expires_at', 'current_result_reservation_id',
|
|
'claim_event_id', 'resolver_token',
|
|
)) or str(queue['resolver_state'] or '') == 'resolving'
|
|
if fences or self.conn.execute(
|
|
'''SELECT 1 FROM target_scans WHERE queue_id = ?
|
|
UNION ALL SELECT 1 FROM result_reservations WHERE queue_id = ?
|
|
LIMIT 1''',
|
|
(queue_id, queue_id),
|
|
).fetchone():
|
|
return 'repository_candidate_fenced'
|
|
events = self.conn.execute(
|
|
'''SELECT * FROM target_queue_policy_events
|
|
WHERE queue_id = ? ORDER BY id LIMIT 3''',
|
|
(queue_id,),
|
|
).fetchall()
|
|
if queue['status'] in ('pending', 'deferred') and not events:
|
|
return None
|
|
if queue['status'] == 'cold' and len(events) == 1:
|
|
event = events[0]
|
|
if (
|
|
event['action'] == 'cold'
|
|
and int(event['experiment_id'] or 0) == int(experiment['id'])
|
|
and str(event['reason_code']) in (
|
|
'docker_depth_experiment_hold',
|
|
'docker_depth_experiment_dynamic_hold',
|
|
)
|
|
and str(event['config_sha256']) == str(experiment['config_sha256'])
|
|
and str(event['policy_sha256'])
|
|
== str(experiment['provenance_policy_sha256'])
|
|
and str(event['manifest_sha256'])
|
|
== str(experiment['hold_manifest_sha256'])
|
|
and str(event['next_status']) == 'cold'
|
|
and str(event['prior_status']) in ('pending', 'deferred')
|
|
and str(event['review_audit_sha256'])
|
|
== self._target_queue_policy_audit_sha256(
|
|
'cold', event['manifest_sha256'], {
|
|
'queue_id': int(queue['id']),
|
|
'source': str(queue['source']),
|
|
'platform': str(queue['platform']),
|
|
'query': str(queue['query']),
|
|
'prior_status': str(event['prior_status']),
|
|
'prior_updated_at': str(event['prior_updated_at']),
|
|
}, int(experiment['id']),
|
|
)
|
|
):
|
|
return None
|
|
if queue['status'] in ('done', 'failed'):
|
|
return 'repository_candidate_terminal'
|
|
if queue['status'] == 'quarantined':
|
|
return 'repository_candidate_quarantined'
|
|
if queue['status'] == 'cold' or events:
|
|
return 'repository_candidate_independently_cold'
|
|
return 'repository_candidate_ineligible'
|
|
|
|
def _docker_depth_repository_candidate_conflict_locked(
|
|
self, experiment, queue_id,
|
|
):
|
|
return self._docker_depth_repository_candidate_conflict(
|
|
experiment, queue_id, lock=True,
|
|
)
|
|
|
|
def _docker_depth_repository_replacement_candidates(
|
|
self, experiment, member, authority, *, lock,
|
|
):
|
|
candidates = self.conn.execute(
|
|
'''SELECT provenance.repository_queue_id,
|
|
MIN(observation.search_rank) AS best_search_rank,
|
|
MIN(observation.page_id) AS eligibility_page_id,
|
|
MIN(queue.normalized_target) AS normalized_target
|
|
FROM docker_repository_query_provenance provenance
|
|
JOIN docker_repository_query_observations observation
|
|
ON observation.source = provenance.source
|
|
AND observation.query = provenance.query
|
|
AND observation.repository_queue_id = provenance.repository_queue_id
|
|
JOIN docker_discovery_pages page ON page.id = observation.page_id
|
|
JOIN docker_discovery_passes discovery_pass ON discovery_pass.id = page.pass_id
|
|
JOIN target_queue queue ON queue.id = provenance.repository_queue_id
|
|
WHERE provenance.source = ? AND provenance.query = ?
|
|
AND provenance.provenance_kind = 'fresh_page'
|
|
AND provenance.fresh_coverage_eligible = 1
|
|
AND provenance.fresh_complete_observation_count > 0
|
|
AND page.query_ordinal = ? AND page.query = ?
|
|
AND discovery_pass.source = ? AND discovery_pass.pass_kind = 'deep'
|
|
AND discovery_pass.collection_generation = ?
|
|
AND discovery_pass.policy_sha256 = ?
|
|
AND discovery_pass.ordered_queries_sha256 = ?
|
|
AND discovery_pass.expected_query_count = ?
|
|
AND discovery_pass.state = 'complete'
|
|
AND queue.source = ? AND queue.platform = 'docker'
|
|
AND queue.target NOT LIKE '%@%' AND queue.normalized_target NOT LIKE '%@%'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_repositories other
|
|
WHERE other.experiment_id = ? AND other.query_ordinal = ?
|
|
AND (other.repository_queue_id = queue.id
|
|
OR other.replacement_repository_queue_id = queue.id)
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_candidate_skips skipped
|
|
WHERE skipped.experiment_repository_id = ?
|
|
AND skipped.candidate_kind = 'repository'
|
|
AND skipped.repository_queue_id = queue.id
|
|
)
|
|
GROUP BY provenance.repository_queue_id
|
|
ORDER BY MIN(observation.search_rank), provenance.repository_queue_id
|
|
LIMIT ?''',
|
|
(
|
|
authority['source'], member['query'], member['query_ordinal'],
|
|
member['query'], authority['source'],
|
|
authority['collection_generation'], authority['provenance_policy_sha256'],
|
|
authority['ordered_queries_sha256'], authority['query_count'],
|
|
authority['source'], experiment['id'], member['query_ordinal'],
|
|
member['id'], 3000,
|
|
),
|
|
).fetchall()
|
|
if lock and candidates:
|
|
ids = sorted({int(row['repository_queue_id']) for row in candidates})
|
|
placeholders = ','.join('?' for _ in ids)
|
|
self.conn.execute(
|
|
f'''SELECT id FROM target_queue WHERE id IN ({placeholders})
|
|
ORDER BY id FOR UPDATE''',
|
|
tuple(ids),
|
|
).fetchall()
|
|
inspected = []
|
|
for candidate in candidates:
|
|
conflict = self._docker_depth_repository_candidate_conflict(
|
|
experiment, int(candidate['repository_queue_id']), lock=lock,
|
|
)
|
|
inspected.append({
|
|
'repository_queue_id': int(candidate['repository_queue_id']),
|
|
'best_search_rank': int(candidate['best_search_rank']),
|
|
'eligibility_page_id': int(candidate['eligibility_page_id']),
|
|
'target_identity_sha256': hashlib.sha256(
|
|
str(candidate['normalized_target']).encode('utf-8')
|
|
).hexdigest(),
|
|
'conflict_reason': str(conflict or ''),
|
|
})
|
|
if not conflict:
|
|
break
|
|
return inspected
|
|
|
|
def _finalize_docker_depth_query_breadth_locked(
|
|
self, experiment, member, now,
|
|
):
|
|
query_row = self.conn.execute(
|
|
'''SELECT selected_repository_count
|
|
FROM docker_depth_experiment_queries
|
|
WHERE experiment_id = ? AND query_ordinal = ? FOR UPDATE''',
|
|
(experiment['id'], member['query_ordinal']),
|
|
).fetchone()
|
|
if not query_row:
|
|
raise ScanEventConflictError('Docker depth query authority is unavailable')
|
|
query_members = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_repositories
|
|
WHERE experiment_id = ? AND query_ordinal = ?
|
|
ORDER BY repository_rank, repository_queue_id, id
|
|
FOR UPDATE''',
|
|
(experiment['id'], member['query_ordinal']),
|
|
).fetchall()
|
|
if len(query_members) != int(query_row['selected_repository_count']):
|
|
raise ScanEventConflictError('Docker depth query cohort membership drifted')
|
|
if not all(
|
|
str(row['work_state']) in ('resolved', 'skipped')
|
|
for row in query_members
|
|
):
|
|
return False
|
|
for row in query_members:
|
|
reason, _evidence_sha256 = (
|
|
self._docker_depth_terminal_repository_evidence_locked(
|
|
experiment, row,
|
|
)
|
|
)
|
|
if reason:
|
|
raise ScanEventConflictError(
|
|
'Docker depth terminal repository evidence drifted'
|
|
)
|
|
from docker_depth_experiment import DOCKER_RANK1_BREADTH_SELECTOR_VERSION
|
|
if str(experiment['selector_version']) == DOCKER_RANK1_BREADTH_SELECTOR_VERSION:
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET is_deep_probe = 0, updated_at = ?
|
|
WHERE experiment_id = ? AND query_ordinal = ?
|
|
AND is_deep_probe <> 0''',
|
|
(now, experiment['id'], member['query_ordinal']),
|
|
)
|
|
return True
|
|
if self.conn.execute(
|
|
'''SELECT 1 FROM docker_depth_experiment_selections
|
|
WHERE experiment_id = ? AND query_ordinal = ? AND image_rank > 1
|
|
LIMIT 1''',
|
|
(experiment['id'], member['query_ordinal']),
|
|
).fetchone():
|
|
return True
|
|
eligible = [
|
|
row for row in query_members
|
|
if str(row['work_state']) == 'resolved'
|
|
and int(row['selected_image_count']) >= 1
|
|
]
|
|
deep_id = None
|
|
if eligible:
|
|
deep = min(eligible, key=lambda row: (
|
|
-int(row['candidate_distinct_graph_count']),
|
|
int(row['repository_rank']), int(row['repository_queue_id']),
|
|
))
|
|
deep_id = int(deep['id'])
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET is_deep_probe = CASE WHEN id = ? THEN 1 ELSE 0 END,
|
|
work_state = CASE
|
|
WHEN id = ? AND candidate_distinct_graph_count > selected_image_count
|
|
AND selected_image_count >= 1
|
|
THEN 'pending' ELSE work_state END,
|
|
resolved_at = CASE
|
|
WHEN id = ? AND candidate_distinct_graph_count > selected_image_count
|
|
AND selected_image_count >= 1
|
|
THEN NULL ELSE resolved_at END,
|
|
updated_at = ?
|
|
WHERE experiment_id = ? AND query_ordinal = ?''',
|
|
(
|
|
deep_id, deep_id, deep_id, now,
|
|
experiment['id'], member['query_ordinal'],
|
|
),
|
|
)
|
|
return True
|
|
|
|
def _replace_docker_depth_repository_locked(
|
|
self, experiment, member, authority, reason, now, candidate_count,
|
|
):
|
|
current_queue_id = int(
|
|
member['replacement_repository_queue_id']
|
|
or member['repository_queue_id']
|
|
)
|
|
self._record_docker_depth_candidate_skip_locked(
|
|
experiment, member, current_queue_id, 'repository',
|
|
int(member['replacement_count']) + 1,
|
|
{'repository_queue_id': current_queue_id}, reason, now,
|
|
)
|
|
candidates = self._docker_depth_repository_replacement_candidates(
|
|
experiment, member, authority, lock=True,
|
|
)
|
|
replacement = None
|
|
for candidate in candidates:
|
|
candidate_queue_id = candidate['repository_queue_id']
|
|
conflict = candidate['conflict_reason']
|
|
if conflict:
|
|
self._record_docker_depth_candidate_skip_locked(
|
|
experiment, member, candidate_queue_id, 'repository',
|
|
int(candidate['best_search_rank']),
|
|
{'repository_queue_id': candidate_queue_id}, conflict, now,
|
|
)
|
|
continue
|
|
replacement = candidate
|
|
break
|
|
if not replacement:
|
|
from docker_depth_experiment import DOCKER_DEPTH_REPOSITORY_SKIP_REASON
|
|
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'skipped', resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = NULL,
|
|
candidate_distinct_graph_count = CASE
|
|
WHEN candidate_distinct_graph_count >= ?
|
|
THEN candidate_distinct_graph_count ELSE ? END,
|
|
selected_image_count = 0, last_error_code = ?,
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ?
|
|
AND work_state = 'resolving'
|
|
AND resolver_generation = ? AND resolver_owner = ?
|
|
AND resolver_token = ? AND resolver_expires_at > ?''',
|
|
(
|
|
candidate_count, candidate_count,
|
|
DOCKER_DEPTH_REPOSITORY_SKIP_REASON, now, now,
|
|
member['id'], experiment['id'], member['resolver_generation'],
|
|
member['resolver_owner'], member['resolver_token'], now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth repository skip lost its lease fence'
|
|
)
|
|
self._finalize_docker_depth_query_breadth_locked(
|
|
experiment, member, now,
|
|
)
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
experiment, selection_reason = (
|
|
self._freeze_docker_depth_runtime_selection_locked(
|
|
experiment, authority, now,
|
|
)
|
|
)
|
|
if selection_reason:
|
|
return self._hold_docker_depth_experiment_locked(
|
|
experiment, selection_reason, now,
|
|
)
|
|
drift_reason = self._docker_depth_persisted_drift_reason_locked(
|
|
experiment, authority, now,
|
|
)
|
|
if drift_reason:
|
|
return self._hold_docker_depth_experiment_locked(
|
|
experiment, drift_reason, now,
|
|
)
|
|
experiment = self._advance_docker_depth_experiment_state_locked(
|
|
experiment, now,
|
|
)
|
|
return {
|
|
'status': 'skipped', 'committed': True,
|
|
'reason': DOCKER_DEPTH_REPOSITORY_SKIP_REASON,
|
|
'experiment_state': str(experiment['state']),
|
|
}
|
|
previous_hash = str(member['replacement_evidence_sha256'] or '')
|
|
replacement_document = {
|
|
'schema': 1,
|
|
'type': 'docker-depth-repository-replacement-v1',
|
|
'experiment_id': int(experiment['id']),
|
|
'experiment_repository_id': int(member['id']),
|
|
'replacement_number': int(member['replacement_count']) + 1,
|
|
'from_repository_queue_id': current_queue_id,
|
|
'to_repository_queue_id': int(replacement['repository_queue_id']),
|
|
'eligibility_page_id': int(replacement['eligibility_page_id']),
|
|
'best_search_rank': int(replacement['best_search_rank']),
|
|
'previous_evidence_sha256': previous_hash,
|
|
'reason_code': str(reason),
|
|
}
|
|
replacement_sha256 = hashlib.sha256(json.dumps(
|
|
replacement_document, ensure_ascii=True, allow_nan=False,
|
|
sort_keys=True, separators=(',', ':'),
|
|
).encode('utf-8')).hexdigest()
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET replacement_repository_queue_id = ?,
|
|
replacement_eligibility_page_id = ?,
|
|
replacement_count = replacement_count + 1,
|
|
replacement_evidence_sha256 = ?, work_state = 'pending',
|
|
resolver_owner = NULL, resolver_token = NULL,
|
|
resolver_expires_at = NULL, resolver_due_at = NULL,
|
|
resolver_attempts = 0, selected_image_count = 0,
|
|
candidate_distinct_graph_count = 0,
|
|
last_error_code = 'repository_candidate_replaced',
|
|
resolved_at = NULL, updated_at = ? WHERE id = ?''',
|
|
(
|
|
replacement['repository_queue_id'], replacement['eligibility_page_id'],
|
|
replacement_sha256, now, member['id'],
|
|
),
|
|
)
|
|
return {
|
|
'status': 'replaced', 'committed': True,
|
|
'reason': 'repository_candidate_replaced',
|
|
}
|
|
|
|
@staticmethod
|
|
def _rerank_docker_depth_selection_record(record, image_rank):
|
|
from docker_depth_experiment import canonical_docker_depth_selection_evidence_hash
|
|
evidence_keys = (
|
|
'schema', 'type', 'selector_version', 'selector_sha256',
|
|
'candidate_distinct_graph_count', 'selection_reason', 'target',
|
|
'repository', 'manifest_digest', 'manifest_media_type',
|
|
'manifest_size_bytes', 'config_digest', 'graph_sha256', 'layers',
|
|
)
|
|
evidence = {key: record[key] for key in evidence_keys}
|
|
original_rank = int(record['image_rank'])
|
|
evidence['image_rank'] = int(image_rank)
|
|
if original_rank != image_rank:
|
|
evidence['selection_reason'] = 'replacement_after_exclusion'
|
|
return {
|
|
**record,
|
|
**evidence,
|
|
'selection_evidence_sha256': canonical_docker_depth_selection_evidence_hash(
|
|
evidence
|
|
),
|
|
}
|
|
|
|
def _docker_depth_target_queue_locked(
|
|
self, experiment_id, source, query, record, now,
|
|
):
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM target_queue
|
|
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
|
|
(source, record['normalized_target']),
|
|
).fetchone()
|
|
inserted = False
|
|
if not row:
|
|
row = self.conn.execute(
|
|
'''INSERT INTO target_queue(
|
|
source, platform, query, target, normalized_target,
|
|
status, created_at, updated_at
|
|
) VALUES (?, 'docker', ?, ?, ?, 'pending', ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO NOTHING
|
|
RETURNING *''',
|
|
(
|
|
source, query, record['target'], record['normalized_target'],
|
|
now, now,
|
|
),
|
|
).fetchone()
|
|
if not row:
|
|
row = self.conn.execute(
|
|
'''SELECT * FROM target_queue
|
|
WHERE source = ? AND normalized_target = ? FOR UPDATE''',
|
|
(source, record['normalized_target']),
|
|
).fetchone()
|
|
else:
|
|
inserted = True
|
|
experiment_target = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ? AND target_queue_id = ? FOR UPDATE''',
|
|
(experiment_id, row['id']),
|
|
).fetchone() if row else None
|
|
audited_reactivation = bool(
|
|
row and not inserted and self._docker_depth_exact_reactivation_locked(
|
|
experiment_id, row,
|
|
)
|
|
)
|
|
if (
|
|
not row
|
|
or row['source'] != source
|
|
or row['platform'] != 'docker'
|
|
or row['status'] not in ('pending', 'deferred')
|
|
or normalize_target(row['target'], 'docker') != record['normalized_target']
|
|
or row['target_scan_id'] is not None
|
|
or row['lease_owner'] is not None
|
|
or row['lease_token'] is not None
|
|
or row['claim_batch'] is not None
|
|
or row['leased_at'] is not None
|
|
or row['lease_expires_at'] is not None
|
|
or row['current_result_reservation_id'] is not None
|
|
or row['claim_event_id'] is not None
|
|
or row['completed_at'] is not None
|
|
or (not inserted and not experiment_target and not audited_reactivation)
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth immutable target has prior or conflicting queue state'
|
|
)
|
|
historical = self.conn.execute(
|
|
'''SELECT
|
|
EXISTS(SELECT 1 FROM target_scans WHERE queue_id = ?) AS scanned,
|
|
EXISTS(SELECT 1 FROM result_reservations WHERE queue_id = ?) AS reserved,
|
|
EXISTS(SELECT 1 FROM target_queue_policy_events WHERE queue_id = ?) AS policy,
|
|
EXISTS(
|
|
SELECT 1 FROM result_reservations reservation
|
|
JOIN pipeline_quarantine quarantine
|
|
ON quarantine.reservation_id = reservation.id
|
|
WHERE reservation.queue_id = ?
|
|
) AS quarantined''',
|
|
(row['id'], row['id'], row['id'], row['id']),
|
|
).fetchone()
|
|
if (
|
|
bool(historical['scanned'])
|
|
or bool(historical['reserved'])
|
|
or bool(historical['quarantined'])
|
|
or (bool(historical['policy']) and not audited_reactivation)
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth immutable target has historical or independently cold state'
|
|
)
|
|
return dict(row)
|
|
|
|
def _persist_docker_depth_manifest_locked(self, queue_id, source, record, now):
|
|
expected = {
|
|
'source': source,
|
|
'repository': record['repository'],
|
|
'manifest_digest': record['manifest_digest'],
|
|
'manifest_media_type': record['manifest_media_type'],
|
|
'config_digest': record['config_digest'],
|
|
'graph_sha256': record['graph_sha256'],
|
|
'manifest_size_bytes': record['manifest_size_bytes'],
|
|
'layer_count': len(record['layers']),
|
|
}
|
|
manifest = self.conn.execute(
|
|
'''SELECT * FROM docker_image_manifests
|
|
WHERE target_queue_id = ? FOR UPDATE''',
|
|
(queue_id,),
|
|
).fetchone()
|
|
if manifest:
|
|
if any(
|
|
(int(manifest[key]) if isinstance(value, int) else str(manifest[key])) != value
|
|
for key, value in expected.items()
|
|
):
|
|
raise ScanEventConflictError('Docker depth immutable manifest metadata conflicts')
|
|
else:
|
|
manifest = self.conn.execute(
|
|
'''INSERT INTO docker_image_manifests(
|
|
target_queue_id, source, repository, manifest_digest,
|
|
manifest_media_type, config_digest, graph_sha256,
|
|
manifest_size_bytes, layer_count, resolved_at, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *''',
|
|
(
|
|
queue_id, source, record['repository'], record['manifest_digest'],
|
|
record['manifest_media_type'], record['config_digest'],
|
|
record['graph_sha256'], record['manifest_size_bytes'],
|
|
len(record['layers']), now, now,
|
|
),
|
|
).fetchone()
|
|
for layer in record['layers']:
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_manifest_layers(
|
|
manifest_id, position_from_base, position_from_top,
|
|
layer_digest, media_type, layer_size_bytes,
|
|
descriptor_sha256, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
manifest['id'], layer['position_from_base'],
|
|
layer['position_from_top'], layer['digest'],
|
|
layer['media_type'], layer['size_bytes'],
|
|
layer['descriptor_sha256'], now,
|
|
),
|
|
)
|
|
stored_layers = self.conn.execute(
|
|
'''SELECT position_from_base, position_from_top, layer_digest,
|
|
media_type, layer_size_bytes, descriptor_sha256
|
|
FROM docker_manifest_layers WHERE manifest_id = ?
|
|
ORDER BY position_from_base''',
|
|
(manifest['id'],),
|
|
).fetchall()
|
|
expected_layers = [(
|
|
layer['position_from_base'], layer['position_from_top'], layer['digest'],
|
|
layer['media_type'], layer['size_bytes'], layer['descriptor_sha256'],
|
|
) for layer in record['layers']]
|
|
if [(
|
|
int(layer['position_from_base']), int(layer['position_from_top']),
|
|
str(layer['layer_digest']), str(layer['media_type']),
|
|
int(layer['layer_size_bytes']), str(layer['descriptor_sha256']),
|
|
) for layer in stored_layers] != expected_layers:
|
|
raise ScanEventConflictError('Docker depth immutable manifest layers conflict')
|
|
return int(manifest['id'])
|
|
|
|
@staticmethod
|
|
def _docker_depth_dispatch_position(query_ordinal, repository_rank, image_rank, query_count):
|
|
if image_rank == 1:
|
|
return 1, (repository_rank - 1) * query_count + query_ordinal + 1
|
|
if image_rank <= 3:
|
|
return 2, (image_rank - 2) * query_count + query_ordinal + 1
|
|
return 3, (image_rank - 4) * query_count + query_ordinal + 1
|
|
|
|
def _defer_or_hold_docker_depth_resolution_conflict(
|
|
self, source, repository_id, resolver_generation, resolver_owner,
|
|
resolver_token, authority, error, *, final_cutover,
|
|
):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS,
|
|
)
|
|
|
|
now = utc_now_iso()
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
|
|
(authority['experiment_key'], source),
|
|
).fetchone()
|
|
member = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_repositories
|
|
WHERE id = ? AND experiment_id = ? FOR UPDATE''',
|
|
(repository_id, experiment['id'] if experiment else 0),
|
|
).fetchone()
|
|
if not (
|
|
experiment
|
|
and str(experiment['state']) == 'resolving'
|
|
and member
|
|
and member['work_state'] == 'resolving'
|
|
and int(member['resolver_generation']) == int(resolver_generation)
|
|
and member['resolver_owner'] == resolver_owner
|
|
and member['resolver_token'] == resolver_token
|
|
and str(member['resolver_expires_at'] or '') > now
|
|
):
|
|
self.conn.rollback()
|
|
return {'status': 'stale', 'committed': False}
|
|
checked, authority_reason = self._locked_docker_depth_experiment_authority(
|
|
authority, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not checked:
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'held', 'committed': True,
|
|
'reason': authority_reason,
|
|
}
|
|
attempts = int(member['resolver_attempts'])
|
|
capacity_conflict = 'capacity' in str(error).lower()
|
|
if capacity_conflict or attempts >= DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS:
|
|
reason = (
|
|
'resolver_capacity_conflict' if capacity_conflict
|
|
else 'resolver_evidence_attempt_limit'
|
|
)
|
|
self._hold_docker_depth_experiment_locked(experiment, reason, now)
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'held', resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = NULL, last_error_code = ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(reason, now, repository_id),
|
|
)
|
|
self.conn.commit()
|
|
return {'status': 'held', 'committed': True, 'reason': reason}
|
|
delay = min(
|
|
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS
|
|
* (2 ** min(8, max(0, attempts - 1))),
|
|
)
|
|
due = datetime.fromtimestamp(
|
|
time.time() + delay, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'pending', resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = ?, last_error_code = 'resolver_evidence_conflict',
|
|
updated_at = ? WHERE id = ?''',
|
|
(due, now, repository_id),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'conflict_retry', 'committed': True,
|
|
'retry_at': due, 'reason': 'resolver_evidence_conflict',
|
|
}
|
|
|
|
def finish_docker_depth_experiment_resolution(
|
|
self, source, repository_id, resolver_generation, resolver_token,
|
|
outcome=None, error='', complete=None, *, resolver_owner=None,
|
|
authority=None, retry_at=None, claim_attempt_consumed=True,
|
|
final_cutover=False,
|
|
):
|
|
if (
|
|
not self.conn
|
|
or not self.conn.is_postgres
|
|
or source != 'dockerhub'
|
|
or repository_id is None
|
|
or not resolver_token
|
|
or not resolver_owner
|
|
):
|
|
return {'status': 'unavailable', 'committed': False}
|
|
if isinstance(authority, dict) and authority.get('enabled') is False:
|
|
held = self._hold_disabled_docker_depth_experiment(authority)
|
|
return {
|
|
'status': 'held' if held else 'unavailable',
|
|
'committed': held, 'reason': 'experiment_disabled',
|
|
}
|
|
if not isinstance(authority, dict) or authority.get('enabled') is not True:
|
|
return {'status': 'unavailable', 'committed': False}
|
|
try:
|
|
normalized = self._normalize_docker_depth_resolver_authority(authority)
|
|
except (TypeError, ValueError):
|
|
return {
|
|
'status': 'invalid', 'committed': False,
|
|
'reason': 'authority_payload_invalid',
|
|
}
|
|
outcome_tags = (
|
|
outcome.get('tags', ()) if isinstance(outcome, dict)
|
|
else getattr(outcome, 'tags', ())
|
|
)
|
|
complete = bool(outcome_tags) if complete is None else bool(complete)
|
|
if not claim_attempt_consumed and (complete or not retry_at):
|
|
return {'status': 'invalid', 'committed': False}
|
|
try:
|
|
if complete:
|
|
self._require_discovery_admission_locked()
|
|
now = utc_now_iso()
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE experiment_key = ? AND source = ? FOR UPDATE''',
|
|
(normalized['experiment_key'], source),
|
|
).fetchone()
|
|
member = self.conn.execute(
|
|
'''SELECT member.*, queue.normalized_target AS repository
|
|
FROM docker_depth_experiment_repositories member
|
|
JOIN target_queue queue ON queue.id = COALESCE(
|
|
member.replacement_repository_queue_id,
|
|
member.repository_queue_id
|
|
)
|
|
WHERE member.id = ? AND member.experiment_id = ?
|
|
FOR UPDATE OF member, queue''',
|
|
(repository_id, experiment['id'] if experiment else 0),
|
|
).fetchone()
|
|
if not (
|
|
experiment
|
|
and str(experiment['state']) == 'resolving'
|
|
and member
|
|
and member['source'] == source
|
|
and member['work_state'] == 'resolving'
|
|
and int(member['resolver_generation']) == int(resolver_generation)
|
|
and member['resolver_owner'] == resolver_owner
|
|
and member['resolver_token'] == resolver_token
|
|
and str(member['resolver_expires_at'] or '') > now
|
|
):
|
|
self.conn.rollback()
|
|
return {'status': 'stale', 'committed': False}
|
|
experiment, authority_reason = self._locked_docker_depth_experiment_authority(
|
|
normalized, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not experiment:
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'held', 'committed': True,
|
|
'reason': authority_reason,
|
|
}
|
|
stage = 'breadth' if int(member['selected_image_count']) == 0 else 'deep'
|
|
if stage == 'deep' and not bool(member['is_deep_probe']):
|
|
raise ScanEventConflictError('Docker depth deep resolver authority conflicts')
|
|
|
|
if not complete:
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS,
|
|
)
|
|
|
|
if (
|
|
claim_attempt_consumed
|
|
and int(member['resolver_attempts'])
|
|
>= DOCKER_DEPTH_RESOLVER_MAX_ATTEMPTS
|
|
):
|
|
terminal = self._terminalize_docker_depth_attempt_limit_locked(
|
|
experiment, member, normalized, now,
|
|
)
|
|
self.conn.commit()
|
|
terminal.pop('experiment', None)
|
|
return terminal
|
|
now_dt = datetime.now(timezone.utc)
|
|
if retry_at:
|
|
try:
|
|
due_dt = datetime.fromisoformat(str(retry_at).replace('Z', '+00:00'))
|
|
if due_dt.tzinfo is None:
|
|
due_dt = due_dt.replace(tzinfo=timezone.utc)
|
|
due_dt = due_dt.astimezone(timezone.utc)
|
|
except (TypeError, ValueError):
|
|
self.conn.rollback()
|
|
return {'status': 'invalid', 'committed': False}
|
|
if due_dt > now_dt + timedelta(seconds=3605):
|
|
self.conn.rollback()
|
|
return {'status': 'invalid', 'committed': False}
|
|
due_dt = max(due_dt, now_dt)
|
|
else:
|
|
exponent = min(8, max(0, int(member['resolver_attempts']) - 1))
|
|
due_dt = datetime.fromtimestamp(
|
|
time.time() + min(
|
|
DOCKER_DEPTH_RESOLVER_RETRY_MAX_SECONDS,
|
|
DOCKER_DEPTH_RESOLVER_RETRY_SECONDS * (2 ** exponent),
|
|
),
|
|
timezone.utc,
|
|
)
|
|
due = due_dt.isoformat(timespec='seconds')
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'pending', resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = ?, resolver_attempts = CASE
|
|
WHEN ? = 0 AND resolver_attempts > 0
|
|
THEN resolver_attempts - 1 ELSE resolver_attempts END,
|
|
last_error_code = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ?
|
|
AND resolver_generation = ? AND resolver_owner = ?
|
|
AND resolver_token = ? AND resolver_expires_at > ?''',
|
|
(
|
|
due, 1 if claim_attempt_consumed else 0,
|
|
first_line(error or 'docker_depth_resolution_deferred', 128),
|
|
now, repository_id, experiment['id'], resolver_generation,
|
|
resolver_owner, resolver_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
self.conn.rollback()
|
|
return {'status': 'stale', 'committed': False}
|
|
self.conn.commit()
|
|
return {'status': 'deferred', 'committed': True, 'retry_at': due}
|
|
|
|
candidate_count, candidate_records = self._normalize_docker_depth_resolution_outcome(
|
|
outcome, str(member['repository']), normalized, stage,
|
|
)
|
|
existing_selections = self.conn.execute(
|
|
'''SELECT selection.image_rank, selection.graph_sha256,
|
|
queue.normalized_target
|
|
FROM docker_depth_experiment_selections selection
|
|
JOIN docker_depth_experiment_targets target
|
|
ON target.id = selection.experiment_target_id
|
|
JOIN target_queue queue ON queue.id = target.target_queue_id
|
|
WHERE selection.experiment_repository_id = ?
|
|
ORDER BY selection.image_rank''',
|
|
(repository_id,),
|
|
).fetchall()
|
|
existing_graphs = {
|
|
str(selection['graph_sha256']) for selection in existing_selections
|
|
}
|
|
existing_targets = {
|
|
str(selection['normalized_target']) for selection in existing_selections
|
|
}
|
|
eligible_records = []
|
|
resolution_repository_queue_id = int(
|
|
member['replacement_repository_queue_id']
|
|
or member['repository_queue_id']
|
|
)
|
|
for candidate in candidate_records:
|
|
if (
|
|
candidate['graph_sha256'] in existing_graphs
|
|
or candidate['normalized_target'] in existing_targets
|
|
):
|
|
continue
|
|
conflict, _candidate_queue_id = (
|
|
self._docker_depth_immutable_candidate_conflict_locked(
|
|
experiment['id'], source, candidate,
|
|
)
|
|
)
|
|
if conflict:
|
|
self._record_docker_depth_candidate_skip_locked(
|
|
experiment, member, resolution_repository_queue_id,
|
|
'image', candidate['image_rank'], {
|
|
'normalized_target': candidate['normalized_target'],
|
|
'graph_sha256': candidate['graph_sha256'],
|
|
}, conflict, now,
|
|
)
|
|
continue
|
|
eligible_records.append(candidate)
|
|
remaining = normalized['images_per_repository'] - len(existing_selections)
|
|
if stage == 'breadth':
|
|
remaining = 1
|
|
records = [
|
|
self._rerank_docker_depth_selection_record(
|
|
record, len(existing_selections) + index,
|
|
)
|
|
for index, record in enumerate(eligible_records[:remaining], 1)
|
|
]
|
|
if stage == 'breadth' and not records:
|
|
replacement = self._replace_docker_depth_repository_locked(
|
|
experiment, member, normalized,
|
|
'no_eligible_physical_target', now, candidate_count,
|
|
)
|
|
self.conn.commit()
|
|
return replacement
|
|
actual_selection_count = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_selections
|
|
WHERE experiment_id = ?''',
|
|
(experiment['id'],),
|
|
).fetchone()['count'])
|
|
actual_target_count = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_targets
|
|
WHERE experiment_id = ?''',
|
|
(experiment['id'],),
|
|
).fetchone()['count'])
|
|
if (
|
|
actual_selection_count != int(experiment['selection_count'])
|
|
or actual_target_count != int(experiment['target_count'])
|
|
):
|
|
raise ScanEventConflictError('Docker depth counter capacity drifted')
|
|
|
|
inserted_selections = 0
|
|
for record in records:
|
|
queue = self._docker_depth_target_queue_locked(
|
|
experiment['id'], source, member['query'], record, now,
|
|
)
|
|
manifest_id = self._persist_docker_depth_manifest_locked(
|
|
queue['id'], source, record, now,
|
|
)
|
|
dispatch_wave, dispatch_order = self._docker_depth_dispatch_position(
|
|
int(member['query_ordinal']), int(member['repository_rank']),
|
|
record['image_rank'], normalized['query_count'],
|
|
)
|
|
capacity = self.reserve_docker_depth_experiment_target_capacity_locked(
|
|
experiment['id'], queue['id'], manifest_id,
|
|
dispatch_wave, dispatch_order, now,
|
|
)
|
|
selection = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiment_selections
|
|
WHERE experiment_repository_id = ? AND image_rank = ?
|
|
FOR UPDATE''',
|
|
(repository_id, record['image_rank']),
|
|
).fetchone()
|
|
expected_selection = {
|
|
'experiment_id': int(experiment['id']),
|
|
'query_ordinal': int(member['query_ordinal']),
|
|
'experiment_target_id': int(capacity['target']['id']),
|
|
'selection_reason': record['selection_reason'],
|
|
'selection_evidence_sha256': record['selection_evidence_sha256'],
|
|
'graph_sha256': record['graph_sha256'],
|
|
}
|
|
if selection:
|
|
if any(
|
|
(int(selection[key]) if isinstance(value, int) else str(selection[key]))
|
|
!= value for key, value in expected_selection.items()
|
|
):
|
|
raise ScanEventConflictError('Docker depth persisted selector evidence conflicts')
|
|
else:
|
|
if actual_selection_count + inserted_selections + 1 > normalized[
|
|
'theoretical_max_targets'
|
|
]:
|
|
raise ScanEventConflictError(
|
|
'Docker depth theoretical selection capacity is exhausted'
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_depth_experiment_selections(
|
|
experiment_id, query_ordinal, experiment_repository_id,
|
|
experiment_target_id, image_rank, selection_reason,
|
|
selection_evidence_sha256, graph_sha256,
|
|
selected_at, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
experiment['id'], member['query_ordinal'], repository_id,
|
|
capacity['target']['id'], record['image_rank'],
|
|
record['selection_reason'], record['selection_evidence_sha256'],
|
|
record['graph_sha256'], now, now,
|
|
),
|
|
)
|
|
inserted_selections += 1
|
|
if inserted_selections:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiments
|
|
SET selection_count = selection_count + ?, updated_at = ?
|
|
WHERE id = ? AND selection_count = ?
|
|
AND selection_count + ? <= ?''',
|
|
(
|
|
inserted_selections, now, experiment['id'],
|
|
actual_selection_count, inserted_selections,
|
|
normalized['theoretical_max_targets'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'Docker depth selection capacity lost its serialized fence'
|
|
)
|
|
selected_image_count = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM docker_depth_experiment_selections
|
|
WHERE experiment_repository_id = ?''',
|
|
(repository_id,),
|
|
).fetchone()['count'])
|
|
cursor = self.conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET work_state = 'resolved', resolver_owner = NULL,
|
|
resolver_token = NULL, resolver_expires_at = NULL,
|
|
resolver_due_at = NULL,
|
|
candidate_distinct_graph_count = CASE
|
|
WHEN candidate_distinct_graph_count >= ?
|
|
THEN candidate_distinct_graph_count ELSE ? END,
|
|
selected_image_count = ?, last_error_code = NULL,
|
|
resolved_at = ?, updated_at = ?
|
|
WHERE id = ? AND experiment_id = ?
|
|
AND resolver_generation = ? AND resolver_owner = ?
|
|
AND resolver_token = ? AND resolver_expires_at > ?''',
|
|
(
|
|
candidate_count, candidate_count, selected_image_count, now, now,
|
|
repository_id, experiment['id'], resolver_generation,
|
|
resolver_owner, resolver_token, now,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('Docker depth resolver completion lost its lease fence')
|
|
|
|
if stage == 'breadth':
|
|
self._finalize_docker_depth_query_breadth_locked(
|
|
experiment, member, now,
|
|
)
|
|
experiment = self.conn.execute(
|
|
'SELECT * FROM docker_depth_experiments WHERE id = ?',
|
|
(experiment['id'],),
|
|
).fetchone()
|
|
experiment, selection_reason = (
|
|
self._freeze_docker_depth_runtime_selection_locked(
|
|
experiment, normalized, now,
|
|
)
|
|
)
|
|
if selection_reason:
|
|
held = self._hold_docker_depth_experiment_locked(
|
|
experiment, selection_reason, now,
|
|
)
|
|
self.conn.commit()
|
|
return held
|
|
drift_reason = self._docker_depth_persisted_drift_reason_locked(
|
|
experiment, normalized, now,
|
|
)
|
|
if drift_reason:
|
|
held = self._hold_docker_depth_experiment_locked(
|
|
experiment, drift_reason, now,
|
|
)
|
|
self.conn.commit()
|
|
return held
|
|
experiment = self._advance_docker_depth_experiment_state_locked(
|
|
experiment, now,
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'status': 'resolved', 'committed': True,
|
|
'stage': stage, 'candidate_distinct_graph_count': candidate_count,
|
|
'selected_image_count': selected_image_count,
|
|
'inserted_selection_count': inserted_selections,
|
|
'experiment_state': str(experiment['state']),
|
|
}
|
|
except DiscoveryPausedError:
|
|
self.conn.rollback()
|
|
raise
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
try:
|
|
self.conn.rollback()
|
|
result = self._defer_or_hold_docker_depth_resolution_conflict(
|
|
source, repository_id, resolver_generation, resolver_owner,
|
|
resolver_token, normalized, str(exc),
|
|
final_cutover=final_cutover,
|
|
)
|
|
self.last_error = str(exc)
|
|
return result
|
|
except Exception as recovery_exc:
|
|
self.last_error = f'{exc}; conflict recovery failed: {recovery_exc}'
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
return {'status': 'error', 'committed': False}
|
|
|
|
def _locked_docker_depth_page_authority(
|
|
self, source, query, observation, authority, *, final_cutover, now,
|
|
):
|
|
rows = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE source = ? AND state <> 'released'
|
|
ORDER BY id FOR UPDATE''',
|
|
(source,),
|
|
).fetchall()
|
|
if not rows:
|
|
return None, None, None
|
|
if len(rows) != 1:
|
|
for row in rows:
|
|
self._hold_docker_depth_experiment_locked(
|
|
row, 'dynamic_hold_authority_ambiguous', now,
|
|
)
|
|
return None, None, 'dynamic_hold_authority_ambiguous'
|
|
row = rows[0]
|
|
if not isinstance(authority, dict) or authority.get('enabled') is not True:
|
|
reason = 'authority_payload_invalid'
|
|
if isinstance(authority, dict) and authority.get('enabled') is False:
|
|
reason = 'experiment_disabled'
|
|
self._hold_docker_depth_experiment_locked(row, reason, now)
|
|
return None, None, reason
|
|
try:
|
|
normalized = self._normalize_docker_depth_resolver_authority(authority)
|
|
except (TypeError, ValueError):
|
|
return None, None, 'authority_payload_invalid'
|
|
if (
|
|
str(row['experiment_key']) != normalized['experiment_key']
|
|
or str(row['source']) != normalized['source']
|
|
):
|
|
self._hold_docker_depth_experiment_locked(
|
|
row, 'experiment_identity_drift', now,
|
|
)
|
|
return None, None, 'experiment_identity_drift'
|
|
experiment, reason = self._locked_docker_depth_experiment_authority(
|
|
normalized, final_cutover=final_cutover, now=now,
|
|
)
|
|
if not experiment:
|
|
return None, None, reason
|
|
query_row = self.conn.execute(
|
|
'''SELECT query_ordinal FROM docker_depth_experiment_queries
|
|
WHERE experiment_id = ? AND source = ? AND query = ?''',
|
|
(experiment['id'], source, query),
|
|
).fetchone()
|
|
if str(experiment['state']) != 'collecting' and (
|
|
observation is None
|
|
or not query_row
|
|
or observation['query_ordinal'] != int(query_row['query_ordinal'])
|
|
or observation['query_count'] != normalized['query_count']
|
|
or observation['collection_generation']
|
|
!= normalized['collection_generation']
|
|
or observation['ordered_query_hash']
|
|
!= normalized['ordered_queries_sha256']
|
|
or observation['policy_sha256']
|
|
!= normalized['provenance_policy_sha256']
|
|
):
|
|
self._hold_docker_depth_experiment_locked(
|
|
experiment, 'dynamic_hold_observation_drift', now,
|
|
)
|
|
return None, None, 'dynamic_hold_observation_drift'
|
|
return experiment, normalized, None
|
|
|
|
def _hold_new_docker_depth_repositories_locked(
|
|
self, source, query, queue_ids, observation, now, *, experiment=None,
|
|
authority=None,
|
|
):
|
|
if not queue_ids or not self.conn.is_postgres or not experiment:
|
|
return 0
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
DOCKER_DEPTH_HOLD_ACTIVE_STATES,
|
|
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
|
|
)
|
|
|
|
if (
|
|
str(experiment['state']) not in DOCKER_DEPTH_HOLD_ACTIVE_STATES
|
|
or experiment['hold_manifest_sha256'] is None
|
|
):
|
|
return 0
|
|
query_row = self.conn.execute(
|
|
'''SELECT query_ordinal FROM docker_depth_experiment_queries
|
|
WHERE experiment_id = ? AND source = ? AND query = ?''',
|
|
(experiment['id'], source, query),
|
|
).fetchone()
|
|
if (
|
|
observation is None
|
|
or authority is None
|
|
or not query_row
|
|
or observation['query_ordinal'] != int(query_row['query_ordinal'])
|
|
or observation['query_count'] != int(experiment['query_count'])
|
|
or observation['collection_generation']
|
|
!= experiment['collection_generation']
|
|
or observation['ordered_query_hash'] != experiment['ordered_queries_sha256']
|
|
or observation['policy_sha256'] != experiment['provenance_policy_sha256']
|
|
or str(experiment['config_sha256']) != authority['config_sha256']
|
|
or str(experiment['selector_sha256']) != authority['selector_sha256']
|
|
):
|
|
raise ScanEventConflictError('Docker depth dynamic hold observation drifted')
|
|
queue_placeholders = ','.join('?' for _ in queue_ids)
|
|
rows = self.conn.execute(
|
|
f'''SELECT queue.* FROM target_queue queue
|
|
WHERE queue.id IN ({queue_placeholders})
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_repositories member
|
|
WHERE member.experiment_id = ?
|
|
AND member.repository_queue_id = queue.id
|
|
)
|
|
ORDER BY queue.id FOR UPDATE''',
|
|
(*sorted(queue_ids), experiment['id']),
|
|
).fetchall()
|
|
entries = []
|
|
for row in rows:
|
|
if (
|
|
row['source'] != source
|
|
or row['platform'] != 'docker'
|
|
or row['query'] != query
|
|
or row['status'] != 'deferred'
|
|
or row['target_scan_id'] is not None
|
|
or '@' in str(row['target'])
|
|
or '@' in str(row['normalized_target'])
|
|
):
|
|
raise ScanEventConflictError('Docker depth dynamic hold row identity conflicts')
|
|
entries.append({
|
|
'queue_id': int(row['id']),
|
|
'source': str(row['source']),
|
|
'platform': str(row['platform']),
|
|
'query': str(row['query']),
|
|
'prior_status': str(row['status']),
|
|
'prior_updated_at': str(row['updated_at']),
|
|
})
|
|
entries = self._normalize_target_queue_policy_entries(
|
|
entries, 'cold', max(1, len(entries)),
|
|
) if entries else []
|
|
if entries:
|
|
active_holds = int(self.conn.execute(
|
|
'''SELECT COUNT(*) AS count
|
|
FROM target_queue_policy_events event
|
|
LEFT JOIN target_queue_policy_events reverse_event
|
|
ON reverse_event.reverses_event_id = event.id
|
|
WHERE event.experiment_id = ? AND event.action = 'cold'
|
|
AND reverse_event.id IS NULL''',
|
|
(experiment['id'],),
|
|
).fetchone()['count'])
|
|
if active_holds + len(entries) > DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS:
|
|
raise ScanEventConflictError(
|
|
'Docker depth dynamic hold capacity is exhausted'
|
|
)
|
|
result = self._cold_target_queue_rows_locked(
|
|
entries,
|
|
reason_code=DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
config_sha256=str(experiment['config_sha256']),
|
|
policy_sha256=str(experiment['provenance_policy_sha256']),
|
|
manifest_sha256=str(experiment['hold_manifest_sha256']),
|
|
experiment_id=int(experiment['id']),
|
|
now=now,
|
|
)
|
|
return int(result['transitioned']) + int(result['duplicates'])
|
|
|
|
@staticmethod
|
|
def _target_queue_policy_audit_sha256(
|
|
action, manifest_sha256, entry, experiment_id=None,
|
|
):
|
|
payload = {
|
|
'action': str(action),
|
|
'manifest_sha256': str(manifest_sha256),
|
|
'entry': dict(entry),
|
|
}
|
|
if experiment_id is not None:
|
|
payload['experiment_id'] = int(experiment_id)
|
|
encoded = json.dumps(
|
|
payload, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
@staticmethod
|
|
def _normalize_target_queue_policy_entries(entries, action, max_rows):
|
|
from docker_depth_experiment import DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS
|
|
|
|
maximum = min(
|
|
DOCKER_DEPTH_REVIEW_MANIFEST_MAX_ROWS,
|
|
max(1, int(max_rows)),
|
|
)
|
|
values = [dict(entry) for entry in entries or ()]
|
|
if not values or len(values) > maximum:
|
|
raise ValueError('target queue policy entry count is outside its bound')
|
|
required = {
|
|
'queue_id', 'source', 'platform', 'query', 'prior_updated_at',
|
|
}
|
|
if action == 'cold':
|
|
required.add('prior_status')
|
|
else:
|
|
required.update(('cold_event_id', 'restore_status'))
|
|
normalized = []
|
|
seen = set()
|
|
for value in values:
|
|
if set(value) != required:
|
|
raise ValueError('target queue policy entry shape is invalid')
|
|
queue_id = int(value['queue_id'])
|
|
if queue_id <= 0 or queue_id in seen:
|
|
raise ValueError('target queue policy queue identity is invalid or duplicated')
|
|
seen.add(queue_id)
|
|
item = {
|
|
'queue_id': queue_id,
|
|
'source': str(value['source'] or '').strip(),
|
|
'platform': str(value['platform'] or '').strip(),
|
|
'query': str(value['query'] or ''),
|
|
'prior_updated_at': str(value['prior_updated_at'] or ''),
|
|
}
|
|
if not all((item['source'], item['platform'], item['query'], item['prior_updated_at'])):
|
|
raise ValueError('target queue policy entry contains an empty identity field')
|
|
if action == 'cold':
|
|
item['prior_status'] = str(value['prior_status'] or '')
|
|
if item['prior_status'] not in ('pending', 'deferred'):
|
|
raise ValueError('cold transition requires a pending or deferred prior status')
|
|
else:
|
|
item['cold_event_id'] = int(value['cold_event_id'])
|
|
item['restore_status'] = str(value['restore_status'] or '')
|
|
if item['cold_event_id'] <= 0 or item['restore_status'] not in ('pending', 'deferred'):
|
|
raise ValueError('reactivation transition identity is invalid')
|
|
normalized.append(item)
|
|
return sorted(normalized, key=lambda item: item['queue_id'])
|
|
|
|
@staticmethod
|
|
def _target_queue_policy_chunks(values, size=500):
|
|
values = list(values)
|
|
for start in range(0, len(values), size):
|
|
yield values[start:start + size]
|
|
|
|
def _locked_target_queue_policy_rows(self, entries):
|
|
rows_by_id = {}
|
|
for chunk in self._target_queue_policy_chunks(entries):
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
rows = self.conn.execute(
|
|
f'''SELECT * FROM target_queue
|
|
WHERE id IN ({placeholders}) ORDER BY id FOR UPDATE''',
|
|
tuple(entry['queue_id'] for entry in chunk),
|
|
).fetchall()
|
|
for row in rows:
|
|
queue_id = int(row['id'])
|
|
if queue_id in rows_by_id:
|
|
raise ScanEventConflictError(
|
|
'target queue policy selection contains duplicate rows'
|
|
)
|
|
rows_by_id[queue_id] = row
|
|
if len(rows_by_id) != len(entries):
|
|
raise ScanEventConflictError('target queue policy selection changed')
|
|
return [rows_by_id[entry['queue_id']] for entry in entries]
|
|
|
|
@staticmethod
|
|
def _normalize_target_queue_policy_experiment_id(experiment_id):
|
|
if experiment_id is None:
|
|
return None
|
|
if isinstance(experiment_id, bool):
|
|
raise ValueError('target queue policy experiment identity is invalid')
|
|
try:
|
|
experiment_id = int(experiment_id)
|
|
except (TypeError, ValueError, OverflowError):
|
|
raise ValueError('target queue policy experiment identity is invalid') from None
|
|
if experiment_id <= 0:
|
|
raise ValueError('target queue policy experiment identity is invalid')
|
|
return experiment_id
|
|
|
|
def _require_target_queue_policy_unfenced(self, row, *, lock_rows=True):
|
|
fenced_fields = (
|
|
'lease_owner', 'lease_token', 'claim_batch', 'leased_at', 'lease_expires_at',
|
|
'current_result_reservation_id', 'claim_event_id', 'resolver_token',
|
|
)
|
|
if any(row[field] is not None for field in fenced_fields):
|
|
raise ScanEventConflictError('target queue policy row has an active claim fence')
|
|
if str(row['resolver_state'] or '') == 'resolving':
|
|
raise ScanEventConflictError('target queue policy row has an active resolver fence')
|
|
lock_suffix = ' FOR UPDATE' if lock_rows else ''
|
|
reservation = self.conn.execute(
|
|
'''SELECT id FROM result_reservations
|
|
WHERE queue_id = ? AND state IN ('scanning','ready','ingesting','db_committed')
|
|
LIMIT 1''' + lock_suffix,
|
|
(int(row['id']),),
|
|
).fetchone()
|
|
if reservation:
|
|
raise ScanEventConflictError('target queue policy row has an active result reservation')
|
|
blob_lock_suffix = ' FOR UPDATE OF blob' if lock_rows else ''
|
|
docker_blob = self.conn.execute(
|
|
'''SELECT 1
|
|
FROM docker_image_blob_coverage coverage
|
|
JOIN docker_content_blobs blob
|
|
ON blob.digest = coverage.blob_digest
|
|
AND blob.coverage_policy_sha256 = coverage.coverage_policy_sha256
|
|
WHERE coverage.queue_id = ? AND blob.state IN ('leased','submitted')
|
|
LIMIT 1''' + blob_lock_suffix,
|
|
(int(row['id']),),
|
|
).fetchone()
|
|
if docker_blob:
|
|
raise ScanEventConflictError('target queue policy row has active Docker content work')
|
|
|
|
def _target_queue_policy_duplicate_count(
|
|
self, action, manifest_sha256, entries, *, reason_code,
|
|
config_sha256, policy_sha256, experiment_id=None,
|
|
):
|
|
entries_by_audit = {}
|
|
for entry in entries:
|
|
audit_sha256 = self._target_queue_policy_audit_sha256(
|
|
action, manifest_sha256, entry, experiment_id,
|
|
)
|
|
entries_by_audit[audit_sha256] = entry
|
|
events_by_audit = {}
|
|
for chunk in self._target_queue_policy_chunks(
|
|
sorted(entries_by_audit),
|
|
):
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
events = self.conn.execute(
|
|
f'''SELECT * FROM target_queue_policy_events
|
|
WHERE review_audit_sha256 IN ({placeholders})
|
|
ORDER BY id FOR UPDATE''',
|
|
tuple(chunk),
|
|
).fetchall()
|
|
for event in events:
|
|
audit_sha256 = str(event['review_audit_sha256'])
|
|
if audit_sha256 in events_by_audit:
|
|
raise ScanEventConflictError(
|
|
'target queue policy audit identity is duplicated'
|
|
)
|
|
events_by_audit[audit_sha256] = event
|
|
if not events_by_audit:
|
|
return 0
|
|
if len(events_by_audit) != len(entries):
|
|
raise ScanEventConflictError(
|
|
'target queue policy manifest was only partially applied'
|
|
)
|
|
reversed_event_ids = set()
|
|
if action == 'cold':
|
|
event_ids = sorted(int(event['id']) for event in events_by_audit.values())
|
|
for chunk in self._target_queue_policy_chunks(event_ids):
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
reversed_rows = self.conn.execute(
|
|
f'''SELECT reverses_event_id FROM target_queue_policy_events
|
|
WHERE reverses_event_id IN ({placeholders})
|
|
ORDER BY reverses_event_id FOR UPDATE''',
|
|
tuple(chunk),
|
|
).fetchall()
|
|
reversed_event_ids.update(
|
|
int(row['reverses_event_id']) for row in reversed_rows
|
|
)
|
|
queues = {
|
|
int(row['id']): row
|
|
for row in self._locked_target_queue_policy_rows(entries)
|
|
}
|
|
for audit_sha256, entry in entries_by_audit.items():
|
|
event = events_by_audit[audit_sha256]
|
|
expected_next = 'cold' if action == 'cold' else entry['restore_status']
|
|
expected_prior = entry['prior_status'] if action == 'cold' else 'cold'
|
|
expected_reverse = None if action == 'cold' else entry['cold_event_id']
|
|
if (
|
|
event['action'] != action
|
|
or int(event['queue_id']) != entry['queue_id']
|
|
or event['prior_status'] != expected_prior
|
|
or event['next_status'] != expected_next
|
|
or event['source'] != entry['source']
|
|
or event['platform'] != entry['platform']
|
|
or event['query'] != entry['query']
|
|
or event['reason_code'] != reason_code
|
|
or event['config_sha256'] != config_sha256
|
|
or event['policy_sha256'] != policy_sha256
|
|
or event['manifest_sha256'] != manifest_sha256
|
|
or event['prior_updated_at'] != entry['prior_updated_at']
|
|
or (
|
|
(event['experiment_id'] is None and experiment_id is not None)
|
|
or (
|
|
event['experiment_id'] is not None
|
|
and int(event['experiment_id']) != experiment_id
|
|
)
|
|
)
|
|
or (
|
|
(event['reverses_event_id'] is None and expected_reverse is not None)
|
|
or (
|
|
event['reverses_event_id'] is not None
|
|
and int(event['reverses_event_id']) != expected_reverse
|
|
)
|
|
)
|
|
):
|
|
raise ScanEventConflictError('target queue policy audit identity conflicts')
|
|
queue = queues.get(entry['queue_id'])
|
|
if not queue or queue['status'] != expected_next:
|
|
raise ScanEventConflictError('target queue policy duplicate state conflicts')
|
|
if action == 'cold' and int(event['id']) in reversed_event_ids:
|
|
raise ScanEventConflictError('target queue cold event was already reversed')
|
|
return len(entries)
|
|
|
|
def _cold_target_queue_rows_locked(
|
|
self, entries, *, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, experiment_id=None, now=None,
|
|
):
|
|
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
|
|
now = str(now or utc_now_iso())
|
|
if not entries:
|
|
return {
|
|
'transitioned': 0, 'duplicates': 0, 'examined': 0,
|
|
'manifest_sha256': manifest_sha256,
|
|
}
|
|
experiment = None
|
|
if experiment_id is not None:
|
|
experiment = self.conn.execute(
|
|
'''SELECT id FROM docker_depth_experiments
|
|
WHERE id = ? FOR UPDATE''',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
if not experiment:
|
|
raise ScanEventConflictError(
|
|
'target queue policy experiment authority is absent'
|
|
)
|
|
rows = self._locked_target_queue_policy_rows(entries)
|
|
duplicates = self._target_queue_policy_duplicate_count(
|
|
'cold', manifest_sha256, entries, reason_code=reason_code,
|
|
config_sha256=config_sha256, policy_sha256=policy_sha256,
|
|
experiment_id=experiment_id,
|
|
)
|
|
if duplicates:
|
|
return {
|
|
'transitioned': 0, 'duplicates': duplicates,
|
|
'examined': len(entries), 'manifest_sha256': manifest_sha256,
|
|
}
|
|
for row, entry in zip(rows, entries):
|
|
if (
|
|
int(row['id']) != entry['queue_id']
|
|
or row['source'] != entry['source']
|
|
or row['platform'] != entry['platform']
|
|
or row['query'] != entry['query']
|
|
or row['status'] != entry['prior_status']
|
|
or row['updated_at'] != entry['prior_updated_at']
|
|
):
|
|
raise ScanEventConflictError('target queue policy evidence does not match')
|
|
self._require_target_queue_policy_unfenced(row)
|
|
for row, entry in zip(rows, entries):
|
|
audit_sha256 = self._target_queue_policy_audit_sha256(
|
|
'cold', manifest_sha256, entry, experiment_id,
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO target_queue_policy_events(
|
|
queue_id, action, prior_status, next_status, source, platform,
|
|
query, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, review_audit_sha256, reverses_event_id,
|
|
experiment_id, prior_updated_at, created_at
|
|
) VALUES (?, 'cold', ?, 'cold', ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)''',
|
|
(
|
|
entry['queue_id'], entry['prior_status'], entry['source'],
|
|
entry['platform'], entry['query'], reason_code, config_sha256,
|
|
policy_sha256, manifest_sha256, audit_sha256, experiment_id,
|
|
entry['prior_updated_at'], now,
|
|
),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'cold', updated_at = ?
|
|
WHERE id = ? AND status = ? AND updated_at = ?''',
|
|
(now, entry['queue_id'], entry['prior_status'], entry['prior_updated_at']),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('target queue cold transition lost its fence')
|
|
return {
|
|
'transitioned': len(entries), 'duplicates': 0,
|
|
'examined': len(entries), 'manifest_sha256': manifest_sha256,
|
|
}
|
|
|
|
def cold_target_queue_rows(
|
|
self, entries, *, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, max_rows=10000, experiment_id=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('target queue cold transition requires PostgreSQL')
|
|
entries = self._normalize_target_queue_policy_entries(entries, 'cold', max_rows)
|
|
hashes = (config_sha256, policy_sha256, manifest_sha256)
|
|
if not all(re.fullmatch(r'[a-f0-9]{64}', str(value or '')) for value in hashes):
|
|
raise ValueError('target queue policy hash identity is invalid')
|
|
reason_code = str(reason_code or '').strip()
|
|
if not reason_code or len(reason_code) > 128:
|
|
raise ValueError('target queue policy reason code is invalid')
|
|
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
|
|
try:
|
|
result = self._cold_target_queue_rows_locked(
|
|
entries, reason_code=reason_code, config_sha256=config_sha256,
|
|
policy_sha256=policy_sha256, manifest_sha256=manifest_sha256,
|
|
experiment_id=experiment_id, now=utc_now_iso(),
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def _reactivate_target_queue_rows_locked(
|
|
self, entries, *, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, experiment_id=None, now=None,
|
|
):
|
|
from docker_depth_experiment import (
|
|
DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
DOCKER_DEPTH_HOLD_REASON,
|
|
)
|
|
|
|
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
|
|
now = str(now or utc_now_iso())
|
|
if not entries:
|
|
return {
|
|
'transitioned': 0, 'duplicates': 0, 'examined': 0,
|
|
'manifest_sha256': manifest_sha256,
|
|
}
|
|
experiment = None
|
|
if experiment_id is not None:
|
|
experiment = self.conn.execute(
|
|
'''SELECT * FROM docker_depth_experiments
|
|
WHERE id = ? FOR UPDATE''',
|
|
(experiment_id,),
|
|
).fetchone()
|
|
if not experiment:
|
|
raise ScanEventConflictError(
|
|
'target queue policy experiment authority is absent'
|
|
)
|
|
rows = self._locked_target_queue_policy_rows(entries)
|
|
duplicates = self._target_queue_policy_duplicate_count(
|
|
'reactivate', manifest_sha256, entries, reason_code=reason_code,
|
|
config_sha256=config_sha256, policy_sha256=policy_sha256,
|
|
experiment_id=experiment_id,
|
|
)
|
|
if duplicates:
|
|
return {
|
|
'transitioned': 0, 'duplicates': duplicates,
|
|
'examined': len(entries), 'manifest_sha256': manifest_sha256,
|
|
}
|
|
cold_events = {}
|
|
cold_event_ids = sorted(entry['cold_event_id'] for entry in entries)
|
|
for chunk in self._target_queue_policy_chunks(cold_event_ids):
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
event_rows = self.conn.execute(
|
|
f'''SELECT * FROM target_queue_policy_events
|
|
WHERE id IN ({placeholders}) ORDER BY id FOR UPDATE''',
|
|
tuple(chunk),
|
|
).fetchall()
|
|
cold_events.update((int(event['id']), event) for event in event_rows)
|
|
reversed_event_ids = set()
|
|
for chunk in self._target_queue_policy_chunks(cold_event_ids):
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
reversed_rows = self.conn.execute(
|
|
f'''SELECT reverses_event_id FROM target_queue_policy_events
|
|
WHERE reverses_event_id IN ({placeholders})
|
|
ORDER BY reverses_event_id FOR UPDATE''',
|
|
tuple(chunk),
|
|
).fetchall()
|
|
reversed_event_ids.update(
|
|
int(event['reverses_event_id']) for event in reversed_rows
|
|
)
|
|
for row, entry in zip(rows, entries):
|
|
cold_event = cold_events.get(entry['cold_event_id'])
|
|
cold_experiment_id = (
|
|
None if not cold_event or cold_event['experiment_id'] is None
|
|
else int(cold_event['experiment_id'])
|
|
)
|
|
cold_entry = {
|
|
'queue_id': entry['queue_id'],
|
|
'source': entry['source'],
|
|
'platform': entry['platform'],
|
|
'query': entry['query'],
|
|
'prior_status': entry['restore_status'],
|
|
'prior_updated_at': (
|
|
str(cold_event['prior_updated_at']) if cold_event else ''
|
|
),
|
|
}
|
|
owned_event_invalid = bool(experiment_id is not None and cold_event) and (
|
|
str(cold_event['reason_code']) not in (
|
|
DOCKER_DEPTH_HOLD_REASON, DOCKER_DEPTH_DYNAMIC_HOLD_REASON,
|
|
)
|
|
or str(cold_event['config_sha256']) != str(config_sha256)
|
|
or str(cold_event['policy_sha256']) != str(policy_sha256)
|
|
or str(cold_event['manifest_sha256'])
|
|
!= str(experiment['hold_manifest_sha256'])
|
|
or str(cold_event['review_audit_sha256'])
|
|
!= self._target_queue_policy_audit_sha256(
|
|
'cold', cold_event['manifest_sha256'], cold_entry,
|
|
experiment_id,
|
|
)
|
|
)
|
|
if (
|
|
int(row['id']) != entry['queue_id']
|
|
or row['source'] != entry['source']
|
|
or row['platform'] != entry['platform']
|
|
or row['query'] != entry['query']
|
|
or row['status'] != 'cold'
|
|
or row['updated_at'] != entry['prior_updated_at']
|
|
or not cold_event
|
|
or cold_event['action'] != 'cold'
|
|
or int(cold_event['queue_id']) != entry['queue_id']
|
|
or cold_event['prior_status'] != entry['restore_status']
|
|
or cold_event['next_status'] != 'cold'
|
|
or cold_experiment_id != experiment_id
|
|
or owned_event_invalid
|
|
or entry['cold_event_id'] in reversed_event_ids
|
|
):
|
|
raise ScanEventConflictError('target queue reactivation evidence does not match')
|
|
self._require_target_queue_policy_unfenced(row)
|
|
for entry in entries:
|
|
audit_sha256 = self._target_queue_policy_audit_sha256(
|
|
'reactivate', manifest_sha256, entry, experiment_id,
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO target_queue_policy_events(
|
|
queue_id, action, prior_status, next_status, source, platform,
|
|
query, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, review_audit_sha256, reverses_event_id,
|
|
experiment_id, prior_updated_at, created_at
|
|
) VALUES (?, 'reactivate', 'cold', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
entry['queue_id'], entry['restore_status'], entry['source'],
|
|
entry['platform'], entry['query'], reason_code, config_sha256,
|
|
policy_sha256, manifest_sha256, audit_sha256,
|
|
entry['cold_event_id'], experiment_id, entry['prior_updated_at'], now,
|
|
),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'cold' AND updated_at = ?''',
|
|
(
|
|
entry['restore_status'], now, entry['queue_id'],
|
|
entry['prior_updated_at'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('target queue reactivation lost its fence')
|
|
return {
|
|
'transitioned': len(entries), 'duplicates': 0,
|
|
'examined': len(entries), 'manifest_sha256': manifest_sha256,
|
|
}
|
|
|
|
def reactivate_cold_target_queue_rows(
|
|
self, entries, *, reason_code, config_sha256, policy_sha256,
|
|
manifest_sha256, max_rows=10000, experiment_id=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('target queue reactivation requires PostgreSQL')
|
|
entries = self._normalize_target_queue_policy_entries(entries, 'reactivate', max_rows)
|
|
hashes = (config_sha256, policy_sha256, manifest_sha256)
|
|
if not all(re.fullmatch(r'[a-f0-9]{64}', str(value or '')) for value in hashes):
|
|
raise ValueError('target queue policy hash identity is invalid')
|
|
reason_code = str(reason_code or '').strip()
|
|
if not reason_code or len(reason_code) > 128:
|
|
raise ValueError('target queue policy reason code is invalid')
|
|
experiment_id = self._normalize_target_queue_policy_experiment_id(experiment_id)
|
|
try:
|
|
result = self._reactivate_target_queue_rows_locked(
|
|
entries, reason_code=reason_code, config_sha256=config_sha256,
|
|
policy_sha256=policy_sha256, manifest_sha256=manifest_sha256,
|
|
experiment_id=experiment_id, now=utc_now_iso(),
|
|
)
|
|
self.conn.commit()
|
|
return result
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _retirement_chain(previous, rows):
|
|
previous = str(previous or '0' * 64)
|
|
payload = json.dumps(
|
|
[dict(row) for row in rows],
|
|
ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
return hashlib.sha256(bytes.fromhex(previous) + hashlib.sha256(payload).digest()).hexdigest()
|
|
|
|
def retire_admission_intents(self, retention_seconds=30 * 86400, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return 0
|
|
now = utc_now_iso()
|
|
cutoff = datetime.fromtimestamp(
|
|
time.time() - max(3600, int(retention_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
limit = min(500, max(1, int(limit)))
|
|
try:
|
|
self.conn.execute(f'SET LOCAL statement_timeout = {RETIREMENT_STATEMENT_TIMEOUT_MS}')
|
|
self.conn.execute(
|
|
'''INSERT INTO admission_intent_retirement(
|
|
id, retired_count, chain_sha256, cursor_token, updated_at
|
|
) VALUES (1, 0, ?, '', ?) ON CONFLICT(id) DO NOTHING''',
|
|
('0' * 64, now),
|
|
)
|
|
summary = self.conn.execute(
|
|
'SELECT * FROM admission_intent_retirement WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
rows = self.conn.execute(
|
|
'''SELECT i.reservation_token, i.intent_sha256, i.state,
|
|
i.resolution_detail, i.created_at, i.updated_at, i.resolved_at
|
|
FROM admission_intents i
|
|
WHERE i.state = 'aborted' AND i.reservation_id IS NULL
|
|
AND i.resolved_at IS NOT NULL AND i.resolved_at <= ?
|
|
AND i.reservation_token > ?
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations r
|
|
WHERE r.reservation_token = i.reservation_token
|
|
)
|
|
ORDER BY i.reservation_token LIMIT ? FOR UPDATE SKIP LOCKED''',
|
|
(cutoff, summary['cursor_token'], limit),
|
|
).fetchall()
|
|
if not rows:
|
|
if summary['cursor_token']:
|
|
self.conn.execute(
|
|
"UPDATE admission_intent_retirement SET cursor_token = '', updated_at = ? WHERE id = 1",
|
|
(now,),
|
|
)
|
|
self.conn.commit()
|
|
return 0
|
|
tokens = [row['reservation_token'] for row in rows]
|
|
placeholders = ','.join('?' for _ in tokens)
|
|
deleted = self.conn.execute(
|
|
f'''DELETE FROM admission_intents i
|
|
WHERE i.reservation_token IN ({placeholders})
|
|
AND i.state = 'aborted' AND i.reservation_id IS NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM result_reservations r
|
|
WHERE r.reservation_token = i.reservation_token
|
|
)''',
|
|
tokens,
|
|
)
|
|
count = int(deleted.rowcount or 0)
|
|
if count != len(rows):
|
|
raise RuntimeError('admission intent retirement lost its exact row fence')
|
|
chain = self._retirement_chain(summary['chain_sha256'], rows)
|
|
self.conn.execute(
|
|
'''UPDATE admission_intent_retirement SET retired_count = retired_count + ?,
|
|
chain_sha256 = ?, cursor_token = ?, updated_at = ? WHERE id = 1''',
|
|
(count, chain, tokens[-1], now),
|
|
)
|
|
self.conn.commit()
|
|
return count
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def retire_deleted_pipeline_artifacts(self, retention_seconds=30 * 86400, limit=100):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return 0
|
|
now = utc_now_iso()
|
|
cutoff = datetime.fromtimestamp(
|
|
time.time() - max(3600, int(retention_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
limit = min(500, max(1, int(limit)))
|
|
try:
|
|
self.conn.execute(f'SET LOCAL statement_timeout = {RETIREMENT_STATEMENT_TIMEOUT_MS}')
|
|
self.conn.execute(
|
|
'''INSERT INTO pipeline_artifact_retirement(
|
|
id, retired_count, chain_sha256, cursor_id, updated_at
|
|
) VALUES (1, 0, ?, 0, ?) ON CONFLICT(id) DO NOTHING''',
|
|
('0' * 64, now),
|
|
)
|
|
summary = self.conn.execute(
|
|
'SELECT * FROM pipeline_artifact_retirement WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
rows = self.conn.execute(
|
|
'''SELECT a.id, a.subsystem, a.artifact_kind, a.owner_id, a.owner_key,
|
|
a.relative_path, a.payload_sha256, a.created_at, a.updated_at, a.deleted_at
|
|
FROM pipeline_artifacts a
|
|
WHERE a.state = 'deleted' AND a.deleted_at IS NOT NULL
|
|
AND a.deleted_at <= ? AND a.id > ?
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM pipeline_quarantine q
|
|
WHERE q.review_status = 'pending' AND (
|
|
(q.object_type = 'projection_tail' AND q.object_id = a.id)
|
|
OR (
|
|
q.object_type = 'result_bundle'
|
|
AND a.subsystem = 'result_bundle'
|
|
AND a.artifact_kind = 'bundle_quarantine'
|
|
AND q.reservation_id = a.owner_id
|
|
)
|
|
)
|
|
)
|
|
ORDER BY a.id LIMIT ? FOR UPDATE SKIP LOCKED''',
|
|
(cutoff, int(summary['cursor_id'] or 0), limit),
|
|
).fetchall()
|
|
if not rows:
|
|
if int(summary['cursor_id'] or 0):
|
|
self.conn.execute(
|
|
'UPDATE pipeline_artifact_retirement SET cursor_id = 0, updated_at = ? WHERE id = 1',
|
|
(now,),
|
|
)
|
|
self.conn.commit()
|
|
return 0
|
|
ids = [int(row['id']) for row in rows]
|
|
placeholders = ','.join('?' for _ in ids)
|
|
deleted = self.conn.execute(
|
|
f'''DELETE FROM pipeline_artifacts a
|
|
WHERE a.id IN ({placeholders}) AND a.state = 'deleted'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM pipeline_quarantine q
|
|
WHERE q.review_status = 'pending' AND (
|
|
(q.object_type = 'projection_tail' AND q.object_id = a.id)
|
|
OR (
|
|
q.object_type = 'result_bundle'
|
|
AND a.subsystem = 'result_bundle'
|
|
AND a.artifact_kind = 'bundle_quarantine'
|
|
AND q.reservation_id = a.owner_id
|
|
)
|
|
)
|
|
)''',
|
|
ids,
|
|
)
|
|
count = int(deleted.rowcount or 0)
|
|
if count != len(rows):
|
|
raise RuntimeError('pipeline artifact retirement lost its exact row fence')
|
|
chain = self._retirement_chain(summary['chain_sha256'], rows)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_artifact_retirement SET retired_count = retired_count + ?,
|
|
chain_sha256 = ?, cursor_id = ?, updated_at = ? WHERE id = 1''',
|
|
(count, chain, ids[-1], now),
|
|
)
|
|
self.conn.commit()
|
|
return count
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def janitor_cursor(self, layout_name):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('janitor cursor authority requires PostgreSQL')
|
|
row = self.conn.execute(
|
|
'SELECT layout_name, last_name, wrap_count FROM janitor_cursors WHERE layout_name = ?',
|
|
(str(layout_name),),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row) if row else {
|
|
'layout_name': str(layout_name), 'last_name': '', 'wrap_count': 0,
|
|
}
|
|
|
|
def advance_janitor_cursor(self, layout_name, last_name, wrapped=False):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('janitor cursor authority requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
self.conn.execute(
|
|
'''INSERT INTO janitor_cursors(layout_name, last_name, wrap_count, updated_at)
|
|
VALUES (?, ?, ?, ?)
|
|
ON CONFLICT(layout_name) DO UPDATE SET
|
|
last_name = excluded.last_name,
|
|
wrap_count = janitor_cursors.wrap_count + excluded.wrap_count,
|
|
updated_at = excluded.updated_at''',
|
|
(str(layout_name), str(last_name), 1 if wrapped else 0, now),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
|
|
def claim_projection_job(self, generation, lease_token, lease_seconds=120):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection job claims require PostgreSQL')
|
|
now = utc_now_iso()
|
|
expires = datetime.fromtimestamp(
|
|
time.time() + max(10, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
token = secrets.token_urlsafe(32)
|
|
try:
|
|
owner = self.conn.execute(
|
|
'''SELECT 1 AS valid FROM pipeline_leases
|
|
WHERE worker_name = 'jsonl_projector' AND generation = ?
|
|
AND lease_token = ? AND state = 'ready' AND lease_expires_at > ? FOR SHARE''',
|
|
(int(generation), str(lease_token), now),
|
|
).fetchone()
|
|
if not owner:
|
|
raise RuntimeError('JSONL projector singleton fence is not valid')
|
|
row = self.conn.execute(
|
|
'''SELECT id FROM projection_jobs
|
|
WHERE (status = 'pending' AND (available_after IS NULL OR available_after <= ?))
|
|
OR (status = 'leased' AND (
|
|
lease_generation IS NULL OR lease_generation <> ?
|
|
OR (lease_expires_at IS NOT NULL AND lease_expires_at <= ?)
|
|
))
|
|
ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED''',
|
|
(now, int(generation), now),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return None
|
|
self.conn.execute(
|
|
'''UPDATE projection_jobs SET status = 'leased', attempts = attempts + 1,
|
|
lease_generation = ?, lease_token = ?, lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(int(generation), token, expires, now, row['id']),
|
|
)
|
|
claimed = self.conn.execute(
|
|
'SELECT * FROM projection_jobs WHERE id = ?', (row['id'],),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(claimed)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def expand_projection_job_capacity(
|
|
self, job_id, job_lease_token, actual_bytes, projection_max_bytes,
|
|
defer_seconds=1,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection capacity expansion requires PostgreSQL')
|
|
actual_bytes = max(0, int(actual_bytes))
|
|
projection_max_bytes = max(0, int(projection_max_bytes))
|
|
now = utc_now_iso()
|
|
available_after = datetime.fromtimestamp(
|
|
time.time() + max(1, int(defer_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
try:
|
|
job = self.conn.execute(
|
|
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
|
|
AND lease_token = ? FOR UPDATE''',
|
|
(int(job_id), str(job_lease_token)),
|
|
).fetchone()
|
|
if not job:
|
|
self.conn.rollback()
|
|
return None
|
|
if actual_bytes <= int(job['capacity_bytes']):
|
|
self.conn.commit()
|
|
return dict(job)
|
|
if self.conn.execute(
|
|
'SELECT 1 AS present FROM projection_appends WHERE job_id = ? LIMIT 1',
|
|
(int(job_id),),
|
|
).fetchone():
|
|
raise RuntimeError(
|
|
'projection capacity cannot expand after append preparation'
|
|
)
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
additional_bytes = actual_bytes - int(job['capacity_bytes'])
|
|
if (
|
|
not capacity
|
|
or actual_bytes > projection_max_bytes
|
|
or int(capacity['projection_bytes']) + additional_bytes
|
|
> projection_max_bytes
|
|
):
|
|
cursor = self.conn.execute(
|
|
'''UPDATE projection_jobs SET status = 'pending',
|
|
available_after = ?, lease_generation = NULL,
|
|
lease_token = NULL, lease_expires_at = NULL,
|
|
last_error_code = 'projection_capacity_backpressure',
|
|
last_error_detail = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_token = ?''',
|
|
(available_after, now, job_id, job_lease_token),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError(
|
|
'projection capacity deferral lost its lease fence'
|
|
)
|
|
self.conn.commit()
|
|
return False
|
|
cursor = self.conn.execute(
|
|
'''UPDATE projection_jobs SET capacity_bytes = ?,
|
|
last_error_code = NULL, last_error_detail = NULL,
|
|
updated_at = ?
|
|
WHERE id = ? AND status = 'leased' AND lease_token = ?
|
|
AND capacity_bytes = ?''',
|
|
(
|
|
actual_bytes, now, job_id, job_lease_token,
|
|
job['capacity_bytes'],
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise RuntimeError(
|
|
'projection capacity expansion lost its lease fence'
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity
|
|
SET projection_bytes = projection_bytes + ?, updated_at = ?
|
|
WHERE id = 1''',
|
|
(additional_bytes, now),
|
|
)
|
|
job = dict(job)
|
|
job['capacity_bytes'] = actual_bytes
|
|
job['updated_at'] = now
|
|
self.conn.commit()
|
|
return job
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def projection_stream_state(self, stream_name):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
row = self.conn.execute(
|
|
'''SELECT s.*, c.generation, c.committed_offset, c.last_append_id,
|
|
c.last_job_id, c.last_event_id, c.last_event_hash
|
|
FROM projection_streams s
|
|
JOIN projection_cursors c ON c.stream_name = s.stream_name
|
|
WHERE s.stream_name = ?''',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def projection_append_for_job(self, job_id, stream_name):
|
|
if not self.conn:
|
|
return None
|
|
row = self.conn.execute(
|
|
'SELECT * FROM projection_appends WHERE job_id = ? AND stream_name = ?',
|
|
(int(job_id), str(stream_name)),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def initialize_projection_stream_offset(self, stream_name, exact_file_size):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection stream offset initialization requires PostgreSQL')
|
|
exact_file_size = max(0, int(exact_file_size))
|
|
now = utc_now_iso()
|
|
try:
|
|
cursor = self.conn.execute(
|
|
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
if not cursor:
|
|
raise ValueError('projection stream cursor is absent')
|
|
if int(cursor['committed_offset']) == exact_file_size:
|
|
self.conn.commit()
|
|
return self.projection_stream_state(stream_name)
|
|
append_count = self.conn.execute(
|
|
'SELECT COUNT(*) AS count FROM projection_appends WHERE stream_name = ?',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
if (
|
|
int(cursor['committed_offset']) != 0
|
|
or cursor['last_append_id'] is not None
|
|
or cursor['last_job_id'] is not None
|
|
or int(append_count['count'] or 0) != 0
|
|
):
|
|
raise RuntimeError('projection stream/file mismatch has append history')
|
|
updated = self.conn.execute(
|
|
'''UPDATE projection_cursors SET committed_offset = ?, updated_at = ?
|
|
WHERE stream_name = ? AND committed_offset = 0
|
|
AND last_append_id IS NULL AND last_job_id IS NULL''',
|
|
(exact_file_size, now, str(stream_name)),
|
|
)
|
|
if int(updated.rowcount or 0) != 1:
|
|
raise RuntimeError('projection stream offset initialization lost its fence')
|
|
self.conn.commit()
|
|
return self.projection_stream_state(stream_name)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def prepare_projection_append(
|
|
self, job_id, job_lease_token, stream_name, generation,
|
|
byte_length, payload_sha256, record_count,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection append preparation requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
job = self.conn.execute(
|
|
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
|
|
AND lease_token = ? FOR UPDATE''',
|
|
(int(job_id), str(job_lease_token)),
|
|
).fetchone()
|
|
if not job:
|
|
self.conn.rollback()
|
|
return None
|
|
existing = self.conn.execute(
|
|
'''SELECT * FROM projection_appends WHERE job_id = ? AND stream_name = ? FOR UPDATE''',
|
|
(int(job_id), str(stream_name)),
|
|
).fetchone()
|
|
if existing:
|
|
self.conn.commit()
|
|
return dict(existing)
|
|
cursor = self.conn.execute(
|
|
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
if not cursor or int(cursor['generation']) != int(generation):
|
|
raise RuntimeError('projection stream generation changed before append preparation')
|
|
append_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO projection_appends(
|
|
job_id, stream_name, event_id, event_hash, generation,
|
|
byte_offset, byte_length, payload_sha256, record_count,
|
|
state, prepared_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'prepared', ?)''',
|
|
(
|
|
job_id, stream_name, job['event_id'], job['event_hash'], generation,
|
|
cursor['committed_offset'], max(0, int(byte_length)),
|
|
str(payload_sha256), max(0, int(record_count)), now,
|
|
),
|
|
)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM projection_appends WHERE id = ?', (append_id,),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def complete_projection_append(self, append_id, job_id, job_lease_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection append completion requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
job = self.conn.execute(
|
|
'''SELECT id FROM projection_jobs WHERE id = ? AND status = 'leased'
|
|
AND lease_token = ? FOR UPDATE''',
|
|
(int(job_id), str(job_lease_token)),
|
|
).fetchone()
|
|
append = self.conn.execute(
|
|
'SELECT * FROM projection_appends WHERE id = ? AND job_id = ? FOR UPDATE',
|
|
(int(append_id), int(job_id)),
|
|
).fetchone()
|
|
if not job or not append:
|
|
self.conn.rollback()
|
|
return False
|
|
expected_offset = int(append['byte_offset']) + int(append['byte_length'])
|
|
cursor = self.conn.execute(
|
|
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
|
|
(append['stream_name'],),
|
|
).fetchone()
|
|
if append['state'] == 'appended':
|
|
valid = bool(
|
|
cursor and int(cursor['generation']) == int(append['generation'])
|
|
and int(cursor['committed_offset']) >= expected_offset
|
|
)
|
|
self.conn.commit()
|
|
return valid
|
|
if not cursor or int(cursor['generation']) != int(append['generation']) or int(cursor['committed_offset']) != int(append['byte_offset']):
|
|
raise RuntimeError('projection cursor no longer matches the prepared append')
|
|
self.conn.execute(
|
|
"UPDATE projection_appends SET state = 'appended', appended_at = ? WHERE id = ?",
|
|
(now, append_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE projection_cursors SET committed_offset = ?, last_append_id = ?,
|
|
last_job_id = ?, last_event_id = ?, last_event_hash = ?, updated_at = ?
|
|
WHERE stream_name = ?''',
|
|
(
|
|
expected_offset, append_id, job_id, append['event_id'],
|
|
append['event_hash'], now, append['stream_name'],
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def complete_projection_job(self, job_id, job_lease_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection job completion requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
job = self.conn.execute(
|
|
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
|
|
AND lease_token = ? FOR UPDATE''',
|
|
(int(job_id), str(job_lease_token)),
|
|
).fetchone()
|
|
if not job:
|
|
self.conn.rollback()
|
|
return False
|
|
stream_count = sum(
|
|
int(bool(int(job['required_stream_mask']) & mask))
|
|
for mask in (1, 2, 4, 8, 16)
|
|
)
|
|
appended = self.conn.execute(
|
|
"SELECT stream_name FROM projection_appends WHERE job_id = ? AND state = 'appended'",
|
|
(int(job_id),),
|
|
).fetchall()
|
|
appended_count = sum(
|
|
int(bool(
|
|
(row['stream_name'] == 'scan_results' and int(job['required_stream_mask']) & 1)
|
|
or (row['stream_name'] == 'found_secrets' and int(job['required_stream_mask']) & 2)
|
|
or (row['stream_name'] == 'scan_errors' and int(job['required_stream_mask']) & 4)
|
|
or (str(row['stream_name']).startswith('keycheck:')
|
|
and str(row['stream_name']).endswith(':results')
|
|
and int(job['required_stream_mask']) & 8)
|
|
or (str(row['stream_name']).startswith('keycheck:')
|
|
and str(row['stream_name']).endswith(':status')
|
|
and int(job['required_stream_mask']) & 16)
|
|
))
|
|
for row in appended
|
|
)
|
|
if appended_count != stream_count:
|
|
raise RuntimeError('projection job does not have every required appended stream')
|
|
if not job['capacity_released']:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['projection_items']) < int(job['capacity_items'])
|
|
or int(capacity['projection_bytes']) < int(job['capacity_bytes'])
|
|
):
|
|
raise RuntimeError('projection completion would make capacity accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET projection_items = projection_items - ?,
|
|
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(job['capacity_items'], job['capacity_bytes'], now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE projection_jobs SET status = 'completed', capacity_released = 1,
|
|
lease_token = NULL, lease_expires_at = NULL, completed_at = ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(now, now, job_id),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def quarantine_projection_job(
|
|
self, job_id, job_lease_token, reason_code, detail='',
|
|
quarantine_max_items=10000, quarantine_max_bytes=1024 * 1024 * 1024,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection quarantine requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
job = self.conn.execute(
|
|
'''SELECT * FROM projection_jobs WHERE id = ? AND status = 'leased'
|
|
AND lease_token = ? FOR UPDATE''',
|
|
(int(job_id), str(job_lease_token)),
|
|
).fetchone()
|
|
if not job:
|
|
self.conn.rollback()
|
|
return False
|
|
prepared = self.conn.execute(
|
|
'''SELECT * FROM projection_appends
|
|
WHERE job_id = ? AND state = 'prepared' ORDER BY id FOR UPDATE''',
|
|
(int(job_id),),
|
|
).fetchall()
|
|
for append in prepared:
|
|
cursor = self.conn.execute(
|
|
'SELECT generation, committed_offset FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
|
|
(append['stream_name'],),
|
|
).fetchone()
|
|
if not cursor or (
|
|
int(cursor['generation']) != int(append['generation'])
|
|
or int(cursor['committed_offset']) != int(append['byte_offset'])
|
|
):
|
|
raise RuntimeError('prepared projection append cannot be safely canceled')
|
|
self.conn.execute(
|
|
'''INSERT INTO projection_append_audit(
|
|
append_id, job_id, stream_name, event_id, event_hash, generation,
|
|
byte_offset, byte_length, payload_sha256, state,
|
|
reason_code, reason_detail, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'quarantined', ?, ?, ?)''',
|
|
(
|
|
append['id'], append['job_id'], append['stream_name'],
|
|
append['event_id'], append['event_hash'], append['generation'],
|
|
append['byte_offset'], append['byte_length'], append['payload_sha256'],
|
|
str(reason_code), first_line(detail, 2000), now,
|
|
),
|
|
)
|
|
self.conn.execute('DELETE FROM projection_appends WHERE id = ?', (append['id'],))
|
|
existing = self.conn.execute(
|
|
'''SELECT id FROM pipeline_quarantine
|
|
WHERE subsystem = 'jsonl_projector' AND object_type = 'projection_job'
|
|
AND object_id = ? AND review_status = 'pending' ''',
|
|
(job_id,),
|
|
).fetchone()
|
|
if not existing:
|
|
self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_quarantine(
|
|
subsystem, object_type, object_id, projection_job_id,
|
|
event_id, payload_sha256, reason_code, reason_detail,
|
|
byte_count, capacity_items, capacity_bytes, detected_at
|
|
) VALUES ('jsonl_projector','projection_job',?,?,?,?,?,?,?,?,?,?)''',
|
|
(
|
|
job_id, job_id, job['event_id'], job['event_hash'], str(reason_code),
|
|
first_line(detail, 2000), job['capacity_bytes'],
|
|
job['capacity_items'], job['capacity_bytes'], now,
|
|
),
|
|
)
|
|
if not job['capacity_released']:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['projection_items']) < int(job['capacity_items'])
|
|
or int(capacity['projection_bytes']) < int(job['capacity_bytes'])
|
|
):
|
|
raise RuntimeError('projection quarantine would make capacity accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET projection_items = projection_items - ?,
|
|
projection_bytes = projection_bytes - ?,
|
|
quarantine_items = quarantine_items + ?,
|
|
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
job['capacity_items'], job['capacity_bytes'], job['capacity_items'],
|
|
job['capacity_bytes'], now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE projection_jobs SET status = 'quarantined', capacity_released = 1,
|
|
last_error_code = ?, last_error_detail = ?, lease_token = NULL,
|
|
lease_expires_at = NULL, completed_at = ?, updated_at = ? WHERE id = ?''',
|
|
(str(reason_code), first_line(detail, 2000), now, now, job_id),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def prepare_projection_rotation(self, stream_name, source_bytes, segment_relative_path):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection rotation requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
stream = self.conn.execute(
|
|
'SELECT * FROM projection_streams WHERE stream_name = ? FOR UPDATE',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
cursor = self.conn.execute(
|
|
'SELECT * FROM projection_cursors WHERE stream_name = ? FOR UPDATE',
|
|
(str(stream_name),),
|
|
).fetchone()
|
|
if not stream or not cursor or int(cursor['committed_offset']) != int(source_bytes):
|
|
raise RuntimeError('projection rotation source size does not match its cursor')
|
|
to_generation = int(stream['current_generation']) + 1
|
|
rotation_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO projection_rotations(
|
|
stream_name, from_generation, to_generation, source_bytes,
|
|
segment_relative_path, state, created_at
|
|
) VALUES (?, ?, ?, ?, ?, 'prepared', ?)
|
|
ON CONFLICT(stream_name, to_generation) DO NOTHING''',
|
|
(
|
|
stream_name, stream['current_generation'], to_generation,
|
|
int(source_bytes), str(segment_relative_path), now,
|
|
),
|
|
)
|
|
if rotation_id is None:
|
|
existing = self.conn.execute(
|
|
'SELECT * FROM projection_rotations WHERE stream_name = ? AND to_generation = ?',
|
|
(stream_name, to_generation),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(existing)
|
|
row = self.conn.execute(
|
|
'SELECT * FROM projection_rotations WHERE id = ?', (rotation_id,),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return dict(row)
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def complete_projection_rotation(self, rotation_id):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('projection rotation completion requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
rotation = self.conn.execute(
|
|
'SELECT * FROM projection_rotations WHERE id = ? FOR UPDATE',
|
|
(int(rotation_id),),
|
|
).fetchone()
|
|
if not rotation:
|
|
self.conn.rollback()
|
|
return False
|
|
self.conn.execute(
|
|
'''UPDATE projection_streams SET current_generation = ?, updated_at = ?
|
|
WHERE stream_name = ? AND current_generation = ?''',
|
|
(
|
|
rotation['to_generation'], now, rotation['stream_name'],
|
|
rotation['from_generation'],
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE projection_cursors SET generation = ?, committed_offset = 0,
|
|
last_append_id = NULL, updated_at = ? WHERE stream_name = ?''',
|
|
(rotation['to_generation'], now, rotation['stream_name']),
|
|
)
|
|
self.conn.execute(
|
|
"UPDATE projection_rotations SET state = 'completed', completed_at = ? WHERE id = ?",
|
|
(now, rotation_id),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def pending_projection_rotations(self, limit=100):
|
|
if not self.conn:
|
|
return []
|
|
rows = self.conn.execute(
|
|
'''SELECT r.*, s.base_relative_path
|
|
FROM projection_rotations r
|
|
JOIN projection_streams s ON s.stream_name = r.stream_name
|
|
WHERE r.state IN ('prepared','renamed') ORDER BY r.id LIMIT ?''',
|
|
(min(1000, max(1, int(limit))),),
|
|
).fetchall()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return [dict(row) for row in rows]
|
|
|
|
def claim_keycheck_candidate(
|
|
self, service, lease_owner, lease_seconds=300,
|
|
result_projection_reserve_bytes=KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES,
|
|
projection_max_items=10000,
|
|
projection_max_bytes=2 * 1024 * 1024 * 1024,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('normal keycheck candidate claims require PostgreSQL')
|
|
service = str(service or '').lower()
|
|
if not service or not lease_owner:
|
|
raise ValueError('keycheck service and lease owner are required')
|
|
now = utc_now_iso()
|
|
expires = datetime.fromtimestamp(
|
|
time.time() + max(30, int(lease_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
token = secrets.token_urlsafe(32)
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT id, result_projection_reserved_bytes,
|
|
result_projection_credit_transferred
|
|
FROM keycheck_candidates
|
|
WHERE service = ? AND (
|
|
state = 'pending'
|
|
OR (state = 'deferred' AND available_after IS NOT NULL AND available_after <= ?)
|
|
OR (state = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?)
|
|
)
|
|
ORDER BY priority DESC, id LIMIT 1 FOR UPDATE SKIP LOCKED''',
|
|
(service, now, now),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return None
|
|
reserve_bytes = max(
|
|
KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES,
|
|
int(result_projection_reserve_bytes),
|
|
)
|
|
if (
|
|
int(row['result_projection_reserved_bytes'] or 0) == 0
|
|
and not row['result_projection_credit_transferred']
|
|
):
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['projection_items']) + 1 > int(projection_max_items)
|
|
or int(capacity['projection_bytes']) + reserve_bytes > int(projection_max_bytes)
|
|
):
|
|
self.conn.rollback()
|
|
return {
|
|
'_capacity_blocked': True,
|
|
'service': service,
|
|
'reason': 'projection_capacity_saturated',
|
|
}
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET projection_items = projection_items + 1,
|
|
projection_bytes = projection_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(reserve_bytes, now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET result_projection_reserved_bytes = ?
|
|
WHERE id = ?''',
|
|
(reserve_bytes, row['id']),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET state = 'leased', attempts = attempts + 1,
|
|
lease_owner = ?, lease_token = ?, lease_expires_at = ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(str(lease_owner), token, expires, now, row['id']),
|
|
)
|
|
candidate = self.conn.execute(
|
|
'''SELECT c.*, kc.candidate_kind, kc.credential_hash,
|
|
kc.provider_key_hash, kc.secret_text,
|
|
kc.secret_json, kc.key_masked, kc.endpoint, kc.principal,
|
|
kc.metadata_json AS credential_metadata_json
|
|
FROM keycheck_candidates c
|
|
JOIN keycheck_credentials kc ON kc.id = c.credential_id
|
|
WHERE c.id = ?''',
|
|
(row['id'],),
|
|
).fetchone()
|
|
finding = None
|
|
if candidate['finding_id'] is not None:
|
|
finding_row = self.conn.execute(
|
|
'''SELECT f.*, cp.raw_value, cp.raw_v2_value, cp.structured_data_json,
|
|
cp.extra_data_json, cp.analysis_info_json, cp.extension_json,
|
|
cp.payload_sha256, cp.payload_bytes, cp.payload_omitted
|
|
FROM findings f
|
|
LEFT JOIN finding_compat_payloads cp ON cp.finding_id = f.id
|
|
WHERE f.id = ?''',
|
|
(candidate['finding_id'],),
|
|
).fetchone()
|
|
if finding_row:
|
|
if finding_row['payload_omitted']:
|
|
finding = {
|
|
'finding_uid': finding_row['finding_uid'],
|
|
'DetectorName': finding_row['detector_name'],
|
|
'finding_omitted': True,
|
|
'payload_sha256': finding_row['payload_sha256'],
|
|
}
|
|
else:
|
|
finding = safe_json_loads(finding_row['extension_json']) or {}
|
|
finding.update({
|
|
'finding_uid': finding_row['finding_uid'],
|
|
'DetectorName': finding_row['detector_name'],
|
|
'DetectorType': finding_row['detector_type'],
|
|
'Verified': bool(finding_row['verified']),
|
|
'Raw': finding_row['raw_value'],
|
|
'RawV2': finding_row['raw_v2_value'],
|
|
})
|
|
for column, key in (
|
|
('structured_data_json', 'StructuredData'),
|
|
('extra_data_json', 'ExtraData'),
|
|
('analysis_info_json', 'AnalysisInfo'),
|
|
):
|
|
value = safe_json_loads(finding_row[column])
|
|
if value is not None:
|
|
finding[key] = value
|
|
output = dict(candidate)
|
|
output['finding'] = finding or {}
|
|
self.conn.commit()
|
|
return output
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def keycheck_candidate_cached_status(self, candidate_id, lease_token):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('cached keycheck status lookup requires PostgreSQL')
|
|
try:
|
|
row = self.conn.execute(
|
|
'''SELECT c.service, c.state, c.lease_token, c.metadata_json,
|
|
s.status, s.status_group, s.last_result_id, s.result_source,
|
|
s.checked_at, s.state_version,
|
|
EXISTS(
|
|
SELECT 1 FROM pipeline_quarantine q
|
|
WHERE q.keycheck_candidate_id = c.id
|
|
AND q.review_status = 'approved_retry'
|
|
) AS approved_retry
|
|
FROM keycheck_candidates c
|
|
LEFT JOIN keycheck_current_state s ON s.credential_id = c.credential_id
|
|
WHERE c.id = ?''',
|
|
(int(candidate_id),),
|
|
).fetchone()
|
|
if (
|
|
not row or row['state'] != 'leased'
|
|
or str(row['lease_token'] or '') != str(lease_token)
|
|
):
|
|
self.conn.rollback()
|
|
return None
|
|
metadata = safe_json_loads(row['metadata_json']) or {}
|
|
reason = ''
|
|
if row['service'] == 'provider_resolver':
|
|
reason = 'provider_resolution_required'
|
|
elif 'recheck_of_status' in metadata or 'recheck_generation' in metadata:
|
|
reason = 'explicit_recheck'
|
|
elif row['approved_retry']:
|
|
reason = 'approved_quarantine_retry'
|
|
elif row['state_version'] is None:
|
|
reason = 'no_current_state'
|
|
self.conn.commit()
|
|
if reason:
|
|
return {'probe_required': True, 'reason': reason}
|
|
return {
|
|
'probe_required': False,
|
|
'status': row['status'],
|
|
'status_group': row['status_group'],
|
|
'last_result_id': int(row['last_result_id']),
|
|
'result_source': row['result_source'],
|
|
'checked_at': row['checked_at'],
|
|
'state_version': int(row['state_version']),
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def defer_keycheck_candidate(self, candidate_id, lease_token, error='', delay_seconds=60):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
return False
|
|
now = utc_now_iso()
|
|
available = datetime.fromtimestamp(
|
|
time.time() + max(1, int(delay_seconds)), timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
try:
|
|
candidate = self.conn.execute(
|
|
'''SELECT * FROM keycheck_candidates
|
|
WHERE id = ? AND state = 'leased' AND lease_token = ? FOR UPDATE''',
|
|
(int(candidate_id), str(lease_token)),
|
|
).fetchone()
|
|
if not candidate:
|
|
self.conn.rollback()
|
|
return False
|
|
reserved = (
|
|
int(candidate['result_projection_reserved_bytes'] or 0)
|
|
if not candidate['result_projection_credit_transferred'] else 0
|
|
)
|
|
if reserved:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if int(capacity['projection_items']) < 1 or int(capacity['projection_bytes']) < reserved:
|
|
raise RuntimeError('keycheck defer would make projection capacity negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET projection_items = projection_items - 1,
|
|
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(reserved, now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET state = 'deferred', available_after = ?,
|
|
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
|
|
result_projection_reserved_bytes = 0,
|
|
last_error = ?, updated_at = ? WHERE id = ?''',
|
|
(available, first_line(error, 1000), now, int(candidate_id)),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def quarantine_keycheck_candidate(self, candidate_id, lease_token, reason_code, detail=''):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('keycheck candidate quarantine requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
try:
|
|
candidate = self.conn.execute(
|
|
'''SELECT * FROM keycheck_candidates
|
|
WHERE id = ? AND state = 'leased' AND lease_token = ? FOR UPDATE''',
|
|
(int(candidate_id), str(lease_token)),
|
|
).fetchone()
|
|
if not candidate:
|
|
self.conn.rollback()
|
|
return False
|
|
existing = self.conn.execute(
|
|
'''SELECT id FROM pipeline_quarantine
|
|
WHERE subsystem = 'keycheck' AND object_type = 'keycheck_candidate'
|
|
AND object_id = ? AND review_status = 'pending' FOR UPDATE''',
|
|
(int(candidate_id),),
|
|
).fetchone()
|
|
if existing:
|
|
self.conn.commit()
|
|
return True
|
|
projection_bytes = (
|
|
int(candidate['result_projection_reserved_bytes'] or 0)
|
|
if not candidate['result_projection_credit_transferred'] else 0
|
|
)
|
|
quarantine_items = (0 if candidate['capacity_released'] else 1) + int(projection_bytes > 0)
|
|
quarantine_bytes = (
|
|
(0 if candidate['capacity_released'] else int(candidate['capacity_bytes']))
|
|
+ projection_bytes
|
|
)
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if (
|
|
int(capacity['keycheck_items']) < (0 if candidate['capacity_released'] else 1)
|
|
or int(capacity['keycheck_bytes']) < (
|
|
0 if candidate['capacity_released'] else int(candidate['capacity_bytes'])
|
|
)
|
|
or int(capacity['projection_items']) < int(projection_bytes > 0)
|
|
or int(capacity['projection_bytes']) < projection_bytes
|
|
):
|
|
raise RuntimeError('keycheck quarantine would make capacity accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET
|
|
keycheck_items = keycheck_items - ?, keycheck_bytes = keycheck_bytes - ?,
|
|
projection_items = projection_items - ?, projection_bytes = projection_bytes - ?,
|
|
quarantine_items = quarantine_items + ?,
|
|
quarantine_bytes = quarantine_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
0 if candidate['capacity_released'] else 1,
|
|
0 if candidate['capacity_released'] else candidate['capacity_bytes'],
|
|
int(projection_bytes > 0), projection_bytes,
|
|
quarantine_items, quarantine_bytes, now,
|
|
),
|
|
)
|
|
self.conn.insert_returning_id(
|
|
'''INSERT INTO pipeline_quarantine(
|
|
subsystem, object_type, object_id, keycheck_candidate_id,
|
|
reason_code, reason_detail, byte_count, capacity_items,
|
|
capacity_bytes, detected_at
|
|
) VALUES ('keycheck','keycheck_candidate',?,?,?,?,?,?,?,?)''',
|
|
(
|
|
candidate_id, candidate_id, str(reason_code), first_line(detail, 2000),
|
|
0, quarantine_items, quarantine_bytes, now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET state = 'quarantined', capacity_released = 1,
|
|
result_projection_credit_transferred = 1,
|
|
lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL,
|
|
last_error = ?, completed_at = ?, updated_at = ? WHERE id = ?''',
|
|
(first_line(detail or reason_code, 1000), now, now, candidate_id),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def complete_keycheck_candidate(
|
|
self, candidate_id, lease_token, event_id, status, status_group,
|
|
checked_at=None, message='', metadata=None, result_source='api_check',
|
|
resolved_service='', cached_state_version=None, cached_last_result_id=None,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('normal keycheck completion requires PostgreSQL')
|
|
metadata = dict(metadata or {})
|
|
resolved_service = str(resolved_service or '').strip().lower()
|
|
cached_completion = (
|
|
cached_state_version is not None or cached_last_result_id is not None
|
|
)
|
|
if cached_completion:
|
|
if cached_state_version is None or cached_last_result_id is None:
|
|
raise ValueError('cached keycheck completion requires an exact current-state fence')
|
|
if str(result_source or '') != 'cached_status':
|
|
raise ValueError('cached keycheck completion requires cached_status result source')
|
|
if resolved_service:
|
|
raise ValueError('provider resolution cannot use cached keycheck completion')
|
|
if (
|
|
int(metadata.get('cached_state_version') or -1) != int(cached_state_version)
|
|
or int(metadata.get('cached_result_id') or -1) != int(cached_last_result_id)
|
|
):
|
|
raise ValueError('cached keycheck metadata conflicts with its current-state fence')
|
|
if resolved_service:
|
|
if not re.fullmatch(r'[a-z][a-z0-9_]{1,63}', resolved_service):
|
|
raise ValueError('resolved keycheck service is invalid')
|
|
if str(metadata.get('resolved_provider') or '').strip().lower() != resolved_service:
|
|
raise ValueError('resolved keycheck service conflicts with result metadata')
|
|
if str(metadata.get('provider_resolution') or '') != 'matched':
|
|
raise ValueError('resolved keycheck service requires a matched provider resolution')
|
|
if not metadata.get('authenticated') and str(status).upper() not in ('VALID', 'ALIVE'):
|
|
raise ValueError('resolved keycheck service has no authenticated provider evidence')
|
|
event_payload = json.dumps({
|
|
'candidate_id': int(candidate_id),
|
|
'event_id': str(event_id or ''),
|
|
'status': str(status).upper(),
|
|
'status_group': str(status_group).lower(),
|
|
'checked_at': str(checked_at or ''),
|
|
'message': first_line(message, 1000),
|
|
'metadata': metadata,
|
|
'result_source': str(result_source or 'api_check'),
|
|
}, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str).encode('utf-8')
|
|
metadata['_event_hash'] = hashlib.sha256(event_payload).hexdigest()
|
|
encoded_metadata = json.dumps(
|
|
metadata, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
if len(encoded_metadata) > 1024 * 1024:
|
|
raise ValueError('keycheck result metadata exceeds its byte bound')
|
|
event_id = str(event_id or '')
|
|
if not re.fullmatch(r'[a-f0-9]{64}', event_id):
|
|
raise ValueError('keycheck event ID must be a stable SHA-256 identity')
|
|
now = utc_now_iso()
|
|
checked = str(checked_at or now)
|
|
try:
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
candidate = self.conn.execute(
|
|
'''SELECT c.*, kc.credential_hash, kc.provider_key_hash,
|
|
kc.key_masked, kc.secret_text, kc.secret_json,
|
|
kc.candidate_kind, kc.endpoint, kc.principal,
|
|
kc.metadata_json AS credential_metadata_json
|
|
FROM keycheck_candidates c
|
|
JOIN keycheck_credentials kc ON kc.id = c.credential_id
|
|
WHERE c.id = ? FOR UPDATE OF c''',
|
|
(int(candidate_id),),
|
|
).fetchone()
|
|
if not candidate:
|
|
raise ValueError('keycheck candidate is absent')
|
|
existing_map = self.conn.execute(
|
|
'SELECT keycheck_result_id FROM keycheck_event_map WHERE event_id = ? FOR UPDATE',
|
|
(event_id,),
|
|
).fetchone()
|
|
if existing_map:
|
|
if int(candidate['keycheck_result_id'] or 0) != int(existing_map['keycheck_result_id'] or 0):
|
|
raise ScanEventConflictError('keycheck event replay conflicts with candidate completion')
|
|
existing_result = self.conn.execute(
|
|
'SELECT metadata_json FROM keycheck_results WHERE id = ?',
|
|
(existing_map['keycheck_result_id'],),
|
|
).fetchone()
|
|
existing_metadata = safe_json_loads(existing_result['metadata_json'] if existing_result else '') or {}
|
|
if existing_metadata.get('_event_hash') != metadata['_event_hash']:
|
|
raise ScanEventConflictError('keycheck event replay has a different payload hash')
|
|
self.conn.commit()
|
|
return {
|
|
'completed': True, 'duplicate': True,
|
|
'keycheck_result_id': existing_map['keycheck_result_id'], 'event_id': event_id,
|
|
}
|
|
if candidate['state'] != 'leased' or str(candidate['lease_token'] or '') != str(lease_token):
|
|
self.conn.rollback()
|
|
return None
|
|
candidate = dict(candidate)
|
|
if cached_completion:
|
|
candidate_metadata = safe_json_loads(candidate['metadata_json']) or {}
|
|
approved_retry = self.conn.execute(
|
|
'''SELECT 1 FROM pipeline_quarantine
|
|
WHERE keycheck_candidate_id = ? AND review_status = 'approved_retry'
|
|
LIMIT 1''',
|
|
(int(candidate_id),),
|
|
).fetchone()
|
|
if (
|
|
candidate['service'] == 'provider_resolver'
|
|
or 'recheck_of_status' in candidate_metadata
|
|
or 'recheck_generation' in candidate_metadata
|
|
or approved_retry
|
|
):
|
|
self.conn.rollback()
|
|
return {'completed': False, 'probe_required': True}
|
|
current_state = self.conn.execute(
|
|
'''SELECT service, status, status_group, last_result_id, state_version
|
|
FROM keycheck_current_state WHERE credential_id = ? FOR UPDATE''',
|
|
(candidate['credential_id'],),
|
|
).fetchone()
|
|
if not current_state:
|
|
self.conn.rollback()
|
|
return {'completed': False, 'probe_required': True}
|
|
if (
|
|
current_state['service'] != candidate['service']
|
|
or int(current_state['state_version']) != int(cached_state_version)
|
|
or int(current_state['last_result_id']) != int(cached_last_result_id)
|
|
or str(current_state['status']).upper() != str(status).upper()
|
|
or str(current_state['status_group']).lower() != str(status_group).lower()
|
|
):
|
|
self.conn.rollback()
|
|
return {'completed': False, 'cached_state_changed': True}
|
|
if resolved_service and resolved_service != candidate['service']:
|
|
allowed_services = {'provider_resolver', 'qwen', 'deepseek', 'kimi', 'zai'}
|
|
if candidate['service'] not in allowed_services or resolved_service not in allowed_services - {'provider_resolver'}:
|
|
raise ValueError('provider resolution cannot reassign this keycheck service')
|
|
if candidate['candidate_kind'] != 'provider_key' or not candidate['secret_text']:
|
|
raise ValueError('provider resolution requires a text provider-key credential')
|
|
probe_material = str(candidate['secret_text'])
|
|
expected_provider_hash = hashlib.sha256(probe_material.encode('utf-8')).hexdigest()
|
|
if expected_provider_hash != candidate['provider_key_hash']:
|
|
raise ScanEventConflictError('provider resolution credential material conflicts with its key hash')
|
|
resolved_credential_hash = hashlib.sha256('|'.join((
|
|
'truf-credential-v2', resolved_service, probe_material,
|
|
)).encode('utf-8')).hexdigest()
|
|
resolved_credential_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO keycheck_credentials(
|
|
service, credential_hash, provider_key_hash, candidate_kind,
|
|
secret_text, secret_json, key_masked, endpoint, principal,
|
|
metadata_json, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(service, provider_key_hash) DO NOTHING''',
|
|
(
|
|
resolved_service, resolved_credential_hash, candidate['provider_key_hash'],
|
|
candidate['candidate_kind'], candidate['secret_text'], candidate['secret_json'],
|
|
candidate['key_masked'], candidate['endpoint'], candidate['principal'],
|
|
candidate['credential_metadata_json'], now, now,
|
|
),
|
|
)
|
|
if resolved_credential_id is None:
|
|
resolved_credential = self.conn.execute(
|
|
'''SELECT id, credential_hash, secret_text, secret_json
|
|
FROM keycheck_credentials
|
|
WHERE service = ? AND provider_key_hash = ? FOR UPDATE''',
|
|
(resolved_service, candidate['provider_key_hash']),
|
|
).fetchone()
|
|
if not resolved_credential or (
|
|
resolved_credential['credential_hash'] != resolved_credential_hash
|
|
or resolved_credential['secret_text'] != candidate['secret_text']
|
|
or resolved_credential['secret_json'] != candidate['secret_json']
|
|
):
|
|
raise ScanEventConflictError('resolved provider credential conflicts with canonical material')
|
|
resolved_credential_id = resolved_credential['id']
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET credential_id = ?, service = ?, updated_at = ?
|
|
WHERE id = ? AND state = 'leased' AND lease_token = ?''',
|
|
(
|
|
resolved_credential_id, resolved_service, now,
|
|
int(candidate_id), str(lease_token),
|
|
),
|
|
)
|
|
candidate['credential_id'] = resolved_credential_id
|
|
candidate['credential_hash'] = resolved_credential_hash
|
|
candidate['service'] = resolved_service
|
|
result_message = first_line(message, 1000)
|
|
projected_result = {
|
|
'event_id': event_id,
|
|
'service': candidate['service'],
|
|
'status': str(status).upper(),
|
|
'status_group': str(status_group).lower(),
|
|
'checked_at': checked,
|
|
'key_hash': candidate['provider_key_hash'],
|
|
'secret_hash': candidate['secret_hash'],
|
|
'key_masked': candidate['key_masked'],
|
|
'finding_uid': candidate['finding_uid'],
|
|
'detector': candidate['detector_name'],
|
|
'source': candidate['source'],
|
|
'message': result_message,
|
|
'metadata': metadata,
|
|
'result_source': str(result_source or 'api_check'),
|
|
}
|
|
result_projection_bytes = len(json.dumps(
|
|
projected_result, ensure_ascii=False, sort_keys=True,
|
|
separators=(',', ':'), default=str,
|
|
).encode('utf-8')) + 1
|
|
projection_capacity_bytes = result_projection_bytes
|
|
reserved_projection_bytes = int(candidate['result_projection_reserved_bytes'] or 0)
|
|
if (
|
|
reserved_projection_bytes < KEYCHECK_RESULT_PROJECTION_RESERVE_BYTES
|
|
or candidate['result_projection_credit_transferred']
|
|
):
|
|
raise RuntimeError('keycheck result has no exact pre-probe projection reservation')
|
|
if projection_capacity_bytes > reserved_projection_bytes:
|
|
raise ValueError(
|
|
'keycheck result compatibility output exceeds its pre-reserved byte capacity'
|
|
)
|
|
result_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO keycheck_results(
|
|
service, status, status_group, checked_at, key_hash, secret_hash,
|
|
key_masked, finding_id, target_scan_id, source, query, target,
|
|
detector_name, found_at, message, metadata_json, event_id, finding_uid,
|
|
link_status, link_attempts, linked_at, link_error,
|
|
candidate_id, credential_id, result_source, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
'linked', 0, ?, '', ?, ?, ?, ?)''',
|
|
(
|
|
candidate['service'], str(status).upper(), str(status_group).lower(), checked,
|
|
candidate['provider_key_hash'], candidate['secret_hash'],
|
|
candidate['key_masked'], candidate['finding_id'], candidate['target_scan_id'],
|
|
candidate['source'], candidate['query'], candidate['target'],
|
|
candidate['detector_name'], candidate['found_at'], result_message,
|
|
encoded_metadata.decode('utf-8'), event_id, candidate['finding_uid'], now,
|
|
candidate_id, candidate['credential_id'], str(result_source or 'api_check'), now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_event_map(event_id, keycheck_result_id, created_at)
|
|
VALUES (?, ?, ?)''',
|
|
(event_id, result_id, now),
|
|
)
|
|
if not cached_completion:
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_current_state(
|
|
credential_id, service, status, status_group, last_result_id,
|
|
result_source, checked_at, recheck_after, state_version,
|
|
metadata_json, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
ON CONFLICT(credential_id) DO UPDATE SET
|
|
service = excluded.service, status = excluded.status,
|
|
status_group = excluded.status_group, last_result_id = excluded.last_result_id,
|
|
result_source = excluded.result_source, checked_at = excluded.checked_at,
|
|
recheck_after = excluded.recheck_after,
|
|
state_version = keycheck_current_state.state_version + 1,
|
|
metadata_json = excluded.metadata_json, updated_at = excluded.updated_at''',
|
|
(
|
|
candidate['credential_id'], candidate['service'], str(status).upper(),
|
|
str(status_group).lower(), result_id, str(result_source or 'api_check'),
|
|
checked, metadata.get('recheck_after'), encoded_metadata.decode('utf-8'), now,
|
|
),
|
|
)
|
|
if not candidate['capacity_released']:
|
|
if (
|
|
int(capacity['keycheck_items']) < 1
|
|
or int(capacity['keycheck_bytes']) < int(candidate['capacity_bytes'])
|
|
or int(capacity['projection_items']) < 1
|
|
or int(capacity['projection_bytes']) < reserved_projection_bytes
|
|
):
|
|
raise RuntimeError('keycheck completion would make candidate capacity negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items - 1,
|
|
keycheck_bytes = keycheck_bytes - ?,
|
|
projection_bytes = projection_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(
|
|
candidate['capacity_bytes'],
|
|
reserved_projection_bytes - projection_capacity_bytes, now,
|
|
),
|
|
)
|
|
for stream_name, relative_path in ((
|
|
f'keycheck:{candidate["service"]}:results',
|
|
f'{candidate["service"]}/{candidate["service"]}Results.jsonl',
|
|
),):
|
|
self.conn.execute(
|
|
'''INSERT INTO projection_streams(
|
|
stream_name, base_relative_path, current_generation,
|
|
rotation_bytes, max_generations, created_at, updated_at
|
|
) VALUES (?, ?, 0, ?, 16, ?, ?)
|
|
ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, relative_path, 32 * 1024 * 1024, now, now),
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO projection_cursors(
|
|
stream_name, generation, committed_offset, updated_at
|
|
) VALUES (?, 0, 0, ?) ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, now),
|
|
)
|
|
projection_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO projection_jobs(
|
|
job_kind, event_id, event_hash, keycheck_result_id, status,
|
|
required_stream_mask, capacity_items, capacity_bytes,
|
|
created_at, updated_at
|
|
) VALUES ('keycheck_event', ?, ?, ?, 'pending', 8, 1, ?, ?, ?)''',
|
|
(
|
|
event_id, hashlib.sha256(encoded_metadata + event_id.encode('ascii')).hexdigest(),
|
|
result_id, projection_capacity_bytes, now, now,
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_candidates SET state = 'completed', keycheck_result_id = ?,
|
|
capacity_released = 1, lease_owner = NULL, lease_token = NULL,
|
|
lease_expires_at = NULL, result_projection_credit_transferred = 1,
|
|
completed_at = ?, updated_at = ? WHERE id = ?''',
|
|
(result_id, now, now, candidate_id),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'completed': True, 'duplicate': False, 'keycheck_result_id': result_id,
|
|
'projection_job_id': projection_id, 'event_id': event_id,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def keycheck_result_for_projection(self, keycheck_result_id):
|
|
if not self.conn:
|
|
return None
|
|
row = self.conn.execute(
|
|
'''SELECT kr.*, kc.candidate_kind, kc.key_masked AS credential_masked,
|
|
kc.secret_text AS credential_secret_text,
|
|
kc.secret_json AS credential_secret_json,
|
|
c.candidate_uid
|
|
FROM keycheck_results kr
|
|
LEFT JOIN keycheck_credentials kc ON kc.id = kr.credential_id
|
|
LEFT JOIN keycheck_candidates c ON c.id = kr.candidate_id
|
|
WHERE kr.id = ?''',
|
|
(int(keycheck_result_id),),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return dict(row) if row else None
|
|
|
|
def enqueue_keycheck_rechecks(
|
|
self, service, status_groups=None, max_items=1000,
|
|
queue_max_items=100000, queue_max_bytes=512 * 1024 * 1024,
|
|
):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('keycheck recheck generation requires PostgreSQL')
|
|
service = str(service or '').lower()
|
|
groups = sorted({str(value).lower() for value in (status_groups or []) if value})
|
|
max_items = min(10000, max(1, int(max_items)))
|
|
now = utc_now_iso()
|
|
status_scope = ','.join(groups) if groups else '*'
|
|
cursor_key = hashlib.sha256(
|
|
f'truf-keycheck-recheck-cursor-v1|{service}|{status_scope}'.encode('utf-8')
|
|
).hexdigest()
|
|
try:
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_recheck_cursors(
|
|
cursor_key, service, status_scope, last_credential_id,
|
|
wrap_count, updated_at
|
|
) VALUES (?, ?, ?, 0, 0, ?) ON CONFLICT(cursor_key) DO NOTHING''',
|
|
(cursor_key, service, status_scope, now),
|
|
)
|
|
cursor = self.conn.execute(
|
|
'SELECT * FROM keycheck_recheck_cursors WHERE cursor_key = ? FOR UPDATE',
|
|
(cursor_key,),
|
|
).fetchone()
|
|
clauses = [
|
|
's.service = ?',
|
|
'''NOT EXISTS (
|
|
SELECT 1 FROM pipeline_quarantine q
|
|
JOIN keycheck_candidates blocked ON blocked.id = q.keycheck_candidate_id
|
|
WHERE blocked.credential_id = s.credential_id
|
|
AND q.review_status = 'pending'
|
|
AND q.reason_code IN (
|
|
'provider_candidate_unconsumed',
|
|
'candidate_provider_route_mismatch'
|
|
)
|
|
)''',
|
|
]
|
|
params = [service]
|
|
if groups:
|
|
clauses.append('s.status_group IN ({})'.format(','.join('?' for _ in groups)))
|
|
params.extend(groups)
|
|
|
|
def select_rows(after_id):
|
|
return self.conn.execute(
|
|
f'''SELECT s.credential_id, s.state_version, s.status, s.status_group,
|
|
c.credential_hash, c.candidate_kind, c.secret_text, c.secret_json,
|
|
r.secret_hash, r.source, r.query, r.target, r.detector_name,
|
|
r.found_at, r.finding_uid
|
|
FROM keycheck_current_state s
|
|
JOIN keycheck_credentials c ON c.id = s.credential_id
|
|
JOIN keycheck_results r ON r.id = s.last_result_id
|
|
WHERE {' AND '.join(clauses)} AND s.credential_id > ?
|
|
ORDER BY s.credential_id LIMIT ?''',
|
|
(*params, int(after_id), max_items),
|
|
).fetchall()
|
|
|
|
rows = select_rows(cursor['last_credential_id'])
|
|
wrapped = False
|
|
if not rows and int(cursor['last_credential_id'] or 0) > 0:
|
|
rows = select_rows(0)
|
|
wrapped = True
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
inserted = 0
|
|
inserted_bytes = 0
|
|
last_scanned = 0 if wrapped else int(cursor['last_credential_id'] or 0)
|
|
generation = int(cursor['wrap_count'] or 0) + (1 if wrapped else 0)
|
|
for row in rows:
|
|
uid = hashlib.sha256('|'.join((
|
|
'truf-keycheck-recheck-v2', service, status_scope,
|
|
str(generation), str(row['credential_id']), str(row['state_version']),
|
|
)).encode('utf-8')).hexdigest()
|
|
capacity_bytes = len(str(row['secret_text'] or row['secret_json'] or '').encode('utf-8')) + 512
|
|
if (
|
|
int(capacity['keycheck_items']) + inserted + 1 > int(queue_max_items)
|
|
or int(capacity['keycheck_bytes']) + inserted_bytes + capacity_bytes > int(queue_max_bytes)
|
|
):
|
|
break
|
|
candidate_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO keycheck_candidates(
|
|
candidate_uid, credential_id, service, routed_service, secret_hash,
|
|
source, query, target, detector_name, found_at, finding_uid,
|
|
metadata_json, state,
|
|
capacity_bytes, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
|
|
ON CONFLICT(candidate_uid) DO NOTHING''',
|
|
(
|
|
uid, row['credential_id'], service, service, row['secret_hash'],
|
|
row['source'], row['query'], row['target'], row['detector_name'],
|
|
row['found_at'], row['finding_uid'],
|
|
json_dumps({
|
|
'provider_hint': service,
|
|
'recheck_of_status': row['status'],
|
|
'state_version': row['state_version'],
|
|
'recheck_generation': generation,
|
|
}),
|
|
capacity_bytes, now, now,
|
|
),
|
|
)
|
|
if candidate_id is not None:
|
|
inserted += 1
|
|
inserted_bytes += capacity_bytes
|
|
last_scanned = int(row['credential_id'])
|
|
if inserted:
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items + ?,
|
|
keycheck_bytes = keycheck_bytes + ?, updated_at = ? WHERE id = 1''',
|
|
(inserted, inserted_bytes, now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE keycheck_recheck_cursors SET last_credential_id = ?,
|
|
wrap_count = wrap_count + ?, updated_at = ? WHERE cursor_key = ?''',
|
|
(last_scanned, 1 if wrapped else 0, now, cursor_key),
|
|
)
|
|
self.conn.commit()
|
|
return inserted
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
def keycheck_service_has_work(self, service):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('keycheck work inspection requires PostgreSQL')
|
|
now = utc_now_iso()
|
|
row = self.conn.execute(
|
|
'''SELECT (
|
|
EXISTS (
|
|
SELECT 1 FROM keycheck_candidates
|
|
WHERE service = ? AND state = 'pending'
|
|
) OR EXISTS (
|
|
SELECT 1 FROM keycheck_candidates
|
|
WHERE service = ? AND state = 'deferred'
|
|
AND available_after IS NOT NULL AND available_after <= ?
|
|
) OR EXISTS (
|
|
SELECT 1 FROM keycheck_candidates
|
|
WHERE service = ? AND state = 'leased'
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
|
|
)
|
|
) AS present''',
|
|
(str(service).lower(), str(service).lower(), now, str(service).lower(), now),
|
|
).fetchone()
|
|
self.conn.commit()
|
|
return bool(row and row['present'])
|
|
|
|
def has_claimable_targets(self, source, platform, max_attempts=0):
|
|
if not self.conn:
|
|
return False
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
max_attempts_value = max(0, int(max_attempts or 0))
|
|
row = self.conn.execute(
|
|
'''SELECT 1 AS present FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND (status = 'pending'
|
|
OR (status = 'deferred' AND available_after IS NOT NULL AND available_after <= ?)
|
|
OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id LIMIT 1''',
|
|
(source, platform, now, now, max_attempts_value, max_attempts_value, now),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return bool(row)
|
|
|
|
return bool(self._safe('has_claimable_targets', op, False))
|
|
|
|
def has_claimable_targets_v2(self, source, platform, max_attempts=0):
|
|
if not self.conn:
|
|
return False
|
|
try:
|
|
now = utc_now_iso()
|
|
maximum = max(0, int(max_attempts or 0))
|
|
row = self.conn.execute(
|
|
HAS_CLAIMABLE_TARGETS_V2_SQL,
|
|
(
|
|
source, platform, now, maximum, maximum,
|
|
source, platform, now, maximum, maximum,
|
|
),
|
|
).fetchone()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
self.last_error = ''
|
|
return bool(row and row['present'])
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self.conn.rollback()
|
|
return False
|
|
|
|
def claim_targets(
|
|
self, source, platform, limit, lease_owner=None, lease_seconds=86400,
|
|
max_attempts=0, return_rows=False, claim_batch=None,
|
|
):
|
|
if not self.conn:
|
|
return None
|
|
batch = str(claim_batch or secrets.token_urlsafe(24))
|
|
self._last_claim_expectation = None
|
|
commit_state = {'started': False}
|
|
|
|
def commit_claim():
|
|
commit_state['started'] = True
|
|
self.conn.commit()
|
|
commit_state['started'] = False
|
|
|
|
def remember_expected(owner, rows):
|
|
self._last_claim_expectation = {
|
|
'claim_batch': batch,
|
|
'lease_owner': str(owner),
|
|
'claims': [
|
|
{
|
|
'id': int(row['id']),
|
|
'lease_token': str(row['lease_token']),
|
|
}
|
|
for row in rows
|
|
],
|
|
}
|
|
|
|
def op():
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
now = utc_now_iso()
|
|
lease_until = datetime.fromtimestamp(time.time() + max(60, int(lease_seconds or 86400)), timezone.utc).isoformat(timespec='seconds')
|
|
limit_value = max(1, int(limit or 1000000))
|
|
max_attempts_value = max(0, int(max_attempts or 0))
|
|
owner = lease_owner or f'{source}:{os.getpid()}'
|
|
existing = self.conn.execute(
|
|
'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
|
|
FROM target_queue WHERE source = ? AND platform = ? AND status = 'in_progress'
|
|
AND current_result_reservation_id IS NULL
|
|
AND lease_owner = ? AND claim_batch = ? ORDER BY id''',
|
|
(source, platform, owner, batch),
|
|
).fetchall()
|
|
if existing:
|
|
remember_expected(owner, existing)
|
|
commit_claim()
|
|
return existing if return_rows else [row['target'] for row in existing]
|
|
candidate_limit = min(
|
|
CLAIM_CANDIDATE_MAX,
|
|
max(CLAIM_CANDIDATE_MIN, limit_value * 8),
|
|
)
|
|
if max_attempts_value:
|
|
if self.conn.is_postgres:
|
|
exhausted = self.conn.execute(
|
|
'''WITH candidates AS MATERIALIZED (
|
|
SELECT id FROM (
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'pending'
|
|
AND current_result_reservation_id IS NULL
|
|
AND attempts >= ?
|
|
ORDER BY id LIMIT ?)
|
|
UNION ALL
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'deferred'
|
|
AND current_result_reservation_id IS NULL
|
|
AND attempts >= ?
|
|
ORDER BY id LIMIT ?)
|
|
UNION ALL
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'in_progress'
|
|
AND current_result_reservation_id IS NULL
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
|
|
AND attempts >= ?
|
|
ORDER BY id LIMIT ?)
|
|
) AS status_candidates
|
|
ORDER BY id LIMIT ?
|
|
)
|
|
SELECT q.id FROM candidates c
|
|
JOIN target_queue q ON q.id = c.id
|
|
ORDER BY q.id FOR UPDATE OF q SKIP LOCKED''',
|
|
(
|
|
source, platform, max_attempts_value, candidate_limit,
|
|
source, platform, max_attempts_value, candidate_limit,
|
|
source, platform, now, max_attempts_value, candidate_limit,
|
|
candidate_limit,
|
|
),
|
|
).fetchall()
|
|
exhausted_ids = [row['id'] for row in exhausted]
|
|
if exhausted_ids:
|
|
placeholders = ','.join('?' for _ in exhausted_ids)
|
|
self.conn.execute(
|
|
f'''UPDATE target_queue SET status = 'failed', completed_at = COALESCE(completed_at, ?),
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = NULL, last_error = COALESCE(last_error, 'target retry attempts exhausted'), updated_at = ?
|
|
WHERE id IN ({placeholders})''',
|
|
(now, now, *exhausted_ids),
|
|
)
|
|
else:
|
|
self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'failed', completed_at = COALESCE(completed_at, ?),
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = NULL, last_error = COALESCE(last_error, 'target retry attempts exhausted'), updated_at = ?
|
|
WHERE source = ? AND platform = ? AND COALESCE(attempts, 0) >= ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND (status IN ('pending', 'deferred')
|
|
OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))''',
|
|
(now, now, source, platform, max_attempts_value, now),
|
|
)
|
|
if self.conn.is_postgres:
|
|
rows = self.conn.execute(
|
|
'''WITH candidates AS MATERIALIZED (
|
|
SELECT id FROM (
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'pending'
|
|
AND current_result_reservation_id IS NULL
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id LIMIT ?)
|
|
UNION ALL
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'deferred'
|
|
AND current_result_reservation_id IS NULL
|
|
AND available_after IS NOT NULL AND available_after <= ?
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id LIMIT ?)
|
|
UNION ALL
|
|
(SELECT id FROM target_queue
|
|
WHERE source = ? AND platform = ? AND status = 'in_progress'
|
|
AND current_result_reservation_id IS NULL
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id LIMIT ?)
|
|
) AS status_candidates
|
|
ORDER BY id LIMIT ?
|
|
)
|
|
SELECT q.id, q.target FROM candidates c
|
|
JOIN target_queue q ON q.id = c.id
|
|
ORDER BY q.id LIMIT ? FOR UPDATE OF q SKIP LOCKED''',
|
|
(
|
|
source, platform, max_attempts_value, max_attempts_value, now, candidate_limit,
|
|
source, platform, now, max_attempts_value, max_attempts_value, candidate_limit,
|
|
source, platform, now, max_attempts_value, max_attempts_value, now, candidate_limit,
|
|
candidate_limit, limit_value,
|
|
),
|
|
).fetchall()
|
|
else:
|
|
rows = self.conn.execute(
|
|
'''SELECT id, target
|
|
FROM target_queue
|
|
WHERE source = ? AND platform = ?
|
|
AND current_result_reservation_id IS NULL
|
|
AND (status = 'pending' OR (status = 'deferred' AND available_after IS NOT NULL AND available_after <= ?) OR (status = 'in_progress' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?))
|
|
AND (? = 0 OR COALESCE(attempts, 0) < ?)
|
|
AND (available_after IS NULL OR available_after <= ?)
|
|
AND (resolver_state IS NULL OR resolver_state = 'resolved')
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_targets experiment_target
|
|
WHERE experiment_target.target_queue_id = target_queue.id
|
|
)
|
|
ORDER BY id LIMIT ?''',
|
|
(source, platform, now, now, max_attempts_value, max_attempts_value, now, limit_value),
|
|
).fetchall()
|
|
ids = [row['id'] for row in rows]
|
|
for row_id in ids:
|
|
lease_token = secrets.token_urlsafe(32)
|
|
self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'in_progress', lease_owner = ?, lease_token = ?, claim_batch = ?, leased_at = ?, lease_expires_at = ?,
|
|
attempts = COALESCE(attempts, 0) + 1,
|
|
scan_remote_modified_at = remote_modified_at, updated_at = ? WHERE id = ?''',
|
|
(owner, lease_token, batch, now, lease_until, now, row_id),
|
|
)
|
|
claimed_rows = None
|
|
if return_rows and ids:
|
|
placeholders = ','.join('?' for _ in ids)
|
|
claimed_rows = self.conn.execute(
|
|
f'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
|
|
FROM target_queue WHERE id IN ({placeholders}) ORDER BY id''',
|
|
ids,
|
|
).fetchall()
|
|
expected_rows = claimed_rows if return_rows and ids else self.conn.execute(
|
|
f'''SELECT id, lease_token FROM target_queue
|
|
WHERE claim_batch = ? AND lease_owner = ? ORDER BY id''',
|
|
(batch, owner),
|
|
).fetchall()
|
|
remember_expected(owner, expected_rows)
|
|
commit_claim()
|
|
if return_rows and ids:
|
|
return claimed_rows
|
|
return [row['target'] for row in rows]
|
|
|
|
last_error = None
|
|
for attempt in range(SQLITE_LOCK_RETRY_ATTEMPTS):
|
|
commit_state['started'] = False
|
|
try:
|
|
result = op()
|
|
self.last_error = ''
|
|
return result
|
|
except Exception as exc:
|
|
last_error = exc
|
|
self.last_error = str(exc)
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
if commit_state['started']:
|
|
raise
|
|
if not sqlite_lock_error(exc) or attempt == SQLITE_LOCK_RETRY_ATTEMPTS - 1:
|
|
raise
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
raise last_error
|
|
|
|
def claim_recovery_expectation(self, claim_batch, lease_owner):
|
|
expectation = self._last_claim_expectation
|
|
if not expectation:
|
|
return None
|
|
if (
|
|
str(expectation.get('claim_batch') or '') != str(claim_batch or '')
|
|
or str(expectation.get('lease_owner') or '') != str(lease_owner or '')
|
|
):
|
|
return None
|
|
return [dict(claim) for claim in expectation.get('claims') or []]
|
|
|
|
def recover_claim_batch(self, claim_batch, lease_owner):
|
|
if not claim_batch or not lease_owner:
|
|
return []
|
|
if not self.conn and not self._reset_connection():
|
|
return []
|
|
|
|
def op():
|
|
rows = self.conn.execute(
|
|
'''SELECT id, target, normalized_target, attempts, lease_owner, lease_token, claim_batch
|
|
FROM target_queue WHERE status = 'in_progress' AND claim_batch = ?
|
|
AND lease_owner = ? ORDER BY id''',
|
|
(str(claim_batch), str(lease_owner)),
|
|
).fetchall()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return rows
|
|
rows = self._safe('recover_claim_batch', op, [])
|
|
if self.last_error:
|
|
first_error = self.last_error
|
|
if not self._reset_connection():
|
|
raise RuntimeError(f'unable to reconnect for committed claim batch recovery: {first_error}')
|
|
rows = self._safe('recover_claim_batch_after_reconnect', op, [])
|
|
if self.last_error:
|
|
raise RuntimeError(f'unable to recover committed claim batch: {self.last_error}')
|
|
return rows
|
|
|
|
def reclaim_target_leases(self, source, platform, lease_owner=None):
|
|
if not self.conn:
|
|
return 0
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending', lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL,
|
|
lease_expires_at = NULL, updated_at = ?
|
|
WHERE source = ? AND platform = ? AND status = 'in_progress'
|
|
AND current_result_reservation_id IS NULL
|
|
AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?''',
|
|
(now, source, platform, now),
|
|
)
|
|
self.conn.commit()
|
|
return int(getattr(cur, 'rowcount', 0) or 0)
|
|
return self._safe('reclaim_target_leases', op, 0)
|
|
|
|
def complete_target_queue_item(self, source, platform, target, target_scan_id=None, status='done', error=None, available_after=None, lease_owner=None, reset_attempts=False, queue_id=None, lease_token=None):
|
|
if not self.conn or queue_id is None or not lease_token:
|
|
return False
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
completed = now if status in ('done', 'failed') else None
|
|
params = [status, target_scan_id, first_line(error, 500) if error else None, available_after, 1 if reset_attempts else 0, completed, now]
|
|
params.extend((queue_id, lease_token))
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET
|
|
status = ?, target_scan_id = COALESCE(?, target_scan_id), last_error = ?,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = ?, attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
|
|
completed_at = COALESCE(?, completed_at), updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
|
|
params,
|
|
)
|
|
self.conn.commit()
|
|
return getattr(cur, 'rowcount', 0) != 0
|
|
return self._safe('complete_target_queue_item', op, False)
|
|
|
|
def renew_target_leases(self, lease_owner, lease_seconds=1800, lease_tokens=None):
|
|
tokens = [str(token) for token in (lease_tokens or []) if token]
|
|
if not self.conn or not lease_owner or not tokens:
|
|
return 0
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
lease_until = datetime.fromtimestamp(
|
|
time.time() + max(60, int(lease_seconds or 1800)), timezone.utc
|
|
).isoformat(timespec='seconds')
|
|
placeholders = ','.join('?' for _ in tokens)
|
|
cur = self.conn.execute(
|
|
f'''UPDATE target_queue SET lease_expires_at = ?, updated_at = ?
|
|
WHERE status = 'in_progress' AND lease_owner = ?
|
|
AND lease_token IN ({placeholders})''',
|
|
(lease_until, now, lease_owner, *tokens),
|
|
)
|
|
self.conn.commit()
|
|
return int(getattr(cur, 'rowcount', 0) or 0)
|
|
return self._safe('renew_target_leases', op, 0)
|
|
|
|
def active_target_lease_tokens(self, lease_owner, lease_tokens=None):
|
|
tokens = [str(token) for token in (lease_tokens or []) if token]
|
|
if not self.conn or not lease_owner or not tokens:
|
|
return set()
|
|
|
|
def op():
|
|
placeholders = ','.join('?' for _ in tokens)
|
|
rows = self.conn.execute(
|
|
f'''SELECT lease_token FROM target_queue
|
|
WHERE status = 'in_progress' AND lease_owner = ?
|
|
AND lease_token IN ({placeholders})''',
|
|
(lease_owner, *tokens),
|
|
).fetchall()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return {str(row['lease_token']) for row in rows if row['lease_token']}
|
|
return self._safe('active_target_lease_tokens', op, set())
|
|
|
|
def refund_target_claims(self, claims, error='infrastructure persistence failure'):
|
|
normalized = []
|
|
for claim in claims or []:
|
|
queue_id = claim.get('id') if isinstance(claim, dict) else claim['id']
|
|
lease_token = claim.get('lease_token') if isinstance(claim, dict) else claim['lease_token']
|
|
if queue_id is None or not lease_token:
|
|
return False
|
|
item = (int(queue_id), str(lease_token))
|
|
if item not in normalized:
|
|
normalized.append(item)
|
|
if not self.conn or not normalized:
|
|
return False
|
|
|
|
def op():
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
now = utc_now_iso()
|
|
for queue_id, lease_token in normalized:
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending',
|
|
attempts = CASE WHEN COALESCE(attempts, 0) > 0 THEN attempts - 1 ELSE 0 END,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = NULL, completed_at = NULL, last_error = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
|
|
(first_line(error, 500), now, queue_id, lease_token),
|
|
)
|
|
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
self.conn.commit()
|
|
return True
|
|
return self._safe('refund_target_claims', op, False)
|
|
|
|
def refund_target_claim(self, queue_id, lease_token, error='infrastructure persistence failure'):
|
|
return self.refund_target_claims(
|
|
[{'id': queue_id, 'lease_token': lease_token}], error,
|
|
)
|
|
|
|
def refund_stopped_result_spool_claims(self, reservation, error='supervisor controlled source stop'):
|
|
claims = [dict(claim) for claim in (reservation or {}).get('claims') or []]
|
|
if not self.conn or len(claims) > 10000:
|
|
return False
|
|
if not claims:
|
|
return True
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
rows = {}
|
|
for claim in claims:
|
|
queue_id = int(claim.get('queue_id'))
|
|
row = self.conn.execute(
|
|
f'''SELECT id, status, lease_owner, lease_token, claim_batch
|
|
FROM target_queue WHERE id = ?{lock_suffix}''',
|
|
(queue_id,),
|
|
).fetchone()
|
|
if not (
|
|
row
|
|
and row['status'] == 'in_progress'
|
|
and row['lease_owner'] == claim.get('lease_owner')
|
|
and row['lease_token'] == claim.get('lease_token')
|
|
and row['claim_batch'] == claim.get('claim_batch')
|
|
and str(claim.get('claim_batch') or '') == str((reservation or {}).get('reservation_id') or '')
|
|
and str(claim.get('lease_owner') or '') == str((reservation or {}).get('owner') or '')
|
|
):
|
|
self.conn.rollback()
|
|
return False
|
|
rows[queue_id] = row
|
|
now = utc_now_iso()
|
|
for queue_id, row in rows.items():
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'pending',
|
|
attempts = CASE WHEN COALESCE(attempts, 0) > 0 THEN attempts - 1 ELSE 0 END,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = NULL, completed_at = NULL, last_error = ?, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress'
|
|
AND lease_owner = ? AND lease_token = ? AND claim_batch = ?''',
|
|
(
|
|
first_line(error, 500), now, queue_id,
|
|
row['lease_owner'], row['lease_token'], row['claim_batch'],
|
|
),
|
|
)
|
|
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
self.conn.commit()
|
|
return True
|
|
except Exception:
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
def claim_docker_resolutions(
|
|
self, source, limit, lease_owner, lease_seconds=300, periodic_limit=0,
|
|
periodic_only=False, allowed_queries=None,
|
|
):
|
|
if not self.conn or not source or not lease_owner:
|
|
return []
|
|
query_values = None
|
|
if allowed_queries is not None:
|
|
query_values = []
|
|
for value in allowed_queries:
|
|
query = str(value or '').strip()
|
|
if not query:
|
|
raise ValueError('Docker resolver allowed query is empty')
|
|
if query not in query_values:
|
|
query_values.append(query)
|
|
|
|
def op():
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
state = self._locked_runtime_control_state(shared=True)
|
|
if state['effective_discovery_paused']:
|
|
self.conn.commit()
|
|
return []
|
|
now = utc_now_iso()
|
|
lease_until = datetime.fromtimestamp(
|
|
time.time() + max(60, int(lease_seconds or 300)), timezone.utc
|
|
).isoformat(timespec='seconds')
|
|
lock_suffix = ' FOR UPDATE SKIP LOCKED' if self.conn.is_postgres else ''
|
|
total_limit = max(1, int(limit or 1))
|
|
query_clause = ''
|
|
query_params = ()
|
|
if query_values is not None:
|
|
if not query_values:
|
|
self.conn.commit()
|
|
return []
|
|
query_clause = ' AND query IN (' + ','.join('?' for _ in query_values) + ')'
|
|
query_params = tuple(query_values)
|
|
retry_rows = [] if periodic_only else self.conn.execute(
|
|
f'''SELECT id, target, COALESCE(resolver_attempts, 0) AS resolver_attempts_before
|
|
FROM target_queue
|
|
WHERE source = ? AND platform = 'docker' AND status = 'deferred'
|
|
AND resolver_state IN ('pending', 'retry', 'resolving')
|
|
AND resolver_due_at IS NOT NULL AND resolver_due_at <= ?{query_clause}
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_repositories member
|
|
JOIN docker_depth_experiments experiment
|
|
ON experiment.id = member.experiment_id
|
|
WHERE (member.repository_queue_id = target_queue.id
|
|
OR member.replacement_repository_queue_id = target_queue.id)
|
|
AND experiment.state IN (
|
|
'planned','holding','resolving','active','draining','held'
|
|
)
|
|
)
|
|
ORDER BY resolver_due_at, id LIMIT ?{lock_suffix}''',
|
|
(source, now, *query_params, total_limit),
|
|
).fetchall()
|
|
rows = [(row, False) for row in retry_rows]
|
|
periodic_count = min(
|
|
max(0, int(periodic_limit or 0)),
|
|
max(0, total_limit - len(rows)),
|
|
)
|
|
if periodic_count:
|
|
periodic_rows = self.conn.execute(
|
|
f'''SELECT id, target, COALESCE(resolver_attempts, 0) AS resolver_attempts_before
|
|
FROM target_queue
|
|
WHERE source = ? AND platform = 'docker' AND status = 'done'
|
|
AND resolver_state = 'resolved' AND resolver_token IS NULL
|
|
AND (resolver_due_at IS NULL OR resolver_due_at <= ?){query_clause}
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM docker_depth_experiment_repositories member
|
|
JOIN docker_depth_experiments experiment
|
|
ON experiment.id = member.experiment_id
|
|
WHERE (member.repository_queue_id = target_queue.id
|
|
OR member.replacement_repository_queue_id = target_queue.id)
|
|
AND experiment.state IN (
|
|
'planned','holding','resolving','active','draining','held'
|
|
)
|
|
)
|
|
ORDER BY COALESCE(resolver_due_at, ''), id
|
|
LIMIT ?{lock_suffix}''',
|
|
(source, now, *query_params, periodic_count),
|
|
).fetchall()
|
|
rows.extend((row, True) for row in periodic_rows)
|
|
output = []
|
|
for row, periodic in rows:
|
|
token = secrets.token_urlsafe(32)
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'resolving',
|
|
resolver_due_at = ?, available_after = ?,
|
|
resolver_token = ?, resolver_attempts = COALESCE(resolver_attempts, 0) + 1,
|
|
updated_at = ? WHERE id = ?''',
|
|
(lease_until, lease_until, token, now, row['id']),
|
|
)
|
|
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
|
|
self.conn.rollback()
|
|
return []
|
|
attempts_before = int(row['resolver_attempts_before'] or 0)
|
|
output.append({
|
|
'id': row['id'],
|
|
'target': row['target'],
|
|
'resolver_token': token,
|
|
'resolver_attempts_before': attempts_before,
|
|
'resolver_attempts': attempts_before + 1,
|
|
'periodic': periodic,
|
|
})
|
|
self.conn.commit()
|
|
return output
|
|
return self._safe('claim_docker_resolutions', op, [])
|
|
|
|
def finish_docker_resolution(
|
|
self, source, queue_id, resolver_token, tagged_targets=None, error='', complete=None,
|
|
*, retry_at=None, claim_attempt_consumed=True, refresh_interval_sec=0,
|
|
):
|
|
if not self.conn or queue_id is None or not resolver_token:
|
|
return False
|
|
tagged_targets = list(tagged_targets or [])
|
|
complete = bool(tagged_targets) if complete is None else bool(complete)
|
|
if not claim_attempt_consumed and (complete or not retry_at):
|
|
return False
|
|
|
|
def op():
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
if tagged_targets:
|
|
self._require_discovery_admission_locked()
|
|
now = utc_now_iso()
|
|
now_dt = datetime.now(timezone.utc)
|
|
retry_due = None
|
|
if retry_at:
|
|
try:
|
|
retry_dt = datetime.fromisoformat(str(retry_at).replace('Z', '+00:00'))
|
|
if retry_dt.tzinfo is None:
|
|
retry_dt = retry_dt.replace(tzinfo=timezone.utc)
|
|
retry_dt = retry_dt.astimezone(timezone.utc)
|
|
except (TypeError, ValueError):
|
|
self.conn.rollback()
|
|
return False
|
|
if retry_dt > now_dt + timedelta(seconds=3605):
|
|
self.conn.rollback()
|
|
return False
|
|
if retry_dt < now_dt:
|
|
retry_dt = now_dt
|
|
retry_due = retry_dt.isoformat(timespec='seconds')
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
row = self.conn.execute(
|
|
f'''SELECT query, resolver_attempts FROM target_queue
|
|
WHERE id = ? AND source = ? AND platform = 'docker'
|
|
AND resolver_state = 'resolving' AND resolver_token = ?{lock_suffix}''',
|
|
(queue_id, source, resolver_token),
|
|
).fetchone()
|
|
if not row:
|
|
self.conn.rollback()
|
|
return False
|
|
if tagged_targets:
|
|
for target in tagged_targets:
|
|
normalized = normalize_target(target, 'docker')
|
|
if not normalized:
|
|
continue
|
|
self.conn.execute(
|
|
'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status,
|
|
created_at, updated_at
|
|
) VALUES (?, 'docker', ?, ?, ?, 'pending', ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO NOTHING''',
|
|
(source, row['query'], target, normalized, now, now),
|
|
)
|
|
if complete:
|
|
refresh_interval = max(0, int(refresh_interval_sec or 0))
|
|
next_due = (
|
|
datetime.fromtimestamp(
|
|
time.time() + refresh_interval, timezone.utc,
|
|
).isoformat(timespec='seconds')
|
|
if refresh_interval else None
|
|
)
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'done', resolver_state = 'resolved',
|
|
resolver_due_at = ?, resolver_token = NULL, resolver_attempts = 0,
|
|
available_after = NULL, last_error = NULL,
|
|
completed_at = COALESCE(completed_at, ?), updated_at = ?
|
|
WHERE id = ? AND resolver_token = ?''',
|
|
(next_due, now, now, queue_id, resolver_token),
|
|
)
|
|
elif retry_due is not None:
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'retry',
|
|
resolver_due_at = ?, resolver_token = NULL, available_after = ?,
|
|
resolver_attempts = CASE
|
|
WHEN ? = 0 AND COALESCE(resolver_attempts, 0) > 0
|
|
THEN resolver_attempts - 1 ELSE COALESCE(resolver_attempts, 0) END,
|
|
last_error = ?, updated_at = ?
|
|
WHERE id = ? AND resolver_token = ?''',
|
|
(
|
|
retry_due, retry_due, 1 if claim_attempt_consumed else 0,
|
|
first_line(error or 'Docker tag resolution deferred', 500),
|
|
now, queue_id, resolver_token,
|
|
),
|
|
)
|
|
else:
|
|
base_delay = max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600))
|
|
max_delay = max(base_delay, env_int('DOCKER_RESOLVER_RETRY_MAX_SEC', 86400))
|
|
exponent = min(16, max(0, int(row['resolver_attempts'] or 1) - 1))
|
|
delay = min(max_delay, base_delay * (2 ** exponent))
|
|
due = datetime.fromtimestamp(time.time() + delay, timezone.utc).isoformat(timespec='seconds')
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'retry',
|
|
resolver_due_at = ?, resolver_token = NULL, available_after = ?,
|
|
last_error = ?, updated_at = ? WHERE id = ? AND resolver_token = ?''',
|
|
(
|
|
due, due,
|
|
first_line(error or 'Docker tag resolution unresolved', 500),
|
|
now, queue_id, resolver_token,
|
|
),
|
|
)
|
|
if int(getattr(cur, 'rowcount', 0) or 0) != 1:
|
|
self.conn.rollback()
|
|
return False
|
|
self.conn.commit()
|
|
return True
|
|
return self._safe('finish_docker_resolution', op, False)
|
|
|
|
def requeue_target_ids(self, queue_ids):
|
|
ids = [int(value) for value in (queue_ids or []) if value is not None]
|
|
if not self.conn or not ids:
|
|
return 0
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
placeholders = ','.join('?' for _ in ids)
|
|
cur = self.conn.execute(
|
|
f'''UPDATE target_queue SET status = 'pending', attempts = 0, available_after = NULL,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
last_error = NULL, completed_at = NULL, updated_at = ?
|
|
WHERE id IN ({placeholders}) AND status = 'deferred' ''',
|
|
[now, *ids],
|
|
)
|
|
self.conn.commit()
|
|
return int(getattr(cur, 'rowcount', 0) or 0)
|
|
return self._safe('requeue_target_ids', op, 0)
|
|
|
|
def target_queue_item(self, source, platform, target):
|
|
if not self.conn:
|
|
return None
|
|
|
|
def op():
|
|
normalized = normalize_target(target, platform)
|
|
return self.conn.execute(
|
|
'''SELECT id, status, attempts, available_after, lease_owner, lease_token, lease_expires_at
|
|
FROM target_queue WHERE source = ? AND normalized_target = ?''',
|
|
(source, normalized),
|
|
).fetchone()
|
|
return self._safe('target_queue_item', op)
|
|
|
|
def target_queue_counts(self, source=None):
|
|
if not self.conn:
|
|
return {
|
|
'counts': {}, 'degraded': True, 'stale': True,
|
|
'reason': 'database_unavailable', 'truncated_statuses': [],
|
|
}
|
|
cache_key = str(source or '*')
|
|
cache = getattr(self, '_target_queue_counts_cache', None)
|
|
if cache is None:
|
|
cache = {}
|
|
self._target_queue_counts_cache = cache
|
|
retry_after_by_key = getattr(self, '_target_queue_counts_retry_after', None)
|
|
if retry_after_by_key is None:
|
|
retry_after_by_key = {}
|
|
self._target_queue_counts_retry_after = retry_after_by_key
|
|
now = time.monotonic()
|
|
retry_after = float(retry_after_by_key.get(cache_key) or 0)
|
|
if retry_after > now:
|
|
cached = cache.get(cache_key)
|
|
snapshot = cached or {
|
|
'counts': {},
|
|
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
|
|
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
|
|
'sampled_rows': 0,
|
|
'truncated_statuses': [],
|
|
}
|
|
return dict(
|
|
snapshot,
|
|
counts=dict(snapshot.get('counts') or {}),
|
|
degraded=True,
|
|
stale=True,
|
|
reason='bounded_count_retry_backoff',
|
|
retry_after_sec=max(1, min(
|
|
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC,
|
|
int(math.ceil(retry_after - now)),
|
|
)),
|
|
truncated_statuses=list(snapshot.get('truncated_statuses') or []),
|
|
)
|
|
try:
|
|
counts = {}
|
|
truncated = []
|
|
if self.conn.is_postgres:
|
|
self.conn.execute(
|
|
f'SET LOCAL statement_timeout = {TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS}'
|
|
)
|
|
for status in TARGET_QUEUE_OBSERVABILITY_STATUSES:
|
|
if source:
|
|
row = self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM (
|
|
SELECT 1 FROM target_queue
|
|
WHERE source = ? AND status = ? LIMIT ?
|
|
) AS sampled''',
|
|
(source, status, TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT),
|
|
).fetchone()
|
|
else:
|
|
row = self.conn.execute(
|
|
'''SELECT COUNT(*) AS count FROM (
|
|
SELECT 1 FROM target_queue
|
|
WHERE status = ? LIMIT ?
|
|
) AS sampled''',
|
|
(status, TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT),
|
|
).fetchone()
|
|
count = int(row['count'] or 0)
|
|
if count:
|
|
counts[status] = count
|
|
if count >= TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT:
|
|
truncated.append(status)
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
snapshot = {
|
|
'counts': counts,
|
|
'degraded': bool(truncated),
|
|
'stale': False,
|
|
'reason': 'bounded_status_sample' if truncated else '',
|
|
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
|
|
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
|
|
'sampled_rows': sum(counts.values()),
|
|
'retry_after_sec': 0,
|
|
'truncated_statuses': truncated,
|
|
}
|
|
cache[cache_key] = snapshot
|
|
retry_after_by_key.pop(cache_key, None)
|
|
self.last_error = ''
|
|
return dict(snapshot, counts=dict(counts), truncated_statuses=list(truncated))
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
rollback_succeeded = False
|
|
try:
|
|
self.conn.rollback()
|
|
rollback_succeeded = True
|
|
except Exception:
|
|
pass
|
|
if rollback_succeeded:
|
|
retry_after_by_key[cache_key] = (
|
|
time.monotonic() + TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC
|
|
)
|
|
cached = cache.get(cache_key)
|
|
logger.warning('Target queue observability degraded after bounded count query failure: %s', type(exc).__name__)
|
|
if cached:
|
|
return dict(
|
|
cached,
|
|
counts=dict(cached.get('counts') or {}),
|
|
degraded=True,
|
|
stale=True,
|
|
reason='bounded_count_query_failed',
|
|
retry_after_sec=(
|
|
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC if rollback_succeeded else 0
|
|
),
|
|
truncated_statuses=list(cached.get('truncated_statuses') or []),
|
|
)
|
|
return {
|
|
'counts': {},
|
|
'degraded': True,
|
|
'stale': True,
|
|
'reason': 'bounded_count_query_failed',
|
|
'sample_limit_per_status': TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT,
|
|
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
|
|
'sampled_rows': 0,
|
|
'retry_after_sec': (
|
|
TARGET_QUEUE_OBSERVABILITY_RETRY_BACKOFF_SEC if rollback_succeeded else 0
|
|
),
|
|
'truncated_statuses': [],
|
|
}
|
|
|
|
def admin_target_queue_health(self, sources):
|
|
if (
|
|
not isinstance(sources, (tuple, list)) or not 1 <= len(sources) <= 16
|
|
or any(type(source) is not str or not source for source in sources)
|
|
or len(set(sources)) != len(sources)
|
|
):
|
|
raise ValueError('admin queue sources are invalid')
|
|
snapshots = [self.target_queue_counts(source) for source in sources]
|
|
counts = {}
|
|
truncated = set()
|
|
for snapshot in snapshots:
|
|
for status, count in (snapshot.get('counts') or {}).items():
|
|
counts[status] = counts.get(status, 0) + int(count)
|
|
truncated.update(snapshot.get('truncated_statuses') or ())
|
|
degraded = any(snapshot.get('degraded') is True for snapshot in snapshots)
|
|
stale = any(snapshot.get('stale') is True for snapshot in snapshots)
|
|
return {
|
|
'counts': counts,
|
|
'degraded': degraded,
|
|
'stale': stale,
|
|
'reason': 'bounded_core_source_sample' if degraded else '',
|
|
'sample_limit_per_status': (
|
|
TARGET_QUEUE_OBSERVABILITY_STATUS_LIMIT * len(sources)
|
|
),
|
|
'timeout_ms': TARGET_QUEUE_OBSERVABILITY_TIMEOUT_MS,
|
|
'sampled_rows': sum(counts.values()),
|
|
'retry_after_sec': max(
|
|
(int(snapshot.get('retry_after_sec') or 0) for snapshot in snapshots),
|
|
default=0,
|
|
),
|
|
'truncated_statuses': sorted(truncated),
|
|
}
|
|
|
|
def known_target_normalizations(self, source, platform=None):
|
|
if not self.conn:
|
|
return set()
|
|
|
|
def op():
|
|
if platform:
|
|
rows = self.conn.execute(
|
|
'''SELECT normalized_target FROM target_queue
|
|
WHERE source = ? AND platform = ?''',
|
|
(source, platform),
|
|
).fetchall()
|
|
else:
|
|
rows = self.conn.execute(
|
|
'SELECT normalized_target FROM target_queue WHERE source = ?',
|
|
(source,),
|
|
).fetchall()
|
|
return {str(row['normalized_target']) for row in rows if row['normalized_target']}
|
|
return self._safe('known_target_normalizations', op, set())
|
|
|
|
def known_target_normalizations_for(self, source, platform, targets, batch_size=KNOWN_TARGET_LOOKUP_BATCH_SIZE):
|
|
"""Return only known identities from the offered bounded discovery batch."""
|
|
if not self.conn:
|
|
return set()
|
|
normalized = []
|
|
seen = set()
|
|
for target in targets or []:
|
|
value = normalize_target(target, platform)
|
|
if value and value not in seen:
|
|
seen.add(value)
|
|
normalized.append(value)
|
|
if not normalized:
|
|
return set()
|
|
|
|
def op():
|
|
known = set()
|
|
size = max(1, min(KNOWN_TARGET_LOOKUP_BATCH_SIZE, int(batch_size or KNOWN_TARGET_LOOKUP_BATCH_SIZE)))
|
|
for start in range(0, len(normalized), size):
|
|
values = normalized[start:start + size]
|
|
rows = self.conn.execute(
|
|
'''SELECT normalized_target FROM target_queue
|
|
WHERE source = ? AND platform = ? AND normalized_target IN ({})'''.format(
|
|
','.join('?' for _ in values)
|
|
),
|
|
(source, platform, *values),
|
|
).fetchall()
|
|
known.update(str(row['normalized_target']) for row in rows if row['normalized_target'])
|
|
return known
|
|
return self._safe('known_target_normalizations_for', op, set())
|
|
|
|
def record_package_repo_candidates(self, run_id, cycle_id, query, candidates):
|
|
offered = list(candidates or [])
|
|
report = {
|
|
'offered_rows': len(offered),
|
|
'committed_rows': 0,
|
|
'skipped_rows': 0,
|
|
'failed': False,
|
|
'failed_chunk_rows': 0,
|
|
'connection_usable': bool(self.conn),
|
|
}
|
|
if not offered or not self.conn:
|
|
report['skipped_rows'] = len(offered)
|
|
report['failed'] = bool(offered)
|
|
return report
|
|
|
|
for start in range(0, len(offered), PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE):
|
|
chunk = offered[start:start + PACKAGE_CANDIDATE_WRITE_CHUNK_SIZE]
|
|
attempts = SQLITE_LOCK_RETRY_ATTEMPTS if self.conn.is_sqlite else 1
|
|
failure = None
|
|
for attempt in range(attempts):
|
|
try:
|
|
now = utc_now_iso()
|
|
for candidate in chunk:
|
|
self.conn.execute(
|
|
'''INSERT INTO package_repo_candidates (
|
|
package_source, package_name, package_version, query, repo_url, provider,
|
|
evidence_json, confidence, first_seen_at, last_seen_at, last_run_id, last_cycle_id
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(package_source, package_name, package_version, repo_url)
|
|
DO UPDATE SET
|
|
query = excluded.query,
|
|
provider = excluded.provider,
|
|
evidence_json = excluded.evidence_json,
|
|
confidence = excluded.confidence,
|
|
last_seen_at = excluded.last_seen_at,
|
|
last_run_id = excluded.last_run_id,
|
|
last_cycle_id = excluded.last_cycle_id''',
|
|
(
|
|
candidate.get('package_source'),
|
|
candidate.get('name') or candidate.get('package_name'),
|
|
candidate.get('version') or candidate.get('package_version') or '',
|
|
query,
|
|
candidate.get('repo_url'),
|
|
candidate.get('provider'),
|
|
json_dumps(candidate.get('evidence') or []),
|
|
candidate.get('confidence') or 'medium',
|
|
now,
|
|
now,
|
|
run_id,
|
|
cycle_id,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
report['committed_rows'] += len(chunk)
|
|
failure = None
|
|
break
|
|
except Exception as exc:
|
|
failure = exc
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
report['connection_usable'] = self._reset_connection()
|
|
if (
|
|
self.conn
|
|
and self.conn.is_sqlite
|
|
and sqlite_lock_error(exc)
|
|
and attempt < attempts - 1
|
|
):
|
|
time.sleep(sqlite_lock_retry_delay(attempt))
|
|
continue
|
|
break
|
|
if failure is not None:
|
|
self.last_error = str(failure)
|
|
report.update({
|
|
'skipped_rows': len(offered) - report['committed_rows'],
|
|
'failed': True,
|
|
'failed_chunk_rows': len(chunk),
|
|
'connection_usable': bool(self.conn) and report['connection_usable'],
|
|
})
|
|
logger.warning(
|
|
'Optional package candidate cache write stopped after a failed bounded chunk: '
|
|
'committed=%s skipped=%s chunk_rows=%s error=%s',
|
|
report['committed_rows'], report['skipped_rows'], len(chunk), type(failure).__name__,
|
|
)
|
|
return report
|
|
|
|
self.last_error = ''
|
|
return report
|
|
|
|
def _bounded_package_candidate_rows(self, query, sources, limit):
|
|
scan_limit = min(
|
|
PACKAGE_CANDIDATE_SCAN_MAX_ROWS,
|
|
max(5000, int(limit) * 4),
|
|
)
|
|
source_set = set(sources)
|
|
exact = []
|
|
if query:
|
|
scope = PACKAGE_REPO_NONEMPTY_PREDICATE
|
|
source_params = []
|
|
if sources:
|
|
scope += ' AND package_source IN ({})'.format(','.join('?' for _ in sources))
|
|
source_params.extend(sources)
|
|
try:
|
|
if self.conn.is_postgres:
|
|
self.conn.execute('SET LOCAL enable_seqscan = off')
|
|
exact = self.conn.execute(
|
|
f'''SELECT id, package_source, package_name AS name,
|
|
package_version AS version, repo_url, provider,
|
|
evidence_json, confidence, last_seen_at
|
|
FROM package_repo_candidates
|
|
WHERE {scope} AND query = ?
|
|
ORDER BY last_seen_at DESC, id DESC LIMIT ?''',
|
|
(*source_params, query, limit),
|
|
).fetchall()
|
|
except Exception as exc:
|
|
if self.conn.is_postgres:
|
|
self.conn.rollback()
|
|
logger.warning(
|
|
'Exact package candidate branch failed; bounded substring results remain available: %s',
|
|
type(exc).__name__,
|
|
)
|
|
matched_ids = []
|
|
examined = 0
|
|
cursor_seen = None
|
|
cursor_id = None
|
|
more_rows = False
|
|
while examined < scan_limit and len(matched_ids) < limit:
|
|
page_limit = min(
|
|
PACKAGE_CANDIDATE_SCAN_PAGE_SIZE,
|
|
scan_limit - examined,
|
|
)
|
|
cursor_clause = ''
|
|
params = []
|
|
if cursor_seen is not None and cursor_id is not None:
|
|
cursor_clause = 'AND (last_seen_at, id) < (?, ?)'
|
|
params.extend((cursor_seen, cursor_id))
|
|
params.append(page_limit)
|
|
page = self.conn.execute(
|
|
f'''SELECT id, package_source, package_name, query, last_seen_at
|
|
FROM package_repo_candidates
|
|
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE} {cursor_clause}
|
|
ORDER BY last_seen_at DESC, id DESC
|
|
LIMIT ?''',
|
|
params,
|
|
).fetchall()
|
|
if not page:
|
|
more_rows = False
|
|
break
|
|
examined += len(page)
|
|
for row in page:
|
|
if source_set and str(row['package_source'] or '').lower() not in source_set:
|
|
continue
|
|
if not query or (
|
|
str(row['query'] or '') != query
|
|
and query in str(row['package_name'] or '')
|
|
):
|
|
matched_ids.append(int(row['id']))
|
|
if len(matched_ids) >= limit:
|
|
break
|
|
cursor_seen = page[-1]['last_seen_at']
|
|
cursor_id = page[-1]['id']
|
|
more_rows = len(page) == page_limit
|
|
if len(page) < page_limit:
|
|
break
|
|
|
|
if len(matched_ids) < limit and examined >= scan_limit and more_rows:
|
|
probe = self.conn.execute(
|
|
f'''SELECT 1 FROM package_repo_candidates
|
|
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE}
|
|
AND (last_seen_at, id) < (?, ?)
|
|
ORDER BY last_seen_at DESC, id DESC
|
|
LIMIT 1''',
|
|
(cursor_seen, cursor_id),
|
|
).fetchone()
|
|
if probe:
|
|
logger.warning(
|
|
'Package candidate substring lookup reached its bounded %s-row metadata scan; '
|
|
'older candidates were not examined',
|
|
scan_limit,
|
|
)
|
|
|
|
rows_by_id = {}
|
|
for start in range(0, len(matched_ids), 64):
|
|
batch = matched_ids[start:start + 64]
|
|
placeholders = ','.join('?' for _ in batch)
|
|
rows = self.conn.execute(
|
|
f'''SELECT id, package_source, package_name AS name, package_version AS version,
|
|
repo_url, provider, evidence_json, confidence, last_seen_at
|
|
FROM package_repo_candidates WHERE id IN ({placeholders})''',
|
|
batch,
|
|
).fetchall()
|
|
rows_by_id.update({int(row['id']): row for row in rows})
|
|
combined = {
|
|
int(row['id']): row
|
|
for row in (*exact, *(rows_by_id[row_id] for row_id in matched_ids if row_id in rows_by_id))
|
|
}
|
|
return sorted(
|
|
combined.values(),
|
|
key=lambda row: (str(row['last_seen_at'] or ''), int(row['id'])),
|
|
reverse=True,
|
|
)[:limit]
|
|
|
|
def get_package_repo_candidates(self, query=None, package_sources=None, limit=1000):
|
|
if not self.conn:
|
|
return []
|
|
def op():
|
|
requested_limit = max(1, int(limit or 1000))
|
|
limit_value = min(PACKAGE_CANDIDATE_LOOKUP_MAX_RESULTS, requested_limit)
|
|
if requested_limit > limit_value:
|
|
logger.warning(
|
|
'Package candidate lookup capped requested result limit from %s to %s',
|
|
requested_limit, limit_value,
|
|
)
|
|
sources = [item.strip().lower() for item in (package_sources or []) if item.strip()]
|
|
search = str(query or '')
|
|
rows = self._bounded_package_candidate_rows(
|
|
search, sources, limit_value,
|
|
)
|
|
candidates = []
|
|
for row in rows:
|
|
try:
|
|
evidence = json.loads(row['evidence_json'] or '[]')
|
|
except json.JSONDecodeError:
|
|
evidence = []
|
|
candidates.append({
|
|
'source': 'package_git',
|
|
'package_source': row['package_source'],
|
|
'name': row['name'],
|
|
'version': row['version'],
|
|
'repo_url': row['repo_url'],
|
|
'provider': row['provider'],
|
|
'evidence': evidence,
|
|
'confidence': row['confidence'],
|
|
})
|
|
return candidates
|
|
return self._safe('get_package_repo_candidates', op, [])
|
|
|
|
@staticmethod
|
|
def _compat_payload(finding, max_bytes=16 * 1024 * 1024):
|
|
mapped = {
|
|
'DetectorName', 'DetectorType', 'Verified', 'Raw', 'RawV2', 'Redacted',
|
|
'StructuredData', 'ExtraData', 'AnalysisInfo', 'finding_uid',
|
|
}
|
|
extension = {key: value for key, value in finding.items() if key not in mapped}
|
|
payload = {
|
|
'raw_value': finding.get('Raw'),
|
|
'raw_v2_value': finding.get('RawV2'),
|
|
'structured_data_json': json_dumps(finding.get('StructuredData')) if finding.get('StructuredData') is not None else None,
|
|
'extra_data_json': json_dumps(finding.get('ExtraData')) if finding.get('ExtraData') is not None else None,
|
|
'analysis_info_json': json_dumps(finding.get('AnalysisInfo')) if finding.get('AnalysisInfo') is not None else None,
|
|
'extension_json': json_dumps(extension) if extension else None,
|
|
}
|
|
encoded = json.dumps(
|
|
finding, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
omitted = len(encoded) > max(1, int(max_bytes))
|
|
if omitted:
|
|
payload = {key: None for key in payload}
|
|
payload.update({
|
|
'payload_sha256': hashlib.sha256(encoded).hexdigest(),
|
|
'payload_bytes': len(encoded),
|
|
'payload_omitted': 1 if omitted else 0,
|
|
})
|
|
return payload
|
|
|
|
def _insert_normalized_finding(
|
|
self, run_id, cycle_id, target_scan_id, source, query, target,
|
|
normalized, finding,
|
|
):
|
|
finding = sanitize_postman_finding(finding)
|
|
raw_secret, secret_hash, detector_secret_hash, fingerprint, location = finding_identity(
|
|
source, normalized, finding,
|
|
)
|
|
finding_uid = str(finding.get('finding_uid') or fingerprint)
|
|
enrichment = enrich_finding(finding)
|
|
compat = self._compat_payload(finding)
|
|
finding_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO findings(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
|
|
detector_name, detector_type, verified, raw_secret, redacted_secret, secret_hash,
|
|
detector_secret_hash, finding_fingerprint, finding_uid, file_path, line_number,
|
|
commit_hash, source_timestamp, source_metadata_type, source_metadata_json,
|
|
raw_finding_json, provider, credential_kind, credential_confidence,
|
|
required_context_missing, principal, username, email, project_id, tenant_id,
|
|
organization, registry, endpoint, scope, resource, enrichment_json,
|
|
raw_payload_sha256, raw_payload_bytes, raw_payload_omitted, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized,
|
|
finding.get('DetectorName'), finding.get('DetectorType'),
|
|
1 if finding.get('Verified', False) else 0, raw_secret,
|
|
extract_redacted_secret(finding), secret_hash, detector_secret_hash,
|
|
fingerprint, finding_uid, location.get('file_path'), location.get('line_number'),
|
|
location.get('commit_hash'), location.get('source_timestamp'),
|
|
location.get('source_metadata_type'), location.get('source_metadata_json'),
|
|
enrichment.get('provider'), enrichment.get('credential_kind'),
|
|
enrichment.get('credential_confidence'), enrichment.get('required_context_missing'),
|
|
enrichment.get('principal'), enrichment.get('username'), enrichment.get('email'),
|
|
enrichment.get('project_id'), enrichment.get('tenant_id'),
|
|
enrichment.get('organization'), enrichment.get('registry'),
|
|
enrichment.get('endpoint'), enrichment.get('scope'), enrichment.get('resource'),
|
|
enrichment.get('enrichment_json'), compat['payload_sha256'],
|
|
compat['payload_bytes'], compat['payload_omitted'], utc_now_iso(),
|
|
),
|
|
)
|
|
if not finding_id:
|
|
raise RuntimeError('normalized finding insert returned no identity')
|
|
self.conn.execute(
|
|
'''INSERT INTO finding_compat_payloads(
|
|
finding_id, raw_value, raw_v2_value, structured_data_json,
|
|
extra_data_json, analysis_info_json, extension_json,
|
|
payload_sha256, payload_bytes, payload_omitted, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
finding_id, compat['raw_value'], compat['raw_v2_value'],
|
|
compat['structured_data_json'], compat['extra_data_json'],
|
|
compat['analysis_info_json'], compat['extension_json'],
|
|
compat['payload_sha256'], compat['payload_bytes'], compat['payload_omitted'],
|
|
utc_now_iso(),
|
|
),
|
|
)
|
|
self.conn.execute(
|
|
'''INSERT INTO finding_uid_map(finding_uid, finding_id, created_at)
|
|
VALUES (?, ?, ?) ON CONFLICT(finding_uid) DO NOTHING''',
|
|
(finding_uid, finding_id, utc_now_iso()),
|
|
)
|
|
return finding_id, finding_uid
|
|
|
|
@staticmethod
|
|
def _docker_finding_layer_evidence(finding, manifest_digest):
|
|
metadata = finding.get('SourceMetadata')
|
|
data = metadata.get('Data') if isinstance(metadata, dict) else None
|
|
if not isinstance(data, dict):
|
|
return None, 'digest_absent'
|
|
|
|
docker = data.get('Docker')
|
|
if 'Docker' in data and not isinstance(docker, dict):
|
|
return None, 'digest_invalid'
|
|
if isinstance(docker, dict) and 'layer' in docker:
|
|
digest = docker.get('layer')
|
|
if not isinstance(digest, str) or not re.fullmatch(
|
|
r'sha256:[0-9a-f]{64}', digest):
|
|
return None, 'digest_invalid'
|
|
return digest, None
|
|
|
|
if 'DockerContent' not in data:
|
|
return None, 'digest_absent'
|
|
docker_content = data.get('DockerContent')
|
|
if not isinstance(docker_content, dict):
|
|
return None, 'digest_invalid'
|
|
if docker_content.get('descriptor_kind') != 'layer':
|
|
return None, 'digest_absent'
|
|
digest = docker_content.get('blob_digest')
|
|
reported_manifest = docker_content.get('manifest_digest')
|
|
if (
|
|
not isinstance(digest, str)
|
|
or not re.fullmatch(r'sha256:[0-9a-f]{64}', digest)
|
|
or not isinstance(reported_manifest, str)
|
|
or not re.fullmatch(r'sha256:[0-9a-f]{64}', reported_manifest)
|
|
):
|
|
return None, 'digest_invalid'
|
|
if reported_manifest != manifest_digest:
|
|
return None, 'manifest_mismatch'
|
|
return digest, None
|
|
|
|
def _insert_docker_finding_layer_attributions_locked(
|
|
self, binding, target_scan_id, finding_id, finding,
|
|
):
|
|
if not binding:
|
|
return 0
|
|
if binding['target_scan_id'] is not None and int(
|
|
binding['target_scan_id']) != int(target_scan_id):
|
|
raise ScanEventConflictError(
|
|
'Docker finding attribution binding has a conflicting scan identity'
|
|
)
|
|
finding_row = self.conn.execute(
|
|
'''SELECT finding.id
|
|
FROM findings finding
|
|
JOIN target_scans scan ON scan.id = finding.target_scan_id
|
|
WHERE finding.id = ? AND finding.target_scan_id = ?
|
|
AND scan.queue_id = ? AND scan.result_reservation_id = ?
|
|
FOR UPDATE OF finding, scan''',
|
|
(
|
|
int(finding_id), int(target_scan_id),
|
|
int(binding['target_queue_id']), int(binding['reservation_id']),
|
|
),
|
|
).fetchone()
|
|
if not finding_row:
|
|
raise ScanEventConflictError(
|
|
'Docker finding attribution lost its exact reservation scan identity'
|
|
)
|
|
|
|
digest, reason = self._docker_finding_layer_evidence(
|
|
finding, str(binding['manifest_digest']),
|
|
)
|
|
layers = []
|
|
if digest is not None:
|
|
layers = self.conn.execute(
|
|
'''SELECT id, position_from_base, position_from_top, layer_digest
|
|
FROM docker_manifest_layers
|
|
WHERE manifest_id = ? AND layer_digest = ?
|
|
ORDER BY position_from_base, id FOR UPDATE''',
|
|
(int(binding['manifest_id']), digest),
|
|
).fetchall()
|
|
if not layers:
|
|
digest = None
|
|
reason = 'digest_not_in_manifest'
|
|
|
|
if digest is None:
|
|
expected_rows = ((
|
|
int(binding['id']), None, 'unattributed', None, reason, None, None,
|
|
),)
|
|
else:
|
|
expected_rows = tuple(
|
|
(
|
|
int(binding['id']), int(layer['id']), 'exact',
|
|
str(layer['layer_digest']), None,
|
|
int(layer['position_from_base']), int(layer['position_from_top']),
|
|
)
|
|
for layer in layers
|
|
)
|
|
|
|
now = utc_now_iso()
|
|
for row in expected_rows:
|
|
self.conn.execute(
|
|
'''INSERT INTO docker_finding_layer_attributions(
|
|
scan_binding_id, finding_id, manifest_layer_id,
|
|
attribution_state, reported_layer_digest,
|
|
unattributed_reason, position_from_base,
|
|
position_from_top, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT DO NOTHING''',
|
|
(row[0], int(finding_id), *row[1:], now),
|
|
)
|
|
actual_rows = self.conn.execute(
|
|
'''SELECT scan_binding_id, manifest_layer_id, attribution_state,
|
|
reported_layer_digest, unattributed_reason,
|
|
position_from_base, position_from_top
|
|
FROM docker_finding_layer_attributions
|
|
WHERE finding_id = ? ORDER BY id FOR UPDATE''',
|
|
(int(finding_id),),
|
|
).fetchall()
|
|
actual = {
|
|
(
|
|
int(row['scan_binding_id']),
|
|
int(row['manifest_layer_id']) if row['manifest_layer_id'] is not None else None,
|
|
str(row['attribution_state']), row['reported_layer_digest'],
|
|
row['unattributed_reason'],
|
|
int(row['position_from_base']) if row['position_from_base'] is not None else None,
|
|
int(row['position_from_top']) if row['position_from_top'] is not None else None,
|
|
)
|
|
for row in actual_rows
|
|
}
|
|
expected = set(expected_rows)
|
|
if len(actual_rows) != len(expected_rows) or actual != expected:
|
|
raise ScanEventConflictError(
|
|
'Docker finding layer attribution conflicts with persisted evidence'
|
|
)
|
|
return len(expected_rows)
|
|
|
|
def _precompute_docker_finding_attribution_rows_locked(self, reader, binding):
|
|
if not binding:
|
|
return 0
|
|
layers = self.conn.execute(
|
|
'''SELECT id, layer_digest, position_from_base, position_from_top
|
|
FROM docker_manifest_layers
|
|
WHERE manifest_id = ?
|
|
ORDER BY position_from_base, id FOR UPDATE''',
|
|
(int(binding['manifest_id']),),
|
|
).fetchall()
|
|
if len(layers) != int(binding['layer_count']):
|
|
raise ScanEventConflictError(
|
|
'Docker attribution manifest layer authority is incomplete'
|
|
)
|
|
positions_by_digest = {}
|
|
for layer in layers:
|
|
positions_by_digest[str(layer['layer_digest'])] = (
|
|
positions_by_digest.get(str(layer['layer_digest']), 0) + 1
|
|
)
|
|
|
|
projected_rows = 0
|
|
finding_count = 0
|
|
for finding in reader.iter_findings():
|
|
finding_count += 1
|
|
digest, _reason = self._docker_finding_layer_evidence(
|
|
finding, str(binding['manifest_digest']),
|
|
)
|
|
projected_rows += max(1, positions_by_digest.get(digest, 0))
|
|
if projected_rows > DOCKER_DEPTH_MAX_ATTRIBUTION_ROWS_PER_BUNDLE:
|
|
raise DockerFindingAttributionLimitError(
|
|
'Docker finding attribution exceeds its strict per-bundle row limit'
|
|
)
|
|
if finding_count != int(reader.validate().finding_count):
|
|
raise ScanEventConflictError(
|
|
'Docker attribution finding count conflicts with its bundle'
|
|
)
|
|
return projected_rows
|
|
|
|
def _existing_normalized_finding_for_replay_locked(
|
|
self, target_scan_id, source, normalized, finding,
|
|
):
|
|
finding = sanitize_postman_finding(finding)
|
|
_, _, _, fingerprint, _ = finding_identity(source, normalized, finding)
|
|
finding_uid = str(finding.get('finding_uid') or fingerprint)
|
|
payload = self._compat_payload(finding)
|
|
rows = self.conn.execute(
|
|
'''SELECT id, finding_fingerprint, raw_payload_sha256
|
|
FROM findings
|
|
WHERE target_scan_id = ? AND finding_uid = ?
|
|
ORDER BY id LIMIT 2 FOR UPDATE''',
|
|
(int(target_scan_id), finding_uid),
|
|
).fetchall()
|
|
if len(rows) != 1 or (
|
|
str(rows[0]['finding_fingerprint']) != fingerprint
|
|
or str(rows[0]['raw_payload_sha256']) != payload['payload_sha256']
|
|
):
|
|
raise ScanEventConflictError(
|
|
'replayed Docker finding identity conflicts with persisted evidence'
|
|
)
|
|
return int(rows[0]['id']), finding_uid
|
|
|
|
def _insert_bundle_candidate(
|
|
self, frame, reservation, target_scan_id, finding_ids,
|
|
capacity_allocation=None, routed_service_supported=True,
|
|
):
|
|
if not isinstance(frame, dict):
|
|
raise ValueError('keycheck candidate frame must be an object')
|
|
service = str(frame.get('service') or '').lower()
|
|
credential_hash = str(frame.get('credential_hash') or '').lower()
|
|
provider_key_hash = str(frame.get('provider_key_hash') or '').lower()
|
|
secret_hash = str(frame.get('secret_hash') or '').lower()
|
|
if (
|
|
not service
|
|
or not re.fullmatch(r'[a-f0-9]{64}', credential_hash)
|
|
or not re.fullmatch(r'[a-f0-9]{64}', provider_key_hash)
|
|
or not re.fullmatch(r'[a-f0-9]{64}', secret_hash)
|
|
):
|
|
raise ValueError('keycheck candidate identity is invalid')
|
|
secret_text = frame.get('secret_text')
|
|
secret_json = frame.get('secret_json')
|
|
if (secret_text is None) == (secret_json is None):
|
|
raise ValueError('keycheck candidate must contain exactly one secret representation')
|
|
metadata = frame.get('metadata') if isinstance(frame.get('metadata'), dict) else {}
|
|
attribution = frame.get('attribution') if isinstance(frame.get('attribution'), dict) else {}
|
|
now = utc_now_iso()
|
|
credential_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO keycheck_credentials(
|
|
service, credential_hash, provider_key_hash,
|
|
candidate_kind, secret_text, secret_json,
|
|
key_masked, endpoint, principal, metadata_json, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(service, provider_key_hash) DO NOTHING''',
|
|
(
|
|
service, credential_hash, provider_key_hash,
|
|
str(frame.get('candidate_kind') or 'raw'),
|
|
secret_text, secret_json, str(frame.get('key_masked') or ''),
|
|
str(frame.get('endpoint') or ''), str(frame.get('principal') or ''),
|
|
json_dumps(metadata), now, now,
|
|
),
|
|
)
|
|
if credential_id is None:
|
|
credential = self.conn.execute(
|
|
'''SELECT id, provider_key_hash, secret_text, secret_json
|
|
FROM keycheck_credentials
|
|
WHERE service = ? AND provider_key_hash = ?''',
|
|
(service, provider_key_hash),
|
|
).fetchone()
|
|
if not credential or (
|
|
credential['provider_key_hash'] != provider_key_hash
|
|
or credential['secret_text'] != secret_text
|
|
or credential['secret_json'] != secret_json
|
|
):
|
|
raise ScanEventConflictError('provider credential identity resolves to conflicting secret material')
|
|
credential_id = credential['id']
|
|
finding_uid_value = str(attribution.get('finding_uid') or metadata.get('finding_uid') or '')
|
|
finding_id = finding_ids.get(finding_uid_value)
|
|
origin = finding_uid_value or str(attribution.get('origin') or frame.get('candidate_kind') or 'structured')
|
|
uid = candidate_uid(reservation['scan_event_id'], origin, service, credential_hash)
|
|
serialized_bytes = len(json.dumps(
|
|
frame, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8'))
|
|
capacity_bytes = max(serialized_bytes, int(capacity_allocation or serialized_bytes))
|
|
if routed_service_supported:
|
|
candidate_sql = '''INSERT INTO keycheck_candidates(
|
|
candidate_uid, credential_id, service, routed_service, secret_hash,
|
|
finding_id, target_scan_id,
|
|
scan_event_id, finding_uid, source, query, target, detector_name,
|
|
found_at, metadata_json, state, capacity_bytes, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
|
|
ON CONFLICT(candidate_uid) DO NOTHING'''
|
|
candidate_values = (
|
|
uid, credential_id, service, service, secret_hash, finding_id, target_scan_id,
|
|
reservation['scan_event_id'], finding_uid_value, reservation['source'],
|
|
reservation['query'], reservation['target'], metadata.get('detector_name'),
|
|
now, json_dumps(metadata), capacity_bytes, now, now,
|
|
)
|
|
else:
|
|
candidate_sql = '''INSERT INTO keycheck_candidates(
|
|
candidate_uid, credential_id, service, secret_hash,
|
|
finding_id, target_scan_id,
|
|
scan_event_id, finding_uid, source, query, target, detector_name,
|
|
found_at, metadata_json, state, capacity_bytes, created_at, updated_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
|
|
ON CONFLICT(candidate_uid) DO NOTHING'''
|
|
candidate_values = (
|
|
uid, credential_id, service, secret_hash, finding_id, target_scan_id,
|
|
reservation['scan_event_id'], finding_uid_value, reservation['source'],
|
|
reservation['query'], reservation['target'], metadata.get('detector_name'),
|
|
now, json_dumps(metadata), capacity_bytes, now, now,
|
|
)
|
|
candidate_id = self.conn.insert_returning_id(candidate_sql, candidate_values)
|
|
return (1, capacity_bytes) if candidate_id is not None else (0, 0)
|
|
|
|
def ingest_result_bundle(self, reader, reservation, bundle_row):
|
|
if not self.conn or not self.conn.is_postgres:
|
|
raise RuntimeError('normalized result bundle ingestion requires PostgreSQL')
|
|
validated = reader.validate()
|
|
reservation_id = int(reservation['id'] if 'id' in reservation else reservation['reservation_id'])
|
|
event_id = str(validated.scan_event_id)
|
|
event_hash = str(validated.scan_event_hash)
|
|
metadata = reader.metadata()
|
|
effective_diagnostics = reader.effective_diagnostics()
|
|
contracts = importlib.import_module('worker_contracts')
|
|
diagnostic_uid_set_sha256 = contracts.ordered_diagnostic_uid_set_sha256(
|
|
effective_diagnostics
|
|
)
|
|
if any(key in metadata for key in ('findings', 'errors')):
|
|
raise ValueError('bundle metadata must not duplicate findings or errors')
|
|
metadata_bytes = json.dumps(
|
|
metadata, ensure_ascii=True, sort_keys=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')
|
|
if len(metadata_bytes) > 16 * 1024 * 1024:
|
|
raise ValueError('scan compatibility metadata exceeds its byte bound')
|
|
routed_service_supported = (
|
|
'routed_service' in self.conn.table_columns('keycheck_candidates')
|
|
)
|
|
try:
|
|
self._lock_docker_depth_experiment_for_reservation(reservation_id)
|
|
current_reservation = self.conn.execute(
|
|
'''SELECT r.*, q.id AS bound_queue_id,
|
|
q.source AS bound_queue_source,
|
|
q.platform AS bound_queue_platform,
|
|
q.query AS bound_queue_query,
|
|
q.target AS bound_queue_target,
|
|
q.normalized_target AS bound_queue_normalized_target,
|
|
q.status AS bound_queue_status,
|
|
q.lease_token AS bound_queue_lease_token,
|
|
q.current_result_reservation_id AS bound_queue_reservation_id,
|
|
q.claim_event_id AS bound_queue_event_id
|
|
FROM result_reservations r
|
|
JOIN target_queue q ON q.id = r.queue_id
|
|
WHERE r.id = ? FOR UPDATE OF r, q''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
current_bundle = self.conn.execute(
|
|
'SELECT * FROM result_bundles WHERE reservation_id = ? FOR UPDATE',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not current_reservation or not current_bundle:
|
|
raise ValueError('bundle reservation state is absent')
|
|
self._validate_result_ingestion_arguments(
|
|
current_reservation, current_bundle, reservation, bundle_row,
|
|
)
|
|
queue_identity = {
|
|
'id': current_reservation['bound_queue_id'],
|
|
'source': current_reservation['bound_queue_source'],
|
|
'platform': current_reservation['bound_queue_platform'],
|
|
'query': current_reservation['bound_queue_query'],
|
|
'target': current_reservation['bound_queue_target'],
|
|
'normalized_target': current_reservation[
|
|
'bound_queue_normalized_target'
|
|
],
|
|
}
|
|
self._validate_locked_reservation_queue_identity(
|
|
current_reservation, queue_identity,
|
|
)
|
|
validated_identity = validated.as_dict()
|
|
validated_identity['relative_path'] = current_bundle['relative_path']
|
|
self._validate_result_ingestion_arguments(
|
|
current_reservation, current_bundle,
|
|
current_reservation, validated_identity,
|
|
)
|
|
self._validate_result_bundle_reservation_identity(
|
|
current_reservation, validated_identity, require_header=True,
|
|
)
|
|
if str(current_reservation['assignment_kind']) == 'remote':
|
|
validate_remote_result_execution_plan(
|
|
current_reservation, metadata,
|
|
)
|
|
projection_version = current_reservation[
|
|
'remote_diagnostic_projection_version'
|
|
]
|
|
if int(projection_version or 0) == 0:
|
|
cursor = self.conn.execute(
|
|
'''UPDATE result_reservations
|
|
SET remote_diagnostic_projection_version = 1,
|
|
remote_diagnostic_count = ?,
|
|
remote_diagnostic_uids_sha256 = ?, updated_at = ?
|
|
WHERE id = ? AND remote_resolution_kind = 'bundle_accepted'
|
|
AND COALESCE(remote_diagnostic_projection_version, 0) = 0''',
|
|
(
|
|
len(effective_diagnostics),
|
|
diagnostic_uid_set_sha256, utc_now_iso(), reservation_id,
|
|
),
|
|
)
|
|
if int(cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError(
|
|
'legacy accepted diagnostic projection fence changed'
|
|
)
|
|
elif (
|
|
int(projection_version) != contracts.DIAGNOSTIC_PROJECTION_VERSION
|
|
or int(current_reservation['remote_diagnostic_count'] or 0)
|
|
!= len(effective_diagnostics)
|
|
or str(current_reservation['remote_diagnostic_uids_sha256'] or '')
|
|
!= diagnostic_uid_set_sha256
|
|
):
|
|
raise ScanEventConflictError(
|
|
'accepted diagnostic projection differs from receipt authority'
|
|
)
|
|
if str(current_bundle['scan_event_hash']) != event_hash:
|
|
raise ScanEventConflictError(
|
|
'validated bundle identity conflicts with its database reservation'
|
|
)
|
|
docker_depth_binding = self._docker_depth_binding_for_reservation_locked(
|
|
current_reservation,
|
|
)
|
|
self._precompute_docker_finding_attribution_rows_locked(
|
|
reader, docker_depth_binding,
|
|
)
|
|
existing = self.conn.execute(
|
|
'''SELECT id, scan_event_hash, result_reservation_id, queue_id,
|
|
claim_lease_token, source, query, target,
|
|
normalized_target, scan_type, raw_result_storage,
|
|
compat_schema_version
|
|
FROM target_scans WHERE scan_event_id = ? FOR UPDATE''',
|
|
(event_id,),
|
|
).fetchone()
|
|
if existing:
|
|
if str(existing['scan_event_hash']) != event_hash:
|
|
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
|
|
if int(existing['result_reservation_id'] or 0) != reservation_id:
|
|
raise ScanEventConflictError('scan event replay belongs to a different reservation')
|
|
if (
|
|
int(existing['queue_id'] or 0)
|
|
!= int(current_reservation['queue_id'])
|
|
or str(existing['claim_lease_token'] or '')
|
|
!= str(current_reservation['claim_lease_token'])
|
|
or str(existing['source'] or '')
|
|
!= str(current_reservation['source'])
|
|
or str(existing['query'] or '')
|
|
!= str(current_reservation['query'] or '')
|
|
or str(existing['target'] or '')
|
|
!= str(current_reservation['target'])
|
|
or str(existing['normalized_target'] or '')
|
|
!= str(current_reservation['normalized_target'])
|
|
or str(existing['scan_type'] or '')
|
|
!= str(current_reservation['platform'])
|
|
or str(existing['raw_result_storage'] or '') != 'normalized_v2'
|
|
or int(existing['compat_schema_version'] or 0) != 2
|
|
):
|
|
raise ScanEventConflictError(
|
|
'authoritative scan replay lost its exact reservation identity'
|
|
)
|
|
if docker_depth_binding:
|
|
if (
|
|
docker_depth_binding['target_scan_id'] is None
|
|
or int(docker_depth_binding['target_scan_id']) != int(existing['id'])
|
|
):
|
|
raise ScanEventConflictError(
|
|
'Docker depth binding scan identity conflicts with replay'
|
|
)
|
|
replay_finding_ids = {}
|
|
for finding in reader.iter_findings():
|
|
finding_id, finding_uid_value = (
|
|
self._existing_normalized_finding_for_replay_locked(
|
|
existing['id'], str(current_reservation['source']),
|
|
str(current_reservation['normalized_target']), finding,
|
|
)
|
|
)
|
|
replay_finding_ids[finding_uid_value] = finding_id
|
|
self._insert_docker_finding_layer_attributions_locked(
|
|
docker_depth_binding, existing['id'], finding_id, finding,
|
|
)
|
|
if len(replay_finding_ids) != validated.finding_count:
|
|
raise ScanEventConflictError(
|
|
'replayed Docker finding identities are missing or duplicated'
|
|
)
|
|
diagnostic_received_at = utc_now_iso()
|
|
for diagnostic in effective_diagnostics:
|
|
self._record_worker_diagnostic(
|
|
reservation_id, diagnostic,
|
|
target_scan_id=int(existing['id']),
|
|
received_at=diagnostic_received_at,
|
|
)
|
|
self.conn.execute(
|
|
"UPDATE result_bundles SET state = 'db_committed', target_scan_id = ?, updated_at = ? WHERE reservation_id = ?",
|
|
(existing['id'], utc_now_iso(), reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
"UPDATE result_reservations SET state = 'db_committed', updated_at = ? WHERE id = ?",
|
|
(utc_now_iso(), reservation_id),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'ingested': True, 'duplicate': True, 'target_scan_id': existing['id'],
|
|
'scan_event_id': event_id, 'scan_event_hash': event_hash,
|
|
}
|
|
if current_reservation['state'] not in ('ready', 'ingesting'):
|
|
raise RuntimeError('bundle reservation is not ingestible')
|
|
if current_bundle['state'] not in ('ready', 'ingesting'):
|
|
raise RuntimeError('bundle row is not ingestible')
|
|
if (
|
|
str(current_reservation['bound_queue_status']) != 'in_progress'
|
|
or str(current_reservation['bound_queue_lease_token'] or '')
|
|
!= str(current_reservation['claim_lease_token'])
|
|
or int(current_reservation['bound_queue_reservation_id'] or 0)
|
|
!= reservation_id
|
|
or str(current_reservation['bound_queue_event_id'] or '') != event_id
|
|
):
|
|
raise ScanEventConflictError(
|
|
'result ingestion lost its exact target queue fence'
|
|
)
|
|
|
|
source = str(current_reservation['source'])
|
|
target = str(current_reservation['target'])
|
|
normalized = str(current_reservation['normalized_target'])
|
|
run_id = current_reservation['run_id']
|
|
cycle_id = current_reservation['cycle_id']
|
|
query = current_reservation['query']
|
|
package = extract_package_metadata(target, metadata, source)
|
|
status = str(metadata.get('status') or 'clean')
|
|
timestamp = str(metadata.get('timestamp') or utc_now_iso())
|
|
target_scan_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO target_scans(
|
|
scan_event_id, scan_event_hash, queue_id, claim_lease_token,
|
|
queue_completion_applied, queue_completion_disposition,
|
|
run_id, cycle_id, source, query, target, normalized_target, scan_type,
|
|
status, started_at, ended_at, duration_sec, scan_options_json,
|
|
package_name, package_version, package_artifact, package_date,
|
|
package_filename, package_type, package_size, findings_count,
|
|
verified_findings_count, error_count, skipped_reason, first_error_summary,
|
|
raw_result_json, result_reservation_id, compat_schema_version,
|
|
raw_result_storage, created_at
|
|
) VALUES (?, ?, ?, ?, 0, 'pending', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, 2, 'normalized_v2', ?)''',
|
|
(
|
|
event_id, event_hash, current_reservation['queue_id'],
|
|
current_reservation['claim_lease_token'], run_id, cycle_id, source, query,
|
|
target, normalized, metadata.get('scan_type'), status,
|
|
metadata.get('scan_started_at') or timestamp, timestamp,
|
|
metadata.get('duration_sec'), json_dumps(redact_config(metadata.get('scan_options') or {})),
|
|
package.get('package_name'), package.get('package_version'),
|
|
package.get('package_artifact'), package.get('package_date'),
|
|
package.get('package_filename'), package.get('package_type'),
|
|
package.get('package_size'), validated.finding_count,
|
|
int(metadata.get('verified_findings_count') or 0), validated.error_count,
|
|
metadata.get('skipped'), metadata.get('first_error_summary'),
|
|
reservation_id, utc_now_iso(),
|
|
),
|
|
)
|
|
if not target_scan_id:
|
|
raise RuntimeError('normalized target scan insert returned no identity')
|
|
self.conn.execute(
|
|
'''INSERT INTO scan_result_compat(
|
|
target_scan_id, schema_version, metadata_json, metadata_sha256,
|
|
metadata_bytes, reconstruction_status, created_at
|
|
) VALUES (?, 2, ?, ?, ?, 'bounded', ?)''',
|
|
(
|
|
target_scan_id, metadata_bytes.decode('utf-8'),
|
|
hashlib.sha256(metadata_bytes).hexdigest(), len(metadata_bytes), utc_now_iso(),
|
|
),
|
|
)
|
|
|
|
diagnostic_received_at = utc_now_iso()
|
|
for diagnostic in effective_diagnostics:
|
|
self._record_worker_diagnostic(
|
|
reservation_id, diagnostic,
|
|
target_scan_id=target_scan_id,
|
|
received_at=diagnostic_received_at,
|
|
)
|
|
|
|
finding_ids = {}
|
|
verified_count = 0
|
|
for finding in reader.iter_findings():
|
|
finding_id, finding_uid_value = self._insert_normalized_finding(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized, finding,
|
|
)
|
|
finding_ids[finding_uid_value] = finding_id
|
|
self._insert_docker_finding_layer_attributions_locked(
|
|
docker_depth_binding, target_scan_id, finding_id, finding,
|
|
)
|
|
verified_count += int(bool(finding.get('Verified')))
|
|
if len(finding_ids) != validated.finding_count:
|
|
raise ValueError('bundle finding identities are missing or duplicated')
|
|
for error in reader.iter_errors():
|
|
self._insert_error(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized, error,
|
|
)
|
|
actual_candidate_items = 0
|
|
actual_candidate_bytes = 0
|
|
per_candidate_capacity = (
|
|
int(current_reservation['reserved_candidate_bytes']) // validated.candidate_count
|
|
if validated.candidate_count else 0
|
|
)
|
|
for frame in reader.iter_candidates():
|
|
inserted_items, inserted_bytes = self._insert_bundle_candidate(
|
|
frame, current_reservation, target_scan_id, finding_ids,
|
|
capacity_allocation=per_candidate_capacity,
|
|
routed_service_supported=routed_service_supported,
|
|
)
|
|
actual_candidate_items += inserted_items
|
|
actual_candidate_bytes += inserted_bytes
|
|
reserved_candidate_items = int(current_reservation['reserved_candidate_items'])
|
|
reserved_candidate_bytes = int(current_reservation['reserved_candidate_bytes'])
|
|
if actual_candidate_items > reserved_candidate_items or actual_candidate_bytes > reserved_candidate_bytes:
|
|
raise ValueError('bundle candidates exceed their pre-reserved capacity')
|
|
|
|
now = utc_now_iso()
|
|
queue_status = str(metadata.get('queue_status') or ('failed' if status == 'error' else 'done'))
|
|
if queue_status not in ('done', 'failed', 'deferred'):
|
|
raise ValueError('bundle queue disposition is invalid')
|
|
self._apply_docker_layer_execution_locked(
|
|
current_reservation, metadata, queue_status, now,
|
|
)
|
|
advance_git_coverage, covered_ref, covered_head = matching_git_coverage(
|
|
current_reservation, metadata, queue_status, validated.error_count,
|
|
)
|
|
completed_at = now if queue_status in ('done', 'failed') else None
|
|
queue_cursor = self.conn.execute(
|
|
'''UPDATE target_queue SET status = ?, target_scan_id = ?, last_error = ?,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL,
|
|
leased_at = NULL, lease_expires_at = NULL, available_after = ?,
|
|
attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
|
|
completed_at = COALESCE(?, completed_at),
|
|
covered_ref = CASE WHEN ? != 0 THEN ? ELSE covered_ref END,
|
|
covered_head = CASE WHEN ? != 0 THEN ? ELSE covered_head END,
|
|
current_result_reservation_id = NULL, claim_event_id = NULL, updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?
|
|
AND current_result_reservation_id = ? AND claim_event_id = ?''',
|
|
(
|
|
queue_status, target_scan_id, first_line(metadata.get('queue_error'), 500),
|
|
metadata.get('available_after'), 1 if metadata.get('reset_attempts') else 0,
|
|
completed_at,
|
|
1 if advance_git_coverage else 0, covered_ref,
|
|
1 if advance_git_coverage else 0, covered_head,
|
|
now, current_reservation['queue_id'],
|
|
current_reservation['claim_lease_token'], reservation_id, event_id,
|
|
),
|
|
)
|
|
if int(queue_cursor.rowcount or 0) != 1:
|
|
raise ScanEventConflictError('fenced target queue completion did not match the reservation')
|
|
self.conn.execute(
|
|
'''UPDATE target_scans SET queue_completion_applied = 1,
|
|
queue_completion_disposition = 'applied',
|
|
verified_findings_count = ? WHERE id = ?''',
|
|
(verified_count, target_scan_id),
|
|
)
|
|
self._transition_docker_depth_binding_locked(
|
|
current_reservation,
|
|
'completed' if queue_status == 'done' else 'failed',
|
|
queue_status if queue_status in ('done', 'failed') else 'pending',
|
|
now,
|
|
target_scan_id=target_scan_id,
|
|
)
|
|
self._insert_derived_postman_targets(metadata.get('derived_postman_targets'), now)
|
|
projection_job_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO projection_jobs(
|
|
job_kind, event_id, event_hash, target_scan_id, status,
|
|
required_stream_mask, capacity_items, capacity_bytes,
|
|
created_at, updated_at
|
|
) VALUES ('scan_event', ?, ?, ?, 'pending', ?, ?, ?, ?, ?)''',
|
|
(
|
|
event_id, event_hash, target_scan_id,
|
|
1 | (2 if validated.finding_count else 0) | (4 if validated.error_count else 0),
|
|
current_reservation['reserved_projection_items'],
|
|
current_reservation['reserved_projection_bytes'], now, now,
|
|
),
|
|
)
|
|
if not projection_job_id:
|
|
raise RuntimeError('projection job insert returned no identity')
|
|
unused_items = reserved_candidate_items - actual_candidate_items
|
|
unused_bytes = reserved_candidate_bytes - actual_candidate_bytes
|
|
# Workload rows are complete before taking the global accounting lock.
|
|
capacity = self.conn.execute(
|
|
'SELECT * FROM pipeline_capacity WHERE id = 1 FOR UPDATE'
|
|
).fetchone()
|
|
if int(capacity['keycheck_items']) < unused_items or int(capacity['keycheck_bytes']) < unused_bytes:
|
|
raise RuntimeError('candidate capacity transfer would make accounting negative')
|
|
self.conn.execute(
|
|
'''UPDATE pipeline_capacity SET keycheck_items = keycheck_items - ?,
|
|
keycheck_bytes = keycheck_bytes - ?, updated_at = ? WHERE id = 1''',
|
|
(unused_items, unused_bytes, now),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_reservations SET state = 'db_committed',
|
|
projection_credit_transferred = 1, candidate_credit_transferred = 1,
|
|
updated_at = ? WHERE id = ?''',
|
|
(now, reservation_id),
|
|
)
|
|
self.conn.execute(
|
|
'''UPDATE result_bundles SET state = 'db_committed', target_scan_id = ?,
|
|
committed_at = ?, updated_at = ? WHERE reservation_id = ?''',
|
|
(target_scan_id, now, now, reservation_id),
|
|
)
|
|
found_count = 1 if validated.finding_count else 0
|
|
clean_count = 1 if not validated.finding_count and not validated.error_count and not metadata.get('skipped') else 0
|
|
skipped_count = 1 if metadata.get('skipped') else 0
|
|
if cycle_id is not None:
|
|
self.conn.execute(
|
|
'''UPDATE source_cycles SET ingested_count = ingested_count + 1,
|
|
scanned_count = scanned_count + 1, clean_count = clean_count + ?,
|
|
found_count = found_count + ?, skipped_count = skipped_count + ?,
|
|
error_count = error_count + ?, findings_count = findings_count + ?,
|
|
verified_findings_count = verified_findings_count + ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(
|
|
clean_count, found_count, skipped_count, validated.error_count,
|
|
validated.finding_count, verified_count, now, cycle_id,
|
|
),
|
|
)
|
|
if run_id is not None:
|
|
self.conn.execute(
|
|
'''UPDATE runs SET total_scanned = total_scanned + 1,
|
|
total_clean = total_clean + ?, total_found = total_found + ?,
|
|
total_skipped = total_skipped + ?, total_errors = total_errors + ?,
|
|
total_findings = total_findings + ?,
|
|
total_verified_findings = total_verified_findings + ?, updated_at = ?
|
|
WHERE id = ?''',
|
|
(
|
|
clean_count, found_count, skipped_count, validated.error_count,
|
|
validated.finding_count, verified_count, now, run_id,
|
|
),
|
|
)
|
|
self.conn.commit()
|
|
return {
|
|
'ingested': True, 'duplicate': False, 'target_scan_id': target_scan_id,
|
|
'projection_job_id': projection_job_id,
|
|
'candidate_count': actual_candidate_items,
|
|
'scan_event_id': event_id, 'scan_event_hash': event_hash,
|
|
}
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
|
|
@staticmethod
|
|
def _compat_finding_from_row(row):
|
|
if row['payload_omitted']:
|
|
return {
|
|
'finding_uid': row['finding_uid'],
|
|
'DetectorName': row['detector_name'],
|
|
'Verified': bool(row['verified']),
|
|
'finding_omitted': True,
|
|
'payload_sha256': row['payload_sha256'],
|
|
'payload_bytes': row['payload_bytes'],
|
|
}
|
|
finding = safe_json_loads(row['extension_json']) or {}
|
|
finding.update({
|
|
'finding_uid': row['finding_uid'],
|
|
'DetectorName': row['detector_name'],
|
|
'DetectorType': row['detector_type'],
|
|
'Verified': bool(row['verified']),
|
|
'Raw': row['raw_value'],
|
|
'RawV2': row['raw_v2_value'],
|
|
'Redacted': row['redacted_secret'],
|
|
})
|
|
for column, key in (
|
|
('structured_data_json', 'StructuredData'),
|
|
('extra_data_json', 'ExtraData'),
|
|
('analysis_info_json', 'AnalysisInfo'),
|
|
):
|
|
value = safe_json_loads(row[column])
|
|
if value is not None:
|
|
finding[key] = value
|
|
return finding
|
|
|
|
def projection_scan_header(self, target_scan_id, max_bytes=16 * 1024 * 1024):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
raw_size_sql = (
|
|
'OCTET_LENGTH(ts.raw_result_json)' if self.conn.is_postgres
|
|
else "LENGTH(CAST(ts.raw_result_json AS BLOB))"
|
|
)
|
|
scan = self.conn.execute(
|
|
f'''SELECT ts.id, ts.scan_event_id, ts.target, ts.scan_type, ts.ended_at,
|
|
ts.raw_result_storage,
|
|
CASE WHEN ts.raw_result_json IS NULL THEN 0
|
|
ELSE {raw_size_sql} END AS raw_result_bytes,
|
|
sc.metadata_sha256, sc.metadata_bytes, sc.reconstruction_status
|
|
FROM target_scans ts
|
|
LEFT JOIN scan_result_compat sc ON sc.target_scan_id = ts.id
|
|
WHERE ts.id = ?''',
|
|
(int(target_scan_id),),
|
|
).fetchone()
|
|
if not scan:
|
|
return None
|
|
if scan['raw_result_storage'] != 'normalized_v2':
|
|
raw_bytes = int(scan['raw_result_bytes'] or 0)
|
|
if raw_bytes <= 0 or raw_bytes > max(1, int(max_bytes)):
|
|
raise ValueError('legacy compatibility result exceeds its reconstruction byte bound')
|
|
payload_size_sql = (
|
|
'OCTET_LENGTH(raw_result_json)' if self.conn.is_postgres
|
|
else 'LENGTH(CAST(raw_result_json AS BLOB))'
|
|
)
|
|
payload = self.conn.execute(
|
|
f'''SELECT raw_result_json FROM target_scans
|
|
WHERE id = ? AND raw_result_storage != 'normalized_v2'
|
|
AND raw_result_json IS NOT NULL
|
|
AND {payload_size_sql} = ?''',
|
|
(int(target_scan_id), raw_bytes),
|
|
).fetchone()
|
|
if not payload:
|
|
raise RuntimeError('legacy compatibility result changed during bounded reconstruction')
|
|
legacy = safe_json_loads(payload['raw_result_json'])
|
|
if isinstance(legacy, dict):
|
|
legacy.setdefault(
|
|
'scan_event_id', scan['scan_event_id'] or f'legacy-target-scan-{scan["id"]}',
|
|
)
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return {'storage': 'legacy', 'result': legacy}
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return None
|
|
if int(scan['metadata_bytes'] or 0) > max(1, int(max_bytes)):
|
|
raise ValueError('normalized compatibility metadata exceeds its reconstruction byte bound')
|
|
metadata_size_sql = (
|
|
'OCTET_LENGTH(metadata_json)' if self.conn.is_postgres
|
|
else 'LENGTH(CAST(metadata_json AS BLOB))'
|
|
)
|
|
metadata = self.conn.execute(
|
|
f'''SELECT metadata_json FROM scan_result_compat
|
|
WHERE target_scan_id = ? AND metadata_bytes = ?
|
|
AND {metadata_size_sql} <= ?''',
|
|
(
|
|
int(target_scan_id), int(scan['metadata_bytes'] or 0),
|
|
max(1, int(max_bytes)),
|
|
),
|
|
).fetchone()
|
|
if not metadata:
|
|
raise RuntimeError('normalized compatibility metadata changed during bounded reconstruction')
|
|
result = safe_json_loads(metadata['metadata_json']) or {}
|
|
if not isinstance(result, dict):
|
|
raise ValueError('normalized compatibility metadata is invalid')
|
|
result['scan_event_id'] = scan['scan_event_id']
|
|
result['target'] = scan['target']
|
|
result['scan_type'] = scan['scan_type']
|
|
result['timestamp'] = scan['ended_at']
|
|
result.pop('findings', None)
|
|
result.pop('errors', None)
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
return {'storage': 'normalized_v2', 'result': result}
|
|
|
|
def iter_projection_findings(self, target_scan_id, max_findings=20000, page_size=4):
|
|
last_id = 0
|
|
count = 0
|
|
page_size = min(64, max(1, int(page_size)))
|
|
while True:
|
|
rows = self.conn.execute(
|
|
'''SELECT f.*, cp.raw_value, cp.raw_v2_value, cp.structured_data_json,
|
|
cp.extra_data_json, cp.analysis_info_json, cp.extension_json,
|
|
cp.payload_sha256, cp.payload_bytes, cp.payload_omitted
|
|
FROM findings f
|
|
LEFT JOIN finding_compat_payloads cp ON cp.finding_id = f.id
|
|
WHERE f.target_scan_id = ? AND f.id > ? ORDER BY f.id LIMIT ?''',
|
|
(int(target_scan_id), last_id, page_size),
|
|
).fetchall()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
if not rows:
|
|
return
|
|
for row in rows:
|
|
count += 1
|
|
if count > max(0, int(max_findings)):
|
|
raise ValueError('normalized compatibility reconstruction exceeds its finding bound')
|
|
last_id = int(row['id'])
|
|
yield self._compat_finding_from_row(row)
|
|
|
|
def iter_projection_errors(self, target_scan_id, max_errors=20000, page_size=64):
|
|
last_id = 0
|
|
count = 0
|
|
page_size = min(256, max(1, int(page_size)))
|
|
while True:
|
|
rows = self.conn.execute(
|
|
'''SELECT id, raw_error FROM errors
|
|
WHERE target_scan_id = ? AND id > ? ORDER BY id LIMIT ?''',
|
|
(int(target_scan_id), last_id, page_size),
|
|
).fetchall()
|
|
if self.conn.is_postgres:
|
|
self.conn.commit()
|
|
if not rows:
|
|
return
|
|
for row in rows:
|
|
count += 1
|
|
if count > max(0, int(max_errors)):
|
|
raise ValueError('normalized compatibility reconstruction exceeds its error bound')
|
|
last_id = int(row['id'])
|
|
yield row['raw_error']
|
|
|
|
def reconstruct_scan_result(
|
|
self, target_scan_id, max_bytes=192 * 1024 * 1024,
|
|
max_findings=20000, max_errors=20000,
|
|
):
|
|
header = self.projection_scan_header(target_scan_id, max_bytes=max_bytes)
|
|
if not header:
|
|
return None
|
|
result = header['result']
|
|
if header['storage'] == 'normalized_v2':
|
|
byte_limit = max(1, int(max_bytes))
|
|
used = len(json.dumps(
|
|
result, ensure_ascii=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')) + 64
|
|
findings = []
|
|
for finding in self.iter_projection_findings(target_scan_id, max_findings):
|
|
used += len(json.dumps(
|
|
finding, ensure_ascii=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')) + 1
|
|
if used > byte_limit:
|
|
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
|
|
findings.append(finding)
|
|
errors = []
|
|
for error in self.iter_projection_errors(target_scan_id, max_errors):
|
|
used += len(json.dumps(
|
|
error, ensure_ascii=True, separators=(',', ':'), default=str,
|
|
).encode('utf-8')) + 1
|
|
if used > byte_limit:
|
|
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
|
|
errors.append(error)
|
|
result['findings'] = findings
|
|
result['errors'] = errors
|
|
encoded = json.dumps(result, ensure_ascii=True, separators=(',', ':'), default=str).encode('utf-8')
|
|
if len(encoded) > max(1, int(max_bytes)):
|
|
raise ValueError('normalized compatibility reconstruction exceeds its aggregate bound')
|
|
return result
|
|
|
|
def record_target_result(self, run_id, cycle_id, source, query, target, result, scan_options=None):
|
|
if not run_id or not cycle_id:
|
|
return None
|
|
def op():
|
|
target_scan_id = self._insert_target_result(run_id, cycle_id, source, query, target, result, scan_options)
|
|
self.conn.commit()
|
|
return target_scan_id
|
|
return self._safe('record_target_result', op)
|
|
|
|
def _insert_target_result(
|
|
self, run_id, cycle_id, source, query, target, result, scan_options=None,
|
|
scan_event_id=None, scan_event_hash_value=None, queue_id=None, claim_lease_token=None,
|
|
queue_completion_applied=0, queue_completion_disposition=None,
|
|
):
|
|
status = target_status(result)
|
|
normalized = normalize_target(target, source)
|
|
findings = [sanitize_postman_finding(finding) for finding in (result.get('findings') or [])]
|
|
persisted_result = dict(result)
|
|
if 'findings' in result or findings:
|
|
persisted_result['findings'] = findings
|
|
errors = result.get('errors') or []
|
|
package = extract_package_metadata(target, result, source)
|
|
timestamp = result.get('timestamp') or utc_now_iso()
|
|
conflict_clause = ' ON CONFLICT DO NOTHING' if scan_event_id else ''
|
|
target_scan_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO target_scans (
|
|
scan_event_id, scan_event_hash, queue_id, claim_lease_token,
|
|
queue_completion_applied, queue_completion_disposition,
|
|
run_id, cycle_id, source, query, target, normalized_target, scan_type, status,
|
|
started_at, ended_at, duration_sec, scan_options_json, package_name, package_version,
|
|
package_artifact, package_date, package_filename, package_type, package_size,
|
|
findings_count, verified_findings_count, error_count, skipped_reason, first_error_summary,
|
|
raw_result_json, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' + conflict_clause,
|
|
(
|
|
scan_event_id, scan_event_hash_value, queue_id, claim_lease_token,
|
|
1 if queue_completion_applied else 0, queue_completion_disposition,
|
|
run_id, cycle_id, source, query, target, normalized, result.get('scan_type'), status,
|
|
result.get('scan_started_at') or timestamp, timestamp, result.get('duration_sec'), json_dumps(redact_config(scan_options or {})),
|
|
package.get('package_name'), package.get('package_version'), package.get('package_artifact'), package.get('package_date'),
|
|
package.get('package_filename'), package.get('package_type'), package.get('package_size'),
|
|
len(findings), sum(1 for finding in findings if finding.get('Verified', False)), len(errors),
|
|
result.get('skipped'), first_error_summary(result), json_dumps(persisted_result), utc_now_iso(),
|
|
),
|
|
)
|
|
if target_scan_id is None and scan_event_id:
|
|
return None
|
|
for finding in findings:
|
|
self._insert_finding(run_id, cycle_id, target_scan_id, source, query, target, normalized, finding)
|
|
for error in errors:
|
|
self._insert_error(run_id, cycle_id, target_scan_id, source, query, target, normalized, error)
|
|
return target_scan_id
|
|
|
|
@staticmethod
|
|
def _unique_violation(exc):
|
|
code = getattr(exc, 'sqlstate', None) or getattr(exc, 'pgcode', None)
|
|
text = str(exc or '').lower()
|
|
return code == '23505' or 'unique constraint' in text or 'duplicate key' in text
|
|
|
|
def _confirmed_scan_event(self, event_id, event_hash):
|
|
row = self.conn.execute(
|
|
'''SELECT id, scan_event_hash, queue_completion_applied, queue_completion_disposition
|
|
FROM target_scans WHERE scan_event_id = ?''',
|
|
(event_id,),
|
|
).fetchone()
|
|
if not row:
|
|
return None
|
|
if str(row['scan_event_hash'] or '') != event_hash:
|
|
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
|
|
outbox = self.conn.execute(
|
|
'SELECT id FROM scan_publication_outbox WHERE target_scan_id = ?',
|
|
(row['id'],),
|
|
).fetchone()
|
|
# Delivery deletes the reference row. Its absence on replay therefore
|
|
# means the authoritative scan event was already projected.
|
|
self.conn.commit()
|
|
applied = bool(row['queue_completion_applied'])
|
|
return {
|
|
'ingested': True,
|
|
'duplicate': True,
|
|
'stale': not applied,
|
|
'queue_completion_applied': applied,
|
|
'queue_completion_disposition': row['queue_completion_disposition'],
|
|
'target_scan_id': row['id'],
|
|
'outbox_id': outbox['id'] if outbox else None,
|
|
'scan_event_id': event_id,
|
|
'scan_event_hash': event_hash,
|
|
}
|
|
|
|
def confirmed_scan_event(self, event_id, event_hash):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
return self._confirmed_scan_event(str(event_id or ''), str(event_hash or ''))
|
|
|
|
def _insert_derived_postman_targets(self, targets, now):
|
|
seen = set()
|
|
for target in targets or []:
|
|
normalized = normalize_target(target, 'postman')
|
|
if not normalized or normalized in seen:
|
|
continue
|
|
seen.add(normalized)
|
|
self.conn.execute(
|
|
'''INSERT INTO target_queue (
|
|
source, platform, query, target, normalized_target, status, created_at, updated_at
|
|
) VALUES ('postman', 'postman', 'harvested', ?, ?, 'pending', ?, ?)
|
|
ON CONFLICT(source, normalized_target) DO NOTHING''',
|
|
(target, normalized, now, now),
|
|
)
|
|
|
|
def ingest_scan_event(self, envelope):
|
|
if not self.conn:
|
|
raise RuntimeError('database connection is unavailable')
|
|
self.require_runtime_safety_schema()
|
|
try:
|
|
event = prepare_scan_event(envelope)
|
|
except SpoolHashConflictError as exc:
|
|
raise ScanEventConflictError(str(exc)) from exc
|
|
event_id = event['scan_event_id']
|
|
event_hash = event['scan_event_hash']
|
|
result = event.get('result')
|
|
if not isinstance(result, dict):
|
|
raise ValueError('scan event result must be an object')
|
|
if str(result.get('scan_event_id') or '') != event_id:
|
|
raise ValueError('scan event result ID does not match its envelope')
|
|
queue_id = event.get('queue_id')
|
|
lease_token = str(event.get('claim_lease_token') or '')
|
|
if queue_id is None or not lease_token:
|
|
raise ValueError('scan event requires queue_id and claim_lease_token')
|
|
queue_status = str(event.get('queue_status') or '')
|
|
if queue_status not in ('done', 'failed', 'deferred'):
|
|
raise ValueError(f'invalid scan event queue status: {queue_status}')
|
|
run_id = event.get('run_id')
|
|
cycle_id = event.get('cycle_id')
|
|
if run_id is None or cycle_id is None:
|
|
raise ValueError('scan event requires run_id and cycle_id')
|
|
|
|
try:
|
|
if self.conn.is_sqlite:
|
|
self.conn.execute('BEGIN IMMEDIATE')
|
|
lock_suffix = ' FOR UPDATE' if self.conn.is_postgres else ''
|
|
existing = self.conn.execute(
|
|
f'''SELECT id, scan_event_hash, queue_completion_applied, queue_completion_disposition
|
|
FROM target_scans WHERE scan_event_id = ?{lock_suffix}''',
|
|
(event_id,),
|
|
).fetchone()
|
|
if existing:
|
|
if str(existing['scan_event_hash'] or '') != event_hash:
|
|
raise ScanEventConflictError(f'scan event {event_id} already exists with a different hash')
|
|
self.conn.commit()
|
|
return self._confirmed_scan_event(event_id, event_hash)
|
|
|
|
target = str(event.get('target') or result.get('target') or '')
|
|
source = str(event.get('source') or '')
|
|
query = event.get('query')
|
|
target_scan_id = self._insert_target_result(
|
|
run_id, cycle_id, source, query, target, result, event.get('scan_options') or {},
|
|
scan_event_id=event_id,
|
|
scan_event_hash_value=event_hash,
|
|
queue_id=queue_id,
|
|
claim_lease_token=lease_token,
|
|
queue_completion_applied=0,
|
|
queue_completion_disposition='pending',
|
|
)
|
|
if target_scan_id is None:
|
|
confirmed = self._confirmed_scan_event(event_id, event_hash)
|
|
if confirmed:
|
|
return confirmed
|
|
raise RuntimeError(f'scan event {event_id} conflict returned no authoritative row')
|
|
now = utc_now_iso()
|
|
completed = now if queue_status in ('done', 'failed') else None
|
|
cur = self.conn.execute(
|
|
'''UPDATE target_queue SET
|
|
status = ?, target_scan_id = ?, last_error = ?,
|
|
lease_owner = NULL, lease_token = NULL, claim_batch = NULL, leased_at = NULL, lease_expires_at = NULL,
|
|
available_after = ?, attempts = CASE WHEN ? != 0 THEN 0 ELSE attempts END,
|
|
completed_at = COALESCE(?, completed_at), updated_at = ?
|
|
WHERE id = ? AND status = 'in_progress' AND lease_token = ?''',
|
|
(
|
|
queue_status, target_scan_id,
|
|
first_line(event.get('queue_error'), 500) if event.get('queue_error') else None,
|
|
event.get('available_after'), 1 if event.get('reset_attempts') else 0,
|
|
completed, now, queue_id, lease_token,
|
|
),
|
|
)
|
|
applied = int(getattr(cur, 'rowcount', 0) or 0) == 1
|
|
disposition = 'applied' if applied else 'stale_lease'
|
|
self.conn.execute(
|
|
'''UPDATE target_scans SET queue_completion_applied = ?, queue_completion_disposition = ?
|
|
WHERE id = ?''',
|
|
(1 if applied else 0, disposition, target_scan_id),
|
|
)
|
|
self._insert_derived_postman_targets(event.get('derived_postman_targets'), now)
|
|
outbox_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO scan_publication_outbox (
|
|
target_scan_id, payload_json, status, attempts, created_at, updated_at
|
|
) VALUES (?, ?, 'pending', 0, ?, ?)''',
|
|
(target_scan_id, '', now, now),
|
|
)
|
|
self.conn.commit()
|
|
self.last_error = ''
|
|
return {
|
|
'ingested': True,
|
|
'duplicate': False,
|
|
'stale': not applied,
|
|
'queue_completion_applied': applied,
|
|
'queue_completion_disposition': disposition,
|
|
'target_scan_id': target_scan_id,
|
|
'outbox_id': outbox_id,
|
|
'scan_event_id': event_id,
|
|
'scan_event_hash': event_hash,
|
|
}
|
|
except ScanEventConflictError:
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
try:
|
|
self.conn.rollback()
|
|
except Exception:
|
|
pass
|
|
if self._unique_violation(exc):
|
|
confirmed = self._confirmed_scan_event(event_id, event_hash)
|
|
if confirmed:
|
|
return confirmed
|
|
raise
|
|
|
|
def record_and_complete_target_result(
|
|
self, run_id, cycle_id, source, query, target, result, scan_options,
|
|
queue_id, lease_owner, queue_status, queue_error=None, available_after=None,
|
|
reset_attempts=False, derived_postman_targets=None, lease_token=None,
|
|
):
|
|
event_id = str(result.get('scan_event_id') or '')
|
|
if not self.conn or not event_id or queue_id is None or not lease_token:
|
|
return None
|
|
event = prepare_scan_event({
|
|
'version': 1,
|
|
'scan_event_id': event_id,
|
|
'run_id': run_id,
|
|
'cycle_id': cycle_id,
|
|
'source': source,
|
|
'query': query,
|
|
'target': target,
|
|
'result': result,
|
|
'scan_options': scan_options or {},
|
|
'queue_id': queue_id,
|
|
'claim_lease_token': lease_token,
|
|
'claim_lease_owner': lease_owner,
|
|
'queue_status': queue_status,
|
|
'queue_error': queue_error,
|
|
'available_after': available_after,
|
|
'reset_attempts': bool(reset_attempts),
|
|
'derived_postman_targets': list(derived_postman_targets or []),
|
|
})
|
|
return self.ingest_scan_event(event)
|
|
|
|
def record_target_results(self, run_id, cycle_id, source, query, results, scan_options=None):
|
|
output = {}
|
|
for result in results or []:
|
|
target = result.get('target', '')
|
|
target_scan_id = self.record_target_result(run_id, cycle_id, source, query, target, result, scan_options)
|
|
if target_scan_id is None:
|
|
output[normalize_target(target, source)] = None
|
|
output[str(target)] = None
|
|
else:
|
|
output[normalize_target(target, source)] = target_scan_id
|
|
output[str(target)] = target_scan_id
|
|
return output
|
|
|
|
def _insert_finding(self, run_id, cycle_id, target_scan_id, source, query, target, normalized, finding):
|
|
finding = sanitize_postman_finding(finding)
|
|
raw_secret, secret_hash, detector_secret_hash, fingerprint, location = finding_identity(source, normalized, finding)
|
|
finding_uid = finding.get('finding_uid') or fingerprint
|
|
enrichment = enrich_finding(finding)
|
|
finding_id = self.conn.insert_returning_id(
|
|
'''INSERT INTO findings (
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
|
|
detector_name, detector_type, verified, raw_secret, redacted_secret, secret_hash,
|
|
detector_secret_hash, finding_fingerprint, finding_uid, file_path, line_number, commit_hash,
|
|
source_timestamp, source_metadata_type, source_metadata_json, raw_finding_json,
|
|
provider, credential_kind, credential_confidence, required_context_missing,
|
|
principal, username, email, project_id, tenant_id, organization, registry,
|
|
endpoint, scope, resource, enrichment_json, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized,
|
|
finding.get('DetectorName'), finding.get('DetectorType'), 1 if finding.get('Verified', False) else 0,
|
|
raw_secret, extract_redacted_secret(finding), secret_hash, detector_secret_hash, fingerprint, finding_uid,
|
|
location.get('file_path'), location.get('line_number'), location.get('commit_hash'), location.get('source_timestamp'),
|
|
location.get('source_metadata_type'), location.get('source_metadata_json'), json_dumps(finding),
|
|
enrichment.get('provider'), enrichment.get('credential_kind'), enrichment.get('credential_confidence'),
|
|
enrichment.get('required_context_missing'), enrichment.get('principal'), enrichment.get('username'),
|
|
enrichment.get('email'), enrichment.get('project_id'), enrichment.get('tenant_id'), enrichment.get('organization'),
|
|
enrichment.get('registry'), enrichment.get('endpoint'), enrichment.get('scope'), enrichment.get('resource'),
|
|
enrichment.get('enrichment_json'), utc_now_iso(),
|
|
),
|
|
)
|
|
if finding_uid and finding_id:
|
|
self.conn.execute(
|
|
'''INSERT INTO finding_uid_map (finding_uid, finding_id, created_at)
|
|
VALUES (?, ?, ?)
|
|
ON CONFLICT(finding_uid) DO NOTHING''',
|
|
(finding_uid, finding_id, utc_now_iso()),
|
|
)
|
|
|
|
def _insert_error(self, run_id, cycle_id, target_scan_id, source, query, target, normalized, error):
|
|
raw_error = str(error)
|
|
self.conn.execute(
|
|
'''INSERT INTO errors (
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized_target,
|
|
category, summary, raw_error, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
(
|
|
run_id, cycle_id, target_scan_id, source, query, target, normalized,
|
|
categorize_error(raw_error), first_line(raw_error), raw_error, utc_now_iso(),
|
|
),
|
|
)
|
|
|
|
def record_keycheck_result(self, service, status, status_group, checked_at=None, key_hash='', secret_hash='', key_masked='', detector_name='', message='', metadata=None, link_findings=True, source_line='', detector_secret_hash=''):
|
|
if not self.conn:
|
|
return
|
|
self.require_runtime_safety_schema()
|
|
|
|
def keycheck_result_columns():
|
|
if self._keycheck_result_columns is None:
|
|
try:
|
|
self._keycheck_result_columns = self.conn.table_columns('keycheck_results')
|
|
except Exception:
|
|
self._keycheck_result_columns = set()
|
|
return self._keycheck_result_columns
|
|
|
|
def matching_findings(hash_value):
|
|
if not hash_value:
|
|
return []
|
|
clauses = ['secret_hash = ?']
|
|
params = [hash_value]
|
|
if detector_name:
|
|
if str(detector_name).lower() == 'googleaistudio':
|
|
extra_name = self.conn.json_extract('raw_finding_json', '$.ExtraData.name')
|
|
clauses.append("""(
|
|
LOWER(detector_name) = LOWER(?)
|
|
OR (
|
|
LOWER(detector_name) = 'customregex'
|
|
AND LOWER(COALESCE({extra_name}, '')) = LOWER(?)
|
|
)
|
|
)""".format(extra_name=extra_name))
|
|
params.extend([detector_name, detector_name])
|
|
else:
|
|
clauses.append('LOWER(detector_name) = LOWER(?)')
|
|
params.append(detector_name)
|
|
row = self.conn.execute(
|
|
f'''SELECT id, run_id, cycle_id, target_scan_id, source, query, target, detector_name, created_at
|
|
FROM findings WHERE {' AND '.join(clauses)} ORDER BY id DESC LIMIT 1''',
|
|
params,
|
|
).fetchone()
|
|
return [row] if row else []
|
|
|
|
def op():
|
|
now = utc_now_iso()
|
|
checked = checked_at or now
|
|
# Inline linking is intentionally disabled for live keychecks. It can
|
|
# misattribute duplicate secrets and should be handled by the bounded
|
|
# repair/linker pipeline instead.
|
|
rows = []
|
|
if not rows:
|
|
rows = [None]
|
|
has_link_columns = {'source_line', 'detector_secret_hash', 'link_status', 'link_attempts', 'linked_at', 'link_error'}.issubset(keycheck_result_columns())
|
|
has_event_columns = {'event_id', 'finding_uid'}.issubset(keycheck_result_columns())
|
|
for row in rows:
|
|
base_values = (
|
|
service, status, status_group, checked, key_hash or '', secret_hash or '', key_masked or '',
|
|
row['id'] if row else None,
|
|
row['target_scan_id'] if row else None,
|
|
row['cycle_id'] if row else None,
|
|
row['run_id'] if row else None,
|
|
row['source'] if row else None,
|
|
row['query'] if row else None,
|
|
row['target'] if row else None,
|
|
row['detector_name'] if row else detector_name,
|
|
row['created_at'] if row else None,
|
|
first_line(message, 1000),
|
|
json_dumps(metadata or {}),
|
|
)
|
|
if has_link_columns and has_event_columns:
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_results (
|
|
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
|
|
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
|
|
found_at, message, metadata_json, source_line, detector_secret_hash,
|
|
event_id, finding_uid, link_status, link_attempts, linked_at, link_error, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
base_values + (
|
|
source_line or (metadata or {}).get('source_line') or '',
|
|
detector_secret_hash or '',
|
|
(metadata or {}).get('event_id') or '',
|
|
(metadata or {}).get('finding_uid') or '',
|
|
'linked' if row else 'pending',
|
|
0,
|
|
now if row else None,
|
|
'',
|
|
now,
|
|
),
|
|
)
|
|
elif has_link_columns:
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_results (
|
|
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
|
|
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
|
|
found_at, message, metadata_json, source_line, detector_secret_hash,
|
|
link_status, link_attempts, linked_at, link_error, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
base_values + (
|
|
source_line or (metadata or {}).get('source_line') or '',
|
|
detector_secret_hash or '',
|
|
'linked' if row else 'pending',
|
|
0,
|
|
now if row else None,
|
|
'',
|
|
now,
|
|
),
|
|
)
|
|
else:
|
|
self.conn.execute(
|
|
'''INSERT INTO keycheck_results (
|
|
service, status, status_group, checked_at, key_hash, secret_hash, key_masked,
|
|
finding_id, target_scan_id, cycle_id, run_id, source, query, target, detector_name,
|
|
found_at, message, metadata_json, created_at
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
|
|
base_values + (now,),
|
|
)
|
|
self.conn.commit()
|
|
return True
|
|
return bool(self._safe('record_keycheck_result', op, False))
|
|
|
|
|
|
def _migration_add_columns(conn, table, additions):
|
|
existing = conn.table_columns(table)
|
|
for name, ddl in additions.items():
|
|
if name not in existing:
|
|
conn.execute(f'ALTER TABLE {table} ADD COLUMN {name} {ddl}')
|
|
|
|
|
|
def _migration_seed_legacy_docker_provenance(
|
|
conn,
|
|
max_rows=DOCKER_LEGACY_PROVENANCE_SEED_MAX_ROWS,
|
|
page_size=DOCKER_LEGACY_PROVENANCE_SEED_PAGE_SIZE,
|
|
):
|
|
max_rows = max(1, int(max_rows))
|
|
page_size = max(1, min(int(page_size), max_rows, 10000))
|
|
after_id = 0
|
|
scanned = 0
|
|
inserted = 0
|
|
predicate = """platform = 'docker' AND query IS NOT NULL AND query <> ''
|
|
AND (resolver_state IS NOT NULL OR target NOT LIKE '%@%')"""
|
|
while scanned < max_rows:
|
|
rows = conn.execute(
|
|
f'''SELECT id, source, query, target, created_at, updated_at
|
|
FROM target_queue
|
|
WHERE id > ? AND {predicate}
|
|
ORDER BY id LIMIT ?''',
|
|
(after_id, min(page_size, max_rows - scanned)),
|
|
).fetchall()
|
|
if not rows:
|
|
break
|
|
for row in rows:
|
|
after_id = int(row['id'])
|
|
target = str(row['target'] or '').strip()
|
|
if not target or '@' in target or ':' in target.rsplit('/', 1)[-1]:
|
|
continue
|
|
cursor = conn.execute(
|
|
'''INSERT INTO docker_repository_query_provenance(
|
|
source, query, repository_queue_id, provenance_kind,
|
|
first_observed_at, last_observed_at, created_at, updated_at
|
|
) VALUES (?, ?, ?, 'legacy_queue', ?, ?, ?, ?)
|
|
ON CONFLICT(source, query, repository_queue_id) DO NOTHING''',
|
|
(
|
|
row['source'], row['query'], row['id'], row['created_at'],
|
|
row['created_at'], row['created_at'], row['updated_at'],
|
|
),
|
|
)
|
|
inserted += max(0, int(getattr(cursor, 'rowcount', 0) or 0))
|
|
scanned += len(rows)
|
|
if scanned >= max_rows and conn.execute(
|
|
f'''SELECT 1 AS present FROM target_queue
|
|
WHERE id > ? AND {predicate} ORDER BY id LIMIT 1''',
|
|
(after_id,),
|
|
).fetchone():
|
|
raise RuntimeSafetySchemaError(
|
|
f'legacy Docker provenance seed exceeds the reviewed {max_rows}-row bound'
|
|
)
|
|
return {'scanned_count': scanned, 'inserted_count': inserted}
|
|
|
|
|
|
def _migration_backfill_keycheck_hashes(conn, max_rows=1000000):
|
|
processed = 0
|
|
while True:
|
|
rows = conn.execute(
|
|
'''SELECT id, service, candidate_kind, secret_text, secret_json, endpoint, principal
|
|
FROM keycheck_credentials WHERE provider_key_hash = '' ORDER BY id LIMIT 250'''
|
|
).fetchall()
|
|
if not rows:
|
|
break
|
|
for row in rows:
|
|
try:
|
|
digest = stored_provider_key_hash(
|
|
row['service'], row['candidate_kind'], row['secret_text'], row['secret_json'],
|
|
row['endpoint'], row['principal'],
|
|
)
|
|
except ValueError as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'keycheck credential {row["id"]} cannot be migrated to provider-canonical identity'
|
|
) from exc
|
|
conn.execute(
|
|
"UPDATE keycheck_credentials SET provider_key_hash = ? WHERE id = ? AND provider_key_hash = ''",
|
|
(digest, row['id']),
|
|
)
|
|
processed += 1
|
|
if processed > max_rows:
|
|
raise RuntimeSafetySchemaError('keycheck credential identity backfill exceeded its row bound')
|
|
|
|
processed = 0
|
|
while True:
|
|
rows = conn.execute(
|
|
'''SELECT c.id,
|
|
(SELECT NULLIF(f.secret_hash, '') FROM findings f WHERE f.id = c.finding_id) AS finding_hash,
|
|
(SELECT NULLIF(r.secret_hash, '') FROM keycheck_results r
|
|
WHERE r.candidate_id = c.id ORDER BY r.id DESC LIMIT 1) AS result_hash,
|
|
kc.provider_key_hash
|
|
FROM keycheck_candidates c
|
|
JOIN keycheck_credentials kc ON kc.id = c.credential_id
|
|
WHERE c.secret_hash = '' ORDER BY c.id LIMIT 250'''
|
|
).fetchall()
|
|
if not rows:
|
|
break
|
|
for row in rows:
|
|
digest = str(row['finding_hash'] or row['result_hash'] or row['provider_key_hash'] or '').lower()
|
|
if not re.fullmatch(r'[a-f0-9]{64}', digest):
|
|
raise RuntimeSafetySchemaError(
|
|
f'keycheck candidate {row["id"]} cannot be migrated to detector secret identity'
|
|
)
|
|
conn.execute(
|
|
"UPDATE keycheck_candidates SET secret_hash = ? WHERE id = ? AND secret_hash = ''",
|
|
(digest, row['id']),
|
|
)
|
|
processed += 1
|
|
if processed > max_rows:
|
|
raise RuntimeSafetySchemaError('keycheck candidate identity backfill exceeded its row bound')
|
|
|
|
processed = 0
|
|
while True:
|
|
rows = conn.execute(
|
|
'''SELECT r.id, kc.provider_key_hash, c.secret_hash
|
|
FROM keycheck_results r
|
|
JOIN keycheck_candidates c ON c.id = r.candidate_id
|
|
JOIN keycheck_credentials kc ON kc.id = c.credential_id
|
|
WHERE COALESCE(r.key_hash, '') != kc.provider_key_hash
|
|
OR COALESCE(r.secret_hash, '') != c.secret_hash
|
|
ORDER BY r.id LIMIT 250'''
|
|
).fetchall()
|
|
if not rows:
|
|
break
|
|
for row in rows:
|
|
conn.execute(
|
|
'UPDATE keycheck_results SET key_hash = ?, secret_hash = ? WHERE id = ?',
|
|
(row['provider_key_hash'], row['secret_hash'], row['id']),
|
|
)
|
|
processed += 1
|
|
if processed > max_rows:
|
|
raise RuntimeSafetySchemaError('keycheck result identity backfill exceeded its row bound')
|
|
|
|
duplicate = conn.execute(
|
|
'''SELECT service, provider_key_hash, COUNT(*) AS count
|
|
FROM keycheck_credentials WHERE provider_key_hash != ''
|
|
GROUP BY service, provider_key_hash HAVING COUNT(*) > 1 LIMIT 1'''
|
|
).fetchone()
|
|
if duplicate:
|
|
raise RuntimeSafetySchemaError(
|
|
'provider-canonical keycheck credentials contain duplicate persisted identities; '
|
|
'reviewed credential merge is required before cutover'
|
|
)
|
|
|
|
|
|
def _migration_backfill_docker_selection_policies(conn, max_reservations=1000000):
|
|
processed = 0
|
|
while True:
|
|
reservations = conn.execute(
|
|
'''SELECT DISTINCT reservation_id
|
|
FROM docker_image_blob_coverage
|
|
WHERE COALESCE(selection_policy_sha256, '') = ?
|
|
ORDER BY reservation_id LIMIT 250''',
|
|
('',),
|
|
).fetchall()
|
|
if not reservations:
|
|
break
|
|
for identity in reservations:
|
|
reservation_id = int(identity['reservation_id'])
|
|
reservation = conn.execute(
|
|
'''SELECT id, docker_layer_plan_json, docker_layer_plan_sha256
|
|
FROM result_reservations WHERE id = ?''',
|
|
(reservation_id,),
|
|
).fetchone()
|
|
if not reservation:
|
|
raise RuntimeSafetySchemaError(
|
|
f'Docker coverage reservation {reservation_id} is absent during selector backfill'
|
|
)
|
|
plan, plan_sha256 = stored_docker_layer_plan(reservation)
|
|
if plan is None:
|
|
raise RuntimeSafetySchemaError(
|
|
f'Docker coverage reservation {reservation_id} has no bound plan'
|
|
)
|
|
coverage_policy_sha256 = docker_layer_plan_coverage_policy_sha256(plan)
|
|
expected = {
|
|
descriptor['position']: descriptor for descriptor in plan['descriptors']
|
|
}
|
|
rows = conn.execute(
|
|
'''SELECT position, blob_digest, coverage_policy_sha256,
|
|
selection_policy_sha256, descriptor_kind, plan_sha256,
|
|
selected, selection_reason
|
|
FROM docker_image_blob_coverage
|
|
WHERE reservation_id = ? ORDER BY position''',
|
|
(reservation_id,),
|
|
).fetchall()
|
|
if len(rows) != len(expected):
|
|
raise RuntimeSafetySchemaError(
|
|
f'Docker coverage reservation {reservation_id} is incomplete during selector backfill'
|
|
)
|
|
empty_count = 0
|
|
for row in rows:
|
|
descriptor = expected.get(int(row['position']))
|
|
existing_selector = str(row['selection_policy_sha256'] or '')
|
|
if (
|
|
not descriptor
|
|
or str(row['blob_digest']) != descriptor['digest']
|
|
or str(row['coverage_policy_sha256']) != coverage_policy_sha256
|
|
or existing_selector not in ('', plan['selection_policy_sha256'])
|
|
or str(row['descriptor_kind']) != descriptor['kind']
|
|
or str(row['plan_sha256']) != plan_sha256
|
|
or bool(row['selected']) != descriptor['selected']
|
|
or str(row['selection_reason']) != descriptor['selection_reason']
|
|
):
|
|
raise RuntimeSafetySchemaError(
|
|
f'Docker coverage reservation {reservation_id} conflicts with its bound plan'
|
|
)
|
|
empty_count += 1 if not existing_selector else 0
|
|
cursor = conn.execute(
|
|
'''UPDATE docker_image_blob_coverage SET selection_policy_sha256 = ?
|
|
WHERE reservation_id = ? AND COALESCE(selection_policy_sha256, '') = ?''',
|
|
(plan['selection_policy_sha256'], reservation_id, ''),
|
|
)
|
|
if int(cursor.rowcount or 0) != empty_count:
|
|
raise RuntimeSafetySchemaError(
|
|
f'Docker coverage reservation {reservation_id} changed during selector backfill'
|
|
)
|
|
processed += 1
|
|
if processed > max_reservations:
|
|
raise RuntimeSafetySchemaError(
|
|
'Docker selection-policy backfill exceeded its reservation bound'
|
|
)
|
|
remaining = conn.execute(
|
|
'''SELECT reservation_id FROM docker_image_blob_coverage
|
|
WHERE COALESCE(selection_policy_sha256, '') = ? LIMIT 1''',
|
|
('',),
|
|
).fetchone()
|
|
if remaining:
|
|
raise RuntimeSafetySchemaError('Docker selection-policy backfill is incomplete')
|
|
|
|
|
|
def _pipeline_quarantine_review_status_constraint(conn):
|
|
if conn.is_postgres:
|
|
rows = conn.execute(
|
|
'''SELECT con.conname AS name,
|
|
pg_catalog.pg_get_constraintdef(con.oid, true) AS definition,
|
|
con.convalidated AS is_valid
|
|
FROM pg_catalog.pg_constraint con
|
|
JOIN pg_catalog.pg_class tbl ON tbl.oid = con.conrelid
|
|
JOIN pg_catalog.pg_namespace n ON n.oid = tbl.relnamespace
|
|
WHERE con.contype = 'c' AND n.nspname = ?
|
|
AND tbl.relname = 'pipeline_quarantine' ''',
|
|
(conn.application_schema,),
|
|
).fetchall()
|
|
matching = [
|
|
row for row in rows
|
|
if 'review_status' in str(row['definition'] or '').lower()
|
|
]
|
|
if len(matching) != 1:
|
|
return {'name': '', 'statuses': None, 'valid': False}
|
|
row = matching[0]
|
|
statuses = frozenset(re.findall(r"'([^']+)'", str(row['definition'] or '')))
|
|
return {
|
|
'name': str(row['name'] or ''),
|
|
'statuses': statuses,
|
|
'valid': bool(row['is_valid']),
|
|
}
|
|
row = conn.execute(
|
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'pipeline_quarantine'"
|
|
).fetchone()
|
|
sql = str(row['sql'] if row else '')
|
|
match = re.search(
|
|
r'review_status\s+TEXT.*?CHECK\s*\(\s*review_status\s+IN\s*\((.*?)\)\s*\)',
|
|
sql, re.IGNORECASE | re.DOTALL,
|
|
)
|
|
statuses = frozenset(re.findall(r"'([^']+)'", match.group(1))) if match else None
|
|
return {'name': '', 'statuses': statuses, 'valid': statuses is not None}
|
|
|
|
|
|
def _migration_ensure_pipeline_quarantine_review_status_constraint(conn):
|
|
constraint = _pipeline_quarantine_review_status_constraint(conn)
|
|
statuses = constraint.get('statuses')
|
|
if statuses == PIPELINE_QUARANTINE_REVIEW_STATUSES:
|
|
if conn.is_postgres and not constraint.get('valid'):
|
|
conn.execute(
|
|
'ALTER TABLE pipeline_quarantine VALIDATE CONSTRAINT '
|
|
+ _quoted_pg_name(constraint['name'])
|
|
)
|
|
return
|
|
if not conn.is_postgres:
|
|
raise RuntimeSafetySchemaError(
|
|
'SQLite pipeline_quarantine review-status constraint requires a reviewed table rebuild'
|
|
)
|
|
if statuses not in (None, PIPELINE_QUARANTINE_LEGACY_REVIEW_STATUSES):
|
|
raise RuntimeSafetySchemaError(
|
|
'PostgreSQL pipeline_quarantine has an unexpected review-status constraint; '
|
|
'manual reviewed repair is required'
|
|
)
|
|
if statuses is not None:
|
|
conn.execute(
|
|
'ALTER TABLE pipeline_quarantine DROP CONSTRAINT '
|
|
+ _quoted_pg_name(constraint['name'])
|
|
)
|
|
conn.execute(
|
|
'''ALTER TABLE pipeline_quarantine
|
|
ADD CONSTRAINT pipeline_quarantine_review_status_check
|
|
CHECK (review_status IN (
|
|
'pending','approved_retry','approved_rescan','discarded','resolved'
|
|
))'''
|
|
)
|
|
|
|
|
|
def _migration_require_pipeline_quiescence(conn):
|
|
if not conn.table_exists('runtime_schema_migrations'):
|
|
return
|
|
applied = {
|
|
str(row['version']) for row in conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations'
|
|
).fetchall()
|
|
}
|
|
if set(PIPELINE_MIGRATION_VERSIONS).issubset(applied) or not all(
|
|
conn.table_exists(table) for table in (
|
|
'pipeline_leases', 'result_reservations', 'target_queue', 'docker_content_blobs',
|
|
)
|
|
):
|
|
return
|
|
capacity_backfill_only = (
|
|
set(PIPELINE_MIGRATION_VERSIONS[:-1]).issubset(applied)
|
|
and REMOTE_ASSIGNMENT_CAPACITY_MIGRATION not in applied
|
|
)
|
|
active = {
|
|
'worker_leases': conn.execute(
|
|
"SELECT COUNT(*) AS count FROM pipeline_leases WHERE state NOT IN ('released','failed')"
|
|
).fetchone()['count'],
|
|
'result_reservations': 0 if capacity_backfill_only else conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM result_reservations
|
|
WHERE state IN ('scanning','ready','ingesting','db_committed')"""
|
|
).fetchone()['count'],
|
|
'queue_leases': 0 if capacity_backfill_only else conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM target_queue q
|
|
LEFT JOIN result_reservations r ON r.id = q.current_result_reservation_id
|
|
WHERE q.status = 'in_progress'
|
|
OR r.state IN ('scanning','ready','ingesting','db_committed')"""
|
|
).fetchone()['count'],
|
|
'blob_leases': conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM docker_content_blobs
|
|
WHERE state IN ('leased','submitted') OR lease_reservation_id IS NOT NULL"""
|
|
).fetchone()['count'],
|
|
}
|
|
if conn.table_exists('discovery_retry_queue'):
|
|
active['discovery_retry_leases'] = conn.execute(
|
|
"SELECT COUNT(*) AS count FROM discovery_retry_queue WHERE status = 'leased'"
|
|
).fetchone()['count']
|
|
queue_columns = set(conn.table_columns('target_queue'))
|
|
if {'resolver_state', 'resolver_token'} <= queue_columns:
|
|
active['docker_resolvers'] = conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM target_queue
|
|
WHERE resolver_state = 'resolving' OR resolver_token IS NOT NULL"""
|
|
).fetchone()['count']
|
|
if conn.table_exists('docker_depth_experiments'):
|
|
active['experiment_authority_fences'] = conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM docker_depth_experiments
|
|
WHERE fence_token IS NOT NULL
|
|
OR state IN ('holding','resolving','active','draining')"""
|
|
).fetchone()['count']
|
|
if conn.table_exists('docker_depth_experiment_repositories'):
|
|
active['experiment_resolvers'] = conn.execute(
|
|
"""SELECT COUNT(*) AS count FROM docker_depth_experiment_repositories
|
|
WHERE work_state = 'resolving' OR resolver_token IS NOT NULL"""
|
|
).fetchone()['count']
|
|
if (
|
|
conn.table_exists('target_queue_policy_events')
|
|
and 'experiment_id' in conn.table_columns('target_queue_policy_events')
|
|
):
|
|
active['experiment_holds'] = conn.execute(
|
|
"""SELECT COUNT(*) AS count
|
|
FROM target_queue_policy_events event
|
|
LEFT JOIN docker_depth_experiments experiment
|
|
ON experiment.id = event.experiment_id
|
|
WHERE event.experiment_id IS NOT NULL AND event.action = 'cold'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM target_queue_policy_events reversal
|
|
WHERE reversal.reverses_event_id = event.id
|
|
)
|
|
AND (
|
|
experiment.id IS NULL
|
|
OR experiment.state NOT IN ('held','completed','released')
|
|
OR experiment.fence_token IS NOT NULL
|
|
)"""
|
|
).fetchone()['count']
|
|
if any(int(value or 0) for value in active.values()):
|
|
summary = ', '.join(
|
|
f'{name}={int(value or 0)}' for name, value in active.items()
|
|
if int(value or 0)
|
|
)
|
|
raise RuntimeSafetySchemaError(
|
|
f'pipeline schema migration requires stopped, reconciled runtime ({summary})'
|
|
)
|
|
|
|
|
|
def _migration_postgres_column_shape(conn, table, specs):
|
|
if not conn.is_postgres:
|
|
return
|
|
type_sql = {'id': 'BIGINT', 'id_ref': 'BIGINT', 'integer': 'INTEGER', 'text': 'TEXT', 'real': 'REAL'}
|
|
details = conn.table_column_details(table)
|
|
for name, (expected_type, expected_not_null) in specs.items():
|
|
actual = details.get(name)
|
|
if not actual:
|
|
raise RuntimeSafetySchemaError(f'migration did not create {table}.{name}')
|
|
if not _schema_type_matches(actual['type'], expected_type, True):
|
|
target_type = type_sql[expected_type]
|
|
actual_type = re.sub(r'\s+', ' ', str(actual['type'] or '').strip().lower())
|
|
safe_sources = {
|
|
'BIGINT': {'integer', 'smallint'},
|
|
'INTEGER': {'smallint'},
|
|
'TEXT': {'character varying', 'character', 'varchar'},
|
|
'REAL': set(),
|
|
}
|
|
if actual_type not in safe_sources[target_type]:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{name} type {actual_type or "<empty>"} cannot be safely '
|
|
f'changed to {target_type}; manual reviewed conversion is required'
|
|
)
|
|
conn.execute(
|
|
f'ALTER TABLE {table} ALTER COLUMN {name} TYPE {target_type} USING {name}::{target_type}'
|
|
)
|
|
if expected_not_null and not actual['not_null']:
|
|
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} SET NOT NULL')
|
|
elif not expected_not_null and actual['not_null'] and not actual['primary_key']:
|
|
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} DROP NOT NULL')
|
|
|
|
|
|
def _migration_require_sqlite_column_shape(conn, table, specs):
|
|
if not conn.is_sqlite:
|
|
return
|
|
details = conn.table_column_details(table)
|
|
problems = []
|
|
for name, (expected_type, expected_not_null) in specs.items():
|
|
actual = details.get(name)
|
|
if not actual:
|
|
problems.append(f'missing {name}')
|
|
continue
|
|
if not _schema_type_matches(actual['type'], expected_type, False):
|
|
problems.append(f'{name} type {actual["type"] or "<empty>"}')
|
|
if bool(actual['not_null']) != bool(expected_not_null):
|
|
problems.append(f'{name} nullability')
|
|
if problems:
|
|
raise RuntimeSafetySchemaError(
|
|
f'SQLite table {table} has a non-additively-repairable schema ({"; ".join(problems)}); '
|
|
'manually rebuild this table from a reviewed backup with the exact runtime schema'
|
|
)
|
|
|
|
|
|
def _migration_ensure_defaults(conn, table, specs, defaults, generated_id=None):
|
|
details = conn.table_column_details(table)
|
|
for name in specs:
|
|
if name == generated_id:
|
|
continue
|
|
expected_present = name in defaults
|
|
expected = defaults.get(name, '')
|
|
actual = details.get(name)
|
|
actual_present = bool(actual and actual.get('has_default', bool(actual.get('default'))))
|
|
if actual and actual_present == expected_present and (
|
|
not expected_present or _normalized_default(actual['default']) == _normalized_default(expected)
|
|
):
|
|
continue
|
|
if conn.is_sqlite:
|
|
raise RuntimeSafetySchemaError(
|
|
f'SQLite table {table}.{name} has default {actual["default"] or "<none>"}; '
|
|
'manually rebuild the table with the exact runtime default'
|
|
)
|
|
if not expected_present:
|
|
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} DROP DEFAULT')
|
|
else:
|
|
default_sql = str(int(expected)) if str(expected).isdigit() else "'" + str(expected).replace("'", "''") + "'"
|
|
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {name} SET DEFAULT {default_sql}')
|
|
|
|
|
|
def _migration_require_primary_key(conn, table, specs, expected):
|
|
details = conn.table_column_details(table)
|
|
actual = [name for name in specs if details.get(name, {}).get('primary_key')]
|
|
if actual != list(expected):
|
|
raise RuntimeSafetySchemaError(
|
|
f'{table} primary key is {actual or "absent"}, expected {list(expected)}; '
|
|
'manual reviewed table rebuild is required'
|
|
)
|
|
if conn.is_postgres:
|
|
primary_indexes = [
|
|
index for index in conn.table_indexes(table).values()
|
|
if index.get('primary')
|
|
]
|
|
if (
|
|
len(primary_indexes) != 1
|
|
or primary_indexes[0].get('columns') != list(expected)
|
|
or not _index_usable(primary_indexes[0])
|
|
):
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table} primary-key index is absent, invalid, or unready; '
|
|
'manual reviewed primary-key constraint rebuild is required'
|
|
)
|
|
|
|
|
|
def _quoted_pg_name(value):
|
|
parts = str(value or '').split('.')
|
|
if not parts or any(not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part) for part in parts):
|
|
raise RuntimeSafetySchemaError(f'unsafe PostgreSQL catalog identifier: {value!r}')
|
|
return '.'.join('"' + part.replace('"', '""') + '"' for part in parts)
|
|
|
|
|
|
def _migration_ensure_generated_id(conn, table, column):
|
|
details = conn.table_column_details(table).get(column)
|
|
if _column_generates_id(details, conn.is_postgres):
|
|
return
|
|
if not conn.is_postgres:
|
|
raise RuntimeSafetySchemaError(
|
|
f'SQLite {table}.{column} does not auto-generate an INTEGER PRIMARY KEY; '
|
|
'manually rebuild this table from a reviewed backup'
|
|
)
|
|
if not details or details.get('type') != 'bigint' or not details.get('primary_key'):
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{column} is not a BIGINT PRIMARY KEY; manual reviewed table repair is required'
|
|
)
|
|
if details.get('identity') or details.get('generated') or details.get('default') or details.get('sequence'):
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{column} has malformed identity/sequence metadata; '
|
|
'manually repair its generated-ID definition before retrying'
|
|
)
|
|
try:
|
|
conn.execute(f'ALTER TABLE {table} ALTER COLUMN {column} ADD GENERATED BY DEFAULT AS IDENTITY')
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{column} cannot be safely converted to an identity column; '
|
|
'manually add a generated sequence/identity and reseed it above MAX(id)'
|
|
) from exc
|
|
repaired = conn.table_column_details(table).get(column) or {}
|
|
sequence = repaired.get('sequence')
|
|
if not _column_generates_id(repaired, True) or not sequence:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL did not create a usable identity sequence for {table}.{column}; '
|
|
'manual identity repair is required'
|
|
)
|
|
row = conn.execute(f'SELECT COALESCE(MAX({column}), 0) + 1 AS next_id FROM {table}').fetchone()
|
|
next_id = max(1, int(row['next_id'] if row else 1))
|
|
try:
|
|
conn.execute(f'ALTER SEQUENCE {_quoted_pg_name(sequence)} RESTART WITH {next_id}')
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL identity sequence {sequence} could not be reseeded to {next_id}; '
|
|
f'manually restart it above MAX({table}.{column})'
|
|
) from exc
|
|
|
|
|
|
def _migration_add_docker_depth_schema_authority_checks(conn, marker_applied):
|
|
if not conn.is_postgres or marker_applied:
|
|
return
|
|
for table, name in DOCKER_DEPTH_SCHEMA_AUTHORITY_CHECKS:
|
|
if name in conn.table_check_constraints(table):
|
|
continue
|
|
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
|
|
try:
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
|
|
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
|
|
)
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
|
|
f'{_quoted_pg_name(name)}'
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{name} cannot be validated against existing rows; '
|
|
'manually repair malformed safety data before retrying migration'
|
|
) from exc
|
|
|
|
|
|
def _migration_add_docker_depth_scarcity_checks(conn, marker_applied):
|
|
if not conn.is_postgres or marker_applied:
|
|
return
|
|
for table, name in DOCKER_DEPTH_SCARCITY_COHORT_CHECKS:
|
|
if name in conn.table_check_constraints(table):
|
|
continue
|
|
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
|
|
try:
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
|
|
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
|
|
)
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
|
|
f'{_quoted_pg_name(name)}'
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{name} cannot be validated against existing rows; '
|
|
'manually repair malformed scarcity cohort data before retrying migration'
|
|
) from exc
|
|
|
|
|
|
def _migration_require_docker_depth_check_constraints(conn):
|
|
problems = _docker_depth_check_constraint_problems(conn)
|
|
prior_reviewed = {
|
|
('docker_depth_experiments', 'docker_depth_experiments_range_check'): (
|
|
'query_count >= 1 AND query_count <= 1000 '
|
|
'AND repositories_per_query >= 1 AND repositories_per_query <= 10 '
|
|
'AND images_per_repository >= 1 AND images_per_repository <= 10 '
|
|
'AND target_limit >= 1 AND target_limit <= 1200 '
|
|
'AND target_count >= 0 AND target_count <= target_limit '
|
|
'AND selection_count >= 0 AND fence_generation >= 0'
|
|
),
|
|
('docker_depth_experiments', 'docker_depth_experiments_capacity_check'): (
|
|
'query_count * (repositories_per_query + images_per_repository - 1) '
|
|
'<= target_limit'
|
|
),
|
|
(
|
|
'docker_depth_experiment_queries',
|
|
'docker_depth_experiment_queries_range_check',
|
|
): (
|
|
'query_ordinal >= 0 AND required_repository_count >= 1 '
|
|
'AND required_repository_count <= 10'
|
|
),
|
|
(
|
|
'docker_depth_experiment_repositories',
|
|
'docker_depth_experiment_repositories_range_check',
|
|
): (
|
|
'query_ordinal >= 0 AND repository_rank >= 1 AND repository_rank <= 10 '
|
|
'AND resolver_generation >= 0 AND resolver_attempts >= 0 '
|
|
'AND selected_image_count >= 0 AND selected_image_count <= 10'
|
|
),
|
|
(
|
|
'docker_depth_experiment_targets',
|
|
'docker_depth_experiment_targets_range_check',
|
|
): (
|
|
'counter_ordinal >= 1 AND counter_ordinal <= 1200 '
|
|
'AND dispatch_wave >= 1 AND dispatch_wave <= 3 '
|
|
'AND dispatch_order >= 1 AND reservation_count >= 0'
|
|
),
|
|
}
|
|
if conn.is_postgres and problems:
|
|
for table, name in prior_reviewed:
|
|
label = f'check constraint {table}.{name}'
|
|
if label not in problems:
|
|
continue
|
|
constraint = conn.table_check_constraints(table).get(name)
|
|
if (
|
|
not constraint
|
|
or not constraint.get('valid', True)
|
|
or _normalized_check_expression(constraint.get('expression'))
|
|
!= _normalized_check_expression(prior_reviewed[(table, name)])
|
|
):
|
|
continue
|
|
expression = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[table][name]
|
|
try:
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} '
|
|
f'DROP CONSTRAINT {_quoted_pg_name(name)}'
|
|
)
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} ADD CONSTRAINT '
|
|
f'{_quoted_pg_name(name)} CHECK ({expression}) NOT VALID'
|
|
)
|
|
conn.execute(
|
|
f'ALTER TABLE {_quoted_pg_name(table)} VALIDATE CONSTRAINT '
|
|
f'{_quoted_pg_name(name)}'
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table}.{name} reviewed widening failed validation'
|
|
) from exc
|
|
problems = _docker_depth_check_constraint_problems(conn)
|
|
if problems:
|
|
raise RuntimeSafetySchemaError(
|
|
'Docker depth experiment schema has missing, invalid, or noncanonical '
|
|
+ '; '.join(problems)
|
|
+ '; only the exact prior reviewed CHECK constraints can be widened'
|
|
)
|
|
|
|
|
|
def _migration_ensure_foreign_keys(conn):
|
|
for table, expected_keys in REQUIRED_FOREIGN_KEYS.items():
|
|
foreign_keys = conn.table_foreign_keys(table)
|
|
for columns, referenced_table, referenced_columns in expected_keys:
|
|
authority_matching = [
|
|
(name, foreign_key)
|
|
for name, foreign_key in foreign_keys.items()
|
|
if tuple(foreign_key.get('columns') or ()) == columns
|
|
and foreign_key.get('referenced_table') == referenced_table
|
|
and tuple(foreign_key.get('referenced_columns') or ()) == referenced_columns
|
|
and foreign_key.get('referenced_schema') in (
|
|
conn.application_schema if conn.is_postgres else 'main',
|
|
)
|
|
]
|
|
if len(authority_matching) > 1:
|
|
raise RuntimeSafetySchemaError(
|
|
f'{table} has duplicate foreign keys for ({", ".join(columns)}); '
|
|
'manual reviewed constraint cleanup is required'
|
|
)
|
|
expected_actions = _required_foreign_key_actions(table, columns)
|
|
if authority_matching and not _foreign_key_actions_match(
|
|
authority_matching[0][1], expected_actions,
|
|
):
|
|
raise RuntimeSafetySchemaError(
|
|
f'{table}({", ".join(columns)}) foreign key actions are not '
|
|
f'ON UPDATE {expected_actions[0]} ON DELETE {expected_actions[1]}; '
|
|
'manual reviewed constraint repair is required'
|
|
)
|
|
if authority_matching:
|
|
name, foreign_key = authority_matching[0]
|
|
if conn.is_postgres and not foreign_key.get('valid', True):
|
|
try:
|
|
conn.execute(f'ALTER TABLE {table} VALIDATE CONSTRAINT {_quoted_pg_name(name)}')
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table} foreign key {name} cannot be validated against existing rows; '
|
|
'manually repair orphaned references before retrying migration'
|
|
) from exc
|
|
continue
|
|
conflicting = [
|
|
foreign_key for foreign_key in foreign_keys.values()
|
|
if tuple(foreign_key.get('columns') or ()) == columns
|
|
]
|
|
if conflicting:
|
|
raise RuntimeSafetySchemaError(
|
|
f'{table}({", ".join(columns)}) has a foreign key to the wrong authority; '
|
|
'manual reviewed constraint repair is required'
|
|
)
|
|
if not conn.is_postgres:
|
|
raise RuntimeSafetySchemaError(
|
|
f'SQLite {table} is missing foreign key ({", ".join(columns)}) -> '
|
|
f'{referenced_table}({", ".join(referenced_columns)}); '
|
|
'manually rebuild this table from a reviewed backup'
|
|
)
|
|
name = f'fk_{table}_{"_".join(columns)}'
|
|
try:
|
|
conn.execute(
|
|
f'ALTER TABLE {table} ADD CONSTRAINT {name} '
|
|
f'FOREIGN KEY ({", ".join(columns)}) REFERENCES '
|
|
f'{POSTGRES_APPLICATION_SCHEMA}.{referenced_table} ({", ".join(referenced_columns)}) '
|
|
f'ON UPDATE {expected_actions[0]} ON DELETE {expected_actions[1]}'
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
f'PostgreSQL {table} cannot add foreign key ({", ".join(columns)}) because '
|
|
'existing rows do not match the referenced authority; manually repair orphaned rows'
|
|
) from exc
|
|
foreign_keys = conn.table_foreign_keys(table)
|
|
|
|
|
|
def _migration_ensure_index(
|
|
conn, table, name, sql, columns, unique=False, predicate='', required_sql_fragments=(),
|
|
):
|
|
current = conn.table_indexes(table).get(name)
|
|
current_sql = str((current or {}).get('sql') or '').lower()
|
|
valid = bool(
|
|
current
|
|
and bool(current['unique']) == bool(unique)
|
|
and current['columns'] == list(columns)
|
|
and _normalized_predicate(current['predicate']) == _normalized_predicate(predicate)
|
|
and _index_usable(current)
|
|
and all(str(fragment).lower() in current_sql for fragment in required_sql_fragments)
|
|
)
|
|
if valid:
|
|
return
|
|
if current:
|
|
conn.execute(f'DROP INDEX {name}')
|
|
conn.execute(sql)
|
|
|
|
|
|
def _ensure_runtime_operations_authority(conn, now=None):
|
|
timestamp = now or utc_now_iso()
|
|
conn.execute(
|
|
'''INSERT INTO runtime_operations_control(
|
|
id, revision, discovery_paused, dispatch_paused, drain_state,
|
|
actor, operation_id, created_at, updated_at
|
|
) VALUES (1, 0, 0, 0, 'normal', 'system:migration', NULL, ?, ?)
|
|
ON CONFLICT(id) DO NOTHING''',
|
|
(timestamp, timestamp),
|
|
)
|
|
if conn.is_postgres:
|
|
conn.execute('''
|
|
CREATE OR REPLACE FUNCTION reject_runtime_audit_event_mutation()
|
|
RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
RAISE EXCEPTION 'runtime_audit_events is append-only';
|
|
END;
|
|
$$
|
|
''')
|
|
conn.execute(
|
|
'DROP TRIGGER IF EXISTS runtime_audit_events_reject_mutation '
|
|
'ON runtime_audit_events'
|
|
)
|
|
conn.execute('''
|
|
CREATE TRIGGER runtime_audit_events_reject_mutation
|
|
BEFORE UPDATE OR DELETE ON runtime_audit_events
|
|
FOR EACH ROW EXECUTE FUNCTION reject_runtime_audit_event_mutation()
|
|
''')
|
|
conn.execute(
|
|
'DROP TRIGGER IF EXISTS runtime_audit_events_reject_truncate '
|
|
'ON runtime_audit_events'
|
|
)
|
|
conn.execute('''
|
|
CREATE TRIGGER runtime_audit_events_reject_truncate
|
|
BEFORE TRUNCATE ON runtime_audit_events
|
|
FOR EACH STATEMENT EXECUTE FUNCTION reject_runtime_audit_event_mutation()
|
|
''')
|
|
return
|
|
conn.execute('DROP TRIGGER IF EXISTS runtime_audit_events_reject_update')
|
|
conn.execute('''
|
|
CREATE TRIGGER runtime_audit_events_reject_update
|
|
BEFORE UPDATE ON runtime_audit_events
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'runtime_audit_events is append-only');
|
|
END
|
|
''')
|
|
conn.execute('DROP TRIGGER IF EXISTS runtime_audit_events_reject_delete')
|
|
conn.execute('''
|
|
CREATE TRIGGER runtime_audit_events_reject_delete
|
|
BEFORE DELETE ON runtime_audit_events
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'runtime_audit_events is append-only');
|
|
END
|
|
''')
|
|
|
|
|
|
def _migration_retain_pg_trgm_without_name_index(conn):
|
|
if not conn.is_postgres:
|
|
return
|
|
extension = conn.execute(
|
|
'''SELECT EXISTS (
|
|
SELECT 1 FROM pg_catalog.pg_available_extensions WHERE name = ?
|
|
) AS available''',
|
|
('pg_trgm',),
|
|
).fetchone()
|
|
if extension and extension['available']:
|
|
conn.execute('SAVEPOINT package_candidate_trgm')
|
|
try:
|
|
conn.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm')
|
|
conn.execute('RELEASE SAVEPOINT package_candidate_trgm')
|
|
except Exception as exc:
|
|
conn.execute('ROLLBACK TO SAVEPOINT package_candidate_trgm')
|
|
conn.execute('RELEASE SAVEPOINT package_candidate_trgm')
|
|
logger.warning(
|
|
'pg_trgm extension retention failed; bounded ordered lookup remains active: %s',
|
|
type(exc).__name__,
|
|
)
|
|
conn.execute('DROP INDEX IF EXISTS idx_package_repo_candidates_name_trgm')
|
|
|
|
|
|
def _migration_reconcile_pipeline_capacity(conn, page_size=1000):
|
|
totals = {
|
|
'bundle_items': 0, 'bundle_bytes': 0,
|
|
'projection_items': 0, 'projection_bytes': 0,
|
|
'keycheck_items': 0, 'keycheck_bytes': 0,
|
|
'quarantine_items': 0, 'quarantine_bytes': 0,
|
|
}
|
|
|
|
def pages(table, columns, where):
|
|
after_id = 0
|
|
while True:
|
|
rows = conn.execute(
|
|
f'''SELECT id, {columns} FROM {table}
|
|
WHERE id > ? AND ({where}) ORDER BY id LIMIT ?''',
|
|
(after_id, max(1, int(page_size))),
|
|
).fetchall()
|
|
if not rows:
|
|
return
|
|
for row in rows:
|
|
after_id = int(row['id'])
|
|
yield row
|
|
|
|
for row in pages(
|
|
'result_reservations',
|
|
'''reserved_bundle_bytes, reserved_projection_items, reserved_projection_bytes,
|
|
reserved_candidate_items, reserved_candidate_bytes, bundle_credit_released,
|
|
projection_credit_transferred, candidate_credit_transferred''',
|
|
"state IN ('scanning','ready','ingesting','db_committed')",
|
|
):
|
|
if not row['bundle_credit_released']:
|
|
totals['bundle_items'] += 1
|
|
totals['bundle_bytes'] += int(row['reserved_bundle_bytes'])
|
|
if not row['projection_credit_transferred']:
|
|
totals['projection_items'] += int(row['reserved_projection_items'])
|
|
totals['projection_bytes'] += int(row['reserved_projection_bytes'])
|
|
if not row['candidate_credit_transferred']:
|
|
totals['keycheck_items'] += int(row['reserved_candidate_items'])
|
|
totals['keycheck_bytes'] += int(row['reserved_candidate_bytes'])
|
|
for row in pages(
|
|
'projection_jobs', 'capacity_items, capacity_bytes',
|
|
"status IN ('pending','leased') AND capacity_released = 0",
|
|
):
|
|
totals['projection_items'] += int(row['capacity_items'])
|
|
totals['projection_bytes'] += int(row['capacity_bytes'])
|
|
for row in pages(
|
|
'keycheck_candidates',
|
|
'capacity_bytes, result_projection_reserved_bytes, result_projection_credit_transferred',
|
|
"state IN ('pending','leased','deferred') AND capacity_released = 0",
|
|
):
|
|
totals['keycheck_items'] += 1
|
|
totals['keycheck_bytes'] += int(row['capacity_bytes'])
|
|
if (
|
|
int(row['result_projection_reserved_bytes'] or 0) > 0
|
|
and not row['result_projection_credit_transferred']
|
|
):
|
|
totals['projection_items'] += 1
|
|
totals['projection_bytes'] += int(row['result_projection_reserved_bytes'])
|
|
for row in pages(
|
|
'pipeline_quarantine', 'capacity_items, capacity_bytes',
|
|
"review_status = 'pending' AND capacity_credit_applied = 1",
|
|
):
|
|
totals['quarantine_items'] += int(row['capacity_items'])
|
|
totals['quarantine_bytes'] += int(row['capacity_bytes'])
|
|
conn.execute(
|
|
'''UPDATE pipeline_capacity SET bundle_items = ?, bundle_bytes = ?,
|
|
projection_items = ?, projection_bytes = ?, keycheck_items = ?,
|
|
keycheck_bytes = ?, quarantine_items = ?, quarantine_bytes = ?,
|
|
updated_at = ? WHERE id = 1''',
|
|
(
|
|
totals['bundle_items'], totals['bundle_bytes'], totals['projection_items'],
|
|
totals['projection_bytes'], totals['keycheck_items'], totals['keycheck_bytes'],
|
|
totals['quarantine_items'], totals['quarantine_bytes'], utc_now_iso(),
|
|
),
|
|
)
|
|
return totals
|
|
|
|
|
|
def migrate_runtime_safety_schema(db, initialize_base=False):
|
|
conn = getattr(db, 'conn', None)
|
|
if not conn:
|
|
raise RuntimeSafetySchemaError('database connection is unavailable')
|
|
try:
|
|
if initialize_base:
|
|
conn.executescript(SCHEMA_SQL)
|
|
for table in ('target_queue', 'target_scans', 'findings'):
|
|
if not conn.table_exists(table):
|
|
raise RuntimeSafetySchemaError(f'base schema table is missing: {table}')
|
|
_migration_require_pipeline_quiescence(conn)
|
|
|
|
_migration_add_columns(conn, 'target_queue', {
|
|
'lease_token': 'TEXT',
|
|
'claim_batch': 'TEXT',
|
|
'resolver_state': 'TEXT',
|
|
'resolver_due_at': 'TEXT',
|
|
'resolver_attempts': 'INTEGER NOT NULL DEFAULT 0',
|
|
'resolver_token': 'TEXT',
|
|
'current_result_reservation_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
|
|
'claim_event_id': 'TEXT',
|
|
'remote_modified_at': 'TEXT',
|
|
'scan_remote_modified_at': 'TEXT',
|
|
'covered_ref': 'TEXT',
|
|
'covered_head': 'TEXT',
|
|
})
|
|
|
|
id_type = 'BIGINT' if conn.is_postgres else 'INTEGER'
|
|
_migration_add_columns(conn, 'target_scans', {
|
|
'scan_event_id': 'TEXT',
|
|
'scan_event_hash': 'TEXT',
|
|
'queue_id': id_type,
|
|
'claim_lease_token': 'TEXT',
|
|
'queue_completion_applied': 'INTEGER NOT NULL DEFAULT 0',
|
|
'queue_completion_disposition': 'TEXT',
|
|
'result_reservation_id': id_type,
|
|
'compat_schema_version': 'INTEGER NOT NULL DEFAULT 2',
|
|
'raw_result_storage': "TEXT NOT NULL DEFAULT 'legacy'",
|
|
})
|
|
|
|
_migration_add_columns(conn, 'findings', {
|
|
'detector_type': 'TEXT',
|
|
'verified': 'INTEGER DEFAULT 0',
|
|
'raw_secret': 'TEXT',
|
|
'redacted_secret': 'TEXT',
|
|
'secret_hash': 'TEXT',
|
|
'finding_uid': 'TEXT',
|
|
'detector_secret_hash': 'TEXT',
|
|
'finding_fingerprint': 'TEXT',
|
|
'file_path': 'TEXT',
|
|
'line_number': 'TEXT',
|
|
'commit_hash': 'TEXT',
|
|
'source_timestamp': 'TEXT',
|
|
'source_metadata_type': 'TEXT',
|
|
'raw_finding_json': 'TEXT',
|
|
'source_metadata_json': 'TEXT',
|
|
'provider': 'TEXT',
|
|
'credential_kind': 'TEXT',
|
|
'credential_confidence': 'TEXT',
|
|
'required_context_missing': 'INTEGER DEFAULT 0',
|
|
'principal': 'TEXT',
|
|
'username': 'TEXT',
|
|
'email': 'TEXT',
|
|
'project_id': 'TEXT',
|
|
'tenant_id': 'TEXT',
|
|
'organization': 'TEXT',
|
|
'registry': 'TEXT',
|
|
'endpoint': 'TEXT',
|
|
'scope': 'TEXT',
|
|
'resource': 'TEXT',
|
|
'enrichment_json': 'TEXT',
|
|
'raw_payload_sha256': 'TEXT',
|
|
'raw_payload_bytes': id_type,
|
|
'raw_payload_omitted': 'INTEGER NOT NULL DEFAULT 0',
|
|
})
|
|
_migration_add_columns(conn, 'runs', {
|
|
'total_staged': 'INTEGER NOT NULL DEFAULT 0',
|
|
'total_quarantined': 'INTEGER NOT NULL DEFAULT 0',
|
|
})
|
|
_migration_add_columns(conn, 'source_cycles', {
|
|
'queued_updated_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
'staged_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
'ingested_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
'quarantined_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
})
|
|
conn.execute(f'''CREATE TABLE IF NOT EXISTS finding_uid_map (
|
|
finding_uid TEXT PRIMARY KEY,
|
|
finding_id {id_type} NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id)
|
|
)''')
|
|
_migration_postgres_column_shape(conn, 'finding_uid_map', {
|
|
'finding_uid': ('text', True), 'finding_id': ('id_ref', True), 'created_at': ('text', True),
|
|
})
|
|
|
|
outbox_id = 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' if conn.is_postgres else 'INTEGER PRIMARY KEY AUTOINCREMENT'
|
|
conn.execute(f'''CREATE TABLE IF NOT EXISTS scan_publication_outbox (
|
|
id {outbox_id},
|
|
target_scan_id {id_type} NOT NULL UNIQUE,
|
|
payload_json TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
attempts INTEGER DEFAULT 0,
|
|
last_error TEXT,
|
|
lease_owner TEXT,
|
|
lease_expires_at TEXT,
|
|
available_after TEXT,
|
|
created_at TEXT NOT NULL,
|
|
delivered_at TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
)''')
|
|
_migration_add_columns(conn, 'scan_publication_outbox', {
|
|
'payload_json': "TEXT NOT NULL DEFAULT ''",
|
|
'status': "TEXT NOT NULL DEFAULT 'pending'",
|
|
'attempts': 'INTEGER DEFAULT 0',
|
|
'last_error': 'TEXT',
|
|
'lease_owner': 'TEXT',
|
|
'lease_expires_at': 'TEXT',
|
|
'available_after': 'TEXT',
|
|
'created_at': "TEXT NOT NULL DEFAULT ''",
|
|
'delivered_at': 'TEXT',
|
|
'updated_at': "TEXT NOT NULL DEFAULT ''",
|
|
})
|
|
_migration_postgres_column_shape(conn, 'scan_publication_outbox', OUTBOX_COLUMN_SPECS)
|
|
compact_delivered_scan_publications(conn)
|
|
conn.execute(
|
|
'''UPDATE scan_publication_outbox SET payload_json = '',
|
|
status = CASE WHEN status = 'delivering' THEN 'delivering' ELSE 'pending' END,
|
|
lease_owner = CASE WHEN status = 'delivering' THEN lease_owner ELSE NULL END,
|
|
lease_expires_at = CASE WHEN status = 'delivering' THEN lease_expires_at ELSE NULL END,
|
|
available_after = CASE WHEN status = 'dead' THEN NULL ELSE available_after END
|
|
WHERE payload_json != '' OR status NOT IN ('pending', 'delivering')'''
|
|
)
|
|
|
|
cursor_integer = 'BIGINT' if conn.is_postgres else 'INTEGER'
|
|
conn.execute(f'''CREATE TABLE IF NOT EXISTS target_queue_reconciliation_cursors (
|
|
source_file TEXT PRIMARY KEY,
|
|
file_identity TEXT NOT NULL,
|
|
file_size {cursor_integer} NOT NULL DEFAULT 0,
|
|
file_mtime_ns {cursor_integer} NOT NULL DEFAULT 0,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
byte_offset {cursor_integer} NOT NULL DEFAULT 0,
|
|
line_number {cursor_integer} NOT NULL DEFAULT 0,
|
|
discarding_oversized INTEGER NOT NULL DEFAULT 0,
|
|
oversized_line_start {cursor_integer},
|
|
cumulative_rows {cursor_integer} NOT NULL DEFAULT 0,
|
|
cumulative_bytes {cursor_integer} NOT NULL DEFAULT 0,
|
|
cumulative_inserted {cursor_integer} NOT NULL DEFAULT 0,
|
|
cumulative_rejected {cursor_integer} NOT NULL DEFAULT 0,
|
|
completed_at TEXT,
|
|
last_report_json TEXT,
|
|
updated_at TEXT NOT NULL
|
|
)''')
|
|
_migration_add_columns(conn, 'target_queue_reconciliation_cursors', {
|
|
'file_size': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'file_mtime_ns': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'discarding_oversized': 'INTEGER NOT NULL DEFAULT 0',
|
|
'oversized_line_start': cursor_integer,
|
|
'cumulative_rows': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'cumulative_bytes': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'cumulative_inserted': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'cumulative_rejected': f'{cursor_integer} NOT NULL DEFAULT 0',
|
|
'completed_at': 'TEXT',
|
|
})
|
|
_migration_postgres_column_shape(conn, 'target_queue_reconciliation_cursors', CURSOR_COLUMN_SPECS)
|
|
|
|
issue_id = 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' if conn.is_postgres else 'INTEGER PRIMARY KEY AUTOINCREMENT'
|
|
conn.execute(f'''CREATE TABLE IF NOT EXISTS target_queue_reconciliation_issues (
|
|
id {issue_id},
|
|
source_file TEXT NOT NULL,
|
|
file_identity TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
line_number {cursor_integer} NOT NULL,
|
|
byte_offset {cursor_integer} NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
target_preview TEXT,
|
|
created_at TEXT NOT NULL,
|
|
resolved_at TEXT
|
|
)''')
|
|
_migration_postgres_column_shape(conn, 'target_queue_reconciliation_issues', RECONCILIATION_ISSUE_COLUMN_SPECS)
|
|
|
|
conn.executescript(KEYCHECK_RESULTS_SQL)
|
|
if 'id' not in conn.table_columns('keycheck_results'):
|
|
raise RuntimeSafetySchemaError('keycheck_results.id is missing and cannot be repaired additively')
|
|
_migration_add_columns(conn, 'keycheck_results', {
|
|
'source_line': 'TEXT',
|
|
'detector_secret_hash': 'TEXT',
|
|
'event_id': 'TEXT',
|
|
'finding_uid': 'TEXT',
|
|
'link_status': "TEXT DEFAULT 'pending'",
|
|
'link_attempts': 'INTEGER DEFAULT 0',
|
|
'linked_at': 'TEXT',
|
|
'link_error': 'TEXT',
|
|
'candidate_id': id_type,
|
|
'credential_id': id_type,
|
|
'result_source': "TEXT NOT NULL DEFAULT 'api_check'",
|
|
})
|
|
conn.execute(f'''CREATE TABLE IF NOT EXISTS keycheck_event_map (
|
|
event_id TEXT PRIMARY KEY,
|
|
keycheck_result_id {id_type},
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
|
|
)''')
|
|
_migration_postgres_column_shape(conn, 'keycheck_results', KEYCHECK_COLUMN_SPECS)
|
|
_migration_postgres_column_shape(conn, 'keycheck_event_map', {
|
|
'event_id': ('text', True), 'keycheck_result_id': ('id_ref', False), 'created_at': ('text', True),
|
|
})
|
|
conn.executescript(KEYCHECK_RESULTS_SQL)
|
|
# The canonical schema creates this index, so an existing marker-13 table
|
|
# needs the additive selector column before the schema script reaches it.
|
|
if conn.table_exists('docker_image_blob_coverage'):
|
|
_migration_add_columns(conn, 'docker_image_blob_coverage', {
|
|
'selection_policy_sha256': "TEXT NOT NULL DEFAULT ''",
|
|
})
|
|
if conn.table_exists('target_queue_policy_events'):
|
|
experiment_reference = (
|
|
'BIGINT' if conn.is_postgres
|
|
else 'INTEGER REFERENCES docker_depth_experiments(id)'
|
|
)
|
|
_migration_add_columns(conn, 'target_queue_policy_events', {
|
|
'experiment_id': experiment_reference,
|
|
})
|
|
if conn.table_exists('discovery_retry_queue'):
|
|
source_cycle_reference = (
|
|
'BIGINT' if conn.is_postgres
|
|
else 'INTEGER REFERENCES source_cycles(id)'
|
|
)
|
|
_migration_add_columns(conn, 'discovery_retry_queue', {
|
|
'source_cycle_id': source_cycle_reference,
|
|
})
|
|
rollout_authority_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
(DOCKER_DEPTH_ROLLOUT_AUTHORITY_MIGRATION,),
|
|
).fetchone()
|
|
schema_selector_authority_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
(DOCKER_DEPTH_SCHEMA_SELECTOR_AUTHORITY_MIGRATION,),
|
|
).fetchone()
|
|
scarcity_cohort_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
(DOCKER_DEPTH_SCARCITY_COHORT_MIGRATION,),
|
|
).fetchone()
|
|
if conn.table_exists('docker_depth_experiments'):
|
|
_migration_add_columns(conn, 'docker_depth_experiments', {
|
|
'collection_generation': (
|
|
"TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1'"
|
|
),
|
|
'selection_sha256': 'TEXT',
|
|
})
|
|
if not rollout_authority_migration:
|
|
conn.execute(
|
|
"UPDATE docker_depth_experiments "
|
|
"SET collection_generation = 'legacy'"
|
|
)
|
|
if conn.table_exists('docker_discovery_passes'):
|
|
_migration_add_columns(conn, 'docker_discovery_passes', {
|
|
'collection_generation': (
|
|
"TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1'"
|
|
),
|
|
})
|
|
if not rollout_authority_migration:
|
|
conn.execute(
|
|
"UPDATE docker_discovery_passes "
|
|
"SET collection_generation = 'legacy'"
|
|
)
|
|
if conn.table_exists('docker_discovery_pages'):
|
|
count_type = 'BIGINT' if conn.is_postgres else 'INTEGER'
|
|
_migration_add_columns(conn, 'docker_discovery_pages', {
|
|
'total_count': count_type,
|
|
})
|
|
if (
|
|
conn.table_exists('docker_finding_layer_attributions')
|
|
and 'unattributed_reason' not in conn.table_columns(
|
|
'docker_finding_layer_attributions'
|
|
)):
|
|
if conn.execute(
|
|
"SELECT id FROM docker_finding_layer_attributions "
|
|
"WHERE attribution_state = 'unattributed' LIMIT 1"
|
|
).fetchone():
|
|
raise RuntimeSafetySchemaError(
|
|
'existing unattributed Docker findings require a reviewed reason backfill'
|
|
)
|
|
reason_check = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[
|
|
'docker_finding_layer_attributions'
|
|
]['docker_finding_layer_attributions_reason_check']
|
|
_migration_add_columns(conn, 'docker_finding_layer_attributions', {
|
|
'unattributed_reason': (
|
|
'TEXT CONSTRAINT docker_finding_layer_attributions_reason_check '
|
|
f'CHECK ({reason_check})'
|
|
),
|
|
})
|
|
if conn.table_exists('docker_depth_experiment_repositories'):
|
|
resolver_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
('20260910_20_docker_depth_resolver',),
|
|
).fetchone()
|
|
_migration_add_columns(conn, 'docker_depth_experiment_repositories', {
|
|
'planned_is_deep_probe': 'INTEGER NOT NULL DEFAULT 0',
|
|
'resolver_due_at': 'TEXT',
|
|
'candidate_distinct_graph_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
'replacement_repository_queue_id': (
|
|
'BIGINT' if conn.is_postgres else 'INTEGER'
|
|
),
|
|
'replacement_eligibility_page_id': (
|
|
'BIGINT' if conn.is_postgres else 'INTEGER'
|
|
),
|
|
'replacement_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
'replacement_evidence_sha256': 'TEXT',
|
|
})
|
|
if not resolver_migration:
|
|
conn.execute(
|
|
'''UPDATE docker_depth_experiment_repositories
|
|
SET planned_is_deep_probe = is_deep_probe'''
|
|
)
|
|
if conn.table_exists('docker_depth_experiment_queries'):
|
|
selection_check = DOCKER_DEPTH_EXPERIMENT_CHECK_SPECS[
|
|
'docker_depth_experiment_queries'
|
|
]['docker_depth_experiment_queries_selection_check']
|
|
_migration_add_columns(conn, 'docker_depth_experiment_queries', {
|
|
'selected_repository_count': (
|
|
'INTEGER NOT NULL DEFAULT 0 '
|
|
'CONSTRAINT docker_depth_experiment_queries_selection_check '
|
|
f'CHECK ({selection_check})'
|
|
),
|
|
})
|
|
if not scarcity_cohort_migration:
|
|
conn.execute(
|
|
'''UPDATE docker_depth_experiment_queries
|
|
SET selected_repository_count = required_repository_count'''
|
|
)
|
|
# Existing installations must gain columns before the schema script
|
|
# creates partial indexes that reference them. Fresh installs create
|
|
# the complete tables directly from PIPELINE_SCHEMA_SQL below.
|
|
remote_capacity_applied = False
|
|
if conn.table_exists('result_reservations'):
|
|
remote_capacity_applied = bool(conn.execute(
|
|
'''SELECT version FROM runtime_schema_migrations
|
|
WHERE version = ?''',
|
|
(REMOTE_ASSIGNMENT_CAPACITY_MIGRATION,),
|
|
).fetchone()) if conn.table_exists('runtime_schema_migrations') else False
|
|
projection_authority_applied = bool(conn.execute(
|
|
'''SELECT version FROM runtime_schema_migrations
|
|
WHERE version = ?''',
|
|
(DIAGNOSTIC_PROJECTION_AUTHORITY_MIGRATION,),
|
|
).fetchone()) if conn.table_exists('runtime_schema_migrations') else False
|
|
_migration_add_columns(conn, 'result_reservations', {
|
|
'reserved_bundle_bytes': (
|
|
'BIGINT CHECK (reserved_bundle_bytes > 0)'
|
|
),
|
|
'assignment_kind': "TEXT NOT NULL DEFAULT 'local' CHECK (assignment_kind IN ('local','remote'))",
|
|
'remote_user_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
|
|
'remote_device_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
|
|
'remote_issued_at': 'TEXT', 'remote_expires_at': 'TEXT',
|
|
'remote_result_upload_body_timeout_seconds': (
|
|
'INTEGER CHECK ('
|
|
'remote_result_upload_body_timeout_seconds IS NULL OR '
|
|
'remote_result_upload_body_timeout_seconds BETWEEN 30 AND 86400)'
|
|
),
|
|
'remote_effective_config_sha256': 'TEXT',
|
|
'remote_client_compat_sha256': 'TEXT',
|
|
'remote_execution_snapshot_json': 'TEXT',
|
|
'remote_execution_snapshot_sha256': 'TEXT',
|
|
'remote_resolution_kind': 'TEXT', 'remote_payload_sha256': 'TEXT',
|
|
'remote_receipt_id': 'TEXT', 'remote_resolution_json': 'TEXT',
|
|
'remote_resolved_at': 'TEXT',
|
|
'remote_diagnostic_projection_version': 'INTEGER',
|
|
'remote_diagnostic_count': 'INTEGER',
|
|
'remote_diagnostic_uids_sha256': 'TEXT',
|
|
})
|
|
if not remote_capacity_applied:
|
|
conn.execute(
|
|
'''UPDATE result_reservations
|
|
SET reserved_bundle_bytes = declared_bundle_bytes
|
|
WHERE reserved_bundle_bytes IS NULL'''
|
|
)
|
|
if conn.is_postgres:
|
|
conn.execute(
|
|
'''ALTER TABLE result_reservations
|
|
ALTER COLUMN reserved_bundle_bytes SET NOT NULL'''
|
|
)
|
|
if not projection_authority_applied:
|
|
conn.execute(
|
|
'''UPDATE result_reservations
|
|
SET remote_diagnostic_projection_version = 0
|
|
WHERE assignment_kind = 'remote'
|
|
AND remote_resolution_kind = 'bundle_accepted'
|
|
AND remote_diagnostic_projection_version IS NULL'''
|
|
)
|
|
if conn.table_exists('admission_intents'):
|
|
_migration_add_columns(conn, 'admission_intents', {
|
|
'remote_user_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
|
|
'remote_device_id': 'BIGINT' if conn.is_postgres else 'INTEGER',
|
|
})
|
|
conn.executescript(PIPELINE_SCHEMA_SQL)
|
|
_ensure_runtime_operations_authority(conn)
|
|
_migration_ensure_pipeline_quarantine_review_status_constraint(conn)
|
|
_migration_add_docker_depth_schema_authority_checks(
|
|
conn, bool(schema_selector_authority_migration),
|
|
)
|
|
_migration_add_docker_depth_scarcity_checks(
|
|
conn, bool(scarcity_cohort_migration),
|
|
)
|
|
_migration_require_docker_depth_check_constraints(conn)
|
|
_migration_add_columns(conn, 'pipeline_quarantine', {
|
|
'capacity_credit_applied': 'INTEGER NOT NULL DEFAULT 1',
|
|
'capacity_items': 'BIGINT NOT NULL DEFAULT 1',
|
|
'capacity_bytes': 'BIGINT NOT NULL DEFAULT 0',
|
|
})
|
|
_migration_add_columns(conn, 'result_reservations', {
|
|
'cleanup_attempts': 'INTEGER NOT NULL DEFAULT 0',
|
|
'cleanup_available_after': 'TEXT',
|
|
'git_scan_plan_json': 'TEXT',
|
|
'git_scan_plan_sha256': 'TEXT',
|
|
'docker_layer_plan_json': 'TEXT',
|
|
'docker_layer_plan_sha256': 'TEXT',
|
|
})
|
|
_migration_add_columns(conn, 'docker_content_blobs', {
|
|
'coverage_policy_sha256': 'TEXT',
|
|
})
|
|
_migration_add_columns(conn, 'docker_image_blob_coverage', {
|
|
'coverage_policy_sha256': 'TEXT',
|
|
'selection_policy_sha256': "TEXT NOT NULL DEFAULT ''",
|
|
})
|
|
_migration_backfill_docker_selection_policies(conn)
|
|
_migration_add_columns(conn, 'pipeline_artifacts', {
|
|
'cleanup_attempts': 'INTEGER NOT NULL DEFAULT 0',
|
|
'cleanup_available_after': 'TEXT',
|
|
'cleanup_last_error': 'TEXT',
|
|
})
|
|
conn.execute(
|
|
'''UPDATE pipeline_quarantine SET capacity_bytes = byte_count
|
|
WHERE capacity_credit_applied = 1 AND capacity_bytes = 0 AND byte_count > 0'''
|
|
)
|
|
_migration_add_columns(conn, 'keycheck_credentials', {
|
|
'provider_key_hash': "TEXT NOT NULL DEFAULT ''",
|
|
})
|
|
_migration_add_columns(conn, 'keycheck_candidates', {
|
|
'secret_hash': "TEXT NOT NULL DEFAULT ''",
|
|
'routed_service': "TEXT NOT NULL DEFAULT ''",
|
|
'result_projection_reserved_bytes': 'BIGINT NOT NULL DEFAULT 0',
|
|
'result_projection_credit_transferred': 'INTEGER NOT NULL DEFAULT 0',
|
|
})
|
|
_migration_add_columns(conn, 'docker_adaptive_shadow_reports', {
|
|
'selection_metrics_json': "TEXT NOT NULL DEFAULT '{}'",
|
|
'sink_checkpoint_count': 'INTEGER NOT NULL DEFAULT 0',
|
|
})
|
|
_migration_backfill_keycheck_hashes(conn)
|
|
conn.execute(
|
|
'''CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_credentials_provider_key
|
|
ON keycheck_credentials(service, provider_key_hash)'''
|
|
)
|
|
|
|
now = utc_now_iso()
|
|
conn.execute(
|
|
'''INSERT INTO pipeline_capacity(id, updated_at) VALUES (1, ?)
|
|
ON CONFLICT(id) DO NOTHING''',
|
|
(now,),
|
|
)
|
|
for stream_name, relative_path, rotation_bytes in (
|
|
('scan_results', 'scan_results.jsonl', 256 * 1024 * 1024),
|
|
('found_secrets', 'found_secrets.jsonl', 128 * 1024 * 1024),
|
|
('scan_errors', 'scan_errors.log', 32 * 1024 * 1024),
|
|
):
|
|
conn.execute(
|
|
'''INSERT INTO projection_streams(
|
|
stream_name, base_relative_path, current_generation,
|
|
rotation_bytes, max_generations, created_at, updated_at
|
|
) VALUES (?, ?, 0, ?, 16, ?, ?)
|
|
ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, relative_path, rotation_bytes, now, now),
|
|
)
|
|
conn.execute(
|
|
'''INSERT INTO projection_cursors(
|
|
stream_name, generation, committed_offset, updated_at
|
|
) VALUES (?, 0, 0, ?)
|
|
ON CONFLICT(stream_name) DO NOTHING''',
|
|
(stream_name, now),
|
|
)
|
|
migration_code = hashlib.sha256(PIPELINE_SCHEMA_SQL.encode('utf-8')).hexdigest()
|
|
capacity_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
(PIPELINE_MIGRATION_VERSIONS[0],),
|
|
).fetchone()
|
|
if not capacity_migration or not remote_capacity_applied:
|
|
_migration_reconcile_pipeline_capacity(conn)
|
|
single_writer_migration = conn.execute(
|
|
'SELECT version FROM runtime_schema_migrations WHERE version = ?',
|
|
('20260729_10_keycheck_projection_single_writer',),
|
|
).fetchone()
|
|
if not single_writer_migration:
|
|
prepared_status = conn.execute(
|
|
'''SELECT a.id FROM projection_appends a
|
|
JOIN projection_jobs j ON j.id = a.job_id
|
|
WHERE j.job_kind = 'keycheck_event'
|
|
AND j.status IN ('pending','leased')
|
|
AND (j.required_stream_mask & 16) <> 0
|
|
AND a.state = 'prepared' AND a.stream_name LIKE 'keycheck:%:status'
|
|
LIMIT 1'''
|
|
).fetchone()
|
|
if prepared_status:
|
|
raise RuntimeSafetySchemaError(
|
|
'keycheck status projection cutover requires reviewed prepared-append recovery'
|
|
)
|
|
conn.execute(
|
|
'''UPDATE projection_jobs
|
|
SET required_stream_mask = required_stream_mask - 16, updated_at = ?
|
|
WHERE job_kind = 'keycheck_event' AND status IN ('pending','leased')
|
|
AND (required_stream_mask & 16) <> 0''',
|
|
(now,),
|
|
)
|
|
for version in PIPELINE_MIGRATION_VERSIONS:
|
|
conn.execute(
|
|
'''INSERT INTO runtime_schema_migrations(version, applied_at, code_sha256)
|
|
VALUES (?, ?, ?) ON CONFLICT(version) DO NOTHING''',
|
|
(version, now, migration_code),
|
|
)
|
|
conn.execute(
|
|
'''UPDATE runtime_schema_migrations SET applied_at = ?, code_sha256 = ?
|
|
WHERE version = ?''',
|
|
(now, migration_code, PIPELINE_MIGRATION_VERSIONS[-1]),
|
|
)
|
|
|
|
for table, specs in RUNTIME_TABLE_SPECS.items():
|
|
if conn.is_postgres:
|
|
_migration_postgres_column_shape(conn, table, specs)
|
|
else:
|
|
_migration_require_sqlite_column_shape(conn, table, specs)
|
|
_migration_require_primary_key(conn, table, specs, RUNTIME_PRIMARY_KEYS[table])
|
|
generated_id = GENERATED_ID_COLUMNS.get(table)
|
|
if generated_id:
|
|
_migration_ensure_generated_id(conn, table, generated_id)
|
|
_migration_ensure_defaults(
|
|
conn, table, specs, RUNTIME_COLUMN_DEFAULTS.get(table, {}), generated_id,
|
|
)
|
|
|
|
_migration_ensure_foreign_keys(conn)
|
|
_migration_seed_legacy_docker_provenance(conn)
|
|
|
|
for table, name, columns, unique, predicate in DOCKER_DEPTH_EXPERIMENT_INDEX_SPECS:
|
|
statement = (
|
|
f'CREATE {"UNIQUE " if unique else ""}INDEX {name} '
|
|
f'ON {table}({", ".join(columns)})'
|
|
)
|
|
if predicate:
|
|
statement += f' WHERE {predicate}'
|
|
_migration_ensure_index(
|
|
conn, table, name, statement, list(columns),
|
|
unique=unique, predicate=predicate,
|
|
)
|
|
|
|
now = utc_now_iso()
|
|
unresolved = conn.execute(
|
|
'''SELECT id, target FROM target_queue
|
|
WHERE platform = 'docker' AND status IN ('pending', 'deferred')
|
|
AND resolver_state IS NULL'''
|
|
).fetchall()
|
|
resolver_due = datetime.fromtimestamp(
|
|
time.time() + max(60, env_int('DOCKER_RESOLVER_RETRY_SEC', 3600)), timezone.utc
|
|
).isoformat(timespec='seconds')
|
|
for row in unresolved:
|
|
target = str(row['target'] or '').strip()
|
|
if target and '@' not in target and ':' not in target.rsplit('/', 1)[-1]:
|
|
conn.execute(
|
|
'''UPDATE target_queue SET status = 'deferred', resolver_state = 'pending', resolver_due_at = ?,
|
|
available_after = COALESCE(available_after, ?), updated_at = ? WHERE id = ?''',
|
|
(resolver_due, resolver_due, now, row['id']),
|
|
)
|
|
conn.execute(
|
|
'''UPDATE target_queue SET
|
|
resolver_state = CASE WHEN resolver_state = 'resolving' THEN 'retry' ELSE resolver_state END,
|
|
resolver_due_at = ?, resolver_token = CASE WHEN resolver_state = 'resolving' THEN NULL ELSE resolver_token END,
|
|
available_after = COALESCE(available_after, ?), updated_at = ?
|
|
WHERE platform = 'docker' AND status = 'deferred'
|
|
AND resolver_state IN ('pending', 'retry', 'resolving')
|
|
AND resolver_due_at IS NULL''',
|
|
(resolver_due, resolver_due, now),
|
|
)
|
|
|
|
_migration_ensure_index(
|
|
conn, 'target_scans', 'uq_target_scans_scan_event_id',
|
|
'''CREATE UNIQUE INDEX uq_target_scans_scan_event_id
|
|
ON target_scans(scan_event_id) WHERE scan_event_id IS NOT NULL''',
|
|
['scan_event_id'], unique=True, predicate='scan_event_id IS NOT NULL',
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'target_scans', 'idx_target_scans_event_hash',
|
|
'CREATE INDEX idx_target_scans_event_hash ON target_scans(scan_event_id, scan_event_hash)',
|
|
['scan_event_id', 'scan_event_hash'],
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'target_scans', 'idx_target_scans_queue_id',
|
|
'CREATE INDEX idx_target_scans_queue_id ON target_scans(queue_id)',
|
|
['queue_id'],
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'target_scans', 'idx_target_scans_result_reservation',
|
|
'''CREATE INDEX idx_target_scans_result_reservation
|
|
ON target_scans(result_reservation_id, id)''',
|
|
['result_reservation_id', 'id'],
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'target_queue', 'idx_target_queue_current_reservation',
|
|
'CREATE INDEX idx_target_queue_current_reservation ON target_queue(current_result_reservation_id)',
|
|
['current_result_reservation_id'],
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'target_queue', 'idx_target_queue_claimable_v2',
|
|
'''CREATE INDEX idx_target_queue_claimable_v2
|
|
ON target_queue(source, platform, status, available_after, id)
|
|
WHERE current_result_reservation_id IS NULL
|
|
AND (status = 'pending' OR status = 'deferred')''',
|
|
['source', 'platform', 'status', 'available_after', 'id'],
|
|
predicate="current_result_reservation_id IS NULL AND (status = 'pending' OR status = 'deferred')",
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'discovery_retry_queue', 'uq_discovery_retry_queue_work_key',
|
|
'''CREATE UNIQUE INDEX uq_discovery_retry_queue_work_key
|
|
ON discovery_retry_queue(work_key)''',
|
|
['work_key'], unique=True,
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_due',
|
|
'''CREATE INDEX idx_discovery_retry_queue_due
|
|
ON discovery_retry_queue(source, available_after, id)
|
|
WHERE status = 'pending' ''',
|
|
['source', 'available_after', 'id'],
|
|
predicate=DISCOVERY_RETRY_DUE_INDEX_PREDICATE,
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_lease',
|
|
'''CREATE INDEX idx_discovery_retry_queue_lease
|
|
ON discovery_retry_queue(lease_expires_at, id)
|
|
WHERE status = 'leased' ''',
|
|
['lease_expires_at', 'id'],
|
|
predicate=DISCOVERY_RETRY_LEASE_INDEX_PREDICATE,
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'discovery_retry_queue', 'idx_discovery_retry_queue_policy',
|
|
'''CREATE INDEX idx_discovery_retry_queue_policy
|
|
ON discovery_retry_queue(source, query, policy_sha256, status, id)''',
|
|
['source', 'query', 'policy_sha256', 'status', 'id'],
|
|
)
|
|
target_queue_indexes = conn.table_indexes('target_queue')
|
|
if not any(
|
|
index['unique'] and index['columns'] == ['source', 'normalized_target']
|
|
and not _normalized_predicate(index['predicate'])
|
|
for index in target_queue_indexes.values()
|
|
):
|
|
try:
|
|
_migration_ensure_index(
|
|
conn, 'target_queue', 'uq_target_queue_source_normalized',
|
|
'CREATE UNIQUE INDEX uq_target_queue_source_normalized ON target_queue(source, normalized_target)',
|
|
['source', 'normalized_target'], unique=True,
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'target_queue cannot be made unique on (source, normalized_target); '
|
|
'manually resolve duplicate rows before retrying migration'
|
|
) from exc
|
|
_migration_ensure_index(
|
|
conn, 'scan_publication_outbox', 'uq_scan_publication_outbox_target_scan_id',
|
|
'CREATE UNIQUE INDEX uq_scan_publication_outbox_target_scan_id ON scan_publication_outbox(target_scan_id)',
|
|
['target_scan_id'], unique=True,
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'keycheck_event_map', 'uq_keycheck_event_map_event_id',
|
|
'CREATE UNIQUE INDEX uq_keycheck_event_map_event_id ON keycheck_event_map(event_id)',
|
|
['event_id'], unique=True,
|
|
)
|
|
_migration_ensure_index(
|
|
conn, 'finding_uid_map', 'uq_finding_uid_map_finding_uid',
|
|
'CREATE UNIQUE INDEX uq_finding_uid_map_finding_uid ON finding_uid_map(finding_uid)',
|
|
['finding_uid'], unique=True,
|
|
)
|
|
package_indexes = conn.table_indexes('package_repo_candidates')
|
|
if not any(
|
|
index['unique']
|
|
and index['columns'] == ['package_source', 'package_name', 'package_version', 'repo_url']
|
|
and not _normalized_predicate(index['predicate'])
|
|
and _index_usable(index)
|
|
for index in package_indexes.values()
|
|
):
|
|
try:
|
|
_migration_ensure_index(
|
|
conn, 'package_repo_candidates', 'uq_package_repo_candidates_identity',
|
|
'''CREATE UNIQUE INDEX uq_package_repo_candidates_identity
|
|
ON package_repo_candidates(package_source, package_name, package_version, repo_url)''',
|
|
['package_source', 'package_name', 'package_version', 'repo_url'], unique=True,
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeSafetySchemaError(
|
|
'package_repo_candidates cannot be made unique on its package/repository identity; '
|
|
'manually resolve duplicate rows before retrying migration'
|
|
) from exc
|
|
index_specs = (
|
|
('runs', 'idx_runs_started_at', 'started_at'),
|
|
('runs', 'idx_runs_status', 'status'),
|
|
('runs', 'idx_runs_selected_source_status', 'selected_source, status, id'),
|
|
('source_cycles', 'idx_source_cycles_run_id', 'run_id'),
|
|
('source_cycles', 'idx_source_cycles_source_query', 'source, query'),
|
|
('source_cycles', 'idx_source_cycles_source_status', 'source, status, id'),
|
|
('target_queue', 'idx_target_queue_source_status', 'source, status, updated_at'),
|
|
('target_queue', 'idx_target_queue_observe_source_status', 'source, status'),
|
|
('target_queue', 'idx_target_queue_lease', 'source, status, lease_expires_at'),
|
|
('target_queue', 'idx_target_queue_platform_status', 'platform, status'),
|
|
('target_queue', 'idx_target_queue_claim_batch', 'claim_batch, lease_owner'),
|
|
('target_queue', 'idx_target_queue_claim', 'source, platform, status, available_after, lease_expires_at, id'),
|
|
('target_queue', 'idx_target_queue_resolver_claim', 'source, platform, status, resolver_state, resolver_due_at, id'),
|
|
('target_queue', 'idx_target_queue_source_platform_normalized', 'source, platform, normalized_target'),
|
|
('scan_publication_outbox', 'idx_scan_publication_outbox_status', 'status, available_after, id'),
|
|
('scan_publication_outbox', 'idx_scan_publication_outbox_age', 'status, created_at, id'),
|
|
('target_scans', 'idx_target_scans_cycle_id', 'cycle_id'),
|
|
('target_scans', 'idx_target_scans_source_status', 'source, status'),
|
|
('target_scans', 'idx_target_scans_source_ended', 'source, ended_at'),
|
|
('target_scans', 'idx_target_scans_source_ended_id', 'source, ended_at, id'),
|
|
('target_scans', 'idx_target_scans_source_skip_ended', 'source, skipped_reason, ended_at'),
|
|
('target_scans', 'idx_target_scans_normalized_target', 'normalized_target'),
|
|
('keycheck_results', 'idx_keycheck_results_checked', 'checked_at'),
|
|
('keycheck_results', 'idx_keycheck_results_service_status', 'service, status_group, status'),
|
|
('keycheck_results', 'idx_keycheck_results_finding', 'finding_id'),
|
|
('keycheck_results', 'idx_keycheck_results_cycle', 'cycle_id'),
|
|
('keycheck_results', 'idx_keycheck_results_source_query', 'source, query'),
|
|
('keycheck_results', 'idx_keycheck_results_key_hash', 'key_hash'),
|
|
('keycheck_results', 'idx_keycheck_results_secret_hash', 'secret_hash'),
|
|
('keycheck_results', 'idx_keycheck_results_link_repair', 'link_status, id'),
|
|
('findings', 'idx_findings_finding_uid', 'finding_uid'),
|
|
('findings', 'idx_findings_target_scan_id_id', 'target_scan_id, id'),
|
|
('findings', 'idx_findings_cycle_id', 'cycle_id'),
|
|
('findings', 'idx_findings_source', 'source'),
|
|
('findings', 'idx_findings_secret_hash', 'secret_hash'),
|
|
('findings', 'idx_findings_provider', 'provider'),
|
|
('findings', 'idx_findings_confidence', 'credential_confidence'),
|
|
('errors', 'idx_errors_cycle_id', 'cycle_id'),
|
|
('errors', 'idx_errors_source_category', 'source, category'),
|
|
('errors', 'idx_errors_target_scan_id', 'target_scan_id'),
|
|
('queue_snapshots', 'idx_queue_snapshots_source_time', 'source, captured_at'),
|
|
('package_repo_candidates', 'idx_package_repo_candidates_query', 'query'),
|
|
('package_repo_candidates', 'idx_package_repo_candidates_repo', 'repo_url'),
|
|
('package_repo_candidates', 'idx_package_repo_candidates_source_seen', 'package_source, last_seen_at'),
|
|
('target_queue_reconciliation_issues', 'idx_reconciliation_issues_open', 'source_file, resolved_at, id'),
|
|
('target_queue_policy_events', 'idx_target_queue_policy_events_queue', 'queue_id, id'),
|
|
('target_queue_policy_events', 'idx_target_queue_policy_events_manifest', 'manifest_sha256, id'),
|
|
('docker_content_blobs', 'idx_docker_content_blobs_reclaim', 'state, available_after, lease_expires_at, digest'),
|
|
('docker_content_blobs', 'idx_docker_content_blobs_reservation', 'lease_reservation_id, digest'),
|
|
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_manifest', 'manifest_digest, coverage_state, position'),
|
|
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_blob', 'blob_digest, coverage_state, queue_id'),
|
|
('docker_image_blob_coverage', 'idx_docker_image_blob_coverage_reservation', 'reservation_id, position'),
|
|
(
|
|
'docker_image_blob_coverage',
|
|
'idx_docker_image_blob_coverage_selection',
|
|
'queue_id, manifest_digest, selection_policy_sha256, position, reservation_id',
|
|
),
|
|
(
|
|
'docker_adaptive_shadow_reports',
|
|
'idx_docker_adaptive_shadow_reports_gate',
|
|
'scan_policy_sha256, execution_policy_sha256, selection_policy_sha256, '
|
|
'state, completed_at, id',
|
|
),
|
|
)
|
|
for table, name, column_sql in index_specs:
|
|
columns = [value.strip() for value in column_sql.split(',')]
|
|
_migration_ensure_index(
|
|
conn, table, name, f'CREATE INDEX {name} ON {table}({column_sql})', columns,
|
|
)
|
|
_migration_ensure_index(
|
|
conn,
|
|
'target_scans',
|
|
'idx_target_scans_cooldown_recent',
|
|
f'''CREATE INDEX idx_target_scans_cooldown_recent
|
|
ON target_scans(source, ended_at DESC, id DESC)
|
|
WHERE {CI_COOLDOWN_INDEX_PREDICATE}''',
|
|
['source', 'ended_at', 'id'],
|
|
predicate=CI_COOLDOWN_INDEX_PREDICATE,
|
|
)
|
|
for name, columns in (
|
|
(
|
|
'idx_package_repo_candidates_query_seen',
|
|
'query, last_seen_at DESC, id DESC',
|
|
),
|
|
(
|
|
'idx_package_repo_candidates_recent_lookup',
|
|
'last_seen_at DESC, id DESC, package_source, query, package_name',
|
|
),
|
|
):
|
|
_migration_ensure_index(
|
|
conn,
|
|
'package_repo_candidates',
|
|
name,
|
|
f'''CREATE INDEX {name} ON package_repo_candidates({columns})
|
|
WHERE {PACKAGE_REPO_NONEMPTY_PREDICATE}''',
|
|
[value.strip().split()[0] for value in columns.split(',')],
|
|
predicate=PACKAGE_REPO_NONEMPTY_PREDICATE,
|
|
)
|
|
_migration_retain_pg_trgm_without_name_index(conn)
|
|
for name, columns, predicate in (
|
|
(
|
|
'idx_target_queue_claim_pending',
|
|
'source, platform, id, attempts, available_after',
|
|
"status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')",
|
|
),
|
|
(
|
|
'idx_target_queue_claim_deferred',
|
|
'source, platform, available_after, id, attempts',
|
|
"status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved')",
|
|
),
|
|
(
|
|
'idx_target_queue_claim_in_progress',
|
|
'source, platform, id, attempts, available_after, lease_expires_at, resolver_state',
|
|
"status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved')",
|
|
),
|
|
(
|
|
'idx_target_queue_active_lease_owner_token',
|
|
'lease_owner, lease_token',
|
|
"status = 'in_progress'",
|
|
),
|
|
(
|
|
'idx_target_queue_exhausted_attempts',
|
|
'source, platform, status, attempts, id, lease_expires_at',
|
|
"status = 'pending' OR status = 'deferred' OR status = 'in_progress'",
|
|
),
|
|
(
|
|
'idx_target_queue_updated_rescan',
|
|
'source, platform, completed_at, remote_modified_at, scan_remote_modified_at, id',
|
|
"status = 'done' AND remote_modified_at IS NOT NULL",
|
|
),
|
|
(
|
|
'idx_target_queue_cold',
|
|
'source, platform, query, id',
|
|
"status = 'cold'",
|
|
),
|
|
):
|
|
_migration_ensure_index(
|
|
conn,
|
|
'target_queue',
|
|
name,
|
|
f'CREATE INDEX {name} ON target_queue({columns}) WHERE {predicate}',
|
|
[value.strip() for value in columns.split(',')],
|
|
predicate=predicate,
|
|
)
|
|
if conn.is_postgres:
|
|
conn.execute(
|
|
'''CREATE STATISTICS IF NOT EXISTS st_target_queue_claim_selectivity
|
|
(dependencies, mcv)
|
|
ON source, platform, status, resolver_state, attempts
|
|
FROM target_queue'''
|
|
)
|
|
conn.execute('ANALYZE target_queue')
|
|
conn.execute('ANALYZE target_scans')
|
|
conn.execute('ANALYZE findings')
|
|
conn.execute('ANALYZE package_repo_candidates')
|
|
_migration_ensure_index(
|
|
conn, 'target_queue_reconciliation_issues', 'uq_reconciliation_issue_identity',
|
|
'''CREATE UNIQUE INDEX uq_reconciliation_issue_identity
|
|
ON target_queue_reconciliation_issues(
|
|
source_file, file_identity, line_number, byte_offset, reason
|
|
)''',
|
|
['source_file', 'file_identity', 'line_number', 'byte_offset', 'reason'], unique=True,
|
|
)
|
|
db._runtime_safety_schema_validated = False
|
|
db.require_runtime_safety_schema(commit=False)
|
|
conn.commit()
|
|
db._docker_depth_experiment_schema_installed_cache = True
|
|
return True
|
|
except Exception:
|
|
try:
|
|
conn.rollback()
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
|
|
def first_line(value, limit=500):
|
|
for line in str(value or '').splitlines():
|
|
line = line.strip()
|
|
if line:
|
|
return line[:limit]
|
|
return ''
|
|
|
|
|
|
def elapsed_seconds(start, end):
|
|
start_dt = parse_time(start)
|
|
end_dt = parse_time(end)
|
|
if not start_dt or not end_dt:
|
|
return None
|
|
return max(0.0, (end_dt - start_dt).total_seconds())
|
|
|
|
|
|
KEYCHECK_RESULTS_SQL = r'''
|
|
CREATE TABLE IF NOT EXISTS keycheck_results (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
service TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
status_group TEXT NOT NULL,
|
|
checked_at TEXT NOT NULL,
|
|
key_hash TEXT,
|
|
secret_hash TEXT,
|
|
key_masked TEXT,
|
|
finding_id INTEGER,
|
|
target_scan_id INTEGER,
|
|
cycle_id INTEGER,
|
|
run_id INTEGER,
|
|
source TEXT,
|
|
query TEXT,
|
|
target TEXT,
|
|
detector_name TEXT,
|
|
found_at TEXT,
|
|
message TEXT,
|
|
metadata_json TEXT,
|
|
source_line TEXT,
|
|
detector_secret_hash TEXT,
|
|
event_id TEXT,
|
|
finding_uid TEXT,
|
|
link_status TEXT DEFAULT 'pending',
|
|
link_attempts INTEGER DEFAULT 0,
|
|
linked_at TEXT,
|
|
link_error TEXT,
|
|
candidate_id INTEGER,
|
|
credential_id INTEGER,
|
|
result_source TEXT NOT NULL DEFAULT 'api_check',
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(candidate_id) REFERENCES keycheck_candidates(id),
|
|
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS keycheck_event_map (
|
|
event_id TEXT PRIMARY KEY,
|
|
keycheck_result_id INTEGER,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_checked ON keycheck_results(checked_at);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_service_status ON keycheck_results(service, status_group, status);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_finding ON keycheck_results(finding_id);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_cycle ON keycheck_results(cycle_id);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_source_query ON keycheck_results(source, query);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_key_hash ON keycheck_results(key_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_secret_hash ON keycheck_results(secret_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_link_repair ON keycheck_results(link_status, id);
|
|
|
|
DROP VIEW IF EXISTS keycheck_latest_state;
|
|
CREATE VIEW IF NOT EXISTS keycheck_latest_state AS
|
|
WITH normalized AS (
|
|
SELECT
|
|
kr.*,
|
|
COALESCE(NULLIF(kr.key_hash, ''), NULLIF(kr.secret_hash, ''), NULLIF(kr.key_masked, '')) AS key_identity,
|
|
CASE
|
|
WHEN NULLIF(kr.key_hash, '') IS NOT NULL THEN 'key_hash'
|
|
WHEN NULLIF(kr.secret_hash, '') IS NOT NULL THEN 'secret_hash'
|
|
WHEN NULLIF(kr.key_masked, '') IS NOT NULL THEN 'key_masked'
|
|
ELSE 'none'
|
|
END AS key_identity_kind
|
|
FROM keycheck_results kr
|
|
), ranked AS (
|
|
SELECT
|
|
normalized.*,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY service, key_identity
|
|
ORDER BY checked_at DESC, id DESC
|
|
) AS latest_rank
|
|
FROM normalized
|
|
WHERE key_identity IS NOT NULL
|
|
)
|
|
SELECT *
|
|
FROM ranked
|
|
WHERE latest_rank = 1;
|
|
'''
|
|
|
|
|
|
DOCKER_DEPTH_EXPERIMENT_SCHEMA_SQL = f'''
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiments (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_key TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
state TEXT NOT NULL DEFAULT 'collecting',
|
|
collection_generation TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1',
|
|
config_sha256 TEXT NOT NULL,
|
|
ordered_queries_sha256 TEXT NOT NULL,
|
|
selector_version TEXT NOT NULL,
|
|
selector_sha256 TEXT NOT NULL,
|
|
provenance_policy_sha256 TEXT NOT NULL,
|
|
query_count INTEGER NOT NULL,
|
|
repositories_per_query INTEGER NOT NULL,
|
|
images_per_repository INTEGER NOT NULL,
|
|
target_limit INTEGER NOT NULL,
|
|
target_count INTEGER NOT NULL DEFAULT 0,
|
|
selection_count INTEGER NOT NULL DEFAULT 0,
|
|
fence_generation INTEGER NOT NULL DEFAULT 0,
|
|
fence_owner TEXT,
|
|
fence_token TEXT,
|
|
fence_expires_at TEXT,
|
|
hold_reason_code TEXT,
|
|
plan_sha256 TEXT,
|
|
selection_sha256 TEXT,
|
|
hold_manifest_sha256 TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
planned_at TEXT,
|
|
activated_at TEXT,
|
|
draining_at TEXT,
|
|
completed_at TEXT,
|
|
released_at TEXT,
|
|
held_at TEXT,
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_state_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_identity_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_capacity_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_selection_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiments', 'docker_depth_experiments_fence_check')}
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_queries (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
source TEXT NOT NULL,
|
|
query_ordinal INTEGER NOT NULL,
|
|
query TEXT NOT NULL,
|
|
query_sha256 TEXT NOT NULL,
|
|
required_repository_count INTEGER NOT NULL DEFAULT 10,
|
|
selected_repository_count INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(experiment_id, query_ordinal),
|
|
UNIQUE(experiment_id, source, query),
|
|
UNIQUE(experiment_id, query_ordinal, source, query),
|
|
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_identity_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_queries', 'docker_depth_experiment_queries_selection_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_discovery_passes (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER,
|
|
pass_token TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
pass_kind TEXT NOT NULL,
|
|
collection_generation TEXT NOT NULL DEFAULT 'docker-depth-provenance-v1',
|
|
policy_sha256 TEXT NOT NULL,
|
|
ordered_queries_sha256 TEXT NOT NULL,
|
|
expected_query_count INTEGER NOT NULL,
|
|
completed_query_count INTEGER NOT NULL DEFAULT 0,
|
|
state TEXT NOT NULL DEFAULT 'collecting',
|
|
started_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_kind_check')},
|
|
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_state_check')},
|
|
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_count_check')},
|
|
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_identity_check')},
|
|
{_docker_depth_check_clause('docker_discovery_passes', 'docker_discovery_passes_complete_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_discovery_pages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
pass_id INTEGER NOT NULL,
|
|
source_cycle_id INTEGER,
|
|
retry_work_id INTEGER,
|
|
query TEXT NOT NULL,
|
|
query_ordinal INTEGER NOT NULL,
|
|
page_number INTEGER NOT NULL,
|
|
result_count INTEGER NOT NULL DEFAULT 0,
|
|
total_count INTEGER,
|
|
admitted_count INTEGER NOT NULL DEFAULT 0,
|
|
query_complete INTEGER NOT NULL DEFAULT 0,
|
|
admission_kind TEXT NOT NULL DEFAULT 'main',
|
|
page_sha256 TEXT NOT NULL,
|
|
observed_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(pass_id, query_ordinal, page_number),
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_range_check')},
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_boolean_check')},
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_kind_check')},
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_identity_check')},
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_evidence_check')},
|
|
{_docker_depth_check_clause('docker_discovery_pages', 'docker_discovery_pages_total_count_check')},
|
|
FOREIGN KEY(pass_id) REFERENCES docker_discovery_passes(id),
|
|
FOREIGN KEY(source_cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(retry_work_id) REFERENCES discovery_retry_queue(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_repository_query_provenance (
|
|
source TEXT NOT NULL,
|
|
query TEXT NOT NULL,
|
|
repository_queue_id INTEGER NOT NULL,
|
|
provenance_kind TEXT NOT NULL,
|
|
first_observed_at TEXT NOT NULL,
|
|
last_observed_at TEXT NOT NULL,
|
|
first_search_rank INTEGER,
|
|
best_search_rank INTEGER,
|
|
last_search_rank INTEGER,
|
|
first_cycle_id INTEGER,
|
|
last_cycle_id INTEGER,
|
|
first_page_id INTEGER,
|
|
last_page_id INTEGER,
|
|
first_policy_sha256 TEXT,
|
|
last_policy_sha256 TEXT,
|
|
observation_count INTEGER NOT NULL DEFAULT 1,
|
|
fresh_observation_count INTEGER NOT NULL DEFAULT 0,
|
|
fresh_complete_observation_count INTEGER NOT NULL DEFAULT 0,
|
|
fresh_coverage_eligible INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY(source, query, repository_queue_id),
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_kind_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_range_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_boolean_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_identity_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_counts_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_legacy_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_fresh_check')},
|
|
{_docker_depth_check_clause('docker_repository_query_provenance', 'docker_repository_query_provenance_eligibility_check')},
|
|
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(first_cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(last_cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(first_page_id) REFERENCES docker_discovery_pages(id),
|
|
FOREIGN KEY(last_page_id) REFERENCES docker_discovery_pages(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_repository_query_observations (
|
|
page_id INTEGER NOT NULL,
|
|
repository_queue_id INTEGER NOT NULL,
|
|
source TEXT NOT NULL,
|
|
query TEXT NOT NULL,
|
|
search_rank INTEGER NOT NULL,
|
|
observed_at TEXT NOT NULL,
|
|
PRIMARY KEY(page_id, repository_queue_id),
|
|
UNIQUE(page_id, repository_queue_id, source, query),
|
|
{_docker_depth_check_clause('docker_repository_query_observations', 'docker_repository_query_observations_rank_check')},
|
|
FOREIGN KEY(page_id) REFERENCES docker_discovery_pages(id),
|
|
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(source, query, repository_queue_id)
|
|
REFERENCES docker_repository_query_provenance(source, query, repository_queue_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_image_manifests (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
target_queue_id INTEGER NOT NULL,
|
|
source TEXT NOT NULL,
|
|
repository TEXT NOT NULL,
|
|
manifest_digest TEXT NOT NULL,
|
|
manifest_media_type TEXT NOT NULL,
|
|
config_digest TEXT,
|
|
graph_sha256 TEXT NOT NULL,
|
|
manifest_size_bytes INTEGER,
|
|
layer_count INTEGER NOT NULL,
|
|
resolved_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(id, target_queue_id),
|
|
{_docker_depth_check_clause('docker_image_manifests', 'docker_image_manifests_range_check')},
|
|
{_docker_depth_check_clause('docker_image_manifests', 'docker_image_manifests_identity_check')},
|
|
FOREIGN KEY(target_queue_id) REFERENCES target_queue(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_manifest_layers (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
manifest_id INTEGER NOT NULL,
|
|
position_from_base INTEGER NOT NULL,
|
|
position_from_top INTEGER NOT NULL,
|
|
layer_digest TEXT NOT NULL,
|
|
media_type TEXT NOT NULL,
|
|
layer_size_bytes INTEGER NOT NULL,
|
|
descriptor_sha256 TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(id, position_from_base, position_from_top, layer_digest),
|
|
{_docker_depth_check_clause('docker_manifest_layers', 'docker_manifest_layers_range_check')},
|
|
{_docker_depth_check_clause('docker_manifest_layers', 'docker_manifest_layers_identity_check')},
|
|
FOREIGN KEY(manifest_id) REFERENCES docker_image_manifests(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_repositories (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
query_ordinal INTEGER NOT NULL,
|
|
source TEXT NOT NULL,
|
|
query TEXT NOT NULL,
|
|
repository_queue_id INTEGER NOT NULL,
|
|
eligibility_page_id INTEGER NOT NULL,
|
|
repository_rank INTEGER NOT NULL,
|
|
is_deep_probe INTEGER NOT NULL DEFAULT 0,
|
|
planned_is_deep_probe INTEGER NOT NULL DEFAULT 0,
|
|
work_state TEXT NOT NULL DEFAULT 'pending',
|
|
resolver_generation INTEGER NOT NULL DEFAULT 0,
|
|
resolver_owner TEXT,
|
|
resolver_token TEXT,
|
|
resolver_expires_at TEXT,
|
|
resolver_attempts INTEGER NOT NULL DEFAULT 0,
|
|
resolver_due_at TEXT,
|
|
candidate_distinct_graph_count INTEGER NOT NULL DEFAULT 0,
|
|
selected_image_count INTEGER NOT NULL DEFAULT 0,
|
|
replacement_repository_queue_id INTEGER,
|
|
replacement_eligibility_page_id INTEGER,
|
|
replacement_count INTEGER NOT NULL DEFAULT 0,
|
|
replacement_evidence_sha256 TEXT,
|
|
last_error_code TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
resolved_at TEXT,
|
|
UNIQUE(experiment_id, query_ordinal, id),
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_boolean_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_state_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_fence_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_resolver_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_repositories', 'docker_depth_experiment_repositories_replacement_check')},
|
|
FOREIGN KEY(experiment_id, query_ordinal, source, query)
|
|
REFERENCES docker_depth_experiment_queries(
|
|
experiment_id, query_ordinal, source, query
|
|
),
|
|
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(source, query, repository_queue_id)
|
|
REFERENCES docker_repository_query_provenance(source, query, repository_queue_id),
|
|
FOREIGN KEY(eligibility_page_id, repository_queue_id, source, query)
|
|
REFERENCES docker_repository_query_observations(
|
|
page_id, repository_queue_id, source, query
|
|
),
|
|
FOREIGN KEY(replacement_repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(
|
|
replacement_eligibility_page_id, replacement_repository_queue_id, source, query
|
|
) REFERENCES docker_repository_query_observations(
|
|
page_id, repository_queue_id, source, query
|
|
)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_targets (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
target_queue_id INTEGER NOT NULL,
|
|
manifest_id INTEGER NOT NULL,
|
|
counter_ordinal INTEGER NOT NULL,
|
|
state TEXT NOT NULL DEFAULT 'pending',
|
|
dispatch_wave INTEGER NOT NULL,
|
|
dispatch_order INTEGER NOT NULL,
|
|
reservation_count INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
terminal_at TEXT,
|
|
UNIQUE(experiment_id, id),
|
|
{_docker_depth_check_clause('docker_depth_experiment_targets', 'docker_depth_experiment_targets_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_targets', 'docker_depth_experiment_targets_state_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
|
|
FOREIGN KEY(target_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(manifest_id, target_queue_id)
|
|
REFERENCES docker_image_manifests(id, target_queue_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_selections (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
query_ordinal INTEGER NOT NULL,
|
|
experiment_repository_id INTEGER NOT NULL,
|
|
experiment_target_id INTEGER NOT NULL,
|
|
image_rank INTEGER NOT NULL,
|
|
selection_reason TEXT NOT NULL,
|
|
selection_evidence_sha256 TEXT NOT NULL,
|
|
graph_sha256 TEXT NOT NULL,
|
|
selected_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_depth_experiment_selections', 'docker_depth_experiment_selections_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_selections', 'docker_depth_experiment_selections_identity_check')},
|
|
FOREIGN KEY(experiment_id, query_ordinal)
|
|
REFERENCES docker_depth_experiment_queries(experiment_id, query_ordinal),
|
|
FOREIGN KEY(experiment_id, query_ordinal, experiment_repository_id)
|
|
REFERENCES docker_depth_experiment_repositories(experiment_id, query_ordinal, id),
|
|
FOREIGN KEY(experiment_id, experiment_target_id)
|
|
REFERENCES docker_depth_experiment_targets(experiment_id, id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_candidate_skips (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
experiment_repository_id INTEGER NOT NULL,
|
|
repository_queue_id INTEGER NOT NULL,
|
|
candidate_kind TEXT NOT NULL,
|
|
candidate_ordinal INTEGER NOT NULL,
|
|
reason_code TEXT NOT NULL,
|
|
candidate_identity_sha256 TEXT NOT NULL,
|
|
evidence_sha256 TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_kind_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_candidate_skips', 'docker_depth_experiment_candidate_skips_identity_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
|
|
FOREIGN KEY(experiment_repository_id)
|
|
REFERENCES docker_depth_experiment_repositories(id),
|
|
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_resolver_attempt_refunds (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
experiment_repository_id INTEGER NOT NULL,
|
|
repository_queue_id INTEGER NOT NULL,
|
|
query_ordinal INTEGER NOT NULL,
|
|
repository_rank INTEGER NOT NULL,
|
|
recovery_kind TEXT NOT NULL,
|
|
manifest_sha256 TEXT NOT NULL,
|
|
entry_evidence_sha256 TEXT NOT NULL,
|
|
log_sha256 TEXT NOT NULL,
|
|
target_identity_sha256 TEXT NOT NULL,
|
|
prior_error_code_sha256 TEXT NOT NULL,
|
|
prior_work_state TEXT NOT NULL,
|
|
next_work_state TEXT NOT NULL,
|
|
prior_resolver_attempts INTEGER NOT NULL,
|
|
refund_attempts INTEGER NOT NULL,
|
|
next_resolver_attempts INTEGER NOT NULL,
|
|
confirmed_bug_event_count INTEGER NOT NULL,
|
|
applied_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_kind_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_state_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_attempt_refunds', 'docker_depth_resolver_attempt_refunds_hash_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
|
|
FOREIGN KEY(experiment_repository_id)
|
|
REFERENCES docker_depth_experiment_repositories(id),
|
|
FOREIGN KEY(repository_queue_id) REFERENCES target_queue(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_resolver_dispositions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_id INTEGER NOT NULL,
|
|
experiment_repository_id INTEGER NOT NULL,
|
|
prior_repository_queue_id INTEGER NOT NULL,
|
|
replacement_repository_queue_id INTEGER,
|
|
replacement_eligibility_page_id INTEGER,
|
|
replacement_best_search_rank INTEGER,
|
|
query_ordinal INTEGER NOT NULL,
|
|
repository_rank INTEGER NOT NULL,
|
|
disposition_kind TEXT NOT NULL,
|
|
outcome TEXT NOT NULL,
|
|
manifest_sha256 TEXT NOT NULL,
|
|
entry_evidence_sha256 TEXT NOT NULL,
|
|
candidate_snapshot_sha256 TEXT NOT NULL,
|
|
prior_target_identity_sha256 TEXT NOT NULL,
|
|
replacement_target_identity_sha256 TEXT,
|
|
prior_error_code_sha256 TEXT NOT NULL,
|
|
prior_work_state TEXT NOT NULL,
|
|
next_work_state TEXT NOT NULL,
|
|
prior_resolver_attempts INTEGER NOT NULL,
|
|
next_resolver_attempts INTEGER NOT NULL,
|
|
applied_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_kind_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_outcome_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_state_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_resolver_dispositions', 'docker_depth_resolver_dispositions_hash_check')},
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id),
|
|
FOREIGN KEY(experiment_repository_id)
|
|
REFERENCES docker_depth_experiment_repositories(id),
|
|
FOREIGN KEY(prior_repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(replacement_repository_queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(replacement_eligibility_page_id) REFERENCES docker_discovery_pages(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_depth_experiment_scan_bindings (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
experiment_target_id INTEGER NOT NULL,
|
|
reservation_id INTEGER NOT NULL,
|
|
target_scan_id INTEGER,
|
|
attempt INTEGER NOT NULL,
|
|
state TEXT NOT NULL DEFAULT 'reserved',
|
|
bound_at TEXT NOT NULL,
|
|
scan_bound_at TEXT,
|
|
completed_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_depth_experiment_scan_bindings', 'docker_depth_experiment_scan_bindings_range_check')},
|
|
{_docker_depth_check_clause('docker_depth_experiment_scan_bindings', 'docker_depth_experiment_scan_bindings_state_check')},
|
|
FOREIGN KEY(experiment_target_id) REFERENCES docker_depth_experiment_targets(id),
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_finding_layer_attributions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
scan_binding_id INTEGER NOT NULL,
|
|
finding_id INTEGER NOT NULL,
|
|
manifest_layer_id INTEGER,
|
|
attribution_state TEXT NOT NULL,
|
|
reported_layer_digest TEXT,
|
|
unattributed_reason TEXT,
|
|
position_from_base INTEGER,
|
|
position_from_top INTEGER,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_state_check')},
|
|
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_evidence_check')},
|
|
{_docker_depth_check_clause('docker_finding_layer_attributions', 'docker_finding_layer_attributions_reason_check')},
|
|
FOREIGN KEY(scan_binding_id) REFERENCES docker_depth_experiment_scan_bindings(id),
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id),
|
|
FOREIGN KEY(
|
|
manifest_layer_id, position_from_base, position_from_top, reported_layer_digest
|
|
) REFERENCES docker_manifest_layers(
|
|
id, position_from_base, position_from_top, layer_digest
|
|
)
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiments_key
|
|
ON docker_depth_experiments(experiment_key);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_depth_experiments_state
|
|
ON docker_depth_experiments(source, state, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiment_queries_ordinal
|
|
ON docker_depth_experiment_queries(experiment_id, query_ordinal);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_experiment_queries_query
|
|
ON docker_depth_experiment_queries(experiment_id, source, query);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_discovery_passes_token
|
|
ON docker_discovery_passes(pass_token);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_discovery_passes_policy
|
|
ON docker_discovery_passes(source, policy_sha256, state, id);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_discovery_passes_experiment
|
|
ON docker_discovery_passes(experiment_id, state, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_discovery_pages_position
|
|
ON docker_discovery_pages(pass_id, query_ordinal, page_number);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_discovery_pages_query
|
|
ON docker_discovery_pages(pass_id, query, query_complete, page_number);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_discovery_pages_retry
|
|
ON docker_discovery_pages(retry_work_id, id) WHERE retry_work_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_docker_repository_provenance_queue
|
|
ON docker_repository_query_provenance(repository_queue_id, source, query);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_repository_provenance_eligible
|
|
ON docker_repository_query_provenance(
|
|
source, query, fresh_coverage_eligible, best_search_rank, repository_queue_id
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_repository_observations_query
|
|
ON docker_repository_query_observations(source, query, repository_queue_id, page_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_image_manifests_queue
|
|
ON docker_image_manifests(target_queue_id);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_image_manifests_digest
|
|
ON docker_image_manifests(manifest_digest, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_manifest_layers_base_position
|
|
ON docker_manifest_layers(manifest_id, position_from_base);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_manifest_layers_top_position
|
|
ON docker_manifest_layers(manifest_id, position_from_top);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_manifest_layers_digest
|
|
ON docker_manifest_layers(layer_digest, manifest_id, position_from_base);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_repositories_member
|
|
ON docker_depth_experiment_repositories(experiment_id, query_ordinal, repository_queue_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_repositories_rank
|
|
ON docker_depth_experiment_repositories(experiment_id, query_ordinal, repository_rank);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_repository_work
|
|
ON docker_depth_experiment_repositories(
|
|
experiment_id, work_state, repository_rank, query_ordinal, id
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_repository_due
|
|
ON docker_depth_experiment_repositories(
|
|
experiment_id, work_state, resolver_due_at, repository_rank, query_ordinal, id
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_targets_queue
|
|
ON docker_depth_experiment_targets(experiment_id, target_queue_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_targets_counter
|
|
ON docker_depth_experiment_targets(experiment_id, counter_ordinal);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_targets_dispatch
|
|
ON docker_depth_experiment_targets(experiment_id, state, dispatch_wave, dispatch_order, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_selections_rank
|
|
ON docker_depth_experiment_selections(experiment_repository_id, image_rank);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_selections_query
|
|
ON docker_depth_experiment_selections(
|
|
experiment_id, query_ordinal, image_rank, experiment_repository_id, id
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_selections_target
|
|
ON docker_depth_experiment_selections(experiment_target_id, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_candidate_skip
|
|
ON docker_depth_experiment_candidate_skips(
|
|
experiment_repository_id, candidate_kind, candidate_identity_sha256
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_candidate_skips
|
|
ON docker_depth_experiment_candidate_skips(
|
|
experiment_id, experiment_repository_id, candidate_kind, id
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_resolver_attempt_refund
|
|
ON docker_depth_resolver_attempt_refunds(experiment_repository_id, recovery_kind);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_depth_resolver_attempt_refunds
|
|
ON docker_depth_resolver_attempt_refunds(experiment_id, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_depth_resolver_disposition
|
|
ON docker_depth_resolver_dispositions(experiment_repository_id, disposition_kind);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_depth_resolver_dispositions
|
|
ON docker_depth_resolver_dispositions(experiment_id, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_reservation
|
|
ON docker_depth_experiment_scan_bindings(reservation_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_attempt
|
|
ON docker_depth_experiment_scan_bindings(experiment_target_id, attempt);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_experiment_bindings_scan
|
|
ON docker_depth_experiment_scan_bindings(target_scan_id) WHERE target_scan_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_docker_experiment_bindings_target
|
|
ON docker_depth_experiment_scan_bindings(experiment_target_id, state, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_finding_layer_exact
|
|
ON docker_finding_layer_attributions(finding_id, manifest_layer_id) WHERE attribution_state = 'exact';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_docker_finding_layer_unattributed
|
|
ON docker_finding_layer_attributions(finding_id) WHERE attribution_state = 'unattributed';
|
|
CREATE INDEX IF NOT EXISTS idx_docker_finding_layer_binding
|
|
ON docker_finding_layer_attributions(scan_binding_id, finding_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_experiment
|
|
ON target_queue_policy_events(experiment_id, id) WHERE experiment_id IS NOT NULL;
|
|
'''
|
|
|
|
|
|
PIPELINE_SCHEMA_SQL = f'''
|
|
CREATE TABLE IF NOT EXISTS runtime_schema_migrations (
|
|
version TEXT PRIMARY KEY,
|
|
applied_at TEXT NOT NULL,
|
|
code_sha256 TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS discovery_retry_queue (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
work_key TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
query TEXT NOT NULL,
|
|
source_cycle_id INTEGER,
|
|
policy_sha256 TEXT NOT NULL,
|
|
pass_kind TEXT NOT NULL,
|
|
work_kind TEXT NOT NULL,
|
|
page_start INTEGER NOT NULL DEFAULT 1,
|
|
page_end INTEGER NOT NULL DEFAULT 1,
|
|
next_page INTEGER NOT NULL DEFAULT 1,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
available_after TEXT,
|
|
lease_owner TEXT,
|
|
lease_token TEXT,
|
|
leased_at TEXT,
|
|
lease_expires_at TEXT,
|
|
last_error_category TEXT,
|
|
held_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
CONSTRAINT discovery_retry_queue_identity_check CHECK (
|
|
length(work_key) = 64 AND work_key = lower(work_key)
|
|
AND length(source) BETWEEN 1 AND 64
|
|
AND length(query) BETWEEN 1 AND 256
|
|
AND length(policy_sha256) = 64 AND policy_sha256 = lower(policy_sha256)
|
|
AND pass_kind IN ('ordinary','deep')
|
|
AND work_kind IN ('query','page','range')
|
|
),
|
|
CONSTRAINT discovery_retry_queue_page_check CHECK (
|
|
page_start BETWEEN 1 AND 30
|
|
AND page_end BETWEEN page_start AND 30
|
|
AND next_page BETWEEN page_start AND page_end
|
|
AND (work_kind <> 'query' OR page_start = 1)
|
|
AND (work_kind <> 'page' OR page_start = page_end)
|
|
),
|
|
CONSTRAINT discovery_retry_queue_status_check CHECK (
|
|
status IN ('pending','leased','held')
|
|
AND attempts BETWEEN 0 AND 1000000
|
|
),
|
|
CONSTRAINT discovery_retry_queue_error_check CHECK (
|
|
last_error_category IS NULL OR last_error_category IN (
|
|
'account_pool_exhausted','auth_forbidden','auth_invalid','auth_unavailable',
|
|
'invalid_payload','network','page_unavailable','policy_mismatch',
|
|
'provider_cooldown','provider_unavailable','query_removed','rate_limit',
|
|
'remote_transient','request_failed','tail_unavailable','transport'
|
|
)
|
|
),
|
|
{_docker_depth_check_clause('discovery_retry_queue', 'discovery_retry_queue_lifecycle_check')},
|
|
{_docker_depth_check_clause('discovery_retry_queue', 'discovery_retry_queue_sha256_check')},
|
|
FOREIGN KEY(source_cycle_id) REFERENCES source_cycles(id)
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_discovery_retry_queue_work_key
|
|
ON discovery_retry_queue(work_key);
|
|
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_due
|
|
ON discovery_retry_queue(source, available_after, id) WHERE status = 'pending';
|
|
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_lease
|
|
ON discovery_retry_queue(lease_expires_at, id) WHERE status = 'leased';
|
|
CREATE INDEX IF NOT EXISTS idx_discovery_retry_queue_policy
|
|
ON discovery_retry_queue(source, query, policy_sha256, status, id);
|
|
|
|
CREATE TABLE IF NOT EXISTS runtime_final_cutover (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
marker TEXT NOT NULL,
|
|
checked_at TEXT NOT NULL,
|
|
evidence_sha256 TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS runtime_operations (
|
|
operation_id TEXT PRIMARY KEY CHECK (length(operation_id) = 36),
|
|
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
|
|
action TEXT NOT NULL CHECK (length(action) BETWEEN 1 AND 128),
|
|
target_kind TEXT NOT NULL CHECK (length(target_kind) BETWEEN 1 AND 64),
|
|
target_ref TEXT NOT NULL DEFAULT '' CHECK (length(target_ref) <= 512),
|
|
status TEXT NOT NULL DEFAULT 'requested' CHECK (
|
|
status IN ('requested','running','succeeded','failed','rolled_back','failed_hold','canceled')
|
|
),
|
|
safe_category TEXT CHECK (safe_category IS NULL OR length(safe_category) <= 128),
|
|
safe_detail TEXT CHECK (safe_detail IS NULL OR length(safe_detail) <= 2048),
|
|
expected_revision INTEGER CHECK (expected_revision IS NULL OR expected_revision >= 0),
|
|
resulting_revision INTEGER CHECK (resulting_revision IS NULL OR resulting_revision >= 0),
|
|
expected_identity_json TEXT NOT NULL DEFAULT '{{}}'
|
|
CHECK (length(expected_identity_json) <= 65536),
|
|
resulting_identity_json TEXT CHECK (
|
|
resulting_identity_json IS NULL OR length(resulting_identity_json) <= 65536
|
|
),
|
|
agent_state TEXT NOT NULL DEFAULT 'not_required' CHECK (
|
|
agent_state IN (
|
|
'not_required','pending','running','succeeded','failed','rolled_back','failed_hold'
|
|
)
|
|
),
|
|
agent_result_sha256 TEXT CHECK (
|
|
agent_result_sha256 IS NULL OR (
|
|
length(agent_result_sha256) = 64 AND agent_result_sha256 = lower(agent_result_sha256)
|
|
)
|
|
),
|
|
requested_at TEXT NOT NULL,
|
|
started_at TEXT,
|
|
completed_at TEXT,
|
|
agent_reconciled_at TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS runtime_operations_control (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
revision INTEGER NOT NULL DEFAULT 0 CHECK (revision >= 0),
|
|
discovery_paused INTEGER NOT NULL DEFAULT 0 CHECK (discovery_paused IN (0,1)),
|
|
dispatch_paused INTEGER NOT NULL DEFAULT 0 CHECK (dispatch_paused IN (0,1)),
|
|
drain_state TEXT NOT NULL DEFAULT 'normal'
|
|
CHECK (drain_state IN ('normal','draining','drained')),
|
|
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
|
|
operation_id TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(operation_id) REFERENCES runtime_operations(operation_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS runtime_audit_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
operation_id TEXT,
|
|
actor TEXT NOT NULL CHECK (length(actor) BETWEEN 1 AND 256),
|
|
action TEXT NOT NULL CHECK (length(action) BETWEEN 1 AND 128),
|
|
target_kind TEXT NOT NULL CHECK (length(target_kind) BETWEEN 1 AND 64),
|
|
target_ref TEXT NOT NULL DEFAULT '' CHECK (length(target_ref) <= 512),
|
|
result TEXT NOT NULL CHECK (
|
|
result IN (
|
|
'accepted','succeeded','rejected','failed','rolled_back','canceled','failed_hold'
|
|
)
|
|
),
|
|
safe_category TEXT CHECK (safe_category IS NULL OR length(safe_category) <= 128),
|
|
before_identity_json TEXT CHECK (
|
|
before_identity_json IS NULL OR length(before_identity_json) <= 65536
|
|
),
|
|
after_identity_json TEXT CHECK (
|
|
after_identity_json IS NULL OR length(after_identity_json) <= 65536
|
|
),
|
|
before_bytes INTEGER CHECK (before_bytes IS NULL OR before_bytes >= 0),
|
|
after_bytes INTEGER CHECK (after_bytes IS NULL OR after_bytes >= 0),
|
|
previous_event_id INTEGER UNIQUE,
|
|
previous_event_sha256 TEXT,
|
|
event_sha256 TEXT NOT NULL UNIQUE CHECK (
|
|
length(event_sha256) = 64 AND event_sha256 = lower(event_sha256)
|
|
),
|
|
created_at TEXT NOT NULL,
|
|
CONSTRAINT runtime_audit_events_chain_check CHECK (
|
|
(previous_event_id IS NULL AND previous_event_sha256 IS NULL)
|
|
OR (previous_event_id IS NOT NULL AND length(previous_event_sha256) = 64
|
|
AND previous_event_sha256 = lower(previous_event_sha256))
|
|
),
|
|
FOREIGN KEY(operation_id) REFERENCES runtime_operations(operation_id),
|
|
FOREIGN KEY(previous_event_id) REFERENCES runtime_audit_events(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_capacity (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
bundle_items BIGINT NOT NULL DEFAULT 0 CHECK (bundle_items >= 0),
|
|
bundle_bytes BIGINT NOT NULL DEFAULT 0 CHECK (bundle_bytes >= 0),
|
|
projection_items BIGINT NOT NULL DEFAULT 0 CHECK (projection_items >= 0),
|
|
projection_bytes BIGINT NOT NULL DEFAULT 0 CHECK (projection_bytes >= 0),
|
|
keycheck_items BIGINT NOT NULL DEFAULT 0 CHECK (keycheck_items >= 0),
|
|
keycheck_bytes BIGINT NOT NULL DEFAULT 0 CHECK (keycheck_bytes >= 0),
|
|
quarantine_items BIGINT NOT NULL DEFAULT 0 CHECK (quarantine_items >= 0),
|
|
quarantine_bytes BIGINT NOT NULL DEFAULT 0 CHECK (quarantine_bytes >= 0),
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS remote_worker_users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_key TEXT NOT NULL UNIQUE,
|
|
active_assignment_cap INTEGER NOT NULL DEFAULT 0
|
|
CHECK (active_assignment_cap BETWEEN 0 AND 10000),
|
|
disabled_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS remote_worker_devices (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
device_key TEXT NOT NULL UNIQUE,
|
|
token_sha256 TEXT NOT NULL UNIQUE CHECK (
|
|
length(token_sha256) = 64 AND token_sha256 = lower(token_sha256)
|
|
),
|
|
revoked_at TEXT,
|
|
last_contact_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(id, user_id),
|
|
FOREIGN KEY(user_id) REFERENCES remote_worker_users(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS admission_intents (
|
|
reservation_token TEXT PRIMARY KEY,
|
|
intent_sha256 TEXT NOT NULL,
|
|
state TEXT NOT NULL CHECK (state IN ('pending','committed','aborted')),
|
|
reservation_id BIGINT,
|
|
resolution_detail TEXT,
|
|
remote_user_id INTEGER,
|
|
remote_device_id INTEGER,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
resolved_at TEXT,
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(remote_device_id, remote_user_id)
|
|
REFERENCES remote_worker_devices(id, user_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS admission_intent_retirement (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
retired_count BIGINT NOT NULL DEFAULT 0,
|
|
chain_sha256 TEXT NOT NULL,
|
|
cursor_token TEXT NOT NULL DEFAULT '',
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_artifact_retirement (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
retired_count BIGINT NOT NULL DEFAULT 0,
|
|
chain_sha256 TEXT NOT NULL,
|
|
cursor_id BIGINT NOT NULL DEFAULT 0,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS result_reservations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
reservation_token TEXT NOT NULL UNIQUE,
|
|
bundle_id TEXT NOT NULL UNIQUE,
|
|
scan_event_id TEXT NOT NULL UNIQUE,
|
|
queue_id INTEGER NOT NULL,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
query TEXT,
|
|
target TEXT NOT NULL,
|
|
normalized_target TEXT NOT NULL,
|
|
claim_lease_owner TEXT NOT NULL,
|
|
claim_lease_token TEXT NOT NULL,
|
|
claim_batch TEXT,
|
|
producer_instance_id TEXT NOT NULL,
|
|
producer_pid BIGINT NOT NULL,
|
|
producer_creation_time TEXT NOT NULL,
|
|
producer_executable TEXT NOT NULL,
|
|
assignment_kind TEXT NOT NULL DEFAULT 'local'
|
|
CHECK (assignment_kind IN ('local','remote')),
|
|
remote_user_id INTEGER,
|
|
remote_device_id INTEGER,
|
|
remote_issued_at TEXT,
|
|
remote_expires_at TEXT,
|
|
remote_result_upload_body_timeout_seconds INTEGER CHECK (
|
|
remote_result_upload_body_timeout_seconds IS NULL
|
|
OR remote_result_upload_body_timeout_seconds BETWEEN 30 AND 86400
|
|
),
|
|
remote_effective_config_sha256 TEXT,
|
|
remote_client_compat_sha256 TEXT,
|
|
remote_execution_snapshot_json TEXT CHECK (
|
|
remote_execution_snapshot_json IS NULL OR length(remote_execution_snapshot_json) <= 65536
|
|
),
|
|
remote_execution_snapshot_sha256 TEXT,
|
|
remote_resolution_kind TEXT CHECK (
|
|
remote_resolution_kind IS NULL OR remote_resolution_kind IN (
|
|
'bundle_accepted','prebundle_report','expired'
|
|
)
|
|
),
|
|
remote_payload_sha256 TEXT,
|
|
remote_receipt_id TEXT,
|
|
remote_resolution_json TEXT CHECK (
|
|
remote_resolution_json IS NULL OR length(remote_resolution_json) <= 16384
|
|
),
|
|
remote_resolved_at TEXT,
|
|
remote_diagnostic_projection_version INTEGER CHECK (
|
|
remote_diagnostic_projection_version IS NULL
|
|
OR remote_diagnostic_projection_version IN (0,1)
|
|
),
|
|
remote_diagnostic_count INTEGER CHECK (
|
|
remote_diagnostic_count IS NULL OR remote_diagnostic_count >= 0
|
|
),
|
|
remote_diagnostic_uids_sha256 TEXT,
|
|
declared_bundle_bytes BIGINT NOT NULL CHECK (declared_bundle_bytes > 0),
|
|
reserved_bundle_bytes BIGINT NOT NULL CHECK (reserved_bundle_bytes > 0),
|
|
reserved_projection_items BIGINT NOT NULL DEFAULT 1,
|
|
reserved_projection_bytes BIGINT NOT NULL,
|
|
reserved_candidate_items BIGINT NOT NULL,
|
|
reserved_candidate_bytes BIGINT NOT NULL,
|
|
ready_relative_path TEXT NOT NULL,
|
|
state TEXT NOT NULL CHECK (
|
|
state IN ('scanning','ready','ingesting','db_committed','acknowledged','refunded','quarantined')
|
|
),
|
|
producer_lease_expires_at TEXT NOT NULL,
|
|
bundle_credit_released INTEGER NOT NULL DEFAULT 0,
|
|
projection_credit_transferred INTEGER NOT NULL DEFAULT 0,
|
|
candidate_credit_transferred INTEGER NOT NULL DEFAULT 0,
|
|
last_error_code TEXT,
|
|
last_error_detail TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
refunded_at TEXT,
|
|
released_at TEXT,
|
|
cleanup_attempts INTEGER NOT NULL DEFAULT 0,
|
|
cleanup_available_after TEXT,
|
|
git_scan_plan_json TEXT,
|
|
git_scan_plan_sha256 TEXT,
|
|
docker_layer_plan_json TEXT,
|
|
docker_layer_plan_sha256 TEXT,
|
|
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(remote_device_id, remote_user_id)
|
|
REFERENCES remote_worker_devices(id, user_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS worker_progress_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
reservation_id INTEGER NOT NULL,
|
|
remote_device_id INTEGER NOT NULL,
|
|
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
sequence INTEGER NOT NULL CHECK (sequence > 0),
|
|
event_type TEXT NOT NULL CHECK (length(event_type) BETWEEN 1 AND 128),
|
|
phase TEXT NOT NULL CHECK (length(phase) BETWEEN 1 AND 64),
|
|
event_timestamp TEXT NOT NULL,
|
|
phase_started_at TEXT,
|
|
instance_id TEXT NOT NULL CHECK (length(instance_id) BETWEEN 1 AND 256),
|
|
slot_id INTEGER NOT NULL CHECK (slot_id >= 0),
|
|
source TEXT NOT NULL CHECK (length(source) BETWEEN 1 AND 64),
|
|
event_json TEXT NOT NULL CHECK (length(event_json) BETWEEN 2 AND 65536),
|
|
event_sha256 TEXT NOT NULL CHECK (
|
|
length(event_sha256) = 64 AND event_sha256 = lower(event_sha256)
|
|
),
|
|
received_at TEXT NOT NULL,
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(remote_device_id) REFERENCES remote_worker_devices(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS worker_diagnostics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
diagnostic_uid TEXT NOT NULL CHECK (length(diagnostic_uid) BETWEEN 1 AND 256),
|
|
reservation_id INTEGER NOT NULL,
|
|
target_scan_id INTEGER,
|
|
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
scan_event_id TEXT CHECK (
|
|
scan_event_id IS NULL OR (
|
|
length(scan_event_id) BETWEEN 32 AND 64
|
|
AND scan_event_id = lower(scan_event_id)
|
|
)
|
|
),
|
|
slot_id INTEGER NOT NULL CHECK (slot_id >= 0),
|
|
attempt INTEGER NOT NULL CHECK (attempt > 0),
|
|
source TEXT NOT NULL CHECK (length(source) BETWEEN 1 AND 64),
|
|
phase TEXT NOT NULL CHECK (length(phase) BETWEEN 1 AND 64),
|
|
kind TEXT NOT NULL CHECK (length(kind) BETWEEN 1 AND 64),
|
|
category TEXT NOT NULL CHECK (length(category) BETWEEN 1 AND 128),
|
|
code TEXT NOT NULL CHECK (length(code) BETWEEN 1 AND 256),
|
|
summary TEXT NOT NULL CHECK (length(summary) BETWEEN 1 AND 2048),
|
|
retryable INTEGER NOT NULL CHECK (retryable IN (0,1)),
|
|
occurred_at TEXT NOT NULL,
|
|
captured_at TEXT NOT NULL,
|
|
envelope_json TEXT NOT NULL CHECK (length(envelope_json) BETWEEN 2 AND 65536),
|
|
envelope_sha256 TEXT NOT NULL CHECK (
|
|
length(envelope_sha256) = 64 AND envelope_sha256 = lower(envelope_sha256)
|
|
),
|
|
body_payload_json TEXT CHECK (
|
|
body_payload_json IS NULL OR length(body_payload_json) <= 65536
|
|
),
|
|
log_payload_json TEXT CHECK (
|
|
log_payload_json IS NULL OR length(log_payload_json) <= 65536
|
|
),
|
|
received_at TEXT NOT NULL,
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_content_blobs (
|
|
digest TEXT NOT NULL,
|
|
coverage_policy_sha256 TEXT NOT NULL,
|
|
descriptor_kind TEXT NOT NULL CHECK (descriptor_kind IN ('config','layer')),
|
|
declared_bytes INTEGER NOT NULL CHECK (declared_bytes >= 0),
|
|
media_type TEXT NOT NULL,
|
|
state TEXT NOT NULL DEFAULT 'pending'
|
|
CHECK (state IN ('pending','leased','submitted','covered','failed')),
|
|
attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
|
|
max_attempts INTEGER NOT NULL CHECK (max_attempts > 0),
|
|
available_after TEXT,
|
|
lease_reservation_id INTEGER,
|
|
lease_token TEXT,
|
|
lease_plan_sha256 TEXT,
|
|
lease_expires_at TEXT,
|
|
covered_reservation_id INTEGER,
|
|
covered_scan_event_id TEXT,
|
|
covered_policy_sha256 TEXT,
|
|
verified_bytes INTEGER,
|
|
covered_at TEXT,
|
|
last_error_code TEXT,
|
|
last_error_detail TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY(digest, coverage_policy_sha256),
|
|
FOREIGN KEY(lease_reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(covered_reservation_id) REFERENCES result_reservations(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_image_blob_coverage (
|
|
queue_id INTEGER NOT NULL,
|
|
manifest_digest TEXT NOT NULL,
|
|
position INTEGER NOT NULL CHECK (position >= 0),
|
|
blob_digest TEXT NOT NULL,
|
|
coverage_policy_sha256 TEXT NOT NULL,
|
|
selection_policy_sha256 TEXT NOT NULL DEFAULT '',
|
|
descriptor_kind TEXT NOT NULL CHECK (descriptor_kind IN ('config','layer')),
|
|
plan_sha256 TEXT NOT NULL,
|
|
reservation_id INTEGER NOT NULL,
|
|
selected INTEGER NOT NULL CHECK (selected IN (0,1)),
|
|
selection_reason TEXT NOT NULL,
|
|
coverage_state TEXT NOT NULL CHECK (
|
|
coverage_state IN (
|
|
'selected','leased','shared_pending','covered',
|
|
'retryable_failed','terminal_failed','skipped'
|
|
)
|
|
),
|
|
covered_at TEXT,
|
|
last_error_code TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY(reservation_id, position),
|
|
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(blob_digest, coverage_policy_sha256)
|
|
REFERENCES docker_content_blobs(digest, coverage_policy_sha256),
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS docker_adaptive_shadow_reports (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
report_token TEXT NOT NULL UNIQUE,
|
|
evaluator_version TEXT NOT NULL,
|
|
state TEXT NOT NULL DEFAULT 'running'
|
|
CHECK (state IN ('running','completed','failed')),
|
|
scan_policy_sha256 TEXT NOT NULL,
|
|
execution_policy_sha256 TEXT NOT NULL,
|
|
selection_policy_sha256 TEXT NOT NULL,
|
|
cohort_size INTEGER NOT NULL CHECK (cohort_size >= 50 AND cohort_size <= 100),
|
|
completed_pairs INTEGER NOT NULL DEFAULT 0 CHECK (completed_pairs >= 0),
|
|
full_routed_count INTEGER NOT NULL DEFAULT 0 CHECK (full_routed_count >= 0),
|
|
adaptive_routed_count INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_routed_count >= 0),
|
|
routed_intersection_count INTEGER NOT NULL DEFAULT 0 CHECK (routed_intersection_count >= 0),
|
|
full_detector_count INTEGER NOT NULL DEFAULT 0 CHECK (full_detector_count >= 0),
|
|
adaptive_detector_count INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_detector_count >= 0),
|
|
detector_intersection_count INTEGER NOT NULL DEFAULT 0
|
|
CHECK (detector_intersection_count >= 0),
|
|
full_slot_ms INTEGER NOT NULL DEFAULT 0 CHECK (full_slot_ms >= 0),
|
|
adaptive_slot_ms INTEGER NOT NULL DEFAULT 0 CHECK (adaptive_slot_ms >= 0),
|
|
omitted_descriptor_count INTEGER NOT NULL DEFAULT 0 CHECK (omitted_descriptor_count >= 0),
|
|
failure_count INTEGER NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
|
|
privacy_violation_count INTEGER NOT NULL DEFAULT 0 CHECK (privacy_violation_count >= 0),
|
|
safety_regression_count INTEGER NOT NULL DEFAULT 0 CHECK (safety_regression_count >= 0),
|
|
selection_metrics_json TEXT NOT NULL DEFAULT '{{}}',
|
|
sink_checkpoint_count INTEGER NOT NULL DEFAULT 0 CHECK (sink_checkpoint_count >= 0),
|
|
routed_recall_ppm INTEGER NOT NULL DEFAULT 0 CHECK (routed_recall_ppm >= 0),
|
|
slot_ratio_ppm INTEGER NOT NULL DEFAULT 0 CHECK (slot_ratio_ppm >= 0),
|
|
recall_threshold_ppm INTEGER NOT NULL DEFAULT 850000
|
|
CHECK (recall_threshold_ppm = 850000),
|
|
slot_threshold_ppm INTEGER NOT NULL DEFAULT 400000
|
|
CHECK (slot_threshold_ppm = 400000),
|
|
passed INTEGER NOT NULL DEFAULT 0 CHECK (passed IN (0,1)),
|
|
lease_owner TEXT NOT NULL,
|
|
lease_token TEXT NOT NULL,
|
|
lease_expires_at TEXT NOT NULL,
|
|
started_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS target_queue_policy_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
queue_id INTEGER NOT NULL,
|
|
action TEXT NOT NULL,
|
|
prior_status TEXT NOT NULL,
|
|
next_status TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
query TEXT NOT NULL,
|
|
reason_code TEXT NOT NULL,
|
|
config_sha256 TEXT NOT NULL,
|
|
policy_sha256 TEXT NOT NULL,
|
|
manifest_sha256 TEXT NOT NULL,
|
|
review_audit_sha256 TEXT NOT NULL UNIQUE,
|
|
reverses_event_id INTEGER UNIQUE,
|
|
experiment_id INTEGER,
|
|
prior_updated_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_transition_check')},
|
|
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_hash_check')},
|
|
{_docker_depth_check_clause('target_queue_policy_events', 'target_queue_policy_events_experiment_ownership_check')},
|
|
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(reverses_event_id) REFERENCES target_queue_policy_events(id),
|
|
FOREIGN KEY(experiment_id) REFERENCES docker_depth_experiments(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS result_bundles (
|
|
reservation_id INTEGER PRIMARY KEY,
|
|
bundle_id TEXT NOT NULL UNIQUE,
|
|
scan_event_id TEXT NOT NULL UNIQUE,
|
|
scan_event_hash TEXT NOT NULL,
|
|
format_version INTEGER NOT NULL CHECK (format_version = 2),
|
|
relative_path TEXT NOT NULL UNIQUE,
|
|
actual_bytes BIGINT NOT NULL CHECK (actual_bytes > 0),
|
|
frame_count INTEGER NOT NULL,
|
|
finding_count INTEGER NOT NULL,
|
|
error_count INTEGER NOT NULL,
|
|
candidate_count INTEGER NOT NULL,
|
|
state TEXT NOT NULL CHECK (state IN ('ready','ingesting','db_committed','acknowledged','quarantined')),
|
|
available_after TEXT,
|
|
ingest_attempts INTEGER NOT NULL DEFAULT 0,
|
|
ingest_lease_generation BIGINT,
|
|
ingest_lease_token TEXT,
|
|
ingest_lease_expires_at TEXT,
|
|
target_scan_id INTEGER,
|
|
ready_at TEXT NOT NULL,
|
|
committed_at TEXT,
|
|
acknowledged_at TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_leases (
|
|
worker_name TEXT PRIMARY KEY,
|
|
generation BIGINT NOT NULL DEFAULT 0,
|
|
lease_token TEXT,
|
|
supervisor_instance_id TEXT,
|
|
owner_pid BIGINT,
|
|
owner_creation_time TEXT,
|
|
owner_executable TEXT,
|
|
state TEXT NOT NULL DEFAULT 'released'
|
|
CHECK (state IN ('starting','recovering','ready','stopping','released','failed')),
|
|
acquired_at TEXT,
|
|
heartbeat_at TEXT,
|
|
lease_expires_at TEXT,
|
|
last_error TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS scan_result_compat (
|
|
target_scan_id INTEGER PRIMARY KEY,
|
|
schema_version INTEGER NOT NULL CHECK (schema_version = 2),
|
|
metadata_json TEXT NOT NULL,
|
|
metadata_sha256 TEXT NOT NULL,
|
|
metadata_bytes BIGINT NOT NULL,
|
|
reconstruction_status TEXT NOT NULL CHECK (reconstruction_status IN ('exact','bounded','legacy')),
|
|
omitted_payload_sha256 TEXT,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS finding_compat_payloads (
|
|
finding_id INTEGER PRIMARY KEY,
|
|
raw_value TEXT,
|
|
raw_v2_value TEXT,
|
|
structured_data_json TEXT,
|
|
extra_data_json TEXT,
|
|
analysis_info_json TEXT,
|
|
extension_json TEXT,
|
|
payload_sha256 TEXT NOT NULL,
|
|
payload_bytes BIGINT NOT NULL,
|
|
payload_omitted INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_jobs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
job_kind TEXT NOT NULL CHECK (job_kind IN ('scan_event','keycheck_event','rebuild')),
|
|
event_id TEXT NOT NULL,
|
|
event_hash TEXT NOT NULL,
|
|
target_scan_id INTEGER,
|
|
keycheck_result_id INTEGER,
|
|
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','leased','completed','quarantined')),
|
|
required_stream_mask INTEGER NOT NULL,
|
|
capacity_items BIGINT NOT NULL,
|
|
capacity_bytes BIGINT NOT NULL,
|
|
capacity_released INTEGER NOT NULL DEFAULT 0,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
available_after TEXT,
|
|
lease_generation BIGINT,
|
|
lease_token TEXT,
|
|
lease_expires_at TEXT,
|
|
last_error_code TEXT,
|
|
last_error_detail TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
UNIQUE(job_kind, event_id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
|
|
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_streams (
|
|
stream_name TEXT PRIMARY KEY,
|
|
base_relative_path TEXT NOT NULL UNIQUE,
|
|
current_generation BIGINT NOT NULL DEFAULT 0,
|
|
rotation_bytes BIGINT NOT NULL,
|
|
max_generations INTEGER NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_cursors (
|
|
stream_name TEXT PRIMARY KEY,
|
|
generation BIGINT NOT NULL,
|
|
committed_offset BIGINT NOT NULL,
|
|
last_append_id INTEGER,
|
|
last_job_id INTEGER,
|
|
last_event_id TEXT,
|
|
last_event_hash TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name),
|
|
FOREIGN KEY(last_append_id) REFERENCES projection_appends(id),
|
|
FOREIGN KEY(last_job_id) REFERENCES projection_jobs(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_appends (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
job_id INTEGER NOT NULL,
|
|
stream_name TEXT NOT NULL,
|
|
event_id TEXT NOT NULL,
|
|
event_hash TEXT NOT NULL,
|
|
generation BIGINT NOT NULL,
|
|
byte_offset BIGINT NOT NULL,
|
|
byte_length BIGINT NOT NULL,
|
|
payload_sha256 TEXT NOT NULL,
|
|
record_count INTEGER NOT NULL,
|
|
state TEXT NOT NULL CHECK (state IN ('prepared','appended')),
|
|
prepared_at TEXT NOT NULL,
|
|
appended_at TEXT,
|
|
UNIQUE(job_id, stream_name),
|
|
UNIQUE(stream_name, generation, byte_offset),
|
|
FOREIGN KEY(job_id) REFERENCES projection_jobs(id),
|
|
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_append_audit (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
append_id BIGINT NOT NULL,
|
|
job_id INTEGER NOT NULL,
|
|
stream_name TEXT NOT NULL,
|
|
event_id TEXT NOT NULL,
|
|
event_hash TEXT NOT NULL,
|
|
generation BIGINT NOT NULL,
|
|
byte_offset BIGINT NOT NULL,
|
|
byte_length BIGINT NOT NULL,
|
|
payload_sha256 TEXT NOT NULL,
|
|
state TEXT NOT NULL CHECK (state IN ('canceled','quarantined')),
|
|
reason_code TEXT NOT NULL,
|
|
reason_detail TEXT,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(job_id) REFERENCES projection_jobs(id),
|
|
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS projection_rotations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
stream_name TEXT NOT NULL,
|
|
from_generation BIGINT NOT NULL,
|
|
to_generation BIGINT NOT NULL,
|
|
source_bytes BIGINT NOT NULL,
|
|
segment_relative_path TEXT NOT NULL,
|
|
state TEXT NOT NULL CHECK (state IN ('prepared','renamed','completed')),
|
|
created_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
UNIQUE(stream_name, to_generation),
|
|
FOREIGN KEY(stream_name) REFERENCES projection_streams(stream_name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS keycheck_credentials (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
service TEXT NOT NULL,
|
|
credential_hash TEXT NOT NULL,
|
|
provider_key_hash TEXT NOT NULL,
|
|
candidate_kind TEXT NOT NULL,
|
|
secret_text TEXT,
|
|
secret_json TEXT,
|
|
key_masked TEXT,
|
|
endpoint TEXT,
|
|
principal TEXT,
|
|
metadata_json TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(service, credential_hash)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS keycheck_candidates (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
candidate_uid TEXT NOT NULL UNIQUE,
|
|
credential_id INTEGER NOT NULL,
|
|
service TEXT NOT NULL,
|
|
routed_service TEXT NOT NULL DEFAULT '',
|
|
secret_hash TEXT NOT NULL,
|
|
finding_id INTEGER,
|
|
target_scan_id INTEGER,
|
|
scan_event_id TEXT,
|
|
finding_uid TEXT,
|
|
source TEXT,
|
|
query TEXT,
|
|
target TEXT,
|
|
detector_name TEXT,
|
|
found_at TEXT,
|
|
metadata_json TEXT,
|
|
state TEXT NOT NULL DEFAULT 'pending'
|
|
CHECK (state IN ('pending','leased','deferred','completed','quarantined')),
|
|
priority INTEGER NOT NULL DEFAULT 0,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
available_after TEXT,
|
|
lease_owner TEXT,
|
|
lease_token TEXT,
|
|
lease_expires_at TEXT,
|
|
keycheck_result_id INTEGER,
|
|
capacity_bytes BIGINT NOT NULL,
|
|
capacity_released INTEGER NOT NULL DEFAULT 0,
|
|
result_projection_reserved_bytes BIGINT NOT NULL DEFAULT 0,
|
|
result_projection_credit_transferred INTEGER NOT NULL DEFAULT 0,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id),
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
|
|
FOREIGN KEY(keycheck_result_id) REFERENCES keycheck_results(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS keycheck_current_state (
|
|
credential_id INTEGER PRIMARY KEY,
|
|
service TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
status_group TEXT NOT NULL,
|
|
last_result_id INTEGER NOT NULL,
|
|
result_source TEXT NOT NULL,
|
|
checked_at TEXT NOT NULL,
|
|
recheck_after TEXT,
|
|
state_version BIGINT NOT NULL DEFAULT 1,
|
|
metadata_json TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(credential_id) REFERENCES keycheck_credentials(id),
|
|
FOREIGN KEY(last_result_id) REFERENCES keycheck_results(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_quarantine (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
subsystem TEXT NOT NULL,
|
|
object_type TEXT NOT NULL,
|
|
object_id INTEGER,
|
|
reservation_id INTEGER,
|
|
projection_job_id INTEGER,
|
|
keycheck_candidate_id INTEGER,
|
|
event_id TEXT,
|
|
payload_sha256 TEXT,
|
|
reason_code TEXT NOT NULL,
|
|
reason_detail TEXT,
|
|
source_relative_path TEXT,
|
|
byte_count BIGINT NOT NULL DEFAULT 0,
|
|
capacity_items BIGINT NOT NULL DEFAULT 1,
|
|
capacity_bytes BIGINT NOT NULL DEFAULT 0,
|
|
capacity_credit_applied INTEGER NOT NULL DEFAULT 1,
|
|
review_status TEXT NOT NULL DEFAULT 'pending'
|
|
CHECK (review_status IN ('pending','approved_retry','approved_rescan','discarded','resolved')),
|
|
detected_at TEXT NOT NULL,
|
|
resolved_at TEXT,
|
|
review_audit_sha256 TEXT,
|
|
FOREIGN KEY(reservation_id) REFERENCES result_reservations(id),
|
|
FOREIGN KEY(projection_job_id) REFERENCES projection_jobs(id),
|
|
FOREIGN KEY(keycheck_candidate_id) REFERENCES keycheck_candidates(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_artifacts (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
subsystem TEXT NOT NULL,
|
|
artifact_kind TEXT NOT NULL,
|
|
owner_id BIGINT NOT NULL,
|
|
owner_key TEXT NOT NULL DEFAULT '',
|
|
relative_path TEXT NOT NULL,
|
|
payload_sha256 TEXT,
|
|
byte_count BIGINT NOT NULL DEFAULT 0,
|
|
state TEXT NOT NULL CHECK (state IN ('expected','present','quarantined','deleted')),
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
deleted_at TEXT,
|
|
cleanup_attempts INTEGER NOT NULL DEFAULT 0,
|
|
cleanup_available_after TEXT,
|
|
cleanup_last_error TEXT,
|
|
UNIQUE(subsystem, artifact_kind, owner_id, owner_key),
|
|
UNIQUE(subsystem, relative_path)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS janitor_cursors (
|
|
layout_name TEXT PRIMARY KEY,
|
|
last_name TEXT NOT NULL DEFAULT '',
|
|
wrap_count BIGINT NOT NULL DEFAULT 0,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS keycheck_recheck_cursors (
|
|
cursor_key TEXT PRIMARY KEY,
|
|
service TEXT NOT NULL,
|
|
status_scope TEXT NOT NULL,
|
|
last_credential_id BIGINT NOT NULL DEFAULT 0,
|
|
wrap_count BIGINT NOT NULL DEFAULT 0,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_result_reservation_queue_lease
|
|
ON result_reservations(queue_id, claim_lease_token);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_worker_progress_reservation_sequence
|
|
ON worker_progress_events(reservation_id, sequence);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_progress_reservation_received
|
|
ON worker_progress_events(reservation_id, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_progress_device_received
|
|
ON worker_progress_events(remote_device_id, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_progress_phase_received
|
|
ON worker_progress_events(phase, received_at, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_worker_diagnostics_uid
|
|
ON worker_diagnostics(diagnostic_uid);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_reservation_received
|
|
ON worker_diagnostics(reservation_id, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_scan_received
|
|
ON worker_diagnostics(target_scan_id, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_phase_received
|
|
ON worker_diagnostics(phase, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_category_code
|
|
ON worker_diagnostics(category, code, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_code_received
|
|
ON worker_diagnostics(code, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_kind_received
|
|
ON worker_diagnostics(kind, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_worker_diagnostics_retryable_received
|
|
ON worker_diagnostics(retryable, received_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_runtime_operations_status_updated
|
|
ON runtime_operations(status, updated_at, operation_id);
|
|
CREATE INDEX IF NOT EXISTS idx_runtime_operations_agent_state_updated
|
|
ON runtime_operations(agent_state, updated_at, operation_id);
|
|
CREATE INDEX IF NOT EXISTS idx_runtime_audit_events_created
|
|
ON runtime_audit_events(created_at DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_runtime_audit_events_operation
|
|
ON runtime_audit_events(operation_id, id DESC);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_users_key
|
|
ON remote_worker_users(user_key);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_devices_key
|
|
ON remote_worker_devices(device_key);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_remote_worker_devices_token
|
|
ON remote_worker_devices(token_sha256);
|
|
CREATE INDEX IF NOT EXISTS idx_admission_intents_remote_device
|
|
ON admission_intents(remote_device_id, created_at) WHERE remote_device_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_active_user
|
|
ON result_reservations(remote_user_id, id)
|
|
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_expiry
|
|
ON result_reservations(remote_expires_at, id)
|
|
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NULL AND state = 'scanning';
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_device_history
|
|
ON result_reservations(remote_device_id, remote_issued_at, id)
|
|
WHERE assignment_kind = 'remote';
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_remote_resolved
|
|
ON result_reservations(remote_resolved_at, id)
|
|
WHERE assignment_kind = 'remote' AND remote_resolved_at IS NOT NULL;
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_result_reservations_remote_receipt
|
|
ON result_reservations(remote_receipt_id) WHERE remote_receipt_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_recovery
|
|
ON result_reservations(state, producer_lease_expires_at, id)
|
|
WHERE state IN ('scanning','ready','ingesting','db_committed');
|
|
CREATE INDEX IF NOT EXISTS idx_result_reservations_queue_history
|
|
ON result_reservations(queue_id, id DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_result_bundles_ready
|
|
ON result_bundles(state, available_after, ready_at, reservation_id)
|
|
WHERE state IN ('ready','ingesting','db_committed');
|
|
CREATE INDEX IF NOT EXISTS idx_result_bundles_ingest_lease
|
|
ON result_bundles(state, ingest_lease_expires_at)
|
|
WHERE state = 'ingesting';
|
|
CREATE INDEX IF NOT EXISTS idx_docker_content_blobs_reclaim
|
|
ON docker_content_blobs(state, available_after, lease_expires_at, digest);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_content_blobs_reservation
|
|
ON docker_content_blobs(lease_reservation_id, digest);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_manifest
|
|
ON docker_image_blob_coverage(manifest_digest, coverage_state, position);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_blob
|
|
ON docker_image_blob_coverage(blob_digest, coverage_state, queue_id);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_reservation
|
|
ON docker_image_blob_coverage(reservation_id, position);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_image_blob_coverage_selection
|
|
ON docker_image_blob_coverage(
|
|
queue_id, manifest_digest, selection_policy_sha256, position, reservation_id
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_docker_adaptive_shadow_reports_gate
|
|
ON docker_adaptive_shadow_reports(
|
|
scan_policy_sha256, execution_policy_sha256, selection_policy_sha256,
|
|
state, completed_at, id
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_queue
|
|
ON target_queue_policy_events(queue_id, id DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_policy_events_manifest
|
|
ON target_queue_policy_events(manifest_sha256, id);
|
|
CREATE INDEX IF NOT EXISTS idx_projection_jobs_claim
|
|
ON projection_jobs(status, available_after, id) WHERE status = 'pending';
|
|
CREATE INDEX IF NOT EXISTS idx_projection_jobs_lease
|
|
ON projection_jobs(status, lease_expires_at, id) WHERE status = 'leased';
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_pending
|
|
ON keycheck_candidates(service, priority DESC, id) WHERE state = 'pending';
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_deferred
|
|
ON keycheck_candidates(service, available_after, id) WHERE state = 'deferred';
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_lease
|
|
ON keycheck_candidates(service, lease_expires_at, id) WHERE state = 'leased';
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_candidates_credential
|
|
ON keycheck_candidates(credential_id, state, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_candidate_finding_credential
|
|
ON keycheck_candidates(service, finding_id, credential_id) WHERE finding_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_pipeline_quarantine_pending
|
|
ON pipeline_quarantine(subsystem, review_status, id) WHERE review_status = 'pending';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_pipeline_quarantine_open_object
|
|
ON pipeline_quarantine(subsystem, object_type, object_id)
|
|
WHERE review_status = 'pending' AND object_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_pipeline_artifacts_owner
|
|
ON pipeline_artifacts(subsystem, owner_id, state, id);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_candidate ON keycheck_results(candidate_id);
|
|
CREATE INDEX IF NOT EXISTS idx_keycheck_results_credential_checked
|
|
ON keycheck_results(credential_id, checked_at DESC, id DESC);
|
|
''' + DOCKER_DEPTH_EXPERIMENT_SCHEMA_SQL
|
|
|
|
|
|
SCHEMA_SQL = r'''
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
started_at TEXT NOT NULL,
|
|
ended_at TEXT,
|
|
duration_sec REAL,
|
|
status TEXT NOT NULL,
|
|
invocation_mode TEXT,
|
|
command_line TEXT,
|
|
argv_json TEXT,
|
|
selected_source TEXT,
|
|
selected_platform TEXT,
|
|
config_path TEXT,
|
|
config_hash TEXT,
|
|
enabled_sources_json TEXT,
|
|
db_path TEXT,
|
|
total_fetched INTEGER DEFAULT 0,
|
|
total_queued_new INTEGER DEFAULT 0,
|
|
total_scan_requested INTEGER DEFAULT 0,
|
|
total_scanned INTEGER DEFAULT 0,
|
|
total_clean INTEGER DEFAULT 0,
|
|
total_found INTEGER DEFAULT 0,
|
|
total_skipped INTEGER DEFAULT 0,
|
|
total_errors INTEGER DEFAULT 0,
|
|
total_findings INTEGER DEFAULT 0,
|
|
total_verified_findings INTEGER DEFAULT 0,
|
|
total_unique_secrets INTEGER DEFAULT 0,
|
|
total_unique_findings INTEGER DEFAULT 0,
|
|
total_staged INTEGER NOT NULL DEFAULT 0,
|
|
total_quarantined INTEGER NOT NULL DEFAULT 0,
|
|
error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS source_cycles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER,
|
|
source TEXT,
|
|
platform TEXT,
|
|
mode TEXT,
|
|
query TEXT,
|
|
query_index INTEGER,
|
|
query_count INTEGER,
|
|
auth_name TEXT,
|
|
started_at TEXT NOT NULL,
|
|
ended_at TEXT,
|
|
duration_sec REAL,
|
|
status TEXT NOT NULL,
|
|
message TEXT,
|
|
config_json TEXT,
|
|
queue_todo_before INTEGER,
|
|
queue_checked_before INTEGER,
|
|
queue_todo_after INTEGER,
|
|
queue_checked_after INTEGER,
|
|
fetched_count INTEGER DEFAULT 0,
|
|
queued_new_count INTEGER DEFAULT 0,
|
|
queued_updated_count INTEGER NOT NULL DEFAULT 0,
|
|
scan_requested_count INTEGER DEFAULT 0,
|
|
scanned_count INTEGER DEFAULT 0,
|
|
clean_count INTEGER DEFAULT 0,
|
|
found_count INTEGER DEFAULT 0,
|
|
skipped_count INTEGER DEFAULT 0,
|
|
error_count INTEGER DEFAULT 0,
|
|
findings_count INTEGER DEFAULT 0,
|
|
verified_findings_count INTEGER DEFAULT 0,
|
|
unique_secrets_count INTEGER DEFAULT 0,
|
|
unique_findings_count INTEGER DEFAULT 0,
|
|
targets_per_hour REAL DEFAULT 0,
|
|
hit_rate REAL DEFAULT 0,
|
|
verified_hit_rate REAL DEFAULT 0,
|
|
error_rate REAL DEFAULT 0,
|
|
staged_count INTEGER NOT NULL DEFAULT 0,
|
|
ingested_count INTEGER NOT NULL DEFAULT 0,
|
|
quarantined_count INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS target_scans (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
scan_event_id TEXT,
|
|
scan_event_hash TEXT,
|
|
queue_id INTEGER,
|
|
claim_lease_token TEXT,
|
|
queue_completion_applied INTEGER NOT NULL DEFAULT 0,
|
|
queue_completion_disposition TEXT,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
source TEXT,
|
|
query TEXT,
|
|
target TEXT,
|
|
normalized_target TEXT,
|
|
scan_type TEXT,
|
|
status TEXT,
|
|
started_at TEXT,
|
|
ended_at TEXT,
|
|
duration_sec REAL,
|
|
scan_options_json TEXT,
|
|
package_name TEXT,
|
|
package_version TEXT,
|
|
package_artifact TEXT,
|
|
package_date TEXT,
|
|
package_filename TEXT,
|
|
package_type TEXT,
|
|
package_size INTEGER,
|
|
findings_count INTEGER DEFAULT 0,
|
|
verified_findings_count INTEGER DEFAULT 0,
|
|
error_count INTEGER DEFAULT 0,
|
|
skipped_reason TEXT,
|
|
first_error_summary TEXT,
|
|
raw_result_json TEXT,
|
|
result_reservation_id INTEGER,
|
|
compat_schema_version INTEGER NOT NULL DEFAULT 2,
|
|
raw_result_storage TEXT NOT NULL DEFAULT 'legacy',
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(queue_id) REFERENCES target_queue(id),
|
|
FOREIGN KEY(result_reservation_id) REFERENCES result_reservations(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS findings (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
target_scan_id INTEGER,
|
|
source TEXT,
|
|
query TEXT,
|
|
target TEXT,
|
|
normalized_target TEXT,
|
|
detector_name TEXT,
|
|
detector_type TEXT,
|
|
verified INTEGER DEFAULT 0,
|
|
raw_secret TEXT,
|
|
redacted_secret TEXT,
|
|
secret_hash TEXT,
|
|
detector_secret_hash TEXT,
|
|
finding_fingerprint TEXT,
|
|
finding_uid TEXT,
|
|
file_path TEXT,
|
|
line_number TEXT,
|
|
commit_hash TEXT,
|
|
source_timestamp TEXT,
|
|
source_metadata_type TEXT,
|
|
source_metadata_json TEXT,
|
|
raw_finding_json TEXT,
|
|
provider TEXT,
|
|
credential_kind TEXT,
|
|
credential_confidence TEXT,
|
|
required_context_missing INTEGER DEFAULT 0,
|
|
principal TEXT,
|
|
username TEXT,
|
|
email TEXT,
|
|
project_id TEXT,
|
|
tenant_id TEXT,
|
|
organization TEXT,
|
|
registry TEXT,
|
|
endpoint TEXT,
|
|
scope TEXT,
|
|
resource TEXT,
|
|
enrichment_json TEXT,
|
|
raw_payload_sha256 TEXT,
|
|
raw_payload_bytes INTEGER,
|
|
raw_payload_omitted INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS finding_uid_map (
|
|
finding_uid TEXT PRIMARY KEY,
|
|
finding_id INTEGER NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(finding_id) REFERENCES findings(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS errors (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
target_scan_id INTEGER,
|
|
source TEXT,
|
|
query TEXT,
|
|
target TEXT,
|
|
normalized_target TEXT,
|
|
category TEXT,
|
|
summary TEXT,
|
|
raw_error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS queue_snapshots (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
source TEXT,
|
|
phase TEXT,
|
|
todo_count INTEGER,
|
|
checked_count INTEGER,
|
|
todo_file TEXT,
|
|
checked_file TEXT,
|
|
captured_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS config_snapshots (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER,
|
|
cycle_id INTEGER,
|
|
scope TEXT,
|
|
source TEXT,
|
|
config_json TEXT,
|
|
captured_at TEXT NOT NULL,
|
|
FOREIGN KEY(run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(cycle_id) REFERENCES source_cycles(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS package_repo_candidates (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
package_source TEXT NOT NULL,
|
|
package_name TEXT NOT NULL,
|
|
package_version TEXT NOT NULL,
|
|
query TEXT,
|
|
repo_url TEXT NOT NULL,
|
|
provider TEXT,
|
|
evidence_json TEXT,
|
|
confidence TEXT,
|
|
first_seen_at TEXT NOT NULL,
|
|
last_seen_at TEXT NOT NULL,
|
|
last_run_id INTEGER,
|
|
last_cycle_id INTEGER,
|
|
UNIQUE(package_source, package_name, package_version, repo_url),
|
|
FOREIGN KEY(last_run_id) REFERENCES runs(id),
|
|
FOREIGN KEY(last_cycle_id) REFERENCES source_cycles(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS target_queue (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
query TEXT,
|
|
target TEXT NOT NULL,
|
|
normalized_target TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
attempts INTEGER DEFAULT 0,
|
|
lease_owner TEXT,
|
|
lease_token TEXT,
|
|
claim_batch TEXT,
|
|
leased_at TEXT,
|
|
lease_expires_at TEXT,
|
|
available_after TEXT,
|
|
target_scan_id INTEGER,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
resolver_state TEXT,
|
|
resolver_due_at TEXT,
|
|
resolver_attempts INTEGER NOT NULL DEFAULT 0,
|
|
resolver_token TEXT,
|
|
current_result_reservation_id INTEGER,
|
|
claim_event_id TEXT,
|
|
remote_modified_at TEXT,
|
|
scan_remote_modified_at TEXT,
|
|
covered_ref TEXT,
|
|
covered_head TEXT,
|
|
UNIQUE(source, normalized_target),
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id),
|
|
FOREIGN KEY(current_result_reservation_id) REFERENCES result_reservations(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS scan_publication_outbox (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
target_scan_id INTEGER NOT NULL UNIQUE,
|
|
payload_json TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
attempts INTEGER DEFAULT 0,
|
|
last_error TEXT,
|
|
lease_owner TEXT,
|
|
lease_expires_at TEXT,
|
|
available_after TEXT,
|
|
created_at TEXT NOT NULL,
|
|
delivered_at TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
FOREIGN KEY(target_scan_id) REFERENCES target_scans(id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS target_queue_reconciliation_cursors (
|
|
source_file TEXT PRIMARY KEY,
|
|
file_identity TEXT NOT NULL,
|
|
file_size INTEGER NOT NULL DEFAULT 0,
|
|
file_mtime_ns INTEGER NOT NULL DEFAULT 0,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
byte_offset INTEGER NOT NULL DEFAULT 0,
|
|
line_number INTEGER NOT NULL DEFAULT 0,
|
|
discarding_oversized INTEGER NOT NULL DEFAULT 0,
|
|
oversized_line_start INTEGER,
|
|
cumulative_rows INTEGER NOT NULL DEFAULT 0,
|
|
cumulative_bytes INTEGER NOT NULL DEFAULT 0,
|
|
cumulative_inserted INTEGER NOT NULL DEFAULT 0,
|
|
cumulative_rejected INTEGER NOT NULL DEFAULT 0,
|
|
completed_at TEXT,
|
|
last_report_json TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS target_queue_reconciliation_issues (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source_file TEXT NOT NULL,
|
|
file_identity TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
line_number INTEGER NOT NULL,
|
|
byte_offset INTEGER NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
target_preview TEXT,
|
|
created_at TEXT NOT NULL,
|
|
resolved_at TEXT
|
|
);
|
|
|
|
''' + KEYCHECK_RESULTS_SQL + PIPELINE_SCHEMA_SQL + r'''
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_selected_source_status ON runs(selected_source, status, id);
|
|
CREATE INDEX IF NOT EXISTS idx_source_cycles_run_id ON source_cycles(run_id);
|
|
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_started ON source_cycles(source, started_at);
|
|
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_query ON source_cycles(source, query);
|
|
CREATE INDEX IF NOT EXISTS idx_source_cycles_source_status ON source_cycles(source, status, id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_cycle_id ON target_scans(cycle_id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_queue_id ON target_scans(queue_id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_result_reservation ON target_scans(result_reservation_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_source_status ON target_scans(source, status);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_source_ended ON target_scans(source, ended_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_source_ended_id ON target_scans(source, ended_at DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_normalized_target ON target_scans(normalized_target);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_source_skip_ended ON target_scans(source, skipped_reason, ended_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_cooldown_recent ON target_scans(source, ended_at DESC, id DESC) WHERE status = 'skipped' AND ((source = 'github_actions' AND (skipped_reason = 'no downloadable workflow logs or artifacts' OR skipped_reason = 'no recent workflow runs')) OR (source = 'gitlab_ci' AND (skipped_reason = 'no downloadable job traces or artifacts' OR skipped_reason = 'no recent pipelines')));
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_target_scans_scan_event_id ON target_scans(scan_event_id) WHERE scan_event_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_target_scans_event_hash ON target_scans(scan_event_id, scan_event_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_cycle_id ON findings(cycle_id);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_target_scan_id_id ON findings(target_scan_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_source ON findings(source);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_detector ON findings(detector_name);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_secret_hash ON findings(secret_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_detector_secret_hash ON findings(detector_secret_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_fingerprint ON findings(finding_fingerprint);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_finding_uid ON findings(finding_uid);
|
|
CREATE INDEX IF NOT EXISTS idx_errors_cycle_id ON errors(cycle_id);
|
|
CREATE INDEX IF NOT EXISTS idx_errors_source_category ON errors(source, category);
|
|
CREATE INDEX IF NOT EXISTS idx_errors_target_scan_id ON errors(target_scan_id);
|
|
CREATE INDEX IF NOT EXISTS idx_queue_snapshots_source_time ON queue_snapshots(source, captured_at);
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_query ON package_repo_candidates(query);
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_repo ON package_repo_candidates(repo_url);
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_provider_seen ON package_repo_candidates(LOWER(provider), last_seen_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_source_seen ON package_repo_candidates(package_source, last_seen_at);
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_query_seen ON package_repo_candidates(query, last_seen_at DESC, id DESC) WHERE repo_url IS NOT NULL AND repo_url <> '';
|
|
CREATE INDEX IF NOT EXISTS idx_package_repo_candidates_recent_lookup ON package_repo_candidates(last_seen_at DESC, id DESC, package_source, query, package_name) WHERE repo_url IS NOT NULL AND repo_url <> '';
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_source_status ON target_queue(source, status, updated_at);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_observe_source_status ON target_queue(source, status);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_lease ON target_queue(source, status, lease_expires_at);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_platform_status ON target_queue(platform, status);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_batch ON target_queue(claim_batch, lease_owner);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claim ON target_queue(source, platform, status, available_after, lease_expires_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_resolver_claim ON target_queue(source, platform, status, resolver_state, resolver_due_at, id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_source_platform_normalized ON target_queue(source, platform, normalized_target);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_cold ON target_queue(source, platform, query, id) WHERE status = 'cold';
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_current_reservation ON target_queue(current_result_reservation_id);
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claimable_v2 ON target_queue(source, platform, status, available_after, id) WHERE current_result_reservation_id IS NULL AND (status = 'pending' OR status = 'deferred');
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_pending ON target_queue(source, platform, id, attempts, available_after) WHERE status = 'pending' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved');
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_deferred ON target_queue(source, platform, available_after, id, attempts) WHERE status = 'deferred' AND current_result_reservation_id IS NULL AND (resolver_state IS NULL OR resolver_state = 'resolved');
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_claim_in_progress ON target_queue(source, platform, id, attempts, available_after, lease_expires_at, resolver_state) WHERE status = 'in_progress' AND (resolver_state IS NULL OR resolver_state = 'resolved');
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_active_lease_owner_token ON target_queue(lease_owner, lease_token) WHERE status = 'in_progress';
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_exhausted_attempts ON target_queue(source, platform, status, attempts, id, lease_expires_at) WHERE status = 'pending' OR status = 'deferred' OR status = 'in_progress';
|
|
CREATE INDEX IF NOT EXISTS idx_target_queue_updated_rescan ON target_queue(source, platform, completed_at, remote_modified_at, scan_remote_modified_at, id) WHERE status = 'done' AND remote_modified_at IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_scan_publication_outbox_status ON scan_publication_outbox(status, available_after, id);
|
|
CREATE INDEX IF NOT EXISTS idx_scan_publication_outbox_age ON scan_publication_outbox(status, created_at, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_scan_publication_outbox_target_scan_id ON scan_publication_outbox(target_scan_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_keycheck_event_map_event_id ON keycheck_event_map(event_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_finding_uid_map_finding_uid ON finding_uid_map(finding_uid);
|
|
CREATE INDEX IF NOT EXISTS idx_reconciliation_issues_open ON target_queue_reconciliation_issues(source_file, resolved_at, id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_reconciliation_issue_identity ON target_queue_reconciliation_issues(source_file, file_identity, line_number, byte_offset, reason);
|
|
'''
|