from datetime import datetime import re REJECTED_QUERY_STATUS = 'rejected_zero_alive' REJECTED_QUERY_KEYS = { 'source', 'query', 'status', 'evidence_cutoff', 'successful_scans', 'findings', 'unique_credentials', 'pending_candidates', 'ever_alive_credentials', 'reviewed_queue_rows', } REJECTED_QUERY_COUNT_KEYS = { 'successful_scans', 'findings', 'unique_credentials', 'pending_candidates', 'ever_alive_credentials', 'reviewed_queue_rows', } OPERATIONAL_QUERY_SENTINELS = { ('github_archive', 'gharchive'), ('github_archive_files', 'gharchive-files'), ('github_gists', 'gists'), ('github_actions', 'logs'), ('gitlab_ci', 'logs'), ('huggingface', 'spaces'), } SOURCE_RE = re.compile(r'[a-z][a-z0-9_]{0,63}') MAX_QUERY_LENGTH = 512 MAX_EVIDENCE_COUNT = (1 << 63) - 1 class QueryPolicyError(ValueError): pass def _active_queries(source, source_config): raw_queries = source_config.get('queries', []) if isinstance(raw_queries, str): raw_queries = raw_queries.split(',') if not isinstance(raw_queries, (list, tuple)): raise QueryPolicyError(f'active query policy is invalid for source {source}') queries = set() for raw_query in raw_queries: query = str(raw_query or '').strip() if not query: raise QueryPolicyError(f'active query policy contains an empty query for source {source}') queries.add(query) return queries def validate_rejected_query_policy(config): config = config or {} raw_policy = config.get('query_policy') if raw_policy is None: return () if not isinstance(raw_policy, dict) or set(raw_policy) != {'rejected'}: raise QueryPolicyError('query_policy must contain only the rejected registry') raw_entries = raw_policy.get('rejected') if not isinstance(raw_entries, list): raise QueryPolicyError('query_policy.rejected must be a list') sources = config.get('sources') or {} if not isinstance(sources, dict): raise QueryPolicyError('configured sources must be a mapping') normalized = [] seen = set() for raw_entry in raw_entries: if not isinstance(raw_entry, dict) or set(raw_entry) != REJECTED_QUERY_KEYS: raise QueryPolicyError('rejected query evidence shape is invalid') source = raw_entry.get('source') query = raw_entry.get('query') if not isinstance(source, str) or not SOURCE_RE.fullmatch(source): raise QueryPolicyError('rejected query source is invalid') if source not in sources or not isinstance(sources[source], dict): raise QueryPolicyError(f'rejected query source is not configured: {source}') if ( not isinstance(query, str) or query != query.strip() or not query or len(query) > MAX_QUERY_LENGTH ): raise QueryPolicyError(f'rejected query text is invalid for source {source}') pair = (source, query) if pair in seen: raise QueryPolicyError('rejected query registry contains a duplicate pair') if pair in OPERATIONAL_QUERY_SENTINELS: raise QueryPolicyError('operational query sentinel cannot be rejected') if query in _active_queries(source, sources[source]): raise QueryPolicyError('active and rejected query policy overlap') if raw_entry.get('status') != REJECTED_QUERY_STATUS: raise QueryPolicyError('rejected query status is invalid') cutoff = raw_entry.get('evidence_cutoff') if not isinstance(cutoff, str) or not cutoff or cutoff != cutoff.strip(): raise QueryPolicyError('rejected query evidence cutoff is invalid') try: parsed_cutoff = datetime.fromisoformat(cutoff.replace('Z', '+00:00')) except ValueError as exc: raise QueryPolicyError('rejected query evidence cutoff is invalid') from exc if parsed_cutoff.tzinfo is None or parsed_cutoff.utcoffset() is None: raise QueryPolicyError('rejected query evidence cutoff must include a timezone') counts = {} for name in REJECTED_QUERY_COUNT_KEYS: value = raw_entry.get(name) if ( isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > MAX_EVIDENCE_COUNT ): raise QueryPolicyError(f'rejected query {name} is invalid') counts[name] = value if counts['successful_scans'] < 1000: raise QueryPolicyError('rejected query has fewer than 1000 successful scans') if counts['pending_candidates'] != 0: raise QueryPolicyError('rejected query still has pending candidates') if counts['ever_alive_credentials'] != 0: raise QueryPolicyError('rejected query has historical alive credentials') seen.add(pair) normalized.append({ 'source': source, 'query': query, 'status': REJECTED_QUERY_STATUS, 'evidence_cutoff': cutoff, **{name: counts[name] for name in sorted(REJECTED_QUERY_COUNT_KEYS)}, }) return tuple(sorted(normalized, key=lambda entry: (entry['source'], entry['query'])))