sashatrask
127 lines
5.2 KiB
Python
127 lines
5.2 KiB
Python
from datetime import datetime
|
|
import re
|
|
|
|
|
|
REJECTED_QUERY_STATUS = 'rejected_zero_alive'
|
|
REJECTED_QUERY_KEYS = {
|
|
'source', 'query', 'status', 'evidence_cutoff', 'successful_scans',
|
|
'findings', 'unique_credentials', 'pending_candidates',
|
|
'ever_alive_credentials', 'reviewed_queue_rows',
|
|
}
|
|
REJECTED_QUERY_COUNT_KEYS = {
|
|
'successful_scans', 'findings', 'unique_credentials',
|
|
'pending_candidates', 'ever_alive_credentials', 'reviewed_queue_rows',
|
|
}
|
|
OPERATIONAL_QUERY_SENTINELS = {
|
|
('github_archive', 'gharchive'),
|
|
('github_archive_files', 'gharchive-files'),
|
|
('github_gists', 'gists'),
|
|
('github_actions', 'logs'),
|
|
('gitlab_ci', 'logs'),
|
|
('huggingface', 'spaces'),
|
|
}
|
|
SOURCE_RE = re.compile(r'[a-z][a-z0-9_]{0,63}')
|
|
MAX_QUERY_LENGTH = 512
|
|
MAX_EVIDENCE_COUNT = (1 << 63) - 1
|
|
|
|
|
|
class QueryPolicyError(ValueError):
|
|
pass
|
|
|
|
|
|
def _active_queries(source, source_config):
|
|
raw_queries = source_config.get('queries', [])
|
|
if isinstance(raw_queries, str):
|
|
raw_queries = raw_queries.split(',')
|
|
if not isinstance(raw_queries, (list, tuple)):
|
|
raise QueryPolicyError(f'active query policy is invalid for source {source}')
|
|
queries = set()
|
|
for raw_query in raw_queries:
|
|
query = str(raw_query or '').strip()
|
|
if not query:
|
|
raise QueryPolicyError(f'active query policy contains an empty query for source {source}')
|
|
queries.add(query)
|
|
return queries
|
|
|
|
|
|
def validate_rejected_query_policy(config):
|
|
config = config or {}
|
|
raw_policy = config.get('query_policy')
|
|
if raw_policy is None:
|
|
return ()
|
|
if not isinstance(raw_policy, dict) or set(raw_policy) != {'rejected'}:
|
|
raise QueryPolicyError('query_policy must contain only the rejected registry')
|
|
raw_entries = raw_policy.get('rejected')
|
|
if not isinstance(raw_entries, list):
|
|
raise QueryPolicyError('query_policy.rejected must be a list')
|
|
|
|
sources = config.get('sources') or {}
|
|
if not isinstance(sources, dict):
|
|
raise QueryPolicyError('configured sources must be a mapping')
|
|
|
|
normalized = []
|
|
seen = set()
|
|
for raw_entry in raw_entries:
|
|
if not isinstance(raw_entry, dict) or set(raw_entry) != REJECTED_QUERY_KEYS:
|
|
raise QueryPolicyError('rejected query evidence shape is invalid')
|
|
source = raw_entry.get('source')
|
|
query = raw_entry.get('query')
|
|
if not isinstance(source, str) or not SOURCE_RE.fullmatch(source):
|
|
raise QueryPolicyError('rejected query source is invalid')
|
|
if source not in sources or not isinstance(sources[source], dict):
|
|
raise QueryPolicyError(f'rejected query source is not configured: {source}')
|
|
if (
|
|
not isinstance(query, str)
|
|
or query != query.strip()
|
|
or not query
|
|
or len(query) > MAX_QUERY_LENGTH
|
|
):
|
|
raise QueryPolicyError(f'rejected query text is invalid for source {source}')
|
|
pair = (source, query)
|
|
if pair in seen:
|
|
raise QueryPolicyError('rejected query registry contains a duplicate pair')
|
|
if pair in OPERATIONAL_QUERY_SENTINELS:
|
|
raise QueryPolicyError('operational query sentinel cannot be rejected')
|
|
if query in _active_queries(source, sources[source]):
|
|
raise QueryPolicyError('active and rejected query policy overlap')
|
|
if raw_entry.get('status') != REJECTED_QUERY_STATUS:
|
|
raise QueryPolicyError('rejected query status is invalid')
|
|
|
|
cutoff = raw_entry.get('evidence_cutoff')
|
|
if not isinstance(cutoff, str) or not cutoff or cutoff != cutoff.strip():
|
|
raise QueryPolicyError('rejected query evidence cutoff is invalid')
|
|
try:
|
|
parsed_cutoff = datetime.fromisoformat(cutoff.replace('Z', '+00:00'))
|
|
except ValueError as exc:
|
|
raise QueryPolicyError('rejected query evidence cutoff is invalid') from exc
|
|
if parsed_cutoff.tzinfo is None or parsed_cutoff.utcoffset() is None:
|
|
raise QueryPolicyError('rejected query evidence cutoff must include a timezone')
|
|
|
|
counts = {}
|
|
for name in REJECTED_QUERY_COUNT_KEYS:
|
|
value = raw_entry.get(name)
|
|
if (
|
|
isinstance(value, bool)
|
|
or not isinstance(value, int)
|
|
or value < 0
|
|
or value > MAX_EVIDENCE_COUNT
|
|
):
|
|
raise QueryPolicyError(f'rejected query {name} is invalid')
|
|
counts[name] = value
|
|
if counts['successful_scans'] < 1000:
|
|
raise QueryPolicyError('rejected query has fewer than 1000 successful scans')
|
|
if counts['pending_candidates'] != 0:
|
|
raise QueryPolicyError('rejected query still has pending candidates')
|
|
if counts['ever_alive_credentials'] != 0:
|
|
raise QueryPolicyError('rejected query has historical alive credentials')
|
|
|
|
seen.add(pair)
|
|
normalized.append({
|
|
'source': source,
|
|
'query': query,
|
|
'status': REJECTED_QUERY_STATUS,
|
|
'evidence_cutoff': cutoff,
|
|
**{name: counts[name] for name in sorted(REJECTED_QUERY_COUNT_KEYS)},
|
|
})
|
|
return tuple(sorted(normalized, key=lambda entry: (entry['source'], entry['query'])))
|