Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,498 @@
|
||||
import sys
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
from itertools import cycle
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||||
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from keycheck_common import (
|
||||
acquire_file_lock,
|
||||
append_checked,
|
||||
append_jsonl,
|
||||
commit_status_transaction,
|
||||
default_input_file,
|
||||
default_proxy_file,
|
||||
env_int,
|
||||
ensure_output_files as ensure_private_output_files,
|
||||
finding_detector_secret_hash,
|
||||
iter_findings,
|
||||
iter_bounded_text_lines,
|
||||
keycheck_input_mode,
|
||||
load_known_statuses,
|
||||
load_checked_statuses,
|
||||
mask_secret,
|
||||
now_iso,
|
||||
physical_jsonl_segments,
|
||||
private_append_writer,
|
||||
private_atomic_writer,
|
||||
reconcile_keycheck_jsonl_segments,
|
||||
record_validation_result,
|
||||
recover_status_transaction,
|
||||
release_file_lock,
|
||||
repair_keycheck_jsonl_tail,
|
||||
require_provider_authority,
|
||||
rotate_jsonl_if_needed,
|
||||
service_output_dir,
|
||||
should_skip_key,
|
||||
sha256_text,
|
||||
write_keycheck_event,
|
||||
)
|
||||
from runtime_security import reject_reparse_components, require_private_file
|
||||
|
||||
# --- Конфигурация ---
|
||||
SERVICE = "openrouter"
|
||||
OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
|
||||
INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
|
||||
ALIVE_FILE = os.path.join(OUTPUT_DIR, "openrouterAlive.txt")
|
||||
DEAD_FILE = os.path.join(OUTPUT_DIR, "openrouterDead.txt")
|
||||
LIMITED_FILE = os.path.join(OUTPUT_DIR, "openrouterLimited.txt")
|
||||
NO_BALANCE_FILE = os.path.join(OUTPUT_DIR, "openrouterNoBalance.txt")
|
||||
NETWORK_FILE = os.path.join(OUTPUT_DIR, "openrouterNetwork.txt")
|
||||
UNKNOWN_FILE = os.path.join(OUTPUT_DIR, "openrouterUnknown.txt")
|
||||
CHECKED_FILE = os.path.join(OUTPUT_DIR, "openrouterChecked.txt")
|
||||
RESULTS_FILE = os.path.join(OUTPUT_DIR, "openrouterResults.jsonl")
|
||||
PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
|
||||
STATUS_FILES = {
|
||||
"VALID": ALIVE_FILE,
|
||||
"NO_BALANCE": NO_BALANCE_FILE,
|
||||
"DEAD": DEAD_FILE,
|
||||
"LIMITED": LIMITED_FILE,
|
||||
"NETWORK": NETWORK_FILE,
|
||||
"UNKNOWN": UNKNOWN_FILE,
|
||||
}
|
||||
|
||||
CREDITS_URL = "https://openrouter.ai/api/v1/credits"
|
||||
|
||||
# --- Вспомогательные функции ---
|
||||
def load_set_from_file(filepath):
|
||||
"""Загружает ключи из файла в set для быстрой проверки."""
|
||||
if not os.path.exists(filepath):
|
||||
return set()
|
||||
return {line.strip().split(':')[0] for line in iter_bounded_text_lines(filepath) if line.strip()}
|
||||
|
||||
def ensure_output_files():
|
||||
ensure_private_output_files((CHECKED_FILE, RESULTS_FILE, *STATUS_FILES.values()))
|
||||
recover_status_transaction(CHECKED_FILE, STATUS_FILES)
|
||||
|
||||
|
||||
def legacy_status_key(line):
|
||||
value = line.strip()
|
||||
if not value:
|
||||
return None
|
||||
if '\t' in value:
|
||||
return value.split('\t', 1)[0].strip()
|
||||
return value.split(':', 1)[0].strip()
|
||||
|
||||
|
||||
def load_openrouter_keys(path):
|
||||
if keycheck_input_mode() == 'postgres':
|
||||
return set()
|
||||
if not os.path.exists(path):
|
||||
return set()
|
||||
return {key for key in (legacy_status_key(line) for line in iter_bounded_text_lines(path)) if key}
|
||||
|
||||
|
||||
def iter_plain_openrouter_keys(paths):
|
||||
if keycheck_input_mode() == 'postgres':
|
||||
return
|
||||
seen = set()
|
||||
items = []
|
||||
for path in paths or []:
|
||||
if not path or not os.path.exists(path):
|
||||
continue
|
||||
for line in iter_bounded_text_lines(path):
|
||||
key = legacy_status_key(line)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append({'raw': key, 'source': path, 'finding': {}})
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
|
||||
def retry_plain_files(args):
|
||||
if keycheck_input_mode() == 'postgres':
|
||||
return []
|
||||
files = list(args.plain or [])
|
||||
if args.recheck_all:
|
||||
files.extend(STATUS_FILES.values())
|
||||
else:
|
||||
if args.retry_valid:
|
||||
files.append(ALIVE_FILE)
|
||||
if args.retry_no_balance:
|
||||
files.append(NO_BALANCE_FILE)
|
||||
if args.retry_limited:
|
||||
files.append(LIMITED_FILE)
|
||||
if args.retry_network:
|
||||
files.append(NETWORK_FILE)
|
||||
if args.retry_unknown:
|
||||
files.append(UNKNOWN_FILE)
|
||||
out = []
|
||||
seen = set()
|
||||
for path in files:
|
||||
if path and path not in seen:
|
||||
seen.add(path)
|
||||
out.append(path)
|
||||
return out
|
||||
|
||||
|
||||
def migrate_legacy_checked():
|
||||
if keycheck_input_mode() == 'postgres':
|
||||
return
|
||||
checked = load_checked_statuses(CHECKED_FILE)
|
||||
for key in sorted(load_openrouter_keys(ALIVE_FILE)):
|
||||
if key not in checked:
|
||||
write_keycheck_event(SERVICE, RESULTS_FILE, key, {'status': 'VALID', 'message': 'legacy alive status migration'}, 'legacy:openrouterAlive.txt', {}, 'OpenRouter', 'legacy_status')
|
||||
append_checked(CHECKED_FILE, key, 'VALID')
|
||||
checked[key] = 'VALID'
|
||||
for key in sorted(load_openrouter_keys(DEAD_FILE)):
|
||||
if key not in checked:
|
||||
write_keycheck_event(SERVICE, RESULTS_FILE, key, {'status': 'DEAD', 'message': 'legacy dead status migration'}, 'legacy:openrouterDead.txt', {}, 'OpenRouter', 'legacy_status')
|
||||
append_checked(CHECKED_FILE, key, 'DEAD')
|
||||
checked[key] = 'DEAD'
|
||||
|
||||
|
||||
def _legacy_alive_checked_at(path):
|
||||
details = os.stat(path, follow_symlinks=False)
|
||||
return datetime.fromtimestamp(details.st_mtime, timezone.utc).isoformat(timespec='seconds')
|
||||
|
||||
|
||||
def _legacy_balance_event_payload(key, balance, checked_at):
|
||||
balance_text = f'{balance:.6f}'
|
||||
key_hash = sha256_text(key)
|
||||
event_id = sha256_text('|'.join([
|
||||
SERVICE,
|
||||
'legacy_status',
|
||||
'openrouterAlive.txt',
|
||||
key_hash,
|
||||
'NO_BALANCE',
|
||||
balance_text,
|
||||
]))
|
||||
return {
|
||||
'key_masked': mask_secret(key),
|
||||
'key_hash': key_hash,
|
||||
'secret_hash': key_hash,
|
||||
'detector_secret_hash': finding_detector_secret_hash({}),
|
||||
'finding_uid': '',
|
||||
'detector': 'OpenRouter',
|
||||
'source': 'legacy:openrouterAlive.txt',
|
||||
'finding': {},
|
||||
'checked_at': checked_at,
|
||||
'result_source': 'legacy_status',
|
||||
'status': 'NO_BALANCE',
|
||||
'message': f'credits={balance_text}',
|
||||
'event_id': event_id,
|
||||
}
|
||||
|
||||
|
||||
def _publish_legacy_no_balance(moved):
|
||||
lock_path = f'{NO_BALANCE_FILE}.lock'
|
||||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||||
try:
|
||||
require_private_file(NO_BALANCE_FILE)
|
||||
existing = load_openrouter_keys(NO_BALANCE_FILE)
|
||||
with private_append_writer(NO_BALANCE_FILE) as handle:
|
||||
for key, balance in moved:
|
||||
if key in existing:
|
||||
continue
|
||||
balance_text = f'{balance:.6f}'
|
||||
handle.write(f'{key}\tNO_BALANCE\tcredits={balance_text}\tmigrated_from_alive\n')
|
||||
existing.add(key)
|
||||
finally:
|
||||
release_file_lock(lock, lock_path)
|
||||
|
||||
|
||||
def _existing_legacy_event_ids(expected):
|
||||
found = set()
|
||||
max_line_bytes = max(1024, env_int('KEYCHECK_INPUT_MAX_LINE_BYTES', 16 * 1024 * 1024))
|
||||
max_file_bytes = max(
|
||||
max_line_bytes,
|
||||
max(1, env_int('KEYCHECK_INPUT_LIST_MAX_BYTES', 32 * 1024 * 1024)),
|
||||
max(0, env_int('KEYCHECK_RESULTS_MAX_MB', 32)) * 1024 * 1024 + max_line_bytes,
|
||||
)
|
||||
paths = [path for _, path in physical_jsonl_segments(RESULTS_FILE)]
|
||||
if os.path.isfile(RESULTS_FILE):
|
||||
paths.append(os.path.abspath(RESULTS_FILE))
|
||||
for path in paths:
|
||||
reject_reparse_components(path)
|
||||
if os.path.getsize(path) > max_file_bytes:
|
||||
raise RuntimeError(f'OpenRouter result file exceeds the bounded migration scan size: {path}')
|
||||
with open(path, 'rb') as handle:
|
||||
while True:
|
||||
raw_line = handle.readline(max_line_bytes + 1)
|
||||
if not raw_line:
|
||||
break
|
||||
if len(raw_line) > max_line_bytes:
|
||||
raise RuntimeError(f'OpenRouter result line exceeds the bounded migration scan size: {path}')
|
||||
if not raw_line.endswith(b'\n'):
|
||||
raise RuntimeError(f'torn OpenRouter result line during legacy migration: {path}')
|
||||
try:
|
||||
payload = json.loads(raw_line.decode('utf-8', errors='replace'))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
event_id = str(payload.get('event_id') or '') if isinstance(payload, dict) else ''
|
||||
if event_id not in expected:
|
||||
continue
|
||||
wanted = expected[event_id]
|
||||
for field in ('key_hash', 'status', 'source', 'result_source'):
|
||||
if str(payload.get(field) or '') != str(wanted.get(field) or ''):
|
||||
raise RuntimeError(f'conflicting OpenRouter legacy migration event: {event_id}')
|
||||
found.add(event_id)
|
||||
return found
|
||||
|
||||
|
||||
def _publish_legacy_balance_events(moved, checked_at):
|
||||
payloads = [_legacy_balance_event_payload(key, balance, checked_at) for key, balance in moved]
|
||||
expected = {payload['event_id']: payload for payload in payloads}
|
||||
lock_path = f'{RESULTS_FILE}.lock'
|
||||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||||
try:
|
||||
require_private_file(RESULTS_FILE)
|
||||
repair_keycheck_jsonl_tail(RESULTS_FILE)
|
||||
reconcile_keycheck_jsonl_segments(RESULTS_FILE)
|
||||
existing = _existing_legacy_event_ids(expected)
|
||||
max_bytes = max(0, env_int('KEYCHECK_RESULTS_MAX_MB', 32)) * 1024 * 1024
|
||||
for payload in payloads:
|
||||
event_id = payload['event_id']
|
||||
if event_id in existing:
|
||||
continue
|
||||
rotate_jsonl_if_needed(RESULTS_FILE, max_bytes)
|
||||
with private_append_writer(RESULTS_FILE) as handle:
|
||||
handle.write(json.dumps(payload, ensure_ascii=False, default=str) + '\n')
|
||||
existing.add(event_id)
|
||||
finally:
|
||||
release_file_lock(lock, lock_path)
|
||||
|
||||
|
||||
def _publish_legacy_checked(moved, checked_at):
|
||||
lock_path = f'{CHECKED_FILE}.lock'
|
||||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||||
try:
|
||||
require_private_file(CHECKED_FILE)
|
||||
existing = set()
|
||||
for line in iter_bounded_text_lines(CHECKED_FILE):
|
||||
parts = line.rstrip('\r\n').split('\t')
|
||||
if len(parts) >= 2:
|
||||
existing.add((parts[0], parts[1]))
|
||||
with private_append_writer(CHECKED_FILE) as handle:
|
||||
for key, _ in moved:
|
||||
identity = (key, 'NO_BALANCE')
|
||||
if identity in existing:
|
||||
continue
|
||||
handle.write(f'{key}\tNO_BALANCE\t{checked_at}\n')
|
||||
existing.add(identity)
|
||||
finally:
|
||||
release_file_lock(lock, lock_path)
|
||||
|
||||
|
||||
def _rewrite_legacy_alive(keep):
|
||||
with private_atomic_writer(ALIVE_FILE, binary=True, suffix='.legacy.tmp') as handle:
|
||||
for line in keep:
|
||||
handle.write(line.encode('utf-8'))
|
||||
|
||||
|
||||
def migrate_legacy_alive_balances():
|
||||
if keycheck_input_mode() == 'postgres':
|
||||
return
|
||||
lock_path = f'{ALIVE_FILE}.lock'
|
||||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||||
try:
|
||||
if not os.path.exists(ALIVE_FILE):
|
||||
return
|
||||
require_private_file(ALIVE_FILE)
|
||||
checked_at = _legacy_alive_checked_at(ALIVE_FILE)
|
||||
keep = []
|
||||
moved = []
|
||||
seen = set()
|
||||
for line in iter_bounded_text_lines(ALIVE_FILE):
|
||||
text = line.strip()
|
||||
if not text:
|
||||
keep.append(line)
|
||||
continue
|
||||
key = legacy_status_key(text)
|
||||
balance = None
|
||||
if ':' in text and '\t' not in text:
|
||||
try:
|
||||
balance = float(text.rsplit(':', 1)[1])
|
||||
except ValueError:
|
||||
balance = None
|
||||
if key and balance is not None and balance <= 0:
|
||||
if key not in seen:
|
||||
moved.append((key, balance))
|
||||
seen.add(key)
|
||||
else:
|
||||
keep.append(line)
|
||||
if not moved:
|
||||
return
|
||||
_publish_legacy_no_balance(moved)
|
||||
_publish_legacy_balance_events(moved, checked_at)
|
||||
_publish_legacy_checked(moved, checked_at)
|
||||
_rewrite_legacy_alive(keep)
|
||||
finally:
|
||||
release_file_lock(lock, lock_path)
|
||||
|
||||
|
||||
def load_proxies(proxy_file=None):
|
||||
"""Загружает и подготавливает прокси."""
|
||||
proxy_file = proxy_file or PROXY_FILE
|
||||
if not os.path.exists(proxy_file):
|
||||
print("ℹ️ Файл proxy.txt не найден, запросы будут идти напрямую.")
|
||||
return None
|
||||
proxies = []
|
||||
with open(proxy_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
try:
|
||||
ip, port, login, password = line.split(':')
|
||||
proxy_url = f"http://{login}:{password}@{ip}:{port}"
|
||||
proxies.append({"http": proxy_url, "https": proxy_url})
|
||||
except ValueError:
|
||||
print(f"⚠️ Неверный формат прокси: '{line}'. Пропускаем.")
|
||||
if not proxies:
|
||||
print("⚠️ Файл proxy.txt пуст. Запросы будут идти напрямую.")
|
||||
return None
|
||||
print(f"✅ Загружено {len(proxies)} прокси.")
|
||||
return cycle(proxies)
|
||||
|
||||
def write_result(key, result, source_line, finding=None, previous_status=None):
|
||||
status = result.get('status') or 'UNKNOWN'
|
||||
credits = result.get('remaining_credits')
|
||||
extra = f"credits={credits:.6f}" if isinstance(credits, (int, float)) else source_line
|
||||
checked_at = now_iso()
|
||||
result = {**result, 'checked_at': result.get('checked_at') or checked_at}
|
||||
write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source_line, finding, 'OpenRouter')
|
||||
commit_status_transaction(
|
||||
CHECKED_FILE, STATUS_FILES, key, status, result.get('message', ''), extra,
|
||||
)
|
||||
record_validation_result(SERVICE, key, result, source_line, finding, 'OpenRouter')
|
||||
|
||||
# --- Функция проверки ---
|
||||
def check_openrouter_key(key, proxy):
|
||||
"""Проверяет один ключ OpenRouter и возвращает normalized result."""
|
||||
headers = {"Authorization": f"Bearer {key}"}
|
||||
try:
|
||||
resp = requests.get(CREDITS_URL, headers=headers, proxies=proxy, timeout=15)
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {'status': 'NETWORK', 'message': str(e)}
|
||||
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
data = resp.json().get("data", {})
|
||||
total = float(data.get("total_credits", 0.0) or 0.0)
|
||||
used = float(data.get("total_usage", 0.0) or 0.0)
|
||||
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
return {'status': 'UNKNOWN', 'http_status': resp.status_code, 'message': f'invalid credits response: {exc}'}
|
||||
remaining = total - used
|
||||
if remaining > 0:
|
||||
return {'status': 'VALID', 'remaining_credits': remaining, 'message': f'credits={remaining:.6f}'}
|
||||
return {'status': 'NO_BALANCE', 'remaining_credits': remaining, 'message': f'credits={remaining:.6f}'}
|
||||
if resp.status_code in (401, 403):
|
||||
return {'status': 'DEAD', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||||
if resp.status_code == 429:
|
||||
return {'status': 'LIMITED', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||||
if 500 <= resp.status_code <= 599:
|
||||
return {'status': 'NETWORK', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||||
return {'status': 'UNKNOWN', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='OpenRouter key checker')
|
||||
parser.add_argument('--input', default=INPUT_FILE)
|
||||
parser.add_argument('--plain', action='append', default=[])
|
||||
parser.add_argument('--proxy-file', default=PROXY_FILE)
|
||||
parser.add_argument('--max-keys', type=int, default=0)
|
||||
parser.add_argument('--retry-network', action='store_true')
|
||||
parser.add_argument('--retry-limited', action='store_true')
|
||||
parser.add_argument('--retry-unknown', action='store_true')
|
||||
parser.add_argument('--retry-no-balance', action='store_true')
|
||||
parser.add_argument('--retry-valid', action='store_true')
|
||||
parser.add_argument('--recheck-all', action='store_true')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
# --- Основной процесс ---
|
||||
def main():
|
||||
require_provider_authority(SERVICE)
|
||||
args = parse_args()
|
||||
ensure_output_files()
|
||||
migrate_legacy_alive_balances()
|
||||
migrate_legacy_checked()
|
||||
print("--- 🚀 Запуск чекера ключей OpenRouter 🚀 ---")
|
||||
proxy_cycler = load_proxies(args.proxy_file)
|
||||
|
||||
checked_statuses = load_checked_statuses(CHECKED_FILE)
|
||||
known_statuses = load_known_statuses(CHECKED_FILE, STATUS_FILES)
|
||||
known_keys = set(known_statuses)
|
||||
for path in STATUS_FILES.values():
|
||||
known_keys.update(load_openrouter_keys(path))
|
||||
retry_statuses = set()
|
||||
if args.retry_network:
|
||||
retry_statuses.add('NETWORK')
|
||||
if args.retry_limited:
|
||||
retry_statuses.add('LIMITED')
|
||||
if args.retry_unknown:
|
||||
retry_statuses.add('UNKNOWN')
|
||||
if args.retry_no_balance:
|
||||
retry_statuses.add('NO_BALANCE')
|
||||
if args.retry_valid:
|
||||
retry_statuses.add('VALID')
|
||||
print(f"📖 Загружено: {len(load_openrouter_keys(ALIVE_FILE))} живых ключей, {len(known_keys)} классифицированных ключей.")
|
||||
|
||||
if keycheck_input_mode() == 'jsonl' and not os.path.exists(args.input):
|
||||
print(f"❌ Файл с секретами {args.input} не найден. Завершение.")
|
||||
return
|
||||
|
||||
processed = 0
|
||||
def candidates():
|
||||
for item in iter_findings(args.input, ["OpenRouter"]):
|
||||
key = item.get("raw") or ""
|
||||
if key:
|
||||
yield item
|
||||
seen = set()
|
||||
for item in iter_plain_openrouter_keys(retry_plain_files(args)):
|
||||
key = item.get("raw") or ""
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
yield item
|
||||
|
||||
for item in candidates():
|
||||
key = item.get("raw") or ""
|
||||
if not key:
|
||||
continue
|
||||
|
||||
source = item.get("source") or args.input
|
||||
finding = item.get("finding") or {}
|
||||
if should_skip_key(
|
||||
key, checked_statuses, known_keys, args, retry_statuses,
|
||||
service=SERVICE, source=source, finding=finding, detector='OpenRouter', known_statuses=known_statuses,
|
||||
):
|
||||
continue
|
||||
if args.max_keys and processed >= args.max_keys:
|
||||
break
|
||||
processed += 1
|
||||
|
||||
print(f"\n[{processed}] 🎯 Новый кандидат: {key[:8]}...{key[-4:]} from {source}")
|
||||
current_proxy = next(proxy_cycler) if proxy_cycler else None
|
||||
result = check_openrouter_key(key, current_proxy)
|
||||
print(f" STATUS: {result.get('status')} | {str(result.get('message', ''))[:200]}")
|
||||
previous_status = known_statuses.get(key) or checked_statuses.get(key)
|
||||
write_result(key, result, source, finding, previous_status)
|
||||
known_keys.add(key)
|
||||
checked_statuses[key] = result.get('status')
|
||||
|
||||
print("\n--- ✅ Проверка завершена. ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user