Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,789 @@
|
||||
import sys
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from keycheck_common import (
|
||||
append_jsonl,
|
||||
append_status,
|
||||
commit_status_transaction,
|
||||
default_input_file,
|
||||
default_proxy_file,
|
||||
ensure_output_files,
|
||||
iter_findings,
|
||||
keycheck_input_mode,
|
||||
load_checked_statuses,
|
||||
load_known_keys,
|
||||
load_known_statuses,
|
||||
load_proxies,
|
||||
mask_secret,
|
||||
recover_status_transaction,
|
||||
request_error_message,
|
||||
record_cached_keycheck_occurrence,
|
||||
record_validation_result,
|
||||
require_provider_authority,
|
||||
service_output_dir,
|
||||
should_skip_key,
|
||||
write_keycheck_event,
|
||||
)
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(SCRIPT_DIR)
|
||||
SERVICE = "gcp"
|
||||
OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
|
||||
|
||||
INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
|
||||
PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
|
||||
PLAIN_FILE = os.path.join(OUTPUT_DIR, "gcp.txt")
|
||||
CHECKED_FILE = os.path.join(OUTPUT_DIR, "gcpChecked.txt")
|
||||
RESULTS_FILE = os.path.join(OUTPUT_DIR, "gcpResults.jsonl")
|
||||
|
||||
STATUS_FILES = {
|
||||
"VALID": os.path.join(OUTPUT_DIR, "gcpAlive.txt"),
|
||||
"VERTEX": os.path.join(OUTPUT_DIR, "gcpVertex.txt"),
|
||||
"DEAD": os.path.join(OUTPUT_DIR, "gcpDead.txt"),
|
||||
"RESTRICTED": os.path.join(OUTPUT_DIR, "gcpRestricted.txt"),
|
||||
"NETWORK": os.path.join(OUTPUT_DIR, "gcpNetwork.txt"),
|
||||
"UNKNOWN": os.path.join(OUTPUT_DIR, "gcpUnknown.txt"),
|
||||
"NO_CONTEXT": os.path.join(OUTPUT_DIR, "gcpNoContext.txt"),
|
||||
}
|
||||
VERTEX_GEMINI_FILE = os.path.join(OUTPUT_DIR, "gcpVertexGemini.txt")
|
||||
VERTEX_ANTHROPIC_FILE = os.path.join(OUTPUT_DIR, "gcpVertexAnthropic.txt")
|
||||
|
||||
GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
|
||||
TRUSTED_GOOGLE_TOKEN_ENDPOINTS = frozenset({
|
||||
GOOGLE_TOKEN_URL,
|
||||
"https://accounts.google.com/o/oauth2/token",
|
||||
})
|
||||
TOKEN_REDIRECT_STATUSES = {301, 302, 303, 307, 308}
|
||||
SA_SCOPE = "https://www.googleapis.com/auth/cloud-platform"
|
||||
MAX_PEM_BYTES = 24 * 1024
|
||||
MAX_DER_BYTES = 16 * 1024
|
||||
MAX_DER_LENGTH_BYTES = 2
|
||||
MIN_RSA_BITS = 2048
|
||||
MAX_RSA_BITS = 8192
|
||||
MAX_RSA_INTEGER_BYTES = MAX_RSA_BITS // 8
|
||||
RSA_ENCRYPTION_OID = bytes.fromhex("2a864886f70d010101")
|
||||
VERTEX_LOCATIONS = ["global", "us", "eu"]
|
||||
VERTEX_MODELS = ["gemini-3.6-flash", "gemini-3.1-pro-preview"]
|
||||
VERTEX_ANTHROPIC_LOCATIONS = ["global", "us", "eu", "us-east5", "europe-west1"]
|
||||
VERTEX_ANTHROPIC_MODELS = ["claude-opus-5", "claude-opus-4-7", "claude-opus-4-6", "claude-fable-5"]
|
||||
|
||||
|
||||
def transaction_status_files():
|
||||
return {
|
||||
**STATUS_FILES,
|
||||
"RATE_LIMITED": STATUS_FILES["UNKNOWN"],
|
||||
"AUX_VERTEX_GEMINI": VERTEX_GEMINI_FILE,
|
||||
"AUX_VERTEX_ANTHROPIC": VERTEX_ANTHROPIC_FILE,
|
||||
}
|
||||
|
||||
|
||||
def ensure_files():
|
||||
ensure_output_files([CHECKED_FILE, RESULTS_FILE, *STATUS_FILES.values(), VERTEX_GEMINI_FILE, VERTEX_ANTHROPIC_FILE, PLAIN_FILE])
|
||||
recover_status_transaction(CHECKED_FILE, transaction_status_files())
|
||||
|
||||
|
||||
def b64url(data):
|
||||
return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def validate_google_token_uri(value):
|
||||
token_uri = str(value or GOOGLE_TOKEN_URL).strip()
|
||||
try:
|
||||
parsed = urlsplit(token_uri)
|
||||
port = parsed.port
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("invalid Google OAuth token_uri") from exc
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or port not in (None, 443)
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or token_uri not in TRUSTED_GOOGLE_TOKEN_ENDPOINTS
|
||||
):
|
||||
raise ValueError("untrusted Google OAuth token_uri")
|
||||
return token_uri
|
||||
|
||||
|
||||
class InvalidRSAPrivateKey(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class DERReader:
|
||||
def __init__(self, data):
|
||||
if not isinstance(data, (bytes, bytearray, memoryview)):
|
||||
raise InvalidRSAPrivateKey("DER value is not binary")
|
||||
if len(data) > MAX_DER_BYTES:
|
||||
raise InvalidRSAPrivateKey("DER value exceeds size limit")
|
||||
self.data = data
|
||||
self.pos = 0
|
||||
|
||||
def read_tlv(self):
|
||||
if len(self.data) - self.pos < 2:
|
||||
raise InvalidRSAPrivateKey("truncated DER tag or length")
|
||||
tag = self.data[self.pos]
|
||||
self.pos += 1
|
||||
first_len = self.data[self.pos]
|
||||
self.pos += 1
|
||||
if first_len & 0x80:
|
||||
length_len = first_len & 0x7F
|
||||
if length_len == 0:
|
||||
raise InvalidRSAPrivateKey("indefinite DER length is not allowed")
|
||||
if length_len > MAX_DER_LENGTH_BYTES:
|
||||
raise InvalidRSAPrivateKey("DER length-of-length exceeds limit")
|
||||
if len(self.data) - self.pos < length_len:
|
||||
raise InvalidRSAPrivateKey("truncated DER length")
|
||||
length_bytes = self.data[self.pos:self.pos + length_len]
|
||||
if length_bytes[0] == 0:
|
||||
raise InvalidRSAPrivateKey("non-minimal DER length")
|
||||
length = int.from_bytes(length_bytes, "big")
|
||||
self.pos += length_len
|
||||
if length < 0x80:
|
||||
raise InvalidRSAPrivateKey("non-minimal DER length")
|
||||
else:
|
||||
length = first_len
|
||||
if length > MAX_DER_BYTES:
|
||||
raise InvalidRSAPrivateKey("DER value length exceeds limit")
|
||||
if length > len(self.data) - self.pos:
|
||||
raise InvalidRSAPrivateKey("truncated DER value")
|
||||
value = self.data[self.pos:self.pos + length]
|
||||
self.pos += length
|
||||
return tag, value
|
||||
|
||||
def expect(self, tag):
|
||||
actual, value = self.read_tlv()
|
||||
if actual != tag:
|
||||
raise InvalidRSAPrivateKey(f"expected DER tag {tag:#x}, got {actual:#x}")
|
||||
return value
|
||||
|
||||
def at_end(self):
|
||||
return self.pos == len(self.data)
|
||||
|
||||
def require_eof(self, context="DER structure"):
|
||||
if not self.at_end():
|
||||
raise InvalidRSAPrivateKey(f"trailing data in {context}")
|
||||
|
||||
|
||||
def der_int(value, name="integer", max_bytes=MAX_RSA_INTEGER_BYTES):
|
||||
if not value:
|
||||
raise InvalidRSAPrivateKey(f"empty RSA {name}")
|
||||
if len(value) > max_bytes + 1:
|
||||
raise InvalidRSAPrivateKey(f"RSA {name} exceeds size limit")
|
||||
if value[0] & 0x80:
|
||||
raise InvalidRSAPrivateKey(f"negative RSA {name}")
|
||||
if value[0] == 0:
|
||||
if len(value) > 1 and not value[1] & 0x80:
|
||||
raise InvalidRSAPrivateKey(f"non-minimal RSA {name}")
|
||||
unsigned = value[1:]
|
||||
else:
|
||||
unsigned = value
|
||||
if len(unsigned) > max_bytes:
|
||||
raise InvalidRSAPrivateKey(f"RSA {name} exceeds size limit")
|
||||
return int.from_bytes(unsigned, "big") if unsigned else 0
|
||||
|
||||
|
||||
def parse_pkcs1_rsa_private_key(data):
|
||||
rsa = DERReader(data)
|
||||
version = der_int(rsa.expect(0x02), "version", 1)
|
||||
if version != 0:
|
||||
raise InvalidRSAPrivateKey("unsupported RSA private key version")
|
||||
n = der_int(rsa.expect(0x02), "modulus")
|
||||
public_exponent = der_int(rsa.expect(0x02), "public exponent")
|
||||
d = der_int(rsa.expect(0x02), "private exponent")
|
||||
for name in ("prime1", "prime2", "exponent1", "exponent2", "coefficient"):
|
||||
der_int(rsa.expect(0x02), name)
|
||||
rsa.require_eof("RSA private key")
|
||||
|
||||
modulus_bits = n.bit_length()
|
||||
if not MIN_RSA_BITS <= modulus_bits <= MAX_RSA_BITS:
|
||||
raise InvalidRSAPrivateKey(
|
||||
f"RSA modulus must be between {MIN_RSA_BITS} and {MAX_RSA_BITS} bits"
|
||||
)
|
||||
if public_exponent == 0:
|
||||
raise InvalidRSAPrivateKey("RSA public exponent is zero")
|
||||
if d == 0 or d >= n:
|
||||
raise InvalidRSAPrivateKey("RSA private exponent is out of range")
|
||||
return n, d
|
||||
|
||||
|
||||
def validate_rsa_algorithm_identifier(data):
|
||||
algorithm = DERReader(data)
|
||||
if algorithm.expect(0x06) != RSA_ENCRYPTION_OID:
|
||||
raise InvalidRSAPrivateKey("PKCS#8 key does not use rsaEncryption")
|
||||
if not algorithm.at_end() and algorithm.expect(0x05):
|
||||
raise InvalidRSAPrivateKey("invalid rsaEncryption parameters")
|
||||
algorithm.require_eof("PKCS#8 algorithm identifier")
|
||||
|
||||
|
||||
def parse_rsa_private_key_from_pem(pem):
|
||||
pem = str(pem or "")
|
||||
if len(pem) > MAX_PEM_BYTES:
|
||||
raise InvalidRSAPrivateKey("PEM private key exceeds size limit")
|
||||
try:
|
||||
pem_bytes = pem.encode("utf-8")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise InvalidRSAPrivateKey("PEM private key is not valid UTF-8") from exc
|
||||
if len(pem_bytes) > MAX_PEM_BYTES:
|
||||
raise InvalidRSAPrivateKey("PEM private key exceeds size limit")
|
||||
pem = pem.replace("\\n", "\n")
|
||||
match = re.fullmatch(
|
||||
r"\s*-----BEGIN (RSA PRIVATE KEY|PRIVATE KEY)-----\s*(.*?)\s*-----END \1-----\s*",
|
||||
pem,
|
||||
re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
raise InvalidRSAPrivateKey("missing complete PEM private key block")
|
||||
body = re.sub(r"\s+", "", match.group(2))
|
||||
if len(body) < 256:
|
||||
raise InvalidRSAPrivateKey("PEM private key body is too short")
|
||||
try:
|
||||
der = base64.b64decode(body + ("=" * (-len(body) % 4)), validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise InvalidRSAPrivateKey("invalid PEM base64") from exc
|
||||
if len(der) > MAX_DER_BYTES:
|
||||
raise InvalidRSAPrivateKey("DER private key exceeds size limit")
|
||||
|
||||
reader = DERReader(der)
|
||||
top_bytes = reader.expect(0x30)
|
||||
reader.require_eof("DER private key")
|
||||
top = DERReader(top_bytes)
|
||||
|
||||
# PKCS#8 PrivateKeyInfo: SEQUENCE(version, alg, OCTET STRING(RSAPrivateKey))
|
||||
first_tag, first_val = top.read_tlv()
|
||||
if first_tag != 0x02:
|
||||
raise InvalidRSAPrivateKey("unexpected private key structure")
|
||||
second_tag, second_val = top.read_tlv()
|
||||
if second_tag == 0x30:
|
||||
if der_int(first_val, "PKCS#8 version", 1) != 0:
|
||||
raise InvalidRSAPrivateKey("unsupported PKCS#8 version")
|
||||
validate_rsa_algorithm_identifier(second_val)
|
||||
private_octet = top.expect(0x04)
|
||||
top.require_eof("PKCS#8 private key")
|
||||
wrapped = DERReader(private_octet)
|
||||
rsa_bytes = wrapped.expect(0x30)
|
||||
wrapped.require_eof("PKCS#8 private key octets")
|
||||
return parse_pkcs1_rsa_private_key(rsa_bytes)
|
||||
if second_tag != 0x02:
|
||||
raise InvalidRSAPrivateKey("unexpected private key structure")
|
||||
return parse_pkcs1_rsa_private_key(top_bytes)
|
||||
|
||||
|
||||
def rsa_pkcs1v15_sha256_sign(message, pem):
|
||||
n, d = parse_rsa_private_key_from_pem(pem)
|
||||
digest = hashlib.sha256(message).digest()
|
||||
digest_info = bytes.fromhex("3031300d060960864801650304020105000420") + digest
|
||||
key_len = (n.bit_length() + 7) // 8
|
||||
if key_len < len(digest_info) + 11:
|
||||
raise InvalidRSAPrivateKey("RSA key too small")
|
||||
encoded = b"\x00\x01" + b"\xff" * (key_len - len(digest_info) - 3) + b"\x00" + digest_info
|
||||
sig = pow(int.from_bytes(encoded, "big"), d, n).to_bytes(key_len, "big")
|
||||
return sig
|
||||
|
||||
|
||||
def make_service_account_assertion(creds):
|
||||
now = int(time.time())
|
||||
token_uri = validate_google_token_uri(creds.get("token_uri"))
|
||||
header = {"alg": "RS256", "typ": "JWT", "kid": creds.get("private_key_id")}
|
||||
payload = {
|
||||
"iss": creds["client_email"],
|
||||
"scope": SA_SCOPE,
|
||||
"aud": token_uri,
|
||||
"iat": now,
|
||||
"exp": now + 3600,
|
||||
}
|
||||
signing_input = (b64url(json.dumps(header, separators=(",", ":")).encode()) + "." + b64url(json.dumps(payload, separators=(",", ":")).encode())).encode()
|
||||
signature = rsa_pkcs1v15_sha256_sign(signing_input, creds["private_key"])
|
||||
return signing_input.decode() + "." + b64url(signature), token_uri
|
||||
|
||||
|
||||
def compact_json(data):
|
||||
return json.dumps(data, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def scanner_context_text(finding):
|
||||
context = finding.get("ScannerContext") if isinstance(finding, dict) else None
|
||||
if isinstance(context, dict):
|
||||
return str(context.get("nearby") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def parse_json_object(text):
|
||||
try:
|
||||
return json.loads(text)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
match = re.search(r"\{.*\}", str(text or ""), re.DOTALL)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_service_account(raw_v2):
|
||||
data = parse_json_object(raw_v2)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
required = ["client_email", "private_key", "private_key_id"]
|
||||
if not all(data.get(item) for item in required):
|
||||
return None
|
||||
private_key = str(data.get("private_key") or "")
|
||||
if "-----BEGIN" not in private_key or "-----END" not in private_key or len(private_key) < 800:
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def parse_adc(raw_v2, finding):
|
||||
data = parse_json_object(scanner_context_text(finding)) or parse_json_object(raw_v2)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
required = ["client_id", "client_secret", "refresh_token"]
|
||||
if not all(data.get(item) for item in required):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def extract_candidates(input_file, plain_file):
|
||||
seen_plain = set()
|
||||
for item in iter_findings(input_file, ["GCP", "GCPApplicationDefaultCredentials"]):
|
||||
if item["detector"] == "GCP":
|
||||
parsed = parse_service_account(item["raw_v2"])
|
||||
detector = "GCP"
|
||||
else:
|
||||
parsed = parse_adc(item["raw_v2"], item["finding"])
|
||||
detector = "GCPApplicationDefaultCredentials"
|
||||
if not parsed:
|
||||
key = f"{detector}:no_context:{item['source']}"
|
||||
yield key, detector, item["source"], item["finding"], None
|
||||
continue
|
||||
key = compact_json(parsed)
|
||||
yield key, detector, item["source"], item["finding"], parsed
|
||||
|
||||
if keycheck_input_mode() == 'jsonl' and os.path.exists(plain_file):
|
||||
with open(plain_file, "r", encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
candidates = []
|
||||
whole_file = parse_json_object(content)
|
||||
if whole_file:
|
||||
candidates.append((plain_file, whole_file))
|
||||
for line_num, line in enumerate(content.splitlines(), 1):
|
||||
key_text = line.split("\t", 1)[0].strip()
|
||||
parsed = parse_json_object(key_text) or parse_json_object(line)
|
||||
if parsed:
|
||||
candidates.append((f"{plain_file}:{line_num}", parsed))
|
||||
for source, parsed in candidates:
|
||||
detector = "GCP" if parsed.get("private_key") else "GCPApplicationDefaultCredentials"
|
||||
key = compact_json(parsed)
|
||||
if key not in seen_plain:
|
||||
seen_plain.add(key)
|
||||
yield key, detector, source, {}, parsed
|
||||
|
||||
|
||||
def vertex_api_host(location):
|
||||
location = str(location or "").strip().lower()
|
||||
if location == "global":
|
||||
return "aiplatform.googleapis.com"
|
||||
if location in ("us", "eu"):
|
||||
return f"aiplatform.{location}.rep.googleapis.com"
|
||||
return f"{location}-aiplatform.googleapis.com"
|
||||
|
||||
|
||||
def probe_vertex_llm(access_token, project_id, proxy, timeout, locations=None, models=None, max_attempts=2):
|
||||
if not project_id:
|
||||
return {"enabled": False, "message": "project_id unavailable"}
|
||||
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
|
||||
payload = {"contents": [{"role": "user", "parts": [{"text": "ping"}]}]}
|
||||
attempts = []
|
||||
accepted = []
|
||||
tried = 0
|
||||
for location in (locations or VERTEX_LOCATIONS):
|
||||
for model in (models or VERTEX_MODELS):
|
||||
if max_attempts and tried >= max_attempts:
|
||||
if accepted:
|
||||
first = accepted[0]
|
||||
return {
|
||||
"enabled": True,
|
||||
"location": first.get("location", ""),
|
||||
"model": first.get("model", ""),
|
||||
"total_tokens": first.get("total_tokens"),
|
||||
"available_models": [f"google/{item['location']}/{item['model']}" for item in accepted],
|
||||
"message": "Vertex countTokens accepted",
|
||||
}
|
||||
return {"enabled": False, "message": "; ".join(attempts[:8]) or "Vertex probe attempt limit reached"}
|
||||
tried += 1
|
||||
url = f"https://{vertex_api_host(location)}/v1/projects/{project_id}/locations/{location}/publishers/google/models/{model}:countTokens"
|
||||
try:
|
||||
response = requests.post(url, headers=headers, json=payload, proxies=proxy, timeout=timeout)
|
||||
except requests.RequestException as exc:
|
||||
attempts.append(f"{location}:{model}:network:{str(exc)[:120]}")
|
||||
continue
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
accepted.append({
|
||||
"location": location,
|
||||
"model": model,
|
||||
"total_tokens": data.get("totalTokens") or data.get("total_tokens"),
|
||||
})
|
||||
continue
|
||||
message = request_error_message(response)
|
||||
if response.status_code in (400, 401, 403, 404, 429):
|
||||
attempts.append(f"{location}:{model}:http_{response.status_code}:{message[:160]}")
|
||||
continue
|
||||
if response.status_code >= 500:
|
||||
attempts.append(f"{location}:{model}:server_{response.status_code}:{message[:160]}")
|
||||
continue
|
||||
attempts.append(f"{location}:{model}:http_{response.status_code}:{message[:160]}")
|
||||
if accepted:
|
||||
first = accepted[0]
|
||||
return {
|
||||
"enabled": True,
|
||||
"location": first.get("location", ""),
|
||||
"model": first.get("model", ""),
|
||||
"total_tokens": first.get("total_tokens"),
|
||||
"available_models": [f"google/{item['location']}/{item['model']}" for item in accepted],
|
||||
"message": "Vertex countTokens accepted",
|
||||
}
|
||||
return {"enabled": False, "message": "; ".join(attempts[:8])}
|
||||
|
||||
|
||||
def probe_vertex_anthropic(access_token, project_id, proxy, timeout, locations=None, models=None, max_attempts=2):
|
||||
if not project_id:
|
||||
return {"enabled": False, "message": "project_id unavailable"}
|
||||
models = models or []
|
||||
if not models:
|
||||
return {"enabled": False, "message": "no Anthropic models configured"}
|
||||
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
|
||||
payload = {
|
||||
"anthropic_version": "vertex-2023-10-16",
|
||||
"messages": [{"role": "user", "content": "ping"}],
|
||||
"max_tokens": 1,
|
||||
}
|
||||
attempts = []
|
||||
accepted = []
|
||||
tried = 0
|
||||
for location in (locations or VERTEX_ANTHROPIC_LOCATIONS):
|
||||
for model in models:
|
||||
if max_attempts and tried >= max_attempts:
|
||||
if accepted:
|
||||
first = accepted[0]
|
||||
return {
|
||||
"enabled": True,
|
||||
"location": first.get("location", ""),
|
||||
"model": first.get("model", ""),
|
||||
"available_models": [f"anthropic/{item['location']}/{item['model']}" for item in accepted],
|
||||
"message": "Vertex Anthropic rawPredict accepted",
|
||||
}
|
||||
return {"enabled": False, "message": "; ".join(attempts[:8]) or "Vertex Anthropic probe attempt limit reached"}
|
||||
tried += 1
|
||||
url = f"https://{vertex_api_host(location)}/v1/projects/{project_id}/locations/{location}/publishers/anthropic/models/{model}:rawPredict"
|
||||
try:
|
||||
response = requests.post(url, headers=headers, json=payload, proxies=proxy, timeout=timeout)
|
||||
except requests.RequestException as exc:
|
||||
attempts.append(f"{location}:{model}:network:{str(exc)[:120]}")
|
||||
continue
|
||||
if response.status_code == 200:
|
||||
accepted.append({"location": location, "model": model})
|
||||
continue
|
||||
message = request_error_message(response)
|
||||
if response.status_code in (400, 401, 403, 404, 429):
|
||||
attempts.append(f"{location}:{model}:http_{response.status_code}:{message[:160]}")
|
||||
continue
|
||||
if response.status_code >= 500:
|
||||
attempts.append(f"{location}:{model}:server_{response.status_code}:{message[:160]}")
|
||||
continue
|
||||
attempts.append(f"{location}:{model}:http_{response.status_code}:{message[:160]}")
|
||||
if accepted:
|
||||
first = accepted[0]
|
||||
return {
|
||||
"enabled": True,
|
||||
"location": first.get("location", ""),
|
||||
"model": first.get("model", ""),
|
||||
"available_models": [f"anthropic/{item['location']}/{item['model']}" for item in accepted],
|
||||
"message": "Vertex Anthropic rawPredict accepted",
|
||||
}
|
||||
return {"enabled": False, "message": "; ".join(attempts[:8])}
|
||||
|
||||
|
||||
def merge_vertex_results(google_vertex, anthropic_vertex):
|
||||
google_vertex = google_vertex or {"enabled": False, "message": ""}
|
||||
anthropic_vertex = anthropic_vertex or {"enabled": False, "message": ""}
|
||||
available = []
|
||||
available.extend(google_vertex.get("available_models") or [])
|
||||
available.extend(anthropic_vertex.get("available_models") or [])
|
||||
first = google_vertex if google_vertex.get("enabled") else anthropic_vertex if anthropic_vertex.get("enabled") else {}
|
||||
messages = []
|
||||
if google_vertex.get("message"):
|
||||
messages.append(f"google: {google_vertex.get('message')}")
|
||||
if anthropic_vertex.get("message"):
|
||||
messages.append(f"anthropic: {anthropic_vertex.get('message')}")
|
||||
return {
|
||||
"enabled": bool(available),
|
||||
"location": first.get("location", ""),
|
||||
"model": first.get("model", ""),
|
||||
"total_tokens": first.get("total_tokens"),
|
||||
"available_models": available,
|
||||
"google_enabled": bool(google_vertex.get("enabled")),
|
||||
"anthropic_enabled": bool(anthropic_vertex.get("enabled")),
|
||||
"message": "; ".join(messages),
|
||||
}
|
||||
|
||||
|
||||
def check_service_account(creds, proxy, timeout, probe_vertex=False, vertex_timeout=6, vertex_locations=None, vertex_models=None, vertex_max_attempts=2, vertex_anthropic_locations=None, vertex_anthropic_models=None, vertex_anthropic_max_attempts=2):
|
||||
try:
|
||||
assertion, token_uri = make_service_account_assertion(creds)
|
||||
except InvalidRSAPrivateKey as exc:
|
||||
return {
|
||||
"status": "DEAD",
|
||||
"classification": "invalid_private_key",
|
||||
"message": f"invalid RSA private key: {exc}",
|
||||
"project_id": creds.get("project_id"),
|
||||
"client_email": creds.get("client_email"),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "UNKNOWN", "message": f"failed to build JWT assertion: {exc}"}
|
||||
data = {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": assertion}
|
||||
try:
|
||||
response = requests.post(
|
||||
token_uri, data=data, proxies=proxy, timeout=timeout, allow_redirects=False,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return {"status": "NETWORK", "message": str(exc), "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
if response.status_code in TOKEN_REDIRECT_STATUSES:
|
||||
return {
|
||||
"status": "UNKNOWN",
|
||||
"http_status": response.status_code,
|
||||
"message": "Google OAuth token endpoint redirect refused",
|
||||
"project_id": creds.get("project_id"),
|
||||
"client_email": creds.get("client_email"),
|
||||
}
|
||||
if response.status_code == 200:
|
||||
payload = response.json()
|
||||
result = {
|
||||
"status": "VALID",
|
||||
"message": "OAuth token issued",
|
||||
"project_id": creds.get("project_id"),
|
||||
"client_email": creds.get("client_email"),
|
||||
"private_key_id": creds.get("private_key_id"),
|
||||
"expires_in": payload.get("expires_in"),
|
||||
}
|
||||
if probe_vertex:
|
||||
google_vertex = probe_vertex_llm(
|
||||
payload.get("access_token"), creds.get("project_id"), proxy,
|
||||
vertex_timeout, vertex_locations, vertex_models, vertex_max_attempts,
|
||||
)
|
||||
anthropic_vertex = probe_vertex_anthropic(
|
||||
payload.get("access_token"), creds.get("project_id"), proxy,
|
||||
vertex_timeout, vertex_anthropic_locations, vertex_anthropic_models, vertex_anthropic_max_attempts,
|
||||
) if vertex_anthropic_models else {"enabled": False, "message": ""}
|
||||
vertex = merge_vertex_results(google_vertex, anthropic_vertex)
|
||||
result.update({
|
||||
"vertex_enabled": vertex.get("enabled"),
|
||||
"vertex_location": vertex.get("location", ""),
|
||||
"vertex_model": vertex.get("model", ""),
|
||||
"vertex_available_models": vertex.get("available_models") or [],
|
||||
"vertex_google_enabled": vertex.get("google_enabled"),
|
||||
"vertex_anthropic_enabled": vertex.get("anthropic_enabled"),
|
||||
"vertex_total_tokens": vertex.get("total_tokens"),
|
||||
"vertex_message": vertex.get("message", ""),
|
||||
})
|
||||
if vertex.get("enabled"):
|
||||
result["status"] = "VERTEX"
|
||||
result["message"] = "OAuth token issued; Vertex countTokens accepted"
|
||||
return result
|
||||
message = request_error_message(response)
|
||||
lower = message.lower()
|
||||
if response.status_code in (400, 401) and any(item in lower for item in ("invalid_grant", "invalid_client", "invalid jwt", "invalid signature")):
|
||||
return {"status": "DEAD", "http_status": response.status_code, "message": message, "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
if response.status_code in (401, 403):
|
||||
return {"status": "RESTRICTED", "http_status": response.status_code, "message": message, "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
if response.status_code == 429:
|
||||
return {"status": "RATE_LIMITED", "http_status": 429, "message": message, "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
if response.status_code >= 500:
|
||||
return {"status": "NETWORK", "http_status": response.status_code, "message": message, "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
return {"status": "UNKNOWN", "http_status": response.status_code, "message": message, "project_id": creds.get("project_id"), "client_email": creds.get("client_email")}
|
||||
|
||||
|
||||
def check_adc(creds, proxy, timeout, probe_vertex=False, vertex_timeout=6, vertex_locations=None, vertex_models=None, vertex_max_attempts=2, vertex_anthropic_locations=None, vertex_anthropic_models=None, vertex_anthropic_max_attempts=2):
|
||||
try:
|
||||
token_uri = validate_google_token_uri(creds.get("token_uri"))
|
||||
except ValueError as exc:
|
||||
return {"status": "UNKNOWN", "message": str(exc), "client_id": creds.get("client_id")}
|
||||
data = {
|
||||
"client_id": creds["client_id"],
|
||||
"client_secret": creds["client_secret"],
|
||||
"refresh_token": creds["refresh_token"],
|
||||
"grant_type": "refresh_token",
|
||||
}
|
||||
try:
|
||||
response = requests.post(
|
||||
token_uri, data=data, proxies=proxy, timeout=timeout, allow_redirects=False,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return {"status": "NETWORK", "message": str(exc), "client_id": creds.get("client_id")}
|
||||
if response.status_code in TOKEN_REDIRECT_STATUSES:
|
||||
return {
|
||||
"status": "UNKNOWN",
|
||||
"http_status": response.status_code,
|
||||
"message": "Google OAuth token endpoint redirect refused",
|
||||
"client_id": creds.get("client_id"),
|
||||
}
|
||||
if response.status_code == 200:
|
||||
payload = response.json()
|
||||
project_id = creds.get("quota_project_id") or creds.get("project_id")
|
||||
result = {"status": "VALID", "message": "refresh token accepted", "client_id": creds.get("client_id"), "project_id": project_id, "expires_in": payload.get("expires_in")}
|
||||
if probe_vertex:
|
||||
google_vertex = probe_vertex_llm(
|
||||
payload.get("access_token"), project_id, proxy,
|
||||
vertex_timeout, vertex_locations, vertex_models, vertex_max_attempts,
|
||||
)
|
||||
anthropic_vertex = probe_vertex_anthropic(
|
||||
payload.get("access_token"), project_id, proxy,
|
||||
vertex_timeout, vertex_anthropic_locations, vertex_anthropic_models, vertex_anthropic_max_attempts,
|
||||
) if vertex_anthropic_models else {"enabled": False, "message": ""}
|
||||
vertex = merge_vertex_results(google_vertex, anthropic_vertex)
|
||||
result.update({
|
||||
"vertex_enabled": vertex.get("enabled"),
|
||||
"vertex_location": vertex.get("location", ""),
|
||||
"vertex_model": vertex.get("model", ""),
|
||||
"vertex_available_models": vertex.get("available_models") or [],
|
||||
"vertex_google_enabled": vertex.get("google_enabled"),
|
||||
"vertex_anthropic_enabled": vertex.get("anthropic_enabled"),
|
||||
"vertex_total_tokens": vertex.get("total_tokens"),
|
||||
"vertex_message": vertex.get("message", ""),
|
||||
})
|
||||
if vertex.get("enabled"):
|
||||
result["status"] = "VERTEX"
|
||||
result["message"] = "refresh token accepted; Vertex countTokens accepted"
|
||||
return result
|
||||
message = request_error_message(response)
|
||||
lower = message.lower()
|
||||
if response.status_code in (400, 401) and any(item in lower for item in ("invalid_grant", "invalid_client", "unauthorized_client")):
|
||||
return {"status": "DEAD", "http_status": response.status_code, "message": message, "client_id": creds.get("client_id")}
|
||||
if response.status_code in (401, 403):
|
||||
return {"status": "RESTRICTED", "http_status": response.status_code, "message": message, "client_id": creds.get("client_id")}
|
||||
if response.status_code == 429:
|
||||
return {"status": "RATE_LIMITED", "http_status": 429, "message": message, "client_id": creds.get("client_id")}
|
||||
if response.status_code >= 500:
|
||||
return {"status": "NETWORK", "http_status": response.status_code, "message": message, "client_id": creds.get("client_id")}
|
||||
return {"status": "UNKNOWN", "http_status": response.status_code, "message": message, "client_id": creds.get("client_id")}
|
||||
|
||||
|
||||
def write_result(key, detector, result, source, finding):
|
||||
write_keycheck_event(SERVICE, RESULTS_FILE, key, {"detector": detector, **result}, source, finding, detector)
|
||||
extra = result.get("client_email") or result.get("client_id") or result.get("project_id") or source
|
||||
message = result.get("message", "")
|
||||
if result.get("status") == "VERTEX":
|
||||
models = result.get("vertex_available_models") or []
|
||||
model_text = ",".join(str(item) for item in models) or f"{result.get('vertex_location', '')}/{result.get('vertex_model', '')}".strip("/")
|
||||
message = f"{message}; models={model_text}"
|
||||
commit_status_transaction(
|
||||
CHECKED_FILE, transaction_status_files(), key, result["status"], message, extra,
|
||||
)
|
||||
if result.get("status") == "VERTEX" and result.get("vertex_google_enabled"):
|
||||
append_status(VERTEX_GEMINI_FILE, key, result["status"], message, extra)
|
||||
if result.get("status") == "VERTEX" and result.get("vertex_anthropic_enabled"):
|
||||
append_status(VERTEX_ANTHROPIC_FILE, key, result["status"], message, extra)
|
||||
record_validation_result(SERVICE, key, {"detector": detector, **result}, source, finding, detector)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="GCP credential checker")
|
||||
parser.add_argument("--input", default=INPUT_FILE)
|
||||
parser.add_argument("--plain", default=PLAIN_FILE)
|
||||
parser.add_argument("--proxy-file", default=PROXY_FILE)
|
||||
parser.add_argument("--timeout", type=int, default=25)
|
||||
parser.add_argument("--max-keys", type=int, default=0)
|
||||
parser.add_argument("--retry-network", action="store_true")
|
||||
parser.add_argument("--retry-limited", action="store_true")
|
||||
parser.add_argument("--retry-unknown", action="store_true")
|
||||
parser.add_argument("--retry-valid", action="store_true")
|
||||
parser.add_argument("--recheck-all", action="store_true")
|
||||
parser.add_argument("--probe-vertex", action="store_true", help="After OAuth succeeds, probe Vertex AI Gemini with countTokens through the configured proxy")
|
||||
parser.add_argument("--vertex-timeout", type=int, default=6, help="Seconds per Vertex countTokens request")
|
||||
parser.add_argument("--vertex-max-attempts", type=int, default=6, help="Maximum location/model countTokens attempts per credential")
|
||||
parser.add_argument("--vertex-locations", default=",".join(VERTEX_LOCATIONS), help="Comma-separated Vertex locations to probe")
|
||||
parser.add_argument("--vertex-models", default=",".join(VERTEX_MODELS), help="Comma-separated Vertex models to probe")
|
||||
parser.add_argument("--vertex-anthropic-locations", default=",".join(VERTEX_ANTHROPIC_LOCATIONS), help="Comma-separated Vertex Anthropic locations to probe")
|
||||
parser.add_argument("--vertex-anthropic-models", default=",".join(VERTEX_ANTHROPIC_MODELS), help="Comma-separated Vertex Anthropic model IDs to probe with rawPredict")
|
||||
parser.add_argument("--vertex-anthropic-max-attempts", type=int, default=20, help="Maximum Anthropic location/model attempts per credential")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
require_provider_authority(SERVICE)
|
||||
args = parse_args()
|
||||
ensure_files()
|
||||
proxy_cycler = load_proxies(args.proxy_file)
|
||||
checked = load_checked_statuses(CHECKED_FILE)
|
||||
known_statuses = load_known_statuses(CHECKED_FILE, STATUS_FILES)
|
||||
known = set(known_statuses)
|
||||
retry_statuses = set()
|
||||
if args.retry_network:
|
||||
retry_statuses.add("NETWORK")
|
||||
if args.retry_limited:
|
||||
retry_statuses.add("RATE_LIMITED")
|
||||
if args.retry_unknown:
|
||||
retry_statuses.add("UNKNOWN")
|
||||
if args.retry_valid:
|
||||
retry_statuses.update({"VALID", "VERTEX"})
|
||||
vertex_locations = [item.strip() for item in str(args.vertex_locations or "").split(",") if item.strip()]
|
||||
vertex_models = [item.strip() for item in str(args.vertex_models or "").split(",") if item.strip()]
|
||||
vertex_anthropic_locations = [item.strip() for item in str(args.vertex_anthropic_locations or "").split(",") if item.strip()]
|
||||
vertex_anthropic_models = [item.strip() for item in str(args.vertex_anthropic_models or "").split(",") if item.strip()]
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
for key, detector, source, finding, parsed in extract_candidates(args.input, args.plain):
|
||||
if should_skip_key(
|
||||
key, checked, known, args, retry_statuses,
|
||||
service=SERVICE, source=source, finding=finding, detector=detector, known_statuses=known_statuses,
|
||||
):
|
||||
skipped += 1
|
||||
continue
|
||||
if args.max_keys and processed >= args.max_keys:
|
||||
break
|
||||
processed += 1
|
||||
print(f"\n[{processed}] {detector} candidate {mask_secret(key)} from {source}", flush=True)
|
||||
proxy = next(proxy_cycler) if proxy_cycler else None
|
||||
if not parsed:
|
||||
result = {"status": "NO_CONTEXT", "message": "credential JSON is incomplete or unavailable"}
|
||||
elif detector == "GCP":
|
||||
result = check_service_account(
|
||||
parsed, proxy, args.timeout, args.probe_vertex,
|
||||
args.vertex_timeout, vertex_locations, vertex_models, args.vertex_max_attempts,
|
||||
vertex_anthropic_locations, vertex_anthropic_models, args.vertex_anthropic_max_attempts,
|
||||
)
|
||||
else:
|
||||
result = check_adc(
|
||||
parsed, proxy, args.timeout, args.probe_vertex,
|
||||
args.vertex_timeout, vertex_locations, vertex_models, args.vertex_max_attempts,
|
||||
vertex_anthropic_locations, vertex_anthropic_models, args.vertex_anthropic_max_attempts,
|
||||
)
|
||||
print(f" STATUS: {result['status']} | {str(result.get('message', ''))[:200]}", flush=True)
|
||||
write_result(key, detector, result, source, finding)
|
||||
known.add(key)
|
||||
checked[key] = result["status"]
|
||||
time.sleep(0.1)
|
||||
print(f"\nDone. Processed={processed}, skipped={skipped}, results={RESULTS_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user