sashatrask
238 lines
9.0 KiB
Python
238 lines
9.0 KiB
Python
import ntpath
|
|
import os
|
|
import re
|
|
|
|
from query_policy import validate_rejected_query_policy
|
|
|
|
|
|
APP_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
CANONICAL_ROOT = os.path.dirname(APP_DIR)
|
|
DEFAULT_TRUFFLEHOG = r"C:\Tools\trufflehog.exe"
|
|
|
|
PLACEHOLDER_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
|
|
|
|
class PathResolutionError(ValueError):
|
|
pass
|
|
|
|
|
|
def _norm(path):
|
|
return os.path.normpath(str(path))
|
|
|
|
|
|
def _config_dir(config_path=None):
|
|
if config_path:
|
|
return os.path.dirname(os.path.abspath(config_path))
|
|
return APP_DIR
|
|
|
|
|
|
def _expand(value, context):
|
|
text = str(value)
|
|
missing = sorted({name for name in PLACEHOLDER_RE.findall(text) if name not in context})
|
|
if missing:
|
|
raise PathResolutionError(f"Unknown path placeholder(s): {', '.join(missing)} in {text!r}")
|
|
for name in PLACEHOLDER_RE.findall(text):
|
|
text = text.replace("{" + name + "}", str(context[name]))
|
|
return os.path.expandvars(os.path.expanduser(text))
|
|
|
|
|
|
def is_command_name(value):
|
|
text = str(value or "")
|
|
return bool(text) and not os.path.isabs(text) and "\\" not in text and "/" not in text
|
|
|
|
|
|
def is_database_url(value):
|
|
return str(value or "").strip().lower().startswith(("postgresql://", "postgres://"))
|
|
|
|
|
|
def resolve_path(value, context=None, base_dir=None, allow_command=False, required=False):
|
|
if value is None or str(value).strip() == "":
|
|
if required:
|
|
raise PathResolutionError("Required path is empty")
|
|
return value
|
|
|
|
context = context or {}
|
|
text = _expand(value, context)
|
|
if os.name != 'nt' and (ntpath.splitdrive(text)[0] or '\\' in text):
|
|
raise PathResolutionError(f"Windows path is not supported on this platform: {text!r}")
|
|
if allow_command and is_command_name(text):
|
|
return text
|
|
if os.path.isabs(text):
|
|
return _norm(text)
|
|
base = base_dir or context.get("project_dir") or context.get("config_dir") or os.getcwd()
|
|
if os.name != 'nt' and (ntpath.splitdrive(str(base))[0] or '\\' in str(base)):
|
|
raise PathResolutionError(f"Windows base path is not supported on this platform: {base!r}")
|
|
return _norm(os.path.join(base, text))
|
|
|
|
|
|
def default_trufflehog_path():
|
|
return DEFAULT_TRUFFLEHOG if os.name == 'nt' and os.path.exists(DEFAULT_TRUFFLEHOG) else "trufflehog"
|
|
|
|
|
|
def resolve_project_paths(global_config=None, config_path=None):
|
|
global_config = global_config or {}
|
|
context = {"config_dir": _config_dir(config_path)}
|
|
|
|
root_raw = (
|
|
global_config.get("root_dir")
|
|
or os.getenv("SCANNER_ROOT_DIR")
|
|
or os.getenv("SCANNER_PROJECT_ROOT")
|
|
or CANONICAL_ROOT
|
|
)
|
|
context["root_dir"] = resolve_path(root_raw, context, base_dir=context["config_dir"], required=True)
|
|
|
|
project_raw = global_config.get("project_dir") or os.getenv("SCANNER_PROJECT_DIR") or context["config_dir"]
|
|
context["project_dir"] = resolve_path(project_raw, context, base_dir=context["config_dir"], required=True)
|
|
|
|
ordered_defaults = [
|
|
("runtime_dir", os.getenv("SCANNER_RUNTIME_DIR") or os.path.join(context["root_dir"], "runtime")),
|
|
("result_bundle_dir", os.getenv("SCANNER_RESULT_BUNDLE_DIR") or "{runtime_dir}/result_bundles"),
|
|
("result_spool_dir", "{runtime_dir}/result_spool"),
|
|
("results_dir", os.getenv("SCAN_RESULTS_DIR") or "{runtime_dir}/results"),
|
|
("queue_dir", "{runtime_dir}/queues"),
|
|
("state_dir", "{runtime_dir}/state"),
|
|
("log_dir", "{runtime_dir}/logs"),
|
|
("control_dir", "{runtime_dir}/control"),
|
|
("keycheck_dir", "{runtime_dir}/keychecks"),
|
|
("postman_cache_dir", "{runtime_dir}/postman_cache"),
|
|
("gharchive_cache_dir", "{state_dir}/gharchive_cache"),
|
|
("work_dir", os.getenv("TRUFFLEHOG_WORK_DIR") or os.path.join(context["root_dir"], "tmp")),
|
|
("proxy_file", "{runtime_dir}/proxy.txt"),
|
|
("database_path", os.getenv("SCANNER_DB_PATH") or os.getenv("SCAN_DB_PATH") or "{results_dir}/scanner.db"),
|
|
("state_file", "{state_dir}/runner_state.json"),
|
|
("secrets_file", "{project_dir}/secrets.yaml"),
|
|
]
|
|
|
|
for key, default in ordered_defaults:
|
|
raw = global_config.get(key) or default
|
|
context[key] = resolve_path(raw, context, base_dir=context["project_dir"], required=True)
|
|
|
|
managed_database_url = os.getenv("TRUF_MANAGED_POSTGRES_DSN") or ""
|
|
database_url = managed_database_url or global_config.get("database_url") or os.getenv("SCANNER_DB_URL") or os.getenv("DATABASE_URL") or ""
|
|
context["database_url"] = _expand(database_url, context) if database_url else ""
|
|
dashboard_db_url = managed_database_url or global_config.get("dashboard_db_url") or os.getenv("SCANNER_DASHBOARD_DB_URL") or context["database_url"]
|
|
context["dashboard_db_url"] = _expand(dashboard_db_url, context) if dashboard_db_url else ""
|
|
|
|
trufflehog_raw = global_config.get("trufflehog_path") or os.getenv("TRUFFLEHOG_PATH") or default_trufflehog_path()
|
|
context["trufflehog_path"] = resolve_path(
|
|
trufflehog_raw,
|
|
context,
|
|
base_dir=context["project_dir"],
|
|
allow_command=True,
|
|
required=True,
|
|
)
|
|
return context
|
|
|
|
|
|
def default_project_paths():
|
|
return resolve_project_paths({}, None)
|
|
|
|
|
|
def resolve_optional_path(value, path_context, base_dir=None, allow_command=False):
|
|
if not value:
|
|
return value
|
|
return resolve_path(value, path_context, base_dir=base_dir or path_context.get("project_dir"), allow_command=allow_command)
|
|
|
|
|
|
def resolve_postgres_data_dir(global_config=None, runtime_dir=None, base_dir=None):
|
|
global_config = global_config or {}
|
|
runtime_dir = runtime_dir or global_config.get('runtime_dir')
|
|
if not runtime_dir:
|
|
root_dir = global_config.get('root_dir') or CANONICAL_ROOT
|
|
runtime_dir = os.path.join(root_dir, 'runtime')
|
|
context = dict(global_config)
|
|
context['runtime_dir'] = runtime_dir
|
|
raw = global_config.get('postgres_data_dir') or os.path.join(runtime_dir, 'postgres', 'data')
|
|
return resolve_path(
|
|
raw,
|
|
context,
|
|
base_dir=base_dir or global_config.get('project_dir') or global_config.get('root_dir'),
|
|
required=True,
|
|
)
|
|
|
|
|
|
def resolve_postgres_bin_dir(global_config=None, runtime_dir=None, base_dir=None):
|
|
global_config = global_config or {}
|
|
runtime_dir = runtime_dir or global_config.get('runtime_dir')
|
|
if not runtime_dir:
|
|
runtime_dir = os.path.join(global_config.get('root_dir') or CANONICAL_ROOT, 'runtime')
|
|
context = dict(global_config, runtime_dir=runtime_dir)
|
|
return resolve_path(
|
|
global_config.get('postgres_bin_dir') or os.path.join(runtime_dir, 'postgres', 'pgsql', 'bin'),
|
|
context,
|
|
base_dir=base_dir or global_config.get('project_dir') or global_config.get('root_dir'),
|
|
required=True,
|
|
)
|
|
|
|
|
|
def apply_path_config(config, config_path=None):
|
|
config = config or {}
|
|
validate_rejected_query_policy(config)
|
|
global_config = config.setdefault("global", {})
|
|
path_context = resolve_project_paths(global_config, config_path)
|
|
|
|
for key, value in path_context.items():
|
|
global_config[key] = value
|
|
|
|
if global_config.get('legacy_result_spool_dir'):
|
|
global_config['legacy_result_spool_dir'] = resolve_path(
|
|
global_config['legacy_result_spool_dir'],
|
|
path_context,
|
|
base_dir=path_context['project_dir'],
|
|
required=True,
|
|
)
|
|
|
|
if global_config.get('postgres_data_dir'):
|
|
global_config['postgres_data_dir'] = resolve_postgres_data_dir(
|
|
global_config,
|
|
path_context['runtime_dir'],
|
|
base_dir=path_context['project_dir'],
|
|
)
|
|
|
|
if global_config.get('postgres_bin_dir'):
|
|
global_config['postgres_bin_dir'] = resolve_postgres_bin_dir(
|
|
global_config,
|
|
path_context['runtime_dir'],
|
|
base_dir=path_context['project_dir'],
|
|
)
|
|
|
|
for key in (
|
|
'api_proxy_file', 'download_proxy_file', 'trufflehog_config',
|
|
'dashboard_db_path', 'scan_limiter_db', 'dockerhub_tag_cache_path',
|
|
):
|
|
if global_config.get(key):
|
|
global_config[key] = resolve_path(
|
|
global_config[key],
|
|
path_context,
|
|
base_dir=path_context['project_dir'],
|
|
required=True,
|
|
)
|
|
|
|
supervisor = config.setdefault("supervisor", {})
|
|
supervisor_defaults = {
|
|
"log_dir": "{log_dir}",
|
|
"control_dir": "{control_dir}",
|
|
"instance_file": "{control_dir}/supervisor.instance.json",
|
|
"lock_file": "{control_dir}/supervisor.lock",
|
|
"supervisor_log": "{log_dir}/supervisor.log",
|
|
"status_file": "{log_dir}/supervisor.status.txt",
|
|
"dashboard_log": "{log_dir}/dashboard.log",
|
|
"state_dir": "{state_dir}",
|
|
}
|
|
for key, default in supervisor_defaults.items():
|
|
supervisor[key] = resolve_path(
|
|
supervisor.get(key) or default,
|
|
path_context,
|
|
base_dir=path_context["project_dir"],
|
|
required=True,
|
|
)
|
|
|
|
return config
|
|
|
|
|
|
def ensure_directories(paths, keys):
|
|
for key in keys:
|
|
path = paths.get(key)
|
|
if path:
|
|
os.makedirs(path, exist_ok=True)
|