Initial server source import
This commit is contained in:
sashatrask
+237
@@ -0,0 +1,237 @@
|
||||
import ntpath
|
||||
import os
|
||||
import re
|
||||
|
||||
from query_policy import validate_rejected_query_policy
|
||||
|
||||
|
||||
APP_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
CANONICAL_ROOT = os.path.dirname(APP_DIR)
|
||||
DEFAULT_TRUFFLEHOG = r"C:\Tools\trufflehog.exe"
|
||||
|
||||
PLACEHOLDER_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||
|
||||
|
||||
class PathResolutionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _norm(path):
|
||||
return os.path.normpath(str(path))
|
||||
|
||||
|
||||
def _config_dir(config_path=None):
|
||||
if config_path:
|
||||
return os.path.dirname(os.path.abspath(config_path))
|
||||
return APP_DIR
|
||||
|
||||
|
||||
def _expand(value, context):
|
||||
text = str(value)
|
||||
missing = sorted({name for name in PLACEHOLDER_RE.findall(text) if name not in context})
|
||||
if missing:
|
||||
raise PathResolutionError(f"Unknown path placeholder(s): {', '.join(missing)} in {text!r}")
|
||||
for name in PLACEHOLDER_RE.findall(text):
|
||||
text = text.replace("{" + name + "}", str(context[name]))
|
||||
return os.path.expandvars(os.path.expanduser(text))
|
||||
|
||||
|
||||
def is_command_name(value):
|
||||
text = str(value or "")
|
||||
return bool(text) and not os.path.isabs(text) and "\\" not in text and "/" not in text
|
||||
|
||||
|
||||
def is_database_url(value):
|
||||
return str(value or "").strip().lower().startswith(("postgresql://", "postgres://"))
|
||||
|
||||
|
||||
def resolve_path(value, context=None, base_dir=None, allow_command=False, required=False):
|
||||
if value is None or str(value).strip() == "":
|
||||
if required:
|
||||
raise PathResolutionError("Required path is empty")
|
||||
return value
|
||||
|
||||
context = context or {}
|
||||
text = _expand(value, context)
|
||||
if os.name != 'nt' and (ntpath.splitdrive(text)[0] or '\\' in text):
|
||||
raise PathResolutionError(f"Windows path is not supported on this platform: {text!r}")
|
||||
if allow_command and is_command_name(text):
|
||||
return text
|
||||
if os.path.isabs(text):
|
||||
return _norm(text)
|
||||
base = base_dir or context.get("project_dir") or context.get("config_dir") or os.getcwd()
|
||||
if os.name != 'nt' and (ntpath.splitdrive(str(base))[0] or '\\' in str(base)):
|
||||
raise PathResolutionError(f"Windows base path is not supported on this platform: {base!r}")
|
||||
return _norm(os.path.join(base, text))
|
||||
|
||||
|
||||
def default_trufflehog_path():
|
||||
return DEFAULT_TRUFFLEHOG if os.name == 'nt' and os.path.exists(DEFAULT_TRUFFLEHOG) else "trufflehog"
|
||||
|
||||
|
||||
def resolve_project_paths(global_config=None, config_path=None):
|
||||
global_config = global_config or {}
|
||||
context = {"config_dir": _config_dir(config_path)}
|
||||
|
||||
root_raw = (
|
||||
global_config.get("root_dir")
|
||||
or os.getenv("SCANNER_ROOT_DIR")
|
||||
or os.getenv("SCANNER_PROJECT_ROOT")
|
||||
or CANONICAL_ROOT
|
||||
)
|
||||
context["root_dir"] = resolve_path(root_raw, context, base_dir=context["config_dir"], required=True)
|
||||
|
||||
project_raw = global_config.get("project_dir") or os.getenv("SCANNER_PROJECT_DIR") or context["config_dir"]
|
||||
context["project_dir"] = resolve_path(project_raw, context, base_dir=context["config_dir"], required=True)
|
||||
|
||||
ordered_defaults = [
|
||||
("runtime_dir", os.getenv("SCANNER_RUNTIME_DIR") or os.path.join(context["root_dir"], "runtime")),
|
||||
("result_bundle_dir", os.getenv("SCANNER_RESULT_BUNDLE_DIR") or "{runtime_dir}/result_bundles"),
|
||||
("result_spool_dir", "{runtime_dir}/result_spool"),
|
||||
("results_dir", os.getenv("SCAN_RESULTS_DIR") or "{runtime_dir}/results"),
|
||||
("queue_dir", "{runtime_dir}/queues"),
|
||||
("state_dir", "{runtime_dir}/state"),
|
||||
("log_dir", "{runtime_dir}/logs"),
|
||||
("control_dir", "{runtime_dir}/control"),
|
||||
("keycheck_dir", "{runtime_dir}/keychecks"),
|
||||
("postman_cache_dir", "{runtime_dir}/postman_cache"),
|
||||
("gharchive_cache_dir", "{state_dir}/gharchive_cache"),
|
||||
("work_dir", os.getenv("TRUFFLEHOG_WORK_DIR") or os.path.join(context["root_dir"], "tmp")),
|
||||
("proxy_file", "{runtime_dir}/proxy.txt"),
|
||||
("database_path", os.getenv("SCANNER_DB_PATH") or os.getenv("SCAN_DB_PATH") or "{results_dir}/scanner.db"),
|
||||
("state_file", "{state_dir}/runner_state.json"),
|
||||
("secrets_file", "{project_dir}/secrets.yaml"),
|
||||
]
|
||||
|
||||
for key, default in ordered_defaults:
|
||||
raw = global_config.get(key) or default
|
||||
context[key] = resolve_path(raw, context, base_dir=context["project_dir"], required=True)
|
||||
|
||||
managed_database_url = os.getenv("TRUF_MANAGED_POSTGRES_DSN") or ""
|
||||
database_url = managed_database_url or global_config.get("database_url") or os.getenv("SCANNER_DB_URL") or os.getenv("DATABASE_URL") or ""
|
||||
context["database_url"] = _expand(database_url, context) if database_url else ""
|
||||
dashboard_db_url = managed_database_url or global_config.get("dashboard_db_url") or os.getenv("SCANNER_DASHBOARD_DB_URL") or context["database_url"]
|
||||
context["dashboard_db_url"] = _expand(dashboard_db_url, context) if dashboard_db_url else ""
|
||||
|
||||
trufflehog_raw = global_config.get("trufflehog_path") or os.getenv("TRUFFLEHOG_PATH") or default_trufflehog_path()
|
||||
context["trufflehog_path"] = resolve_path(
|
||||
trufflehog_raw,
|
||||
context,
|
||||
base_dir=context["project_dir"],
|
||||
allow_command=True,
|
||||
required=True,
|
||||
)
|
||||
return context
|
||||
|
||||
|
||||
def default_project_paths():
|
||||
return resolve_project_paths({}, None)
|
||||
|
||||
|
||||
def resolve_optional_path(value, path_context, base_dir=None, allow_command=False):
|
||||
if not value:
|
||||
return value
|
||||
return resolve_path(value, path_context, base_dir=base_dir or path_context.get("project_dir"), allow_command=allow_command)
|
||||
|
||||
|
||||
def resolve_postgres_data_dir(global_config=None, runtime_dir=None, base_dir=None):
|
||||
global_config = global_config or {}
|
||||
runtime_dir = runtime_dir or global_config.get('runtime_dir')
|
||||
if not runtime_dir:
|
||||
root_dir = global_config.get('root_dir') or CANONICAL_ROOT
|
||||
runtime_dir = os.path.join(root_dir, 'runtime')
|
||||
context = dict(global_config)
|
||||
context['runtime_dir'] = runtime_dir
|
||||
raw = global_config.get('postgres_data_dir') or os.path.join(runtime_dir, 'postgres', 'data')
|
||||
return resolve_path(
|
||||
raw,
|
||||
context,
|
||||
base_dir=base_dir or global_config.get('project_dir') or global_config.get('root_dir'),
|
||||
required=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_postgres_bin_dir(global_config=None, runtime_dir=None, base_dir=None):
|
||||
global_config = global_config or {}
|
||||
runtime_dir = runtime_dir or global_config.get('runtime_dir')
|
||||
if not runtime_dir:
|
||||
runtime_dir = os.path.join(global_config.get('root_dir') or CANONICAL_ROOT, 'runtime')
|
||||
context = dict(global_config, runtime_dir=runtime_dir)
|
||||
return resolve_path(
|
||||
global_config.get('postgres_bin_dir') or os.path.join(runtime_dir, 'postgres', 'pgsql', 'bin'),
|
||||
context,
|
||||
base_dir=base_dir or global_config.get('project_dir') or global_config.get('root_dir'),
|
||||
required=True,
|
||||
)
|
||||
|
||||
|
||||
def apply_path_config(config, config_path=None):
|
||||
config = config or {}
|
||||
validate_rejected_query_policy(config)
|
||||
global_config = config.setdefault("global", {})
|
||||
path_context = resolve_project_paths(global_config, config_path)
|
||||
|
||||
for key, value in path_context.items():
|
||||
global_config[key] = value
|
||||
|
||||
if global_config.get('legacy_result_spool_dir'):
|
||||
global_config['legacy_result_spool_dir'] = resolve_path(
|
||||
global_config['legacy_result_spool_dir'],
|
||||
path_context,
|
||||
base_dir=path_context['project_dir'],
|
||||
required=True,
|
||||
)
|
||||
|
||||
if global_config.get('postgres_data_dir'):
|
||||
global_config['postgres_data_dir'] = resolve_postgres_data_dir(
|
||||
global_config,
|
||||
path_context['runtime_dir'],
|
||||
base_dir=path_context['project_dir'],
|
||||
)
|
||||
|
||||
if global_config.get('postgres_bin_dir'):
|
||||
global_config['postgres_bin_dir'] = resolve_postgres_bin_dir(
|
||||
global_config,
|
||||
path_context['runtime_dir'],
|
||||
base_dir=path_context['project_dir'],
|
||||
)
|
||||
|
||||
for key in (
|
||||
'api_proxy_file', 'download_proxy_file', 'trufflehog_config',
|
||||
'dashboard_db_path', 'scan_limiter_db', 'dockerhub_tag_cache_path',
|
||||
):
|
||||
if global_config.get(key):
|
||||
global_config[key] = resolve_path(
|
||||
global_config[key],
|
||||
path_context,
|
||||
base_dir=path_context['project_dir'],
|
||||
required=True,
|
||||
)
|
||||
|
||||
supervisor = config.setdefault("supervisor", {})
|
||||
supervisor_defaults = {
|
||||
"log_dir": "{log_dir}",
|
||||
"control_dir": "{control_dir}",
|
||||
"instance_file": "{control_dir}/supervisor.instance.json",
|
||||
"lock_file": "{control_dir}/supervisor.lock",
|
||||
"supervisor_log": "{log_dir}/supervisor.log",
|
||||
"status_file": "{log_dir}/supervisor.status.txt",
|
||||
"dashboard_log": "{log_dir}/dashboard.log",
|
||||
"state_dir": "{state_dir}",
|
||||
}
|
||||
for key, default in supervisor_defaults.items():
|
||||
supervisor[key] = resolve_path(
|
||||
supervisor.get(key) or default,
|
||||
path_context,
|
||||
base_dir=path_context["project_dir"],
|
||||
required=True,
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def ensure_directories(paths, keys):
|
||||
for key in keys:
|
||||
path = paths.get(key)
|
||||
if path:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
Reference in New Issue
Block a user