Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,41 @@
|
||||
ARG BASE_IMAGE=truf-local:runtime
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/capacity_model.py /opt/truf/app/capacity_model.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/scanner_db.py /opt/truf/app/scanner_db.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/worker_assignment.py /opt/truf/app/worker_assignment.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/worker_api.py /opt/truf/app/worker_api.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/jsonl_projector.py /opt/truf/app/jsonl_projector.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/runtime_document.py /opt/truf/app/runtime_document.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/lifecycle_authority.py /opt/truf/app/lifecycle_authority.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/config.linux.yaml /opt/truf/app/config.linux.yaml
|
||||
|
||||
RUN python3 -I -S -B - <<'PY'
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
root = Path('/opt/truf/app')
|
||||
names = (
|
||||
'capacity_model.py',
|
||||
'scanner_db.py',
|
||||
'worker_assignment.py',
|
||||
'worker_api.py',
|
||||
'jsonl_projector.py',
|
||||
'runtime_document.py',
|
||||
'lifecycle_authority.py',
|
||||
)
|
||||
for name in names:
|
||||
ast.parse((root / name).read_text(encoding='utf-8'), filename=name)
|
||||
sys.path.insert(0, str(root))
|
||||
from capacity_model import ( # noqa: E402
|
||||
MAX_RESULT_BUNDLE_BYTES,
|
||||
REMOTE_ASSIGNMENT_BASELINE_BYTES,
|
||||
REMOTE_ASSIGNMENT_MAX_ACTIVE,
|
||||
)
|
||||
assert (MAX_RESULT_BUNDLE_BYTES, REMOTE_ASSIGNMENT_BASELINE_BYTES, REMOTE_ASSIGNMENT_MAX_ACTIVE) == (
|
||||
64 * 1024 * 1024,
|
||||
2 * 1024 * 1024,
|
||||
50,
|
||||
)
|
||||
PY
|
||||
@@ -0,0 +1,481 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
MODE="${1:-}"
|
||||
STAGE="${2:-}"
|
||||
if [[ "$MODE" != plan && "$MODE" != apply ]]; then
|
||||
echo 'usage: deploy.sh plan|apply STAGE' >&2
|
||||
exit 64
|
||||
fi
|
||||
if [[ ! "$STAGE" =~ ^/var/lib/truf-deploy/stage/capacity50\.[A-Za-z0-9]+$ ]] || [[ ! -d "$STAGE" ]]; then
|
||||
echo 'invalid deployment stage' >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
readonly RELEASE_ID='capacity50-20260930'
|
||||
readonly EXPECTED_IMAGE='sha256:46f1cf1b92d1a7d93d06f690309d8c7eca45f64dc45ca310861c70fb419bb035'
|
||||
readonly EXPECTED_CONFIG_SHA256='12bd9a60cc56c3d6cbad18435523e8229b0cd8fdccc2d73922ea7e580ce7441c'
|
||||
readonly CANDIDATE_TAG="truf-local:runtime-${RELEASE_ID}"
|
||||
readonly ROLLBACK_TAG="truf-local:runtime-pre-${RELEASE_ID}"
|
||||
readonly ACTIVE_CONFIG='/etc/truf/runtime/config.yaml'
|
||||
readonly ACTIVE_SECRETS='/etc/truf/runtime/secrets.yaml'
|
||||
readonly SOURCE_ROOT='/opt/truf'
|
||||
readonly HISTORY="/var/lib/truf-deploy/history/${RELEASE_ID}"
|
||||
readonly TEST_USER='operator-trace-windows-20260925'
|
||||
readonly TEST_USER_ORIGINAL_CAP='2'
|
||||
readonly APP_FILES=(
|
||||
capacity_model.py
|
||||
scanner_db.py
|
||||
worker_assignment.py
|
||||
worker_api.py
|
||||
jsonl_projector.py
|
||||
runtime_document.py
|
||||
lifecycle_authority.py
|
||||
config.linux.yaml
|
||||
)
|
||||
readonly COMPOSE=(
|
||||
docker compose
|
||||
--project-name truf-docker
|
||||
--project-directory /opt/truf
|
||||
--env-file /etc/truf-edge/edge.env
|
||||
--file /opt/truf/compose.yaml
|
||||
--file /opt/truf/compose.shared-host.yaml
|
||||
)
|
||||
|
||||
PHASE='preflight'
|
||||
MUTATED=0
|
||||
PHASE_A_HEALTHY=0
|
||||
SOURCE_INSTALLED=0
|
||||
USER_CAP_CHANGED=0
|
||||
DEPLOY_SUCCEEDED=0
|
||||
RESUME=0
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$RELEASE_ID" "$*"
|
||||
}
|
||||
|
||||
runtime_id() {
|
||||
"${COMPOSE[@]}" ps --quiet runtime
|
||||
}
|
||||
|
||||
psql() {
|
||||
local container
|
||||
container="$(runtime_id)"
|
||||
[[ -n "$container" ]] || return 1
|
||||
docker exec "$container" /usr/lib/postgresql/16/bin/psql \
|
||||
-h /run/truf-postgres -U truf -d truf -v ON_ERROR_STOP=1 -At "$@"
|
||||
}
|
||||
|
||||
current_image() {
|
||||
docker image inspect --format '{{.Id}}' truf-local:runtime
|
||||
}
|
||||
|
||||
config_sha256() {
|
||||
sha256sum "$ACTIVE_CONFIG" | cut -d' ' -f1
|
||||
}
|
||||
|
||||
require_baseline() {
|
||||
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
|
||||
echo 'runtime image identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
[[ "$(config_sha256)" == "$EXPECTED_CONFIG_SHA256" ]] || {
|
||||
echo 'active config identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
local state
|
||||
state="$(psql -F '|' -c \
|
||||
"SELECT revision, discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
if ((RESUME)); then
|
||||
[[ "$state" =~ ^[0-9]+\|0\|0\|(normal|drained)$ ]] || {
|
||||
echo "resumed runtime control is neither open nor drained: $state" >&2
|
||||
return 1
|
||||
}
|
||||
elif [[ ! "$state" =~ ^[0-9]+\|0\|0\|normal$ ]]; then
|
||||
echo "runtime control is not open: $state" >&2
|
||||
return 1
|
||||
fi
|
||||
local debt
|
||||
debt="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
[[ "$debt" == '0|0|0|0|0|0|0|0|0' ]] || {
|
||||
echo "pipeline is not reconciled: $debt" >&2
|
||||
return 1
|
||||
}
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}' AND disabled_at IS NULL;")" == "$TEST_USER_ORIGINAL_CAP" ]] || {
|
||||
echo 'temporary validation user identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
edge_value() {
|
||||
local name="$1"
|
||||
sed -n "s/^${name}=//p" /etc/truf-edge/edge.env
|
||||
}
|
||||
|
||||
admin_material() {
|
||||
local marker host page token revision
|
||||
marker="$(edge_value TRUF_ADMIN_EDGE_MARKER)"
|
||||
host="$(edge_value TRUF_EDGE_HOST)"
|
||||
[[ "$marker" =~ ^[a-f0-9]{64}$ ]] || return 1
|
||||
[[ "$host" =~ ^[A-Za-z0-9.-]+$ ]] || return 1
|
||||
page="$(curl --fail --silent --show-error --max-time 20 \
|
||||
--header "X-Truf-Admin-Edge: ${marker}" \
|
||||
--header 'X-Truf-Admin-Operator: deploy-runtime' \
|
||||
http://127.0.0.1:8766/admin-internal/)"
|
||||
token="$(python3 -c \
|
||||
'import re,sys; values=set(re.findall(r"name=\"csrf_token\" value=\"([^\"]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
|
||||
<<<"$page")"
|
||||
revision="$(python3 -c \
|
||||
'import re,sys; values=set(re.findall(r"name=\"expected_revision\" value=\"([0-9]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
|
||||
<<<"$page")"
|
||||
[[ "$token" =~ ^[A-Za-z0-9_-]{32,128}$ && "$revision" =~ ^[0-9]+$ ]] || return 1
|
||||
printf '%s|%s|%s|%s\n' "$marker" "$host" "$token" "$revision"
|
||||
}
|
||||
|
||||
admin_post() {
|
||||
local route="$1"
|
||||
shift
|
||||
local material marker host token revision operation
|
||||
material="$(admin_material)"
|
||||
IFS='|' read -r marker host token revision <<<"$material"
|
||||
operation="$(cat /proc/sys/kernel/random/uuid)"
|
||||
local arguments=(
|
||||
--fail --silent --show-error --max-time 30
|
||||
--request POST
|
||||
--header "X-Truf-Admin-Edge: ${marker}"
|
||||
--header 'X-Truf-Admin-Operator: deploy-runtime'
|
||||
--header "Origin: https://${host}"
|
||||
--header 'Content-Type: application/x-www-form-urlencoded'
|
||||
--data-urlencode "csrf_token=${token}"
|
||||
--data-urlencode "operation_id=${operation}"
|
||||
)
|
||||
if [[ "$route" == dispatch/* || "$route" == search/discovery/* ]]; then
|
||||
arguments+=(--data-urlencode "expected_revision=${revision}")
|
||||
fi
|
||||
while (($#)); do
|
||||
arguments+=(--data-urlencode "$1")
|
||||
shift
|
||||
done
|
||||
curl "${arguments[@]}" "http://127.0.0.1:8766/admin-internal/${route}" >/dev/null
|
||||
}
|
||||
|
||||
wait_for_drain() {
|
||||
local deadline=$((SECONDS + 600)) state debt
|
||||
while ((SECONDS < deadline)); do
|
||||
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
debt="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
if [[ "$state" == drained && "$debt" == '0|0|0|0|0|0|0|0|0' ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo 'runtime did not drain within 600 seconds' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
quiesce_pipeline_workers() {
|
||||
local source deadline active container
|
||||
for source in result-ingester jsonl-projector; do
|
||||
admin_post supervisor/sources/stop "source_id=${source}"
|
||||
done
|
||||
container="$(runtime_id)"
|
||||
[[ -n "$container" ]] || return 1
|
||||
docker exec --interactive "$container" /usr/local/bin/python3 -I -S -B - \
|
||||
<"$STAGE/release_stopped_pipeline_leases.py"
|
||||
deadline=$((SECONDS + 120))
|
||||
while ((SECONDS < deadline)); do
|
||||
active="$(psql -c \
|
||||
"SELECT count(*) FROM pipeline_leases WHERE state NOT IN ('released','failed');")"
|
||||
if [[ "$active" == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo 'pipeline worker leases did not release within 120 seconds' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
install_config() {
|
||||
local source="$1" temporary
|
||||
temporary="/etc/truf/runtime/.config.yaml.${RELEASE_ID}.tmp"
|
||||
install -o root -g root -m 0600 "$source" "$temporary"
|
||||
chown 10001:10001 "$temporary"
|
||||
mv -f "$temporary" "$ACTIVE_CONFIG"
|
||||
}
|
||||
|
||||
stop_stack() {
|
||||
"${COMPOSE[@]}" stop --timeout 30 edge
|
||||
"${COMPOSE[@]}" stop --timeout 600 runtime
|
||||
local container state
|
||||
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
|
||||
state="$(docker inspect --format '{{.State.Status}}|{{.State.ExitCode}}|{{.State.OOMKilled}}' "$container")"
|
||||
[[ "$state" == 'exited|0|false' ]] || {
|
||||
echo "runtime stop was not clean: $state" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
wait_runtime_health() {
|
||||
local deadline=$((SECONDS + 420)) container state status
|
||||
while ((SECONDS < deadline)); do
|
||||
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
|
||||
if [[ -n "$container" ]]; then
|
||||
state="$(docker inspect --format '{{.State.Status}}' "$container")"
|
||||
[[ "$state" != exited && "$state" != dead ]] || return 1
|
||||
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")"
|
||||
if [[ "$status" == healthy ]] && docker exec "$container" \
|
||||
/usr/local/bin/python3 -I -S -B /opt/truf/app/container_runtime.py \
|
||||
health --config /data/config/config.yaml --require-worker-api >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
[[ "$status" != unhealthy ]] || return 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo 'runtime health timed out' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
start_stack() {
|
||||
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate runtime
|
||||
wait_runtime_health
|
||||
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate edge
|
||||
sleep 3
|
||||
local edge_container edge_state host public_code
|
||||
edge_container="$("${COMPOSE[@]}" ps --quiet edge)"
|
||||
edge_state="$(docker inspect --format '{{.State.Status}}|{{.State.Running}}|{{.State.OOMKilled}}' "$edge_container")"
|
||||
[[ "$edge_state" == 'running|true|false' ]] || return 1
|
||||
host="$(edge_value TRUF_EDGE_HOST)"
|
||||
public_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--max-time 20 "https://${host}/")"
|
||||
[[ "$public_code" == 401 || "$public_code" == 404 ]] || {
|
||||
echo "unexpected public edge response: $public_code" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
validate_candidate_config() {
|
||||
local path="$1"
|
||||
docker run --rm --network none --read-only --user 10001:10001 \
|
||||
--cap-drop ALL --security-opt no-new-privileges:true \
|
||||
--tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \
|
||||
--volume "$path:/data/config/config.yaml:ro" \
|
||||
--volume "$ACTIVE_SECRETS:/data/config/secrets.yaml:ro" \
|
||||
--volume /etc/truf/worker-packages:/data/worker-packages:ro \
|
||||
--entrypoint /usr/local/bin/python3 "$CANDIDATE_TAG" -I -S -B -c \
|
||||
"import sys,sysconfig;sys.path.append(sysconfig.get_paths()['purelib']);sys.path.insert(0,'/opt/truf/app');from runtime_document_io import validate_managed_runtime_files;print(validate_managed_runtime_files('/data/config/config.yaml').config_sha256)" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
install_sources() {
|
||||
local name destination temporary
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
destination="${SOURCE_ROOT}/app/${name}"
|
||||
temporary="${destination}.${RELEASE_ID}.tmp"
|
||||
install -o root -g root -m 0644 "$STAGE/payload/app/$name" "$temporary"
|
||||
mv -f "$temporary" "$destination"
|
||||
done
|
||||
SOURCE_INSTALLED=1
|
||||
}
|
||||
|
||||
restore_sources() {
|
||||
local name
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
if [[ -f "$HISTORY/source/$name" ]]; then
|
||||
install -o root -g root -m 0644 "$HISTORY/source/$name" "${SOURCE_ROOT}/app/$name"
|
||||
else
|
||||
rm -f "${SOURCE_ROOT}/app/$name"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_user_cap() {
|
||||
if ((USER_CAP_CHANGED)); then
|
||||
admin_post users/cap "user_key=${TEST_USER}" "active_assignment_cap=${TEST_USER_ORIGINAL_CAP}" || true
|
||||
USER_CAP_CHANGED=0
|
||||
fi
|
||||
}
|
||||
|
||||
cancel_drain() {
|
||||
local state
|
||||
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;" 2>/dev/null || true)"
|
||||
if [[ "$state" == draining || "$state" == drained ]]; then
|
||||
admin_post dispatch/drain/cancel || return 1
|
||||
fi
|
||||
}
|
||||
|
||||
rollback() {
|
||||
set +e
|
||||
log "rollback from phase ${PHASE}"
|
||||
restore_user_cap
|
||||
stop_stack
|
||||
if ((PHASE_A_HEALTHY)); then
|
||||
docker image tag "$CANDIDATE_TAG" truf-local:runtime
|
||||
install_config "$HISTORY/config.conservative.yaml"
|
||||
else
|
||||
docker image tag "$EXPECTED_IMAGE" truf-local:runtime
|
||||
install_config "$HISTORY/config.original.yaml"
|
||||
fi
|
||||
((SOURCE_INSTALLED)) && restore_sources
|
||||
if start_stack; then
|
||||
cancel_drain
|
||||
log 'rollback restored a healthy runtime'
|
||||
else
|
||||
log 'rollback could not prove health; runtime remains contained' >&2
|
||||
fi
|
||||
set -e
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local code=$?
|
||||
trap - EXIT ERR INT TERM
|
||||
if ((code != 0 && MUTATED && !DEPLOY_SUCCEEDED)); then
|
||||
rollback
|
||||
fi
|
||||
exit "$code"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
exec 9>/run/lock/truf-runtime-deploy.lock
|
||||
flock -n 9 || {
|
||||
echo 'another runtime deployment is active' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ "$MODE" == apply && -d "$HISTORY" ]] \
|
||||
&& docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1 \
|
||||
&& [[ "$(docker image inspect --format '{{.Id}}' "$ROLLBACK_TAG" 2>/dev/null || true)" == "$EXPECTED_IMAGE" ]]; then
|
||||
RESUME=1
|
||||
fi
|
||||
require_baseline
|
||||
available_kb="$(df -Pk /var/lib/docker | awk 'NR==2 {print $4}')"
|
||||
[[ "$available_kb" =~ ^[0-9]+$ && "$available_kb" -ge 786432 ]] || {
|
||||
echo 'less than 768 MiB is available for the derived image' >&2
|
||||
exit 1
|
||||
}
|
||||
log "plan image=${EXPECTED_IMAGE#sha256:} config=${EXPECTED_CONFIG_SHA256} free_kib=${available_kb}"
|
||||
if [[ "$MODE" == plan ]]; then
|
||||
log 'plan passed; no runtime state changed'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ((RESUME)); then
|
||||
log 'resuming a verified pre-cutover release'
|
||||
[[ "$(sha256sum "$HISTORY/config.original.yaml" | cut -d' ' -f1)" == "$EXPECTED_CONFIG_SHA256" ]] || exit 1
|
||||
[[ -f "$HISTORY/config.conservative.yaml" && -f "$HISTORY/config.final.yaml" ]] || exit 1
|
||||
validate_candidate_config "$HISTORY/config.conservative.yaml"
|
||||
validate_candidate_config "$HISTORY/config.final.yaml"
|
||||
if [[ "$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")" == normal ]]; then
|
||||
admin_post dispatch/drain/start
|
||||
MUTATED=1
|
||||
wait_for_drain
|
||||
else
|
||||
MUTATED=1
|
||||
fi
|
||||
else
|
||||
install -d -o root -g root -m 0700 /var/lib/truf-deploy /var/lib/truf-deploy/history
|
||||
if [[ -e "$HISTORY" ]]; then
|
||||
echo 'release history already exists' >&2
|
||||
exit 1
|
||||
fi
|
||||
install -d -o root -g root -m 0700 "$HISTORY" "$HISTORY/source"
|
||||
install -o root -g root -m 0600 "$ACTIVE_CONFIG" "$HISTORY/config.original.yaml"
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
if [[ -f "${SOURCE_ROOT}/app/$name" ]]; then
|
||||
install -o root -g root -m 0600 "${SOURCE_ROOT}/app/$name" "$HISTORY/source/$name"
|
||||
fi
|
||||
done
|
||||
|
||||
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
|
||||
--output "$HISTORY/config.conservative.yaml" --mode conservative \
|
||||
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
|
||||
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
|
||||
--output "$HISTORY/config.final.yaml" --mode final \
|
||||
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
|
||||
chown 10001:10001 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
|
||||
chmod 0600 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
|
||||
|
||||
if docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1; then
|
||||
echo 'candidate image tag already exists' >&2
|
||||
exit 1
|
||||
fi
|
||||
PHASE='candidate-build'
|
||||
docker build --network none --build-arg BASE_IMAGE=truf-local:runtime \
|
||||
--file "$STAGE/Dockerfile" --tag "$CANDIDATE_TAG" "$STAGE"
|
||||
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
|
||||
echo 'runtime tag changed during candidate build' >&2
|
||||
exit 1
|
||||
}
|
||||
validate_candidate_config "$HISTORY/config.conservative.yaml"
|
||||
validate_candidate_config "$HISTORY/config.final.yaml"
|
||||
docker image tag "$EXPECTED_IMAGE" "$ROLLBACK_TAG"
|
||||
|
||||
PHASE='drain'
|
||||
admin_post dispatch/drain/start
|
||||
MUTATED=1
|
||||
wait_for_drain
|
||||
fi
|
||||
|
||||
PHASE='conservative-cutover'
|
||||
quiesce_pipeline_workers
|
||||
stop_stack
|
||||
install_config "$HISTORY/config.conservative.yaml"
|
||||
docker image tag "$CANDIDATE_TAG" truf-local:runtime
|
||||
start_stack
|
||||
schema_state="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM information_schema.columns WHERE table_name='result_reservations' AND column_name='reserved_bundle_bytes'), (SELECT count(*) FROM runtime_schema_migrations WHERE version='20260930_33_remote_assignment_capacity');")"
|
||||
[[ "$schema_state" == '1|1' ]] || {
|
||||
echo "capacity migration was not applied: $schema_state" >&2
|
||||
exit 1
|
||||
}
|
||||
PHASE_A_HEALTHY=1
|
||||
|
||||
PHASE='capacity50-cutover'
|
||||
quiesce_pipeline_workers
|
||||
stop_stack
|
||||
install_config "$HISTORY/config.final.yaml"
|
||||
start_stack
|
||||
|
||||
final_values="$(psql -F '|' -c \
|
||||
"SELECT bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
[[ "$final_values" == '0|0|0|0|0|0|0|0' ]] || {
|
||||
echo "post-deploy capacity is not reconciled: $final_values" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
PHASE='temporary-user-cap-validation'
|
||||
admin_post users/cap "user_key=${TEST_USER}" 'active_assignment_cap=50'
|
||||
USER_CAP_CHANGED=1
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == 50 ]] || exit 1
|
||||
restore_user_cap
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == "$TEST_USER_ORIGINAL_CAP" ]] || exit 1
|
||||
|
||||
PHASE='source-install'
|
||||
install_sources
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
cmp -s "$STAGE/payload/app/$name" "${SOURCE_ROOT}/app/$name" || exit 1
|
||||
done
|
||||
|
||||
PHASE='resume'
|
||||
cancel_drain
|
||||
post_control="$(psql -F '|' -c \
|
||||
"SELECT discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
[[ "$post_control" == '0|0|normal' ]] || {
|
||||
echo "runtime control did not resume: $post_control" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
cat >"$HISTORY/result.txt" <<EOF
|
||||
release=${RELEASE_ID}
|
||||
runtime_image=$(current_image)
|
||||
config_sha256=$(config_sha256)
|
||||
schema=${schema_state}
|
||||
capacity=${final_values}
|
||||
control=${post_control}
|
||||
EOF
|
||||
chmod 0600 "$HISTORY/result.txt"
|
||||
DEPLOY_SUCCEEDED=1
|
||||
log "applied image=$(current_image) config=$(config_sha256)"
|
||||
@@ -0,0 +1,74 @@
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
|
||||
|
||||
for path in glob.glob('/proc/[0-9]*/cmdline'):
|
||||
try:
|
||||
command = open(path, 'rb').read().replace(b'\0', b' ')
|
||||
except (FileNotFoundError, PermissionError, ProcessLookupError):
|
||||
continue
|
||||
if b'/opt/truf/app/result_ingester.py' in command or b'/opt/truf/app/jsonl_projector.py' in command:
|
||||
raise SystemExit('a pipeline worker process is still active')
|
||||
|
||||
sys.path.append(sysconfig.get_paths()['purelib'])
|
||||
sys.path.insert(0, '/opt/truf/app')
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from db_backend import DatabaseConnection
|
||||
from scanner_db import (
|
||||
PIPELINE_ADVISORY_CLASS,
|
||||
PIPELINE_ADVISORY_OBJECTS,
|
||||
ScannerDB,
|
||||
)
|
||||
|
||||
|
||||
connection = psycopg.connect(
|
||||
dbname='truf',
|
||||
user='truf',
|
||||
host='/run/truf-postgres',
|
||||
row_factory=dict_row,
|
||||
options='-c search_path=public -c statement_timeout=30000 -c lock_timeout=5000',
|
||||
)
|
||||
database = ScannerDB(enabled=False)
|
||||
database.conn = DatabaseConnection('postgres', connection, application_schema='public')
|
||||
released = 0
|
||||
try:
|
||||
rows = database.conn.execute(
|
||||
"SELECT worker_name, generation, lease_token FROM pipeline_leases "
|
||||
"WHERE state NOT IN ('released','failed') ORDER BY worker_name"
|
||||
).fetchall()
|
||||
expected = {'result_ingester', 'jsonl_projector'}
|
||||
if not {row['worker_name'] for row in rows} <= expected:
|
||||
raise RuntimeError('an unexpected pipeline lease is active')
|
||||
for row in rows:
|
||||
name = row['worker_name']
|
||||
advisory = PIPELINE_ADVISORY_OBJECTS[name]
|
||||
locked = database.conn.execute(
|
||||
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
|
||||
(PIPELINE_ADVISORY_CLASS, advisory),
|
||||
).fetchone()
|
||||
if not locked or not locked['acquired']:
|
||||
raise RuntimeError('a stopped pipeline worker still owns its advisory lock')
|
||||
database._pipeline_advisory_held.add(name)
|
||||
if not database.release_pipeline_lease(
|
||||
name,
|
||||
row['generation'],
|
||||
row['lease_token'],
|
||||
state='released',
|
||||
error='deployment_quiesce',
|
||||
):
|
||||
raise RuntimeError('pipeline lease identity changed during release')
|
||||
released += 1
|
||||
remaining = database.conn.execute(
|
||||
"SELECT count(*) AS count FROM pipeline_leases "
|
||||
"WHERE state NOT IN ('released','failed')"
|
||||
).fetchone()['count']
|
||||
if remaining:
|
||||
raise RuntimeError('pipeline lease reconciliation is incomplete')
|
||||
print(f'released_pipeline_leases={released}')
|
||||
finally:
|
||||
database.close()
|
||||
@@ -0,0 +1,73 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
VALUES = {
|
||||
'conservative': {
|
||||
'remote_assignment_reserve_bytes': 64 * 1024 * 1024,
|
||||
'remote_assignment_max_active': 1,
|
||||
'keycheck_queue_max_items': 8192,
|
||||
'keycheck_queue_max_bytes': 64 * 1024 * 1024,
|
||||
},
|
||||
'final': {
|
||||
'remote_assignment_reserve_bytes': 2 * 1024 * 1024,
|
||||
'remote_assignment_max_active': 50,
|
||||
'keycheck_queue_max_items': 131072,
|
||||
'keycheck_queue_max_bytes': 128 * 1024 * 1024,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(payload, mode):
|
||||
text = payload.decode('utf-8', errors='strict')
|
||||
if '\r' in text:
|
||||
raise ValueError('configuration must use LF line endings')
|
||||
lines = text.splitlines(keepends=True)
|
||||
values = VALUES[mode]
|
||||
hard_limit = ' result_bundle_max_event_bytes: 67108864\n'
|
||||
if lines.count(hard_limit) != 1:
|
||||
raise ValueError('unexpected hard result limit')
|
||||
for field in ('remote_assignment_reserve_bytes', 'remote_assignment_max_active'):
|
||||
if any(line.startswith(f' {field}:') for line in lines):
|
||||
raise ValueError(f'active configuration already contains {field}')
|
||||
index = lines.index(hard_limit) + 1
|
||||
lines[index:index] = [
|
||||
f" remote_assignment_reserve_bytes: {values['remote_assignment_reserve_bytes']}\n",
|
||||
f" remote_assignment_max_active: {values['remote_assignment_max_active']}\n",
|
||||
]
|
||||
replacements = {
|
||||
' keycheck_queue_max_items: 8192\n': (
|
||||
f" keycheck_queue_max_items: {values['keycheck_queue_max_items']}\n"
|
||||
),
|
||||
' keycheck_queue_max_bytes: 67108864\n': (
|
||||
f" keycheck_queue_max_bytes: {values['keycheck_queue_max_bytes']}\n"
|
||||
),
|
||||
}
|
||||
for before, after in replacements.items():
|
||||
if lines.count(before) != 1:
|
||||
raise ValueError(f'unexpected active configuration field: {before.strip()}')
|
||||
lines[lines.index(before)] = after
|
||||
return ''.join(lines).encode('utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--input', required=True)
|
||||
parser.add_argument('--output', required=True)
|
||||
parser.add_argument('--mode', choices=tuple(VALUES), required=True)
|
||||
parser.add_argument('--expected-sha256', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
source = Path(args.input)
|
||||
destination = Path(args.output)
|
||||
payload = source.read_bytes()
|
||||
if hashlib.sha256(payload).hexdigest() != args.expected_sha256:
|
||||
raise SystemExit('active configuration identity changed')
|
||||
rendered = render(payload, args.mode)
|
||||
destination.write_bytes(rendered)
|
||||
print(hashlib.sha256(rendered).hexdigest())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
{
|
||||
admin unix//run/caddy-admin.sock
|
||||
auto_https disable_redirects
|
||||
skip_install_trust
|
||||
}
|
||||
|
||||
(admin_security) {
|
||||
header {
|
||||
Cache-Control "no-store"
|
||||
Pragma "no-cache"
|
||||
Referrer-Policy "same-origin"
|
||||
Content-Security-Policy "default-src 'none'; style-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'"
|
||||
X-Content-Type-Options "nosniff"
|
||||
}
|
||||
}
|
||||
|
||||
(admin_gate) {
|
||||
import {$TRUF_ADMIN_DENYLIST_FILE:/etc/caddy/denylist/admin-denylist.caddy}
|
||||
basic_auth bcrypt "truf-admin" {
|
||||
{$TRUF_ADMIN_USER} {$TRUF_ADMIN_PASSWORD_HASH}
|
||||
}
|
||||
}
|
||||
|
||||
{$TRUF_EDGE_HOST} {
|
||||
import {$TRUF_EDGE_TLS_INCLUDE:/etc/caddy/tls/automatic.caddy}
|
||||
import admin_security
|
||||
header Strict-Transport-Security "max-age=63072000; includeSubDomains"
|
||||
|
||||
log routine_access {
|
||||
output discard
|
||||
}
|
||||
|
||||
log admin_auth_failures {
|
||||
no_hostname
|
||||
output file {$TRUF_ADMIN_AUTH_LOG_FILE:/var/log/caddy/admin-auth-failures.json} {
|
||||
roll_size 8MiB
|
||||
roll_keep 10
|
||||
roll_keep_for 240h
|
||||
}
|
||||
format filter {
|
||||
request delete
|
||||
bytes_read delete
|
||||
user_id delete
|
||||
duration delete
|
||||
size delete
|
||||
resp_headers delete
|
||||
wrap json
|
||||
}
|
||||
}
|
||||
|
||||
@worker path /api/v1/worker/*
|
||||
handle @worker {
|
||||
reverse_proxy 127.0.0.1:8766 {
|
||||
header_up -X-Truf-Admin-Edge
|
||||
header_up -X-Truf-Admin-Operator
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
}
|
||||
}
|
||||
|
||||
@admin_root path /{$TRUF_ADMIN_PREFIX}
|
||||
handle @admin_root {
|
||||
route {
|
||||
import admin_security
|
||||
import admin_gate
|
||||
redir * /{$TRUF_ADMIN_PREFIX}/ 308
|
||||
}
|
||||
}
|
||||
|
||||
@admin path /{$TRUF_ADMIN_PREFIX}/*
|
||||
handle @admin {
|
||||
route {
|
||||
import admin_security
|
||||
request_header -X-Truf-Admin-Edge
|
||||
request_header -X-Truf-Admin-Operator
|
||||
import admin_gate
|
||||
uri strip_prefix /{$TRUF_ADMIN_PREFIX}
|
||||
uri path_regexp ^ /admin-internal
|
||||
reverse_proxy 127.0.0.1:8766 {
|
||||
header_up -Authorization
|
||||
header_up X-Truf-Admin-Edge {$TRUF_ADMIN_EDGE_MARKER}
|
||||
header_up X-Truf-Admin-Operator {http.auth.user.id}
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
header_down -Strict-Transport-Security
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
handle {
|
||||
respond "" 404
|
||||
}
|
||||
|
||||
handle_errors {
|
||||
@bad_admin_credentials {
|
||||
path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
header Authorization *
|
||||
expression {err.status_code} == 401
|
||||
}
|
||||
handle @bad_admin_credentials {
|
||||
route {
|
||||
import admin_security
|
||||
log_name admin_auth_failures
|
||||
log_append event admin_auth_failure
|
||||
log_append remote_ip {http.request.remote.host}
|
||||
header WWW-Authenticate "Basic realm=\"truf-admin\""
|
||||
respond "" 401
|
||||
}
|
||||
}
|
||||
|
||||
@admin_unauthorized {
|
||||
path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
expression {err.status_code} == 401
|
||||
}
|
||||
handle @admin_unauthorized {
|
||||
route {
|
||||
import admin_security
|
||||
header WWW-Authenticate "Basic realm=\"truf-admin\""
|
||||
respond "" 401
|
||||
}
|
||||
}
|
||||
|
||||
@admin_error path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
handle @admin_error {
|
||||
import admin_security
|
||||
respond "" {err.status_code}
|
||||
}
|
||||
|
||||
handle {
|
||||
respond "" {err.status_code}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
{
|
||||
admin unix//run/caddy-admin.sock
|
||||
auto_https off
|
||||
skip_install_trust
|
||||
servers {
|
||||
trusted_proxies static 127.0.0.1/32 ::1/128
|
||||
trusted_proxies_strict
|
||||
client_ip_headers X-Forwarded-For
|
||||
}
|
||||
}
|
||||
|
||||
(admin_security) {
|
||||
header {
|
||||
Cache-Control "no-store"
|
||||
Pragma "no-cache"
|
||||
Referrer-Policy "same-origin"
|
||||
Content-Security-Policy "default-src 'none'; style-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'"
|
||||
X-Content-Type-Options "nosniff"
|
||||
}
|
||||
}
|
||||
|
||||
(admin_gate) {
|
||||
import {$TRUF_ADMIN_DENYLIST_FILE:/etc/caddy/denylist/admin-denylist.caddy}
|
||||
basic_auth bcrypt "truf-admin" {
|
||||
{$TRUF_ADMIN_USER} {$TRUF_ADMIN_PASSWORD_HASH}
|
||||
}
|
||||
}
|
||||
|
||||
http://:18766 {
|
||||
bind 127.0.0.1
|
||||
|
||||
log routine_access {
|
||||
output discard
|
||||
}
|
||||
|
||||
log admin_auth_failures {
|
||||
no_hostname
|
||||
output file {$TRUF_ADMIN_AUTH_LOG_FILE:/var/log/caddy/admin-auth-failures.json} {
|
||||
roll_size 8MiB
|
||||
roll_keep 10
|
||||
roll_keep_for 240h
|
||||
}
|
||||
format filter {
|
||||
request delete
|
||||
bytes_read delete
|
||||
user_id delete
|
||||
duration delete
|
||||
size delete
|
||||
resp_headers delete
|
||||
wrap json
|
||||
}
|
||||
}
|
||||
|
||||
route {
|
||||
@invalid_ingress not header X-Truf-Shared-Ingress {$TRUF_SHARED_INGRESS_MARKER}
|
||||
respond @invalid_ingress "" 403
|
||||
request_header -X-Truf-Shared-Ingress
|
||||
|
||||
import admin_security
|
||||
header Strict-Transport-Security "max-age=63072000; includeSubDomains"
|
||||
|
||||
@worker path /api/v1/worker/*
|
||||
handle @worker {
|
||||
reverse_proxy 127.0.0.1:8766 {
|
||||
header_up -X-Truf-Admin-Edge
|
||||
header_up -X-Truf-Admin-Operator
|
||||
header_up -X-Truf-Shared-Ingress
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
}
|
||||
}
|
||||
|
||||
@admin_root path /{$TRUF_ADMIN_PREFIX}
|
||||
handle @admin_root {
|
||||
route {
|
||||
import admin_security
|
||||
import admin_gate
|
||||
redir * /{$TRUF_ADMIN_PREFIX}/ 308
|
||||
}
|
||||
}
|
||||
|
||||
@admin path /{$TRUF_ADMIN_PREFIX}/*
|
||||
handle @admin {
|
||||
route {
|
||||
import admin_security
|
||||
request_header -X-Truf-Admin-Edge
|
||||
request_header -X-Truf-Admin-Operator
|
||||
import admin_gate
|
||||
uri strip_prefix /{$TRUF_ADMIN_PREFIX}
|
||||
uri path_regexp ^ /admin-internal
|
||||
reverse_proxy 127.0.0.1:8766 {
|
||||
header_up -Authorization
|
||||
header_up X-Truf-Admin-Edge {$TRUF_ADMIN_EDGE_MARKER}
|
||||
header_up X-Truf-Admin-Operator {http.auth.user.id}
|
||||
header_up -X-Truf-Shared-Ingress
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
header_down -Strict-Transport-Security
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
handle {
|
||||
respond "" 404
|
||||
}
|
||||
}
|
||||
|
||||
handle_errors {
|
||||
@bad_admin_credentials {
|
||||
path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
header Authorization *
|
||||
expression {err.status_code} == 401
|
||||
}
|
||||
handle @bad_admin_credentials {
|
||||
route {
|
||||
import admin_security
|
||||
log_name admin_auth_failures
|
||||
log_append event admin_auth_failure
|
||||
log_append remote_ip {http.request.client_ip}
|
||||
header WWW-Authenticate "Basic realm=\"truf-admin\""
|
||||
respond "" 401
|
||||
}
|
||||
}
|
||||
|
||||
@admin_unauthorized {
|
||||
path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
expression {err.status_code} == 401
|
||||
}
|
||||
handle @admin_unauthorized {
|
||||
route {
|
||||
import admin_security
|
||||
header WWW-Authenticate "Basic realm=\"truf-admin\""
|
||||
respond "" 401
|
||||
}
|
||||
}
|
||||
|
||||
@admin_error path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
handle @admin_error {
|
||||
import admin_security
|
||||
respond "" {err.status_code}
|
||||
}
|
||||
|
||||
handle {
|
||||
respond "" {err.status_code}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM caddy:2.10.2-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d AS edge
|
||||
|
||||
USER 0:0
|
||||
RUN setcap -r /usr/bin/caddy \
|
||||
&& install -d -o 10001 -g 10001 -m 0700 /data /config /var/log/caddy /etc/caddy/denylist /etc/caddy/tls
|
||||
COPY --chown=0:0 --chmod=0444 deploy/edge/Caddyfile /etc/caddy/Caddyfile
|
||||
COPY --chown=0:0 --chmod=0444 deploy/edge/Caddyfile.shared-host /etc/caddy/Caddyfile.shared-host
|
||||
COPY --chown=0:0 --chmod=0444 deploy/edge/admin-denylist.caddy /etc/caddy/denylist/admin-denylist.caddy
|
||||
COPY --chown=0:0 --chmod=0444 deploy/edge/automatic-tls.caddy /etc/caddy/tls/automatic.caddy
|
||||
COPY --chown=0:0 --chmod=0555 deploy/edge/entrypoint.sh /usr/local/bin/truf-edge-entrypoint
|
||||
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/truf-edge-entrypoint"]
|
||||
CMD []
|
||||
|
||||
FROM edge AS edge-e2e
|
||||
|
||||
USER 0:0
|
||||
RUN chmod 0644 /etc/caddy/Caddyfile \
|
||||
&& sed -i 's#^[[:space:]]*admin unix//run/caddy-admin.sock$#\tadmin 127.0.0.1:2019#' /etc/caddy/Caddyfile \
|
||||
&& chmod 0444 /etc/caddy/Caddyfile \
|
||||
&& grep -Fx ' admin 127.0.0.1:2019' /etc/caddy/Caddyfile >/dev/null
|
||||
USER 10001:10001
|
||||
@@ -0,0 +1,253 @@
|
||||
# Production edge deployment
|
||||
|
||||
This opt-in deployment keeps PostgreSQL, supervisor control, the standalone dashboard,
|
||||
the Worker API, and its typed admin backend on the runtime container's loopback. The
|
||||
root-owned deployment profile selects one of two exact Caddy topologies. Both keep the
|
||||
private Caddy admin API on an unpublished Unix socket and preserve the same Worker API,
|
||||
admin authentication, operator attribution, denylist, and header contract.
|
||||
|
||||
## Deployment profiles
|
||||
|
||||
If `/etc/truf/deployment-profile` is absent, `standalone-edge-v1` is selected. The
|
||||
standalone profile publishes runtime TCP 443 and gives the managed edge only
|
||||
`NET_BIND_SERVICE`.
|
||||
|
||||
For a host whose existing root-owned Caddy must remain the sole owner of ports 80/443,
|
||||
install the shared profile before running the host-agent installer:
|
||||
|
||||
```sh
|
||||
printf '%s\n' shared-host-edge-v1 | sudo install -m 0444 -o root -g root /dev/stdin /etc/truf/deployment-profile
|
||||
```
|
||||
|
||||
`shared-host-edge-v1` runs the runtime in the host network namespace with no Docker
|
||||
published ports. The managed Truf edge shares that namespace, has no capabilities, and
|
||||
binds plain HTTP only at `127.0.0.1:18766`. The existing host Caddy imports the fixed
|
||||
route-only `deploy/edge/host-caddy-shared.caddy` snippet inside the reviewed public site.
|
||||
Install that import before any catch-all handler. It handles only `/api/v1/worker/*` and
|
||||
the exact random admin prefix; it does not define a listener, TLS policy, global option,
|
||||
or route for another application. The host agent never restarts or reconfigures host
|
||||
Caddy or X-UI.
|
||||
|
||||
Profile changes are maintenance operations: stop the host agent first, require no active
|
||||
apply or failed hold, install the exact root-owned mode-0444 value, validate the selected
|
||||
Compose projection and host-Caddy configuration, then restart the agent. Never expose
|
||||
profile selection through the admin or host-agent request.
|
||||
|
||||
### Choosing a topology
|
||||
|
||||
Use `standalone-edge-v1` on a dedicated host where the managed edge can own public TCP
|
||||
443. The request path is:
|
||||
|
||||
```text
|
||||
Internet -> managed Caddy :443 -> private runtime :8766
|
||||
```
|
||||
|
||||
Use `shared-host-edge-v1` only when an existing root-owned Caddy must remain the sole
|
||||
owner of public ports and TLS. The request path is:
|
||||
|
||||
```text
|
||||
Internet -> host Caddy :443 -> 127.0.0.1:18766 -> managed Caddy -> private runtime :8766
|
||||
```
|
||||
|
||||
Shared-host mode adds a one-time integration boundary, not a second public edge. The
|
||||
operator installs the fixed route snippet, places its import before every catch-all,
|
||||
supplies the independent ingress marker, and validates the complete host Caddy
|
||||
configuration. After that bootstrap, runtime restart and document apply use the same
|
||||
host-agent lifecycle as standalone mode. The host agent never owns the host Caddy
|
||||
configuration or service lifecycle.
|
||||
|
||||
These are the only supported production topologies. Nginx, Traefik, an arbitrary Caddy
|
||||
layout, or an ad-hoc Compose override is not equivalent to either profile. Add and test a
|
||||
new exact deployment profile instead of translating private headers approximately. The
|
||||
host agent validates the selected fixed Compose projection and rejects metadata drift.
|
||||
|
||||
The shared-host projection currently carries the constrained-host runtime limits declared
|
||||
in `compose.shared-host.yaml`. A materially different CPU or memory envelope also requires
|
||||
a reviewed profile change; do not hide it in an unvalidated local override.
|
||||
|
||||
### End-to-end host bootstrap
|
||||
|
||||
The repository provides fixed deployment components, not a universal VPS installer,
|
||||
Ansible role, public image registry, or infrastructure module. Bootstrap a new host from
|
||||
one reviewed release checkout as follows:
|
||||
|
||||
1. Install the reviewed Linux, Docker Engine and Compose plugin, systemd, Python 3, and
|
||||
fail2ban prerequisites; provision DNS and the selected TLS ownership boundary.
|
||||
2. Install the release checkout root-owned at `/opt/truf` and choose exactly one deployment
|
||||
profile before installing the host agent.
|
||||
3. In shared-host mode, install the fixed host-Caddy import, validate the complete host
|
||||
configuration, and prove an unavailable loopback edge cannot fall through to another
|
||||
application.
|
||||
4. Create the protected edge directories, denylist state, and mode-0600 edge environment
|
||||
described below. Generate independent admin, edge, and shared-ingress values rather
|
||||
than copying values from another host.
|
||||
5. Install and validate the fixed host agent. Its first install seeds an absent active
|
||||
config from `app/config.linux.yaml` and an absent secrets document as an empty mapping;
|
||||
repeat installation never replaces active documents.
|
||||
6. Install every trusted worker-package manifest referenced by the runtime config beneath
|
||||
`/etc/truf/worker-packages` with the exact ownership and mode described below.
|
||||
7. Review the private active config and secrets, then build `runtime` and `edge` from the
|
||||
same checkout with the exact base and selected profile Compose files.
|
||||
8. Start the stack, explicitly enable Worker API and admin only after their private
|
||||
configuration is complete, and verify PostgreSQL, runtime, edge, HTTPS, admin, Worker
|
||||
API, host-agent, and unrelated host applications.
|
||||
|
||||
Initial host bootstrap is therefore intentionally more manual than later operation.
|
||||
Normal config apply, restart, rollback, status, and audit are performed through the typed
|
||||
control plane and fixed host agent after this trust boundary is established.
|
||||
|
||||
## Host agent and fixed runtime paths
|
||||
|
||||
Install the root-owned checkout at `/opt/truf`. Before invoking the host-agent installer,
|
||||
prepare the edge state below and create the complete protected environment file that its
|
||||
fixed combined-Compose validation consumes.
|
||||
|
||||
UID/GID 10001 is the numeric edge identity. The denylist directory is mounted, rather than
|
||||
its file, so atomic replacement remains visible in the container.
|
||||
|
||||
```sh
|
||||
sudo install -d -o 10001 -g 10001 -m 0700 /var/log/truf-edge
|
||||
sudo install -d -o root -g 10001 -m 2750 /etc/truf-edge/denylist
|
||||
sudo install -m 0640 -o root -g 10001 deploy/edge/admin-denylist.caddy /etc/truf-edge/denylist/admin-denylist.caddy
|
||||
sudo install -d -o root -g root -m 0700 /var/lib/truf-edge
|
||||
sudo install -m 0750 -o root -g root deploy/fail2ban/truf_caddy_admin_denylist.py /usr/local/sbin/truf-caddy-admin-denylist
|
||||
```
|
||||
|
||||
Create `/etc/truf-edge/edge.env` as root with mode 0600. Generate a new admin segment
|
||||
with `openssl rand -hex 32`. It must be exactly 64 lowercase hex characters (256 random
|
||||
bits). Generate the bcrypt value interactively with the pinned edge image's
|
||||
`caddy hash-password` command; never put the plaintext password in a command, file, or
|
||||
Compose variable.
|
||||
|
||||
```dotenv
|
||||
TRUF_EDGE_HOST=edge.example.net
|
||||
TRUF_ADMIN_PREFIX=replace_with_64_lowercase_hex_characters
|
||||
TRUF_ADMIN_USER=operator
|
||||
TRUF_ADMIN_PASSWORD_HASH='$2a$14$replace_with_a_real_caddy_bcrypt_hash'
|
||||
TRUF_ADMIN_EDGE_MARKER=replace_with_a_second_independent_64_character_hex_secret
|
||||
TRUF_SHARED_INGRESS_MARKER=replace_with_a_third_independent_64_character_hex_secret
|
||||
TRUF_EDGE_AUTH_LOG_DIR=/var/log/truf-edge
|
||||
TRUF_EDGE_DENYLIST_DIR=/etc/truf-edge/denylist
|
||||
```
|
||||
|
||||
`TRUF_SHARED_INGRESS_MARKER` is required only by `shared-host-edge-v1`. Host Caddy strips
|
||||
any inbound transit/private headers, injects this marker and its observed client address,
|
||||
and proxies to loopback. The managed edge rejects a missing marker before trusting that
|
||||
address and removes the marker before proxying to the application.
|
||||
|
||||
Now install and validate the fixed host agent. The installer creates the fixed candidate,
|
||||
result, PostgreSQL socket, and active-document paths and enables
|
||||
`/run/truf/host-agent.sock`; Compose refuses to create missing bind sources.
|
||||
|
||||
```sh
|
||||
sudo /usr/bin/python3 -I -S -B /opt/truf/deploy/host-agent/truf_host_agent_install.py install
|
||||
sudo /usr/bin/python3 -I -S -B /opt/truf/deploy/host-agent/truf_host_agent_install.py validate
|
||||
```
|
||||
|
||||
The active `/etc/truf/runtime/config.yaml` and `secrets.yaml` are UID/GID 10001 mode 0600
|
||||
documents and are never overwritten by repeat installation. Any package manifest referenced
|
||||
by the config must be installed beneath `/etc/truf/worker-packages` as a root-owned,
|
||||
root:root mode 0644 regular file before validation. The runtime maps that immutable authority
|
||||
read-only at `/data/worker-packages`; do not place manifests in the private active-document
|
||||
directory.
|
||||
|
||||
Automatic TLS remains the default. A deployment that must use operator-provided
|
||||
certificates can mount a root-owned, non-link `*.caddy` file under `/etc/caddy/tls` and
|
||||
set `TRUF_EDGE_TLS_INCLUDE` to that absolute container path in a reviewed Compose
|
||||
override. The include should contain only the site's `tls CERT KEY` directive. Never use
|
||||
the repository's localhost test certificate or key in a deployment.
|
||||
|
||||
The normal `compose.yaml` remains private and unchanged. Confirm the host-agent socket is
|
||||
active and rerun installer validation immediately before starting production edge. Always
|
||||
supply the base file, the exact selected profile file, and the protected environment file.
|
||||
For standalone:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /etc/truf-edge/edge.env -f compose.yaml -f compose.edge.yaml build runtime edge
|
||||
docker compose --env-file /etc/truf-edge/edge.env -f compose.yaml -f compose.edge.yaml up -d
|
||||
```
|
||||
|
||||
For shared host:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /etc/truf-edge/edge.env -f compose.yaml -f compose.shared-host.yaml build runtime edge
|
||||
docker compose --env-file /etc/truf-edge/edge.env -f compose.yaml -f compose.shared-host.yaml up -d
|
||||
```
|
||||
|
||||
Before starting shared host, validate the complete existing host Caddy configuration with
|
||||
the snippet import in place. A matching request must fail at that Truf route if the
|
||||
loopback edge is unavailable; it must never fall through to X-UI or another upstream.
|
||||
|
||||
Provisioning creates the private `/data/managed-files` namespace in the named data volume.
|
||||
Each configured writable root must be a reviewed immediate child such as
|
||||
`/data/managed-files/exports`, created with UID/GID 10001 and mode 0700 while the runtime is
|
||||
stopped. Arbitrary host bind paths are not managed-file roots.
|
||||
|
||||
Worker admission remains disabled by `app/config.linux.yaml`. Configure the private
|
||||
runtime config's worker sources, compatibility profiles, and hashed device credentials
|
||||
before explicitly enabling `supervisor.worker_api.enabled`. The edge does not enable it.
|
||||
The typed admin backend is disabled independently under `supervisor.worker_api.admin`.
|
||||
Set its exact `origin` to `https://TRUF_EDGE_HOST`, set `edge_marker` to the same independent
|
||||
256-bit value as `TRUF_ADMIN_EDGE_MARKER`, then explicitly enable it. Both authenticated
|
||||
surfaces share the private runtime loopback port 8766. Caddy strips any inbound
|
||||
`X-Truf-Admin-Edge` and `X-Truf-Admin-Operator`, sets the configured marker and the
|
||||
authenticated Basic-auth username only after authentication, and rewrites the public
|
||||
random prefix to the private `/admin-internal` backend path. The backend accepts the
|
||||
operator identity only together with the private marker.
|
||||
Worker API requests receive neither private admin header.
|
||||
|
||||
Build and client bootstrap instructions for Windows and Linux remote workers are in
|
||||
`docs/remote-worker-operations.md`. Worker executables and images must be produced from a
|
||||
reviewed release checkout; operators must not assemble Python, Git, TruffleHog, detector
|
||||
policy, or dependencies manually on each worker.
|
||||
|
||||
## Fail2ban
|
||||
|
||||
Install the host files under their conventional names and enable fail2ban plus the expiry
|
||||
timer. The jail counts only redacted `admin_auth_failure` JSON records. An initial Basic
|
||||
challenge without credentials, Worker API authentication failures, and unrelated 404s do
|
||||
not enter that log. The action changes only the matcher imported inside the secret admin
|
||||
route; it does not create firewall rules and therefore does not block workers sharing an IP.
|
||||
|
||||
```sh
|
||||
sudo install -m 0644 deploy/fail2ban/filter.d-truf-admin-auth.conf /etc/fail2ban/filter.d/truf-admin-auth.conf
|
||||
sudo install -m 0644 deploy/fail2ban/jail.d-truf-admin-auth.local /etc/fail2ban/jail.d/truf-admin-auth.local
|
||||
sudo install -m 0644 deploy/fail2ban/action.d-truf-caddy-admin-denylist.conf /etc/fail2ban/action.d/truf-caddy-admin-denylist.conf
|
||||
sudo install -m 0644 deploy/fail2ban/fail2ban.d-truf-persistence.local /etc/fail2ban/fail2ban.d/truf-persistence.local
|
||||
sudo install -m 0644 deploy/systemd/truf-caddy-admin-denylist-expire.service /etc/systemd/system/
|
||||
sudo install -m 0644 deploy/systemd/truf-caddy-admin-denylist-expire.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now fail2ban truf-caddy-admin-denylist-expire.timer
|
||||
sudo fail2ban-client status truf-admin-auth
|
||||
```
|
||||
|
||||
Fail2ban persists jail state in `/var/lib/fail2ban/fail2ban.sqlite3`. The updater persists
|
||||
canonical IPs and expiry timestamps in private
|
||||
`/var/lib/truf-edge/admin-denylist.json`. It validates the complete Caddyfile in the running
|
||||
edge container, reloads it through the private admin endpoint, and restores/reloads the
|
||||
previous state if a command fails. Reload uses Caddy's private `/run/caddy-admin.sock` inside the edge
|
||||
container. The socket is not mounted or published. Shared mode renders denylist matchers
|
||||
against the marker-authenticated client address; standalone mode uses the direct peer.
|
||||
|
||||
## SSH recovery
|
||||
|
||||
Use fail2ban's normal unban first so its database and Caddy agree:
|
||||
|
||||
```sh
|
||||
sudo fail2ban-client set truf-admin-auth unbanip 203.0.113.10
|
||||
sudo /usr/local/sbin/truf-caddy-admin-denylist status
|
||||
sudo /usr/local/sbin/truf-caddy-admin-denylist expire
|
||||
```
|
||||
|
||||
If fail2ban is unavailable, run the updater's explicit unban over SSH:
|
||||
|
||||
```sh
|
||||
sudo /usr/local/sbin/truf-caddy-admin-denylist unban 203.0.113.10
|
||||
```
|
||||
|
||||
For recovery from a damaged generated snippet, stop the expiry timer and fail2ban, restore
|
||||
`deploy/edge/admin-denylist.caddy` to `/etc/truf-edge/denylist/admin-denylist.caddy`, then
|
||||
run Caddy validation and reload through the private Unix admin socket only after validation
|
||||
succeeds. Reconcile each
|
||||
remaining address with the updater before re-enabling the services. Do not use a global
|
||||
firewall ban as a shortcut.
|
||||
@@ -0,0 +1 @@
|
||||
# Managed by truf-caddy-admin-denylist. Admin-route import only.
|
||||
@@ -0,0 +1 @@
|
||||
# Empty by design: Caddy's automatic TLS remains the production default.
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
fail() {
|
||||
echo "edge configuration rejected: $1" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
host=${TRUF_EDGE_HOST:-}
|
||||
prefix=${TRUF_ADMIN_PREFIX:-}
|
||||
user=${TRUF_ADMIN_USER:-}
|
||||
password_hash=${TRUF_ADMIN_PASSWORD_HASH:-}
|
||||
edge_marker=${TRUF_ADMIN_EDGE_MARKER:-}
|
||||
edge_mode=${TRUF_EDGE_MODE:-standalone-edge-v1}
|
||||
ingress_marker=${TRUF_SHARED_INGRESS_MARKER:-}
|
||||
tls_include=${TRUF_EDGE_TLS_INCLUDE:-}
|
||||
|
||||
case "$edge_mode" in
|
||||
standalone-edge-v1) caddyfile=/etc/caddy/Caddyfile ;;
|
||||
shared-host-edge-v1)
|
||||
caddyfile=/etc/caddy/Caddyfile.shared-host
|
||||
[ "${#ingress_marker}" -eq 64 ] || fail "TRUF_SHARED_INGRESS_MARKER must encode 256 random bits"
|
||||
printf '%s' "$ingress_marker" | grep -Eq '^[0-9a-f]{64}$' \
|
||||
|| fail "TRUF_SHARED_INGRESS_MARKER must encode 256 random bits"
|
||||
;;
|
||||
*) fail "TRUF_EDGE_MODE is unsupported" ;;
|
||||
esac
|
||||
|
||||
if [ "$host" = localhost ]; then
|
||||
[ -n "$tls_include" ] || fail "localhost requires an explicit static TLS include"
|
||||
else
|
||||
case "$host" in
|
||||
''|*://*|*/*|*:*|.*|*..*|*.) fail "TRUF_EDGE_HOST must be one DNS hostname" ;;
|
||||
esac
|
||||
printf '%s' "$host" | awk -F. '
|
||||
length($0) > 253 || NF < 2 { exit 1 }
|
||||
{ for (i = 1; i <= NF; i++) if (length($i) > 63 || $i !~ /^[A-Za-z0-9-]+$/ || $i ~ /^-/ || $i ~ /-$/) exit 1 }
|
||||
' \
|
||||
|| fail "TRUF_EDGE_HOST must be one DNS hostname"
|
||||
fi
|
||||
|
||||
if [ -n "$tls_include" ]; then
|
||||
case "$tls_include" in
|
||||
/etc/caddy/tls/*.caddy) ;;
|
||||
*) fail "TRUF_EDGE_TLS_INCLUDE must be an absolute Caddy TLS include" ;;
|
||||
esac
|
||||
[ -f "$tls_include" ] && [ ! -L "$tls_include" ] \
|
||||
|| fail "TRUF_EDGE_TLS_INCLUDE must be a regular non-link file"
|
||||
fi
|
||||
|
||||
[ "${#prefix}" -eq 64 ] || fail "TRUF_ADMIN_PREFIX must encode 256 random bits"
|
||||
printf '%s' "$prefix" | grep -Eq '^[0-9a-f]{64}$' \
|
||||
|| fail "TRUF_ADMIN_PREFIX must encode 256 random bits"
|
||||
|
||||
printf '%s' "$user" | grep -Eq '^[A-Za-z0-9_.-]{1,64}$' \
|
||||
|| fail "TRUF_ADMIN_USER has an unsupported form"
|
||||
printf '%s' "$password_hash" | grep -Eq '^\$2[aby]\$(0[4-9]|[12][0-9]|3[01])\$[./A-Za-z0-9]{53}$' \
|
||||
|| fail "TRUF_ADMIN_PASSWORD_HASH must be a supported bcrypt hash"
|
||||
[ "${#edge_marker}" -eq 64 ] || fail "TRUF_ADMIN_EDGE_MARKER must encode 256 random bits"
|
||||
printf '%s' "$edge_marker" | grep -Eq '^[0-9a-f]{64}$' \
|
||||
|| fail "TRUF_ADMIN_EDGE_MARKER must encode 256 random bits"
|
||||
|
||||
[ -f /etc/caddy/denylist/admin-denylist.caddy ] \
|
||||
|| fail "the managed admin denylist snippet is missing"
|
||||
[ ! -L /etc/caddy/denylist/admin-denylist.caddy ] \
|
||||
|| fail "the managed admin denylist snippet must not be a link"
|
||||
[ -d /var/log/caddy ] && [ -w /var/log/caddy ] \
|
||||
|| fail "the authentication log directory is not writable"
|
||||
|
||||
umask 077
|
||||
exec caddy run --config "$caddyfile" --adapter caddyfile
|
||||
@@ -0,0 +1,26 @@
|
||||
# Import this route-only snippet inside the reviewed public site block.
|
||||
@truf_worker path /api/v1/worker/*
|
||||
handle @truf_worker {
|
||||
reverse_proxy 127.0.0.1:18766 {
|
||||
header_up X-Truf-Shared-Ingress {$TRUF_SHARED_INGRESS_MARKER}
|
||||
header_up -X-Truf-Admin-Edge
|
||||
header_up -X-Truf-Admin-Operator
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
header_up -X-Forwarded-For
|
||||
header_up X-Forwarded-For {http.request.remote.host}
|
||||
}
|
||||
}
|
||||
|
||||
@truf_admin path /{$TRUF_ADMIN_PREFIX} /{$TRUF_ADMIN_PREFIX}/*
|
||||
handle @truf_admin {
|
||||
reverse_proxy 127.0.0.1:18766 {
|
||||
header_up X-Truf-Shared-Ingress {$TRUF_SHARED_INGRESS_MARKER}
|
||||
header_up -X-Truf-Admin-Edge
|
||||
header_up -X-Truf-Admin-Operator
|
||||
header_up -Forwarded
|
||||
header_up -X-Real-IP
|
||||
header_up -X-Forwarded-For
|
||||
header_up X-Forwarded-For {http.request.remote.host}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
FROM caddy:2.10.2-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d AS caddy-edge-e2e
|
||||
|
||||
FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS fail2ban-edge-e2e
|
||||
|
||||
COPY --from=caddy-edge-e2e --chown=0:0 --chmod=0444 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
RUN <<'SH'
|
||||
set -eu
|
||||
rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources
|
||||
printf '%s\n' \
|
||||
'Types: deb' \
|
||||
'URIs: http://snapshot.debian.org/archive/debian/20260914T000000Z/' \
|
||||
'Suites: bookworm bookworm-updates' \
|
||||
'Components: main' \
|
||||
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
||||
'Check-Valid-Until: no' \
|
||||
'' \
|
||||
'Types: deb' \
|
||||
'URIs: http://snapshot.debian.org/archive/debian-security/20260914T000000Z/' \
|
||||
'Suites: bookworm-security' \
|
||||
'Components: main' \
|
||||
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
||||
'Check-Valid-Until: no' \
|
||||
> /etc/apt/sources.list.d/debian.sources
|
||||
printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
|
||||
chmod 0755 /usr/sbin/policy-rc.d
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get -o Acquire::Retries=3 -o Acquire::https::Timeout=30 -o APT::Update::Error-Mode=any update
|
||||
apt-get install -y --no-install-recommends fail2ban=1.0.2-2
|
||||
rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /var/log/apt/*
|
||||
find /etc/fail2ban/jail.d -type f -delete
|
||||
install -d -o 0 -g 0 -m 0755 \
|
||||
/etc/caddy /etc/caddy/denylist /etc/caddy/tls /etc/fail2ban/action.d /etc/fail2ban/fail2ban.d \
|
||||
/etc/fail2ban/filter.d /etc/fail2ban/jail.d /etc/truf-edge/denylist \
|
||||
/run/fail2ban /var/lib/fail2ban /var/lib/truf-edge /var/log/caddy /var/log/truf-edge
|
||||
SH
|
||||
|
||||
COPY --from=caddy-edge-e2e --chown=0:0 --chmod=0555 /usr/bin/caddy /usr/bin/caddy
|
||||
RUN cp /usr/bin/caddy /usr/bin/caddy-edge-e2e \
|
||||
&& rm /usr/bin/caddy \
|
||||
&& mv /usr/bin/caddy-edge-e2e /usr/bin/caddy \
|
||||
&& chmod 0555 /usr/bin/caddy
|
||||
COPY --chown=0:0 --chmod=0444 deploy/edge/Caddyfile /etc/caddy/Caddyfile
|
||||
RUN chmod 0644 /etc/caddy/Caddyfile \
|
||||
&& sed -i 's/^[[:space:]]*admin off$/\tadmin 127.0.0.1:2019/' /etc/caddy/Caddyfile \
|
||||
&& chmod 0444 /etc/caddy/Caddyfile \
|
||||
&& grep -Fx ' admin 127.0.0.1:2019' /etc/caddy/Caddyfile >/dev/null
|
||||
COPY --chown=0:0 --chmod=0444 deploy/fail2ban/filter.d-truf-admin-auth.conf /etc/fail2ban/filter.d/truf-admin-auth.conf
|
||||
COPY --chown=0:0 --chmod=0444 deploy/fail2ban/jail.d-truf-admin-auth.local /etc/fail2ban/jail.d/truf-admin-auth.local
|
||||
COPY --chown=0:0 --chmod=0444 deploy/fail2ban/action.d-truf-caddy-admin-denylist.conf /etc/fail2ban/action.d/truf-caddy-admin-denylist.conf
|
||||
COPY --chown=0:0 --chmod=0444 deploy/fail2ban/fail2ban.d-truf-persistence.local /etc/fail2ban/fail2ban.d/truf-persistence.local
|
||||
COPY --chown=0:0 --chmod=0444 deploy/fail2ban/fail2ban.d-edge-e2e.local /etc/fail2ban/fail2ban.d/edge-e2e.local
|
||||
COPY --chown=0:0 --chmod=0555 deploy/fail2ban/truf_caddy_admin_denylist.py /usr/local/sbin/truf-caddy-admin-denylist
|
||||
COPY --chown=0:0 --chmod=0555 deploy/fail2ban/edge_e2e_docker_shim.py /usr/local/bin/docker
|
||||
|
||||
RUN fail2ban-server --version 2>&1 | grep -F 'v1.0.2' >/dev/null \
|
||||
&& test "$(find /etc/fail2ban/jail.d -type f | wc -l)" -eq 1
|
||||
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/usr/bin/fail2ban-server", "-f", "-x"]
|
||||
CMD []
|
||||
@@ -0,0 +1,4 @@
|
||||
[Definition]
|
||||
actionstart = /usr/local/sbin/truf-caddy-admin-denylist expire
|
||||
actionban = /usr/local/sbin/truf-caddy-admin-denylist ban '<ip>'
|
||||
actionunban = /usr/local/sbin/truf-caddy-admin-denylist unban '<ip>'
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/python3
|
||||
"""Constrain production denylist reload commands to the colocated E2E Caddy."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ENV_FILE = Path("/etc/truf-edge/edge.env")
|
||||
AUDIT_PATH = Path("/var/lib/truf-edge/edge-e2e-reload.audit")
|
||||
VALIDATION_ERROR_PATH = Path("/var/lib/truf-edge/edge-e2e-validation.error")
|
||||
COMPOSE_PREFIX = (
|
||||
"compose", "--ansi", "never", "--env-file", str(ENV_FILE),
|
||||
"--project-directory", "/opt/truf",
|
||||
"--file", "/opt/truf/compose.yaml",
|
||||
"--file", "/opt/truf/compose.edge.yaml",
|
||||
)
|
||||
VALIDATE_COMMAND = COMPOSE_PREFIX + (
|
||||
"exec", "-T", "edge", "caddy", "validate",
|
||||
"--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile",
|
||||
)
|
||||
RELOAD_COMMAND = COMPOSE_PREFIX + ("kill", "--signal", "SIGUSR1", "edge")
|
||||
REQUIRED_ENV = {
|
||||
"TRUF_EDGE_HOST",
|
||||
"TRUF_EDGE_TLS_INCLUDE",
|
||||
"TRUF_ADMIN_PREFIX",
|
||||
"TRUF_ADMIN_USER",
|
||||
"TRUF_ADMIN_PASSWORD_HASH",
|
||||
"TRUF_ADMIN_EDGE_MARKER",
|
||||
}
|
||||
|
||||
|
||||
def classify_command(arguments):
|
||||
command = tuple(arguments)
|
||||
if command == VALIDATE_COMMAND:
|
||||
return "validate"
|
||||
if command == RELOAD_COMMAND:
|
||||
return "reload"
|
||||
raise ValueError("unsupported command")
|
||||
|
||||
|
||||
def audit(operation, result):
|
||||
payload = f"{operation}:{result}\n".encode("ascii")
|
||||
flags = (
|
||||
os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
|
||||
| getattr(os, "O_BINARY", 0)
|
||||
)
|
||||
descriptor = os.open(AUDIT_PATH, flags, 0o600)
|
||||
try:
|
||||
details = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(details.st_mode) or details.st_size + len(payload) > 4096:
|
||||
raise ValueError("invalid audit file")
|
||||
os.write(descriptor, payload)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def record_validation_error(content, environment):
|
||||
if len(content) > 65536:
|
||||
content = b"caddy validation error exceeded evidence bound\n"
|
||||
text = content.decode("utf-8", errors="replace")
|
||||
for value in environment.values():
|
||||
if value:
|
||||
text = text.replace(value, "[redacted]")
|
||||
text = re.sub(r"\$2[aby]\$(?:0[4-9]|[12][0-9]|3[01])\$[./A-Za-z0-9]{53}", "[redacted]", text)
|
||||
text = re.sub(r"(?<![0-9a-f])[0-9a-f]{64}(?![0-9a-f])", "[redacted]", text)
|
||||
payload = text.encode("utf-8", errors="replace")[:4096]
|
||||
descriptor = os.open(
|
||||
VALIDATION_ERROR_PATH,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
|
||||
| getattr(os, "O_BINARY", 0),
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
|
||||
raise ValueError("invalid validation evidence file")
|
||||
os.write(descriptor, payload)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def load_environment(path=ENV_FILE):
|
||||
details = path.lstat()
|
||||
if not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_size > 8192:
|
||||
raise ValueError("invalid environment file")
|
||||
values = {}
|
||||
for raw_line in path.read_text(encoding="ascii").splitlines():
|
||||
if not raw_line or raw_line.startswith("#"):
|
||||
continue
|
||||
name, separator, value = raw_line.partition("=")
|
||||
if not separator or name not in REQUIRED_ENV or name in values or "\x00" in value:
|
||||
raise ValueError("invalid environment entry")
|
||||
values[name] = value
|
||||
if set(values) != REQUIRED_ENV:
|
||||
raise ValueError("incomplete environment")
|
||||
if (
|
||||
values["TRUF_EDGE_HOST"] != "localhost"
|
||||
or values["TRUF_EDGE_TLS_INCLUDE"] != "/etc/caddy/tls/static-tls.caddy"
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", values["TRUF_ADMIN_PREFIX"])
|
||||
or not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", values["TRUF_ADMIN_USER"])
|
||||
or not re.fullmatch(r"\$2[aby]\$(?:0[4-9]|[12][0-9]|3[01])\$[./A-Za-z0-9]{53}", values["TRUF_ADMIN_PASSWORD_HASH"])
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", values["TRUF_ADMIN_EDGE_MARKER"])
|
||||
):
|
||||
raise ValueError("unsupported environment")
|
||||
return {
|
||||
**values,
|
||||
"HOME": "/tmp",
|
||||
"LANG": "C.UTF-8",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
"PATH": "/usr/bin:/bin",
|
||||
}
|
||||
|
||||
|
||||
def validate():
|
||||
try:
|
||||
environment = load_environment()
|
||||
except Exception:
|
||||
audit("environment", 64)
|
||||
raise
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
(
|
||||
"/usr/bin/caddy", "validate", "--config", "/etc/caddy/Caddyfile",
|
||||
"--adapter", "caddyfile",
|
||||
),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
env=environment,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
except Exception:
|
||||
audit("caddy-exec", 64)
|
||||
raise
|
||||
if completed.returncode:
|
||||
record_validation_error(completed.stderr, environment)
|
||||
return completed.returncode
|
||||
|
||||
|
||||
def reload_caddy():
|
||||
try:
|
||||
command = Path("/proc/1/cmdline").read_bytes()
|
||||
except Exception:
|
||||
audit("reload-proc", 64)
|
||||
raise
|
||||
if (
|
||||
len(command) > 4096
|
||||
or command.rstrip(b"\0").split(b"\0")
|
||||
not in (
|
||||
[b"caddy", b"run", b"--config", b"/etc/caddy/Caddyfile", b"--adapter", b"caddyfile"],
|
||||
[b"/usr/bin/caddy", b"run", b"--config", b"/etc/caddy/Caddyfile", b"--adapter", b"caddyfile"],
|
||||
)
|
||||
):
|
||||
audit("reload-identity", 64)
|
||||
raise ValueError("unexpected pid namespace")
|
||||
completed = subprocess.run(
|
||||
(
|
||||
"/usr/bin/caddy", "reload", "--config", "/etc/caddy/Caddyfile",
|
||||
"--adapter", "caddyfile", "--address", "127.0.0.1:2019",
|
||||
),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env=load_environment(),
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
return completed.returncode
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
operation = classify_command((argv or sys.argv)[1:])
|
||||
result = validate() if operation == "validate" else reload_caddy()
|
||||
except Exception:
|
||||
if "operation" in locals():
|
||||
try:
|
||||
audit(operation, 64)
|
||||
except Exception:
|
||||
pass
|
||||
return 64
|
||||
try:
|
||||
audit(operation, result)
|
||||
except Exception:
|
||||
return 64
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,5 @@
|
||||
[Definition]
|
||||
loglevel = INFO
|
||||
logtarget = STDOUT
|
||||
socket = /run/fail2ban/fail2ban.sock
|
||||
pidfile = /run/fail2ban/fail2ban.pid
|
||||
@@ -0,0 +1,3 @@
|
||||
[Definition]
|
||||
dbfile = /var/lib/fail2ban/fail2ban.sqlite3
|
||||
dbpurgeage = 7d
|
||||
@@ -0,0 +1,4 @@
|
||||
[Definition]
|
||||
failregex = ^(?=.{1,1024}$)(?=.*"event"\s*:\s*"admin_auth_failure")(?=.*"status"\s*:\s*401)(?=.*"remote_ip"\s*:\s*"<HOST>")(?!.*"(?:request|uri|headers|authorization|password|token|prefix)"\s*:).*\s*$
|
||||
ignoreregex =
|
||||
datepattern = "ts":{EPOCH}
|
||||
@@ -0,0 +1,9 @@
|
||||
[truf-admin-auth]
|
||||
enabled = true
|
||||
filter = truf-admin-auth
|
||||
logpath = /var/log/truf-edge/admin-auth-failures.json
|
||||
backend = auto
|
||||
maxretry = 2
|
||||
findtime = 10m
|
||||
bantime = 24h
|
||||
action = truf-caddy-admin-denylist
|
||||
@@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintain the Caddy admin-only IP denylist with durable expiry state."""
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
BAN_SECONDS = 24 * 60 * 60
|
||||
MAX_BANS = 4096
|
||||
MAX_FILE_BYTES = 512 * 1024
|
||||
STATE_VERSION = 1
|
||||
EMPTY_SNIPPET = "# Managed by truf-caddy-admin-denylist. Admin-route import only.\n"
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
PROFILE_PATH = Path("/etc/truf/deployment-profile")
|
||||
STANDALONE_PROFILE = "standalone-edge-v1"
|
||||
SHARED_HOST_PROFILE = "shared-host-edge-v1"
|
||||
|
||||
|
||||
class UpdateError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class CommandFailure(UpdateError):
|
||||
pass
|
||||
|
||||
|
||||
class RollbackFailure(UpdateError):
|
||||
pass
|
||||
|
||||
|
||||
def canonical_ip(value):
|
||||
text = str(value or "")
|
||||
if not text or len(text) > 64 or "%" in text or any(char.isspace() for char in text):
|
||||
raise UpdateError("invalid IP address")
|
||||
try:
|
||||
address = ipaddress.ip_address(text)
|
||||
except ValueError as exc:
|
||||
raise UpdateError("invalid IP address") from exc
|
||||
if address.is_unspecified or address.is_multicast:
|
||||
raise UpdateError("unsupported IP address")
|
||||
return address.compressed.lower()
|
||||
|
||||
|
||||
def render_snippet(bans, matcher="remote_ip"):
|
||||
if matcher not in {"remote_ip", "client_ip"}:
|
||||
raise UpdateError("unsupported denylist matcher")
|
||||
addresses = sorted(
|
||||
(ipaddress.ip_address(address) for address in bans),
|
||||
key=lambda address: (address.version, int(address)),
|
||||
)
|
||||
if not addresses:
|
||||
return EMPTY_SNIPPET.encode("ascii")
|
||||
lines = [EMPTY_SNIPPET.rstrip("\n")]
|
||||
for offset in range(0, len(addresses), 64):
|
||||
name = f"truf_admin_denied_{offset // 64:04d}"
|
||||
values = " ".join(address.compressed.lower() for address in addresses[offset:offset + 64])
|
||||
lines.append(f"@{name} {matcher} {values}")
|
||||
lines.append(f'respond @{name} "" 403')
|
||||
return ("\n".join(lines) + "\n").encode("ascii")
|
||||
|
||||
|
||||
def _digest(content):
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def _check_parent(path):
|
||||
parent = path.parent
|
||||
details = parent.lstat()
|
||||
if not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode):
|
||||
raise UpdateError("managed parent must be a real directory")
|
||||
if os.name == "posix" and stat.S_IMODE(details.st_mode) & 0o002:
|
||||
raise UpdateError("managed parent must not be world-writable")
|
||||
|
||||
|
||||
def _read_optional(path):
|
||||
_check_parent(path)
|
||||
try:
|
||||
details = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode):
|
||||
raise UpdateError("managed path must be a regular file")
|
||||
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(path, flags)
|
||||
try:
|
||||
current = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(current.st_mode) or current.st_size > MAX_FILE_BYTES:
|
||||
raise UpdateError("managed file is invalid or too large")
|
||||
chunks = []
|
||||
remaining = MAX_FILE_BYTES + 1
|
||||
while remaining:
|
||||
chunk = os.read(descriptor, min(65536, remaining))
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
content = b"".join(chunks)
|
||||
if len(content) > MAX_FILE_BYTES:
|
||||
raise UpdateError("managed file is too large")
|
||||
return content
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _sync_parent(parent):
|
||||
if os.name != "posix":
|
||||
return
|
||||
descriptor = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _atomic_write(path, content, mode):
|
||||
_check_parent(path)
|
||||
if len(content) > MAX_FILE_BYTES:
|
||||
raise UpdateError("managed content is too large")
|
||||
try:
|
||||
existing = path.lstat()
|
||||
except FileNotFoundError:
|
||||
existing = None
|
||||
if existing is not None and (not stat.S_ISREG(existing.st_mode) or stat.S_ISLNK(existing.st_mode)):
|
||||
raise UpdateError("managed path must be a regular file")
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".truf-denylist-", dir=path.parent)
|
||||
temporary_path = Path(temporary)
|
||||
try:
|
||||
if hasattr(os, "fchmod"):
|
||||
os.fchmod(descriptor, mode)
|
||||
else:
|
||||
os.chmod(temporary_path, mode)
|
||||
with os.fdopen(descriptor, "wb", closefd=True) as handle:
|
||||
descriptor = -1
|
||||
handle.write(content)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
_sync_parent(path.parent)
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
temporary_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def _restore(path, content, mode):
|
||||
if content is not None:
|
||||
_atomic_write(path, content, mode)
|
||||
return
|
||||
try:
|
||||
details = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode):
|
||||
raise UpdateError("managed path changed during rollback")
|
||||
path.unlink()
|
||||
_sync_parent(path.parent)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive_lock(path):
|
||||
_check_parent(path)
|
||||
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
try:
|
||||
details = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(details.st_mode):
|
||||
raise UpdateError("lock path must be a regular file")
|
||||
if os.name == "posix":
|
||||
import fcntl
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _load_state(content):
|
||||
if content is None:
|
||||
return {"version": STATE_VERSION, "bans": {}, "applied_sha256": ""}
|
||||
try:
|
||||
value = json.loads(content.decode("ascii"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise UpdateError("denylist state is not valid JSON") from exc
|
||||
if not isinstance(value, dict) or set(value) != {"version", "bans", "applied_sha256"}:
|
||||
raise UpdateError("denylist state has an invalid schema")
|
||||
if value["version"] != STATE_VERSION or not isinstance(value["bans"], dict):
|
||||
raise UpdateError("denylist state has an unsupported version")
|
||||
if len(value["bans"]) > MAX_BANS:
|
||||
raise UpdateError("denylist state exceeds its entry bound")
|
||||
applied = value["applied_sha256"]
|
||||
if not isinstance(applied, str) or (applied and not SHA256_RE.fullmatch(applied)):
|
||||
raise UpdateError("denylist state has an invalid applied digest")
|
||||
bans = {}
|
||||
for address, expires_at in value["bans"].items():
|
||||
canonical = canonical_ip(address)
|
||||
if canonical != address or isinstance(expires_at, bool) or not isinstance(expires_at, int):
|
||||
raise UpdateError("denylist state has a noncanonical entry")
|
||||
if expires_at <= 0 or expires_at > 253402300799:
|
||||
raise UpdateError("denylist state has an invalid expiry")
|
||||
bans[canonical] = expires_at
|
||||
return {"version": STATE_VERSION, "bans": bans, "applied_sha256": applied}
|
||||
|
||||
|
||||
def _encode_state(state):
|
||||
return (json.dumps(state, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii")
|
||||
|
||||
|
||||
def _subprocess_runner(command):
|
||||
environment = {
|
||||
"HOME": "/root",
|
||||
"LANG": "C.UTF-8",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
}
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env=environment,
|
||||
timeout=45,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
class DenylistUpdater:
|
||||
def __init__(
|
||||
self,
|
||||
state_path,
|
||||
snippet_path,
|
||||
project_directory="/opt/truf",
|
||||
env_file="/etc/truf-edge/edge.env",
|
||||
profile=None,
|
||||
runner=None,
|
||||
clock=None,
|
||||
):
|
||||
self.state_path = Path(state_path)
|
||||
self.snippet_path = Path(snippet_path)
|
||||
self.lock_path = self.state_path.with_suffix(self.state_path.suffix + ".lock")
|
||||
if profile is None:
|
||||
try:
|
||||
profile = PROFILE_PATH.read_text(encoding="ascii").strip()
|
||||
except FileNotFoundError:
|
||||
profile = STANDALONE_PROFILE
|
||||
except (OSError, UnicodeError):
|
||||
raise UpdateError("deployment profile is unreadable") from None
|
||||
if profile not in {STANDALONE_PROFILE, SHARED_HOST_PROFILE}:
|
||||
raise UpdateError("deployment profile is unsupported")
|
||||
compose_file = (
|
||||
"compose.shared-host.yaml"
|
||||
if profile == SHARED_HOST_PROFILE else "compose.edge.yaml"
|
||||
)
|
||||
caddyfile = (
|
||||
"/etc/caddy/Caddyfile.shared-host"
|
||||
if profile == SHARED_HOST_PROFILE else "/etc/caddy/Caddyfile"
|
||||
)
|
||||
self.matcher = "client_ip" if profile == SHARED_HOST_PROFILE else "remote_ip"
|
||||
compose = (
|
||||
"docker", "compose", "--ansi", "never", "--env-file", str(env_file),
|
||||
"--project-directory", str(project_directory),
|
||||
"--file", str(Path(project_directory) / "compose.yaml"),
|
||||
"--file", str(Path(project_directory) / compose_file),
|
||||
)
|
||||
self.validate_command = compose + (
|
||||
"exec", "-T", "edge", "caddy", "validate",
|
||||
"--config", caddyfile, "--adapter", "caddyfile",
|
||||
)
|
||||
self.reload_command = compose + (
|
||||
"exec", "-T", "edge", "caddy", "reload",
|
||||
"--config", caddyfile, "--adapter", "caddyfile",
|
||||
"--address", "unix//run/caddy-admin.sock",
|
||||
)
|
||||
self.runner = runner or _subprocess_runner
|
||||
self.clock = clock or time.time
|
||||
|
||||
def _run(self, command, phase):
|
||||
try:
|
||||
succeeded = self.runner(command)
|
||||
except Exception as exc:
|
||||
raise CommandFailure(f"{phase} command failed") from exc
|
||||
if not succeeded:
|
||||
raise CommandFailure(f"{phase} command failed")
|
||||
|
||||
def update(self, operation, address=None):
|
||||
if operation not in {"ban", "unban", "expire", "status"}:
|
||||
raise UpdateError("unsupported operation")
|
||||
canonical = canonical_ip(address) if operation in {"ban", "unban"} else None
|
||||
now = int(self.clock())
|
||||
if now <= 0:
|
||||
raise UpdateError("system clock is invalid")
|
||||
|
||||
with _exclusive_lock(self.lock_path):
|
||||
old_state_content = _read_optional(self.state_path)
|
||||
old_snippet_content = _read_optional(self.snippet_path)
|
||||
state = _load_state(old_state_content)
|
||||
bans = {
|
||||
ip: expires_at for ip, expires_at in state["bans"].items()
|
||||
if expires_at > now
|
||||
}
|
||||
expired = len(state["bans"]) - len(bans)
|
||||
|
||||
if operation == "ban":
|
||||
if canonical not in bans and len(bans) >= MAX_BANS:
|
||||
raise UpdateError("denylist entry bound reached")
|
||||
bans[canonical] = max(bans.get(canonical, 0), now + BAN_SECONDS)
|
||||
elif operation == "unban":
|
||||
bans.pop(canonical, None)
|
||||
|
||||
desired_snippet = render_snippet(bans, self.matcher)
|
||||
desired_digest = _digest(desired_snippet)
|
||||
pending_state = {
|
||||
"version": STATE_VERSION,
|
||||
"bans": bans,
|
||||
"applied_sha256": state["applied_sha256"],
|
||||
}
|
||||
pending_content = _encode_state(pending_state)
|
||||
needs_reload = (
|
||||
old_snippet_content != desired_snippet
|
||||
or state["applied_sha256"] != desired_digest
|
||||
)
|
||||
needs_state_write = old_state_content != pending_content
|
||||
|
||||
if needs_reload:
|
||||
reload_attempted = False
|
||||
try:
|
||||
_atomic_write(self.state_path, pending_content, 0o600)
|
||||
_atomic_write(self.snippet_path, desired_snippet, 0o640)
|
||||
self._run(self.validate_command, "validation")
|
||||
reload_attempted = True
|
||||
self._run(self.reload_command, "reload")
|
||||
pending_state["applied_sha256"] = desired_digest
|
||||
_atomic_write(self.state_path, _encode_state(pending_state), 0o600)
|
||||
except Exception as original:
|
||||
try:
|
||||
_restore(self.state_path, old_state_content, 0o600)
|
||||
_restore(self.snippet_path, old_snippet_content, 0o640)
|
||||
if reload_attempted:
|
||||
self._run(self.validate_command, "rollback validation")
|
||||
self._run(self.reload_command, "rollback reload")
|
||||
except Exception as rollback:
|
||||
raise RollbackFailure("denylist rollback failed") from rollback
|
||||
if isinstance(original, UpdateError):
|
||||
raise
|
||||
raise UpdateError("denylist update failed") from original
|
||||
elif needs_state_write:
|
||||
pending_state["applied_sha256"] = desired_digest
|
||||
_atomic_write(self.state_path, _encode_state(pending_state), 0o600)
|
||||
|
||||
return {
|
||||
"operation": operation,
|
||||
"ip": canonical,
|
||||
"expired": expired,
|
||||
"bans": dict(bans),
|
||||
}
|
||||
|
||||
|
||||
def _emit(result):
|
||||
bans = result["bans"]
|
||||
if result["operation"] == "status":
|
||||
addresses = sorted(
|
||||
bans, key=lambda value: (ipaddress.ip_address(value).version, int(ipaddress.ip_address(value)))
|
||||
)
|
||||
payload = {
|
||||
"active": len(addresses),
|
||||
"bans": [
|
||||
{"ip": address, "expires_at": bans[address]}
|
||||
for address in addresses[:256]
|
||||
],
|
||||
"event": "admin_denylist_status",
|
||||
"truncated": len(addresses) > 256,
|
||||
}
|
||||
else:
|
||||
payload = {
|
||||
"active": len(bans),
|
||||
"event": "admin_denylist_" + result["operation"],
|
||||
"expired": result["expired"],
|
||||
}
|
||||
if result["ip"] is not None:
|
||||
payload["ip"] = result["ip"]
|
||||
print(json.dumps(payload, sort_keys=True, separators=(",", ":")), flush=True)
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False)
|
||||
parser.add_argument("--state-path", default="/var/lib/truf-edge/admin-denylist.json")
|
||||
parser.add_argument("--snippet-path", default="/etc/truf-edge/denylist/admin-denylist.caddy")
|
||||
parser.add_argument("--project-directory", default="/opt/truf")
|
||||
parser.add_argument("--env-file", default="/etc/truf-edge/edge.env")
|
||||
commands = parser.add_subparsers(dest="operation", required=True)
|
||||
for name in ("ban", "unban"):
|
||||
command = commands.add_parser(name, allow_abbrev=False)
|
||||
command.add_argument("ip")
|
||||
commands.add_parser("expire", allow_abbrev=False)
|
||||
commands.add_parser("status", allow_abbrev=False)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = parse_args(argv)
|
||||
updater = DenylistUpdater(
|
||||
args.state_path,
|
||||
args.snippet_path,
|
||||
project_directory=args.project_directory,
|
||||
env_file=args.env_file,
|
||||
)
|
||||
try:
|
||||
result = updater.update(args.operation, getattr(args, "ip", None))
|
||||
except Exception as exc:
|
||||
payload = {
|
||||
"event": "admin_denylist_error",
|
||||
"operation": args.operation,
|
||||
"reason": type(exc).__name__,
|
||||
}
|
||||
print(json.dumps(payload, sort_keys=True, separators=(",", ":")), file=sys.stderr)
|
||||
return 1
|
||||
_emit(result)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
d /etc/truf/runtime 0755 root root -
|
||||
d /etc/truf/worker-packages 0755 root root -
|
||||
d /var/lib/truf/runtime-document-candidates 0700 10001 10001 -
|
||||
d /var/lib/truf/host-agent 0700 root root -
|
||||
d /var/lib/truf/host-agent/backups 0700 root root -
|
||||
d /var/lib/truf/host-agent/operations 0700 root root -
|
||||
d /var/lib/truf/host-agent/results 0750 root 10001 -
|
||||
d /run/truf-postgres 0700 10001 10001 -
|
||||
@@ -0,0 +1,50 @@
|
||||
[Unit]
|
||||
Description=Truf privileged host operations agent
|
||||
Requires=truf-host-agent.socket docker.service
|
||||
After=truf-host-agent.socket docker.service
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=root
|
||||
Group=root
|
||||
UMask=0077
|
||||
WorkingDirectory=/
|
||||
ExecStartPre=/usr/bin/python3 -I -B /usr/lib/truf-host-agent/truf_host_agent_install.py validate
|
||||
ExecStart=/usr/bin/python3 -I -B /usr/lib/truf-host-agent/truf_host_agent.py
|
||||
RuntimeDirectory=truf-host-agent
|
||||
RuntimeDirectoryMode=0700
|
||||
NoNewPrivileges=yes
|
||||
CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_FOWNER
|
||||
AmbientCapabilities=
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
PrivateNetwork=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
ProcSubset=pid
|
||||
RestrictAddressFamilies=AF_UNIX
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
MemoryDenyWriteExecute=yes
|
||||
SystemCallArchitectures=native
|
||||
ReadWritePaths=/etc/truf/runtime
|
||||
ReadWritePaths=/var/lib/truf/host-agent
|
||||
ReadWritePaths=/run/truf-host-agent
|
||||
ReadWritePaths=/run/docker.sock
|
||||
ReadOnlyPaths=/usr/lib/truf-host-agent
|
||||
ReadOnlyPaths=/opt/truf
|
||||
ReadOnlyPaths=/var/lib/truf/runtime-document-candidates
|
||||
ReadOnlyPaths=/run/truf-postgres
|
||||
Restart=no
|
||||
TimeoutStopSec=45min
|
||||
StandardOutput=null
|
||||
StandardError=journal
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Truf privileged host operations socket
|
||||
|
||||
[Socket]
|
||||
ListenStream=/run/truf/host-agent.sock
|
||||
SocketUser=root
|
||||
SocketGroup=truf-runtime
|
||||
SocketMode=0660
|
||||
DirectoryMode=0755
|
||||
Service=truf-host-agent.service
|
||||
Accept=no
|
||||
RemoveOnStop=yes
|
||||
Backlog=8
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
APP_DIRECTORY = Path('/opt/truf/app')
|
||||
sys.path.insert(0, str(APP_DIRECTORY))
|
||||
|
||||
from host_agent_runtime import FixedHostOperationDispatcher
|
||||
from host_agent_server import (
|
||||
HostAgentServerError,
|
||||
inherited_systemd_listener,
|
||||
serve_forever,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 1:
|
||||
raise HostAgentServerError('arguments_forbidden')
|
||||
listener = inherited_systemd_listener()
|
||||
stop_event = threading.Event()
|
||||
dispatcher = FixedHostOperationDispatcher()
|
||||
|
||||
def request_stop(_signum, _frame):
|
||||
stop_event.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, request_stop)
|
||||
signal.signal(signal.SIGINT, request_stop)
|
||||
try:
|
||||
with listener:
|
||||
serve_forever(
|
||||
listener, handler=dispatcher.handle, stop_event=stop_event,
|
||||
)
|
||||
finally:
|
||||
dispatcher.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
result = main()
|
||||
except BaseException as exc:
|
||||
if not isinstance(exc, Exception):
|
||||
raise
|
||||
print(
|
||||
'host operations agent failed (' + type(exc).__name__ + '); details withheld',
|
||||
file=sys.stderr,
|
||||
)
|
||||
result = 1
|
||||
raise SystemExit(result)
|
||||
@@ -0,0 +1,571 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install or validate the fixed Truf host-agent deployment."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
PROJECT = Path('/opt/truf')
|
||||
DEPLOY = PROJECT / 'deploy/host-agent'
|
||||
INSTALL_ROOT = Path('/usr/lib/truf-host-agent')
|
||||
SYSTEMD = Path('/etc/systemd/system')
|
||||
TMPFILES = Path('/etc/tmpfiles.d/truf-host-agent.conf')
|
||||
ACTIVE = Path('/etc/truf/runtime')
|
||||
WORKER_PACKAGES = Path('/etc/truf/worker-packages')
|
||||
DEPLOYMENT_PROFILE = Path('/etc/truf/deployment-profile')
|
||||
AGENT_SOCKET = Path('/run/truf/host-agent.sock')
|
||||
RUNTIME_UID = RUNTIME_GID = 10001
|
||||
RUNTIME_GROUP = 'truf-runtime'
|
||||
MAX_COPY_BYTES = 4 * 1024 * 1024
|
||||
MAX_COMPOSE_OUTPUT_BYTES = 4 * 1024 * 1024
|
||||
UNITS = ('truf-host-agent.socket', 'truf-host-agent.service')
|
||||
SCRIPTS = ('truf_host_agent.py', 'truf_host_agent_install.py')
|
||||
FIXED_ENV = {
|
||||
'PATH': '/usr/sbin:/usr/bin:/sbin:/bin',
|
||||
'LANG': 'C',
|
||||
'LC_ALL': 'C',
|
||||
}
|
||||
STANDALONE_PROFILE = {
|
||||
'name': 'standalone-edge-v1',
|
||||
'compose_files': ('compose.yaml', 'compose.edge.yaml'),
|
||||
'runtime_network': None,
|
||||
'runtime_ports': [{
|
||||
'mode': 'host', 'target': 443, 'published': '443', 'protocol': 'tcp',
|
||||
}],
|
||||
'runtime_cpus': 2.0,
|
||||
'runtime_mem_limit': str(6 * 1024 ** 3),
|
||||
'edge_cap_add': ['NET_BIND_SERVICE'],
|
||||
'data_volume': {'name': 'truf-docker_data'},
|
||||
}
|
||||
SHARED_HOST_PROFILE = {
|
||||
'name': 'shared-host-edge-v1',
|
||||
'compose_files': ('compose.yaml', 'compose.shared-host.yaml'),
|
||||
'runtime_network': 'host',
|
||||
'runtime_ports': None,
|
||||
'runtime_cpus': 0.9,
|
||||
'runtime_mem_limit': str(720 * 1024 ** 2),
|
||||
'edge_cap_add': None,
|
||||
'data_volume': {'name': 'truf-remote-server-data', 'external': True},
|
||||
}
|
||||
|
||||
|
||||
def _compose_config_command(profile):
|
||||
return (
|
||||
'/usr/bin/docker', 'compose', '--ansi', 'never', '--project-name',
|
||||
'truf-docker', '--env-file', '/etc/truf-edge/edge.env',
|
||||
'--project-directory', '/opt/truf',
|
||||
*(item for name in profile['compose_files'] for item in (
|
||||
'--file', '/opt/truf/' + name,
|
||||
)),
|
||||
'config', '--format', 'json',
|
||||
)
|
||||
|
||||
|
||||
COMPOSE_CONFIG_COMMAND = _compose_config_command(STANDALONE_PROFILE)
|
||||
EXPECTED_RUNTIME_MOUNTS = (
|
||||
('volume', 'data', '/data', False, None),
|
||||
('bind', '/etc/truf/runtime', '/data/config', True, False),
|
||||
('bind', '/etc/truf/worker-packages', '/data/worker-packages', True, False),
|
||||
(
|
||||
'bind', '/var/lib/truf/runtime-document-candidates',
|
||||
'/data/runtime-document-candidates', False, False,
|
||||
),
|
||||
('bind', '/run/truf/host-agent.sock', '/run/truf/host-agent.sock', True, False),
|
||||
(
|
||||
'bind', '/var/lib/truf/host-agent/results',
|
||||
'/data/host-agent-results', True, False,
|
||||
),
|
||||
('bind', '/run/truf-postgres', '/run/truf-postgres', False, False),
|
||||
)
|
||||
|
||||
|
||||
class InstallError(RuntimeError):
|
||||
def __init__(self, category):
|
||||
self.category = str(category)
|
||||
super().__init__('host-agent deployment validation failed')
|
||||
|
||||
|
||||
def _deployment_profile():
|
||||
descriptor = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
DEPLOYMENT_PROFILE,
|
||||
os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0)
|
||||
| getattr(os, 'O_NOFOLLOW', 0),
|
||||
)
|
||||
before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode) or before.st_uid != 0
|
||||
or before.st_gid != 0 or stat.S_IMODE(before.st_mode) != 0o444
|
||||
or before.st_nlink != 1 or before.st_size > 64
|
||||
):
|
||||
raise InstallError('profile')
|
||||
payload = os.read(descriptor, 65)
|
||||
after = os.fstat(descriptor)
|
||||
if (
|
||||
len(payload) > 64 or before.st_dev != after.st_dev
|
||||
or before.st_ino != after.st_ino or before.st_mode != after.st_mode
|
||||
or before.st_uid != after.st_uid or before.st_gid != after.st_gid
|
||||
or before.st_nlink != after.st_nlink or before.st_size != after.st_size
|
||||
or before.st_mtime_ns != after.st_mtime_ns
|
||||
):
|
||||
raise InstallError('profile')
|
||||
except FileNotFoundError:
|
||||
return STANDALONE_PROFILE
|
||||
except InstallError:
|
||||
raise
|
||||
except OSError:
|
||||
raise InstallError('profile') from None
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
name = payload.decode('ascii').strip()
|
||||
except UnicodeDecodeError:
|
||||
raise InstallError('profile') from None
|
||||
profiles = {
|
||||
STANDALONE_PROFILE['name']: STANDALONE_PROFILE,
|
||||
SHARED_HOST_PROFILE['name']: SHARED_HOST_PROFILE,
|
||||
}
|
||||
if name not in profiles:
|
||||
raise InstallError('profile')
|
||||
return profiles[name]
|
||||
|
||||
|
||||
def _run(command, timeout=120):
|
||||
try:
|
||||
subprocess.run(
|
||||
tuple(command), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, cwd='/', env=FIXED_ENV, shell=False,
|
||||
timeout=timeout, check=True,
|
||||
)
|
||||
except Exception:
|
||||
raise InstallError('command') from None
|
||||
|
||||
|
||||
def _capture(command, timeout=120):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
tuple(command), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, cwd='/', env=FIXED_ENV, shell=False,
|
||||
timeout=timeout, check=True,
|
||||
)
|
||||
if len(result.stdout) > MAX_COMPOSE_OUTPUT_BYTES:
|
||||
raise InstallError('command')
|
||||
return result.stdout
|
||||
except InstallError:
|
||||
raise
|
||||
except Exception:
|
||||
raise InstallError('command') from None
|
||||
|
||||
|
||||
def _unit_active(unit):
|
||||
if unit not in UNITS:
|
||||
raise InstallError('command')
|
||||
try:
|
||||
result = subprocess.run(
|
||||
('/usr/bin/systemctl', 'is-active', '--quiet', unit),
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, cwd='/', env=FIXED_ENV, shell=False,
|
||||
timeout=120, check=False,
|
||||
)
|
||||
except Exception:
|
||||
raise InstallError('command') from None
|
||||
if result.returncode not in (0, 3, 4):
|
||||
raise InstallError('command')
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _validate_compose_projection(payload, profile=None):
|
||||
profile = profile or STANDALONE_PROFILE
|
||||
try:
|
||||
projection = json.loads(payload)
|
||||
if type(projection) is not dict or projection.get('name') != 'truf-docker':
|
||||
raise InstallError('compose')
|
||||
volume_definitions = projection.get('volumes')
|
||||
if (
|
||||
type(volume_definitions) is not dict
|
||||
or volume_definitions.get('data') != profile['data_volume']
|
||||
):
|
||||
raise InstallError('compose')
|
||||
services = projection.get('services')
|
||||
runtime = services.get('runtime') if type(services) is dict else None
|
||||
edge = services.get('edge') if type(services) is dict else None
|
||||
if (
|
||||
type(runtime) is not dict or type(edge) is not dict
|
||||
or runtime.get('network_mode') != profile['runtime_network']
|
||||
or runtime.get('ports') != profile['runtime_ports']
|
||||
or runtime.get('cpus') != profile['runtime_cpus']
|
||||
or runtime.get('mem_limit') != profile['runtime_mem_limit']
|
||||
or edge.get('network_mode') != 'service:runtime'
|
||||
or edge.get('cap_add') != profile['edge_cap_add']
|
||||
or edge.get('image') != 'truf-local:edge'
|
||||
):
|
||||
raise InstallError('compose')
|
||||
mounts = runtime.get('volumes') if type(runtime) is dict else None
|
||||
if type(mounts) is not list:
|
||||
raise InstallError('compose')
|
||||
observed = []
|
||||
for mount in mounts:
|
||||
if type(mount) is not dict or type(mount.get('read_only', False)) is not bool:
|
||||
raise InstallError('compose')
|
||||
kind = mount.get('type')
|
||||
if kind == 'volume':
|
||||
if set(mount) - {'type', 'source', 'target', 'read_only'}:
|
||||
raise InstallError('compose')
|
||||
create_host_path = None
|
||||
elif kind == 'bind':
|
||||
if set(mount) - {'type', 'source', 'target', 'read_only', 'bind'}:
|
||||
raise InstallError('compose')
|
||||
binding = mount.get('bind')
|
||||
if binding not in ({}, {'create_host_path': False}):
|
||||
raise InstallError('compose')
|
||||
create_host_path = False
|
||||
else:
|
||||
raise InstallError('compose')
|
||||
observed.append((
|
||||
kind, mount.get('source'), mount.get('target'),
|
||||
mount.get('read_only', False), create_host_path,
|
||||
))
|
||||
if tuple(observed) != EXPECTED_RUNTIME_MOUNTS:
|
||||
raise InstallError('compose')
|
||||
except InstallError:
|
||||
raise
|
||||
except Exception:
|
||||
raise InstallError('compose') from None
|
||||
|
||||
|
||||
def _details(path, *, directory, uid, gid, mode):
|
||||
try:
|
||||
value = os.stat(path, follow_symlinks=False)
|
||||
except OSError:
|
||||
raise InstallError('metadata') from None
|
||||
expected = stat.S_ISDIR if directory else stat.S_ISREG
|
||||
if (
|
||||
not expected(value.st_mode) or value.st_uid != uid or value.st_gid != gid
|
||||
or stat.S_IMODE(value.st_mode) != mode
|
||||
or (not directory and value.st_nlink != 1)
|
||||
):
|
||||
raise InstallError('metadata')
|
||||
return value
|
||||
|
||||
|
||||
def _read_source(path, maximum=MAX_COPY_BYTES):
|
||||
descriptor = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
path, os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0)
|
||||
| getattr(os, 'O_NOFOLLOW', 0),
|
||||
)
|
||||
before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode) or before.st_nlink != 1
|
||||
or before.st_uid != 0 or stat.S_IMODE(before.st_mode) & 0o022
|
||||
):
|
||||
raise InstallError('source')
|
||||
with os.fdopen(descriptor, 'rb') as handle:
|
||||
descriptor = None
|
||||
payload = handle.read(maximum + 1)
|
||||
after = os.fstat(handle.fileno())
|
||||
if len(payload) > maximum or (before.st_dev, before.st_ino, before.st_size) != (
|
||||
after.st_dev, after.st_ino, after.st_size,
|
||||
):
|
||||
raise InstallError('source')
|
||||
return payload
|
||||
except InstallError:
|
||||
raise
|
||||
except Exception:
|
||||
raise InstallError('source') from None
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _secure_tree(path):
|
||||
try:
|
||||
root = os.stat(path, follow_symlinks=False)
|
||||
if (
|
||||
not stat.S_ISDIR(root.st_mode)
|
||||
or root.st_uid != 0
|
||||
or stat.S_IMODE(root.st_mode) & 0o022
|
||||
):
|
||||
raise InstallError('project')
|
||||
for current, directories, files in os.walk(path, topdown=True, followlinks=False):
|
||||
current_path = Path(current)
|
||||
entries = ((name, True) for name in directories)
|
||||
entries = tuple(entries) + tuple((name, False) for name in files)
|
||||
current_details = os.stat(current_path, follow_symlinks=False)
|
||||
if (
|
||||
not stat.S_ISDIR(current_details.st_mode)
|
||||
or current_details.st_uid != 0
|
||||
or stat.S_IMODE(current_details.st_mode) & 0o022
|
||||
):
|
||||
raise InstallError('project')
|
||||
for name, directory in entries:
|
||||
details = os.stat(current_path / name, follow_symlinks=False)
|
||||
expected = stat.S_ISDIR if directory else stat.S_ISREG
|
||||
if (
|
||||
not expected(details.st_mode)
|
||||
or details.st_uid != 0
|
||||
or stat.S_IMODE(details.st_mode) & 0o022
|
||||
or (not directory and details.st_nlink != 1)
|
||||
):
|
||||
raise InstallError('project')
|
||||
except InstallError:
|
||||
raise
|
||||
except Exception:
|
||||
raise InstallError('project') from None
|
||||
|
||||
|
||||
def _same_file(installed, source):
|
||||
if not _read_source(installed) == _read_source(source):
|
||||
raise InstallError('installed_content')
|
||||
|
||||
|
||||
def _ensure_install_root():
|
||||
try:
|
||||
INSTALL_ROOT.mkdir(mode=0o755)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except OSError:
|
||||
raise InstallError('write') from None
|
||||
_details(INSTALL_ROOT, directory=True, uid=0, gid=0, mode=0o755)
|
||||
|
||||
|
||||
def _root_directory(path):
|
||||
try:
|
||||
details = os.stat(path, follow_symlinks=False)
|
||||
except OSError:
|
||||
raise InstallError('metadata') from None
|
||||
if (
|
||||
not stat.S_ISDIR(details.st_mode)
|
||||
or details.st_uid != 0
|
||||
or stat.S_IMODE(details.st_mode) & 0o022
|
||||
):
|
||||
raise InstallError('metadata')
|
||||
|
||||
|
||||
def _secure_executable(path):
|
||||
try:
|
||||
link = os.lstat(path)
|
||||
resolved = os.path.realpath(path)
|
||||
target = os.stat(path)
|
||||
parent = os.stat(Path(path).parent, follow_symlinks=False)
|
||||
except OSError:
|
||||
raise InstallError('executable') from None
|
||||
if (
|
||||
link.st_uid != 0
|
||||
or not (stat.S_ISREG(link.st_mode) or stat.S_ISLNK(link.st_mode))
|
||||
or not resolved.startswith(('/usr/bin/', '/usr/sbin/'))
|
||||
or not stat.S_ISREG(target.st_mode) or target.st_uid != 0
|
||||
or stat.S_IMODE(target.st_mode) & 0o022
|
||||
or not stat.S_ISDIR(parent.st_mode) or parent.st_uid != 0
|
||||
or stat.S_IMODE(parent.st_mode) & 0o022
|
||||
):
|
||||
raise InstallError('executable')
|
||||
|
||||
|
||||
def _runtime_group_exists():
|
||||
if sys.platform != 'linux':
|
||||
raise InstallError('group')
|
||||
try:
|
||||
import grp
|
||||
named = grp.getgrnam(RUNTIME_GROUP)
|
||||
numbered = grp.getgrgid(RUNTIME_GID)
|
||||
except KeyError:
|
||||
return False
|
||||
except Exception:
|
||||
raise InstallError('group') from None
|
||||
if named.gr_gid != RUNTIME_GID or numbered.gr_name != RUNTIME_GROUP:
|
||||
raise InstallError('group')
|
||||
return True
|
||||
|
||||
|
||||
def _ensure_runtime_group():
|
||||
if _runtime_group_exists():
|
||||
return
|
||||
try:
|
||||
import grp
|
||||
grp.getgrgid(RUNTIME_GID)
|
||||
except KeyError:
|
||||
pass
|
||||
except Exception:
|
||||
raise InstallError('group') from None
|
||||
else:
|
||||
raise InstallError('group')
|
||||
_secure_executable('/usr/sbin/groupadd')
|
||||
_run(('/usr/sbin/groupadd', '--system', '--gid', str(RUNTIME_GID), RUNTIME_GROUP))
|
||||
if not _runtime_group_exists():
|
||||
raise InstallError('group')
|
||||
|
||||
|
||||
def _validate_agent_socket():
|
||||
try:
|
||||
details = os.stat(AGENT_SOCKET, follow_symlinks=False)
|
||||
except OSError:
|
||||
raise InstallError('socket') from None
|
||||
if (
|
||||
not stat.S_ISSOCK(details.st_mode)
|
||||
or details.st_uid != 0
|
||||
or details.st_gid != RUNTIME_GID
|
||||
or stat.S_IMODE(details.st_mode) != 0o660
|
||||
):
|
||||
raise InstallError('socket')
|
||||
|
||||
|
||||
def _write(path, payload, *, uid, gid, mode, replace):
|
||||
temporary = path.parent / ('.' + path.name + '.truf-install')
|
||||
descriptor = None
|
||||
try:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
descriptor = os.open(
|
||||
temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
| getattr(os, 'O_NOFOLLOW', 0), mode,
|
||||
)
|
||||
os.fchmod(descriptor, mode)
|
||||
os.fchown(descriptor, uid, gid)
|
||||
view = memoryview(payload)
|
||||
while view:
|
||||
written = os.write(descriptor, view)
|
||||
if written <= 0:
|
||||
raise OSError('short write')
|
||||
view = view[written:]
|
||||
os.fsync(descriptor)
|
||||
os.close(descriptor)
|
||||
descriptor = None
|
||||
if not replace and path.exists():
|
||||
os.unlink(temporary)
|
||||
return
|
||||
os.replace(temporary, path)
|
||||
parent = os.open(path.parent, os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0))
|
||||
try:
|
||||
os.fsync(parent)
|
||||
finally:
|
||||
os.close(parent)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except OSError:
|
||||
pass
|
||||
raise InstallError('write') from None
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
payload = None
|
||||
|
||||
|
||||
def validate(*, require_socket=True):
|
||||
if sys.platform != 'linux' or not hasattr(os, 'geteuid') or os.geteuid() != 0:
|
||||
raise InstallError('root')
|
||||
_root_directory(PROJECT.parent)
|
||||
_root_directory(INSTALL_ROOT.parent)
|
||||
if not _runtime_group_exists():
|
||||
raise InstallError('group')
|
||||
_details(PROJECT, directory=True, uid=0, gid=0, mode=0o755)
|
||||
_details(DEPLOY, directory=True, uid=0, gid=0, mode=0o755)
|
||||
_details(INSTALL_ROOT, directory=True, uid=0, gid=0, mode=0o755)
|
||||
_secure_tree(PROJECT / 'app')
|
||||
_details(ACTIVE, directory=True, uid=0, gid=0, mode=0o755)
|
||||
_details(WORKER_PACKAGES, directory=True, uid=0, gid=0, mode=0o755)
|
||||
_details(ACTIVE / 'config.yaml', directory=False, uid=RUNTIME_UID, gid=RUNTIME_GID, mode=0o600)
|
||||
_details(ACTIVE / 'secrets.yaml', directory=False, uid=RUNTIME_UID, gid=RUNTIME_GID, mode=0o600)
|
||||
layouts = (
|
||||
('/var/lib/truf/runtime-document-candidates', RUNTIME_UID, RUNTIME_GID, 0o700),
|
||||
('/var/lib/truf/host-agent', 0, 0, 0o700),
|
||||
('/var/lib/truf/host-agent/backups', 0, 0, 0o700),
|
||||
('/var/lib/truf/host-agent/operations', 0, 0, 0o700),
|
||||
('/var/lib/truf/host-agent/results', 0, RUNTIME_GID, 0o750),
|
||||
('/run/truf-postgres', RUNTIME_UID, RUNTIME_GID, 0o700),
|
||||
)
|
||||
for path, uid, gid, mode in layouts:
|
||||
_details(Path(path), directory=True, uid=uid, gid=gid, mode=mode)
|
||||
for executable in (
|
||||
'/usr/bin/python3', '/usr/bin/docker', '/usr/bin/systemctl',
|
||||
'/usr/bin/systemd-analyze', '/usr/bin/systemd-tmpfiles',
|
||||
'/usr/sbin/groupadd',
|
||||
):
|
||||
_secure_executable(executable)
|
||||
for unit in UNITS:
|
||||
_details(SYSTEMD / unit, directory=False, uid=0, gid=0, mode=0o644)
|
||||
_same_file(SYSTEMD / unit, DEPLOY / unit)
|
||||
_details(TMPFILES, directory=False, uid=0, gid=0, mode=0o644)
|
||||
_same_file(TMPFILES, DEPLOY / 'truf-host-agent.conf')
|
||||
for script in SCRIPTS:
|
||||
_details(INSTALL_ROOT / script, directory=False, uid=0, gid=0, mode=0o755)
|
||||
_same_file(INSTALL_ROOT / script, DEPLOY / script)
|
||||
profile = _deployment_profile()
|
||||
for compose_file in profile['compose_files']:
|
||||
_read_source(PROJECT / compose_file)
|
||||
try:
|
||||
docker_socket = os.stat('/run/docker.sock', follow_symlinks=False)
|
||||
except OSError:
|
||||
raise InstallError('socket') from None
|
||||
if (
|
||||
not stat.S_ISSOCK(docker_socket.st_mode)
|
||||
or docker_socket.st_uid != 0
|
||||
or stat.S_IMODE(docker_socket.st_mode) & 0o002
|
||||
):
|
||||
raise InstallError('socket')
|
||||
if require_socket:
|
||||
_validate_agent_socket()
|
||||
_run(('/usr/bin/python3', '-I', '-B', '-c', 'import psycopg, yaml'))
|
||||
_run(('/usr/bin/docker', 'compose', 'version'))
|
||||
_run(('/usr/bin/systemd-analyze', 'verify', *(str(SYSTEMD / unit) for unit in UNITS)))
|
||||
_validate_compose_projection(
|
||||
_capture(_compose_config_command(profile)), profile,
|
||||
)
|
||||
|
||||
|
||||
def install():
|
||||
if sys.platform != 'linux' or not hasattr(os, 'geteuid') or os.geteuid() != 0:
|
||||
raise InstallError('root')
|
||||
_ensure_runtime_group()
|
||||
for unit in UNITS:
|
||||
if _unit_active(unit):
|
||||
_run(('/usr/bin/systemctl', 'stop', unit))
|
||||
_ensure_install_root()
|
||||
for script in SCRIPTS:
|
||||
_write(INSTALL_ROOT / script, _read_source(DEPLOY / script), uid=0, gid=0, mode=0o755, replace=True)
|
||||
for unit in UNITS:
|
||||
_write(SYSTEMD / unit, _read_source(DEPLOY / unit), uid=0, gid=0, mode=0o644, replace=True)
|
||||
_write(TMPFILES, _read_source(DEPLOY / 'truf-host-agent.conf'), uid=0, gid=0, mode=0o644, replace=True)
|
||||
_run(('/usr/bin/systemd-tmpfiles', '--create', str(TMPFILES)))
|
||||
_write(
|
||||
ACTIVE / 'config.yaml', _read_source(PROJECT / 'app/config.linux.yaml'),
|
||||
uid=RUNTIME_UID, gid=RUNTIME_GID, mode=0o600, replace=False,
|
||||
)
|
||||
_write(
|
||||
ACTIVE / 'secrets.yaml', b'{}\n', uid=RUNTIME_UID, gid=RUNTIME_GID,
|
||||
mode=0o600, replace=False,
|
||||
)
|
||||
validate(require_socket=False)
|
||||
_run(('/usr/bin/systemctl', 'daemon-reload'))
|
||||
_run(('/usr/bin/systemctl', 'enable', 'truf-host-agent.socket'))
|
||||
_run(('/usr/bin/systemctl', 'restart', 'truf-host-agent.socket'))
|
||||
_validate_agent_socket()
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in ('install', 'validate'):
|
||||
raise InstallError('arguments')
|
||||
install() if sys.argv[1] == 'install' else validate()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
result = main()
|
||||
except Exception as exc:
|
||||
print(
|
||||
'host-agent deployment failed (' + type(exc).__name__ + '); details withheld',
|
||||
file=sys.stderr,
|
||||
)
|
||||
result = 1
|
||||
raise SystemExit(result)
|
||||
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Expire Truf Caddy admin-only denylist entries
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
Group=root
|
||||
ExecStart=/usr/local/sbin/truf-caddy-admin-denylist expire
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/truf-edge /etc/truf-edge/denylist
|
||||
RestrictAddressFamilies=AF_UNIX
|
||||
LockPersonality=true
|
||||
MemoryDenyWriteExecute=true
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Expire Truf Caddy admin-only denylist entries every minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=1min
|
||||
Persistent=true
|
||||
AccuracySec=10s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,27 @@
|
||||
name: truf-worker
|
||||
|
||||
services:
|
||||
worker:
|
||||
image: truf-remote-worker:linux-x86_64
|
||||
container_name: truf-worker
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 256
|
||||
stop_grace_period: 10m
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,nodev,noexec,size=128m,mode=1777
|
||||
environment:
|
||||
XDG_DATA_HOME: /data/client
|
||||
XDG_STATE_HOME: /data/state-base
|
||||
volumes:
|
||||
- truf-worker-data:/data
|
||||
command:
|
||||
- run
|
||||
|
||||
volumes:
|
||||
truf-worker-data:
|
||||
name: truf-worker-data
|
||||
@@ -0,0 +1,3 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
& docker compose exec worker /opt/truf-worker/truf-worker @args
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec docker compose exec worker /opt/truf-worker/truf-worker "$@"
|
||||
Reference in New Issue
Block a user