Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,41 @@
|
||||
ARG BASE_IMAGE=truf-local:runtime
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/capacity_model.py /opt/truf/app/capacity_model.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/scanner_db.py /opt/truf/app/scanner_db.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/worker_assignment.py /opt/truf/app/worker_assignment.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/worker_api.py /opt/truf/app/worker_api.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/jsonl_projector.py /opt/truf/app/jsonl_projector.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/runtime_document.py /opt/truf/app/runtime_document.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/lifecycle_authority.py /opt/truf/app/lifecycle_authority.py
|
||||
COPY --chown=10001:10001 --chmod=0600 payload/app/config.linux.yaml /opt/truf/app/config.linux.yaml
|
||||
|
||||
RUN python3 -I -S -B - <<'PY'
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
root = Path('/opt/truf/app')
|
||||
names = (
|
||||
'capacity_model.py',
|
||||
'scanner_db.py',
|
||||
'worker_assignment.py',
|
||||
'worker_api.py',
|
||||
'jsonl_projector.py',
|
||||
'runtime_document.py',
|
||||
'lifecycle_authority.py',
|
||||
)
|
||||
for name in names:
|
||||
ast.parse((root / name).read_text(encoding='utf-8'), filename=name)
|
||||
sys.path.insert(0, str(root))
|
||||
from capacity_model import ( # noqa: E402
|
||||
MAX_RESULT_BUNDLE_BYTES,
|
||||
REMOTE_ASSIGNMENT_BASELINE_BYTES,
|
||||
REMOTE_ASSIGNMENT_MAX_ACTIVE,
|
||||
)
|
||||
assert (MAX_RESULT_BUNDLE_BYTES, REMOTE_ASSIGNMENT_BASELINE_BYTES, REMOTE_ASSIGNMENT_MAX_ACTIVE) == (
|
||||
64 * 1024 * 1024,
|
||||
2 * 1024 * 1024,
|
||||
50,
|
||||
)
|
||||
PY
|
||||
@@ -0,0 +1,481 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
MODE="${1:-}"
|
||||
STAGE="${2:-}"
|
||||
if [[ "$MODE" != plan && "$MODE" != apply ]]; then
|
||||
echo 'usage: deploy.sh plan|apply STAGE' >&2
|
||||
exit 64
|
||||
fi
|
||||
if [[ ! "$STAGE" =~ ^/var/lib/truf-deploy/stage/capacity50\.[A-Za-z0-9]+$ ]] || [[ ! -d "$STAGE" ]]; then
|
||||
echo 'invalid deployment stage' >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
readonly RELEASE_ID='capacity50-20260930'
|
||||
readonly EXPECTED_IMAGE='sha256:46f1cf1b92d1a7d93d06f690309d8c7eca45f64dc45ca310861c70fb419bb035'
|
||||
readonly EXPECTED_CONFIG_SHA256='12bd9a60cc56c3d6cbad18435523e8229b0cd8fdccc2d73922ea7e580ce7441c'
|
||||
readonly CANDIDATE_TAG="truf-local:runtime-${RELEASE_ID}"
|
||||
readonly ROLLBACK_TAG="truf-local:runtime-pre-${RELEASE_ID}"
|
||||
readonly ACTIVE_CONFIG='/etc/truf/runtime/config.yaml'
|
||||
readonly ACTIVE_SECRETS='/etc/truf/runtime/secrets.yaml'
|
||||
readonly SOURCE_ROOT='/opt/truf'
|
||||
readonly HISTORY="/var/lib/truf-deploy/history/${RELEASE_ID}"
|
||||
readonly TEST_USER='operator-trace-windows-20260925'
|
||||
readonly TEST_USER_ORIGINAL_CAP='2'
|
||||
readonly APP_FILES=(
|
||||
capacity_model.py
|
||||
scanner_db.py
|
||||
worker_assignment.py
|
||||
worker_api.py
|
||||
jsonl_projector.py
|
||||
runtime_document.py
|
||||
lifecycle_authority.py
|
||||
config.linux.yaml
|
||||
)
|
||||
readonly COMPOSE=(
|
||||
docker compose
|
||||
--project-name truf-docker
|
||||
--project-directory /opt/truf
|
||||
--env-file /etc/truf-edge/edge.env
|
||||
--file /opt/truf/compose.yaml
|
||||
--file /opt/truf/compose.shared-host.yaml
|
||||
)
|
||||
|
||||
PHASE='preflight'
|
||||
MUTATED=0
|
||||
PHASE_A_HEALTHY=0
|
||||
SOURCE_INSTALLED=0
|
||||
USER_CAP_CHANGED=0
|
||||
DEPLOY_SUCCEEDED=0
|
||||
RESUME=0
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$RELEASE_ID" "$*"
|
||||
}
|
||||
|
||||
runtime_id() {
|
||||
"${COMPOSE[@]}" ps --quiet runtime
|
||||
}
|
||||
|
||||
psql() {
|
||||
local container
|
||||
container="$(runtime_id)"
|
||||
[[ -n "$container" ]] || return 1
|
||||
docker exec "$container" /usr/lib/postgresql/16/bin/psql \
|
||||
-h /run/truf-postgres -U truf -d truf -v ON_ERROR_STOP=1 -At "$@"
|
||||
}
|
||||
|
||||
current_image() {
|
||||
docker image inspect --format '{{.Id}}' truf-local:runtime
|
||||
}
|
||||
|
||||
config_sha256() {
|
||||
sha256sum "$ACTIVE_CONFIG" | cut -d' ' -f1
|
||||
}
|
||||
|
||||
require_baseline() {
|
||||
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
|
||||
echo 'runtime image identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
[[ "$(config_sha256)" == "$EXPECTED_CONFIG_SHA256" ]] || {
|
||||
echo 'active config identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
local state
|
||||
state="$(psql -F '|' -c \
|
||||
"SELECT revision, discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
if ((RESUME)); then
|
||||
[[ "$state" =~ ^[0-9]+\|0\|0\|(normal|drained)$ ]] || {
|
||||
echo "resumed runtime control is neither open nor drained: $state" >&2
|
||||
return 1
|
||||
}
|
||||
elif [[ ! "$state" =~ ^[0-9]+\|0\|0\|normal$ ]]; then
|
||||
echo "runtime control is not open: $state" >&2
|
||||
return 1
|
||||
fi
|
||||
local debt
|
||||
debt="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
[[ "$debt" == '0|0|0|0|0|0|0|0|0' ]] || {
|
||||
echo "pipeline is not reconciled: $debt" >&2
|
||||
return 1
|
||||
}
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}' AND disabled_at IS NULL;")" == "$TEST_USER_ORIGINAL_CAP" ]] || {
|
||||
echo 'temporary validation user identity changed' >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
edge_value() {
|
||||
local name="$1"
|
||||
sed -n "s/^${name}=//p" /etc/truf-edge/edge.env
|
||||
}
|
||||
|
||||
admin_material() {
|
||||
local marker host page token revision
|
||||
marker="$(edge_value TRUF_ADMIN_EDGE_MARKER)"
|
||||
host="$(edge_value TRUF_EDGE_HOST)"
|
||||
[[ "$marker" =~ ^[a-f0-9]{64}$ ]] || return 1
|
||||
[[ "$host" =~ ^[A-Za-z0-9.-]+$ ]] || return 1
|
||||
page="$(curl --fail --silent --show-error --max-time 20 \
|
||||
--header "X-Truf-Admin-Edge: ${marker}" \
|
||||
--header 'X-Truf-Admin-Operator: deploy-runtime' \
|
||||
http://127.0.0.1:8766/admin-internal/)"
|
||||
token="$(python3 -c \
|
||||
'import re,sys; values=set(re.findall(r"name=\"csrf_token\" value=\"([^\"]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
|
||||
<<<"$page")"
|
||||
revision="$(python3 -c \
|
||||
'import re,sys; values=set(re.findall(r"name=\"expected_revision\" value=\"([0-9]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
|
||||
<<<"$page")"
|
||||
[[ "$token" =~ ^[A-Za-z0-9_-]{32,128}$ && "$revision" =~ ^[0-9]+$ ]] || return 1
|
||||
printf '%s|%s|%s|%s\n' "$marker" "$host" "$token" "$revision"
|
||||
}
|
||||
|
||||
admin_post() {
|
||||
local route="$1"
|
||||
shift
|
||||
local material marker host token revision operation
|
||||
material="$(admin_material)"
|
||||
IFS='|' read -r marker host token revision <<<"$material"
|
||||
operation="$(cat /proc/sys/kernel/random/uuid)"
|
||||
local arguments=(
|
||||
--fail --silent --show-error --max-time 30
|
||||
--request POST
|
||||
--header "X-Truf-Admin-Edge: ${marker}"
|
||||
--header 'X-Truf-Admin-Operator: deploy-runtime'
|
||||
--header "Origin: https://${host}"
|
||||
--header 'Content-Type: application/x-www-form-urlencoded'
|
||||
--data-urlencode "csrf_token=${token}"
|
||||
--data-urlencode "operation_id=${operation}"
|
||||
)
|
||||
if [[ "$route" == dispatch/* || "$route" == search/discovery/* ]]; then
|
||||
arguments+=(--data-urlencode "expected_revision=${revision}")
|
||||
fi
|
||||
while (($#)); do
|
||||
arguments+=(--data-urlencode "$1")
|
||||
shift
|
||||
done
|
||||
curl "${arguments[@]}" "http://127.0.0.1:8766/admin-internal/${route}" >/dev/null
|
||||
}
|
||||
|
||||
wait_for_drain() {
|
||||
local deadline=$((SECONDS + 600)) state debt
|
||||
while ((SECONDS < deadline)); do
|
||||
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
debt="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
if [[ "$state" == drained && "$debt" == '0|0|0|0|0|0|0|0|0' ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo 'runtime did not drain within 600 seconds' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
quiesce_pipeline_workers() {
|
||||
local source deadline active container
|
||||
for source in result-ingester jsonl-projector; do
|
||||
admin_post supervisor/sources/stop "source_id=${source}"
|
||||
done
|
||||
container="$(runtime_id)"
|
||||
[[ -n "$container" ]] || return 1
|
||||
docker exec --interactive "$container" /usr/local/bin/python3 -I -S -B - \
|
||||
<"$STAGE/release_stopped_pipeline_leases.py"
|
||||
deadline=$((SECONDS + 120))
|
||||
while ((SECONDS < deadline)); do
|
||||
active="$(psql -c \
|
||||
"SELECT count(*) FROM pipeline_leases WHERE state NOT IN ('released','failed');")"
|
||||
if [[ "$active" == 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo 'pipeline worker leases did not release within 120 seconds' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
install_config() {
|
||||
local source="$1" temporary
|
||||
temporary="/etc/truf/runtime/.config.yaml.${RELEASE_ID}.tmp"
|
||||
install -o root -g root -m 0600 "$source" "$temporary"
|
||||
chown 10001:10001 "$temporary"
|
||||
mv -f "$temporary" "$ACTIVE_CONFIG"
|
||||
}
|
||||
|
||||
stop_stack() {
|
||||
"${COMPOSE[@]}" stop --timeout 30 edge
|
||||
"${COMPOSE[@]}" stop --timeout 600 runtime
|
||||
local container state
|
||||
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
|
||||
state="$(docker inspect --format '{{.State.Status}}|{{.State.ExitCode}}|{{.State.OOMKilled}}' "$container")"
|
||||
[[ "$state" == 'exited|0|false' ]] || {
|
||||
echo "runtime stop was not clean: $state" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
wait_runtime_health() {
|
||||
local deadline=$((SECONDS + 420)) container state status
|
||||
while ((SECONDS < deadline)); do
|
||||
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
|
||||
if [[ -n "$container" ]]; then
|
||||
state="$(docker inspect --format '{{.State.Status}}' "$container")"
|
||||
[[ "$state" != exited && "$state" != dead ]] || return 1
|
||||
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")"
|
||||
if [[ "$status" == healthy ]] && docker exec "$container" \
|
||||
/usr/local/bin/python3 -I -S -B /opt/truf/app/container_runtime.py \
|
||||
health --config /data/config/config.yaml --require-worker-api >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
[[ "$status" != unhealthy ]] || return 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo 'runtime health timed out' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
start_stack() {
|
||||
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate runtime
|
||||
wait_runtime_health
|
||||
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate edge
|
||||
sleep 3
|
||||
local edge_container edge_state host public_code
|
||||
edge_container="$("${COMPOSE[@]}" ps --quiet edge)"
|
||||
edge_state="$(docker inspect --format '{{.State.Status}}|{{.State.Running}}|{{.State.OOMKilled}}' "$edge_container")"
|
||||
[[ "$edge_state" == 'running|true|false' ]] || return 1
|
||||
host="$(edge_value TRUF_EDGE_HOST)"
|
||||
public_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
--max-time 20 "https://${host}/")"
|
||||
[[ "$public_code" == 401 || "$public_code" == 404 ]] || {
|
||||
echo "unexpected public edge response: $public_code" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
validate_candidate_config() {
|
||||
local path="$1"
|
||||
docker run --rm --network none --read-only --user 10001:10001 \
|
||||
--cap-drop ALL --security-opt no-new-privileges:true \
|
||||
--tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \
|
||||
--volume "$path:/data/config/config.yaml:ro" \
|
||||
--volume "$ACTIVE_SECRETS:/data/config/secrets.yaml:ro" \
|
||||
--volume /etc/truf/worker-packages:/data/worker-packages:ro \
|
||||
--entrypoint /usr/local/bin/python3 "$CANDIDATE_TAG" -I -S -B -c \
|
||||
"import sys,sysconfig;sys.path.append(sysconfig.get_paths()['purelib']);sys.path.insert(0,'/opt/truf/app');from runtime_document_io import validate_managed_runtime_files;print(validate_managed_runtime_files('/data/config/config.yaml').config_sha256)" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
install_sources() {
|
||||
local name destination temporary
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
destination="${SOURCE_ROOT}/app/${name}"
|
||||
temporary="${destination}.${RELEASE_ID}.tmp"
|
||||
install -o root -g root -m 0644 "$STAGE/payload/app/$name" "$temporary"
|
||||
mv -f "$temporary" "$destination"
|
||||
done
|
||||
SOURCE_INSTALLED=1
|
||||
}
|
||||
|
||||
restore_sources() {
|
||||
local name
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
if [[ -f "$HISTORY/source/$name" ]]; then
|
||||
install -o root -g root -m 0644 "$HISTORY/source/$name" "${SOURCE_ROOT}/app/$name"
|
||||
else
|
||||
rm -f "${SOURCE_ROOT}/app/$name"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_user_cap() {
|
||||
if ((USER_CAP_CHANGED)); then
|
||||
admin_post users/cap "user_key=${TEST_USER}" "active_assignment_cap=${TEST_USER_ORIGINAL_CAP}" || true
|
||||
USER_CAP_CHANGED=0
|
||||
fi
|
||||
}
|
||||
|
||||
cancel_drain() {
|
||||
local state
|
||||
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;" 2>/dev/null || true)"
|
||||
if [[ "$state" == draining || "$state" == drained ]]; then
|
||||
admin_post dispatch/drain/cancel || return 1
|
||||
fi
|
||||
}
|
||||
|
||||
rollback() {
|
||||
set +e
|
||||
log "rollback from phase ${PHASE}"
|
||||
restore_user_cap
|
||||
stop_stack
|
||||
if ((PHASE_A_HEALTHY)); then
|
||||
docker image tag "$CANDIDATE_TAG" truf-local:runtime
|
||||
install_config "$HISTORY/config.conservative.yaml"
|
||||
else
|
||||
docker image tag "$EXPECTED_IMAGE" truf-local:runtime
|
||||
install_config "$HISTORY/config.original.yaml"
|
||||
fi
|
||||
((SOURCE_INSTALLED)) && restore_sources
|
||||
if start_stack; then
|
||||
cancel_drain
|
||||
log 'rollback restored a healthy runtime'
|
||||
else
|
||||
log 'rollback could not prove health; runtime remains contained' >&2
|
||||
fi
|
||||
set -e
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local code=$?
|
||||
trap - EXIT ERR INT TERM
|
||||
if ((code != 0 && MUTATED && !DEPLOY_SUCCEEDED)); then
|
||||
rollback
|
||||
fi
|
||||
exit "$code"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
exec 9>/run/lock/truf-runtime-deploy.lock
|
||||
flock -n 9 || {
|
||||
echo 'another runtime deployment is active' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ "$MODE" == apply && -d "$HISTORY" ]] \
|
||||
&& docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1 \
|
||||
&& [[ "$(docker image inspect --format '{{.Id}}' "$ROLLBACK_TAG" 2>/dev/null || true)" == "$EXPECTED_IMAGE" ]]; then
|
||||
RESUME=1
|
||||
fi
|
||||
require_baseline
|
||||
available_kb="$(df -Pk /var/lib/docker | awk 'NR==2 {print $4}')"
|
||||
[[ "$available_kb" =~ ^[0-9]+$ && "$available_kb" -ge 786432 ]] || {
|
||||
echo 'less than 768 MiB is available for the derived image' >&2
|
||||
exit 1
|
||||
}
|
||||
log "plan image=${EXPECTED_IMAGE#sha256:} config=${EXPECTED_CONFIG_SHA256} free_kib=${available_kb}"
|
||||
if [[ "$MODE" == plan ]]; then
|
||||
log 'plan passed; no runtime state changed'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ((RESUME)); then
|
||||
log 'resuming a verified pre-cutover release'
|
||||
[[ "$(sha256sum "$HISTORY/config.original.yaml" | cut -d' ' -f1)" == "$EXPECTED_CONFIG_SHA256" ]] || exit 1
|
||||
[[ -f "$HISTORY/config.conservative.yaml" && -f "$HISTORY/config.final.yaml" ]] || exit 1
|
||||
validate_candidate_config "$HISTORY/config.conservative.yaml"
|
||||
validate_candidate_config "$HISTORY/config.final.yaml"
|
||||
if [[ "$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")" == normal ]]; then
|
||||
admin_post dispatch/drain/start
|
||||
MUTATED=1
|
||||
wait_for_drain
|
||||
else
|
||||
MUTATED=1
|
||||
fi
|
||||
else
|
||||
install -d -o root -g root -m 0700 /var/lib/truf-deploy /var/lib/truf-deploy/history
|
||||
if [[ -e "$HISTORY" ]]; then
|
||||
echo 'release history already exists' >&2
|
||||
exit 1
|
||||
fi
|
||||
install -d -o root -g root -m 0700 "$HISTORY" "$HISTORY/source"
|
||||
install -o root -g root -m 0600 "$ACTIVE_CONFIG" "$HISTORY/config.original.yaml"
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
if [[ -f "${SOURCE_ROOT}/app/$name" ]]; then
|
||||
install -o root -g root -m 0600 "${SOURCE_ROOT}/app/$name" "$HISTORY/source/$name"
|
||||
fi
|
||||
done
|
||||
|
||||
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
|
||||
--output "$HISTORY/config.conservative.yaml" --mode conservative \
|
||||
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
|
||||
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
|
||||
--output "$HISTORY/config.final.yaml" --mode final \
|
||||
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
|
||||
chown 10001:10001 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
|
||||
chmod 0600 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
|
||||
|
||||
if docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1; then
|
||||
echo 'candidate image tag already exists' >&2
|
||||
exit 1
|
||||
fi
|
||||
PHASE='candidate-build'
|
||||
docker build --network none --build-arg BASE_IMAGE=truf-local:runtime \
|
||||
--file "$STAGE/Dockerfile" --tag "$CANDIDATE_TAG" "$STAGE"
|
||||
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
|
||||
echo 'runtime tag changed during candidate build' >&2
|
||||
exit 1
|
||||
}
|
||||
validate_candidate_config "$HISTORY/config.conservative.yaml"
|
||||
validate_candidate_config "$HISTORY/config.final.yaml"
|
||||
docker image tag "$EXPECTED_IMAGE" "$ROLLBACK_TAG"
|
||||
|
||||
PHASE='drain'
|
||||
admin_post dispatch/drain/start
|
||||
MUTATED=1
|
||||
wait_for_drain
|
||||
fi
|
||||
|
||||
PHASE='conservative-cutover'
|
||||
quiesce_pipeline_workers
|
||||
stop_stack
|
||||
install_config "$HISTORY/config.conservative.yaml"
|
||||
docker image tag "$CANDIDATE_TAG" truf-local:runtime
|
||||
start_stack
|
||||
schema_state="$(psql -F '|' -c \
|
||||
"SELECT (SELECT count(*) FROM information_schema.columns WHERE table_name='result_reservations' AND column_name='reserved_bundle_bytes'), (SELECT count(*) FROM runtime_schema_migrations WHERE version='20260930_33_remote_assignment_capacity');")"
|
||||
[[ "$schema_state" == '1|1' ]] || {
|
||||
echo "capacity migration was not applied: $schema_state" >&2
|
||||
exit 1
|
||||
}
|
||||
PHASE_A_HEALTHY=1
|
||||
|
||||
PHASE='capacity50-cutover'
|
||||
quiesce_pipeline_workers
|
||||
stop_stack
|
||||
install_config "$HISTORY/config.final.yaml"
|
||||
start_stack
|
||||
|
||||
final_values="$(psql -F '|' -c \
|
||||
"SELECT bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
|
||||
[[ "$final_values" == '0|0|0|0|0|0|0|0' ]] || {
|
||||
echo "post-deploy capacity is not reconciled: $final_values" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
PHASE='temporary-user-cap-validation'
|
||||
admin_post users/cap "user_key=${TEST_USER}" 'active_assignment_cap=50'
|
||||
USER_CAP_CHANGED=1
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == 50 ]] || exit 1
|
||||
restore_user_cap
|
||||
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == "$TEST_USER_ORIGINAL_CAP" ]] || exit 1
|
||||
|
||||
PHASE='source-install'
|
||||
install_sources
|
||||
for name in "${APP_FILES[@]}"; do
|
||||
cmp -s "$STAGE/payload/app/$name" "${SOURCE_ROOT}/app/$name" || exit 1
|
||||
done
|
||||
|
||||
PHASE='resume'
|
||||
cancel_drain
|
||||
post_control="$(psql -F '|' -c \
|
||||
"SELECT discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
|
||||
[[ "$post_control" == '0|0|normal' ]] || {
|
||||
echo "runtime control did not resume: $post_control" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
cat >"$HISTORY/result.txt" <<EOF
|
||||
release=${RELEASE_ID}
|
||||
runtime_image=$(current_image)
|
||||
config_sha256=$(config_sha256)
|
||||
schema=${schema_state}
|
||||
capacity=${final_values}
|
||||
control=${post_control}
|
||||
EOF
|
||||
chmod 0600 "$HISTORY/result.txt"
|
||||
DEPLOY_SUCCEEDED=1
|
||||
log "applied image=$(current_image) config=$(config_sha256)"
|
||||
@@ -0,0 +1,74 @@
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
|
||||
|
||||
for path in glob.glob('/proc/[0-9]*/cmdline'):
|
||||
try:
|
||||
command = open(path, 'rb').read().replace(b'\0', b' ')
|
||||
except (FileNotFoundError, PermissionError, ProcessLookupError):
|
||||
continue
|
||||
if b'/opt/truf/app/result_ingester.py' in command or b'/opt/truf/app/jsonl_projector.py' in command:
|
||||
raise SystemExit('a pipeline worker process is still active')
|
||||
|
||||
sys.path.append(sysconfig.get_paths()['purelib'])
|
||||
sys.path.insert(0, '/opt/truf/app')
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from db_backend import DatabaseConnection
|
||||
from scanner_db import (
|
||||
PIPELINE_ADVISORY_CLASS,
|
||||
PIPELINE_ADVISORY_OBJECTS,
|
||||
ScannerDB,
|
||||
)
|
||||
|
||||
|
||||
connection = psycopg.connect(
|
||||
dbname='truf',
|
||||
user='truf',
|
||||
host='/run/truf-postgres',
|
||||
row_factory=dict_row,
|
||||
options='-c search_path=public -c statement_timeout=30000 -c lock_timeout=5000',
|
||||
)
|
||||
database = ScannerDB(enabled=False)
|
||||
database.conn = DatabaseConnection('postgres', connection, application_schema='public')
|
||||
released = 0
|
||||
try:
|
||||
rows = database.conn.execute(
|
||||
"SELECT worker_name, generation, lease_token FROM pipeline_leases "
|
||||
"WHERE state NOT IN ('released','failed') ORDER BY worker_name"
|
||||
).fetchall()
|
||||
expected = {'result_ingester', 'jsonl_projector'}
|
||||
if not {row['worker_name'] for row in rows} <= expected:
|
||||
raise RuntimeError('an unexpected pipeline lease is active')
|
||||
for row in rows:
|
||||
name = row['worker_name']
|
||||
advisory = PIPELINE_ADVISORY_OBJECTS[name]
|
||||
locked = database.conn.execute(
|
||||
'SELECT pg_try_advisory_lock(?, ?) AS acquired',
|
||||
(PIPELINE_ADVISORY_CLASS, advisory),
|
||||
).fetchone()
|
||||
if not locked or not locked['acquired']:
|
||||
raise RuntimeError('a stopped pipeline worker still owns its advisory lock')
|
||||
database._pipeline_advisory_held.add(name)
|
||||
if not database.release_pipeline_lease(
|
||||
name,
|
||||
row['generation'],
|
||||
row['lease_token'],
|
||||
state='released',
|
||||
error='deployment_quiesce',
|
||||
):
|
||||
raise RuntimeError('pipeline lease identity changed during release')
|
||||
released += 1
|
||||
remaining = database.conn.execute(
|
||||
"SELECT count(*) AS count FROM pipeline_leases "
|
||||
"WHERE state NOT IN ('released','failed')"
|
||||
).fetchone()['count']
|
||||
if remaining:
|
||||
raise RuntimeError('pipeline lease reconciliation is incomplete')
|
||||
print(f'released_pipeline_leases={released}')
|
||||
finally:
|
||||
database.close()
|
||||
@@ -0,0 +1,73 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
VALUES = {
|
||||
'conservative': {
|
||||
'remote_assignment_reserve_bytes': 64 * 1024 * 1024,
|
||||
'remote_assignment_max_active': 1,
|
||||
'keycheck_queue_max_items': 8192,
|
||||
'keycheck_queue_max_bytes': 64 * 1024 * 1024,
|
||||
},
|
||||
'final': {
|
||||
'remote_assignment_reserve_bytes': 2 * 1024 * 1024,
|
||||
'remote_assignment_max_active': 50,
|
||||
'keycheck_queue_max_items': 131072,
|
||||
'keycheck_queue_max_bytes': 128 * 1024 * 1024,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(payload, mode):
|
||||
text = payload.decode('utf-8', errors='strict')
|
||||
if '\r' in text:
|
||||
raise ValueError('configuration must use LF line endings')
|
||||
lines = text.splitlines(keepends=True)
|
||||
values = VALUES[mode]
|
||||
hard_limit = ' result_bundle_max_event_bytes: 67108864\n'
|
||||
if lines.count(hard_limit) != 1:
|
||||
raise ValueError('unexpected hard result limit')
|
||||
for field in ('remote_assignment_reserve_bytes', 'remote_assignment_max_active'):
|
||||
if any(line.startswith(f' {field}:') for line in lines):
|
||||
raise ValueError(f'active configuration already contains {field}')
|
||||
index = lines.index(hard_limit) + 1
|
||||
lines[index:index] = [
|
||||
f" remote_assignment_reserve_bytes: {values['remote_assignment_reserve_bytes']}\n",
|
||||
f" remote_assignment_max_active: {values['remote_assignment_max_active']}\n",
|
||||
]
|
||||
replacements = {
|
||||
' keycheck_queue_max_items: 8192\n': (
|
||||
f" keycheck_queue_max_items: {values['keycheck_queue_max_items']}\n"
|
||||
),
|
||||
' keycheck_queue_max_bytes: 67108864\n': (
|
||||
f" keycheck_queue_max_bytes: {values['keycheck_queue_max_bytes']}\n"
|
||||
),
|
||||
}
|
||||
for before, after in replacements.items():
|
||||
if lines.count(before) != 1:
|
||||
raise ValueError(f'unexpected active configuration field: {before.strip()}')
|
||||
lines[lines.index(before)] = after
|
||||
return ''.join(lines).encode('utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--input', required=True)
|
||||
parser.add_argument('--output', required=True)
|
||||
parser.add_argument('--mode', choices=tuple(VALUES), required=True)
|
||||
parser.add_argument('--expected-sha256', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
source = Path(args.input)
|
||||
destination = Path(args.output)
|
||||
payload = source.read_bytes()
|
||||
if hashlib.sha256(payload).hexdigest() != args.expected_sha256:
|
||||
raise SystemExit('active configuration identity changed')
|
||||
rendered = render(payload, args.mode)
|
||||
destination.write_bytes(rendered)
|
||||
print(hashlib.sha256(rendered).hexdigest())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user