rene.fichtmueller/transceiver-db
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): Fleet-Crash-Loop-Watchdog — Pi-Heartbeat (fleet_health) + Erik-Detector (v_fleet_health) schließt den Self-Healing-Blindspot (Crash-Loop tarnt sich als online)

Browse Source
This commit is contained in:
Rene Fichtmueller 2026-07-07 23:02:20 +02:00
parent 4806cdfd1c
commit 8d11a4d24d
2 changed files with 64 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/fleet-health-monitor.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the
# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot
# where a crash-looping pm2 process reports "online" and a naive liveness check
# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way.
# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL).
set -uo pipefail
LOG=/opt/tip/logs/fleet-health.log
mkdir -p "$(dirname "$LOG")"
TS=$(date '+%Y-%m-%d %H:%M:%S')
# shellcheck disable=SC1091
source /opt/tip/.env 2>/dev/null || true
Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host"
ALERTS=$(docker exec -e PGPASSWORD="${POSTGRES_PASSWORD:-}" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null)
if [ -n "$ALERTS" ]; then
N=$(printf '%s\n' "$ALERTS" | grep -c .)
echo "[$TS] FLEET ALERT ($N):" >> "$LOG"
printf '%s\n' "$ALERTS" | sed 's/^/[ ] /' >> "$LOG"
# best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet)
if [ -n "${FLEET_NTFY_URL:-}" ]; then
printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true
fi
else
echo "[$TS] fleet ok" >> "$LOG"
fi

37
scripts/fleet-heartbeat.mjs Normal file
View File

@ -0,0 +1,37 @@
/**
* Pi/host fleet heartbeat: self-report local pm2 process health to Postgres so a
* central detector on Erik can catch crash-loops (restart_time velocity + near-0
* uptime) and stale/dead hosts — which a naive "pm2 status == online" check misses.
* Runs on each Pi via cron (every 5 min). DB via the local tunnel (127.0.0.1:5433).
*/
import { execFileSync } from "node:child_process";
import os from "node:os";
import { config } from "dotenv";
import pg from "pg";
config({ path: new URL("../.env", import.meta.url) }); // ~/tip/.env
const host = os.hostname();
let procs = [];
try { procs = JSON.parse(execFileSync("pm2", ["jlist"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); }
catch { process.exit(0); }
const pool = new pg.Pool({
host: "127.0.0.1", port: 5433,
user: process.env.POSTGRES_USER || "tip",
database: process.env.POSTGRES_DB || "transceiver_db",
password: process.env.POSTGRES_PASSWORD,
max: 1, connectionTimeoutMillis: 8000, statement_timeout: 8000,
});
try {
for (const p of procs) {
const e = p.pm2_env || {};
const uptimeS = Math.round((Date.now() - (e.pm_uptime || Date.now())) / 1000);
await pool.query(
"insert into fleet_health(host,pm_name,status,restart_time,uptime_s) values($1,$2,$3,$4,$5)",
[host, p.name, e.status || null, e.restart_time ?? 0, uptimeS],
);
}
} catch { /* heartbeat is best-effort; never crash the host */ }
finally { await pool.end().catch(() => {}); }