diff --git a/scripts/fleet-health-monitor.sh b/scripts/fleet-health-monitor.sh new file mode 100644 index 0000000..49eccda --- /dev/null +++ b/scripts/fleet-health-monitor.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the +# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot +# where a crash-looping pm2 process reports "online" and a naive liveness check +# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way. +# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL). +set -uo pipefail +LOG=/opt/tip/logs/fleet-health.log +mkdir -p "$(dirname "$LOG")" +TS=$(date '+%Y-%m-%d %H:%M:%S') + +# shellcheck disable=SC1091 +source /opt/tip/.env 2>/dev/null || true +Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host" +ALERTS=$(docker exec -e PGPASSWORD="${POSTGRES_PASSWORD:-}" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null) + +if [ -n "$ALERTS" ]; then + N=$(printf '%s\n' "$ALERTS" | grep -c .) + echo "[$TS] FLEET ALERT ($N):" >> "$LOG" + printf '%s\n' "$ALERTS" | sed 's/^/[ ] /' >> "$LOG" + # best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet) + if [ -n "${FLEET_NTFY_URL:-}" ]; then + printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true + fi +else + echo "[$TS] fleet ok" >> "$LOG" +fi diff --git a/scripts/fleet-heartbeat.mjs b/scripts/fleet-heartbeat.mjs new file mode 100644 index 0000000..7639a59 --- /dev/null +++ b/scripts/fleet-heartbeat.mjs @@ -0,0 +1,37 @@ +/** + * Pi/host fleet heartbeat: self-report local pm2 process health to Postgres so a + * central detector on Erik can catch crash-loops (restart_time velocity + near-0 + * uptime) and stale/dead hosts — which a naive "pm2 status == online" check misses. + * Runs on each Pi via cron (every 5 min). DB via the local tunnel (127.0.0.1:5433). + */ +import { execFileSync } from "node:child_process"; +import os from "node:os"; +import { config } from "dotenv"; +import pg from "pg"; + +config({ path: new URL("../.env", import.meta.url) }); // ~/tip/.env + +const host = os.hostname(); +let procs = []; +try { procs = JSON.parse(execFileSync("pm2", ["jlist"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); } +catch { process.exit(0); } + +const pool = new pg.Pool({ + host: "127.0.0.1", port: 5433, + user: process.env.POSTGRES_USER || "tip", + database: process.env.POSTGRES_DB || "transceiver_db", + password: process.env.POSTGRES_PASSWORD, + max: 1, connectionTimeoutMillis: 8000, statement_timeout: 8000, +}); + +try { + for (const p of procs) { + const e = p.pm2_env || {}; + const uptimeS = Math.round((Date.now() - (e.pm_uptime || Date.now())) / 1000); + await pool.query( + "insert into fleet_health(host,pm_name,status,restart_time,uptime_s) values($1,$2,$3,$4,$5)", + [host, p.name, e.status || null, e.restart_time ?? 0, uptimeS], + ); + } +} catch { /* heartbeat is best-effort; never crash the host */ } +finally { await pool.end().catch(() => {}); }