29 lines
1.5 KiB
Bash
29 lines
1.5 KiB
Bash
#!/bin/bash
|
|
# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the
|
|
# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot
|
|
# where a crash-looping pm2 process reports "online" and a naive liveness check
|
|
# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way.
|
|
# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL).
|
|
set -uo pipefail
|
|
LOG=/opt/tip/logs/fleet-health.log
|
|
mkdir -p "$(dirname "$LOG")"
|
|
TS=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
# shellcheck disable=SC1091
|
|
source /opt/tip/.env 2>/dev/null || true
|
|
Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host"
|
|
PW="${POSTGRES_PASSWORD:-}" # pass DB password to docker via a short-named var (no literal)
|
|
ALERTS=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null)
|
|
|
|
if [ -n "$ALERTS" ]; then
|
|
N=$(printf '%s\n' "$ALERTS" | grep -c .)
|
|
echo "[$TS] FLEET ALERT ($N):" >> "$LOG"
|
|
printf '%s\n' "$ALERTS" | sed 's/^/[ ] /' >> "$LOG"
|
|
# best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet)
|
|
if [ -n "${FLEET_NTFY_URL:-}" ]; then
|
|
printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true
|
|
fi
|
|
else
|
|
echo "[$TS] fleet ok" >> "$LOG"
|
|
fi
|