#!/bin/bash
# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the
# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot
# where a crash-looping pm2 process reports "online" and a naive liveness check
# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way.
# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL).
set -uo pipefail
LOG=/opt/tip/logs/fleet-health.log
mkdir -p "$(dirname "$LOG")"
TS=$(date '+%Y-%m-%d %H:%M:%S')

# shellcheck disable=SC1091
source /opt/tip/.env 2>/dev/null || true
Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host"
PW="${POSTGRES_PASSWORD:-}"   # pass DB password to docker via a short-named var (no literal)
ALERTS=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null)

if [ -n "$ALERTS" ]; then
  N=$(printf '%s\n' "$ALERTS" | grep -c .)
  echo "[$TS] FLEET ALERT ($N):" >> "$LOG"
  printf '%s\n' "$ALERTS" | sed 's/^/[    ] /' >> "$LOG"
  # best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet)
  if [ -n "${FLEET_NTFY_URL:-}" ]; then
    printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true
  fi
else
  echo "[$TS] fleet ok" >> "$LOG"
fi