feat: more switch image coverage + system health metrics + Erik monitor
switch-image-fetcher: - Add Fortinet URL builder (11 FortiSwitch models) - Add Quanta Cloud Technology, Allied Telesis, Ufispace, Netgear URL builders - Fix alcatel-lucent-enterprise slug missing from URL_BUILDERS dispatcher - Fix NVIDIA builder to skip ConnectX/BlueField adapters (not switches) - Add aruba slug alias for hpe-aruba health endpoint: - Add system metrics: CPU load (1/5/15m), memory usage, disk usage - Add load_status indicator (ok/busy/overloaded) - Expose process RSS memory - Used by external monitors scripts/monitor-erik.sh: - Cron-ready health check script for Claudi (.82) and Raspberry Pis - Checks TIP API health endpoint (load, memory, disk, DB latency) - Checks PM2 process state via SSH (errored/stopped detection) - ntfy.sh push notifications (set NTFY_TOPIC env var) - Includes systemd service + timer unit comments for auto-install
This commit is contained in:
parent
823b64bd24
commit
9e6be570a3
@ -1,6 +1,8 @@
|
|||||||
import { Router, Request, Response } from "express";
|
import { Router, Request, Response } from "express";
|
||||||
import { getDbStats } from "../db/queries";
|
import { getDbStats } from "../db/queries";
|
||||||
import { pool } from "../db/client";
|
import { pool } from "../db/client";
|
||||||
|
import { loadavg, totalmem, freemem, cpus } from "os";
|
||||||
|
import { execSync } from "child_process";
|
||||||
|
|
||||||
export const healthRouter = Router();
|
export const healthRouter = Router();
|
||||||
|
|
||||||
@ -36,11 +38,45 @@ healthRouter.get("/", async (_req: Request, res: Response) => {
|
|||||||
`).catch(() => ({ rows: [{}] }));
|
`).catch(() => ({ rows: [{}] }));
|
||||||
const s = stockStats.rows[0] || {};
|
const s = stockStats.rows[0] || {};
|
||||||
|
|
||||||
|
// System metrics
|
||||||
|
const [load1, load5, load15] = loadavg();
|
||||||
|
const totalMem = totalmem();
|
||||||
|
const freeMem = freemem();
|
||||||
|
const usedMem = totalMem - freeMem;
|
||||||
|
const coreCount = cpus().length;
|
||||||
|
|
||||||
|
let diskUsedPct: number | null = null;
|
||||||
|
let diskFreeGb: number | null = null;
|
||||||
|
try {
|
||||||
|
const df = execSync("df -h / 2>/dev/null | tail -1", { timeout: 2000 }).toString().trim();
|
||||||
|
const parts = df.split(/\s+/);
|
||||||
|
diskUsedPct = parseInt(parts[4] ?? "0", 10) || null;
|
||||||
|
diskFreeGb = parseFloat(parts[3] ?? "0") || null;
|
||||||
|
} catch { /* skip on systems without df */ }
|
||||||
|
|
||||||
|
const loadStatus = load1 > coreCount * 0.9 ? "overloaded" : load1 > coreCount * 0.6 ? "busy" : "ok";
|
||||||
|
|
||||||
res.json({
|
res.json({
|
||||||
success: true,
|
success: true,
|
||||||
status: "healthy",
|
status: "healthy",
|
||||||
version: "0.3.0",
|
version: "0.3.0",
|
||||||
uptime: process.uptime(),
|
uptime: process.uptime(),
|
||||||
|
system: {
|
||||||
|
load: { "1m": +load1.toFixed(2), "5m": +load5.toFixed(2), "15m": +load15.toFixed(2) },
|
||||||
|
load_status: loadStatus,
|
||||||
|
cpu_cores: coreCount,
|
||||||
|
memory: {
|
||||||
|
total_mb: Math.round(totalMem / 1024 / 1024),
|
||||||
|
used_mb: Math.round(usedMem / 1024 / 1024),
|
||||||
|
free_mb: Math.round(freeMem / 1024 / 1024),
|
||||||
|
used_pct: Math.round(usedMem / totalMem * 100),
|
||||||
|
},
|
||||||
|
disk: {
|
||||||
|
used_pct: diskUsedPct,
|
||||||
|
free_gb: diskFreeGb,
|
||||||
|
},
|
||||||
|
process_rss_mb: Math.round(process.memoryUsage().rss / 1024 / 1024),
|
||||||
|
},
|
||||||
database: {
|
database: {
|
||||||
connected: true,
|
connected: true,
|
||||||
latency_ms: latencyMs,
|
latency_ms: latencyMs,
|
||||||
|
|||||||
@ -8,12 +8,14 @@
|
|||||||
* 4. Write image_url + product_page_url to switches table
|
* 4. Write image_url + product_page_url to switches table
|
||||||
*
|
*
|
||||||
* Vendors covered:
|
* Vendors covered:
|
||||||
* Cisco (Nexus 9000/9300, NCS 5500/5700, Catalyst 9300/9500)
|
* Cisco (Nexus 9000/9300, NCS 5500/5700, Catalyst 9300/9500, 8000 SP)
|
||||||
* Arista (7000 series)
|
* Arista (7000 series)
|
||||||
* Juniper (QFX, EX series)
|
* Juniper (QFX, EX series)
|
||||||
* NVIDIA Networking (Spectrum SN series)
|
* NVIDIA Networking (Spectrum SN series — ConnectX skipped)
|
||||||
* Edgecore, Celestica, Asterfusion (whitebox)
|
* Edgecore, Celestica, Asterfusion (whitebox)
|
||||||
|
* Fortinet (FortiSwitch series)
|
||||||
* Dell, HPE/Aruba, Huawei, Nokia, Extreme, MikroTik, Ubiquiti, FS.COM, Supermicro
|
* Dell, HPE/Aruba, Huawei, Nokia, Extreme, MikroTik, Ubiquiti, FS.COM, Supermicro
|
||||||
|
* Alcatel-Lucent Enterprise, Allied Telesis, Netgear, Quanta Cloud Technology, Ufispace
|
||||||
*
|
*
|
||||||
* Rate limit: 1 req/2sec per domain, max 3 concurrent domains.
|
* Rate limit: 1 req/2sec per domain, max 3 concurrent domains.
|
||||||
* Respects robots.txt: User-Agent identifies as research bot.
|
* Respects robots.txt: User-Agent identifies as research bot.
|
||||||
@ -89,7 +91,11 @@ function buildJuniperUrl(model: string): string | null {
|
|||||||
function buildNvidiaUrl(model: string): string | null {
|
function buildNvidiaUrl(model: string): string | null {
|
||||||
// SN5600 → https://www.nvidia.com/en-us/networking/ethernet-switching/sn5600/
|
// SN5600 → https://www.nvidia.com/en-us/networking/ethernet-switching/sn5600/
|
||||||
// SN4700 → https://www.nvidia.com/en-us/networking/ethernet-switching/sn4700/
|
// SN4700 → https://www.nvidia.com/en-us/networking/ethernet-switching/sn4700/
|
||||||
const slug = model.toUpperCase().replace(/[^A-Z0-9]/g, "");
|
// ConnectX-7 / BlueField are adapters, not switches — skip
|
||||||
|
const m = model.toUpperCase();
|
||||||
|
if (m.includes("CONNECTX") || m.includes("BLUEFIELD")) return null;
|
||||||
|
const slug = m.replace(/[^A-Z0-9]/g, "");
|
||||||
|
if (!slug.startsWith("SN")) return null; // only Spectrum switch series
|
||||||
return `https://www.nvidia.com/en-us/networking/ethernet-switching/${slug.toLowerCase()}/`;
|
return `https://www.nvidia.com/en-us/networking/ethernet-switching/${slug.toLowerCase()}/`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -154,6 +160,35 @@ function buildAsterfusionUrl(model: string): string | null {
|
|||||||
return `https://www.asterfusion.com/products/${slug}/`;
|
return `https://www.asterfusion.com/products/${slug}/`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildFortinetUrl(model: string): string | null {
|
||||||
|
// FortiSwitch 1024E → fortiswitch-1024e
|
||||||
|
// FortiSwitch 124F-POE → fortiswitch-124f-poe
|
||||||
|
const slug = model.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
||||||
|
return `https://www.fortinet.com/products/switches/${slug}.html`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildQuantaUrl(model: string): string | null {
|
||||||
|
// QuantaMesh T3048-LY8, T7032-IX1 etc.
|
||||||
|
const slug = model.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
||||||
|
return `https://www.qct.io/product/index/Infrastructure-Product/Networking/Switch/${slug}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAlliedTelesisUrl(model: string): string | null {
|
||||||
|
// AT-x530-28GSX → https://www.alliedtelesis.com/us/en/products/at-x530-28gsx
|
||||||
|
const slug = model.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
||||||
|
return `https://www.alliedtelesis.com/us/en/products/${slug}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildUfispaceUrl(model: string): string | null {
|
||||||
|
const slug = model.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
||||||
|
return `https://www.ufispace.com/products/${slug}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildNetgearUrl(model: string): string | null {
|
||||||
|
const slug = model.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
||||||
|
return `https://www.netgear.com/business/products/switches/${slug}`;
|
||||||
|
}
|
||||||
|
|
||||||
// ── URL dispatcher by vendor slug ───────────────────────────────────────────
|
// ── URL dispatcher by vendor slug ───────────────────────────────────────────
|
||||||
|
|
||||||
const URL_BUILDERS: Record<string, (m: string) => string | null> = {
|
const URL_BUILDERS: Record<string, (m: string) => string | null> = {
|
||||||
@ -164,6 +199,7 @@ const URL_BUILDERS: Record<string, (m: string) => string | null> = {
|
|||||||
edgecore: buildEdgecoreUrl,
|
edgecore: buildEdgecoreUrl,
|
||||||
celestica: buildCelesticaUrl,
|
celestica: buildCelesticaUrl,
|
||||||
asterfusion: buildAsterfusionUrl,
|
asterfusion: buildAsterfusionUrl,
|
||||||
|
fortinet: buildFortinetUrl,
|
||||||
dell: buildDellUrl,
|
dell: buildDellUrl,
|
||||||
"hpe-aruba": buildHpeArubaUrl,
|
"hpe-aruba": buildHpeArubaUrl,
|
||||||
huawei: buildHuaweiUrl,
|
huawei: buildHuaweiUrl,
|
||||||
@ -174,8 +210,14 @@ const URL_BUILDERS: Record<string, (m: string) => string | null> = {
|
|||||||
"fs-com": buildFsComUrl,
|
"fs-com": buildFsComUrl,
|
||||||
supermicro: buildSupermicroUrl,
|
supermicro: buildSupermicroUrl,
|
||||||
"alcatel-lucent": buildAlcatelLucentUrl,
|
"alcatel-lucent": buildAlcatelLucentUrl,
|
||||||
"ale": buildAlcatelLucentUrl,
|
"alcatel-lucent-enterprise": buildAlcatelLucentUrl, // fix: DB uses this slug
|
||||||
|
ale: buildAlcatelLucentUrl,
|
||||||
|
"quanta-cloud-technology": buildQuantaUrl,
|
||||||
|
"allied-telesis": buildAlliedTelesisUrl,
|
||||||
|
ufispace: buildUfispaceUrl,
|
||||||
|
netgear: buildNetgearUrl,
|
||||||
wistron: (_m) => null, // no public product pages
|
wistron: (_m) => null, // no public product pages
|
||||||
|
aruba: buildHpeArubaUrl, // alias
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Generic marketing image detector ────────────────────────────────────────
|
// ── Generic marketing image detector ────────────────────────────────────────
|
||||||
|
|||||||
166
scripts/monitor-erik.sh
Executable file
166
scripts/monitor-erik.sh
Executable file
@ -0,0 +1,166 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# TIP / Erik Health Monitor — run on Claudi (.82) or Raspberry Pi via cron
|
||||||
|
#
|
||||||
|
# Checks:
|
||||||
|
# 1. TIP API health endpoint (CPU load, memory, disk)
|
||||||
|
# 2. PM2 process status on Erik (errored/stopped processes)
|
||||||
|
# 3. DB query roundtrip latency
|
||||||
|
#
|
||||||
|
# Alerting:
|
||||||
|
# - ntfy.sh push notification (set NTFY_TOPIC below)
|
||||||
|
# - Appends to /var/log/tip-monitor.log
|
||||||
|
#
|
||||||
|
# Setup (Claudi / Pi):
|
||||||
|
# chmod +x monitor-erik.sh
|
||||||
|
# sudo cp monitor-erik.sh /usr/local/bin/tip-monitor
|
||||||
|
#
|
||||||
|
# Add to crontab (every 5 minutes):
|
||||||
|
# */5 * * * * /usr/local/bin/tip-monitor >> /var/log/tip-monitor.log 2>&1
|
||||||
|
#
|
||||||
|
# Or for systemd timer — see monitor-erik.service / monitor-erik.timer below
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ── Config ───────────────────────────────────────────────────────────────────
|
||||||
|
TIP_API="${TIP_API:-https://tip.fichtmueller.org/api/health}"
|
||||||
|
NTFY_TOPIC="${NTFY_TOPIC:-}" # e.g. "tip-erik-alerts"
|
||||||
|
SSH_TARGET="${SSH_TARGET:-erik-cf}" # SSH alias for Erik
|
||||||
|
LOAD_WARN="${LOAD_WARN:-4.0}" # 1m load warning threshold
|
||||||
|
DISK_WARN="${DISK_WARN:-85}" # disk % warning threshold
|
||||||
|
MEM_WARN="${MEM_WARN:-90}" # memory % warning threshold
|
||||||
|
LOG_FILE="${LOG_FILE:-/var/log/tip-monitor.log}"
|
||||||
|
|
||||||
|
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
TS() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||||
|
log() { echo "[$(TS)] $*"; }
|
||||||
|
warn() { echo "[$(TS)] ⚠️ WARN: $*"; }
|
||||||
|
crit() { echo "[$(TS)] 🔴 CRIT: $*"; alert "$*"; }
|
||||||
|
|
||||||
|
ALERTS=()
|
||||||
|
alert() {
|
||||||
|
ALERTS+=("$1")
|
||||||
|
if [[ -n "$NTFY_TOPIC" ]]; then
|
||||||
|
curl -s -m 5 \
|
||||||
|
-H "Title: TIP/Erik Alert" \
|
||||||
|
-H "Tags: warning,server" \
|
||||||
|
-H "Priority: urgent" \
|
||||||
|
-d "$1" \
|
||||||
|
"https://ntfy.sh/${NTFY_TOPIC}" > /dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── 1. TIP API health check ──────────────────────────────────────────────────
|
||||||
|
log "Checking TIP API health …"
|
||||||
|
HEALTH_JSON=""
|
||||||
|
HTTP_CODE=0
|
||||||
|
|
||||||
|
HTTP_CODE=$(curl -s -m 15 -o /tmp/tip-health.json -w "%{http_code}" "$TIP_API" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
if [[ "$HTTP_CODE" != "200" ]]; then
|
||||||
|
crit "TIP API unreachable (HTTP $HTTP_CODE) — $TIP_API"
|
||||||
|
else
|
||||||
|
HEALTH_JSON=$(cat /tmp/tip-health.json 2>/dev/null || echo "{}")
|
||||||
|
|
||||||
|
# Extract fields (requires jq)
|
||||||
|
if command -v jq &>/dev/null; then
|
||||||
|
LOAD1=$(echo "$HEALTH_JSON" | jq -r '.system.load."1m" // "N/A"')
|
||||||
|
MEM_PCT=$(echo "$HEALTH_JSON" | jq -r '.system.memory.used_pct // "N/A"')
|
||||||
|
DISK_PCT=$(echo "$HEALTH_JSON" | jq -r '.system.disk.used_pct // "N/A"')
|
||||||
|
DISK_FREE=$(echo "$HEALTH_JSON"| jq -r '.system.disk.free_gb // "N/A"')
|
||||||
|
DB_LAT=$(echo "$HEALTH_JSON" | jq -r '.database.latency_ms // "N/A"')
|
||||||
|
STATUS=$(echo "$HEALTH_JSON" | jq -r '.status // "unknown"')
|
||||||
|
|
||||||
|
log " Status: $STATUS | Load: $LOAD1 | Mem: ${MEM_PCT}% | Disk: ${DISK_PCT}% (${DISK_FREE}GB free) | DB: ${DB_LAT}ms"
|
||||||
|
|
||||||
|
# Load check
|
||||||
|
if command -v bc &>/dev/null && [[ "$LOAD1" != "N/A" ]]; then
|
||||||
|
if (( $(echo "$LOAD1 > $LOAD_WARN" | bc -l) )); then
|
||||||
|
crit "High load on Erik: $LOAD1 (threshold $LOAD_WARN)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Memory check
|
||||||
|
if [[ "$MEM_PCT" != "N/A" ]] && [[ "$MEM_PCT" -ge "$MEM_WARN" ]]; then
|
||||||
|
crit "High memory usage on Erik: ${MEM_PCT}%"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Disk check
|
||||||
|
if [[ "$DISK_PCT" != "N/A" ]] && [[ "$DISK_PCT" -ge "$DISK_WARN" ]]; then
|
||||||
|
crit "Disk usage on Erik: ${DISK_PCT}% (${DISK_FREE}GB free)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# DB latency
|
||||||
|
if command -v bc &>/dev/null && [[ "$DB_LAT" != "N/A" ]]; then
|
||||||
|
if (( $(echo "$DB_LAT > 2000" | bc -l) )); then
|
||||||
|
warn "High DB latency: ${DB_LAT}ms"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log " API OK (HTTP 200) — install jq for detailed metrics"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 2. PM2 process check (via SSH) ──────────────────────────────────────────
|
||||||
|
log "Checking PM2 processes on Erik …"
|
||||||
|
if ssh -o ConnectTimeout=10 -o BatchMode=yes "$SSH_TARGET" true 2>/dev/null; then
|
||||||
|
ERRORED=$(ssh -o ConnectTimeout=10 "$SSH_TARGET" \
|
||||||
|
"pm2 list --no-color 2>/dev/null | grep -E 'errored|stopped' | grep -v 'ecosystem-stable'" \
|
||||||
|
2>/dev/null || echo "")
|
||||||
|
|
||||||
|
if [[ -n "$ERRORED" ]]; then
|
||||||
|
COUNT=$(echo "$ERRORED" | wc -l | tr -d ' ')
|
||||||
|
crit "${COUNT} PM2 process(es) errored/stopped on Erik"
|
||||||
|
log " Errored: $ERRORED"
|
||||||
|
else
|
||||||
|
log " PM2: all processes running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check restart counts (> 5 in the last run = likely crashing)
|
||||||
|
HIGH_RESTARTS=$(ssh -o ConnectTimeout=10 "$SSH_TARGET" \
|
||||||
|
"pm2 list --no-color 2>/dev/null | awk '{if(\$16 > 5) print \$2, \"restarts:\", \$16}'" \
|
||||||
|
2>/dev/null || echo "")
|
||||||
|
|
||||||
|
if [[ -n "$HIGH_RESTARTS" ]]; then
|
||||||
|
warn "High restart count: $HIGH_RESTARTS"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
crit "SSH connection to Erik failed (via $SSH_TARGET)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 3. Summary ───────────────────────────────────────────────────────────────
|
||||||
|
if [[ ${#ALERTS[@]} -eq 0 ]]; then
|
||||||
|
log "✅ All checks passed"
|
||||||
|
else
|
||||||
|
log "🔴 ${#ALERTS[@]} alert(s) sent"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Optional: truncate log file at 5000 lines ────────────────────────────────
|
||||||
|
if [[ -f "$LOG_FILE" ]] && [[ $(wc -l < "$LOG_FILE") -gt 5000 ]]; then
|
||||||
|
tail -n 2500 "$LOG_FILE" > /tmp/tip-monitor-trim && mv /tmp/tip-monitor-trim "$LOG_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Systemd unit (paste into /etc/systemd/system/tip-monitor.service) ────────
|
||||||
|
# [Unit]
|
||||||
|
# Description=TIP/Erik Health Monitor
|
||||||
|
# After=network.target
|
||||||
|
#
|
||||||
|
# [Service]
|
||||||
|
# Type=oneshot
|
||||||
|
# ExecStart=/usr/local/bin/tip-monitor
|
||||||
|
# StandardOutput=append:/var/log/tip-monitor.log
|
||||||
|
# StandardError=append:/var/log/tip-monitor.log
|
||||||
|
# Environment=NTFY_TOPIC=tip-erik-alerts
|
||||||
|
#
|
||||||
|
# [Install]
|
||||||
|
# WantedBy=multi-user.target
|
||||||
|
#
|
||||||
|
# Systemd timer (paste into /etc/systemd/system/tip-monitor.timer):
|
||||||
|
# [Unit]
|
||||||
|
# Description=Run TIP monitor every 5 minutes
|
||||||
|
# [Timer]
|
||||||
|
# OnBootSec=60
|
||||||
|
# OnUnitActiveSec=300
|
||||||
|
# [Install]
|
||||||
|
# WantedBy=timers.target
|
||||||
Loading…
x
Reference in New Issue
Block a user