e0b9fa1f53
New package @llm-gateway/ctx-health (packages/ctx-health/) — a TypeScript infrastructure monitoring and auto-healing daemon. Monitors 8 subsystems every 60s (PM2, PostgreSQL, Ollama, Cloudflare tunnel, disk, memory, network, WireGuard), gets AI-powered root cause analysis via the gateway (ctxhealer caller / ctx_health_diagnose task_type), executes healing actions with cooldown (5min) and escalation guards (3+ failures → human escalation), persists all incidents to ctx_health_incidents and ctx_health_status tables. Dry-run mode via CTX_HEALTH_DRY_RUN=true. Runs as ctx-health PM2 process on Erik server.
93 lines
3.2 KiB
TypeScript
93 lines
3.2 KiB
TypeScript
/**
|
|
* CtxHealth — Infrastructure Self-Healing Daemon
|
|
*
|
|
* Monitors infrastructure every 60 seconds:
|
|
* - PM2 processes, PostgreSQL, Ollama, Cloudflare tunnel,
|
|
* disk space, memory, network, WireGuard
|
|
*
|
|
* When a check fails:
|
|
* 1. Calls LLM Gateway for AI-powered root cause analysis
|
|
* 2. Executes auto-healing actions (with cooldown + escalation guards)
|
|
* 3. Logs all incidents to PostgreSQL
|
|
*
|
|
* Safety rules:
|
|
* - After 3+ consecutive failures: escalate to human, skip healing
|
|
* - Cooldown: no re-healing same check within 5 minutes
|
|
* - Dry-run mode: CTX_HEALTH_DRY_RUN=true → diagnose but don't heal
|
|
*/
|
|
|
|
import cron from 'node-cron';
|
|
import { logger } from './observability/logger.js';
|
|
import { closePool } from './db/client.js';
|
|
import { runMigrations } from './db/incidents.js';
|
|
import { runHealthCycle } from './runner.js';
|
|
|
|
const DRY_RUN = process.env['CTX_HEALTH_DRY_RUN'] === 'true';
|
|
const CRON_SCHEDULE = process.env['CTX_HEALTH_CRON'] ?? '*/1 * * * *';
|
|
|
|
// ─── Startup ─────────────────────────────────────────────────────────────────
|
|
|
|
async function startup(): Promise<void> {
|
|
logger.info({ dryRun: DRY_RUN, schedule: CRON_SCHEDULE }, 'CtxHealth daemon starting');
|
|
|
|
try {
|
|
await runMigrations();
|
|
} catch (err) {
|
|
logger.error({ err }, 'Migration failed — proceeding without DB (degraded mode)');
|
|
}
|
|
|
|
// Run one immediate cycle on start
|
|
await safeRun();
|
|
|
|
cron.schedule(CRON_SCHEDULE, () => {
|
|
void safeRun();
|
|
});
|
|
|
|
logger.info('CtxHealth daemon running — press Ctrl+C to stop');
|
|
}
|
|
|
|
let cycleRunning = false;
|
|
|
|
async function safeRun(): Promise<void> {
|
|
if (cycleRunning) {
|
|
logger.warn('Previous health cycle still running — skipping this tick');
|
|
return;
|
|
}
|
|
cycleRunning = true;
|
|
const start = Date.now();
|
|
try {
|
|
await runHealthCycle(DRY_RUN);
|
|
logger.debug({ durationMs: Date.now() - start }, 'Health cycle completed');
|
|
} catch (err) {
|
|
logger.error({ err, durationMs: Date.now() - start }, 'Health cycle crashed unexpectedly');
|
|
} finally {
|
|
cycleRunning = false;
|
|
}
|
|
}
|
|
|
|
// ─── Graceful shutdown ───────────────────────────────────────────────────────
|
|
|
|
async function shutdown(signal: string): Promise<void> {
|
|
logger.info({ signal }, 'CtxHealth shutting down gracefully');
|
|
try {
|
|
await closePool();
|
|
} catch (err) {
|
|
logger.warn({ err }, 'Error closing DB pool during shutdown');
|
|
}
|
|
process.exit(0);
|
|
}
|
|
|
|
process.on('SIGTERM', () => void shutdown('SIGTERM'));
|
|
process.on('SIGINT', () => void shutdown('SIGINT'));
|
|
|
|
process.on('unhandledRejection', (reason) => {
|
|
logger.error({ reason }, 'Unhandled promise rejection in ctx-health');
|
|
});
|
|
|
|
// ─── Boot ────────────────────────────────────────────────────────────────────
|
|
|
|
void startup().catch((err) => {
|
|
logger.error({ err }, 'Fatal startup error');
|
|
process.exit(1);
|
|
});
|