/** * CtxHealth — Infrastructure Self-Healing Daemon * * Monitors infrastructure every 60 seconds: * - PM2 processes, PostgreSQL, Ollama, Cloudflare tunnel, * disk space, memory, network, WireGuard * * When a check fails: * 1. Calls LLM Gateway for AI-powered root cause analysis * 2. Executes auto-healing actions (with cooldown + escalation guards) * 3. Logs all incidents to PostgreSQL * * Safety rules: * - After 3+ consecutive failures: escalate to human, skip healing * - Cooldown: no re-healing same check within 5 minutes * - Dry-run mode: CTX_HEALTH_DRY_RUN=true → diagnose but don't heal */ import cron from 'node-cron'; import { logger } from './observability/logger.js'; import { closePool } from './db/client.js'; import { runMigrations } from './db/incidents.js'; import { runHealthCycle } from './runner.js'; const DRY_RUN = process.env['CTX_HEALTH_DRY_RUN'] === 'true'; const CRON_SCHEDULE = process.env['CTX_HEALTH_CRON'] ?? '*/1 * * * *'; // ─── Startup ───────────────────────────────────────────────────────────────── async function startup(): Promise { logger.info({ dryRun: DRY_RUN, schedule: CRON_SCHEDULE }, 'CtxHealth daemon starting'); try { await runMigrations(); } catch (err) { logger.error({ err }, 'Migration failed — proceeding without DB (degraded mode)'); } // Run one immediate cycle on start await safeRun(); cron.schedule(CRON_SCHEDULE, () => { void safeRun(); }); logger.info('CtxHealth daemon running — press Ctrl+C to stop'); } let cycleRunning = false; async function safeRun(): Promise { if (cycleRunning) { logger.warn('Previous health cycle still running — skipping this tick'); return; } cycleRunning = true; const start = Date.now(); try { await runHealthCycle(DRY_RUN); logger.debug({ durationMs: Date.now() - start }, 'Health cycle completed'); } catch (err) { logger.error({ err, durationMs: Date.now() - start }, 'Health cycle crashed unexpectedly'); } finally { cycleRunning = false; } } // ─── Graceful shutdown ─────────────────────────────────────────────────────── async function shutdown(signal: string): Promise { logger.info({ signal }, 'CtxHealth shutting down gracefully'); try { await closePool(); } catch (err) { logger.warn({ err }, 'Error closing DB pool during shutdown'); } process.exit(0); } process.on('SIGTERM', () => void shutdown('SIGTERM')); process.on('SIGINT', () => void shutdown('SIGINT')); process.on('unhandledRejection', (reason) => { logger.error({ reason }, 'Unhandled promise rejection in ctx-health'); }); // ─── Boot ──────────────────────────────────────────────────────────────────── void startup().catch((err) => { logger.error({ err }, 'Fatal startup error'); process.exit(1); });