60cffb3585
Close the missing feedback loop. Detection (v_scraper_health, v_data_quality_alerts, check_switch_quality) was accurate but inert; dispatch was blind fixed-cadence (boss.send() is never called from any health path). tip-dqo is a separate, read-mostly control plane that closes DETECT -> PRIORITIZE -> DISPATCH -> VERIFY -> RECHECK against observation-table freshness (never pg-boss job state), with DB-enforced guardrails. - sql/129: source_registry (signal->queue map + per-source SLA + guardrail contract + global kill-switch), self_heal_dispatch append-only ledger, scraper_run_log, acknowledged_exceptions, v_dqo_targets ranked view. - orchestrator/: control-loop, self-heal-guards (pure, unit-tested), escalation-sink (ntfy + no-op fallback), index entrypoint. Reuses the verification-robots dispatch idiom (extracted shared dispatchQueues) and the quality views as-is. - Guardrails: kill-switch, plan-mode default, per-source cooldown/budget/backoff/ circuit-breaker, load-guard, erik-safe profile cap, delta-gated escalation. Root-cause routing escalates code_fix/blocked (e.g. ATGBICS silent stock-write break) to a human instead of looping retries; only staleness is auto-dispatched. - security: remove hardcoded 'tip_dev_2026' DB password fallback across 6 sites -> fail-closed requireDbPassword() / dbConnectionString(). SELF_HEAL_PLAN_ONLY defaults true (dispatches nothing; writes plan rows for review). Verified against the live DB: detects ATGBICS-stock dead/54d -> escalate, Flexoptix-stock stale/30d -> plan-dispatch; guards + ledger correct; 8/8 guard unit tests pass.
79 lines
3.2 KiB
TypeScript
79 lines
3.2 KiB
TypeScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
isLoadAcceptable,
|
|
minIntervalOk,
|
|
budgetOk,
|
|
backoffElapsed,
|
|
breakerTripped,
|
|
canDispatch,
|
|
type GuardConfig,
|
|
type LedgerFacts,
|
|
} from "./self-heal-guards";
|
|
|
|
const CFG: GuardConfig = {
|
|
cooldown_minutes: 180,
|
|
budget_per_day: 4,
|
|
max_backoff_hours: 24,
|
|
circuit_breaker_strikes: 3,
|
|
};
|
|
const NOW = new Date("2026-07-06T12:00:00Z");
|
|
const fresh: LedgerFacts = { last_dispatch_at: null, last_strikes: 0, dispatches_today: 0 };
|
|
|
|
test("load guard", () => {
|
|
assert.equal(isLoadAcceptable(1.2).allow, true);
|
|
assert.equal(isLoadAcceptable(3.0).allow, false);
|
|
assert.equal(isLoadAcceptable(2.5).allow, true); // boundary: not greater than
|
|
});
|
|
|
|
test("min-interval blocks within cooldown, allows after", () => {
|
|
const recent = { ...fresh, last_dispatch_at: new Date("2026-07-06T11:00:00Z") }; // 60min ago
|
|
assert.equal(minIntervalOk(recent, CFG, NOW).allow, false);
|
|
const old = { ...fresh, last_dispatch_at: new Date("2026-07-06T08:00:00Z") }; // 240min ago
|
|
assert.equal(minIntervalOk(old, CFG, NOW).allow, true);
|
|
assert.equal(minIntervalOk(fresh, CFG, NOW).allow, true); // never dispatched
|
|
});
|
|
|
|
test("daily budget", () => {
|
|
assert.equal(budgetOk({ ...fresh, dispatches_today: 3 }, CFG).allow, true);
|
|
assert.equal(budgetOk({ ...fresh, dispatches_today: 4 }, CFG).allow, false);
|
|
assert.equal(budgetOk({ ...fresh, dispatches_today: 9 }, CFG).allow, false);
|
|
});
|
|
|
|
test("exponential backoff scales with strikes and caps", () => {
|
|
// 2 strikes → 4h backoff. Dispatched 1h ago → still backing off.
|
|
const s2recent = { last_dispatch_at: new Date("2026-07-06T11:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
|
assert.equal(backoffElapsed(s2recent, CFG, NOW).allow, false);
|
|
// 2 strikes → 4h backoff. Dispatched 5h ago → elapsed.
|
|
const s2old = { last_dispatch_at: new Date("2026-07-06T07:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
|
assert.equal(backoffElapsed(s2old, CFG, NOW).allow, true);
|
|
// 0 strikes → no backoff
|
|
assert.equal(backoffElapsed(fresh, CFG, NOW).allow, true);
|
|
// huge strikes → capped at max_backoff_hours (24h), dispatched 25h ago → elapsed
|
|
const capped = { last_dispatch_at: new Date("2026-07-05T11:00:00Z"), last_strikes: 10, dispatches_today: 0 };
|
|
assert.equal(backoffElapsed(capped, CFG, NOW).allow, true);
|
|
});
|
|
|
|
test("circuit breaker opens at threshold", () => {
|
|
assert.equal(breakerTripped({ ...fresh, last_strikes: 2 }, CFG).allow, true);
|
|
assert.equal(breakerTripped({ ...fresh, last_strikes: 3 }, CFG).allow, false);
|
|
assert.equal(breakerTripped({ ...fresh, last_strikes: 5 }, CFG).allow, false);
|
|
});
|
|
|
|
test("canDispatch: clean source under low load passes", () => {
|
|
assert.equal(canDispatch(fresh, CFG, NOW, 1.0).allow, true);
|
|
});
|
|
|
|
test("canDispatch: breaker is reported before other brakes", () => {
|
|
const tripped = { last_dispatch_at: new Date("2026-07-06T11:59:00Z"), last_strikes: 3, dispatches_today: 9 };
|
|
const v = canDispatch(tripped, CFG, NOW, 5.0);
|
|
assert.equal(v.allow, false);
|
|
assert.match(v.reason, /breaker OPEN/);
|
|
});
|
|
|
|
test("canDispatch: high load blocks an otherwise-clean source", () => {
|
|
const v = canDispatch(fresh, CFG, NOW, 9.0);
|
|
assert.equal(v.allow, false);
|
|
assert.match(v.reason, /load/);
|
|
});
|