PeerCortex/server/__smoke__/compare-responses.js

#!/usr/bin/env node
// Regression gate for the server.js module-extraction refactor (see
// /Users/renefichtmueller/.claude/plans/linked-riding-newell.md). Not shipped
// to prod -- local dev tool only.
//
// Usage:
//   node server/__smoke__/compare-responses.js capture   # save baseline (run once, before Phase A)
//   node server/__smoke__/compare-responses.js diff      # run after every extraction step

const http = require("http");
const fs = require("fs");
const path = require("path");
const { spawn, execFileSync } = require("child_process");

// server.js writes real snapshots into these git-tracked files at startup
// (ASPA adoption history) and at runtime (hijack alerts, webhook subs) --
// snapshot + restore around every local boot so smoke-testing never leaves
// lasting mutations on data that's supposed to hold genuine production
// history, not local test noise.
const REPO_ROOT = path.join(__dirname, "..", "..");
const MUTABLE_TRACKED_FILES = ["aspa-adoption-history.json", "hijack-alerts.json", "webhook-subs.json"];

function restoreTrackedFiles() {
  try {
    execFileSync("git", ["checkout", "--", ...MUTABLE_TRACKED_FILES], { cwd: REPO_ROOT, stdio: "ignore" });
  } catch {
    // fine if a file has no uncommitted changes (git checkout on a clean path is a no-op anyway)
  }
}

const PORT = 3199;
const BASE = `http://127.0.0.1:${PORT}`;
const BASELINE_DIR = path.join(__dirname, "baseline");
const ASN = 13335; // Cloudflare -- stable, public, always has real data
const ASN2 = 15169; // Google -- for compare/relationships

// One representative, read-only (GET) request per route from the exploration
// map. POST/DELETE mutating routes (feedback POST, webhooks POST/DELETE,
// hijacks resolve) are intentionally excluded -- not idempotent-safe to
// re-run repeatedly across every extraction step.
const ENDPOINTS = [
  ["root", "/"],
  ["search", `/api/search?q=cloudflare`],
  ["visitors", "/api/visitors"],
  ["health", "/api/health"],
  ["aspa-verify", `/api/aspa/verify?asn=${ASN}`],
  ["aspa-legacy", `/api/aspa?asn=${ASN}`],
  ["bgp", `/api/bgp?asn=${ASN}`],
  ["validate", `/api/validate?asn=${ASN}`],
  ["lookup", `/api/lookup?asn=${ASN}`],
  ["relationships", `/api/relationships?asn=${ASN}`],
  ["compare", `/api/compare?asn1=${ASN}&asn2=${ASN2}`],
  ["quick-ix", `/api/quick-ix?asn=${ASN}`],
  ["peers-find", `/api/peers/find?asn=${ASN}`],
  ["prefix-detail", "/api/prefix/detail?prefix=1.1.1.0/24"],
  ["ix-detail", "/api/ix/detail?asn=${ASN}"],
  ["topology", `/api/topology?asn=${ASN}`],
  ["whois", `/api/whois?asn=${ASN}`],
  ["enrich", `/api/enrich?asn=${ASN}`],
  ["changelog", "/changelog"],
  ["webhooks-list", "/api/webhooks"],
  ["hijacks-list", "/api/hijacks"],
  ["hijack-alerts", "/api/hijack-alerts"],
  ["aspa-adoption-page", "/aspa-adoption"],
  ["aspa-adoption-stats", "/api/aspa-adoption-stats?period=30d"],
  ["ipv6-adoption-stats", "/api/ipv6-adoption-stats"],
  ["atlas-coverage", "/api/atlas/coverage"],
  // proxy stubs -- expected to fail identically (peercortex-api not running
  // locally either); still worth diffing so the *error shape* stays stable
  ["submarine-cables", "/api/submarine-cables"],
  ["communities", "/api/communities?asn=" + ASN],
];

// Fields that legitimately change between runs -- excluded from the diff.
// Two sources of real (non-bug) drift observed empirically by diffing an
// unmodified server.js against itself: (1) process-local counters (uptime,
// memory), and (2) live third-party data that changes minute-to-minute
// (RIPE Atlas probe counts/IDs, Cloudflare RPKI feed size, health_score
// insofar as it's derived from live external checks like MANRS that can
// time out differently run to run).
const NONDETERMINISTIC_KEY_PATTERN =
  /^(generated_at|timestamp|checked_at|fetched_at|query_time|cache_age|age_seconds|age_minutes|uptime_seconds|memory_mb|response_time_ms|duration_ms|last_updated|health_score|delta_last|history_samples|total_probes|total_connected|unique_asns_with_probes|atlas_asns_total|atlas_asns_with_aspa|coverage_pct_atlas|asns_with_probes|roa_count|aspa_objects)$/i;

// Whole sub-trees excluded by dotted path (relative to the endpoint's body),
// found empirically by diffing an unmodified server.js against itself twice.
// Each of these depends on a live third-party call that can legitimately
// time out or return fresh-but-different data run to run (RIPE Stat routing
// status, RIS prefix history, PeeringDB unauthenticated rate limits, RIPE
// Atlas probe churn, source-fetch timing diagnostics) -- none of it reflects
// server.js's own logic, so a diff here is noise, not a regression signal.
const VOLATILE_PATHS = new Set([
  "lookup.source_timing",
  "validate.validations.visibility",
  // redundant summary of validations.* (already diffed in detail) that also
  // embeds the visibility check's flaky pass/info status
  "validate.score_breakdown",
  "prefix-detail.origins",
  "prefix-detail.first_seen",
  "aspa-adoption-stats.forecast",
  "quick-ix.ix_connections",
  "quick-ix.name",
  "atlas-coverage.by_country",
  // live Cloudflare RPKI feed object count ticks between runs; date-rollover
  // (midnight) shifts the adoption tracker's day-over-day delta too
  "health.caches.aspa_map.entries",
  "health.aspa_adoption.total_objects",
  "aspa-adoption-stats.current.delta_from_previous",
]);

function stripNondeterministic(obj, pathPrefix) {
  if (Array.isArray(obj)) return obj.map((v) => stripNondeterministic(v, pathPrefix));
  if (obj && typeof obj === "object") {
    const out = {};
    for (const [k, v] of Object.entries(obj)) {
      if (NONDETERMINISTIC_KEY_PATTERN.test(k)) continue;
      const fullPath = pathPrefix ? `${pathPrefix}.${k}` : k;
      if (VOLATILE_PATHS.has(fullPath)) continue;
      out[k] = stripNondeterministic(v, fullPath);
    }
    return out;
  }
  return obj;
}

function fetchOne(name, pathAndQuery) {
  return new Promise((resolve) => {
    const req = http.get(BASE + pathAndQuery, { timeout: 20000 }, (res) => {
      let body = "";
      res.on("data", (c) => (body += c));
      res.on("end", () => {
        let parsed;
        try {
          parsed = JSON.parse(body);
        } catch {
          parsed = { __non_json__: true, status: res.statusCode, bodyLength: body.length };
        }
        resolve({ status: res.statusCode, body: stripNondeterministic(parsed, name) });
      });
    });
    req.on("timeout", () => {
      req.destroy();
      resolve({ status: 0, body: { __timeout__: true } });
    });
    req.on("error", (err) => resolve({ status: 0, body: { __error__: err.message } }));
  });
}

function waitForServer(maxWaitMs = 100000) {
  // Startup does real, sequential network fetches (Cloudflare RPKI/ROA feed
  // ~972k entries, full paginated RIPE Atlas probe list, IPv6 RIR delegation
  // stats) before calling server.listen() -- observed ~60s locally.
  const start = Date.now();
  return new Promise((resolve, reject) => {
    const tryOnce = () => {
      const req = http.get(`${BASE}/api/health`, { timeout: 2000 }, (res) => {
        res.resume();
        resolve();
      });
      req.on("error", () => {
        if (Date.now() - start > maxWaitMs) return reject(new Error("server didn't come up in time"));
        setTimeout(tryOnce, 500);
      });
      req.on("timeout", () => {
        req.destroy();
        if (Date.now() - start > maxWaitMs) return reject(new Error("server didn't come up in time"));
        setTimeout(tryOnce, 500);
      });
    };
    tryOnce();
  });
}

async function runAgainstLiveServer() {
  const results = {};
  for (const [name, pathAndQuery] of ENDPOINTS) {
    results[name] = await fetchOne(name, pathAndQuery);
  }
  return results;
}

function leafDiffs(a, b, pathPrefix, out) {
  if (JSON.stringify(a) === JSON.stringify(b)) return;
  const bothPlainObjects =
    a && b && typeof a === "object" && typeof b === "object" && !Array.isArray(a) && !Array.isArray(b);
  if (bothPlainObjects) {
    for (const k of new Set([...Object.keys(a), ...Object.keys(b)])) {
      leafDiffs(a[k], b[k], pathPrefix + "." + k, out);
    }
  } else {
    out.push(`${pathPrefix}: ${JSON.stringify(a)} -> ${JSON.stringify(b)}`);
  }
}

function diffResults(baseline, current) {
  const diffs = [];
  for (const [name] of ENDPOINTS) {
    const b = JSON.stringify(baseline[name]);
    const c = JSON.stringify(current[name]);
    if (b !== c) {
      const leaves = [];
      leafDiffs(baseline[name], current[name], name, leaves);
      diffs.push({ name, leaves });
    }
  }
  return diffs;
}

async function main() {
  const mode = process.argv[2];
  if (mode !== "capture" && mode !== "diff") {
    console.error("Usage: node compare-responses.js <capture|diff>");
    process.exit(1);
  }

  restoreTrackedFiles();

  const serverEntry = path.join(__dirname, "..", "..", "server.js");
  const child = spawn(process.execPath, [serverEntry], {
    cwd: path.join(__dirname, "..", ".."),
    env: {
      ...process.env,
      PORT: String(PORT),
      // Node 22's stricter header validation throws on Authorization:
      // undefined, which server.js's fetchPdbOrgCountries() produces when
      // PEERINGDB_API_KEY is unset (line ~6689). This is a real bug (tracked
      // separately, fixed during the pdb-org-countries.js extraction in
      // Phase B) but it blocks the server from even booting for local
      // testing -- work around it here at the test-harness level only, so
      // the baseline reflects real behavior rather than a crash.
      PEERINGDB_API_KEY: process.env.PEERINGDB_API_KEY || "smoke-test-placeholder",
      // Force the proxy-to-peercortex-api path to genuinely fail closed
      // instead of cross-talking with whatever unrelated dev server happens
      // to be listening on the real default port (3102) on this machine --
      // observed proxying straight into an unrelated Next.js app's 404 page.
      API_SERVER_PORT: "39102",
    },
    stdio: ["ignore", "pipe", "pipe"],
  });

  let stderrBuf = "";
  child.stderr.on("data", (c) => (stderrBuf += c));

  const cleanup = () => {
    try {
      child.kill("SIGTERM");
    } catch {}
  };
  process.on("exit", cleanup);

  try {
    await waitForServer();
    console.log(`Server up on port ${PORT}, running ${ENDPOINTS.length} requests...`);
    const results = await runAgainstLiveServer();

    if (mode === "capture") {
      fs.mkdirSync(BASELINE_DIR, { recursive: true });
      fs.writeFileSync(path.join(BASELINE_DIR, "baseline.json"), JSON.stringify(results, null, 2));
      console.log(`Baseline captured: ${ENDPOINTS.length} endpoints -> ${BASELINE_DIR}/baseline.json`);
    } else {
      const baselinePath = path.join(BASELINE_DIR, "baseline.json");
      if (!fs.existsSync(baselinePath)) {
        console.error("No baseline found -- run `capture` first.");
        process.exit(1);
      }
      const baseline = JSON.parse(fs.readFileSync(baselinePath, "utf8"));
      const diffs = diffResults(baseline, results);
      if (diffs.length === 0) {
        console.log(`PASS -- all ${ENDPOINTS.length} endpoints match baseline.`);
      } else {
        console.log(`FAIL -- ${diffs.length}/${ENDPOINTS.length} endpoint(s) differ from baseline:`);
        for (const d of diffs) {
          console.log(`\n=== ${d.name} (${d.leaves.length} leaf change(s)) ===`);
          for (const line of d.leaves.slice(0, 20)) console.log("  " + line);
          if (d.leaves.length > 20) console.log(`  ... and ${d.leaves.length - 20} more`);
        }
        process.exitCode = 1;
      }
    }
  } catch (err) {
    console.error("Smoke test failed to run:", err.message);
    if (stderrBuf) console.error("--- server stderr ---\n" + stderrBuf.slice(0, 3000));
    process.exitCode = 1;
  } finally {
    cleanup();
    restoreTrackedFiles();
  }
}

main();