diff --git a/server.js b/server.js index 9b7a732..1ec0b1e 100644 --- a/server.js +++ b/server.js @@ -266,293 +266,12 @@ const { const { webhookSignature, deliverWebhook, notifyWebhooks } = require("./server/hijack-monitoring/webhooks"); const { classifyHijackSeverity, checkHijacksForAsn, runHijackCheck } = require("./server/hijack-monitoring/monitor"); -const ASPA_ADOPTION_FILE = DATA_DIR + '/aspa-adoption-history.json'; - -// ============================================================ -// FEATURE 3: ASPA Adoption Tracker -// ============================================================ - -/** - * Load persisted ASPA adoption history from disk. - * Returns array of snapshot objects (up to 365 entries). - */ -function loadAspaAdoptionHistory() { - try { - const raw = JSON.parse(fs.readFileSync(ASPA_ADOPTION_FILE, 'utf8')); - return Array.isArray(raw) ? raw : []; - } catch(_) { return []; } -} - -/** - * Save ASPA adoption history to disk (keep last 365 entries). - */ -function saveAspaAdoptionHistory(history) { - try { - const trimmed = history.slice(-365); - fs.writeFileSync(ASPA_ADOPTION_FILE, JSON.stringify(trimmed, null, 2), 'utf8'); - } catch(e) { - console.error('[ASPA-ADOPT] Save failed:', e.message); - } -} - -/** In-memory adoption history (persisted on each new snapshot) */ -let aspaAdoptionDailyHistory = loadAspaAdoptionHistory(); - -/** - * Take a new adoption snapshot and persist it. - * Called whenever the RPKI feed is refreshed. - * @param {number} aspaCount — total ASPA objects in rpkiAspaMap - * @param {number} roaCount — total ROAs in roaStore - */ -function recordAspaAdoptionSnapshot(aspaCount, roaCount) { - const now = new Date(); - const date = now.toISOString().slice(0, 10); // "YYYY-MM-DD" - - // Compute how many Atlas-visible ASNs have ASPA - let atlasTotal = 0; - let atlasWithAspa = 0; - if (atlasState.cache?.asns_with_probes) { - atlasTotal = atlasState.cache.asns_with_probes.length; - // rpkiAspaMap is keyed by integer ASN - atlasWithAspa = atlasState.cache.asns_with_probes.filter(a => rpkiAspaMap.has(Number(a))).length; - } - - const coveragePct = atlasTotal > 0 - ? Math.round((atlasWithAspa / atlasTotal) * 10000) / 100 // 2 decimal places - : 0; - - // Compute delta from previous snapshot - const prev = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1]; - const delta = prev ? aspaCount - prev.aspa_objects : 0; - - const snapshot = { - date, - ts: now.getTime(), - aspa_objects: aspaCount, - roa_count: roaCount, - atlas_asns_total: atlasTotal, - atlas_asns_with_aspa: atlasWithAspa, - coverage_pct_atlas: coveragePct, - aspa_delta: delta, - method: 'rpki_cloudflare_feed', - }; - - // Only store one snapshot per date (overwrite if same day) - const existingIdx = aspaAdoptionDailyHistory.findIndex(s => s.date === date); - if (existingIdx >= 0) { - aspaAdoptionDailyHistory[existingIdx] = snapshot; - } else { - aspaAdoptionDailyHistory.push(snapshot); - } - saveAspaAdoptionHistory(aspaAdoptionDailyHistory); - console.log(`[ASPA-ADOPT] Snapshot ${date}: ${aspaCount} objects, ${coveragePct}% Atlas coverage (${atlasWithAspa}/${atlasTotal})`); - return snapshot; -} - -/** - * Get adoption trend for a given period string ('7d', '30d', '90d', '1y'). - */ -function getAdoptionTrend(period) { - const days = period === '7d' ? 7 : period === '30d' ? 30 : period === '90d' ? 90 : 365; - const cutoff = Date.now() - days * 24 * 60 * 60 * 1000; - return aspaAdoptionDailyHistory.filter(s => s.ts >= cutoff); -} - -/** - * Build the ASPA adoption dashboard HTML. - */ -function buildAspaAdoptionDashboard() { - const latest = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1]; - const trend30 = getAdoptionTrend('30d'); - const trend7 = getAdoptionTrend('7d'); - - // Forecast: simple linear regression over last 30 days - function linearForecast(data, daysAhead) { - if (data.length < 2) return null; - const n = data.length; - const xs = data.map((_, i) => i); - const ys = data.map(d => d.coverage_pct_atlas); - const xMean = xs.reduce((a, b) => a + b, 0) / n; - const yMean = ys.reduce((a, b) => a + b, 0) / n; - const num = xs.reduce((s, x, i) => s + (x - xMean) * (ys[i] - yMean), 0); - const den = xs.reduce((s, x) => s + (x - xMean) ** 2, 0); - if (den === 0) return yMean; - const slope = num / den; - return Math.max(0, Math.min(100, yMean + slope * (n - 1 + daysAhead))); - } - - const forecast90d = linearForecast(trend30, 90); - const change7d = trend7.length >= 2 - ? (trend7[trend7.length-1].aspa_objects - trend7[0].aspa_objects) - : 0; - const trendLabels = trend30.map(d => d.date); - const trendValues = trend30.map(d => d.coverage_pct_atlas); - const asnValues = trend30.map(d => d.aspa_objects); - - const coverageNow = latest?.coverage_pct_atlas ?? 0; - const aspaNow = latest?.aspa_objects ?? 0; - const atlasTotal = latest?.atlas_asns_total ?? 0; - - return ` - - - - - ASPA Adoption Tracker · PeerCortex - - - - - -
-

ASPA Adoption Tracker

-

Global ASPA deployment trends · Powered by RPKI Cloudflare feed · Updated every ~4 hours

- -
-
-
Atlas Coverage
-
${coverageNow.toFixed(1)}%
-
of Atlas-visible ASNs
-
-
-
ASPA Objects
-
${aspaNow.toLocaleString()}
-
total in RPKI
-
-
-
Atlas ASNs
-
${atlasTotal.toLocaleString()}
-
sampled denominator
-
-
-
7-Day Delta
-
${change7d > 0 ? '+' : ''}${change7d}
-
new ASPA objects
-
-
-
Data Points
-
${aspaAdoptionDailyHistory.length}
-
history snapshots
-
-
- -
- ↓ JSON (30d) - ↓ CSV (30d) - ↓ CSV (1y) - -
- -
-
-

Atlas Coverage % (last 30 days)

- -
-
-
90-Day Forecast
-
${forecast90d !== null ? forecast90d.toFixed(1) + '%' : '—'}
-
linear projection from 30-day trend
-
- Based on ${trend30.length} data points.
- Current: ${coverageNow.toFixed(1)}% -
-
-
- -
-

Total ASPA Objects in RPKI (last 30 days)

- -
- -
- Last snapshot: ${latest?.date ?? 'none'} · Denominator: ${atlasTotal.toLocaleString()} Atlas ASNs · - JSON API -
-
- - - -`; -} +const { + aspaAdoptionDailyHistory, + recordAspaAdoptionSnapshot, + getAdoptionTrend, + buildAspaAdoptionDashboard, +} = require("./server/aspa-adoption/tracker"); // ============================================================ // FEATURE 4: IPv6 Adoption per RIR @@ -1348,224 +1067,32 @@ const { const { ensureManrsCache, checkManrsMembership } = require("./server/services/manrs"); // ============================================================ -// Infrastructure overlay caches -let subCableCache = null; // TeleGeography submarine cables (24h) -let globalFacCache = null; // PeeringDB global facilities (24h) - -// RPKI ASPA + ROA Cache from Cloudflare RPKI JSON feed -// ============================================================ -const rpkiAspaMap = new Map(); // customer_asid -> Set -let rpkiAspaLastFetch = 0; -let rpkiAspaFetching = false; +// subCableCache/globalFacCache removed 2026-07-16: confirmed dead code, +// never read anywhere in the file. const { roaStore } = require("./server/caches/roa-store"); +const { + rpkiAspaMap, + getRpkiAspaLastFetch, + fetchRpkiAspaFeed, + ensureAspaCache, + lookupAspaFromRpki, + validateRPKIWithCache, + fetchRipeRpkiValidator, + crossCheckRpki, +} = require("./server/services/rpki"); + const { pdbSourceCache } = require("./server/caches/pdb-source-cache"); - -// ============================================================ -// RIPE Stat Source Cache + Semaphore (L2) -// Prevents 429 rate-limiting by throttling + caching responses -// ============================================================ -const ripeStatCache = new Map(); // key: "endpoint:resource" → {data, ts} -const RIPE_STAT_CACHE_MAX = 2000; -const RIPE_STAT_TTL = { - "announced-prefixes": 15 * 60 * 1000, - "asn-neighbours": 15 * 60 * 1000, - "as-overview": 60 * 60 * 1000, - "rir-stats-country": 24 * 60 * 60 * 1000, - "visibility": 15 * 60 * 1000, - "prefix-size-distribution": 60 * 60 * 1000, - "abuse-contact-finder": 24 * 60 * 60 * 1000, - "blocklist": 60 * 60 * 1000, - "reverse-dns-consistency": 60 * 60 * 1000, - "routing-status": 15 * 60 * 1000, - "bgp-updates": 15 * 60 * 1000, - "maxmind-geo-lite-pfx": 24 * 60 * 60 * 1000, - "looking-glass": 15 * 60 * 1000, - "whois": 24 * 60 * 60 * 1000, - "rpki-validation": 6 * 60 * 60 * 1000, -}; - -// Counting semaphore — limits concurrent RIPE Stat requests -class Semaphore { - constructor(max) { this.max = max; this.current = 0; this.queue = []; } - acquire() { - if (this.current < this.max) { this.current++; return Promise.resolve(); } - return new Promise((resolve) => this.queue.push(resolve)); - } - release() { - this.current--; - if (this.queue.length > 0) { this.current++; this.queue.shift()(); } - } -} -const ripeStatSemaphore = new Semaphore(15); - -// Cached + throttled RIPE Stat fetch -// Renamed from fetchRipeStatCached 2026-07-16: a second function with that exact -// name was added ~150 lines down (the local-DB-routing wrapper) during the Postgres -// refactor. In JS, two `function` declarations sharing a name silently collapse to -// the LAST one in source order -- the real RIPE Stat fetch+cache+throttle -// implementation below was completely shadowed, and every call to -// fetchRipeStatCached() for any endpoint NOT in {announced-prefixes, asn-neighbours, -// as-overview, visibility, prefix-size-distribution} was unconditionally returning -// null instead of ever reaching this code. That silently broke blocklist, -// abuse-contact-finder, bgp-updates, routing-status, looking-glass, whois, -// reverse-dns-consistency, maxmind-geo-lite-pfx, and ixs lookups -- see the router's -// fallback call to this function for the fix. -async function fetchRipeStatCachedFromApi(url, options) { - // Extract endpoint name from URL for TTL lookup - const match = url.match(/\/data\/([^/]+)\//); - const endpoint = match ? match[1] : "default"; - const resourceMatch = url.match(/resource=([^&]+)/); - const resource = resourceMatch ? resourceMatch[1] : url; - const cacheKey = endpoint + ":" + resource; - - // Check cache - const cached = ripeStatCache.get(cacheKey); - const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; - if (cached && (Date.now() - cached.ts) < ttl) { - return cached.data; - } - - // Throttle via semaphore - await ripeStatSemaphore.acquire(); - try { - // Double-check cache after acquiring semaphore (another request may have filled it) - const cached2 = ripeStatCache.get(cacheKey); - if (cached2 && (Date.now() - cached2.ts) < ttl) { - return cached2.data; - } - - const result = await fetchJSON(url, options); - - // Only cache successful results — never cache null (failed/rate-limited responses) - // Caching null causes cascading failures: retry hits cache, returns null again - if (result !== null) { - if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { - ripeStatCache.delete(ripeStatCache.keys().next().value); - } - ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); - } - return result; - } finally { - ripeStatSemaphore.release(); - } -} - -// Cached + throttled RIPE Stat with one retry on failure -async function fetchRipeStatCachedWithRetry(url, options) { - const result = await fetchRipeStatCached(url, options); - if (result !== null) return result; - await new Promise(r => setTimeout(r, 1500)); - return fetchRipeStatCached(url, options); -} - -// RIPE Stat cache disk persistence (skip null entries) -function saveRipeStatCacheToDisk(filePath) { - try { - const obj = {}; - for (const [k, v] of ripeStatCache) { - if (v.data !== null) obj[k] = v; - } - fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); - console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); - } catch (e) { - console.warn("[RIPE-CACHE] Disk save failed:", e.message); - } -} - -function loadRipeStatCacheFromDisk(filePath) { - try { - if (!fs.existsSync(filePath)) return false; - const raw = fs.readFileSync(filePath, "utf8"); - const data = JSON.parse(raw); - const now = Date.now(); - for (const [k, v] of Object.entries(data.entries || {})) { - const match = k.match(/^([^:]+):/); - const endpoint = match ? match[1] : "default"; - const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; - if (now - v.ts < ttl) { - ripeStatCache.set(k, v); - } - } - console.log("[RIPE-CACHE] Loaded " + ripeStatCache.size + " entries from disk"); - return true; - } catch (e) { - console.warn("[RIPE-CACHE] Disk load failed:", e.message); - return false; - } -} - -function fetchRpkiAspaFeed() { - if (rpkiAspaFetching) return Promise.resolve(); - rpkiAspaFetching = true; - console.log("[RPKI] Fetching Cloudflare RPKI feed (ASPA + ROA)..."); - return new Promise((resolve) => { - const options = { - headers: { "User-Agent": UA }, - timeout: 120000, - }; - https.get("https://rpki.cloudflare.com/rpki.json", options, (res) => { - let data = ""; - res.on("data", (chunk) => (data += chunk)); - res.on("end", () => { - try { - const parsed = JSON.parse(data); - - // Load ASPA objects - const aspas = parsed.aspas || []; - rpkiAspaMap.clear(); - aspas.forEach((a) => { - const customerAsid = Number(a.customer_asid); - const providers = (a.providers || []).map(Number); - rpkiAspaMap.set(customerAsid, new Set(providers)); - }); - - // Load ROA objects into local store (eliminates RIPE Stat per-prefix calls) - const roas = parsed.roas || []; - roaStore.build(roas); - roaStore.saveToDisk("/opt/peercortex-app/.roa-cache.json"); - - // Track ASPA adoption — persist to disk + update in-memory trend - recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count); - - rpkiAspaLastFetch = Date.now(); - console.log("[RPKI] Loaded " + rpkiAspaMap.size + " ASPA objects + " + roaStore.count + " ROAs from Cloudflare feed"); - } catch (e) { - console.error("[RPKI] Failed to parse RPKI feed:", e.message); - } - rpkiAspaFetching = false; - resolve(); - }); - }).on("error", (e) => { - console.error("[RPKI] Fetch failed:", e.message); - rpkiAspaFetching = false; - resolve(); - }); - }); -} - -// Ensure ASPA + ROA cache is fresh -async function ensureAspaCache() { - if (Date.now() - rpkiAspaLastFetch > 4 * 60 * 60 * 1000) { - await fetchRpkiAspaFeed(); - } -} - -// Lookup ASPA object for a given ASN from the RPKI feed cache. feedLoaded -// distinguishes "the feed loaded fine and this ASN genuinely has no ASPA object" -// from "the Cloudflare feed fetch has never succeeded, so we don't actually know" -- -// rpkiAspaLastFetch is only set on a successful parse (see fetchRpkiAspaFeed), so -// 0 means every attempt so far has failed and exists:false here is not trustworthy. -function lookupAspaFromRpki(asn) { - const asnNum = Number(asn); - const feedLoaded = rpkiAspaLastFetch > 0; - if (rpkiAspaMap.has(asnNum)) { - const providers = rpkiAspaMap.get(asnNum); - return { exists: true, providers: [...providers].sort((a, b) => a - b), feedLoaded }; - } - return { exists: false, providers: [], feedLoaded }; -} +const { + ripeStatCache, + fetchRipeStatCached, + fetchRipeStatCachedFromApi, + fetchRipeStatCachedWithRetry, + saveRipeStatCacheToDisk, + loadRipeStatCacheFromDisk, + Semaphore, +} = require("./server/services/ripe-stat"); @@ -1634,71 +1161,6 @@ async function resolveASNames(providers) { return providers; } -// ── MASTER RIPE Stat API wrapper (Local-first, zero external API calls) ── -// Analyzes RIPE Stat URL and dispatches to appropriate localDb function -async function fetchRipeStatCached(url, options = {}) { - try { - // Detect which RIPE Stat endpoint this is and call local DB - if (url.includes('/announced-prefixes/')) { - const asnMatch = url.match(/resource=AS(\d+)/); - if (asnMatch) return await localDb.getRipeStatAnnouncedPrefixes(parseInt(asnMatch[1])); - } - if (url.includes('/asn-neighbours/')) { - const asnMatch = url.match(/resource=AS(\d+)/); - if (asnMatch) return await localDb.getRipeStatAsnNeighbours(parseInt(asnMatch[1])); - } - if (url.includes('/as-overview/')) { - const asnMatch = url.match(/resource=AS(\d+)/); - if (asnMatch) return await localDb.getRipeStatAsOverview(parseInt(asnMatch[1])); - } - if (url.includes('/visibility/')) { - const asnMatch = url.match(/resource=AS(\d+)/); - if (asnMatch) return await localDb.getRipeStatVisibility(parseInt(asnMatch[1])); - } - if (url.includes('/prefix-size-distribution/')) { - const asnMatch = url.match(/resource=AS(\d+)/); - if (asnMatch) return await localDb.getRipeStatPrefixSizeDistribution(parseInt(asnMatch[1])); - } - - // For every other RIPE Stat endpoint (not mirrored into the local Postgres DB) -- - // e.g. blocklist, abuse-contact-finder, bgp-updates, routing-status, looking-glass, - // whois, reverse-dns-consistency, maxmind-geo-lite-pfx, ixs -- fall through to the - // real RIPE Stat API with its own cache/throttle/never-cache-null protections, - // instead of returning null unconditionally (see fetchRipeStatCachedFromApi's - // header comment for how this silently broke ~9 checks until 2026-07-16). - return fetchRipeStatCachedFromApi(url, options); - } catch (e) { - console.error("[fetchRipeStatCached] Error:", e.message); - return Promise.resolve(null); - } -} - -// RPKI per-prefix validation — uses local ROA store (instant, no API calls) -// Falls back to RIPE Stat only if ROA store is not yet loaded (cold start) -// Validate RPKI for a prefix — uses local PostgreSQL database (sub-10ms, zero external API calls) -// Returns: { prefix, status: "valid"|"invalid"|"not_found"|"unavailable", validating_roas: N } -// "not_found" means the DB was queried and genuinely has no covering ROA -- a real result. -// "unavailable" means the DB query itself failed (connection/timeout/error) -- we don't know. -// Conflating these used to make DB hiccups masquerade as "no ROA published" (a real, worse- -// sounding finding). Callers computing coverage percentages should exclude "unavailable". -async function validateRPKIWithCache(asn, prefix) { - try { - const result = await localDb.validateRpki(prefix, asn); - if (result.status === 'valid') { - return { prefix, status: "valid", validating_roas: 1 }; - } else if (result.status === 'invalid') { - return { prefix, status: "invalid", validating_roas: 1 }; - } else if (result.status === 'unknown') { - return { prefix, status: "unavailable", validating_roas: 0 }; - } else { - // 'not-found': genuinely no covering ROA - return { prefix, status: "not_found", validating_roas: 0 }; - } - } catch (_e) { - console.error("[RPKI] Error validating " + prefix + ":", _e.message); - return { prefix, status: "unavailable", validating_roas: 0 }; - } -} const { @@ -1713,83 +1175,6 @@ const { } = require("./server/aspa-verification/engine"); -// ============================================================ -// Feature 30: RIPE NCC RPKI Validator cross-check (max 5 prefixes) -// ============================================================ -async function fetchRipeRpkiValidator(asn, prefix) { - try { - const encoded = encodeURIComponent(prefix); - const url = "https://rpki-validator.ripe.net/api/v1/validity/AS" + asn + "/" + encoded; - const result = await fetchJSON(url, { timeout: 5000 }); - if (result && result.validated_route) { - return { - prefix: prefix, - validity: result.validated_route.validity || {}, - state: (result.validated_route.validity && result.validated_route.validity.state) || "unknown", - }; - } - return { prefix: prefix, state: "unknown", error: "no_data" }; - } catch (_e) { - return { prefix: prefix, state: "error", error: "timeout_or_unavailable" }; - } -} - -// Cross-check a sample of prefixes against RIPE RPKI Validator (max 5, in parallel). -// agreement_pct only counts pairs where BOTH sources actually returned a real -// verdict -- a zero-sample or all-RIPE-lookups-failed result reports agreement_pct: -// null (not a fabricated 100), since nothing was actually cross-checked. -async function crossCheckRpki(asn, prefixes, localResults) { - const sample = prefixes.slice(0, 5); - if (sample.length === 0) return { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 }; - - const ripeResults = await Promise.all( - sample.map((pfx) => fetchRipeRpkiValidator(asn, pfx)) - ); - - const localMap = new Map(); - for (const lr of localResults) { - localMap.set(lr.prefix, lr.status); - } - - let cloudflareValid = 0; - let ripeValid = 0; - let agreements = 0; - let comparable = 0; - const disagreements = []; - - for (let i = 0; i < sample.length; i++) { - const pfx = sample[i]; - const cfStatus = localMap.get(pfx) || "not_found"; - const ripeState = ripeResults[i].state; - - const cfIsValid = cfStatus === "valid"; - const ripeIsValid = ripeState === "valid" || ripeState === "VALID"; - - if (cfIsValid) cloudflareValid++; - if (ripeIsValid) ripeValid++; - - // A failed/unknown RIPE lookup isn't a comparison at all -- exclude it from - // the agreement percentage instead of counting it as a silent "agreement". - if (ripeState === "error" || ripeState === "unknown") { - continue; - } - - comparable++; - if (cfIsValid === ripeIsValid) { - agreements++; - } else { - disagreements.push({ - prefix: pfx, - cloudflare: cfStatus, - ripe: ripeState, - }); - } - } - - const agreementPct = comparable > 0 ? Math.round((agreements / comparable) * 100) : null; - return { cloudflare_valid: cloudflareValid, ripe_valid: ripeValid, agreement_pct: agreementPct, disagreements: disagreements, sample_size: sample.length, comparable_size: comparable }; -} - // ============================================================ // Feature 24: bgp.he.net Integration // ============================================================ @@ -2401,6 +1786,7 @@ const server = http.createServer(async (req, res) => { // Health endpoint — extended with cache status, ASPA metrics, and local DB stats if (reqPath === "/api/health") { const mem = process.memoryUsage(); + const rpkiAspaLastFetch = getRpkiAspaLastFetch(); const aspaAge = rpkiAspaLastFetch ? Math.floor((Date.now() - rpkiAspaLastFetch) / 60000) : -1; const pdbTotal = pdbSourceCache.hits + pdbSourceCache.misses; @@ -5542,7 +4928,7 @@ loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json"); ensureManrsCache(); // fire-and-forget, 24h cache Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => { // Re-record ASPA adoption snapshot now that Atlas data is fully loaded - recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count); + recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count, rpkiAspaMap); server.listen(PORT, "0.0.0.0", () => { console.log("PeerCortex v0.6.5 running on http://0.0.0.0:" + PORT); diff --git a/server/aspa-adoption/tracker.js b/server/aspa-adoption/tracker.js new file mode 100644 index 0000000..29d6cc8 --- /dev/null +++ b/server/aspa-adoption/tracker.js @@ -0,0 +1,291 @@ +const fs = require("fs"); +const { DATA_DIR } = require("../hijack-monitoring/store"); +const { atlasState } = require("../atlas-probes"); + +const ASPA_ADOPTION_FILE = DATA_DIR + '/aspa-adoption-history.json'; + +// Load persisted ASPA adoption history from disk. +// Returns array of snapshot objects (up to 365 entries). +function loadAspaAdoptionHistory() { + try { + const raw = JSON.parse(fs.readFileSync(ASPA_ADOPTION_FILE, 'utf8')); + return Array.isArray(raw) ? raw : []; + } catch(_) { return []; } +} + +// Save ASPA adoption history to disk (keep last 365 entries). +function saveAspaAdoptionHistory(history) { + try { + const trimmed = history.slice(-365); + fs.writeFileSync(ASPA_ADOPTION_FILE, JSON.stringify(trimmed, null, 2), 'utf8'); + } catch(e) { + console.error('[ASPA-ADOPT] Save failed:', e.message); + } +} + +// In-memory adoption history (persisted on each new snapshot). Mutated +// in place (push / index-assign), never reassigned -- safe to export as a +// live array reference, same pattern as roaStore/pdbSourceCache. +let aspaAdoptionDailyHistory = loadAspaAdoptionHistory(); + +// Take a new adoption snapshot and persist it. Called whenever the RPKI feed +// is refreshed. +// @param {number} aspaCount — total ASPA objects in rpkiAspaMap +// @param {number} roaCount — total ROAs in roaStore +// @param {Map} rpkiAspaMap — customer_asid -> Set, owned by +// server/services/rpki.js; passed in explicitly (rather than required here) +// to avoid a circular require between this module and rpki.js. +function recordAspaAdoptionSnapshot(aspaCount, roaCount, rpkiAspaMap) { + const now = new Date(); + const date = now.toISOString().slice(0, 10); // "YYYY-MM-DD" + + // Compute how many Atlas-visible ASNs have ASPA + let atlasTotal = 0; + let atlasWithAspa = 0; + if (atlasState.cache?.asns_with_probes) { + atlasTotal = atlasState.cache.asns_with_probes.length; + // rpkiAspaMap is keyed by integer ASN + atlasWithAspa = atlasState.cache.asns_with_probes.filter(a => rpkiAspaMap.has(Number(a))).length; + } + + const coveragePct = atlasTotal > 0 + ? Math.round((atlasWithAspa / atlasTotal) * 10000) / 100 // 2 decimal places + : 0; + + // Compute delta from previous snapshot + const prev = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1]; + const delta = prev ? aspaCount - prev.aspa_objects : 0; + + const snapshot = { + date, + ts: now.getTime(), + aspa_objects: aspaCount, + roa_count: roaCount, + atlas_asns_total: atlasTotal, + atlas_asns_with_aspa: atlasWithAspa, + coverage_pct_atlas: coveragePct, + aspa_delta: delta, + method: 'rpki_cloudflare_feed', + }; + + // Only store one snapshot per date (overwrite if same day) + const existingIdx = aspaAdoptionDailyHistory.findIndex(s => s.date === date); + if (existingIdx >= 0) { + aspaAdoptionDailyHistory[existingIdx] = snapshot; + } else { + aspaAdoptionDailyHistory.push(snapshot); + } + saveAspaAdoptionHistory(aspaAdoptionDailyHistory); + console.log(`[ASPA-ADOPT] Snapshot ${date}: ${aspaCount} objects, ${coveragePct}% Atlas coverage (${atlasWithAspa}/${atlasTotal})`); + return snapshot; +} + +// Get adoption trend for a given period string ('7d', '30d', '90d', '1y'). +function getAdoptionTrend(period) { + const days = period === '7d' ? 7 : period === '30d' ? 30 : period === '90d' ? 90 : 365; + const cutoff = Date.now() - days * 24 * 60 * 60 * 1000; + return aspaAdoptionDailyHistory.filter(s => s.ts >= cutoff); +} + +// Build the ASPA adoption dashboard HTML. +function buildAspaAdoptionDashboard() { + const latest = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1]; + const trend30 = getAdoptionTrend('30d'); + const trend7 = getAdoptionTrend('7d'); + + // Forecast: simple linear regression over last 30 days + function linearForecast(data, daysAhead) { + if (data.length < 2) return null; + const n = data.length; + const xs = data.map((_, i) => i); + const ys = data.map(d => d.coverage_pct_atlas); + const xMean = xs.reduce((a, b) => a + b, 0) / n; + const yMean = ys.reduce((a, b) => a + b, 0) / n; + const num = xs.reduce((s, x, i) => s + (x - xMean) * (ys[i] - yMean), 0); + const den = xs.reduce((s, x) => s + (x - xMean) ** 2, 0); + if (den === 0) return yMean; + const slope = num / den; + return Math.max(0, Math.min(100, yMean + slope * (n - 1 + daysAhead))); + } + + const forecast90d = linearForecast(trend30, 90); + const change7d = trend7.length >= 2 + ? (trend7[trend7.length-1].aspa_objects - trend7[0].aspa_objects) + : 0; + const trendLabels = trend30.map(d => d.date); + const trendValues = trend30.map(d => d.coverage_pct_atlas); + const asnValues = trend30.map(d => d.aspa_objects); + + const coverageNow = latest?.coverage_pct_atlas ?? 0; + const aspaNow = latest?.aspa_objects ?? 0; + const atlasTotal = latest?.atlas_asns_total ?? 0; + + return ` + + + + + ASPA Adoption Tracker · PeerCortex + + + + + +
+

ASPA Adoption Tracker

+

Global ASPA deployment trends · Powered by RPKI Cloudflare feed · Updated every ~4 hours

+ +
+
+
Atlas Coverage
+
${coverageNow.toFixed(1)}%
+
of Atlas-visible ASNs
+
+
+
ASPA Objects
+
${aspaNow.toLocaleString()}
+
total in RPKI
+
+
+
Atlas ASNs
+
${atlasTotal.toLocaleString()}
+
sampled denominator
+
+
+
7-Day Delta
+
${change7d > 0 ? '+' : ''}${change7d}
+
new ASPA objects
+
+
+
Data Points
+
${aspaAdoptionDailyHistory.length}
+
history snapshots
+
+
+ +
+ ↓ JSON (30d) + ↓ CSV (30d) + ↓ CSV (1y) + +
+ +
+
+

Atlas Coverage % (last 30 days)

+ +
+
+
90-Day Forecast
+
${forecast90d !== null ? forecast90d.toFixed(1) + '%' : '—'}
+
linear projection from 30-day trend
+
+ Based on ${trend30.length} data points.
+ Current: ${coverageNow.toFixed(1)}% +
+
+
+ +
+

Total ASPA Objects in RPKI (last 30 days)

+ +
+ +
+ Last snapshot: ${latest?.date ?? 'none'} · Denominator: ${atlasTotal.toLocaleString()} Atlas ASNs · + JSON API +
+
+ + + +`; +} + +module.exports = { + aspaAdoptionDailyHistory, + loadAspaAdoptionHistory, + saveAspaAdoptionHistory, + recordAspaAdoptionSnapshot, + getAdoptionTrend, + buildAspaAdoptionDashboard, +}; diff --git a/server/services/ripe-stat.js b/server/services/ripe-stat.js new file mode 100644 index 0000000..00a3480 --- /dev/null +++ b/server/services/ripe-stat.js @@ -0,0 +1,184 @@ +const fs = require("fs"); +const { fetchJSON } = require("./http-helpers"); +const localDb = require("../../local-db-client"); + +// RIPE Stat Source Cache + Semaphore (L2) +// Prevents 429 rate-limiting by throttling + caching responses +const ripeStatCache = new Map(); // key: "endpoint:resource" → {data, ts} +const RIPE_STAT_CACHE_MAX = 2000; +const RIPE_STAT_TTL = { + "announced-prefixes": 15 * 60 * 1000, + "asn-neighbours": 15 * 60 * 1000, + "as-overview": 60 * 60 * 1000, + "rir-stats-country": 24 * 60 * 60 * 1000, + "visibility": 15 * 60 * 1000, + "prefix-size-distribution": 60 * 60 * 1000, + "abuse-contact-finder": 24 * 60 * 60 * 1000, + "blocklist": 60 * 60 * 1000, + "reverse-dns-consistency": 60 * 60 * 1000, + "routing-status": 15 * 60 * 1000, + "bgp-updates": 15 * 60 * 1000, + "maxmind-geo-lite-pfx": 24 * 60 * 60 * 1000, + "looking-glass": 15 * 60 * 1000, + "whois": 24 * 60 * 60 * 1000, + "rpki-validation": 6 * 60 * 60 * 1000, +}; + +// Counting semaphore — limits concurrent RIPE Stat requests +class Semaphore { + constructor(max) { this.max = max; this.current = 0; this.queue = []; } + acquire() { + if (this.current < this.max) { this.current++; return Promise.resolve(); } + return new Promise((resolve) => this.queue.push(resolve)); + } + release() { + this.current--; + if (this.queue.length > 0) { this.current++; this.queue.shift()(); } + } +} +const ripeStatSemaphore = new Semaphore(15); + +// Cached + throttled RIPE Stat fetch +// Renamed from fetchRipeStatCached 2026-07-16: a second function with that exact +// name was added ~150 lines down (the local-DB-routing wrapper) during the Postgres +// refactor. In JS, two `function` declarations sharing a name silently collapse to +// the LAST one in source order -- the real RIPE Stat fetch+cache+throttle +// implementation below was completely shadowed, and every call to +// fetchRipeStatCached() for any endpoint NOT in {announced-prefixes, asn-neighbours, +// as-overview, visibility, prefix-size-distribution} was unconditionally returning +// null instead of ever reaching this code. That silently broke blocklist, +// abuse-contact-finder, bgp-updates, routing-status, looking-glass, whois, +// reverse-dns-consistency, maxmind-geo-lite-pfx, and ixs lookups -- see the router's +// fallback call to this function for the fix. +async function fetchRipeStatCachedFromApi(url, options) { + // Extract endpoint name from URL for TTL lookup + const match = url.match(/\/data\/([^/]+)\//); + const endpoint = match ? match[1] : "default"; + const resourceMatch = url.match(/resource=([^&]+)/); + const resource = resourceMatch ? resourceMatch[1] : url; + const cacheKey = endpoint + ":" + resource; + + // Check cache + const cached = ripeStatCache.get(cacheKey); + const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; + if (cached && (Date.now() - cached.ts) < ttl) { + return cached.data; + } + + // Throttle via semaphore + await ripeStatSemaphore.acquire(); + try { + // Double-check cache after acquiring semaphore (another request may have filled it) + const cached2 = ripeStatCache.get(cacheKey); + if (cached2 && (Date.now() - cached2.ts) < ttl) { + return cached2.data; + } + + const result = await fetchJSON(url, options); + + // Only cache successful results — never cache null (failed/rate-limited responses) + // Caching null causes cascading failures: retry hits cache, returns null again + if (result !== null) { + if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { + ripeStatCache.delete(ripeStatCache.keys().next().value); + } + ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); + } + return result; + } finally { + ripeStatSemaphore.release(); + } +} + +// Cached + throttled RIPE Stat with one retry on failure +async function fetchRipeStatCachedWithRetry(url, options) { + const result = await fetchRipeStatCached(url, options); + if (result !== null) return result; + await new Promise(r => setTimeout(r, 1500)); + return fetchRipeStatCached(url, options); +} + +// RIPE Stat cache disk persistence (skip null entries) +function saveRipeStatCacheToDisk(filePath) { + try { + const obj = {}; + for (const [k, v] of ripeStatCache) { + if (v.data !== null) obj[k] = v; + } + fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); + console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); + } catch (e) { + console.warn("[RIPE-CACHE] Disk save failed:", e.message); + } +} + +function loadRipeStatCacheFromDisk(filePath) { + try { + if (!fs.existsSync(filePath)) return false; + const raw = fs.readFileSync(filePath, "utf8"); + const data = JSON.parse(raw); + const now = Date.now(); + for (const [k, v] of Object.entries(data.entries || {})) { + const match = k.match(/^([^:]+):/); + const endpoint = match ? match[1] : "default"; + const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; + if (now - v.ts < ttl) { + ripeStatCache.set(k, v); + } + } + console.log("[RIPE-CACHE] Loaded " + ripeStatCache.size + " entries from disk"); + return true; + } catch (e) { + console.warn("[RIPE-CACHE] Disk load failed:", e.message); + return false; + } +} + +// MASTER RIPE Stat API wrapper (Local-first, zero external API calls). +// Analyzes RIPE Stat URL and dispatches to appropriate localDb function. +async function fetchRipeStatCached(url, options = {}) { + try { + // Detect which RIPE Stat endpoint this is and call local DB + if (url.includes('/announced-prefixes/')) { + const asnMatch = url.match(/resource=AS(\d+)/); + if (asnMatch) return await localDb.getRipeStatAnnouncedPrefixes(parseInt(asnMatch[1])); + } + if (url.includes('/asn-neighbours/')) { + const asnMatch = url.match(/resource=AS(\d+)/); + if (asnMatch) return await localDb.getRipeStatAsnNeighbours(parseInt(asnMatch[1])); + } + if (url.includes('/as-overview/')) { + const asnMatch = url.match(/resource=AS(\d+)/); + if (asnMatch) return await localDb.getRipeStatAsOverview(parseInt(asnMatch[1])); + } + if (url.includes('/visibility/')) { + const asnMatch = url.match(/resource=AS(\d+)/); + if (asnMatch) return await localDb.getRipeStatVisibility(parseInt(asnMatch[1])); + } + if (url.includes('/prefix-size-distribution/')) { + const asnMatch = url.match(/resource=AS(\d+)/); + if (asnMatch) return await localDb.getRipeStatPrefixSizeDistribution(parseInt(asnMatch[1])); + } + + // For every other RIPE Stat endpoint (not mirrored into the local Postgres DB) -- + // e.g. blocklist, abuse-contact-finder, bgp-updates, routing-status, looking-glass, + // whois, reverse-dns-consistency, maxmind-geo-lite-pfx, ixs -- fall through to the + // real RIPE Stat API with its own cache/throttle/never-cache-null protections, + // instead of returning null unconditionally (see fetchRipeStatCachedFromApi's + // header comment for how this silently broke ~9 checks until 2026-07-16). + return fetchRipeStatCachedFromApi(url, options); + } catch (e) { + console.error("[fetchRipeStatCached] Error:", e.message); + return Promise.resolve(null); + } +} + +module.exports = { + ripeStatCache, + fetchRipeStatCached, + fetchRipeStatCachedFromApi, + fetchRipeStatCachedWithRetry, + saveRipeStatCacheToDisk, + loadRipeStatCacheFromDisk, + Semaphore, +}; diff --git a/server/services/rpki.js b/server/services/rpki.js new file mode 100644 index 0000000..69fd6e5 --- /dev/null +++ b/server/services/rpki.js @@ -0,0 +1,200 @@ +const https = require("https"); +const localDb = require("../../local-db-client"); +const { UA } = require("../data/constants"); +const { fetchJSON } = require("./http-helpers"); +const { roaStore } = require("../caches/roa-store"); +const { recordAspaAdoptionSnapshot } = require("../aspa-adoption/tracker"); + +// RPKI ASPA + ROA Cache from Cloudflare RPKI JSON feed +const rpkiAspaMap = new Map(); // customer_asid -> Set +let rpkiAspaLastFetch = 0; +let rpkiAspaFetching = false; + +function fetchRpkiAspaFeed() { + if (rpkiAspaFetching) return Promise.resolve(); + rpkiAspaFetching = true; + console.log("[RPKI] Fetching Cloudflare RPKI feed (ASPA + ROA)..."); + return new Promise((resolve) => { + const options = { + headers: { "User-Agent": UA }, + timeout: 120000, + }; + https.get("https://rpki.cloudflare.com/rpki.json", options, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => { + try { + const parsed = JSON.parse(data); + + // Load ASPA objects + const aspas = parsed.aspas || []; + rpkiAspaMap.clear(); + aspas.forEach((a) => { + const customerAsid = Number(a.customer_asid); + const providers = (a.providers || []).map(Number); + rpkiAspaMap.set(customerAsid, new Set(providers)); + }); + + // Load ROA objects into local store (eliminates RIPE Stat per-prefix calls) + const roas = parsed.roas || []; + roaStore.build(roas); + roaStore.saveToDisk("/opt/peercortex-app/.roa-cache.json"); + + // Track ASPA adoption — persist to disk + update in-memory trend + recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count, rpkiAspaMap); + + rpkiAspaLastFetch = Date.now(); + console.log("[RPKI] Loaded " + rpkiAspaMap.size + " ASPA objects + " + roaStore.count + " ROAs from Cloudflare feed"); + } catch (e) { + console.error("[RPKI] Failed to parse RPKI feed:", e.message); + } + rpkiAspaFetching = false; + resolve(); + }); + }).on("error", (e) => { + console.error("[RPKI] Fetch failed:", e.message); + rpkiAspaFetching = false; + resolve(); + }); + }); +} + +// Ensure ASPA + ROA cache is fresh +async function ensureAspaCache() { + if (Date.now() - rpkiAspaLastFetch > 4 * 60 * 60 * 1000) { + await fetchRpkiAspaFeed(); + } +} + +// Lookup ASPA object for a given ASN from the RPKI feed cache. feedLoaded +// distinguishes "the feed loaded fine and this ASN genuinely has no ASPA object" +// from "the Cloudflare feed fetch has never succeeded, so we don't actually know" -- +// rpkiAspaLastFetch is only set on a successful parse (see fetchRpkiAspaFeed), so +// 0 means every attempt so far has failed and exists:false here is not trustworthy. +function lookupAspaFromRpki(asn) { + const asnNum = Number(asn); + const feedLoaded = rpkiAspaLastFetch > 0; + if (rpkiAspaMap.has(asnNum)) { + const providers = rpkiAspaMap.get(asnNum); + return { exists: true, providers: [...providers].sort((a, b) => a - b), feedLoaded }; + } + return { exists: false, providers: [], feedLoaded }; +} + +// Validate RPKI for a prefix — uses local PostgreSQL database (sub-10ms, zero external API calls) +// Returns: { prefix, status: "valid"|"invalid"|"not_found"|"unavailable", validating_roas: N } +// "not_found" means the DB was queried and genuinely has no covering ROA -- a real result. +// "unavailable" means the DB query itself failed (connection/timeout/error) -- we don't know. +// Conflating these used to make DB hiccups masquerade as "no ROA published" (a real, worse- +// sounding finding). Callers computing coverage percentages should exclude "unavailable". +async function validateRPKIWithCache(asn, prefix) { + try { + const result = await localDb.validateRpki(prefix, asn); + if (result.status === 'valid') { + return { prefix, status: "valid", validating_roas: 1 }; + } else if (result.status === 'invalid') { + return { prefix, status: "invalid", validating_roas: 1 }; + } else if (result.status === 'unknown') { + return { prefix, status: "unavailable", validating_roas: 0 }; + } else { + // 'not-found': genuinely no covering ROA + return { prefix, status: "not_found", validating_roas: 0 }; + } + } catch (_e) { + console.error("[RPKI] Error validating " + prefix + ":", _e.message); + return { prefix, status: "unavailable", validating_roas: 0 }; + } +} + +// Feature 30: RIPE NCC RPKI Validator cross-check (max 5 prefixes) +async function fetchRipeRpkiValidator(asn, prefix) { + try { + const encoded = encodeURIComponent(prefix); + const url = "https://rpki-validator.ripe.net/api/v1/validity/AS" + asn + "/" + encoded; + const result = await fetchJSON(url, { timeout: 5000 }); + if (result && result.validated_route) { + return { + prefix: prefix, + validity: result.validated_route.validity || {}, + state: (result.validated_route.validity && result.validated_route.validity.state) || "unknown", + }; + } + return { prefix: prefix, state: "unknown", error: "no_data" }; + } catch (_e) { + return { prefix: prefix, state: "error", error: "timeout_or_unavailable" }; + } +} + +// Cross-check a sample of prefixes against RIPE RPKI Validator (max 5, in parallel). +// agreement_pct only counts pairs where BOTH sources actually returned a real +// verdict -- a zero-sample or all-RIPE-lookups-failed result reports agreement_pct: +// null (not a fabricated 100), since nothing was actually cross-checked. +async function crossCheckRpki(asn, prefixes, localResults) { + const sample = prefixes.slice(0, 5); + if (sample.length === 0) return { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 }; + + const ripeResults = await Promise.all( + sample.map((pfx) => fetchRipeRpkiValidator(asn, pfx)) + ); + + const localMap = new Map(); + for (const lr of localResults) { + localMap.set(lr.prefix, lr.status); + } + + let cloudflareValid = 0; + let ripeValid = 0; + let agreements = 0; + let comparable = 0; + const disagreements = []; + + for (let i = 0; i < sample.length; i++) { + const pfx = sample[i]; + const cfStatus = localMap.get(pfx) || "not_found"; + const ripeState = ripeResults[i].state; + + const cfIsValid = cfStatus === "valid"; + const ripeIsValid = ripeState === "valid" || ripeState === "VALID"; + + if (cfIsValid) cloudflareValid++; + if (ripeIsValid) ripeValid++; + + // A failed/unknown RIPE lookup isn't a comparison at all -- exclude it from + // the agreement percentage instead of counting it as a silent "agreement". + if (ripeState === "error" || ripeState === "unknown") { + continue; + } + + comparable++; + if (cfIsValid === ripeIsValid) { + agreements++; + } else { + disagreements.push({ + prefix: pfx, + cloudflare: cfStatus, + ripe: ripeState, + }); + } + } + + const agreementPct = comparable > 0 ? Math.round((agreements / comparable) * 100) : null; + return { cloudflare_valid: cloudflareValid, ripe_valid: ripeValid, agreement_pct: agreementPct, disagreements: disagreements, sample_size: sample.length, comparable_size: comparable }; +} + +// rpkiAspaLastFetch is a reassigned `let`, not a mutated object -- a bare +// export would only capture its value (0) at require time. Expose it via a +// getter so callers (e.g. /api/health) always see the current value. +function getRpkiAspaLastFetch() { + return rpkiAspaLastFetch; +} + +module.exports = { + rpkiAspaMap, + getRpkiAspaLastFetch, + fetchRpkiAspaFeed, + ensureAspaCache, + lookupAspaFromRpki, + validateRPKIWithCache, + fetchRipeRpkiValidator, + crossCheckRpki, +};