const fs = require("fs"); const { fetchJSON } = require("./http-helpers"); const localDb = require("../../local-db-client"); // RIPE Stat Source Cache + Semaphore (L2) // Prevents 429 rate-limiting by throttling + caching responses const ripeStatCache = new Map(); // key: "endpoint:resource" → {data, ts} const RIPE_STAT_CACHE_MAX = 2000; const RIPE_STAT_TTL = { "announced-prefixes": 15 * 60 * 1000, "asn-neighbours": 15 * 60 * 1000, "as-overview": 60 * 60 * 1000, "rir-stats-country": 24 * 60 * 60 * 1000, "visibility": 15 * 60 * 1000, "prefix-size-distribution": 60 * 60 * 1000, "abuse-contact-finder": 24 * 60 * 60 * 1000, "blocklist": 60 * 60 * 1000, "reverse-dns-consistency": 60 * 60 * 1000, "routing-status": 15 * 60 * 1000, "bgp-updates": 15 * 60 * 1000, "maxmind-geo-lite-pfx": 24 * 60 * 60 * 1000, "looking-glass": 15 * 60 * 1000, "whois": 24 * 60 * 60 * 1000, "rpki-validation": 6 * 60 * 60 * 1000, }; // Counting semaphore — limits concurrent RIPE Stat requests class Semaphore { constructor(max) { this.max = max; this.current = 0; this.queue = []; } acquire() { if (this.current < this.max) { this.current++; return Promise.resolve(); } return new Promise((resolve) => this.queue.push(resolve)); } release() { this.current--; if (this.queue.length > 0) { this.current++; this.queue.shift()(); } } } const ripeStatSemaphore = new Semaphore(15); // Cached + throttled RIPE Stat fetch // Renamed from fetchRipeStatCached 2026-07-16: a second function with that exact // name was added ~150 lines down (the local-DB-routing wrapper) during the Postgres // refactor. In JS, two `function` declarations sharing a name silently collapse to // the LAST one in source order -- the real RIPE Stat fetch+cache+throttle // implementation below was completely shadowed, and every call to // fetchRipeStatCached() for any endpoint NOT in {announced-prefixes, asn-neighbours, // as-overview, visibility, prefix-size-distribution} was unconditionally returning // null instead of ever reaching this code. That silently broke blocklist, // abuse-contact-finder, bgp-updates, routing-status, looking-glass, whois, // reverse-dns-consistency, maxmind-geo-lite-pfx, and ixs lookups -- see the router's // fallback call to this function for the fix. async function fetchRipeStatCachedFromApi(url, options) { // Extract endpoint name from URL for TTL lookup const match = url.match(/\/data\/([^/]+)\//); const endpoint = match ? match[1] : "default"; const resourceMatch = url.match(/resource=([^&]+)/); const resource = resourceMatch ? resourceMatch[1] : url; const cacheKey = endpoint + ":" + resource; // Check cache const cached = ripeStatCache.get(cacheKey); const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; if (cached && (Date.now() - cached.ts) < ttl) { return cached.data; } // Throttle via semaphore await ripeStatSemaphore.acquire(); try { // Double-check cache after acquiring semaphore (another request may have filled it) const cached2 = ripeStatCache.get(cacheKey); if (cached2 && (Date.now() - cached2.ts) < ttl) { return cached2.data; } const result = await fetchJSON(url, options); // Only cache successful results — never cache null (failed/rate-limited responses) // Caching null causes cascading failures: retry hits cache, returns null again if (result !== null) { if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { ripeStatCache.delete(ripeStatCache.keys().next().value); } ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); } return result; } finally { ripeStatSemaphore.release(); } } // Cached + throttled RIPE Stat with one retry on failure async function fetchRipeStatCachedWithRetry(url, options) { const result = await fetchRipeStatCached(url, options); if (result !== null) return result; await new Promise(r => setTimeout(r, 1500)); return fetchRipeStatCached(url, options); } // RIPE Stat cache disk persistence (skip null entries) function saveRipeStatCacheToDisk(filePath) { try { const obj = {}; for (const [k, v] of ripeStatCache) { if (v.data !== null) obj[k] = v; } fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); } catch (e) { console.warn("[RIPE-CACHE] Disk save failed:", e.message); } } function loadRipeStatCacheFromDisk(filePath) { try { if (!fs.existsSync(filePath)) return false; const raw = fs.readFileSync(filePath, "utf8"); const data = JSON.parse(raw); const now = Date.now(); for (const [k, v] of Object.entries(data.entries || {})) { const match = k.match(/^([^:]+):/); const endpoint = match ? match[1] : "default"; const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000; if (now - v.ts < ttl) { ripeStatCache.set(k, v); } } console.log("[RIPE-CACHE] Loaded " + ripeStatCache.size + " entries from disk"); return true; } catch (e) { console.warn("[RIPE-CACHE] Disk load failed:", e.message); return false; } } // MASTER RIPE Stat API wrapper (Local-first, zero external API calls). // Analyzes RIPE Stat URL and dispatches to appropriate localDb function. async function fetchRipeStatCached(url, options = {}) { try { // Detect which RIPE Stat endpoint this is and call local DB if (url.includes('/announced-prefixes/')) { const asnMatch = url.match(/resource=AS(\d+)/); if (asnMatch) return await localDb.getRipeStatAnnouncedPrefixes(parseInt(asnMatch[1])); } if (url.includes('/asn-neighbours/')) { const asnMatch = url.match(/resource=AS(\d+)/); if (asnMatch) return await localDb.getRipeStatAsnNeighbours(parseInt(asnMatch[1])); } if (url.includes('/as-overview/')) { const asnMatch = url.match(/resource=AS(\d+)/); if (asnMatch) return await localDb.getRipeStatAsOverview(parseInt(asnMatch[1])); } if (url.includes('/visibility/')) { const asnMatch = url.match(/resource=AS(\d+)/); if (asnMatch) return await localDb.getRipeStatVisibility(parseInt(asnMatch[1])); } if (url.includes('/prefix-size-distribution/')) { const asnMatch = url.match(/resource=AS(\d+)/); if (asnMatch) return await localDb.getRipeStatPrefixSizeDistribution(parseInt(asnMatch[1])); } // For every other RIPE Stat endpoint (not mirrored into the local Postgres DB) -- // e.g. blocklist, abuse-contact-finder, bgp-updates, routing-status, looking-glass, // whois, reverse-dns-consistency, maxmind-geo-lite-pfx, ixs -- fall through to the // real RIPE Stat API with its own cache/throttle/never-cache-null protections, // instead of returning null unconditionally (see fetchRipeStatCachedFromApi's // header comment for how this silently broke ~9 checks until 2026-07-16). return fetchRipeStatCachedFromApi(url, options); } catch (e) { console.error("[fetchRipeStatCached] Error:", e.message); return Promise.resolve(null); } } module.exports = { ripeStatCache, fetchRipeStatCached, fetchRipeStatCachedFromApi, fetchRipeStatCachedWithRetry, saveRipeStatCacheToDisk, loadRipeStatCacheFromDisk, Semaphore, };