From 8f51f32dc3ce98b528e58dc4171ca6cd47a9ec41 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Mon, 30 Mar 2026 07:58:24 +0200 Subject: [PATCH] fix: never cache null responses + increase RIPE Stat timeout for large carriers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of neighbour=0 for large carriers (AS9002, AS3491, AS12956): 1. RIPE Stat asn-neighbours returns 5000+ entries for Tier-1 carriers, exceeding the 30s timeout → fetchJSON returns null 2. null was cached in ripeStatCache for 15 minutes (the endpoint TTL) 3. All subsequent requests hit the null cache → perpetual 0 neighbours Fixes: - Never cache null results in ripeStatCache (only successful responses) - Never persist null entries to disk cache - Increase RIPE Stat timeout from 30s to 45s for prefix/neighbour queries - Increase RIPE Stat semaphore from 10 to 15 concurrent requests Verified: AS9002 up=146 down=2702, AS3491 up=90 down=710 --- deploy/server.js | 27 ++++++++++++++++----------- server.js | 27 ++++++++++++++++----------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/deploy/server.js b/deploy/server.js index 76cf005..0158c84 100644 --- a/deploy/server.js +++ b/deploy/server.js @@ -445,7 +445,7 @@ class Semaphore { if (this.queue.length > 0) { this.current++; this.queue.shift()(); } } } -const ripeStatSemaphore = new Semaphore(10); +const ripeStatSemaphore = new Semaphore(15); // Cached + throttled RIPE Stat fetch async function fetchRipeStatCached(url, options) { @@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) { const result = await fetchJSON(url, options); - // Store in cache (evict oldest if full) - if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { - ripeStatCache.delete(ripeStatCache.keys().next().value); + // Only cache successful results — never cache null (failed/rate-limited responses) + // Caching null causes cascading failures: retry hits cache, returns null again + if (result !== null) { + if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { + ripeStatCache.delete(ripeStatCache.keys().next().value); + } + ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); } - ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); return result; } finally { ripeStatSemaphore.release(); @@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) { async function fetchRipeStatCachedWithRetry(url, options) { const result = await fetchRipeStatCached(url, options); if (result !== null) return result; - await new Promise(r => setTimeout(r, 1000)); + await new Promise(r => setTimeout(r, 1500)); return fetchRipeStatCached(url, options); } -// RIPE Stat cache disk persistence +// RIPE Stat cache disk persistence (skip null entries) function saveRipeStatCacheToDisk(filePath) { try { const obj = {}; - for (const [k, v] of ripeStatCache) obj[k] = v; + for (const [k, v] of ripeStatCache) { + if (v.data !== null) obj[k] = v; + } fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); - console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); + console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); } catch (e) { console.warn("[RIPE-CACHE] Disk save failed:", e.message); } @@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => { let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; const promises = [ - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"), diff --git a/server.js b/server.js index 76cf005..0158c84 100644 --- a/server.js +++ b/server.js @@ -445,7 +445,7 @@ class Semaphore { if (this.queue.length > 0) { this.current++; this.queue.shift()(); } } } -const ripeStatSemaphore = new Semaphore(10); +const ripeStatSemaphore = new Semaphore(15); // Cached + throttled RIPE Stat fetch async function fetchRipeStatCached(url, options) { @@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) { const result = await fetchJSON(url, options); - // Store in cache (evict oldest if full) - if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { - ripeStatCache.delete(ripeStatCache.keys().next().value); + // Only cache successful results — never cache null (failed/rate-limited responses) + // Caching null causes cascading failures: retry hits cache, returns null again + if (result !== null) { + if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { + ripeStatCache.delete(ripeStatCache.keys().next().value); + } + ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); } - ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); return result; } finally { ripeStatSemaphore.release(); @@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) { async function fetchRipeStatCachedWithRetry(url, options) { const result = await fetchRipeStatCached(url, options); if (result !== null) return result; - await new Promise(r => setTimeout(r, 1000)); + await new Promise(r => setTimeout(r, 1500)); return fetchRipeStatCached(url, options); } -// RIPE Stat cache disk persistence +// RIPE Stat cache disk persistence (skip null entries) function saveRipeStatCacheToDisk(filePath) { try { const obj = {}; - for (const [k, v] of ripeStatCache) obj[k] = v; + for (const [k, v] of ripeStatCache) { + if (v.data !== null) obj[k] = v; + } fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); - console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); + console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); } catch (e) { console.warn("[RIPE-CACHE] Disk save failed:", e.message); } @@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => { let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; const promises = [ - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),