diff --git a/deploy/server.js b/deploy/server.js index 76cf005..0158c84 100644 --- a/deploy/server.js +++ b/deploy/server.js @@ -445,7 +445,7 @@ class Semaphore { if (this.queue.length > 0) { this.current++; this.queue.shift()(); } } } -const ripeStatSemaphore = new Semaphore(10); +const ripeStatSemaphore = new Semaphore(15); // Cached + throttled RIPE Stat fetch async function fetchRipeStatCached(url, options) { @@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) { const result = await fetchJSON(url, options); - // Store in cache (evict oldest if full) - if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { - ripeStatCache.delete(ripeStatCache.keys().next().value); + // Only cache successful results — never cache null (failed/rate-limited responses) + // Caching null causes cascading failures: retry hits cache, returns null again + if (result !== null) { + if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { + ripeStatCache.delete(ripeStatCache.keys().next().value); + } + ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); } - ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); return result; } finally { ripeStatSemaphore.release(); @@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) { async function fetchRipeStatCachedWithRetry(url, options) { const result = await fetchRipeStatCached(url, options); if (result !== null) return result; - await new Promise(r => setTimeout(r, 1000)); + await new Promise(r => setTimeout(r, 1500)); return fetchRipeStatCached(url, options); } -// RIPE Stat cache disk persistence +// RIPE Stat cache disk persistence (skip null entries) function saveRipeStatCacheToDisk(filePath) { try { const obj = {}; - for (const [k, v] of ripeStatCache) obj[k] = v; + for (const [k, v] of ripeStatCache) { + if (v.data !== null) obj[k] = v; + } fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); - console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); + console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); } catch (e) { console.warn("[RIPE-CACHE] Disk save failed:", e.message); } @@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => { let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; const promises = [ - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"), diff --git a/server.js b/server.js index 76cf005..0158c84 100644 --- a/server.js +++ b/server.js @@ -445,7 +445,7 @@ class Semaphore { if (this.queue.length > 0) { this.current++; this.queue.shift()(); } } } -const ripeStatSemaphore = new Semaphore(10); +const ripeStatSemaphore = new Semaphore(15); // Cached + throttled RIPE Stat fetch async function fetchRipeStatCached(url, options) { @@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) { const result = await fetchJSON(url, options); - // Store in cache (evict oldest if full) - if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { - ripeStatCache.delete(ripeStatCache.keys().next().value); + // Only cache successful results — never cache null (failed/rate-limited responses) + // Caching null causes cascading failures: retry hits cache, returns null again + if (result !== null) { + if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { + ripeStatCache.delete(ripeStatCache.keys().next().value); + } + ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); } - ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); return result; } finally { ripeStatSemaphore.release(); @@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) { async function fetchRipeStatCachedWithRetry(url, options) { const result = await fetchRipeStatCached(url, options); if (result !== null) return result; - await new Promise(r => setTimeout(r, 1000)); + await new Promise(r => setTimeout(r, 1500)); return fetchRipeStatCached(url, options); } -// RIPE Stat cache disk persistence +// RIPE Stat cache disk persistence (skip null entries) function saveRipeStatCacheToDisk(filePath) { try { const obj = {}; - for (const [k, v] of ripeStatCache) obj[k] = v; + for (const [k, v] of ripeStatCache) { + if (v.data !== null) obj[k] = v; + } fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); - console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); + console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk"); } catch (e) { console.warn("[RIPE-CACHE] Disk save failed:", e.message); } @@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => { let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; const promises = [ - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), - fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }), + fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),