fix: never cache null responses + increase RIPE Stat timeout for large carriers
Root cause of neighbour=0 for large carriers (AS9002, AS3491, AS12956): 1. RIPE Stat asn-neighbours returns 5000+ entries for Tier-1 carriers, exceeding the 30s timeout → fetchJSON returns null 2. null was cached in ripeStatCache for 15 minutes (the endpoint TTL) 3. All subsequent requests hit the null cache → perpetual 0 neighbours Fixes: - Never cache null results in ripeStatCache (only successful responses) - Never persist null entries to disk cache - Increase RIPE Stat timeout from 30s to 45s for prefix/neighbour queries - Increase RIPE Stat semaphore from 10 to 15 concurrent requests Verified: AS9002 up=146 down=2702, AS3491 up=90 down=710
This commit is contained in:
Rene Fichtmueller
parent
9bc1292bac
commit
8f51f32dc3
@ -445,7 +445,7 @@ class Semaphore {
|
||||
if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
|
||||
}
|
||||
}
|
||||
const ripeStatSemaphore = new Semaphore(10);
|
||||
const ripeStatSemaphore = new Semaphore(15);
|
||||
|
||||
// Cached + throttled RIPE Stat fetch
|
||||
async function fetchRipeStatCached(url, options) {
|
||||
@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) {
|
||||
|
||||
const result = await fetchJSON(url, options);
|
||||
|
||||
// Store in cache (evict oldest if full)
|
||||
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
||||
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
||||
// Only cache successful results — never cache null (failed/rate-limited responses)
|
||||
// Caching null causes cascading failures: retry hits cache, returns null again
|
||||
if (result !== null) {
|
||||
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
||||
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
||||
}
|
||||
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
||||
}
|
||||
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
||||
return result;
|
||||
} finally {
|
||||
ripeStatSemaphore.release();
|
||||
@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) {
|
||||
async function fetchRipeStatCachedWithRetry(url, options) {
|
||||
const result = await fetchRipeStatCached(url, options);
|
||||
if (result !== null) return result;
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
return fetchRipeStatCached(url, options);
|
||||
}
|
||||
|
||||
// RIPE Stat cache disk persistence
|
||||
// RIPE Stat cache disk persistence (skip null entries)
|
||||
function saveRipeStatCacheToDisk(filePath) {
|
||||
try {
|
||||
const obj = {};
|
||||
for (const [k, v] of ripeStatCache) obj[k] = v;
|
||||
for (const [k, v] of ripeStatCache) {
|
||||
if (v.data !== null) obj[k] = v;
|
||||
}
|
||||
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
|
||||
console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk");
|
||||
console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
|
||||
} catch (e) {
|
||||
console.warn("[RIPE-CACHE] Disk save failed:", e.message);
|
||||
}
|
||||
@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => {
|
||||
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
|
||||
|
||||
const promises = [
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn),
|
||||
fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),
|
||||
|
||||
27
server.js
27
server.js
@ -445,7 +445,7 @@ class Semaphore {
|
||||
if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
|
||||
}
|
||||
}
|
||||
const ripeStatSemaphore = new Semaphore(10);
|
||||
const ripeStatSemaphore = new Semaphore(15);
|
||||
|
||||
// Cached + throttled RIPE Stat fetch
|
||||
async function fetchRipeStatCached(url, options) {
|
||||
@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) {
|
||||
|
||||
const result = await fetchJSON(url, options);
|
||||
|
||||
// Store in cache (evict oldest if full)
|
||||
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
||||
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
||||
// Only cache successful results — never cache null (failed/rate-limited responses)
|
||||
// Caching null causes cascading failures: retry hits cache, returns null again
|
||||
if (result !== null) {
|
||||
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
||||
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
||||
}
|
||||
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
||||
}
|
||||
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
||||
return result;
|
||||
} finally {
|
||||
ripeStatSemaphore.release();
|
||||
@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) {
|
||||
async function fetchRipeStatCachedWithRetry(url, options) {
|
||||
const result = await fetchRipeStatCached(url, options);
|
||||
if (result !== null) return result;
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
return fetchRipeStatCached(url, options);
|
||||
}
|
||||
|
||||
// RIPE Stat cache disk persistence
|
||||
// RIPE Stat cache disk persistence (skip null entries)
|
||||
function saveRipeStatCacheToDisk(filePath) {
|
||||
try {
|
||||
const obj = {};
|
||||
for (const [k, v] of ripeStatCache) obj[k] = v;
|
||||
for (const [k, v] of ripeStatCache) {
|
||||
if (v.data !== null) obj[k] = v;
|
||||
}
|
||||
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
|
||||
console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk");
|
||||
console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
|
||||
} catch (e) {
|
||||
console.warn("[RIPE-CACHE] Disk save failed:", e.message);
|
||||
}
|
||||
@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => {
|
||||
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
|
||||
|
||||
const promises = [
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }),
|
||||
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn),
|
||||
fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user