rene/PeerCortex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: reduce all remaining long RIPE Stat timeouts, add validate result cache

Browse Source 
- reverse-dns-consistency: 15s → 5s per prefix
- route-leak asn-neighbours: 30s → 8s
- comparison endpoint: 4x fetchRipeStatCached 30s → 8s
- validate result cache: 15min TTL, ~18ms hit vs 700ms+ cold

Prevents semaphore starvation — slow validate calls were blocking
ASPA/WHOIS/bgproutes from acquiring semaphore slots.
This commit is contained in:
Rene Fichtmueller 2026-04-09 08:09:03 +02:00
parent 487b032661
commit e1dcbe517f
2 changed files with 37 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG_PENDING.md
View File

@ -16,3 +16,5 @@
{"d":"2026-04-08","t":"FIX","m":"Peering Recommendations: replace 20 concurrent full lookup calls with new /api/quick-ix endpoint — was hanging indefinitely on every new lookup"} {"d":"2026-04-08","t":"FIX","m":"Peering Recommendations: replace 20 concurrent full lookup calls with new /api/quick-ix endpoint — was hanging indefinitely on every new lookup"}
{"d":"2026-04-08","t":"FIX","m":"ASPA: reduce looking-glass timeout 8s→5s and hard cap 18s→12s — faster response for slow RIPE Stat endpoints"} {"d":"2026-04-08","t":"FIX","m":"ASPA: reduce looking-glass timeout 8s→5s and hard cap 18s→12s — faster response for slow RIPE Stat endpoints"}
{"d":"2026-04-08","t":"FEAT","m":"New /api/quick-ix endpoint: lightweight PeeringDB IX connections + network name, 1h cache"} {"d":"2026-04-08","t":"FEAT","m":"New /api/quick-ix endpoint: lightweight PeeringDB IX connections + network name, 1h cache"}
{"d":"2026-04-08","t":"FIX","m":"validate: reduce reverse-dns timeout 15s→5s, route-leak asn-neighbours 30s→8s, comparison endpoint 4x 30s→8s — prevent semaphore starvation"}
{"d":"2026-04-08","t":"FEAT","m":"validate: add 15min result cache — subsequent lookups return in ~18ms vs 700ms+ cold"}

40
server.js
View File

@ -549,6 +549,7 @@ const BGPROUTES_VP_TTL = 60 * 60 * 1000; // 1 hour
// ============================================================ // ============================================================
const bgproutesResultCache = new Map(); const bgproutesResultCache = new Map();
const aspaResultCache = new Map(); const aspaResultCache = new Map();
const validateResultCache = new Map();
const RESULT_CACHE_TTL = 15 * 60 * 1000; // 15 minutes const RESULT_CACHE_TTL = 15 * 60 * 1000; // 15 minutes
function resultCacheGet(map, key) { function resultCacheGet(map, key) {
const e = map.get(String(key)); const e = map.get(String(key));
@ -2917,15 +2918,20 @@ const server = http.createServer(async (req, res) => {
res.writeHead(400); res.writeHead(400);
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" })); return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
} }
const cachedValidate = resultCacheGet(validateResultCache, rawAsn);
if (cachedValidate !== undefined) {
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
return res.end(JSON.stringify(cachedValidate));
}
const start = Date.now(); const start = Date.now();
const targetAsn = parseInt(rawAsn); const targetAsn = parseInt(rawAsn);
try { try {
// Phase 1: Fetch core data needed by multiple validations // Phase 1: Fetch core data needed by multiple validations
const [prefixData, pdbNet, neighbourData, overviewData] = await Promise.all([ const [prefixData, pdbNet, neighbourData, overviewData] = await Promise.all([
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + rawAsn, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + rawAsn, { timeout: 8000 }),
fetchPeeringDB("/net?asn=" + rawAsn), fetchPeeringDB("/net?asn=" + rawAsn),
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, { timeout: 8000 }),
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + rawAsn), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + rawAsn),
]); ]);
@ -3058,7 +3064,7 @@ const server = http.createServer(async (req, res) => {
var rdnsSampleSize = Math.min(20, samplePrefixes.length); var rdnsSampleSize = Math.min(20, samplePrefixes.length);
validationPromises.rdns = Promise.all( validationPromises.rdns = Promise.all(
samplePrefixes.slice(0, rdnsSampleSize).map(function(pfx) { samplePrefixes.slice(0, rdnsSampleSize).map(function(pfx) {
return fetchRipeStatCached("https://stat.ripe.net/data/reverse-dns-consistency/data.json?resource=" + encodeURIComponent(pfx), { timeout: 15000 }).then(function(data) { return fetchRipeStatCached("https://stat.ripe.net/data/reverse-dns-consistency/data.json?resource=" + encodeURIComponent(pfx), { timeout: 5000 }).then(function(data) {
var pfxData = data && data.data && data.data.prefixes ? data.data.prefixes : {}; var pfxData = data && data.data && data.data.prefixes ? data.data.prefixes : {};
var hasDelegation = false; var hasDelegation = false;
var details = []; var details = [];
@ -3094,7 +3100,7 @@ const server = http.createServer(async (req, res) => {
}).catch(function(e) { return { status: "error", error: String(e) }; }); }).catch(function(e) { return { status: "error", error: String(e) }; });
// 18. BGP Visibility (uses routing-status API which is more reliable than visibility API) // 18. BGP Visibility (uses routing-status API which is more reliable than visibility API)
validationPromises.visibility = fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=AS" + rawAsn, { timeout: 20000 }).then(function(rsData) { validationPromises.visibility = fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=AS" + rawAsn, { timeout: 8000 }).then(function(rsData) {
var vis = rsData && rsData.data && rsData.data.visibility ? rsData.data.visibility : {}; var vis = rsData && rsData.data && rsData.data.visibility ? rsData.data.visibility : {};
var v4 = vis.v4 || {}; var v4 = vis.v4 || {};
var v6 = vis.v6 || {}; var v6 = vis.v6 || {};
@ -3345,9 +3351,7 @@ const server = http.createServer(async (req, res) => {
.slice(0, 30) .slice(0, 30)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; }); .map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
return res.end( const validateResult = {
JSON.stringify(
{
meta: { query: "AS" + rawAsn, duration_ms: duration, timestamp: new Date().toISOString(), total_prefixes: allPrefixes.length, prefixes_sampled: samplePrefixes.length }, meta: { query: "AS" + rawAsn, duration_ms: duration, timestamp: new Date().toISOString(), total_prefixes: allPrefixes.length, prefixes_sampled: samplePrefixes.length },
asn: targetAsn, asn: targetAsn,
name: net.name || (overviewData && overviewData.data ? overviewData.data.holder : "") || "Unknown", name: net.name || (overviewData && overviewData.data ? overviewData.data.holder : "") || "Unknown",
@ -3362,11 +3366,9 @@ const server = http.createServer(async (req, res) => {
source: "RIPE Stat asn-neighbours", source: "RIPE Stat asn-neighbours",
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.", note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
}, },
}, };
null, resultCacheSet(validateResultCache, rawAsn, validateResult);
2 return res.end(JSON.stringify(validateResult, null, 2));
)
);
} catch (err) { } catch (err) {
res.writeHead(500); res.writeHead(500);
return res.end(JSON.stringify({ error: "Validation failed", message: err.message })); return res.end(JSON.stringify({ error: "Validation failed", message: err.message }));
@ -3438,8 +3440,8 @@ const server = http.createServer(async (req, res) => {
]).then(d => { rdapCacheSet(asn, d); return d; }); ]).then(d => { rdapCacheSet(asn, d); return d; });
const promises = [ const promises = [
timedFetch("RIPE Stat Prefixes", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 20000 })), timedFetch("RIPE Stat Prefixes", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 8000 })),
timedFetch("RIPE Stat Neighbours", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 20000 })), timedFetch("RIPE Stat Neighbours", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 8000 })),
timedFetch("RIPE Stat Overview", fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn)), timedFetch("RIPE Stat Overview", fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn)),
timedFetch("RIPE Stat RIR", fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn)), timedFetch("RIPE Stat RIR", fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn)),
timedFetch("RIPE Atlas", fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500")), timedFetch("RIPE Atlas", fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500")),
@ -3983,7 +3985,7 @@ const server = http.createServer(async (req, res) => {
try { try {
const neighbourData = await fetchRipeStatCached( const neighbourData = await fetchRipeStatCached(
"https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1", "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1",
{ timeout: 30000 } { timeout: 8000 }
); );
const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || []; const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || [];
const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {}; const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {};
@ -4062,10 +4064,10 @@ const server = http.createServer(async (req, res) => {
const [pdb1, pdb2, nb1Data, nb2Data, pfx1Data, pfx2Data] = await Promise.all([ const [pdb1, pdb2, nb1Data, nb2Data, pfx1Data, pfx2Data] = await Promise.all([
fetchPeeringDB("/net?asn=" + asn1), fetchPeeringDB("/net?asn=" + asn1),
fetchPeeringDB("/net?asn=" + asn2), fetchPeeringDB("/net?asn=" + asn2),
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 8000 }),
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 8000 }),
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 8000 }),
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 30000 }), fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 8000 }),
]); ]);
const net1 = pdb1?.data?.[0] || {}; const net1 = pdb1?.data?.[0] || {};