fix: reduce all remaining long RIPE Stat timeouts, add validate result cache
- reverse-dns-consistency: 15s → 5s per prefix - route-leak asn-neighbours: 30s → 8s - comparison endpoint: 4x fetchRipeStatCached 30s → 8s - validate result cache: 15min TTL, ~18ms hit vs 700ms+ cold Prevents semaphore starvation — slow validate calls were blocking ASPA/WHOIS/bgproutes from acquiring semaphore slots.
This commit is contained in:
Rene Fichtmueller
parent
487b032661
commit
e1dcbe517f
@ -16,3 +16,5 @@
|
||||
{"d":"2026-04-08","t":"FIX","m":"Peering Recommendations: replace 20 concurrent full lookup calls with new /api/quick-ix endpoint — was hanging indefinitely on every new lookup"}
|
||||
{"d":"2026-04-08","t":"FIX","m":"ASPA: reduce looking-glass timeout 8s→5s and hard cap 18s→12s — faster response for slow RIPE Stat endpoints"}
|
||||
{"d":"2026-04-08","t":"FEAT","m":"New /api/quick-ix endpoint: lightweight PeeringDB IX connections + network name, 1h cache"}
|
||||
{"d":"2026-04-08","t":"FIX","m":"validate: reduce reverse-dns timeout 15s→5s, route-leak asn-neighbours 30s→8s, comparison endpoint 4x 30s→8s — prevent semaphore starvation"}
|
||||
{"d":"2026-04-08","t":"FEAT","m":"validate: add 15min result cache — subsequent lookups return in ~18ms vs 700ms+ cold"}
|
||||
|
||||
68
server.js
68
server.js
@ -549,6 +549,7 @@ const BGPROUTES_VP_TTL = 60 * 60 * 1000; // 1 hour
|
||||
// ============================================================
|
||||
const bgproutesResultCache = new Map();
|
||||
const aspaResultCache = new Map();
|
||||
const validateResultCache = new Map();
|
||||
const RESULT_CACHE_TTL = 15 * 60 * 1000; // 15 minutes
|
||||
function resultCacheGet(map, key) {
|
||||
const e = map.get(String(key));
|
||||
@ -2917,15 +2918,20 @@ const server = http.createServer(async (req, res) => {
|
||||
res.writeHead(400);
|
||||
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
|
||||
}
|
||||
const cachedValidate = resultCacheGet(validateResultCache, rawAsn);
|
||||
if (cachedValidate !== undefined) {
|
||||
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
|
||||
return res.end(JSON.stringify(cachedValidate));
|
||||
}
|
||||
const start = Date.now();
|
||||
const targetAsn = parseInt(rawAsn);
|
||||
|
||||
try {
|
||||
// Phase 1: Fetch core data needed by multiple validations
|
||||
const [prefixData, pdbNet, neighbourData, overviewData] = await Promise.all([
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + rawAsn, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + rawAsn, { timeout: 8000 }),
|
||||
fetchPeeringDB("/net?asn=" + rawAsn),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, { timeout: 8000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + rawAsn),
|
||||
]);
|
||||
|
||||
@ -3058,7 +3064,7 @@ const server = http.createServer(async (req, res) => {
|
||||
var rdnsSampleSize = Math.min(20, samplePrefixes.length);
|
||||
validationPromises.rdns = Promise.all(
|
||||
samplePrefixes.slice(0, rdnsSampleSize).map(function(pfx) {
|
||||
return fetchRipeStatCached("https://stat.ripe.net/data/reverse-dns-consistency/data.json?resource=" + encodeURIComponent(pfx), { timeout: 15000 }).then(function(data) {
|
||||
return fetchRipeStatCached("https://stat.ripe.net/data/reverse-dns-consistency/data.json?resource=" + encodeURIComponent(pfx), { timeout: 5000 }).then(function(data) {
|
||||
var pfxData = data && data.data && data.data.prefixes ? data.data.prefixes : {};
|
||||
var hasDelegation = false;
|
||||
var details = [];
|
||||
@ -3094,7 +3100,7 @@ const server = http.createServer(async (req, res) => {
|
||||
}).catch(function(e) { return { status: "error", error: String(e) }; });
|
||||
|
||||
// 18. BGP Visibility (uses routing-status API which is more reliable than visibility API)
|
||||
validationPromises.visibility = fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=AS" + rawAsn, { timeout: 20000 }).then(function(rsData) {
|
||||
validationPromises.visibility = fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=AS" + rawAsn, { timeout: 8000 }).then(function(rsData) {
|
||||
var vis = rsData && rsData.data && rsData.data.visibility ? rsData.data.visibility : {};
|
||||
var v4 = vis.v4 || {};
|
||||
var v6 = vis.v6 || {};
|
||||
@ -3345,28 +3351,24 @@ const server = http.createServer(async (req, res) => {
|
||||
.slice(0, 30)
|
||||
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
|
||||
|
||||
return res.end(
|
||||
JSON.stringify(
|
||||
{
|
||||
meta: { query: "AS" + rawAsn, duration_ms: duration, timestamp: new Date().toISOString(), total_prefixes: allPrefixes.length, prefixes_sampled: samplePrefixes.length },
|
||||
asn: targetAsn,
|
||||
name: net.name || (overviewData && overviewData.data ? overviewData.data.holder : "") || "Unknown",
|
||||
health_score: healthScore,
|
||||
score_breakdown: checkResults,
|
||||
validations: validations,
|
||||
relationships: {
|
||||
counts: { upstreams: relNeighbours.left || relUpstreams.length, downstreams: relNeighbours.right || relDownstreams.length, peers: relNeighbours.unique || relPeers.length, uncertain: relNeighbours.uncertain || 0 },
|
||||
upstreams: relUpstreams,
|
||||
downstreams: relDownstreams,
|
||||
top_peers: relPeers,
|
||||
source: "RIPE Stat asn-neighbours",
|
||||
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
|
||||
},
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
const validateResult = {
|
||||
meta: { query: "AS" + rawAsn, duration_ms: duration, timestamp: new Date().toISOString(), total_prefixes: allPrefixes.length, prefixes_sampled: samplePrefixes.length },
|
||||
asn: targetAsn,
|
||||
name: net.name || (overviewData && overviewData.data ? overviewData.data.holder : "") || "Unknown",
|
||||
health_score: healthScore,
|
||||
score_breakdown: checkResults,
|
||||
validations: validations,
|
||||
relationships: {
|
||||
counts: { upstreams: relNeighbours.left || relUpstreams.length, downstreams: relNeighbours.right || relDownstreams.length, peers: relNeighbours.unique || relPeers.length, uncertain: relNeighbours.uncertain || 0 },
|
||||
upstreams: relUpstreams,
|
||||
downstreams: relDownstreams,
|
||||
top_peers: relPeers,
|
||||
source: "RIPE Stat asn-neighbours",
|
||||
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
|
||||
},
|
||||
};
|
||||
resultCacheSet(validateResultCache, rawAsn, validateResult);
|
||||
return res.end(JSON.stringify(validateResult, null, 2));
|
||||
} catch (err) {
|
||||
res.writeHead(500);
|
||||
return res.end(JSON.stringify({ error: "Validation failed", message: err.message }));
|
||||
@ -3438,8 +3440,8 @@ const server = http.createServer(async (req, res) => {
|
||||
]).then(d => { rdapCacheSet(asn, d); return d; });
|
||||
|
||||
const promises = [
|
||||
timedFetch("RIPE Stat Prefixes", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 20000 })),
|
||||
timedFetch("RIPE Stat Neighbours", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 20000 })),
|
||||
timedFetch("RIPE Stat Prefixes", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 8000 })),
|
||||
timedFetch("RIPE Stat Neighbours", fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 8000 })),
|
||||
timedFetch("RIPE Stat Overview", fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn)),
|
||||
timedFetch("RIPE Stat RIR", fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn)),
|
||||
timedFetch("RIPE Atlas", fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500")),
|
||||
@ -3983,7 +3985,7 @@ const server = http.createServer(async (req, res) => {
|
||||
try {
|
||||
const neighbourData = await fetchRipeStatCached(
|
||||
"https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1",
|
||||
{ timeout: 30000 }
|
||||
{ timeout: 8000 }
|
||||
);
|
||||
const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || [];
|
||||
const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {};
|
||||
@ -4062,10 +4064,10 @@ const server = http.createServer(async (req, res) => {
|
||||
const [pdb1, pdb2, nb1Data, nb2Data, pfx1Data, pfx2Data] = await Promise.all([
|
||||
fetchPeeringDB("/net?asn=" + asn1),
|
||||
fetchPeeringDB("/net?asn=" + asn2),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 30000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 8000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 8000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 8000 }),
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 8000 }),
|
||||
]);
|
||||
|
||||
const net1 = pdb1?.data?.[0] || {};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user