refactor: decompose /api/lookup into server/routes/lookup/{index,facilities,enrich-neighbours,rir-country,routing-info,cross-check}.js
The 662-line handler (the largest in the file) split into one function per data-source/merge stage: facility+IX-location coordinate resolution (facilities.js), neighbour name-resolution + threat-intel enrichment (enrich-neighbours.js), RIR/country derivation cascade (rir-country.js), visibility/prefix-size computation (routing-info.js), multi-source cross-checks + data-quality scoring (cross-check.js), and a slim orchestrator (index.js) running Phase 0/1 fetch and final result assembly. Verified via node --check, eslint no-undef (2 known bugs remain: compare, webhooks), and smoke-test harness (28/28 match, including a byte-for-byte match on /api/lookup's own large response body -- the strongest possible confirmation this, the most complex handler in the file, was decomposed without changing behavior). server.js: 2917 -> 1734 lines (6838 originally -- 75% reduction so far).
This commit is contained in:
parent
144457f5d2
commit
cba7680de0
668
server.js
668
server.js
@ -226,6 +226,7 @@ const aspaVerifyRoute = require("./server/routes/aspa-verify");
|
||||
const aspaLegacyRoute = require("./server/routes/aspa-legacy");
|
||||
const bgpRoute = require("./server/routes/bgp");
|
||||
const validateRoute = require("./server/routes/validate");
|
||||
const lookupRoute = require("./server/routes/lookup");
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
res.setHeader("Access-Control-Allow-Origin", "*");
|
||||
@ -339,676 +340,11 @@ const server = http.createServer(async (req, res) => {
|
||||
return validateRoute.handleValidate(req, res, url);
|
||||
}
|
||||
|
||||
|
||||
// ============================================================
|
||||
// Main lookup endpoint: /api/lookup?asn=X
|
||||
// ============================================================
|
||||
if (reqPath === "/api/lookup") {
|
||||
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
|
||||
if (!rawAsn) {
|
||||
res.writeHead(400);
|
||||
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
|
||||
}
|
||||
const asn = rawAsn;
|
||||
const cacheKey = "lookup:" + asn;
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) {
|
||||
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
|
||||
return res.end(JSON.stringify(cached));
|
||||
}
|
||||
const start = Date.now();
|
||||
|
||||
try {
|
||||
// Phase 0: Get PDB net first — check L2 cache, then API with retry
|
||||
let pdbNet = pdbSourceCache.get("net", asn);
|
||||
if (!pdbNet) {
|
||||
pdbNet = await fetchPeeringDBWithRetry("/net?asn=" + asn);
|
||||
if (pdbNet) pdbSourceCache.set("net", asn, pdbNet);
|
||||
}
|
||||
const net = pdbNet?.data?.[0] || {};
|
||||
const netId = net.id;
|
||||
|
||||
// Phase 1: ALL calls in parallel — RIPE Stat (cached+throttled) + PDB IX/Fac (cached) + Atlas + bgp.he.net
|
||||
const ixQuery = netId
|
||||
? "/netixlan?net_id=" + netId + "&limit=1000"
|
||||
: "/netixlan?asn=" + asn + "&limit=1000";
|
||||
const ixCacheKey = netId ? String(netId) : "asn:" + asn;
|
||||
|
||||
// Check PDB source cache for IX/Fac data
|
||||
let cachedIxlan = pdbSourceCache.get("netixlan", ixCacheKey);
|
||||
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
|
||||
|
||||
// Per-source timing tracking — 9s hard cap per source to prevent long-tail blocking
|
||||
const sourceTiming = {};
|
||||
function timedFetch(name, promise) {
|
||||
const ts = Date.now();
|
||||
return Promise.race([
|
||||
Promise.resolve(promise),
|
||||
new Promise(function(r) { setTimeout(function() { r(null); }, 9000); }),
|
||||
])
|
||||
.then(function(r) { sourceTiming[name] = Date.now() - ts; return r; })
|
||||
.catch(function() { sourceTiming[name] = null; return null; });
|
||||
}
|
||||
|
||||
const pocQuery = netId ? "/poc?net_id=" + netId + "&limit=25" : null;
|
||||
|
||||
// RDAP: check module-level cache first, only hit RIR endpoints on cache miss
|
||||
const rdapCached = rdapCacheGet(asn);
|
||||
const rdapPromise = rdapCached !== undefined
|
||||
? Promise.resolve(rdapCached)
|
||||
: Promise.race([
|
||||
...["https://rdap.db.ripe.net/autnum/"+asn, "https://rdap.arin.net/registry/autnum/"+asn,
|
||||
"https://rdap.apnic.net/autnum/"+asn, "https://rdap.lacnic.net/rdap/autnum/"+asn,
|
||||
"https://rdap.afrinic.net/rdap/autnum/"+asn].map(url =>
|
||||
fetchJSON(url, { timeout: 4000 })
|
||||
.then(d => (d && !d.errorCode && d.handle) ? d : new Promise(() => {}))
|
||||
.catch(() => new Promise(() => {}))
|
||||
),
|
||||
new Promise(resolve => setTimeout(() => resolve(null), 5000)),
|
||||
]).then(d => { rdapCacheSet(asn, d); return d; });
|
||||
|
||||
const promises = [
|
||||
timedFetch("RIPE Stat Prefixes", localDb ? localDb.getRipeStatAnnouncedPrefixes(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Neighbours", localDb ? localDb.getRipeStatAsnNeighbours(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Overview", localDb ? localDb.getRipeStatAsOverview(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat RIR", Promise.resolve(null)),
|
||||
timedFetch("RIPE Atlas", Promise.resolve(null)),
|
||||
timedFetch("bgp.he.net", Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Visibility", localDb ? localDb.getRipeStatVisibility(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat PrefixSize", localDb ? localDb.getRipeStatPrefixSizeDistribution(asn) : Promise.resolve(null)),
|
||||
timedFetch("PeeringDB IXLan", cachedIxlan ? Promise.resolve(cachedIxlan) : fetchPeeringDBWithRetry(ixQuery)),
|
||||
timedFetch("PeeringDB Facilities", cachedFac ? Promise.resolve(cachedFac) : (netId ? fetchPeeringDBWithRetry("/netfac?net_id=" + netId + "&limit=1000") : Promise.resolve(null))),
|
||||
timedFetch("PeeringDB Contacts", pocQuery ? fetchPeeringDB(pocQuery).catch(() => null) : Promise.resolve(null)),
|
||||
timedFetch("RDAP Registration", rdapPromise),
|
||||
];
|
||||
const [prefixData, neighbourData, overviewData, rirData, atlasProbeData, bgpHeData, visibilityData, prefixSizeData, ixlanData, facData, pocData, rdapData] = await Promise.all(promises);
|
||||
|
||||
// Store PDB results in L2 source cache for future lookups
|
||||
if (!cachedIxlan && ixlanData) pdbSourceCache.set("netixlan", ixCacheKey, ixlanData);
|
||||
if (!cachedFac && facData) pdbSourceCache.set("netfac", String(netId), facData);
|
||||
|
||||
// local-db-client's getRipeStat* functions return status:"error" (not "ok") when
|
||||
// the local Postgres DB query itself failed -- track which sources actually
|
||||
// failed so we don't cache a DB hiccup's fake-empty result as if it were a
|
||||
// genuine "this ASN has 0 prefixes" answer for the full 5-minute TTL.
|
||||
const degradedSources = [
|
||||
["RIPE Stat Prefixes", prefixData], ["RIPE Stat Neighbours", neighbourData],
|
||||
["RIPE Stat Overview", overviewData], ["RIPE Stat Visibility", visibilityData],
|
||||
["RIPE Stat PrefixSize", prefixSizeData],
|
||||
].filter(([, v]) => v && v.status === "error").map(([name]) => name);
|
||||
|
||||
const prefixes = prefixData?.data?.prefixes || [];
|
||||
const neighbours = neighbourData?.data?.neighbours || [];
|
||||
const overview = overviewData?.data || {};
|
||||
const rirEntries = rirData?.data?.located_resources || rirData?.data?.rir_stats || [];
|
||||
|
||||
// Bug 6 fix: Atlas probe status uses status.name (object), not status_name (flat)
|
||||
const atlasProbes = atlasProbeData?.results || [];
|
||||
const atlasConnected = atlasProbes.filter(p => {
|
||||
const sName = (p.status_name || (p.status && p.status.name) || "").toLowerCase();
|
||||
return sName === "connected";
|
||||
});
|
||||
const atlasAnchors = atlasProbes.filter(p => p.is_anchor === true);
|
||||
|
||||
// RPKI: validate ALL prefixes using local Cloudflare RPKI data (all 5 RIRs, instant)
|
||||
await ensureAspaCache();
|
||||
const allPrefixes = prefixes.map((p) => p.prefix);
|
||||
const rpkiAllResults = await Promise.all(allPrefixes.map((pfx) => validateRPKIWithCache(asn, pfx)));
|
||||
|
||||
const ixConnections = (ixlanData?.data || [])
|
||||
.map((ix) => ({
|
||||
ix_name: ix.name || "",
|
||||
ix_id: ix.ix_id,
|
||||
speed_mbps: ix.speed || 0,
|
||||
ipv4: ix.ipaddr4 || null,
|
||||
ipv6: ix.ipaddr6 || null,
|
||||
city: ix.city || "",
|
||||
is_rs_peer: ix.is_rs_peer === true,
|
||||
}))
|
||||
.sort((a, b) => b.speed_mbps - a.speed_mbps);
|
||||
|
||||
const facilitiesRaw = (facData?.data || []).map((f) => ({
|
||||
fac_id: f.fac_id,
|
||||
name: f.name || "",
|
||||
city: f.city || "",
|
||||
country: f.country || "",
|
||||
}));
|
||||
|
||||
// Batch-fetch facility coordinates for map (max 50 facilities)
|
||||
const facIds = facilitiesRaw.map(f => f.fac_id).filter(Boolean).slice(0, 50);
|
||||
let facCoordMap = {};
|
||||
if (facIds.length > 0) {
|
||||
try {
|
||||
const chunks = [];
|
||||
for (let i = 0; i < facIds.length; i += 25) chunks.push(facIds.slice(i, i + 25));
|
||||
const coordResults = await Promise.race([
|
||||
Promise.all(chunks.map(chunk =>
|
||||
fetchPeeringDB("/fac?id__in=" + chunk.join(",") + "&fields=id,latitude,longitude").catch(() => null)
|
||||
)),
|
||||
new Promise(r => setTimeout(() => r([]), 5000))
|
||||
]);
|
||||
(coordResults || []).forEach(res => {
|
||||
(res?.data || []).forEach(f => { if (f.latitude && f.longitude) facCoordMap[f.id] = { lat: f.latitude, lon: f.longitude }; });
|
||||
});
|
||||
} catch(e) { /* graceful degradation */ }
|
||||
}
|
||||
const facilities = facilitiesRaw.map(f => ({
|
||||
...f,
|
||||
latitude: facCoordMap[f.fac_id] ? facCoordMap[f.fac_id].lat : null,
|
||||
longitude: facCoordMap[f.fac_id] ? facCoordMap[f.fac_id].lon : null,
|
||||
}));
|
||||
|
||||
// Get IX locations for map via ixfac -> fac coordinates (max 20 IXs)
|
||||
const uniqueIxIds = [...new Set(ixConnections.map(c => c.ix_id))].filter(Boolean).slice(0, 20);
|
||||
let ixLocations = [];
|
||||
if (uniqueIxIds.length > 0) {
|
||||
try {
|
||||
const ixFacData = await Promise.race([
|
||||
fetchPeeringDB("/ixfac?ix_id__in=" + uniqueIxIds.join(",")),
|
||||
new Promise(r => setTimeout(() => r(null), 5000))
|
||||
]);
|
||||
const ixFacs = ixFacData?.data || [];
|
||||
// Collect unique fac_ids we don't already have coords for
|
||||
const extraFacIds = [...new Set(ixFacs.map(f => f.fac_id).filter(id => id && !facCoordMap[id]))].slice(0, 30);
|
||||
if (extraFacIds.length > 0) {
|
||||
const extraChunks = [];
|
||||
for (let i = 0; i < extraFacIds.length; i += 25) extraChunks.push(extraFacIds.slice(i, i + 25));
|
||||
const extraRes = await Promise.race([
|
||||
Promise.all(extraChunks.map(chunk =>
|
||||
fetchPeeringDB("/fac?id__in=" + chunk.join(",") + "&fields=id,latitude,longitude").catch(() => null)
|
||||
)),
|
||||
new Promise(r => setTimeout(() => r([]), 4000))
|
||||
]);
|
||||
(extraRes || []).forEach(res => {
|
||||
(res?.data || []).forEach(f => { if (f.latitude && f.longitude) facCoordMap[f.id] = { lat: f.latitude, lon: f.longitude }; });
|
||||
});
|
||||
}
|
||||
// Build IX locations: pick first facility with coords per IX
|
||||
const ixNameMap = {};
|
||||
ixConnections.forEach(c => { if (c.ix_id && c.ix_name) ixNameMap[c.ix_id] = c.ix_name; });
|
||||
const seenIx = {};
|
||||
ixFacs.forEach(f => {
|
||||
if (seenIx[f.ix_id]) return;
|
||||
const coords = facCoordMap[f.fac_id];
|
||||
if (coords) {
|
||||
seenIx[f.ix_id] = true;
|
||||
ixLocations.push({ ix_id: f.ix_id, name: ixNameMap[f.ix_id] || f.name || "", city: f.city || "", country: f.country || "", latitude: coords.lat, longitude: coords.lon });
|
||||
}
|
||||
});
|
||||
} catch(e) { /* graceful degradation */ }
|
||||
}
|
||||
|
||||
const rpkiStatuses = rpkiAllResults;
|
||||
const rpkiValid = rpkiStatuses.filter((r) => r.status === "valid").length;
|
||||
const rpkiInvalid = rpkiStatuses.filter((r) => r.status === "invalid").length;
|
||||
const rpkiUnavailable = rpkiStatuses.filter((r) => r.status === "unavailable").length;
|
||||
const rpkiNotFound = rpkiStatuses.filter((r) => r.status !== "valid" && r.status !== "invalid" && r.status !== "unavailable").length;
|
||||
// Exclude "unavailable" (DB error) from the coverage denominator -- a DB hiccup
|
||||
// must not lower a network's displayed RPKI coverage percentage.
|
||||
const rpkiTotal = rpkiStatuses.length - rpkiUnavailable;
|
||||
const rpkiCoverage = rpkiTotal > 0 ? Math.round((rpkiValid / rpkiTotal) * 100) : 0;
|
||||
|
||||
let upstreams = neighbours
|
||||
.filter((n) => n.type === "left")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
let downstreams = neighbours
|
||||
.filter((n) => n.type === "right")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
let peers = neighbours
|
||||
.filter((n) => n.type === "uncertain" || n.type === "peer")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
|
||||
// Resolve empty AS names — all in parallel, with 3s timeout
|
||||
const emptyNameNeighbours = [...upstreams, ...downstreams, ...peers].filter(n => !n.name);
|
||||
if (emptyNameNeighbours.length > 0) {
|
||||
const resolvePromise = Promise.all(
|
||||
emptyNameNeighbours.map(n =>
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 })
|
||||
.then(r => { if (r?.data?.holder) n.name = r.data.holder; })
|
||||
.catch(() => {})
|
||||
)
|
||||
);
|
||||
await Promise.race([resolvePromise, new Promise(r => setTimeout(r, 3000))]);
|
||||
}
|
||||
|
||||
// ---- Threat Intelligence Enrichment for Neighbors ----
|
||||
// Enrich neighbor data with threat status from local threat_intel table
|
||||
const threatEnrichNeighbors = async (neighbors) => {
|
||||
const allNeighbors = [...neighbors];
|
||||
const threatMap = {};
|
||||
|
||||
// Batch threat intel lookups (cap at 50 to avoid overwhelming DB)
|
||||
const toCheck = allNeighbors.slice(0, 50);
|
||||
const threatPromises = toCheck.map(async (n) => {
|
||||
try {
|
||||
// Try to get threat intel by AS number or typical AS IP pattern
|
||||
// For now, we'll mark neighbors without direct IP threat data
|
||||
const asNum = String(n.asn);
|
||||
const threat = await localDb.getThreatIntel(asNum);
|
||||
if (threat) {
|
||||
threatMap[n.asn] = {
|
||||
threat_level: threat.threat_level,
|
||||
confidence_score: threat.confidence_score,
|
||||
source: threat.source,
|
||||
cached_at: threat.cached_at,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
// Gracefully skip on error
|
||||
console.error(`[Threat Lookup] Error checking ASN ${n.asn}:`, e.message);
|
||||
}
|
||||
});
|
||||
|
||||
// Run threat lookups with 4s timeout
|
||||
await Promise.race([
|
||||
Promise.all(threatPromises),
|
||||
new Promise(r => setTimeout(r, 4000)),
|
||||
]);
|
||||
|
||||
return threatMap;
|
||||
};
|
||||
|
||||
const threatMap = await threatEnrichNeighbors([...upstreams, ...downstreams, ...peers]);
|
||||
|
||||
// Attach threat status to neighbor objects
|
||||
const addThreatToNeighbor = (n) => ({
|
||||
...n,
|
||||
threat_level: threatMap[n.asn]?.threat_level || null,
|
||||
threat_confidence: threatMap[n.asn]?.confidence_score || null,
|
||||
threat_source: threatMap[n.asn]?.source || null,
|
||||
});
|
||||
|
||||
upstreams = upstreams.map(addThreatToNeighbor);
|
||||
downstreams = downstreams.map(addThreatToNeighbor);
|
||||
peers = peers.map(addThreatToNeighbor);
|
||||
|
||||
let rir = "";
|
||||
let country = "";
|
||||
// RIPE Stat rir-stats-country uses 'location' field (not 'country' or 'rir')
|
||||
if (Array.isArray(rirEntries) && rirEntries.length > 0) {
|
||||
country = rirEntries[0]?.location || rirEntries[0]?.country || "";
|
||||
rir = rirEntries[0]?.rir || "";
|
||||
}
|
||||
if (!rir && rirData?.data) {
|
||||
const rirField = rirData.data.rirs || [];
|
||||
if (rirField.length > 0) rir = rirField[0]?.rir || "";
|
||||
}
|
||||
// Derive RIR from rdapData.port43 (e.g. "whois.ripe.net" → "RIPE")
|
||||
if (!rir && rdapData && rdapData.port43) {
|
||||
const p43 = (rdapData.port43 || "").toLowerCase();
|
||||
if (p43.includes("ripe")) rir = "RIPE";
|
||||
else if (p43.includes("arin")) rir = "ARIN";
|
||||
else if (p43.includes("apnic")) rir = "APNIC";
|
||||
else if (p43.includes("lacnic")) rir = "LACNIC";
|
||||
else if (p43.includes("afrinic")) rir = "AFRINIC";
|
||||
}
|
||||
// Also derive RIR from RDAP links (URL of the RDAP endpoint that responded)
|
||||
if (!rir && rdapData && rdapData.links) {
|
||||
const selfLink = (rdapData.links.find(l => l.rel === "self") || {}).href || "";
|
||||
if (selfLink.includes("ripe")) rir = "RIPE";
|
||||
else if (selfLink.includes("arin")) rir = "ARIN";
|
||||
else if (selfLink.includes("apnic")) rir = "APNIC";
|
||||
else if (selfLink.includes("lacnic")) rir = "LACNIC";
|
||||
else if (selfLink.includes("afrinic")) rir = "AFRINIC";
|
||||
}
|
||||
// bgp.he.net country_code fallback (for unannounced/reserve ASNs)
|
||||
if (!country && bgpHeData && bgpHeData.country_code) {
|
||||
country = bgpHeData.country_code;
|
||||
}
|
||||
// Last resort: derive RIR from country code (common assignments)
|
||||
if (!rir && country) {
|
||||
const { ARIN_CC, APNIC_CC, LACNIC_CC, AFRINIC_CC } = require("./server/data/rir-country-codes");
|
||||
if (ARIN_CC.has(country)) rir = "ARIN";
|
||||
else if (APNIC_CC.has(country)) rir = "APNIC";
|
||||
else if (LACNIC_CC.has(country)) rir = "LACNIC";
|
||||
else if (AFRINIC_CC.has(country)) rir = "AFRINIC";
|
||||
else rir = "RIPE"; // Europe + rest = RIPE NCC
|
||||
}
|
||||
|
||||
const duration = Date.now() - start;
|
||||
|
||||
// Compute routing visibility and prefix size distribution
|
||||
const routingInfo = await (async function() {
|
||||
const ipv4Prefixes = prefixes.filter(function(p) { return !p.prefix.includes(":"); });
|
||||
const ipv6Prefixes = prefixes.filter(function(p) { return p.prefix.includes(":"); });
|
||||
var ipv4VisAvg = 0, ipv6VisAvg = 0, totalRisPeersV4 = 0, totalRisPeersV6 = 0;
|
||||
|
||||
// Visibility API returns per-RIS-collector data
|
||||
// Each collector has ipv4_full_table_peer_count and ipv4_full_table_peers_not_seeing[]
|
||||
// Bug 3 fix: visibility API may timeout for large ASNs — handle gracefully
|
||||
var visibilities = (visibilityData && visibilityData.data && visibilityData.data.visibilities) || [];
|
||||
var v4Seeing = 0, v4Total = 0, v6Seeing = 0, v6Total = 0;
|
||||
var visTimedOut = !visibilityData || !visibilityData.data;
|
||||
visibilities.forEach(function(v) {
|
||||
if (!v || !v.probe) return;
|
||||
var v4PeerCount = v.ipv4_full_table_peer_count || 0;
|
||||
var v4NotSeeing = (v.ipv4_full_table_peers_not_seeing || []).length;
|
||||
var v6PeerCount = v.ipv6_full_table_peer_count || 0;
|
||||
var v6NotSeeing = (v.ipv6_full_table_peers_not_seeing || []).length;
|
||||
v4Total += v4PeerCount;
|
||||
v4Seeing += (v4PeerCount - v4NotSeeing);
|
||||
v6Total += v6PeerCount;
|
||||
v6Seeing += (v6PeerCount - v6NotSeeing);
|
||||
});
|
||||
if (v4Total > 0) ipv4VisAvg = Math.round((v4Seeing / v4Total) * 1000) / 10;
|
||||
if (v6Total > 0) ipv6VisAvg = Math.round((v6Seeing / v6Total) * 1000) / 10;
|
||||
// If visibility API timed out but we have prefixes, try bgproutes.io fallback
|
||||
if (visTimedOut && prefixes.length > 0) {
|
||||
var fallbackPrefix = prefixes.find(function(p) { return !p.prefix.includes(":"); });
|
||||
if (!fallbackPrefix) fallbackPrefix = prefixes[0];
|
||||
if (fallbackPrefix) {
|
||||
var bgprFallback = await fetchBgproutesVisibility(fallbackPrefix.prefix);
|
||||
if (bgprFallback && bgprFallback.vps_seeing > 0) {
|
||||
// Estimate visibility: % of VPs seeing the prefix (assume ~300 total RIS-equivalent VPs)
|
||||
var estimatedTotal = Math.max(bgprFallback.vps_seeing, 300);
|
||||
ipv4VisAvg = Math.round((bgprFallback.vps_seeing / estimatedTotal) * 1000) / 10;
|
||||
ipv6VisAvg = -1; // bgproutes fallback is per-prefix, not per-AF aggregate
|
||||
totalRisPeersV4 = bgprFallback.vps_seeing;
|
||||
console.log("[Visibility] RIPE Stat timed out, used bgproutes.io fallback for " + fallbackPrefix.prefix + ": " + bgprFallback.vps_seeing + " VPs seeing it");
|
||||
} else {
|
||||
ipv4VisAvg = -1;
|
||||
ipv6VisAvg = -1;
|
||||
console.log("[Visibility] RIPE Stat timed out and bgproutes.io fallback returned no data");
|
||||
}
|
||||
} else {
|
||||
ipv4VisAvg = -1;
|
||||
ipv6VisAvg = -1;
|
||||
}
|
||||
}
|
||||
totalRisPeersV4 = v4Total;
|
||||
totalRisPeersV6 = v6Total;
|
||||
|
||||
// Prefix size distribution: data.ipv4[] and data.ipv6[] arrays with {size, count}
|
||||
var psdData = (prefixSizeData && prefixSizeData.data) || {};
|
||||
var psV4 = (psdData.ipv4 || []).map(function(e) { return { size: e.size, count: e.count }; }).sort(function(a,b){ return a.size - b.size; });
|
||||
var psV6 = (psdData.ipv6 || []).map(function(e) { return { size: e.size, count: e.count }; }).sort(function(a,b){ return a.size - b.size; });
|
||||
|
||||
return {
|
||||
ipv4_prefixes: ipv4Prefixes.length,
|
||||
ipv6_prefixes: ipv6Prefixes.length,
|
||||
ipv4_visibility_avg: ipv4VisAvg,
|
||||
ipv6_visibility_avg: ipv6VisAvg,
|
||||
total_ris_peers_v4: totalRisPeersV4,
|
||||
total_ris_peers_v6: totalRisPeersV6,
|
||||
prefix_sizes_v4: psV4,
|
||||
prefix_sizes_v6: psV6,
|
||||
};
|
||||
})();
|
||||
|
||||
// ============================================================
|
||||
// Multi-source cross-checks (run in parallel, non-blocking)
|
||||
// ============================================================
|
||||
let rpkiCrossCheck = { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 };
|
||||
let prefixCrossCheck = { ripe_stat: prefixes.length, bgp_he_net: null, agreement: null, note: "" };
|
||||
let neighbourCrossCheck = { ripe_stat_total: neighbours.length, bgp_he_net_total: null };
|
||||
|
||||
try {
|
||||
// RPKI cross-check: sample up to 5 prefixes against RIPE Validator (with 8s total timeout)
|
||||
const rpkiCrossPromise = crossCheckRpki(asn, allPrefixes, rpkiStatuses);
|
||||
const rpkiCrossResult = await Promise.race([
|
||||
rpkiCrossPromise,
|
||||
new Promise((resolve) => setTimeout(() => resolve(null), 8000)),
|
||||
]);
|
||||
if (rpkiCrossResult) rpkiCrossCheck = rpkiCrossResult;
|
||||
} catch (_e) { /* cross-check failed, keep defaults */ }
|
||||
|
||||
// bgp.he.net fetching is deliberately disabled (see the hardcoded
|
||||
// Promise.resolve(null) a few hundred lines up, "bgp.he.net" timedFetch) --
|
||||
// bgpHeData is therefore always null and this whole branch, along with the
|
||||
// prefix/neighbour cross-checks below, never actually runs. Left in place
|
||||
// (not deleted) in case bgp.he.net fetching is re-enabled later, but
|
||||
// overall_confidence below must not claim a 2-source agreement that never
|
||||
// happened -- see the sources:1/2 handling further down.
|
||||
// Prefix count cross-check: compare RIPE Stat vs bgp.he.net
|
||||
if (bgpHeData) {
|
||||
const heV4 = bgpHeData.prefixes_v4 || 0;
|
||||
const heV6 = bgpHeData.prefixes_v6 || 0;
|
||||
const heTotal = heV4 + heV6;
|
||||
if (heTotal > 0) {
|
||||
prefixCrossCheck.bgp_he_net = heTotal;
|
||||
const ripeStat = prefixes.length;
|
||||
if (ripeStat > 0 && heTotal > 0) {
|
||||
const ratio = Math.min(ripeStat, heTotal) / Math.max(ripeStat, heTotal);
|
||||
prefixCrossCheck.agreement = ratio >= 0.9;
|
||||
const diff = Math.abs(ripeStat - heTotal);
|
||||
prefixCrossCheck.note = diff === 0
|
||||
? "Exact match"
|
||||
: "Difference of " + diff + " prefixes (" + Math.round((1 - ratio) * 100) + "% divergence)";
|
||||
}
|
||||
} else {
|
||||
prefixCrossCheck.note = "bgp.he.net prefix count unavailable";
|
||||
}
|
||||
|
||||
// Neighbour cross-check: compare RIPE Stat vs bgp.he.net peer_count
|
||||
if (bgpHeData.peer_count != null) {
|
||||
neighbourCrossCheck.bgp_he_net_total = bgpHeData.peer_count;
|
||||
}
|
||||
} else {
|
||||
prefixCrossCheck.note = "bgp.he.net data unavailable";
|
||||
}
|
||||
|
||||
// Compute overall data quality. Only push a score for a cross-check that
|
||||
// actually ran (agreement_pct !== null) -- an unrun/inconclusive check must
|
||||
// not silently count as a perfect 100 in the average, and confidence must
|
||||
// reflect "nothing was actually cross-checked" rather than defaulting high.
|
||||
const crossCheckScores = [];
|
||||
if (rpkiCrossCheck.agreement_pct !== null) crossCheckScores.push(rpkiCrossCheck.agreement_pct);
|
||||
// Prefix agreement: convert to percentage
|
||||
if (prefixCrossCheck.bgp_he_net != null && prefixes.length > 0) {
|
||||
const pfxRatio = Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net);
|
||||
crossCheckScores.push(Math.round(pfxRatio * 100));
|
||||
}
|
||||
// Neighbour agreement
|
||||
if (neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0) {
|
||||
const nbrRatio = Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total);
|
||||
crossCheckScores.push(Math.round(nbrRatio * 100));
|
||||
}
|
||||
const avgAgreement = crossCheckScores.length > 0
|
||||
? Math.round(crossCheckScores.reduce((a, b) => a + b, 0) / crossCheckScores.length)
|
||||
: null;
|
||||
const overallConfidence = avgAgreement === null ? "unknown" : avgAgreement > 90 ? "high" : avgAgreement >= 70 ? "medium" : "low";
|
||||
|
||||
const dataQuality = {
|
||||
sources_queried: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator"],
|
||||
cross_checks: {
|
||||
rpki: { sources: rpkiCrossCheck.agreement_pct !== null ? 2 : 1, agreement_pct: rpkiCrossCheck.agreement_pct, sample_size: rpkiCrossCheck.sample_size, comparable_size: rpkiCrossCheck.comparable_size, disagreements: rpkiCrossCheck.disagreements },
|
||||
prefixes: { sources: prefixCrossCheck.bgp_he_net != null ? 2 : 1, agreement_pct: prefixCrossCheck.bgp_he_net != null ? Math.round((Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net || 1)) * 100) : null, ripe_stat: prefixCrossCheck.ripe_stat, bgp_he_net: prefixCrossCheck.bgp_he_net, note: prefixCrossCheck.note },
|
||||
neighbours: { sources: neighbourCrossCheck.bgp_he_net_total != null ? 2 : 1, agreement_pct: neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0 ? Math.round((Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total)) * 100) : null, ripe_stat_total: neighbourCrossCheck.ripe_stat_total, bgp_he_net_total: neighbourCrossCheck.bgp_he_net_total },
|
||||
},
|
||||
overall_confidence: overallConfidence,
|
||||
overall_agreement_pct: avgAgreement,
|
||||
};
|
||||
|
||||
// === IX Location Geocode Fallback ===
|
||||
// Some IXPs have no facility coordinates in PeeringDB.
|
||||
// Use ix_name city extraction + hard-coded IX→city map as fallback.
|
||||
var ixIdsWithCoords = new Set(ixLocations.map(function(l) { return l.ix_id; }));
|
||||
ixConnections.forEach(function(conn) {
|
||||
if (ixIdsWithCoords.has(conn.ix_id)) return;
|
||||
var name = conn.ix_name || "";
|
||||
if (name) {
|
||||
var words = name.toLowerCase().replace(/[^a-z\s]/g, " ").split(/\s+/).filter(Boolean);
|
||||
for (var w = 0; w < words.length; w++) {
|
||||
if (CITY_COORDS[words[w]]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: name, city: words[w].charAt(0).toUpperCase() + words[w].slice(1), country: "", latitude: CITY_COORDS[words[w]][0], longitude: CITY_COORDS[words[w]][1], source: "name_geocode" });
|
||||
ixIdsWithCoords.add(conn.ix_id);
|
||||
return;
|
||||
}
|
||||
if (w < words.length - 1) {
|
||||
var tw = words[w] + " " + words[w + 1];
|
||||
if (CITY_COORDS[tw]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: name, city: tw, country: "", latitude: CITY_COORDS[tw][0], longitude: CITY_COORDS[tw][1], source: "name_geocode" });
|
||||
ixIdsWithCoords.add(conn.ix_id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// Hard-coded IX ID → city for well-known IXPs whose names don't contain city
|
||||
var { IX_CITY_MAP } = require("./server/data/ix-city-map");
|
||||
ixConnections.forEach(function(conn) {
|
||||
if (ixIdsWithCoords.has(conn.ix_id)) return;
|
||||
var city = IX_CITY_MAP[conn.ix_id];
|
||||
if (city && CITY_COORDS[city]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: conn.ix_name || ("IX " + conn.ix_id), city: city.charAt(0).toUpperCase() + city.slice(1), country: "", latitude: CITY_COORDS[city][0], longitude: CITY_COORDS[city][1], source: "ix_city_map" });
|
||||
}
|
||||
});
|
||||
|
||||
const result = {
|
||||
meta: {
|
||||
service: "PeerCortex",
|
||||
version: "0.6.9",
|
||||
query: "AS" + asn,
|
||||
duration_ms: duration,
|
||||
sources: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator", "Route Views"],
|
||||
timestamp: new Date().toISOString(),
|
||||
rpki_prefixes_checked: rpkiTotal,
|
||||
total_prefixes: prefixes.length,
|
||||
degraded_sources: degradedSources,
|
||||
},
|
||||
network: {
|
||||
asn: parseInt(asn),
|
||||
name: net.name || overview?.holder || (bgpHeData && bgpHeData.name_from_title) || "Unknown",
|
||||
aka: net.aka || "",
|
||||
org_name: (net.org && net.org.name) ? net.org.name : "",
|
||||
website: net.website || "",
|
||||
type: net.info_type || "",
|
||||
policy: net.policy_general || "",
|
||||
traffic: net.info_traffic || "",
|
||||
ratio: net.info_ratio || "",
|
||||
scope: net.info_scope || "",
|
||||
notes: net.notes ? net.notes.substring(0, 500) : "",
|
||||
peeringdb_id: netId || null,
|
||||
rir: rir,
|
||||
country: country,
|
||||
city: net.city || "",
|
||||
latitude: (net.latitude != null) ? net.latitude : null,
|
||||
longitude: (net.longitude != null) ? net.longitude : null,
|
||||
looking_glass: net.looking_glass || "",
|
||||
route_server: net.route_server || "",
|
||||
info_prefixes4: net.info_prefixes4 || 0,
|
||||
info_prefixes6: net.info_prefixes6 || 0,
|
||||
status: net.status || "",
|
||||
peeringdb_created: net.created ? net.created.slice(0, 10) : "",
|
||||
peeringdb_updated: net.updated ? net.updated.slice(0, 10) : "",
|
||||
},
|
||||
prefixes: {
|
||||
total: prefixes.length,
|
||||
ipv4: prefixes.filter((p) => !p.prefix.includes(":")).length,
|
||||
ipv6: prefixes.filter((p) => p.prefix.includes(":")).length,
|
||||
list: prefixes.map((p) => p.prefix),
|
||||
cross_check: prefixCrossCheck,
|
||||
},
|
||||
rpki: {
|
||||
coverage_percent: rpkiCoverage,
|
||||
valid: rpkiValid,
|
||||
invalid: rpkiInvalid,
|
||||
not_found: rpkiNotFound,
|
||||
unavailable: rpkiUnavailable,
|
||||
checked: rpkiTotal,
|
||||
details: rpkiStatuses,
|
||||
cross_check: rpkiCrossCheck,
|
||||
},
|
||||
neighbours: {
|
||||
total: neighbours.length,
|
||||
upstream_count: upstreams.length,
|
||||
downstream_count: downstreams.length,
|
||||
peer_count: peers.length,
|
||||
upstreams: upstreams.slice(0, 20),
|
||||
downstreams: downstreams.slice(0, 20),
|
||||
peers: peers.slice(0, 20),
|
||||
cross_check: neighbourCrossCheck,
|
||||
},
|
||||
ix_presence: {
|
||||
total_connections: ixConnections.length,
|
||||
unique_ixps: [...new Set(ixConnections.map((ix) => ix.ix_id))].length,
|
||||
connections: ixConnections,
|
||||
},
|
||||
ix_locations: ixLocations,
|
||||
facilities: {
|
||||
total: facilities.length,
|
||||
list: facilities,
|
||||
},
|
||||
routing: routingInfo,
|
||||
resilience_score: computeResilienceScore(upstreams, peers, ixConnections, prefixes),
|
||||
route_leak: computeRouteLeakDetection(upstreams, downstreams, peers),
|
||||
bgp_he_net: bgpHeData || null,
|
||||
atlas: {
|
||||
total_probes: atlasProbes.length,
|
||||
connected: atlasConnected.length,
|
||||
disconnected: atlasProbes.length - atlasConnected.length,
|
||||
anchors: atlasAnchors.length,
|
||||
probes: atlasProbes.slice(0, 100).map(p => ({
|
||||
id: p.id,
|
||||
status: p.status_name || p.status || "Unknown",
|
||||
is_anchor: p.is_anchor || false,
|
||||
country: p.country_code || "",
|
||||
prefix_v4: p.prefix_v4 || "",
|
||||
prefix_v6: p.prefix_v6 || "",
|
||||
description: p.description || "",
|
||||
})),
|
||||
},
|
||||
data_quality: dataQuality,
|
||||
source_timing: sourceTiming,
|
||||
contacts: (() => {
|
||||
const pocs = (pocData && pocData.data) ? pocData.data : [];
|
||||
return pocs.slice(0, 20).map(p => ({
|
||||
role: p.role || "",
|
||||
name: p.name || "",
|
||||
email: p.email || "",
|
||||
url: p.url || "",
|
||||
visible: p.visible || "",
|
||||
}));
|
||||
})(),
|
||||
registration: (() => {
|
||||
const events = (rdapData && rdapData.events) ? rdapData.events : [];
|
||||
const created = (events.find(e => e.eventAction === "registration") || {}).eventDate || "";
|
||||
const lastChg = (events.find(e => e.eventAction === "last changed") || {}).eventDate || "";
|
||||
return {
|
||||
created: created ? created.slice(0, 10) : "",
|
||||
last_modified: lastChg ? lastChg.slice(0, 10) : "",
|
||||
rir: rir || "",
|
||||
handle: (rdapData && rdapData.handle) ? rdapData.handle : ("AS" + asn),
|
||||
rdap_source: (rdapData && rdapData.port43) ? rdapData.port43 : "",
|
||||
};
|
||||
})(),
|
||||
_provenance: {
|
||||
prefixes: { source: "RIPE Stat announced-prefixes", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net prefix count daily" },
|
||||
neighbours: { source: "RIPE Stat asn-neighbours", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net peer count daily" },
|
||||
rpki: { source: "Cloudflare RPKI + RIPE Validator", validation: "cross-validated", confidence: "high", note: "Two independent RPKI sources compared" },
|
||||
ix_presence: { source: "PeeringDB netixlan (local mirror)", validation: "cross-validated", confidence: "high", note: "PeeringDB mirror refreshed daily" },
|
||||
facilities: { source: "PeeringDB netfac (local mirror)", validation: "single-source", confidence: "medium" },
|
||||
bgp_he_net: { source: "bgp.he.net HTML scrape", validation: "single-source", confidence: "medium", note: "HTML scrape, no official API — may have parsing drift" },
|
||||
atlas: { source: "RIPE Atlas API", validation: "single-source", confidence: "medium", note: "Probe availability varies by region" },
|
||||
resilience_score: { source: "Computed from RIPE Stat + PeeringDB", validation: "computed", confidence: "high", note: "All inputs cross-validated daily" },
|
||||
route_leak: { source: "RIPE Stat asn-neighbours heuristic", validation: "heuristic", confidence: "medium", note: "Pattern-based, not real-time — false positives possible" },
|
||||
registration: { source: "RDAP (RIR registry)", validation: "single-source", confidence: "high" },
|
||||
contacts: { source: "PeeringDB POC API", validation: "single-source", confidence: "medium", note: "Subject to PeeringDB rate limiting" },
|
||||
},
|
||||
};
|
||||
|
||||
// Update duration to include cross-check time
|
||||
result.meta.duration_ms = Date.now() - start;
|
||||
|
||||
// A DB hiccup shouldn't get cached as this ASN's answer for the full 5min --
|
||||
// retry soon instead of repeating a degraded result to every visitor.
|
||||
cacheSet(cacheKey, result, degradedSources.length > 0 ? CACHE_TTL_LOOKUP_DEGRADED : CACHE_TTL_LOOKUP);
|
||||
res.end(JSON.stringify(result, null, 2));
|
||||
} catch (err) {
|
||||
const duration = Date.now() - start;
|
||||
res.writeHead(500);
|
||||
res.end(JSON.stringify({ error: "Lookup failed", message: err.message, duration_ms: duration }));
|
||||
}
|
||||
return;
|
||||
return lookupRoute.handleLookup(req, res, url);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// AS Relationships endpoint: /api/relationships?asn=X
|
||||
// Returns upstream providers, downstream customers, and peers
|
||||
// with resolved names. Based on RIPE Stat asn-neighbours.
|
||||
// ============================================================
|
||||
if (reqPath === "/api/relationships") {
|
||||
|
||||
90
server/routes/lookup/cross-check.js
Normal file
90
server/routes/lookup/cross-check.js
Normal file
@ -0,0 +1,90 @@
|
||||
const { crossCheckRpki } = require("../../services/rpki");
|
||||
|
||||
// Multi-source cross-checks (RPKI vs RIPE Validator, prefix/neighbour counts
|
||||
// vs bgp.he.net) plus the overall data-quality summary derived from them.
|
||||
async function computeCrossChecks(asn, allPrefixes, rpkiStatuses, prefixes, neighbours, bgpHeData) {
|
||||
let rpkiCrossCheck = { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 };
|
||||
let prefixCrossCheck = { ripe_stat: prefixes.length, bgp_he_net: null, agreement: null, note: "" };
|
||||
let neighbourCrossCheck = { ripe_stat_total: neighbours.length, bgp_he_net_total: null };
|
||||
|
||||
try {
|
||||
// RPKI cross-check: sample up to 5 prefixes against RIPE Validator (with 8s total timeout)
|
||||
const rpkiCrossPromise = crossCheckRpki(asn, allPrefixes, rpkiStatuses);
|
||||
const rpkiCrossResult = await Promise.race([
|
||||
rpkiCrossPromise,
|
||||
new Promise((resolve) => setTimeout(() => resolve(null), 8000)),
|
||||
]);
|
||||
if (rpkiCrossResult) rpkiCrossCheck = rpkiCrossResult;
|
||||
} catch (_e) { /* cross-check failed, keep defaults */ }
|
||||
|
||||
// bgp.he.net fetching is deliberately disabled (see the hardcoded
|
||||
// Promise.resolve(null) in index.js's timedFetch("bgp.he.net", ...)) --
|
||||
// bgpHeData is therefore always null and this whole branch, along with the
|
||||
// prefix/neighbour cross-checks below, never actually runs. Left in place
|
||||
// (not deleted) in case bgp.he.net fetching is re-enabled later, but
|
||||
// overall_confidence below must not claim a 2-source agreement that never
|
||||
// happened -- see the sources:1/2 handling further down.
|
||||
// Prefix count cross-check: compare RIPE Stat vs bgp.he.net
|
||||
if (bgpHeData) {
|
||||
const heV4 = bgpHeData.prefixes_v4 || 0;
|
||||
const heV6 = bgpHeData.prefixes_v6 || 0;
|
||||
const heTotal = heV4 + heV6;
|
||||
if (heTotal > 0) {
|
||||
prefixCrossCheck.bgp_he_net = heTotal;
|
||||
const ripeStat = prefixes.length;
|
||||
if (ripeStat > 0 && heTotal > 0) {
|
||||
const ratio = Math.min(ripeStat, heTotal) / Math.max(ripeStat, heTotal);
|
||||
prefixCrossCheck.agreement = ratio >= 0.9;
|
||||
const diff = Math.abs(ripeStat - heTotal);
|
||||
prefixCrossCheck.note = diff === 0
|
||||
? "Exact match"
|
||||
: "Difference of " + diff + " prefixes (" + Math.round((1 - ratio) * 100) + "% divergence)";
|
||||
}
|
||||
} else {
|
||||
prefixCrossCheck.note = "bgp.he.net prefix count unavailable";
|
||||
}
|
||||
|
||||
// Neighbour cross-check: compare RIPE Stat vs bgp.he.net peer_count
|
||||
if (bgpHeData.peer_count != null) {
|
||||
neighbourCrossCheck.bgp_he_net_total = bgpHeData.peer_count;
|
||||
}
|
||||
} else {
|
||||
prefixCrossCheck.note = "bgp.he.net data unavailable";
|
||||
}
|
||||
|
||||
// Compute overall data quality. Only push a score for a cross-check that
|
||||
// actually ran (agreement_pct !== null) -- an unrun/inconclusive check must
|
||||
// not silently count as a perfect 100 in the average, and confidence must
|
||||
// reflect "nothing was actually cross-checked" rather than defaulting high.
|
||||
const crossCheckScores = [];
|
||||
if (rpkiCrossCheck.agreement_pct !== null) crossCheckScores.push(rpkiCrossCheck.agreement_pct);
|
||||
// Prefix agreement: convert to percentage
|
||||
if (prefixCrossCheck.bgp_he_net != null && prefixes.length > 0) {
|
||||
const pfxRatio = Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net);
|
||||
crossCheckScores.push(Math.round(pfxRatio * 100));
|
||||
}
|
||||
// Neighbour agreement
|
||||
if (neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0) {
|
||||
const nbrRatio = Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total);
|
||||
crossCheckScores.push(Math.round(nbrRatio * 100));
|
||||
}
|
||||
const avgAgreement = crossCheckScores.length > 0
|
||||
? Math.round(crossCheckScores.reduce((a, b) => a + b, 0) / crossCheckScores.length)
|
||||
: null;
|
||||
const overallConfidence = avgAgreement === null ? "unknown" : avgAgreement > 90 ? "high" : avgAgreement >= 70 ? "medium" : "low";
|
||||
|
||||
const dataQuality = {
|
||||
sources_queried: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator"],
|
||||
cross_checks: {
|
||||
rpki: { sources: rpkiCrossCheck.agreement_pct !== null ? 2 : 1, agreement_pct: rpkiCrossCheck.agreement_pct, sample_size: rpkiCrossCheck.sample_size, comparable_size: rpkiCrossCheck.comparable_size, disagreements: rpkiCrossCheck.disagreements },
|
||||
prefixes: { sources: prefixCrossCheck.bgp_he_net != null ? 2 : 1, agreement_pct: prefixCrossCheck.bgp_he_net != null ? Math.round((Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net || 1)) * 100) : null, ripe_stat: prefixCrossCheck.ripe_stat, bgp_he_net: prefixCrossCheck.bgp_he_net, note: prefixCrossCheck.note },
|
||||
neighbours: { sources: neighbourCrossCheck.bgp_he_net_total != null ? 2 : 1, agreement_pct: neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0 ? Math.round((Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total)) * 100) : null, ripe_stat_total: neighbourCrossCheck.ripe_stat_total, bgp_he_net_total: neighbourCrossCheck.bgp_he_net_total },
|
||||
},
|
||||
overall_confidence: overallConfidence,
|
||||
overall_agreement_pct: avgAgreement,
|
||||
};
|
||||
|
||||
return { rpkiCrossCheck, prefixCrossCheck, neighbourCrossCheck, dataQuality };
|
||||
}
|
||||
|
||||
module.exports = { computeCrossChecks };
|
||||
73
server/routes/lookup/enrich-neighbours.js
Normal file
73
server/routes/lookup/enrich-neighbours.js
Normal file
@ -0,0 +1,73 @@
|
||||
const localDb = require("../../../local-db-client");
|
||||
const { fetchRipeStatCached } = require("../../services/ripe-stat");
|
||||
|
||||
// Resolve empty AS names — all in parallel, with 3s timeout. Mutates entries
|
||||
// in place (matches original control flow).
|
||||
async function resolveEmptyNames(neighbourLists) {
|
||||
const emptyNameNeighbours = neighbourLists.flat().filter(n => !n.name);
|
||||
if (emptyNameNeighbours.length === 0) return;
|
||||
const resolvePromise = Promise.all(
|
||||
emptyNameNeighbours.map(n =>
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 })
|
||||
.then(r => { if (r?.data?.holder) n.name = r.data.holder; })
|
||||
.catch(() => {})
|
||||
)
|
||||
);
|
||||
await Promise.race([resolvePromise, new Promise(r => setTimeout(r, 3000))]);
|
||||
}
|
||||
|
||||
// Threat Intelligence Enrichment for Neighbors: enrich neighbor data with
|
||||
// threat status from the local threat_intel table. Batch lookups (cap at 50
|
||||
// to avoid overwhelming DB), 4s timeout.
|
||||
async function buildThreatMap(neighbors) {
|
||||
const threatMap = {};
|
||||
const toCheck = neighbors.slice(0, 50);
|
||||
const threatPromises = toCheck.map(async (n) => {
|
||||
try {
|
||||
const asNum = String(n.asn);
|
||||
const threat = await localDb.getThreatIntel(asNum);
|
||||
if (threat) {
|
||||
threatMap[n.asn] = {
|
||||
threat_level: threat.threat_level,
|
||||
confidence_score: threat.confidence_score,
|
||||
source: threat.source,
|
||||
cached_at: threat.cached_at,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
// Gracefully skip on error
|
||||
console.error(`[Threat Lookup] Error checking ASN ${n.asn}:`, e.message);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.race([
|
||||
Promise.all(threatPromises),
|
||||
new Promise(r => setTimeout(r, 4000)),
|
||||
]);
|
||||
|
||||
return threatMap;
|
||||
}
|
||||
|
||||
function attachThreat(n, threatMap) {
|
||||
return {
|
||||
...n,
|
||||
threat_level: threatMap[n.asn]?.threat_level || null,
|
||||
threat_confidence: threatMap[n.asn]?.confidence_score || null,
|
||||
threat_source: threatMap[n.asn]?.source || null,
|
||||
};
|
||||
}
|
||||
|
||||
// Full enrichment pipeline: resolve empty names, then attach threat intel to
|
||||
// upstreams/downstreams/peers. Returns new arrays (does not mutate the input
|
||||
// arrays themselves, though resolveEmptyNames does mutate entry.name).
|
||||
async function enrichNeighbours(upstreams, downstreams, peers) {
|
||||
await resolveEmptyNames([upstreams, downstreams, peers]);
|
||||
const threatMap = await buildThreatMap([...upstreams, ...downstreams, ...peers]);
|
||||
return {
|
||||
upstreams: upstreams.map(n => attachThreat(n, threatMap)),
|
||||
downstreams: downstreams.map(n => attachThreat(n, threatMap)),
|
||||
peers: peers.map(n => attachThreat(n, threatMap)),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { enrichNeighbours };
|
||||
114
server/routes/lookup/facilities.js
Normal file
114
server/routes/lookup/facilities.js
Normal file
@ -0,0 +1,114 @@
|
||||
const { fetchPeeringDB } = require("../../services/peeringdb");
|
||||
const { CITY_COORDS } = require("../../data/city-coords");
|
||||
const { IX_CITY_MAP } = require("../../data/ix-city-map");
|
||||
|
||||
// Batch-fetch facility lat/lon by ID, 25 per request (PeeringDB /fac?id__in= limit),
|
||||
// bounded by an overall race timeout. Returns {fac_id: {lat, lon}}.
|
||||
async function fetchFacCoordsBatch(facIds, timeoutMs) {
|
||||
const coordMap = {};
|
||||
if (facIds.length === 0) return coordMap;
|
||||
try {
|
||||
const chunks = [];
|
||||
for (let i = 0; i < facIds.length; i += 25) chunks.push(facIds.slice(i, i + 25));
|
||||
const coordResults = await Promise.race([
|
||||
Promise.all(chunks.map(chunk =>
|
||||
fetchPeeringDB("/fac?id__in=" + chunk.join(",") + "&fields=id,latitude,longitude").catch(() => null)
|
||||
)),
|
||||
new Promise(r => setTimeout(() => r([]), timeoutMs)),
|
||||
]);
|
||||
(coordResults || []).forEach(res => {
|
||||
(res?.data || []).forEach(f => { if (f.latitude && f.longitude) coordMap[f.id] = { lat: f.latitude, lon: f.longitude }; });
|
||||
});
|
||||
} catch (e) { /* graceful degradation */ }
|
||||
return coordMap;
|
||||
}
|
||||
|
||||
// Resolve facility coordinates for the map, then IX locations via ixfac -> fac
|
||||
// coordinates (max 20 IXs), then the name/CITY_COORDS/IX_CITY_MAP geocode
|
||||
// fallbacks for IXs PeeringDB has no facility coordinates for.
|
||||
async function resolveFacilitiesAndIxLocations(facilitiesRaw, ixConnections) {
|
||||
// Batch-fetch facility coordinates for map (max 50 facilities)
|
||||
const facIds = facilitiesRaw.map(f => f.fac_id).filter(Boolean).slice(0, 50);
|
||||
let facCoordMap = await fetchFacCoordsBatch(facIds, 5000);
|
||||
|
||||
const facilities = facilitiesRaw.map(f => ({
|
||||
...f,
|
||||
latitude: facCoordMap[f.fac_id] ? facCoordMap[f.fac_id].lat : null,
|
||||
longitude: facCoordMap[f.fac_id] ? facCoordMap[f.fac_id].lon : null,
|
||||
}));
|
||||
|
||||
// Get IX locations for map via ixfac -> fac coordinates (max 20 IXs)
|
||||
const uniqueIxIds = [...new Set(ixConnections.map(c => c.ix_id))].filter(Boolean).slice(0, 20);
|
||||
let ixLocations = [];
|
||||
if (uniqueIxIds.length > 0) {
|
||||
try {
|
||||
const ixFacData = await Promise.race([
|
||||
fetchPeeringDB("/ixfac?ix_id__in=" + uniqueIxIds.join(",")),
|
||||
new Promise(r => setTimeout(() => r(null), 5000)),
|
||||
]);
|
||||
const ixFacs = ixFacData?.data || [];
|
||||
// Collect unique fac_ids we don't already have coords for
|
||||
const extraFacIds = [...new Set(ixFacs.map(f => f.fac_id).filter(id => id && !facCoordMap[id]))].slice(0, 30);
|
||||
if (extraFacIds.length > 0) {
|
||||
const extraCoords = await fetchFacCoordsBatch(extraFacIds, 4000);
|
||||
facCoordMap = { ...facCoordMap, ...extraCoords };
|
||||
}
|
||||
// Build IX locations: pick first facility with coords per IX
|
||||
const ixNameMap = {};
|
||||
ixConnections.forEach(c => { if (c.ix_id && c.ix_name) ixNameMap[c.ix_id] = c.ix_name; });
|
||||
const seenIx = {};
|
||||
ixFacs.forEach(f => {
|
||||
if (seenIx[f.ix_id]) return;
|
||||
const coords = facCoordMap[f.fac_id];
|
||||
if (coords) {
|
||||
seenIx[f.ix_id] = true;
|
||||
ixLocations.push({ ix_id: f.ix_id, name: ixNameMap[f.ix_id] || f.name || "", city: f.city || "", country: f.country || "", latitude: coords.lat, longitude: coords.lon });
|
||||
}
|
||||
});
|
||||
} catch (e) { /* graceful degradation */ }
|
||||
}
|
||||
|
||||
applyIxGeocodeFallback(ixLocations, ixConnections);
|
||||
|
||||
return { facilities, ixLocations };
|
||||
}
|
||||
|
||||
// === IX Location Geocode Fallback ===
|
||||
// Some IXPs have no facility coordinates in PeeringDB.
|
||||
// Use ix_name city extraction + hard-coded IX→city map as fallback.
|
||||
// Mutates ixLocations in place (matches original control flow).
|
||||
function applyIxGeocodeFallback(ixLocations, ixConnections) {
|
||||
var ixIdsWithCoords = new Set(ixLocations.map(function(l) { return l.ix_id; }));
|
||||
ixConnections.forEach(function(conn) {
|
||||
if (ixIdsWithCoords.has(conn.ix_id)) return;
|
||||
var name = conn.ix_name || "";
|
||||
if (name) {
|
||||
var words = name.toLowerCase().replace(/[^a-z\s]/g, " ").split(/\s+/).filter(Boolean);
|
||||
for (var w = 0; w < words.length; w++) {
|
||||
if (CITY_COORDS[words[w]]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: name, city: words[w].charAt(0).toUpperCase() + words[w].slice(1), country: "", latitude: CITY_COORDS[words[w]][0], longitude: CITY_COORDS[words[w]][1], source: "name_geocode" });
|
||||
ixIdsWithCoords.add(conn.ix_id);
|
||||
return;
|
||||
}
|
||||
if (w < words.length - 1) {
|
||||
var tw = words[w] + " " + words[w + 1];
|
||||
if (CITY_COORDS[tw]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: name, city: tw, country: "", latitude: CITY_COORDS[tw][0], longitude: CITY_COORDS[tw][1], source: "name_geocode" });
|
||||
ixIdsWithCoords.add(conn.ix_id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// Hard-coded IX ID → city for well-known IXPs whose names don't contain city
|
||||
ixConnections.forEach(function(conn) {
|
||||
if (ixIdsWithCoords.has(conn.ix_id)) return;
|
||||
var city = IX_CITY_MAP[conn.ix_id];
|
||||
if (city && CITY_COORDS[city]) {
|
||||
ixLocations.push({ ix_id: conn.ix_id, name: conn.ix_name || ("IX " + conn.ix_id), city: city.charAt(0).toUpperCase() + city.slice(1), country: "", latitude: CITY_COORDS[city][0], longitude: CITY_COORDS[city][1], source: "ix_city_map" });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { resolveFacilitiesAndIxLocations };
|
||||
337
server/routes/lookup/index.js
Normal file
337
server/routes/lookup/index.js
Normal file
@ -0,0 +1,337 @@
|
||||
const localDb = require("../../../local-db-client");
|
||||
const { fetchJSON } = require("../../services/http-helpers");
|
||||
const { fetchPeeringDB, fetchPeeringDBWithRetry } = require("../../services/peeringdb");
|
||||
const { ensureAspaCache, validateRPKIWithCache } = require("../../services/rpki");
|
||||
const { computeResilienceScore } = require("../../resilience-score");
|
||||
const { computeRouteLeakDetection } = require("../../route-leak");
|
||||
const { pdbSourceCache } = require("../../caches/pdb-source-cache");
|
||||
const {
|
||||
cacheGet, cacheSet, CACHE_TTL_LOOKUP, CACHE_TTL_LOOKUP_DEGRADED,
|
||||
rdapCacheGet, rdapCacheSet,
|
||||
} = require("../../caches/response-caches");
|
||||
const { resolveFacilitiesAndIxLocations } = require("./facilities");
|
||||
const { enrichNeighbours } = require("./enrich-neighbours");
|
||||
const { deriveRirAndCountry } = require("./rir-country");
|
||||
const { computeRoutingInfo } = require("./routing-info");
|
||||
const { computeCrossChecks } = require("./cross-check");
|
||||
|
||||
// Main lookup endpoint: /api/lookup?asn=X
|
||||
async function handleLookup(req, res, url) {
|
||||
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
|
||||
if (!rawAsn) {
|
||||
res.writeHead(400);
|
||||
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
|
||||
}
|
||||
const asn = rawAsn;
|
||||
const cacheKey = "lookup:" + asn;
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) {
|
||||
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
|
||||
return res.end(JSON.stringify(cached));
|
||||
}
|
||||
const start = Date.now();
|
||||
|
||||
try {
|
||||
// Phase 0: Get PDB net first — check L2 cache, then API with retry
|
||||
let pdbNet = pdbSourceCache.get("net", asn);
|
||||
if (!pdbNet) {
|
||||
pdbNet = await fetchPeeringDBWithRetry("/net?asn=" + asn);
|
||||
if (pdbNet) pdbSourceCache.set("net", asn, pdbNet);
|
||||
}
|
||||
const net = pdbNet?.data?.[0] || {};
|
||||
const netId = net.id;
|
||||
|
||||
// Phase 1: ALL calls in parallel — RIPE Stat (cached+throttled) + PDB IX/Fac (cached) + Atlas + bgp.he.net
|
||||
const ixQuery = netId
|
||||
? "/netixlan?net_id=" + netId + "&limit=1000"
|
||||
: "/netixlan?asn=" + asn + "&limit=1000";
|
||||
const ixCacheKey = netId ? String(netId) : "asn:" + asn;
|
||||
|
||||
// Check PDB source cache for IX/Fac data
|
||||
let cachedIxlan = pdbSourceCache.get("netixlan", ixCacheKey);
|
||||
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
|
||||
|
||||
// Per-source timing tracking — 9s hard cap per source to prevent long-tail blocking
|
||||
const sourceTiming = {};
|
||||
function timedFetch(name, promise) {
|
||||
const ts = Date.now();
|
||||
return Promise.race([
|
||||
Promise.resolve(promise),
|
||||
new Promise(function(r) { setTimeout(function() { r(null); }, 9000); }),
|
||||
])
|
||||
.then(function(r) { sourceTiming[name] = Date.now() - ts; return r; })
|
||||
.catch(function() { sourceTiming[name] = null; return null; });
|
||||
}
|
||||
|
||||
const pocQuery = netId ? "/poc?net_id=" + netId + "&limit=25" : null;
|
||||
|
||||
// RDAP: check module-level cache first, only hit RIR endpoints on cache miss
|
||||
const rdapCached = rdapCacheGet(asn);
|
||||
const rdapPromise = rdapCached !== undefined
|
||||
? Promise.resolve(rdapCached)
|
||||
: Promise.race([
|
||||
...["https://rdap.db.ripe.net/autnum/"+asn, "https://rdap.arin.net/registry/autnum/"+asn,
|
||||
"https://rdap.apnic.net/autnum/"+asn, "https://rdap.lacnic.net/rdap/autnum/"+asn,
|
||||
"https://rdap.afrinic.net/rdap/autnum/"+asn].map(url =>
|
||||
fetchJSON(url, { timeout: 4000 })
|
||||
.then(d => (d && !d.errorCode && d.handle) ? d : new Promise(() => {}))
|
||||
.catch(() => new Promise(() => {}))
|
||||
),
|
||||
new Promise(resolve => setTimeout(() => resolve(null), 5000)),
|
||||
]).then(d => { rdapCacheSet(asn, d); return d; });
|
||||
|
||||
const promises = [
|
||||
timedFetch("RIPE Stat Prefixes", localDb ? localDb.getRipeStatAnnouncedPrefixes(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Neighbours", localDb ? localDb.getRipeStatAsnNeighbours(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Overview", localDb ? localDb.getRipeStatAsOverview(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat RIR", Promise.resolve(null)),
|
||||
timedFetch("RIPE Atlas", Promise.resolve(null)),
|
||||
timedFetch("bgp.he.net", Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat Visibility", localDb ? localDb.getRipeStatVisibility(asn) : Promise.resolve(null)),
|
||||
timedFetch("RIPE Stat PrefixSize", localDb ? localDb.getRipeStatPrefixSizeDistribution(asn) : Promise.resolve(null)),
|
||||
timedFetch("PeeringDB IXLan", cachedIxlan ? Promise.resolve(cachedIxlan) : fetchPeeringDBWithRetry(ixQuery)),
|
||||
timedFetch("PeeringDB Facilities", cachedFac ? Promise.resolve(cachedFac) : (netId ? fetchPeeringDBWithRetry("/netfac?net_id=" + netId + "&limit=1000") : Promise.resolve(null))),
|
||||
timedFetch("PeeringDB Contacts", pocQuery ? fetchPeeringDB(pocQuery).catch(() => null) : Promise.resolve(null)),
|
||||
timedFetch("RDAP Registration", rdapPromise),
|
||||
];
|
||||
const [prefixData, neighbourData, overviewData, rirData, atlasProbeData, bgpHeData, visibilityData, prefixSizeData, ixlanData, facData, pocData, rdapData] = await Promise.all(promises);
|
||||
|
||||
// Store PDB results in L2 source cache for future lookups
|
||||
if (!cachedIxlan && ixlanData) pdbSourceCache.set("netixlan", ixCacheKey, ixlanData);
|
||||
if (!cachedFac && facData) pdbSourceCache.set("netfac", String(netId), facData);
|
||||
|
||||
// local-db-client's getRipeStat* functions return status:"error" (not "ok") when
|
||||
// the local Postgres DB query itself failed -- track which sources actually
|
||||
// failed so we don't cache a DB hiccup's fake-empty result as if it were a
|
||||
// genuine "this ASN has 0 prefixes" answer for the full 5-minute TTL.
|
||||
const degradedSources = [
|
||||
["RIPE Stat Prefixes", prefixData], ["RIPE Stat Neighbours", neighbourData],
|
||||
["RIPE Stat Overview", overviewData], ["RIPE Stat Visibility", visibilityData],
|
||||
["RIPE Stat PrefixSize", prefixSizeData],
|
||||
].filter(([, v]) => v && v.status === "error").map(([name]) => name);
|
||||
|
||||
const prefixes = prefixData?.data?.prefixes || [];
|
||||
const neighbours = neighbourData?.data?.neighbours || [];
|
||||
const overview = overviewData?.data || {};
|
||||
const rirEntries = rirData?.data?.located_resources || rirData?.data?.rir_stats || [];
|
||||
|
||||
// Bug 6 fix: Atlas probe status uses status.name (object), not status_name (flat)
|
||||
const atlasProbes = atlasProbeData?.results || [];
|
||||
const atlasConnected = atlasProbes.filter(p => {
|
||||
const sName = (p.status_name || (p.status && p.status.name) || "").toLowerCase();
|
||||
return sName === "connected";
|
||||
});
|
||||
const atlasAnchors = atlasProbes.filter(p => p.is_anchor === true);
|
||||
|
||||
// RPKI: validate ALL prefixes using local Cloudflare RPKI data (all 5 RIRs, instant)
|
||||
await ensureAspaCache();
|
||||
const allPrefixes = prefixes.map((p) => p.prefix);
|
||||
const rpkiAllResults = await Promise.all(allPrefixes.map((pfx) => validateRPKIWithCache(asn, pfx)));
|
||||
|
||||
const ixConnections = (ixlanData?.data || [])
|
||||
.map((ix) => ({
|
||||
ix_name: ix.name || "",
|
||||
ix_id: ix.ix_id,
|
||||
speed_mbps: ix.speed || 0,
|
||||
ipv4: ix.ipaddr4 || null,
|
||||
ipv6: ix.ipaddr6 || null,
|
||||
city: ix.city || "",
|
||||
is_rs_peer: ix.is_rs_peer === true,
|
||||
}))
|
||||
.sort((a, b) => b.speed_mbps - a.speed_mbps);
|
||||
|
||||
const facilitiesRaw = (facData?.data || []).map((f) => ({
|
||||
fac_id: f.fac_id,
|
||||
name: f.name || "",
|
||||
city: f.city || "",
|
||||
country: f.country || "",
|
||||
}));
|
||||
|
||||
const { facilities, ixLocations } = await resolveFacilitiesAndIxLocations(facilitiesRaw, ixConnections);
|
||||
|
||||
const rpkiStatuses = rpkiAllResults;
|
||||
const rpkiValid = rpkiStatuses.filter((r) => r.status === "valid").length;
|
||||
const rpkiInvalid = rpkiStatuses.filter((r) => r.status === "invalid").length;
|
||||
const rpkiUnavailable = rpkiStatuses.filter((r) => r.status === "unavailable").length;
|
||||
const rpkiNotFound = rpkiStatuses.filter((r) => r.status !== "valid" && r.status !== "invalid" && r.status !== "unavailable").length;
|
||||
// Exclude "unavailable" (DB error) from the coverage denominator -- a DB hiccup
|
||||
// must not lower a network's displayed RPKI coverage percentage.
|
||||
const rpkiTotal = rpkiStatuses.length - rpkiUnavailable;
|
||||
const rpkiCoverage = rpkiTotal > 0 ? Math.round((rpkiValid / rpkiTotal) * 100) : 0;
|
||||
|
||||
let upstreams = neighbours
|
||||
.filter((n) => n.type === "left")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
let downstreams = neighbours
|
||||
.filter((n) => n.type === "right")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
let peers = neighbours
|
||||
.filter((n) => n.type === "uncertain" || n.type === "peer")
|
||||
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
||||
.sort((a, b) => b.power - a.power);
|
||||
|
||||
const enriched = await enrichNeighbours(upstreams, downstreams, peers);
|
||||
upstreams = enriched.upstreams;
|
||||
downstreams = enriched.downstreams;
|
||||
peers = enriched.peers;
|
||||
|
||||
const { rir, country } = deriveRirAndCountry(rirEntries, rirData, rdapData, bgpHeData);
|
||||
|
||||
const duration = Date.now() - start;
|
||||
|
||||
const routingInfo = await computeRoutingInfo(prefixes, visibilityData, prefixSizeData);
|
||||
|
||||
const { rpkiCrossCheck, prefixCrossCheck, neighbourCrossCheck, dataQuality } =
|
||||
await computeCrossChecks(asn, allPrefixes, rpkiStatuses, prefixes, neighbours, bgpHeData);
|
||||
|
||||
const result = {
|
||||
meta: {
|
||||
service: "PeerCortex",
|
||||
version: "0.6.9",
|
||||
query: "AS" + asn,
|
||||
duration_ms: duration,
|
||||
sources: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator", "Route Views"],
|
||||
timestamp: new Date().toISOString(),
|
||||
rpki_prefixes_checked: rpkiTotal,
|
||||
total_prefixes: prefixes.length,
|
||||
degraded_sources: degradedSources,
|
||||
},
|
||||
network: {
|
||||
asn: parseInt(asn),
|
||||
name: net.name || overview?.holder || (bgpHeData && bgpHeData.name_from_title) || "Unknown",
|
||||
aka: net.aka || "",
|
||||
org_name: (net.org && net.org.name) ? net.org.name : "",
|
||||
website: net.website || "",
|
||||
type: net.info_type || "",
|
||||
policy: net.policy_general || "",
|
||||
traffic: net.info_traffic || "",
|
||||
ratio: net.info_ratio || "",
|
||||
scope: net.info_scope || "",
|
||||
notes: net.notes ? net.notes.substring(0, 500) : "",
|
||||
peeringdb_id: netId || null,
|
||||
rir: rir,
|
||||
country: country,
|
||||
city: net.city || "",
|
||||
latitude: (net.latitude != null) ? net.latitude : null,
|
||||
longitude: (net.longitude != null) ? net.longitude : null,
|
||||
looking_glass: net.looking_glass || "",
|
||||
route_server: net.route_server || "",
|
||||
info_prefixes4: net.info_prefixes4 || 0,
|
||||
info_prefixes6: net.info_prefixes6 || 0,
|
||||
status: net.status || "",
|
||||
peeringdb_created: net.created ? net.created.slice(0, 10) : "",
|
||||
peeringdb_updated: net.updated ? net.updated.slice(0, 10) : "",
|
||||
},
|
||||
prefixes: {
|
||||
total: prefixes.length,
|
||||
ipv4: prefixes.filter((p) => !p.prefix.includes(":")).length,
|
||||
ipv6: prefixes.filter((p) => p.prefix.includes(":")).length,
|
||||
list: prefixes.map((p) => p.prefix),
|
||||
cross_check: prefixCrossCheck,
|
||||
},
|
||||
rpki: {
|
||||
coverage_percent: rpkiCoverage,
|
||||
valid: rpkiValid,
|
||||
invalid: rpkiInvalid,
|
||||
not_found: rpkiNotFound,
|
||||
unavailable: rpkiUnavailable,
|
||||
checked: rpkiTotal,
|
||||
details: rpkiStatuses,
|
||||
cross_check: rpkiCrossCheck,
|
||||
},
|
||||
neighbours: {
|
||||
total: neighbours.length,
|
||||
upstream_count: upstreams.length,
|
||||
downstream_count: downstreams.length,
|
||||
peer_count: peers.length,
|
||||
upstreams: upstreams.slice(0, 20),
|
||||
downstreams: downstreams.slice(0, 20),
|
||||
peers: peers.slice(0, 20),
|
||||
cross_check: neighbourCrossCheck,
|
||||
},
|
||||
ix_presence: {
|
||||
total_connections: ixConnections.length,
|
||||
unique_ixps: [...new Set(ixConnections.map((ix) => ix.ix_id))].length,
|
||||
connections: ixConnections,
|
||||
},
|
||||
ix_locations: ixLocations,
|
||||
facilities: {
|
||||
total: facilities.length,
|
||||
list: facilities,
|
||||
},
|
||||
routing: routingInfo,
|
||||
resilience_score: computeResilienceScore(upstreams, peers, ixConnections, prefixes),
|
||||
route_leak: computeRouteLeakDetection(upstreams, downstreams, peers),
|
||||
bgp_he_net: bgpHeData || null,
|
||||
atlas: {
|
||||
total_probes: atlasProbes.length,
|
||||
connected: atlasConnected.length,
|
||||
disconnected: atlasProbes.length - atlasConnected.length,
|
||||
anchors: atlasAnchors.length,
|
||||
probes: atlasProbes.slice(0, 100).map(p => ({
|
||||
id: p.id,
|
||||
status: p.status_name || p.status || "Unknown",
|
||||
is_anchor: p.is_anchor || false,
|
||||
country: p.country_code || "",
|
||||
prefix_v4: p.prefix_v4 || "",
|
||||
prefix_v6: p.prefix_v6 || "",
|
||||
description: p.description || "",
|
||||
})),
|
||||
},
|
||||
data_quality: dataQuality,
|
||||
source_timing: sourceTiming,
|
||||
contacts: (() => {
|
||||
const pocs = (pocData && pocData.data) ? pocData.data : [];
|
||||
return pocs.slice(0, 20).map(p => ({
|
||||
role: p.role || "",
|
||||
name: p.name || "",
|
||||
email: p.email || "",
|
||||
url: p.url || "",
|
||||
visible: p.visible || "",
|
||||
}));
|
||||
})(),
|
||||
registration: (() => {
|
||||
const events = (rdapData && rdapData.events) ? rdapData.events : [];
|
||||
const created = (events.find(e => e.eventAction === "registration") || {}).eventDate || "";
|
||||
const lastChg = (events.find(e => e.eventAction === "last changed") || {}).eventDate || "";
|
||||
return {
|
||||
created: created ? created.slice(0, 10) : "",
|
||||
last_modified: lastChg ? lastChg.slice(0, 10) : "",
|
||||
rir: rir || "",
|
||||
handle: (rdapData && rdapData.handle) ? rdapData.handle : ("AS" + asn),
|
||||
rdap_source: (rdapData && rdapData.port43) ? rdapData.port43 : "",
|
||||
};
|
||||
})(),
|
||||
_provenance: {
|
||||
prefixes: { source: "RIPE Stat announced-prefixes", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net prefix count daily" },
|
||||
neighbours: { source: "RIPE Stat asn-neighbours", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net peer count daily" },
|
||||
rpki: { source: "Cloudflare RPKI + RIPE Validator", validation: "cross-validated", confidence: "high", note: "Two independent RPKI sources compared" },
|
||||
ix_presence: { source: "PeeringDB netixlan (local mirror)", validation: "cross-validated", confidence: "high", note: "PeeringDB mirror refreshed daily" },
|
||||
facilities: { source: "PeeringDB netfac (local mirror)", validation: "single-source", confidence: "medium" },
|
||||
bgp_he_net: { source: "bgp.he.net HTML scrape", validation: "single-source", confidence: "medium", note: "HTML scrape, no official API — may have parsing drift" },
|
||||
atlas: { source: "RIPE Atlas API", validation: "single-source", confidence: "medium", note: "Probe availability varies by region" },
|
||||
resilience_score: { source: "Computed from RIPE Stat + PeeringDB", validation: "computed", confidence: "high", note: "All inputs cross-validated daily" },
|
||||
route_leak: { source: "RIPE Stat asn-neighbours heuristic", validation: "heuristic", confidence: "medium", note: "Pattern-based, not real-time — false positives possible" },
|
||||
registration: { source: "RDAP (RIR registry)", validation: "single-source", confidence: "high" },
|
||||
contacts: { source: "PeeringDB POC API", validation: "single-source", confidence: "medium", note: "Subject to PeeringDB rate limiting" },
|
||||
},
|
||||
};
|
||||
|
||||
// Update duration to include cross-check time
|
||||
result.meta.duration_ms = Date.now() - start;
|
||||
|
||||
// A DB hiccup shouldn't get cached as this ASN's answer for the full 5min --
|
||||
// retry soon instead of repeating a degraded result to every visitor.
|
||||
cacheSet(cacheKey, result, degradedSources.length > 0 ? CACHE_TTL_LOOKUP_DEGRADED : CACHE_TTL_LOOKUP);
|
||||
res.end(JSON.stringify(result, null, 2));
|
||||
} catch (err) {
|
||||
const duration = Date.now() - start;
|
||||
res.writeHead(500);
|
||||
res.end(JSON.stringify({ error: "Lookup failed", message: err.message, duration_ms: duration }));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
module.exports = { handleLookup };
|
||||
51
server/routes/lookup/rir-country.js
Normal file
51
server/routes/lookup/rir-country.js
Normal file
@ -0,0 +1,51 @@
|
||||
const { ARIN_CC, APNIC_CC, LACNIC_CC, AFRINIC_CC } = require("../../data/rir-country-codes");
|
||||
|
||||
// Derive RIR and country through a cascade of fallback sources: RIPE Stat
|
||||
// rir-stats-country, RDAP port43, RDAP self-link, bgp.he.net country_code,
|
||||
// then a country-code-to-RIR table as a last resort.
|
||||
function deriveRirAndCountry(rirEntries, rirData, rdapData, bgpHeData) {
|
||||
let rir = "";
|
||||
let country = "";
|
||||
// RIPE Stat rir-stats-country uses 'location' field (not 'country' or 'rir')
|
||||
if (Array.isArray(rirEntries) && rirEntries.length > 0) {
|
||||
country = rirEntries[0]?.location || rirEntries[0]?.country || "";
|
||||
rir = rirEntries[0]?.rir || "";
|
||||
}
|
||||
if (!rir && rirData?.data) {
|
||||
const rirField = rirData.data.rirs || [];
|
||||
if (rirField.length > 0) rir = rirField[0]?.rir || "";
|
||||
}
|
||||
// Derive RIR from rdapData.port43 (e.g. "whois.ripe.net" → "RIPE")
|
||||
if (!rir && rdapData && rdapData.port43) {
|
||||
const p43 = (rdapData.port43 || "").toLowerCase();
|
||||
if (p43.includes("ripe")) rir = "RIPE";
|
||||
else if (p43.includes("arin")) rir = "ARIN";
|
||||
else if (p43.includes("apnic")) rir = "APNIC";
|
||||
else if (p43.includes("lacnic")) rir = "LACNIC";
|
||||
else if (p43.includes("afrinic")) rir = "AFRINIC";
|
||||
}
|
||||
// Also derive RIR from RDAP links (URL of the RDAP endpoint that responded)
|
||||
if (!rir && rdapData && rdapData.links) {
|
||||
const selfLink = (rdapData.links.find(l => l.rel === "self") || {}).href || "";
|
||||
if (selfLink.includes("ripe")) rir = "RIPE";
|
||||
else if (selfLink.includes("arin")) rir = "ARIN";
|
||||
else if (selfLink.includes("apnic")) rir = "APNIC";
|
||||
else if (selfLink.includes("lacnic")) rir = "LACNIC";
|
||||
else if (selfLink.includes("afrinic")) rir = "AFRINIC";
|
||||
}
|
||||
// bgp.he.net country_code fallback (for unannounced/reserve ASNs)
|
||||
if (!country && bgpHeData && bgpHeData.country_code) {
|
||||
country = bgpHeData.country_code;
|
||||
}
|
||||
// Last resort: derive RIR from country code (common assignments)
|
||||
if (!rir && country) {
|
||||
if (ARIN_CC.has(country)) rir = "ARIN";
|
||||
else if (APNIC_CC.has(country)) rir = "APNIC";
|
||||
else if (LACNIC_CC.has(country)) rir = "LACNIC";
|
||||
else if (AFRINIC_CC.has(country)) rir = "AFRINIC";
|
||||
else rir = "RIPE"; // Europe + rest = RIPE NCC
|
||||
}
|
||||
return { rir, country };
|
||||
}
|
||||
|
||||
module.exports = { deriveRirAndCountry };
|
||||
71
server/routes/lookup/routing-info.js
Normal file
71
server/routes/lookup/routing-info.js
Normal file
@ -0,0 +1,71 @@
|
||||
const { fetchBgproutesVisibility } = require("../../services/http-helpers");
|
||||
|
||||
// Compute routing visibility and prefix size distribution.
|
||||
async function computeRoutingInfo(prefixes, visibilityData, prefixSizeData) {
|
||||
const ipv4Prefixes = prefixes.filter(function(p) { return !p.prefix.includes(":"); });
|
||||
const ipv6Prefixes = prefixes.filter(function(p) { return p.prefix.includes(":"); });
|
||||
var ipv4VisAvg = 0, ipv6VisAvg = 0, totalRisPeersV4 = 0, totalRisPeersV6 = 0;
|
||||
|
||||
// Visibility API returns per-RIS-collector data
|
||||
// Each collector has ipv4_full_table_peer_count and ipv4_full_table_peers_not_seeing[]
|
||||
// Bug 3 fix: visibility API may timeout for large ASNs — handle gracefully
|
||||
var visibilities = (visibilityData && visibilityData.data && visibilityData.data.visibilities) || [];
|
||||
var v4Seeing = 0, v4Total = 0, v6Seeing = 0, v6Total = 0;
|
||||
var visTimedOut = !visibilityData || !visibilityData.data;
|
||||
visibilities.forEach(function(v) {
|
||||
if (!v || !v.probe) return;
|
||||
var v4PeerCount = v.ipv4_full_table_peer_count || 0;
|
||||
var v4NotSeeing = (v.ipv4_full_table_peers_not_seeing || []).length;
|
||||
var v6PeerCount = v.ipv6_full_table_peer_count || 0;
|
||||
var v6NotSeeing = (v.ipv6_full_table_peers_not_seeing || []).length;
|
||||
v4Total += v4PeerCount;
|
||||
v4Seeing += (v4PeerCount - v4NotSeeing);
|
||||
v6Total += v6PeerCount;
|
||||
v6Seeing += (v6PeerCount - v6NotSeeing);
|
||||
});
|
||||
if (v4Total > 0) ipv4VisAvg = Math.round((v4Seeing / v4Total) * 1000) / 10;
|
||||
if (v6Total > 0) ipv6VisAvg = Math.round((v6Seeing / v6Total) * 1000) / 10;
|
||||
// If visibility API timed out but we have prefixes, try bgproutes.io fallback
|
||||
if (visTimedOut && prefixes.length > 0) {
|
||||
var fallbackPrefix = prefixes.find(function(p) { return !p.prefix.includes(":"); });
|
||||
if (!fallbackPrefix) fallbackPrefix = prefixes[0];
|
||||
if (fallbackPrefix) {
|
||||
var bgprFallback = await fetchBgproutesVisibility(fallbackPrefix.prefix);
|
||||
if (bgprFallback && bgprFallback.vps_seeing > 0) {
|
||||
// Estimate visibility: % of VPs seeing the prefix (assume ~300 total RIS-equivalent VPs)
|
||||
var estimatedTotal = Math.max(bgprFallback.vps_seeing, 300);
|
||||
ipv4VisAvg = Math.round((bgprFallback.vps_seeing / estimatedTotal) * 1000) / 10;
|
||||
ipv6VisAvg = -1; // bgproutes fallback is per-prefix, not per-AF aggregate
|
||||
totalRisPeersV4 = bgprFallback.vps_seeing;
|
||||
console.log("[Visibility] RIPE Stat timed out, used bgproutes.io fallback for " + fallbackPrefix.prefix + ": " + bgprFallback.vps_seeing + " VPs seeing it");
|
||||
} else {
|
||||
ipv4VisAvg = -1;
|
||||
ipv6VisAvg = -1;
|
||||
console.log("[Visibility] RIPE Stat timed out and bgproutes.io fallback returned no data");
|
||||
}
|
||||
} else {
|
||||
ipv4VisAvg = -1;
|
||||
ipv6VisAvg = -1;
|
||||
}
|
||||
}
|
||||
totalRisPeersV4 = v4Total;
|
||||
totalRisPeersV6 = v6Total;
|
||||
|
||||
// Prefix size distribution: data.ipv4[] and data.ipv6[] arrays with {size, count}
|
||||
var psdData = (prefixSizeData && prefixSizeData.data) || {};
|
||||
var psV4 = (psdData.ipv4 || []).map(function(e) { return { size: e.size, count: e.count }; }).sort(function(a,b){ return a.size - b.size; });
|
||||
var psV6 = (psdData.ipv6 || []).map(function(e) { return { size: e.size, count: e.count }; }).sort(function(a,b){ return a.size - b.size; });
|
||||
|
||||
return {
|
||||
ipv4_prefixes: ipv4Prefixes.length,
|
||||
ipv6_prefixes: ipv6Prefixes.length,
|
||||
ipv4_visibility_avg: ipv4VisAvg,
|
||||
ipv6_visibility_avg: ipv6VisAvg,
|
||||
total_ris_peers_v4: totalRisPeersV4,
|
||||
total_ris_peers_v6: totalRisPeersV6,
|
||||
prefix_sizes_v4: psV4,
|
||||
prefix_sizes_v6: psV6,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { computeRoutingInfo };
|
||||
Loading…
x
Reference in New Issue
Block a user