50dc4e6b6a
All checks were successful
build-verify / build-verify (push) Successful in 25s
/api/lookup's deriveRirAndCountry() can only populate country from two
inputs: rirEntries (RIPE Stat rir-stats-country) and bgpHeData.country_code
(bgp.he.net). Both of their fetches in the Phase 1 Promise.all are
hardcoded to Promise.resolve(null) (see the timedFetch("RIPE Stat RIR", ...)
and timedFetch("bgp.he.net", ...) lines), so country has been unpopulated
for every single lookup regardless of ASN. rir stayed populated separately
via the RDAP port43/links fallback, which masked the issue -- rir looked
fine, country silently didn't.
Confirmed via today's rotating_audit.py run: 100/100 sampled ASNs flagged
country_empty.
Fix: fall back to PeeringDB's org-level country (pdbOrgState, loaded at
startup from /api/org, already used successfully by lia.js's coverage
endpoint for the same purpose) when the RIR-derived country is empty.
Does not touch the two dead stubs -- those look like deliberate stand-ins
for sources that were never wired up, out of scope here.
Verified live post-deploy: AS680 DE, AS847 US, AS1251 BR now populated
(previously all three were empty). AS4628 still empty -- its PeeringDB org
itself has no country set, a genuine upstream data gap, not something this
fallback can fix. npm test: 215/215 passing.
344 lines
17 KiB
JavaScript
344 lines
17 KiB
JavaScript
const localDb = require("../../../local-db-client");
|
|
const { fetchJSON } = require("../../services/http-helpers");
|
|
const { fetchPeeringDB, fetchPeeringDBWithRetry } = require("../../services/peeringdb");
|
|
const { ensureAspaCache, validateRPKIWithCache } = require("../../services/rpki");
|
|
const { computeResilienceScore } = require("../../resilience-score");
|
|
const { computeRouteLeakDetection } = require("../../route-leak");
|
|
const { pdbSourceCache } = require("../../caches/pdb-source-cache");
|
|
const {
|
|
cacheGet, cacheSet, CACHE_TTL_LOOKUP, CACHE_TTL_LOOKUP_DEGRADED,
|
|
rdapCacheGet, rdapCacheSet,
|
|
} = require("../../caches/response-caches");
|
|
const { resolveFacilitiesAndIxLocations } = require("./facilities");
|
|
const { enrichNeighbours } = require("./enrich-neighbours");
|
|
const { deriveRirAndCountry } = require("./rir-country");
|
|
const { computeRoutingInfo } = require("./routing-info");
|
|
const { computeCrossChecks } = require("./cross-check");
|
|
const { pdbOrgState } = require("../../pdb-org-countries");
|
|
|
|
// Main lookup endpoint: /api/lookup?asn=X
|
|
async function handleLookup(req, res, url) {
|
|
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
|
|
if (!rawAsn) {
|
|
res.writeHead(400);
|
|
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
|
|
}
|
|
const asn = rawAsn;
|
|
const cacheKey = "lookup:" + asn;
|
|
const cached = cacheGet(cacheKey);
|
|
if (cached) {
|
|
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
|
|
return res.end(JSON.stringify(cached));
|
|
}
|
|
const start = Date.now();
|
|
|
|
try {
|
|
// Phase 0: Get PDB net first — check L2 cache, then API with retry
|
|
let pdbNet = pdbSourceCache.get("net", asn);
|
|
if (!pdbNet) {
|
|
pdbNet = await fetchPeeringDBWithRetry("/net?asn=" + asn);
|
|
if (pdbNet) pdbSourceCache.set("net", asn, pdbNet);
|
|
}
|
|
const net = pdbNet?.data?.[0] || {};
|
|
const netId = net.id;
|
|
|
|
// Phase 1: ALL calls in parallel — RIPE Stat (cached+throttled) + PDB IX/Fac (cached) + Atlas + bgp.he.net
|
|
const ixQuery = netId
|
|
? "/netixlan?net_id=" + netId + "&limit=1000"
|
|
: "/netixlan?asn=" + asn + "&limit=1000";
|
|
const ixCacheKey = netId ? String(netId) : "asn:" + asn;
|
|
|
|
// Check PDB source cache for IX/Fac data
|
|
let cachedIxlan = pdbSourceCache.get("netixlan", ixCacheKey);
|
|
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
|
|
|
|
// Per-source timing tracking — 9s hard cap per source to prevent long-tail blocking
|
|
const sourceTiming = {};
|
|
function timedFetch(name, promise) {
|
|
const ts = Date.now();
|
|
return Promise.race([
|
|
Promise.resolve(promise),
|
|
new Promise(function(r) { setTimeout(function() { r(null); }, 9000); }),
|
|
])
|
|
.then(function(r) { sourceTiming[name] = Date.now() - ts; return r; })
|
|
.catch(function() { sourceTiming[name] = null; return null; });
|
|
}
|
|
|
|
const pocQuery = netId ? "/poc?net_id=" + netId + "&limit=25" : null;
|
|
|
|
// RDAP: check module-level cache first, only hit RIR endpoints on cache miss
|
|
const rdapCached = rdapCacheGet(asn);
|
|
const rdapPromise = rdapCached !== undefined
|
|
? Promise.resolve(rdapCached)
|
|
: Promise.race([
|
|
...["https://rdap.db.ripe.net/autnum/"+asn, "https://rdap.arin.net/registry/autnum/"+asn,
|
|
"https://rdap.apnic.net/autnum/"+asn, "https://rdap.lacnic.net/rdap/autnum/"+asn,
|
|
"https://rdap.afrinic.net/rdap/autnum/"+asn].map(url =>
|
|
fetchJSON(url, { timeout: 4000 })
|
|
.then(d => (d && !d.errorCode && d.handle) ? d : new Promise(() => {}))
|
|
.catch(() => new Promise(() => {}))
|
|
),
|
|
new Promise(resolve => setTimeout(() => resolve(null), 5000)),
|
|
]).then(d => { rdapCacheSet(asn, d); return d; });
|
|
|
|
const promises = [
|
|
timedFetch("RIPE Stat Prefixes", localDb ? localDb.getRipeStatAnnouncedPrefixes(asn) : Promise.resolve(null)),
|
|
timedFetch("RIPE Stat Neighbours", localDb ? localDb.getRipeStatAsnNeighbours(asn) : Promise.resolve(null)),
|
|
timedFetch("RIPE Stat Overview", localDb ? localDb.getRipeStatAsOverview(asn) : Promise.resolve(null)),
|
|
timedFetch("RIPE Stat RIR", Promise.resolve(null)),
|
|
timedFetch("RIPE Atlas", Promise.resolve(null)),
|
|
timedFetch("bgp.he.net", Promise.resolve(null)),
|
|
timedFetch("RIPE Stat Visibility", localDb ? localDb.getRipeStatVisibility(asn) : Promise.resolve(null)),
|
|
timedFetch("RIPE Stat PrefixSize", localDb ? localDb.getRipeStatPrefixSizeDistribution(asn) : Promise.resolve(null)),
|
|
timedFetch("PeeringDB IXLan", cachedIxlan ? Promise.resolve(cachedIxlan) : fetchPeeringDBWithRetry(ixQuery)),
|
|
timedFetch("PeeringDB Facilities", cachedFac ? Promise.resolve(cachedFac) : (netId ? fetchPeeringDBWithRetry("/netfac?net_id=" + netId + "&limit=1000") : Promise.resolve(null))),
|
|
timedFetch("PeeringDB Contacts", pocQuery ? fetchPeeringDB(pocQuery).catch(() => null) : Promise.resolve(null)),
|
|
timedFetch("RDAP Registration", rdapPromise),
|
|
];
|
|
const [prefixData, neighbourData, overviewData, rirData, atlasProbeData, bgpHeData, visibilityData, prefixSizeData, ixlanData, facData, pocData, rdapData] = await Promise.all(promises);
|
|
|
|
// Store PDB results in L2 source cache for future lookups
|
|
if (!cachedIxlan && ixlanData) pdbSourceCache.set("netixlan", ixCacheKey, ixlanData);
|
|
if (!cachedFac && facData) pdbSourceCache.set("netfac", String(netId), facData);
|
|
|
|
// local-db-client's getRipeStat* functions return status:"error" (not "ok") when
|
|
// the local Postgres DB query itself failed -- track which sources actually
|
|
// failed so we don't cache a DB hiccup's fake-empty result as if it were a
|
|
// genuine "this ASN has 0 prefixes" answer for the full 5-minute TTL.
|
|
const degradedSources = [
|
|
["RIPE Stat Prefixes", prefixData], ["RIPE Stat Neighbours", neighbourData],
|
|
["RIPE Stat Overview", overviewData], ["RIPE Stat Visibility", visibilityData],
|
|
["RIPE Stat PrefixSize", prefixSizeData],
|
|
].filter(([, v]) => v && v.status === "error").map(([name]) => name);
|
|
|
|
const prefixes = prefixData?.data?.prefixes || [];
|
|
const neighbours = neighbourData?.data?.neighbours || [];
|
|
const overview = overviewData?.data || {};
|
|
const rirEntries = rirData?.data?.located_resources || rirData?.data?.rir_stats || [];
|
|
|
|
// Bug 6 fix: Atlas probe status uses status.name (object), not status_name (flat)
|
|
const atlasProbes = atlasProbeData?.results || [];
|
|
const atlasConnected = atlasProbes.filter(p => {
|
|
const sName = (p.status_name || (p.status && p.status.name) || "").toLowerCase();
|
|
return sName === "connected";
|
|
});
|
|
const atlasAnchors = atlasProbes.filter(p => p.is_anchor === true);
|
|
|
|
// RPKI: validate ALL prefixes using local Cloudflare RPKI data (all 5 RIRs, instant)
|
|
await ensureAspaCache();
|
|
const allPrefixes = prefixes.map((p) => p.prefix);
|
|
const rpkiAllResults = await Promise.all(allPrefixes.map((pfx) => validateRPKIWithCache(asn, pfx)));
|
|
|
|
const ixConnections = (ixlanData?.data || [])
|
|
.map((ix) => ({
|
|
ix_name: ix.name || "",
|
|
ix_id: ix.ix_id,
|
|
speed_mbps: ix.speed || 0,
|
|
ipv4: ix.ipaddr4 || null,
|
|
ipv6: ix.ipaddr6 || null,
|
|
city: ix.city || "",
|
|
is_rs_peer: ix.is_rs_peer === true,
|
|
}))
|
|
.sort((a, b) => b.speed_mbps - a.speed_mbps);
|
|
|
|
const facilitiesRaw = (facData?.data || []).map((f) => ({
|
|
fac_id: f.fac_id,
|
|
name: f.name || "",
|
|
city: f.city || "",
|
|
country: f.country || "",
|
|
}));
|
|
|
|
const { facilities, ixLocations } = await resolveFacilitiesAndIxLocations(facilitiesRaw, ixConnections);
|
|
|
|
const rpkiStatuses = rpkiAllResults;
|
|
const rpkiValid = rpkiStatuses.filter((r) => r.status === "valid").length;
|
|
const rpkiInvalid = rpkiStatuses.filter((r) => r.status === "invalid").length;
|
|
const rpkiUnavailable = rpkiStatuses.filter((r) => r.status === "unavailable").length;
|
|
const rpkiNotFound = rpkiStatuses.filter((r) => r.status !== "valid" && r.status !== "invalid" && r.status !== "unavailable").length;
|
|
// Exclude "unavailable" (DB error) from the coverage denominator -- a DB hiccup
|
|
// must not lower a network's displayed RPKI coverage percentage.
|
|
const rpkiTotal = rpkiStatuses.length - rpkiUnavailable;
|
|
const rpkiCoverage = rpkiTotal > 0 ? Math.round((rpkiValid / rpkiTotal) * 100) : 0;
|
|
|
|
let upstreams = neighbours
|
|
.filter((n) => n.type === "left")
|
|
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
|
.sort((a, b) => b.power - a.power);
|
|
let downstreams = neighbours
|
|
.filter((n) => n.type === "right")
|
|
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
|
.sort((a, b) => b.power - a.power);
|
|
let peers = neighbours
|
|
.filter((n) => n.type === "uncertain" || n.type === "peer")
|
|
.map((n) => ({ asn: n.asn, name: n.as_name || "", power: n.power || 0 }))
|
|
.sort((a, b) => b.power - a.power);
|
|
|
|
const enriched = await enrichNeighbours(upstreams, downstreams, peers);
|
|
upstreams = enriched.upstreams;
|
|
downstreams = enriched.downstreams;
|
|
peers = enriched.peers;
|
|
|
|
const { rir, country: derivedCountry } = deriveRirAndCountry(rirEntries, rirData, rdapData, bgpHeData);
|
|
// "RIPE Stat RIR" and "bgp.he.net" sources above are stubbed to always
|
|
// resolve null (see Phase 1 promises), so derivedCountry is currently
|
|
// never populated -- fall back to PeeringDB's org-level country map
|
|
// (pdbOrgState, loaded at startup), the same source lia.js already uses.
|
|
const country = derivedCountry || (pdbOrgState.map.get(net.org_id) || {}).country || "";
|
|
|
|
const duration = Date.now() - start;
|
|
|
|
const routingInfo = await computeRoutingInfo(prefixes, visibilityData, prefixSizeData);
|
|
|
|
const { rpkiCrossCheck, prefixCrossCheck, neighbourCrossCheck, dataQuality } =
|
|
await computeCrossChecks(asn, allPrefixes, rpkiStatuses, prefixes, neighbours, bgpHeData);
|
|
|
|
const result = {
|
|
meta: {
|
|
service: "PeerCortex",
|
|
version: "0.6.9",
|
|
query: "AS" + asn,
|
|
duration_ms: duration,
|
|
sources: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator", "Route Views"],
|
|
timestamp: new Date().toISOString(),
|
|
rpki_prefixes_checked: rpkiTotal,
|
|
total_prefixes: prefixes.length,
|
|
degraded_sources: degradedSources,
|
|
},
|
|
network: {
|
|
asn: parseInt(asn),
|
|
name: net.name || overview?.holder || (bgpHeData && bgpHeData.name_from_title) || "Unknown",
|
|
aka: net.aka || "",
|
|
org_name: (net.org && net.org.name) ? net.org.name : "",
|
|
website: net.website || "",
|
|
type: net.info_type || "",
|
|
policy: net.policy_general || "",
|
|
traffic: net.info_traffic || "",
|
|
ratio: net.info_ratio || "",
|
|
scope: net.info_scope || "",
|
|
notes: net.notes ? net.notes.substring(0, 500) : "",
|
|
peeringdb_id: netId || null,
|
|
rir: rir,
|
|
country: country,
|
|
city: net.city || "",
|
|
latitude: (net.latitude != null) ? net.latitude : null,
|
|
longitude: (net.longitude != null) ? net.longitude : null,
|
|
looking_glass: net.looking_glass || "",
|
|
route_server: net.route_server || "",
|
|
info_prefixes4: net.info_prefixes4 || 0,
|
|
info_prefixes6: net.info_prefixes6 || 0,
|
|
status: net.status || "",
|
|
peeringdb_created: net.created ? net.created.slice(0, 10) : "",
|
|
peeringdb_updated: net.updated ? net.updated.slice(0, 10) : "",
|
|
},
|
|
prefixes: {
|
|
total: prefixes.length,
|
|
ipv4: prefixes.filter((p) => !p.prefix.includes(":")).length,
|
|
ipv6: prefixes.filter((p) => p.prefix.includes(":")).length,
|
|
list: prefixes.map((p) => p.prefix),
|
|
cross_check: prefixCrossCheck,
|
|
},
|
|
rpki: {
|
|
coverage_percent: rpkiCoverage,
|
|
valid: rpkiValid,
|
|
invalid: rpkiInvalid,
|
|
not_found: rpkiNotFound,
|
|
unavailable: rpkiUnavailable,
|
|
checked: rpkiTotal,
|
|
details: rpkiStatuses,
|
|
cross_check: rpkiCrossCheck,
|
|
},
|
|
neighbours: {
|
|
total: neighbours.length,
|
|
upstream_count: upstreams.length,
|
|
downstream_count: downstreams.length,
|
|
peer_count: peers.length,
|
|
upstreams: upstreams.slice(0, 20),
|
|
downstreams: downstreams.slice(0, 20),
|
|
peers: peers.slice(0, 20),
|
|
cross_check: neighbourCrossCheck,
|
|
},
|
|
ix_presence: {
|
|
total_connections: ixConnections.length,
|
|
unique_ixps: [...new Set(ixConnections.map((ix) => ix.ix_id))].length,
|
|
connections: ixConnections,
|
|
},
|
|
ix_locations: ixLocations,
|
|
facilities: {
|
|
total: facilities.length,
|
|
list: facilities,
|
|
},
|
|
routing: routingInfo,
|
|
resilience_score: computeResilienceScore(upstreams, peers, ixConnections, prefixes),
|
|
route_leak: computeRouteLeakDetection(upstreams, downstreams, peers),
|
|
bgp_he_net: bgpHeData || null,
|
|
atlas: {
|
|
total_probes: atlasProbes.length,
|
|
connected: atlasConnected.length,
|
|
disconnected: atlasProbes.length - atlasConnected.length,
|
|
anchors: atlasAnchors.length,
|
|
probes: atlasProbes.slice(0, 100).map(p => ({
|
|
id: p.id,
|
|
status: p.status_name || p.status || "Unknown",
|
|
is_anchor: p.is_anchor || false,
|
|
country: p.country_code || "",
|
|
prefix_v4: p.prefix_v4 || "",
|
|
prefix_v6: p.prefix_v6 || "",
|
|
description: p.description || "",
|
|
})),
|
|
},
|
|
data_quality: dataQuality,
|
|
source_timing: sourceTiming,
|
|
contacts: (() => {
|
|
const pocs = (pocData && pocData.data) ? pocData.data : [];
|
|
return pocs.slice(0, 20).map(p => ({
|
|
role: p.role || "",
|
|
name: p.name || "",
|
|
email: p.email || "",
|
|
url: p.url || "",
|
|
visible: p.visible || "",
|
|
}));
|
|
})(),
|
|
registration: (() => {
|
|
const events = (rdapData && rdapData.events) ? rdapData.events : [];
|
|
const created = (events.find(e => e.eventAction === "registration") || {}).eventDate || "";
|
|
const lastChg = (events.find(e => e.eventAction === "last changed") || {}).eventDate || "";
|
|
return {
|
|
created: created ? created.slice(0, 10) : "",
|
|
last_modified: lastChg ? lastChg.slice(0, 10) : "",
|
|
rir: rir || "",
|
|
handle: (rdapData && rdapData.handle) ? rdapData.handle : ("AS" + asn),
|
|
rdap_source: (rdapData && rdapData.port43) ? rdapData.port43 : "",
|
|
};
|
|
})(),
|
|
_provenance: {
|
|
prefixes: { source: "RIPE Stat announced-prefixes", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net prefix count daily" },
|
|
neighbours: { source: "RIPE Stat asn-neighbours", validation: "cross-validated", confidence: "high", note: "Cross-checked with bgp.he.net peer count daily" },
|
|
rpki: { source: "Cloudflare RPKI + RIPE Validator", validation: "cross-validated", confidence: "high", note: "Two independent RPKI sources compared" },
|
|
ix_presence: { source: "PeeringDB netixlan (local mirror)", validation: "cross-validated", confidence: "high", note: "PeeringDB mirror refreshed daily" },
|
|
facilities: { source: "PeeringDB netfac (local mirror)", validation: "single-source", confidence: "medium" },
|
|
bgp_he_net: { source: "bgp.he.net HTML scrape", validation: "single-source", confidence: "medium", note: "HTML scrape, no official API — may have parsing drift" },
|
|
atlas: { source: "RIPE Atlas API", validation: "single-source", confidence: "medium", note: "Probe availability varies by region" },
|
|
resilience_score: { source: "Computed from RIPE Stat + PeeringDB", validation: "computed", confidence: "high", note: "All inputs cross-validated daily" },
|
|
route_leak: { source: "RIPE Stat asn-neighbours heuristic", validation: "heuristic", confidence: "medium", note: "Pattern-based, not real-time — false positives possible" },
|
|
registration: { source: "RDAP (RIR registry)", validation: "single-source", confidence: "high" },
|
|
contacts: { source: "PeeringDB POC API", validation: "single-source", confidence: "medium", note: "Subject to PeeringDB rate limiting" },
|
|
},
|
|
};
|
|
|
|
// Update duration to include cross-check time
|
|
result.meta.duration_ms = Date.now() - start;
|
|
|
|
// A DB hiccup shouldn't get cached as this ASN's answer for the full 5min --
|
|
// retry soon instead of repeating a degraded result to every visitor.
|
|
cacheSet(cacheKey, result, degradedSources.length > 0 ? CACHE_TTL_LOOKUP_DEGRADED : CACHE_TTL_LOOKUP);
|
|
res.end(JSON.stringify(result, null, 2));
|
|
} catch (err) {
|
|
const duration = Date.now() - start;
|
|
res.writeHead(500);
|
|
res.end(JSON.stringify({ error: "Lookup failed", message: err.message, duration_ms: duration }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
module.exports = { handleLookup };
|