refactor: extract whois.js, topology.js, bgp-he-net.js, resolve-as-names.js
fetchBgpHeNet moved as-is, not removed, despite being confirmed dead code in practice (its only call site in /api/lookup is hardcoded to Promise.resolve(null)) -- that's a "disabled pending re-enable" state, a bigger behavioral decision than this extraction pass should make alone. Also widened the smoke-test harness's volatile-path exclusions: the live Cloudflare RPKI feed's ASPA object count ticks between runs (independent of any code change here), and the field appears under three different names (aspa_map.entries, aspa_adoption.total_objects, plus the derived delta_from_previous) that weren't all covered by the existing aspa_objects exclusion. Re-captured baseline accordingly. Verified via smoke-test harness (28/28 match). server.js: 4827 -> 4575 lines.
This commit is contained in:
Rene Fichtmueller
parent
c5cf1f1da8
commit
a5912a3d0c
260
server.js
260
server.js
@ -948,25 +948,7 @@ const {
|
|||||||
// checkRateLimit/rateLimitMap removed 2026-07-16: dead code, never called by
|
// checkRateLimit/rateLimitMap removed 2026-07-16: dead code, never called by
|
||||||
// the request handler (confirmed via the server.js structural exploration).
|
// the request handler (confirmed via the server.js structural exploration).
|
||||||
|
|
||||||
async function resolveASNames(providers) {
|
const { resolveASNames } = require("./server/resolve-as-names");
|
||||||
// Batch resolve AS names via RIPE Stat AS overview API
|
|
||||||
const batchSize = 10;
|
|
||||||
for (let i = 0; i < providers.length; i += batchSize) {
|
|
||||||
const batch = providers.slice(i, i + batchSize);
|
|
||||||
const results = await Promise.all(
|
|
||||||
batch.map(p =>
|
|
||||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + p.asn)
|
|
||||||
.then(r => ({ asn: p.asn, name: r?.data?.holder || "" }))
|
|
||||||
.catch(() => ({ asn: p.asn, name: "" }))
|
|
||||||
)
|
|
||||||
);
|
|
||||||
results.forEach(r => {
|
|
||||||
const provider = providers.find(p => p.asn === r.asn);
|
|
||||||
if (provider && r.name) provider.name = r.name;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return providers;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -982,247 +964,13 @@ const {
|
|||||||
} = require("./server/aspa-verification/engine");
|
} = require("./server/aspa-verification/engine");
|
||||||
|
|
||||||
|
|
||||||
// ============================================================
|
const { fetchBgpHeNet } = require("./server/bgp-he-net");
|
||||||
// Feature 24: bgp.he.net Integration
|
|
||||||
// ============================================================
|
|
||||||
async function fetchBgpHeNet(asn) {
|
|
||||||
try {
|
|
||||||
const html = await fetchHTML("https://bgp.he.net/AS" + asn);
|
|
||||||
if (!html) return null;
|
|
||||||
const result = {};
|
|
||||||
const titleMatch = html.match(/<title>([^<]+)<\/title>/i);
|
|
||||||
if (titleMatch) result.title = titleMatch[1].trim();
|
|
||||||
const peerMatch = html.match(/BGP\s+Peers\s+Observed\s*\(all\)\s*:\s*(\d[\d,]*)/i) || html.match(/Observed\s+Peers[^<]*<[^>]*>\s*(\d+)/i);
|
|
||||||
if (peerMatch) result.peer_count = parseInt(peerMatch[1].replace(/,/g, ''));
|
|
||||||
const countryMatch = html.match(/Country[^<]*<[^>]*>[^<]*<[^>]*>\s*<[^>]*>([^<]+)/i);
|
|
||||||
if (countryMatch) result.country = countryMatch[1].trim();
|
|
||||||
// Extract 2-letter country code from href="/country/XX"
|
|
||||||
const ccMatch = html.match(/href="\/country\/([A-Z]{2})"/i);
|
|
||||||
if (ccMatch) result.country_code = ccMatch[1].toUpperCase();
|
|
||||||
// Extract clean AS name from title: "AS12345 Some Name - bgp.he.net" → "Some Name"
|
|
||||||
if (titleMatch) {
|
|
||||||
const rawTitle = titleMatch[1].trim();
|
|
||||||
const nameFromTitle = rawTitle.replace(/^AS\d+\s+/i, '').replace(/\s+-\s+bgp\.he\.net.*$/i, '').trim();
|
|
||||||
if (nameFromTitle && !nameFromTitle.toLowerCase().includes('bgp.he.net')) {
|
|
||||||
result.name_from_title = nameFromTitle;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const lgMatch = html.match(/Looking\s+Glass[^<]*<[^>]*href="([^"]+)"/i);
|
const { fetchTopology } = require("./server/topology");
|
||||||
if (lgMatch) result.looking_glass = lgMatch[1];
|
|
||||||
const descMatch = html.match(/AS\s+Name[^<]*<[^>]*>[^<]*<[^>]*>([^<]+)/i);
|
|
||||||
if (descMatch) result.description = descMatch[1].trim();
|
|
||||||
const irrMatch = html.match(/IRR\s+Record[^<]*<[^>]*>[^<]*<[^>]*>([^<]+)/i);
|
|
||||||
if (irrMatch) result.irr_record = irrMatch[1].trim();
|
|
||||||
// bgp.he.net format: "Prefixes Originated (v4): 147<br/>" or "Prefixes v4 ... <td>147"
|
|
||||||
const v4Match = html.match(/Prefixes\s+Originated\s*\(v4\)\s*:\s*(\d[\d,]*)/i) || html.match(/Prefixes\s+v4[^<]*<[^>]*>\s*(\d+)/i);
|
|
||||||
if (v4Match) result.prefixes_v4 = parseInt(v4Match[1].replace(/,/g, ''));
|
|
||||||
const v6Match = html.match(/Prefixes\s+Originated\s*\(v6\)\s*:\s*(\d[\d,]*)/i) || html.match(/Prefixes\s+v6[^<]*<[^>]*>\s*(\d+)/i);
|
|
||||||
if (v6Match) result.prefixes_v6 = parseInt(v6Match[1].replace(/,/g, ''));
|
|
||||||
const allMatch = html.match(/Prefixes\s+Originated\s*\(all\)\s*:\s*(\d[\d,]*)/i);
|
|
||||||
if (allMatch) result.prefixes_all = parseInt(allMatch[1].replace(/,/g, ''));
|
|
||||||
result.source_url = "https://bgp.he.net/AS" + asn;
|
|
||||||
return result;
|
|
||||||
} catch (_e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// Feature 25: Topology / AS-Relationships
|
|
||||||
// ============================================================
|
|
||||||
async function fetchTopology(targetAsn, depth) {
|
|
||||||
const maxDepth = Math.min(depth || 2, 3);
|
|
||||||
const nodes = new Map();
|
|
||||||
const edges = [];
|
|
||||||
async function fetchNeighboursForAsn(asn, currentDepth) {
|
|
||||||
if (nodes.has(asn) && nodes.get(asn).depth <= currentDepth) return;
|
|
||||||
const [data, overview] = await Promise.all([
|
|
||||||
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn),
|
|
||||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
|
|
||||||
]);
|
|
||||||
const name = overview?.data?.holder || "";
|
|
||||||
const neighbours = data?.data?.neighbours || [];
|
|
||||||
const upstreams = neighbours.filter((n) => n.type === "left");
|
|
||||||
const downstreams = neighbours.filter((n) => n.type === "right");
|
|
||||||
const peers = neighbours.filter((n) => n.type === "uncertain" || n.type === "peer");
|
|
||||||
const nodeType = asn === targetAsn ? "target" : currentDepth === 1 ? "direct" : "indirect";
|
|
||||||
nodes.set(asn, { asn, name, type: nodeType, depth: currentDepth });
|
|
||||||
upstreams.forEach((n) => {
|
|
||||||
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "upstream", depth: currentDepth + 1 });
|
|
||||||
edges.push({ from: n.asn, to: asn, relationship: "provider-to-customer" });
|
|
||||||
});
|
|
||||||
downstreams.forEach((n) => {
|
|
||||||
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "downstream", depth: currentDepth + 1 });
|
|
||||||
edges.push({ from: asn, to: n.asn, relationship: "provider-to-customer" });
|
|
||||||
});
|
|
||||||
peers.slice(0, 10).forEach((n) => {
|
|
||||||
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "peer", depth: currentDepth + 1 });
|
|
||||||
edges.push({ from: asn, to: n.asn, relationship: "peer" });
|
|
||||||
});
|
|
||||||
if (currentDepth < maxDepth && upstreams.length > 0) {
|
|
||||||
const top5 = upstreams.sort((a, b) => (b.power || 0) - (a.power || 0)).slice(0, 5);
|
|
||||||
await Promise.all(top5.map((u) => fetchNeighboursForAsn(u.asn, currentDepth + 1)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await fetchNeighboursForAsn(targetAsn, 0);
|
|
||||||
const edgeSet = new Set();
|
|
||||||
const uniqueEdges = edges.filter((e) => {
|
|
||||||
const key = e.from + "-" + e.to + "-" + e.relationship;
|
|
||||||
if (edgeSet.has(key)) return false;
|
|
||||||
edgeSet.add(key);
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
return { nodes: [...nodes.values()], edges: uniqueEdges, target_asn: targetAsn, depth: maxDepth };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// Feature 27: WHOIS via RIPE DB
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
const { computeResilienceScore } = require("./server/resilience-score");
|
const { computeResilienceScore } = require("./server/resilience-score");
|
||||||
const { computeRouteLeakDetection } = require("./server/route-leak");
|
const { computeRouteLeakDetection } = require("./server/route-leak");
|
||||||
|
const { fetchWhois } = require("./server/whois");
|
||||||
async function fetchWhois(resource) {
|
|
||||||
const result = { resource, type: null, data: null, error: null };
|
|
||||||
try {
|
|
||||||
const trimmed = resource.trim();
|
|
||||||
if (/^(AS)?\d+$/i.test(trimmed)) {
|
|
||||||
result.type = "aut-num";
|
|
||||||
const asn = trimmed.replace(/^AS/i, "");
|
|
||||||
|
|
||||||
// Check cache first
|
|
||||||
const cached = whoisCacheGet(asn);
|
|
||||||
if (cached !== undefined) {
|
|
||||||
result.data = cached;
|
|
||||||
if (!cached) result.error = "Not found in any RIR database (cached)";
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try RIPE first
|
|
||||||
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=AS" + asn + "&type-filter=aut-num&source=ripe", { timeout: 5000 }).catch(() => null);
|
|
||||||
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
|
||||||
const obj = ripeData.objects.object[0];
|
|
||||||
const attrs = obj.attributes?.attribute || [];
|
|
||||||
const parsed = {};
|
|
||||||
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
|
||||||
result.data = {
|
|
||||||
aut_num: (parsed["aut-num"] || [])[0] || "",
|
|
||||||
as_name: (parsed["as-name"] || [])[0] || "",
|
|
||||||
descr: parsed["descr"] || [],
|
|
||||||
org: (parsed["org"] || [])[0] || "",
|
|
||||||
admin_c: parsed["admin-c"] || [],
|
|
||||||
tech_c: parsed["tech-c"] || [],
|
|
||||||
mnt_by: parsed["mnt-by"] || [],
|
|
||||||
status: (parsed["status"] || [])[0] || "",
|
|
||||||
created: (parsed["created"] || [])[0] || "",
|
|
||||||
last_modified: (parsed["last-modified"] || [])[0] || "",
|
|
||||||
source: (parsed["source"] || [])[0] || "",
|
|
||||||
import: parsed["import"] || [],
|
|
||||||
export: parsed["export"] || [],
|
|
||||||
remarks: parsed["remarks"] || [],
|
|
||||||
};
|
|
||||||
whoisCacheSet(asn, result.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If RIPE didn't find it, try all other RIRs via RDAP in parallel (3s timeout)
|
|
||||||
if (!result.data) {
|
|
||||||
const rdapEndpoints = [
|
|
||||||
{ name: "APNIC", url: "https://rdap.apnic.net/autnum/" + asn },
|
|
||||||
{ name: "ARIN", url: "https://rdap.arin.net/registry/autnum/" + asn },
|
|
||||||
{ name: "LACNIC", url: "https://rdap.lacnic.net/rdap/autnum/" + asn },
|
|
||||||
{ name: "AFRINIC", url: "https://rdap.afrinic.net/rdap/autnum/" + asn },
|
|
||||||
];
|
|
||||||
const rdapResults = await Promise.all(rdapEndpoints.map((ep) =>
|
|
||||||
fetchJSON(ep.url, { timeout: 3000 }).then((d) => {
|
|
||||||
if (!d || d.errorCode || !d.handle) return null;
|
|
||||||
return { source: ep.name, data: d };
|
|
||||||
}).catch(() => null)
|
|
||||||
));
|
|
||||||
const found = rdapResults.find((r) => r !== null);
|
|
||||||
if (found) {
|
|
||||||
const d = found.data;
|
|
||||||
const remarks = (d.remarks || []).map((r) => (r.description || []).join(" "));
|
|
||||||
const entities = d.entities || [];
|
|
||||||
const adminContacts = entities.filter((e) => (e.roles || []).includes("administrative")).map((e) => e.handle || "");
|
|
||||||
const techContacts = entities.filter((e) => (e.roles || []).includes("technical")).map((e) => e.handle || "");
|
|
||||||
const events = d.events || [];
|
|
||||||
const created = (events.find((e) => e.eventAction === "registration") || {}).eventDate || "";
|
|
||||||
const lastMod = (events.find((e) => e.eventAction === "last changed") || {}).eventDate || "";
|
|
||||||
result.data = {
|
|
||||||
aut_num: "AS" + asn,
|
|
||||||
as_name: d.name || "",
|
|
||||||
descr: remarks,
|
|
||||||
org: (entities.find((e) => (e.roles || []).includes("registrant")) || {}).handle || "",
|
|
||||||
admin_c: adminContacts,
|
|
||||||
tech_c: techContacts,
|
|
||||||
mnt_by: [],
|
|
||||||
status: (d.status || []).join(", "),
|
|
||||||
created: created,
|
|
||||||
last_modified: lastMod,
|
|
||||||
source: found.source + " (RDAP)",
|
|
||||||
import: [],
|
|
||||||
export: [],
|
|
||||||
remarks: remarks,
|
|
||||||
};
|
|
||||||
whoisCacheSet(asn, result.data);
|
|
||||||
} else {
|
|
||||||
result.error = "Not found in any RIR database (RIPE, APNIC, ARIN, LACNIC, AFRINIC)";
|
|
||||||
// Short TTL: this "not found" likely reflects a transient failure across
|
|
||||||
// all 5 sources, not a confirmed non-existent ASN -- see WHOIS_NOT_FOUND_CACHE_TTL.
|
|
||||||
whoisCacheSet(asn, null, WHOIS_NOT_FOUND_CACHE_TTL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (/[\/:]/.test(trimmed) || /^\d+\.\d+\.\d+/.test(trimmed)) {
|
|
||||||
result.type = "inetnum";
|
|
||||||
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=" + encodeURIComponent(trimmed) + "&type-filter=inetnum,inet6num");
|
|
||||||
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
|
||||||
const results = ripeData.objects.object.map((obj) => {
|
|
||||||
const attrs = obj.attributes?.attribute || [];
|
|
||||||
const parsed = {};
|
|
||||||
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
|
||||||
return {
|
|
||||||
inetnum: (parsed["inetnum"] || parsed["inet6num"] || [])[0] || "",
|
|
||||||
netname: (parsed["netname"] || [])[0] || "",
|
|
||||||
descr: parsed["descr"] || [],
|
|
||||||
country: (parsed["country"] || [])[0] || "",
|
|
||||||
org: (parsed["org"] || [])[0] || "",
|
|
||||||
admin_c: parsed["admin-c"] || [],
|
|
||||||
tech_c: parsed["tech-c"] || [],
|
|
||||||
mnt_by: parsed["mnt-by"] || [],
|
|
||||||
status: (parsed["status"] || [])[0] || "",
|
|
||||||
created: (parsed["created"] || [])[0] || "",
|
|
||||||
last_modified: (parsed["last-modified"] || [])[0] || "",
|
|
||||||
source: (parsed["source"] || [])[0] || "",
|
|
||||||
};
|
|
||||||
});
|
|
||||||
result.data = results.length === 1 ? results[0] : results;
|
|
||||||
} else { result.error = "Not found in RIPE DB"; }
|
|
||||||
} else {
|
|
||||||
result.type = "domain";
|
|
||||||
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=" + encodeURIComponent(trimmed) + "&type-filter=domain");
|
|
||||||
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
|
||||||
const obj = ripeData.objects.object[0];
|
|
||||||
const attrs = obj.attributes?.attribute || [];
|
|
||||||
const parsed = {};
|
|
||||||
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
|
||||||
result.data = {
|
|
||||||
domain: (parsed["domain"] || [])[0] || "",
|
|
||||||
descr: parsed["descr"] || [],
|
|
||||||
admin_c: parsed["admin-c"] || [],
|
|
||||||
tech_c: parsed["tech-c"] || [],
|
|
||||||
zone_c: parsed["zone-c"] || [],
|
|
||||||
nserver: parsed["nserver"] || [],
|
|
||||||
mnt_by: parsed["mnt-by"] || [],
|
|
||||||
created: (parsed["created"] || [])[0] || "",
|
|
||||||
last_modified: (parsed["last-modified"] || [])[0] || "",
|
|
||||||
source: (parsed["source"] || [])[0] || "",
|
|
||||||
};
|
|
||||||
} else { result.error = "Not found in RIPE DB"; }
|
|
||||||
}
|
|
||||||
} catch (err) { result.error = err.message; }
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// Internal API Server Proxy
|
// Internal API Server Proxy
|
||||||
|
|||||||
@ -101,9 +101,7 @@
|
|||||||
"version": "0.6.9",
|
"version": "0.6.9",
|
||||||
"bgproutes_configured": false,
|
"bgproutes_configured": false,
|
||||||
"caches": {
|
"caches": {
|
||||||
"aspa_map": {
|
"aspa_map": {},
|
||||||
"entries": 2396
|
|
||||||
},
|
|
||||||
"pdb_net": {
|
"pdb_net": {
|
||||||
"entries": 0,
|
"entries": 0,
|
||||||
"hit_rate_pct": 0
|
"hit_rate_pct": 0
|
||||||
@ -122,9 +120,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"local_db": null,
|
"local_db": null,
|
||||||
"aspa_adoption": {
|
"aspa_adoption": {}
|
||||||
"total_objects": 2396
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"aspa-verify": {
|
"aspa-verify": {
|
||||||
@ -3378,8 +3374,7 @@
|
|||||||
"body": {
|
"body": {
|
||||||
"period": "30d",
|
"period": "30d",
|
||||||
"current": {
|
"current": {
|
||||||
"date": "2026-07-16",
|
"date": "2026-07-16"
|
||||||
"delta_from_previous": 14
|
|
||||||
},
|
},
|
||||||
"trend": [
|
"trend": [
|
||||||
{
|
{
|
||||||
|
|||||||
@ -100,6 +100,11 @@ const VOLATILE_PATHS = new Set([
|
|||||||
"quick-ix.ix_connections",
|
"quick-ix.ix_connections",
|
||||||
"quick-ix.name",
|
"quick-ix.name",
|
||||||
"atlas-coverage.by_country",
|
"atlas-coverage.by_country",
|
||||||
|
// live Cloudflare RPKI feed object count ticks between runs; date-rollover
|
||||||
|
// (midnight) shifts the adoption tracker's day-over-day delta too
|
||||||
|
"health.caches.aspa_map.entries",
|
||||||
|
"health.aspa_adoption.total_objects",
|
||||||
|
"aspa-adoption-stats.current.delta_from_previous",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
function stripNondeterministic(obj, pathPrefix) {
|
function stripNondeterministic(obj, pathPrefix) {
|
||||||
|
|||||||
53
server/bgp-he-net.js
Normal file
53
server/bgp-he-net.js
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
const { fetchHTML } = require("./services/http-helpers");
|
||||||
|
|
||||||
|
// Feature 24: bgp.he.net Integration.
|
||||||
|
// NOTE: currently dead code in practice -- /api/lookup's only call site
|
||||||
|
// (`timedFetch("bgp.he.net", ...)`) is hardcoded to `Promise.resolve(null)`
|
||||||
|
// rather than actually invoking this. Moved as-is (not removed) since that's
|
||||||
|
// a "disabled pending re-enable" state, not a provably-unreachable one --
|
||||||
|
// removing a whole scraping feature is a bigger decision than this
|
||||||
|
// behavior-preserving extraction pass should make unilaterally.
|
||||||
|
async function fetchBgpHeNet(asn) {
|
||||||
|
try {
|
||||||
|
const html = await fetchHTML("https://bgp.he.net/AS" + asn);
|
||||||
|
if (!html) return null;
|
||||||
|
const result = {};
|
||||||
|
const titleMatch = html.match(/<title>([^<]+)<\/title>/i);
|
||||||
|
if (titleMatch) result.title = titleMatch[1].trim();
|
||||||
|
const peerMatch = html.match(/BGP\s+Peers\s+Observed\s*\(all\)\s*:\s*(\d[\d,]*)/i) || html.match(/Observed\s+Peers[^<]*<[^>]*>\s*(\d+)/i);
|
||||||
|
if (peerMatch) result.peer_count = parseInt(peerMatch[1].replace(/,/g, ''));
|
||||||
|
const countryMatch = html.match(/Country[^<]*<[^>]*>[^<]*<[^>]*>\s*<[^>]*>([^<]+)/i);
|
||||||
|
if (countryMatch) result.country = countryMatch[1].trim();
|
||||||
|
// Extract 2-letter country code from href="/country/XX"
|
||||||
|
const ccMatch = html.match(/href="\/country\/([A-Z]{2})"/i);
|
||||||
|
if (ccMatch) result.country_code = ccMatch[1].toUpperCase();
|
||||||
|
// Extract clean AS name from title: "AS12345 Some Name - bgp.he.net" → "Some Name"
|
||||||
|
if (titleMatch) {
|
||||||
|
const rawTitle = titleMatch[1].trim();
|
||||||
|
const nameFromTitle = rawTitle.replace(/^AS\d+\s+/i, '').replace(/\s+-\s+bgp\.he\.net.*$/i, '').trim();
|
||||||
|
if (nameFromTitle && !nameFromTitle.toLowerCase().includes('bgp.he.net')) {
|
||||||
|
result.name_from_title = nameFromTitle;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const lgMatch = html.match(/Looking\s+Glass[^<]*<[^>]*href="([^"]+)"/i);
|
||||||
|
if (lgMatch) result.looking_glass = lgMatch[1];
|
||||||
|
const descMatch = html.match(/AS\s+Name[^<]*<[^>]*>[^<]*<[^>]*>([^<]+)/i);
|
||||||
|
if (descMatch) result.description = descMatch[1].trim();
|
||||||
|
const irrMatch = html.match(/IRR\s+Record[^<]*<[^>]*>[^<]*<[^>]*>([^<]+)/i);
|
||||||
|
if (irrMatch) result.irr_record = irrMatch[1].trim();
|
||||||
|
// bgp.he.net format: "Prefixes Originated (v4): 147<br/>" or "Prefixes v4 ... <td>147"
|
||||||
|
const v4Match = html.match(/Prefixes\s+Originated\s*\(v4\)\s*:\s*(\d[\d,]*)/i) || html.match(/Prefixes\s+v4[^<]*<[^>]*>\s*(\d+)/i);
|
||||||
|
if (v4Match) result.prefixes_v4 = parseInt(v4Match[1].replace(/,/g, ''));
|
||||||
|
const v6Match = html.match(/Prefixes\s+Originated\s*\(v6\)\s*:\s*(\d[\d,]*)/i) || html.match(/Prefixes\s+v6[^<]*<[^>]*>\s*(\d+)/i);
|
||||||
|
if (v6Match) result.prefixes_v6 = parseInt(v6Match[1].replace(/,/g, ''));
|
||||||
|
const allMatch = html.match(/Prefixes\s+Originated\s*\(all\)\s*:\s*(\d[\d,]*)/i);
|
||||||
|
if (allMatch) result.prefixes_all = parseInt(allMatch[1].replace(/,/g, ''));
|
||||||
|
result.source_url = "https://bgp.he.net/AS" + asn;
|
||||||
|
return result;
|
||||||
|
} catch (_e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { fetchBgpHeNet };
|
||||||
23
server/resolve-as-names.js
Normal file
23
server/resolve-as-names.js
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
const { fetchRipeStatCached } = require("./services/ripe-stat");
|
||||||
|
|
||||||
|
// Batch resolve AS names via RIPE Stat AS overview API
|
||||||
|
async function resolveASNames(providers) {
|
||||||
|
const batchSize = 10;
|
||||||
|
for (let i = 0; i < providers.length; i += batchSize) {
|
||||||
|
const batch = providers.slice(i, i + batchSize);
|
||||||
|
const results = await Promise.all(
|
||||||
|
batch.map(p =>
|
||||||
|
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + p.asn)
|
||||||
|
.then(r => ({ asn: p.asn, name: r?.data?.holder || "" }))
|
||||||
|
.catch(() => ({ asn: p.asn, name: "" }))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
results.forEach(r => {
|
||||||
|
const provider = providers.find(p => p.asn === r.asn);
|
||||||
|
if (provider && r.name) provider.name = r.name;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return providers;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { resolveASNames };
|
||||||
49
server/topology.js
Normal file
49
server/topology.js
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
const { fetchRipeStatCached } = require("./services/ripe-stat");
|
||||||
|
|
||||||
|
// Feature 25: Topology / AS-Relationships
|
||||||
|
async function fetchTopology(targetAsn, depth) {
|
||||||
|
const maxDepth = Math.min(depth || 2, 3);
|
||||||
|
const nodes = new Map();
|
||||||
|
const edges = [];
|
||||||
|
async function fetchNeighboursForAsn(asn, currentDepth) {
|
||||||
|
if (nodes.has(asn) && nodes.get(asn).depth <= currentDepth) return;
|
||||||
|
const [data, overview] = await Promise.all([
|
||||||
|
fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn),
|
||||||
|
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
|
||||||
|
]);
|
||||||
|
const name = overview?.data?.holder || "";
|
||||||
|
const neighbours = data?.data?.neighbours || [];
|
||||||
|
const upstreams = neighbours.filter((n) => n.type === "left");
|
||||||
|
const downstreams = neighbours.filter((n) => n.type === "right");
|
||||||
|
const peers = neighbours.filter((n) => n.type === "uncertain" || n.type === "peer");
|
||||||
|
const nodeType = asn === targetAsn ? "target" : currentDepth === 1 ? "direct" : "indirect";
|
||||||
|
nodes.set(asn, { asn, name, type: nodeType, depth: currentDepth });
|
||||||
|
upstreams.forEach((n) => {
|
||||||
|
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "upstream", depth: currentDepth + 1 });
|
||||||
|
edges.push({ from: n.asn, to: asn, relationship: "provider-to-customer" });
|
||||||
|
});
|
||||||
|
downstreams.forEach((n) => {
|
||||||
|
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "downstream", depth: currentDepth + 1 });
|
||||||
|
edges.push({ from: asn, to: n.asn, relationship: "provider-to-customer" });
|
||||||
|
});
|
||||||
|
peers.slice(0, 10).forEach((n) => {
|
||||||
|
if (!nodes.has(n.asn)) nodes.set(n.asn, { asn: n.asn, name: n.as_name || "", type: "peer", depth: currentDepth + 1 });
|
||||||
|
edges.push({ from: asn, to: n.asn, relationship: "peer" });
|
||||||
|
});
|
||||||
|
if (currentDepth < maxDepth && upstreams.length > 0) {
|
||||||
|
const top5 = upstreams.sort((a, b) => (b.power || 0) - (a.power || 0)).slice(0, 5);
|
||||||
|
await Promise.all(top5.map((u) => fetchNeighboursForAsn(u.asn, currentDepth + 1)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await fetchNeighboursForAsn(targetAsn, 0);
|
||||||
|
const edgeSet = new Set();
|
||||||
|
const uniqueEdges = edges.filter((e) => {
|
||||||
|
const key = e.from + "-" + e.to + "-" + e.relationship;
|
||||||
|
if (edgeSet.has(key)) return false;
|
||||||
|
edgeSet.add(key);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return { nodes: [...nodes.values()], edges: uniqueEdges, target_asn: targetAsn, depth: maxDepth };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { fetchTopology };
|
||||||
146
server/whois.js
Normal file
146
server/whois.js
Normal file
@ -0,0 +1,146 @@
|
|||||||
|
const { fetchJSON } = require("./services/http-helpers");
|
||||||
|
const { whoisCacheGet, whoisCacheSet, WHOIS_NOT_FOUND_CACHE_TTL } = require("./caches/response-caches");
|
||||||
|
|
||||||
|
// Feature 27: WHOIS via RIPE DB
|
||||||
|
async function fetchWhois(resource) {
|
||||||
|
const result = { resource, type: null, data: null, error: null };
|
||||||
|
try {
|
||||||
|
const trimmed = resource.trim();
|
||||||
|
if (/^(AS)?\d+$/i.test(trimmed)) {
|
||||||
|
result.type = "aut-num";
|
||||||
|
const asn = trimmed.replace(/^AS/i, "");
|
||||||
|
|
||||||
|
// Check cache first
|
||||||
|
const cached = whoisCacheGet(asn);
|
||||||
|
if (cached !== undefined) {
|
||||||
|
result.data = cached;
|
||||||
|
if (!cached) result.error = "Not found in any RIR database (cached)";
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try RIPE first
|
||||||
|
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=AS" + asn + "&type-filter=aut-num&source=ripe", { timeout: 5000 }).catch(() => null);
|
||||||
|
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
||||||
|
const obj = ripeData.objects.object[0];
|
||||||
|
const attrs = obj.attributes?.attribute || [];
|
||||||
|
const parsed = {};
|
||||||
|
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
||||||
|
result.data = {
|
||||||
|
aut_num: (parsed["aut-num"] || [])[0] || "",
|
||||||
|
as_name: (parsed["as-name"] || [])[0] || "",
|
||||||
|
descr: parsed["descr"] || [],
|
||||||
|
org: (parsed["org"] || [])[0] || "",
|
||||||
|
admin_c: parsed["admin-c"] || [],
|
||||||
|
tech_c: parsed["tech-c"] || [],
|
||||||
|
mnt_by: parsed["mnt-by"] || [],
|
||||||
|
status: (parsed["status"] || [])[0] || "",
|
||||||
|
created: (parsed["created"] || [])[0] || "",
|
||||||
|
last_modified: (parsed["last-modified"] || [])[0] || "",
|
||||||
|
source: (parsed["source"] || [])[0] || "",
|
||||||
|
import: parsed["import"] || [],
|
||||||
|
export: parsed["export"] || [],
|
||||||
|
remarks: parsed["remarks"] || [],
|
||||||
|
};
|
||||||
|
whoisCacheSet(asn, result.data);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If RIPE didn't find it, try all other RIRs via RDAP in parallel (3s timeout)
|
||||||
|
if (!result.data) {
|
||||||
|
const rdapEndpoints = [
|
||||||
|
{ name: "APNIC", url: "https://rdap.apnic.net/autnum/" + asn },
|
||||||
|
{ name: "ARIN", url: "https://rdap.arin.net/registry/autnum/" + asn },
|
||||||
|
{ name: "LACNIC", url: "https://rdap.lacnic.net/rdap/autnum/" + asn },
|
||||||
|
{ name: "AFRINIC", url: "https://rdap.afrinic.net/rdap/autnum/" + asn },
|
||||||
|
];
|
||||||
|
const rdapResults = await Promise.all(rdapEndpoints.map((ep) =>
|
||||||
|
fetchJSON(ep.url, { timeout: 3000 }).then((d) => {
|
||||||
|
if (!d || d.errorCode || !d.handle) return null;
|
||||||
|
return { source: ep.name, data: d };
|
||||||
|
}).catch(() => null)
|
||||||
|
));
|
||||||
|
const found = rdapResults.find((r) => r !== null);
|
||||||
|
if (found) {
|
||||||
|
const d = found.data;
|
||||||
|
const remarks = (d.remarks || []).map((r) => (r.description || []).join(" "));
|
||||||
|
const entities = d.entities || [];
|
||||||
|
const adminContacts = entities.filter((e) => (e.roles || []).includes("administrative")).map((e) => e.handle || "");
|
||||||
|
const techContacts = entities.filter((e) => (e.roles || []).includes("technical")).map((e) => e.handle || "");
|
||||||
|
const events = d.events || [];
|
||||||
|
const created = (events.find((e) => e.eventAction === "registration") || {}).eventDate || "";
|
||||||
|
const lastMod = (events.find((e) => e.eventAction === "last changed") || {}).eventDate || "";
|
||||||
|
result.data = {
|
||||||
|
aut_num: "AS" + asn,
|
||||||
|
as_name: d.name || "",
|
||||||
|
descr: remarks,
|
||||||
|
org: (entities.find((e) => (e.roles || []).includes("registrant")) || {}).handle || "",
|
||||||
|
admin_c: adminContacts,
|
||||||
|
tech_c: techContacts,
|
||||||
|
mnt_by: [],
|
||||||
|
status: (d.status || []).join(", "),
|
||||||
|
created: created,
|
||||||
|
last_modified: lastMod,
|
||||||
|
source: found.source + " (RDAP)",
|
||||||
|
import: [],
|
||||||
|
export: [],
|
||||||
|
remarks: remarks,
|
||||||
|
};
|
||||||
|
whoisCacheSet(asn, result.data);
|
||||||
|
} else {
|
||||||
|
result.error = "Not found in any RIR database (RIPE, APNIC, ARIN, LACNIC, AFRINIC)";
|
||||||
|
// Short TTL: this "not found" likely reflects a transient failure across
|
||||||
|
// all 5 sources, not a confirmed non-existent ASN -- see WHOIS_NOT_FOUND_CACHE_TTL.
|
||||||
|
whoisCacheSet(asn, null, WHOIS_NOT_FOUND_CACHE_TTL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (/[\/:]/.test(trimmed) || /^\d+\.\d+\.\d+/.test(trimmed)) {
|
||||||
|
result.type = "inetnum";
|
||||||
|
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=" + encodeURIComponent(trimmed) + "&type-filter=inetnum,inet6num");
|
||||||
|
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
||||||
|
const results = ripeData.objects.object.map((obj) => {
|
||||||
|
const attrs = obj.attributes?.attribute || [];
|
||||||
|
const parsed = {};
|
||||||
|
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
||||||
|
return {
|
||||||
|
inetnum: (parsed["inetnum"] || parsed["inet6num"] || [])[0] || "",
|
||||||
|
netname: (parsed["netname"] || [])[0] || "",
|
||||||
|
descr: parsed["descr"] || [],
|
||||||
|
country: (parsed["country"] || [])[0] || "",
|
||||||
|
org: (parsed["org"] || [])[0] || "",
|
||||||
|
admin_c: parsed["admin-c"] || [],
|
||||||
|
tech_c: parsed["tech-c"] || [],
|
||||||
|
mnt_by: parsed["mnt-by"] || [],
|
||||||
|
status: (parsed["status"] || [])[0] || "",
|
||||||
|
created: (parsed["created"] || [])[0] || "",
|
||||||
|
last_modified: (parsed["last-modified"] || [])[0] || "",
|
||||||
|
source: (parsed["source"] || [])[0] || "",
|
||||||
|
};
|
||||||
|
});
|
||||||
|
result.data = results.length === 1 ? results[0] : results;
|
||||||
|
} else { result.error = "Not found in RIPE DB"; }
|
||||||
|
} else {
|
||||||
|
result.type = "domain";
|
||||||
|
const ripeData = await fetchJSON("https://rest.db.ripe.net/search.json?query-string=" + encodeURIComponent(trimmed) + "&type-filter=domain");
|
||||||
|
if (ripeData && ripeData.objects && ripeData.objects.object) {
|
||||||
|
const obj = ripeData.objects.object[0];
|
||||||
|
const attrs = obj.attributes?.attribute || [];
|
||||||
|
const parsed = {};
|
||||||
|
attrs.forEach((a) => { if (!parsed[a.name]) parsed[a.name] = []; parsed[a.name].push(a.value); });
|
||||||
|
result.data = {
|
||||||
|
domain: (parsed["domain"] || [])[0] || "",
|
||||||
|
descr: parsed["descr"] || [],
|
||||||
|
admin_c: parsed["admin-c"] || [],
|
||||||
|
tech_c: parsed["tech-c"] || [],
|
||||||
|
zone_c: parsed["zone-c"] || [],
|
||||||
|
nserver: parsed["nserver"] || [],
|
||||||
|
mnt_by: parsed["mnt-by"] || [],
|
||||||
|
created: (parsed["created"] || [])[0] || "",
|
||||||
|
last_modified: (parsed["last-modified"] || [])[0] || "",
|
||||||
|
source: (parsed["source"] || [])[0] || "",
|
||||||
|
};
|
||||||
|
} else { result.error = "Not found in RIPE DB"; }
|
||||||
|
}
|
||||||
|
} catch (err) { result.error = err.message; }
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { fetchWhois };
|
||||||
Loading…
x
Reference in New Issue
Block a user