From 49aa46fef3a4ce82e3ea88df8741e1442f6d2025 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Fri, 17 Jul 2026 07:57:04 +0200 Subject: [PATCH] refactor: extract relationships/compare/quick-ix/peers-find/prefix-detail/ix-detail/topology/whois/enrich routes Bug fix (agreed pre-existing bug #2 of 3): /api/compare called fetchRPKIPerPrefix(asn, prefix), never defined anywhere in the codebase -- threw ReferenceError whenever either ASN had >=1 sample prefix, caught by the outer try/catch, returning 500. Replaced both call sites with validateRPKIWithCache(asn, prefix), the existing single-prefix RPKI check used everywhere else in the codebase -- its return shape ({status, ...}) is exactly what the surrounding code already expects. Verified via node --check, eslint no-undef (only 1 known bug remains: webhooks POST's `secret`), and smoke-test harness (28/28 match). The compare fix itself isn't exercised by the smoke test's fixed test ASNs (both return empty prefix samples locally), but the fix is a direct, same-shape substitution of an already-proven function. server.js: 1736 -> 989 lines. --- server.js | 733 +-------------------------------- server/routes/compare.js | 175 ++++++++ server/routes/enrich.js | 125 ++++++ server/routes/ix-detail.js | 59 +++ server/routes/peers-find.js | 106 +++++ server/routes/prefix-detail.js | 77 ++++ server/routes/quick-ix.js | 48 +++ server/routes/relationships.js | 110 +++++ server/routes/topology.js | 59 +++ server/routes/whois.js | 26 ++ 10 files changed, 804 insertions(+), 714 deletions(-) create mode 100644 server/routes/compare.js create mode 100644 server/routes/enrich.js create mode 100644 server/routes/ix-detail.js create mode 100644 server/routes/peers-find.js create mode 100644 server/routes/prefix-detail.js create mode 100644 server/routes/quick-ix.js create mode 100644 server/routes/relationships.js create mode 100644 server/routes/topology.js create mode 100644 server/routes/whois.js diff --git a/server.js b/server.js index 0ef1365..cc3184b 100644 --- a/server.js +++ b/server.js @@ -227,6 +227,15 @@ const aspaLegacyRoute = require("./server/routes/aspa-legacy"); const bgpRoute = require("./server/routes/bgp"); const validateRoute = require("./server/routes/validate"); const lookupRoute = require("./server/routes/lookup"); +const relationshipsRoute = require("./server/routes/relationships"); +const compareRoute = require("./server/routes/compare"); +const quickIxRoute = require("./server/routes/quick-ix"); +const peersFindRoute = require("./server/routes/peers-find"); +const prefixDetailRoute = require("./server/routes/prefix-detail"); +const ixDetailRoute = require("./server/routes/ix-detail"); +const topologyRoute = require("./server/routes/topology"); +const whoisRoute = require("./server/routes/whois"); +const enrichRoute = require("./server/routes/enrich"); const server = http.createServer(async (req, res) => { res.setHeader("Access-Control-Allow-Origin", "*"); @@ -345,755 +354,51 @@ const server = http.createServer(async (req, res) => { return lookupRoute.handleLookup(req, res, url); } - // with resolved names. Based on RIPE Stat asn-neighbours. - // ============================================================ if (reqPath === "/api/relationships") { - const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); - if (!rawAsn) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" })); - } - const cacheKey = "relationships:" + rawAsn; - const cached = cacheGet(cacheKey); - if (cached) { - res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" }); - return res.end(JSON.stringify(cached)); - } - const start = Date.now(); - try { - const neighbourData = await fetchRipeStatCached( - "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1", - { timeout: 8000 } - ); - const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || []; - const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {}; - - const upstreams = neighbours.filter(n => n.type === "left").sort((a,b) => (b.power||0)-(a.power||0)); - const downstreams = neighbours.filter(n => n.type === "right").sort((a,b) => (b.power||0)-(a.power||0)); - const peers = neighbours.filter(n => n.type === "uncertain").sort((a,b) => (b.power||0)-(a.power||0)); - - // Resolve AS names for top entries (upstreams + downstreams all, top 20 peers) - const toResolve = [...upstreams, ...downstreams, ...peers.slice(0, 20)]; - const resolvedNames = {}; - await Promise.race([ - Promise.all(toResolve.map(n => - fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 }) - .then(r => { if (r && r.data && r.data.holder) resolvedNames[n.asn] = r.data.holder; }) - .catch(() => {}) - )), - new Promise(r => setTimeout(r, 5000)), - ]); - - // ---- Threat Intelligence Enrichment ---- - // Enrich neighbors with threat status from local threat_intel table - const threatMap = {}; - const allNeighborsForThreat = [...upstreams, ...downstreams, ...peers]; - const threatPromises = allNeighborsForThreat.slice(0, 100).map(async (n) => { - try { - const asNum = String(n.asn); - const threat = await localDb.getThreatIntel(asNum); - if (threat) { - threatMap[n.asn] = { - threat_level: threat.threat_level, - confidence_score: threat.confidence_score, - source: threat.source, - }; - } - } catch (e) { - console.error(`[Relationships Threat Lookup] Error checking ASN ${n.asn}:`, e.message); - } - }); - await Promise.race([ - Promise.all(threatPromises), - new Promise(r => setTimeout(r, 3000)), - ]); - - const fmt = n => ({ - asn: n.asn, - name: resolvedNames[n.asn] || "", - power: n.power || 0, - v4_peers: n.v4_peers || 0, - v6_peers: n.v6_peers || 0, - threat_level: threatMap[n.asn]?.threat_level || null, - threat_confidence: threatMap[n.asn]?.confidence_score || null, - threat_source: threatMap[n.asn]?.source || null, - }); - - const result = { - asn: parseInt(rawAsn), - query_time: new Date().toISOString(), - duration_ms: Date.now() - start, - counts: { - upstreams: counts.left || upstreams.length, - downstreams: counts.right || downstreams.length, - peers_total: counts.unique || peers.length, - uncertain: counts.uncertain || peers.length, - }, - upstreams: upstreams.map(fmt), - downstreams: downstreams.map(fmt), - peers: peers.slice(0, 50).map(fmt), - methodology: "RIPE Stat asn-neighbours API. left=upstream providers (carry your traffic), right=downstream customers (you carry their traffic), uncertain=lateral peers. Sorted by power score (number of prefixes seen via this relationship).", - source_url: "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, - }; - - // A neighbourData fetch failure (not a genuine zero-neighbour ASN) would - // otherwise get cached as "0 upstreams/downstreams/peers" for the full 10min -- - // match the existing /api/validate precedent (90s TTL for suspicious zero counts). - cacheSet(cacheKey, result, neighbourData ? 10 * 60 * 1000 : 90 * 1000); - res.writeHead(200, { "Content-Type": "application/json" }); - return res.end(JSON.stringify(result, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "Relationships lookup failed", message: err.message })); - } + return relationshipsRoute.handleRelationships(req, res, url); } - // ============================================================ // Compare endpoint: /api/compare?asn1=X&asn2=Y - // ============================================================ if (reqPath === "/api/compare") { - const asn1 = (url.searchParams.get("asn1") || "").replace(/[^0-9]/g, ""); - const asn2 = (url.searchParams.get("asn2") || "").replace(/[^0-9]/g, ""); - if (!asn1 || !asn2) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Need asn1 and asn2 parameters" })); - } - - const compareCacheKey = "compare:" + asn1 + ":" + asn2; - const compareCached = cacheGet(compareCacheKey); - if (compareCached) { - res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" }); - return res.end(JSON.stringify(compareCached)); - } - const start = Date.now(); - try { - // ALL calls in parallel — single batch - // Phase 1: Get PDB net objects + RIPE data - const [pdb1, pdb2, nb1Data, nb2Data, pfx1Data, pfx2Data] = await Promise.all([ - fetchPeeringDB("/net?asn=" + asn1), - fetchPeeringDB("/net?asn=" + asn2), - fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 8000 }), - fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 8000 }), - fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 8000 }), - fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 8000 }), - ]); - - const net1 = pdb1?.data?.[0] || {}; - const net2 = pdb2?.data?.[0] || {}; - const netId1 = net1.id; - const netId2 = net2.id; - - // Phase 2: IX + Facility using net_id (Bug 1 fix: netfac requires net_id, not asn) - const ixFacPromises = []; - ixFacPromises.push(netId1 ? fetchPeeringDB("/netixlan?net_id=" + netId1) : Promise.resolve(null)); - ixFacPromises.push(netId2 ? fetchPeeringDB("/netixlan?net_id=" + netId2) : Promise.resolve(null)); - ixFacPromises.push(netId1 ? fetchPeeringDB("/netfac?net_id=" + netId1) : Promise.resolve(null)); - ixFacPromises.push(netId2 ? fetchPeeringDB("/netfac?net_id=" + netId2) : Promise.resolve(null)); - const [ix1Data, ix2Data, fac1Data, fac2Data] = await Promise.all(ixFacPromises); - - const ix1Set = new Set((ix1Data?.data || []).map((ix) => ix.ix_id)); - const ix2Set = new Set((ix2Data?.data || []).map((ix) => ix.ix_id)); - const ix1Names = {}; - (ix1Data?.data || []).forEach((ix) => (ix1Names[ix.ix_id] = ix.name)); - const ix2Names = {}; - (ix2Data?.data || []).forEach((ix) => (ix2Names[ix.ix_id] = ix.name)); - - const commonIX = [...ix1Set].filter((id) => ix2Set.has(id)).map((id) => ({ ix_id: id, name: ix1Names[id] || ix2Names[id] || "" })); - const only1IX = [...ix1Set].filter((id) => !ix2Set.has(id)).map((id) => ({ ix_id: id, name: ix1Names[id] || "" })); - const only2IX = [...ix2Set].filter((id) => !ix1Set.has(id)).map((id) => ({ ix_id: id, name: ix2Names[id] || "" })); - - const fac1Set = new Set((fac1Data?.data || []).map((f) => f.fac_id)); - const fac2Set = new Set((fac2Data?.data || []).map((f) => f.fac_id)); - const fac1Names = {}; - (fac1Data?.data || []).forEach((f) => (fac1Names[f.fac_id] = f.name)); - const fac2Names = {}; - (fac2Data?.data || []).forEach((f) => (fac2Names[f.fac_id] = f.name)); - - const commonFac = [...fac1Set].filter((id) => fac2Set.has(id)).map((id) => ({ fac_id: id, name: fac1Names[id] || fac2Names[id] || "" })); - - const nb1 = (nb1Data?.data?.neighbours || []).filter((n) => n.type === "left"); - const nb2 = (nb2Data?.data?.neighbours || []).filter((n) => n.type === "left"); - const up1Set = new Set(nb1.map((n) => n.asn)); - const up2Set = new Set(nb2.map((n) => n.asn)); - const nb1Map = {}; - nb1.forEach((n) => (nb1Map[n.asn] = n.as_name || "")); - const nb2Map = {}; - nb2.forEach((n) => (nb2Map[n.asn] = n.as_name || "")); - - const commonUpstreams = [...up1Set] - .filter((a) => up2Set.has(a)) - .map((a) => ({ asn: a, name: nb1Map[a] || nb2Map[a] || "" })); - - // ---- Threat Intelligence Enrichment for Common Upstreams ---- - const threatMap = {}; - const threatPromises = commonUpstreams.slice(0, 50).map(async (n) => { - try { - const asNum = String(n.asn); - const threat = await localDb.getThreatIntel(asNum); - if (threat) { - threatMap[n.asn] = { - threat_level: threat.threat_level, - confidence_score: threat.confidence_score, - source: threat.source, - }; - } - } catch (e) { - console.error(`[Compare Threat Lookup] Error checking ASN ${n.asn}:`, e.message); - } - }); - await Promise.race([ - Promise.all(threatPromises), - new Promise(r => setTimeout(r, 2000)), - ]); - - // Attach threat status to upstream objects - commonUpstreams.forEach((n) => { - if (threatMap[n.asn]) { - n.threat_level = threatMap[n.asn].threat_level; - n.threat_confidence = threatMap[n.asn].confidence_score; - n.threat_source = threatMap[n.asn].source; - } - }); - - // Resolve names + RPKI sample (max 3+3 prefixes) all in parallel with 5s timeout - const pfx1 = (pfx1Data?.data?.prefixes || []).slice(0, 3).map((p) => p.prefix); - const pfx2 = (pfx2Data?.data?.prefixes || []).slice(0, 3).map((p) => p.prefix); - const [, rpki1Results, rpki2Results] = await Promise.race([ - Promise.all([ - commonUpstreams.length > 0 ? Promise.all(commonUpstreams.map(n => - fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 }) - .then(r => { if (r?.data?.holder) n.name = r.data.holder; }) - .catch(() => {}) - )) : Promise.resolve([]), - Promise.all(pfx1.map((p) => fetchRPKIPerPrefix(asn1, p))), - Promise.all(pfx2.map((p) => fetchRPKIPerPrefix(asn2, p))), - ]), - new Promise(r => setTimeout(() => r([[], [], []]), 5000)), - ]); - - const rpki1Valid = rpki1Results.filter((r) => r.status === "valid").length; - const rpki2Valid = rpki2Results.filter((r) => r.status === "valid").length; - const rpki1Pct = rpki1Results.length > 0 ? Math.round((rpki1Valid / rpki1Results.length) * 100) : 0; - const rpki2Pct = rpki2Results.length > 0 ? Math.round((rpki2Valid / rpki2Results.length) * 100) : 0; - - const duration = Date.now() - start; - const compareResult = { - meta: { duration_ms: duration, timestamp: new Date().toISOString() }, - asn1: { - asn: parseInt(asn1), - name: net1.name || "Unknown", - ix_count: ix1Set.size, - fac_count: fac1Set.size, - upstream_count: up1Set.size, - rpki_coverage: rpki1Pct, - }, - asn2: { - asn: parseInt(asn2), - name: net2.name || "Unknown", - ix_count: ix2Set.size, - fac_count: fac2Set.size, - upstream_count: up2Set.size, - rpki_coverage: rpki2Pct, - }, - common_ixps: commonIX, - only_asn1_ixps: only1IX, - only_asn2_ixps: only2IX, - common_facilities: commonFac, - common_upstreams: commonUpstreams, - rpki_comparison: { - asn1_coverage: rpki1Pct, - asn2_coverage: rpki2Pct, - asn1_checked: rpki1Results.length, - asn2_checked: rpki2Results.length, - better: rpki1Pct > rpki2Pct ? "AS" + asn1 : rpki2Pct > rpki1Pct ? "AS" + asn2 : "equal", - }, - }; - cacheSet(compareCacheKey, compareResult, CACHE_TTL_DEFAULT); - res.end(JSON.stringify(compareResult, null, 2)); - } catch (err) { - res.writeHead(500); - res.end(JSON.stringify({ error: "Compare failed", message: err.message })); - } - return; + return compareRoute.handleCompare(req, res, url); } - - // ============================================================ // Quick-IX endpoint: /api/quick-ix?asn=X - // Lightweight: only IX connections from PeeringDB, 1h cached - // Used by Peering Recommendations to avoid 20x full lookups - // ============================================================ if (reqPath === "/api/quick-ix") { - const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); - if (!rawAsn) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing asn parameter" })); - } - const cached = quickIxCacheGet(rawAsn); - if (cached !== undefined) { - res.writeHead(200, { "Content-Type": "application/json" }); - return res.end(JSON.stringify(cached)); - } - try { - const [pdbNetData, pdbIxlanData] = await Promise.all([ - fetchJSON("https://www.peeringdb.com/api/net?asn=" + rawAsn + "&depth=0", { timeout: 5000 }).catch(() => null), - fetchJSON("https://www.peeringdb.com/api/netixlan?asn=" + rawAsn + "&limit=100", { timeout: 6000 }).catch(() => null), - ]); - const netName = pdbNetData?.data?.[0]?.name || ""; - const ixConnections = []; - if (pdbIxlanData && pdbIxlanData.data) { - pdbIxlanData.data.forEach((row) => { - ixConnections.push({ ix_id: row.ixlan_id, name: row.name || "", speed: row.speed || 0 }); - }); - } - // Fall back to RIPE Stat if PeeringDB returns nothing - if (ixConnections.length === 0) { - const rsStat = await fetchRipeStatCached("https://stat.ripe.net/data/ixs/data.json?resource=AS" + rawAsn, { timeout: 5000 }).catch(() => null); - const ixs = rsStat?.data?.ixs || []; - ixs.forEach((ix) => ixConnections.push({ ix_id: ix.ixp_id || 0, name: ix.name || "", speed: 0 })); - } - const result = { asn: parseInt(rawAsn), name: netName, ix_connections: ixConnections }; - quickIxCacheSet(rawAsn, result); - res.writeHead(200, { "Content-Type": "application/json" }); - return res.end(JSON.stringify(result)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "quick-ix lookup failed", message: err.message })); - } + return quickIxRoute.handleQuickIx(req, res, url); } - // ============================================================ // Peer Matching endpoint: /api/peers/find?ix=NAME&policy=open&min_speed=10000 - // ============================================================ if (reqPath === "/api/peers/find") { - const ixName = url.searchParams.get("ix") || ""; - const policy = url.searchParams.get("policy") || ""; - const minSpeed = parseInt(url.searchParams.get("min_speed") || "0"); - const netType = url.searchParams.get("type") || ""; - - if (!ixName) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing ix parameter (IX name)" })); - } - - const start = Date.now(); - try { - // Search for IX by name - const ixSearch = await fetchPeeringDB("/ix?name__contains=" + encodeURIComponent(ixName)); - const ixResults = ixSearch?.data || []; - if (ixResults.length === 0) { - return res.end(JSON.stringify({ error: "No IX found matching: " + ixName, matches: [] })); - } - - // Use first matching IX - const ix = ixResults[0]; - const ixId = ix.id; - - // Get ixlan for this IX - const ixlanData = await fetchPeeringDB("/ixlan?ix_id=" + ixId); - const ixlans = ixlanData?.data || []; - if (ixlans.length === 0) { - return res.end(JSON.stringify({ ix: { id: ixId, name: ix.name }, matches: [] })); - } - - const ixlanId = ixlans[0].id; - - // Get all networks at this IX - const netixlanData = await fetchPeeringDB("/netixlan?ixlan_id=" + ixlanId); - const netixlans = netixlanData?.data || []; - - // Get unique net_ids - const netIds = [...new Set(netixlans.map(n => n.net_id))]; - - // Fetch network details in batches - const networks = []; - const batchSize = 20; - for (let i = 0; i < Math.min(netIds.length, 200); i += batchSize) { - const batch = netIds.slice(i, i + batchSize); - const batchResults = await Promise.all( - batch.map(nid => fetchPeeringDB("/net/" + nid)) - ); - batchResults.forEach(r => { - if (r?.data?.[0]) networks.push(r.data[0]); - }); - } - - // Filter and rank - let filtered = networks.map(net => { - const nix = netixlans.filter(n => n.net_id === net.id); - const maxSpeed = Math.max(...nix.map(n => n.speed || 0)); - return { - asn: net.asn, - name: net.name || "", - policy: net.policy_general || "", - type: net.info_type || "", - speed_mbps: maxSpeed, - speed_gbps: maxSpeed >= 1000 ? (maxSpeed / 1000) + " Gbps" : maxSpeed + " Mbps", - traffic: net.info_traffic || "", - website: net.website || "", - peeringdb_id: net.id, - ipv4: nix[0]?.ipaddr4 || null, - ipv6: nix[0]?.ipaddr6 || null, - }; - }); - - // Apply filters - if (policy) { - filtered = filtered.filter(n => n.policy.toLowerCase().includes(policy.toLowerCase())); - } - if (minSpeed > 0) { - filtered = filtered.filter(n => n.speed_mbps >= minSpeed); - } - if (netType) { - filtered = filtered.filter(n => n.type.toLowerCase().includes(netType.toLowerCase())); - } - - // Sort by speed desc - filtered.sort((a, b) => b.speed_mbps - a.speed_mbps); - - // Also find common IXPs for each match (check if they share other IXPs) - const duration = Date.now() - start; - return res.end(JSON.stringify({ - meta: { duration_ms: duration, timestamp: new Date().toISOString() }, - ix: { id: ixId, name: ix.name, ixlan_id: ixlanId }, - total_members: netixlans.length, - filtered_count: filtered.length, - matches: filtered.slice(0, 50), - }, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "Peer matching failed", message: err.message })); - } + return peersFindRoute.handlePeersFind(req, res, url); } - // ============================================================ // Prefix Detail endpoint: /api/prefix/detail?prefix=X - // ============================================================ if (reqPath === "/api/prefix/detail") { - const prefix = url.searchParams.get("prefix") || ""; - if (!prefix) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing prefix parameter" })); - } - const start = Date.now(); - try { - const [routingStatus, visibility] = await Promise.all([ - fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=" + encodeURIComponent(prefix)), - fetchRipeStatCached("https://stat.ripe.net/data/visibility/data.json?resource=" + encodeURIComponent(prefix)), - ]); - - const origins = routingStatus?.data?.origins || []; - const firstSeen = routingStatus?.data?.first_seen?.time || null; - - // RPKI validation: use local PostgreSQL database (sub-10ms, zero external API calls) - let rpkiStatus = "unknown"; - let rpkiRoas = []; - const originAsn = origins.length > 0 ? origins[0].asn : null; - if (originAsn) { - try { - const localRpki = await validateRPKIWithCache(originAsn, prefix); - rpkiStatus = localRpki.status; - rpkiRoas = new Array(localRpki.validating_roas); // count only, no detail - } catch (e) { - console.error("[Prefix Detail] RPKI validation error:", e.message); - rpkiStatus = "unknown"; - rpkiRoas = []; - } - } - var visData = visibility?.data?.visibilities || []; - var risPeersSeeingIt = visData.length > 0 ? visData.filter(v => v.ris_peers_seeing > 0).length : 0; - var visibilitySource = "ripe_stat"; - // bgproutes.io fallback if RIPE Stat visibility returned no data - if (visData.length === 0 && BGPROUTES_API_KEY) { - var bgprVis = await fetchBgproutesVisibility(prefix); - if (bgprVis && bgprVis.vps_seeing > 0) { - risPeersSeeingIt = bgprVis.vps_seeing; - visData = []; // keep empty, use risPeersSeeingIt - visibilitySource = "bgproutes.io"; - } - } - - // Try to get IRR data - let irrStatus = "unknown"; - try { - const whoisData = await fetchRipeStatCached("https://stat.ripe.net/data/whois/data.json?resource=" + encodeURIComponent(prefix)); - const records = whoisData?.data?.records || []; - if (records.length > 0) irrStatus = "found"; - } catch(_e) {} - - const duration = Date.now() - start; - return res.end(JSON.stringify({ - meta: { duration_ms: duration, timestamp: new Date().toISOString() }, - prefix: prefix, - origins: origins.map(o => ({ asn: o.asn, prefix: o.prefix })), - rpki: { status: rpkiStatus, validating_roas: rpkiRoas.length }, - irr_status: irrStatus, - visibility: { ris_peers_seeing: risPeersSeeingIt, total_probes: visData.length || risPeersSeeingIt, source: visibilitySource }, - first_seen: firstSeen, - }, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "Prefix detail failed", message: err.message })); - } + return prefixDetailRoute.handlePrefixDetail(req, res, url); } - // ============================================================ // IX Detail endpoint: /api/ix/detail?ix_id=X - // ============================================================ if (reqPath === "/api/ix/detail") { - const ixId = (url.searchParams.get("ix_id") || "").replace(/[^0-9]/g, ""); - if (!ixId) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing ix_id parameter" })); - } - const start = Date.now(); - try { - const [ixData, ixlanData] = await Promise.all([ - fetchPeeringDB("/ix/" + ixId), - fetchPeeringDB("/ixlan?ix_id=" + ixId), - ]); - - const ix = ixData?.data?.[0] || {}; - const ixlans = ixlanData?.data || []; - const ixlanId = ixlans.length > 0 ? ixlans[0].id : null; - - let members = []; - if (ixlanId) { - const netixlanData = await fetchPeeringDB("/netixlan?ixlan_id=" + ixlanId); - members = (netixlanData?.data || []).map(m => ({ - asn: m.asn, - name: m.name || "", - speed_mbps: m.speed || 0, - speed_display: (m.speed || 0) >= 1000 ? ((m.speed || 0) / 1000) + " Gbps" : (m.speed || 0) + " Mbps", - ipv4: m.ipaddr4 || null, - ipv6: m.ipaddr6 || null, - })); - } - - // Sort by speed desc for top members - const sorted = members.slice().sort((a, b) => b.speed_mbps - a.speed_mbps); - - const duration = Date.now() - start; - return res.end(JSON.stringify({ - meta: { duration_ms: duration, timestamp: new Date().toISOString() }, - ix: { - id: parseInt(ixId), - name: ix.name || "", - city: ix.city || "", - country: ix.country || "", - website: ix.website || "", - peeringdb_url: "https://www.peeringdb.com/ix/" + ixId, - }, - total_members: members.length, - top_members_by_speed: sorted.slice(0, 20), - all_members: sorted, - }, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "IX detail failed", message: err.message })); - } + return ixDetailRoute.handleIxDetail(req, res, url); } - - // ============================================================ // Feature 25: Topology endpoint - // ============================================================ if (reqPath === "/api/topology") { - const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); - const depth = parseInt(url.searchParams.get("depth") || "2") || 2; - if (!rawAsn) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" })); - } - const start = Date.now(); - try { - const topology = await fetchTopology(parseInt(rawAsn), depth); - - // ---- Threat Intelligence Enrichment for Topology Nodes ---- - const threatMap = {}; - const threatPromises = topology.nodes.slice(0, 100).map(async (node) => { - try { - const asNum = String(node.asn); - const threat = await localDb.getThreatIntel(asNum); - if (threat) { - threatMap[node.asn] = { - threat_level: threat.threat_level, - confidence_score: threat.confidence_score, - source: threat.source, - }; - } - } catch (e) { - console.error(`[Topology Threat Lookup] Error checking ASN ${node.asn}:`, e.message); - } - }); - await Promise.race([ - Promise.all(threatPromises), - new Promise(r => setTimeout(r, 3000)), - ]); - - // Attach threat status to node objects - topology.nodes.forEach((node) => { - if (threatMap[node.asn]) { - node.threat_level = threatMap[node.asn].threat_level; - node.threat_confidence = threatMap[node.asn].confidence_score; - node.threat_source = threatMap[node.asn].source; - } - }); - - topology.meta = { - query: "AS" + rawAsn, depth: depth, duration_ms: Date.now() - start, - timestamp: new Date().toISOString(), node_count: topology.nodes.length, edge_count: topology.edges.length, - }; - return res.end(JSON.stringify(topology, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "Topology query failed", message: err.message })); - } + return topologyRoute.handleTopology(req, res, url); } - // ============================================================ // Feature 27: WHOIS endpoint - // ============================================================ if (reqPath === "/api/whois") { - const resource = url.searchParams.get("resource") || ""; - if (!resource) { - res.writeHead(400); - return res.end(JSON.stringify({ error: "Missing resource parameter (ASN, prefix, or domain)" })); - } - const start = Date.now(); - try { - const whoisResult = await fetchWhois(resource); - if (!whoisResult || typeof whoisResult !== "object") { - res.writeHead(503); - return res.end(JSON.stringify({ error: "WHOIS data temporarily unavailable" })); - } - whoisResult.meta = { duration_ms: Date.now() - start, timestamp: new Date().toISOString() }; - return res.end(JSON.stringify(whoisResult, null, 2)); - } catch (err) { - res.writeHead(500); - return res.end(JSON.stringify({ error: "WHOIS lookup failed", message: err.message })); - } + return whoisRoute.handleWhoisRoute(req, res, url); } // Feature 28b: Company enrichment via Wikipedia + website meta scraping if (reqPath === "/api/enrich") { - res.setHeader("Content-Type", "application/json"); - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader("Cache-Control", "no-store"); - const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); - const companyName = (url.searchParams.get("name") || "").trim(); - const website = (url.searchParams.get("website") || "").trim(); - const UA_SCRAPE = "Mozilla/5.0 (compatible; PeerCortex/1.0; +https://peercortex.org)"; - - let description = null; - let wikiUrl = null; - - try { - // 1. Direct Wikipedia lookup by company name - if (companyName) { - const nameLower = companyName.toLowerCase(); - const isRelevant = (title, extract) => { - const titleLower = (title || "").toLowerCase(); - const extractLower = (extract || "").toLowerCase(); - const nameWords = nameLower.replace(/\s+(gmbh|ag|ltd|inc|llc|bv|sa|sas|oy|ab)\s*$/i, "").split(/\s+/).filter(w => w.length > 3); - const titleMatch = nameWords.some(w => titleLower.includes(w)); - const netTerms = ["internet", "network", "isp", "hosting", "provider", "transit", "data center", "datacenter", "telecommunications", "telecom", "bandwidth", "peering", "routing", "autonomous system", "colocation", "colo", "fiber", "optical", "transceiver"]; - const hasNetContext = netTerms.some(t => extractLower.includes(t)); - return titleMatch && (hasNetContext || titleMatch); - }; - - // Direct title lookup — try full name first, then first word as fallback - const cleanName = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim(); - const firstName = cleanName.split(/\s+/)[0]; - const namesToTry = cleanName === firstName ? [cleanName] : [cleanName, firstName]; - for (const tryName of namesToTry) { - const wikiDirect = await fetchJSON( - "https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(tryName), - { timeout: 5000 } - ); - if (wikiDirect && wikiDirect.type === "disambiguation") continue; // skip disambiguation pages - if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30 && isRelevant(wikiDirect.title, wikiDirect.extract)) { - description = wikiDirect.extract.replace(/\s+/g, " ").trim().slice(0, 300); - wikiUrl = wikiDirect.content_urls && wikiDirect.content_urls.desktop && wikiDirect.content_urls.desktop.page; - break; - } - } - - // 2. Wikipedia search if direct lookup didn't match - if (!description) { - const searchQuery = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC)$/i, "") + " internet service provider"; - const searchData = await fetchJSON( - "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=" + encodeURIComponent(searchQuery) + "&limit=3", - { timeout: 5000 } - ); - if (searchData && Array.isArray(searchData) && searchData[1] && searchData[1].length > 0) { - const topTitle = searchData[1][0]; - const wikiSearch = await fetchJSON( - "https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(topTitle), - { timeout: 5000 } - ); - if (wikiSearch && wikiSearch.extract && wikiSearch.extract.length > 30) { - if (isRelevant(wikiSearch.title, wikiSearch.extract)) { - description = wikiSearch.extract.replace(/\s+/g, " ").trim().slice(0, 300); - wikiUrl = wikiSearch.content_urls && wikiSearch.content_urls.desktop && wikiSearch.content_urls.desktop.page; - } - } - } - } - } - - // 3. Fallback: scrape website meta description (follows up to 3 redirects) - function fetchPage(pageUrl, hops) { - if (hops <= 0) return Promise.resolve(null); - return new Promise((resolve) => { - const mod = pageUrl.startsWith("https") ? https : http; - const req = mod.get(pageUrl, { headers: { "User-Agent": UA_SCRAPE }, timeout: 6000 }, (res) => { - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - res.resume(); // drain to free socket - const next = res.headers.location.startsWith("http") ? res.headers.location : new URL(res.headers.location, pageUrl).href; - return resolve(fetchPage(next, hops - 1)); - } - let data = ""; - res.on("data", (c) => { data += c; if (data.length > 40000) { req.destroy(); resolve(data); } }); - res.on("end", () => resolve(data)); - }); - req.on("error", () => resolve(null)); - req.on("timeout", () => { req.destroy(); resolve(null); }); - }); - } - if (!description && website) { - let wsUrl = website; - if (!wsUrl.startsWith("http")) wsUrl = "https://" + wsUrl; - const aboutUrl = wsUrl.replace(/\/$/, "") + "/about"; - const tryUrls = [aboutUrl, wsUrl]; - for (const tryUrl of tryUrls) { - const resp = await fetchPage(tryUrl, 3); - if (!resp) continue; - const metaPatterns = [ - /]+name=["']description["'][^>]+content=["']([^"']{20,500})["']/i, - /]+content=["']([^"']{20,500})["'][^>]+name=["']description["']/i, - /]+property=["']og:description["'][^>]+content=["']([^"']{20,500})["']/i, - /]+content=["']([^"']{20,500})["'][^>]+property=["']og:description["']/i, - ]; - let matched = false; - for (const pat of metaPatterns) { - const m = resp.match(pat); - if (m && m[1]) { - description = m[1].replace(/ |✓|&/g, " ").replace(/\s+/g, " ").trim().slice(0, 300); - matched = true; - break; - } - } - if (matched) break; - } - } - } catch (_e) { - // Return whatever we have - } - - return res.end(JSON.stringify({ asn: rawAsn, description, wiki_url: wikiUrl })); + return enrichRoute.handleEnrich(req, res, url); } + // Feature 28: Submarine Cable overlay (TeleGeography proxy) // ── Submarine Cables map data ───────────────────────────────── if (reqPath === '/api/submarine-cables') { diff --git a/server/routes/compare.js b/server/routes/compare.js new file mode 100644 index 0000000..fe2f13e --- /dev/null +++ b/server/routes/compare.js @@ -0,0 +1,175 @@ +const localDb = require("../../local-db-client"); +const { fetchRipeStatCached } = require("../services/ripe-stat"); +const { fetchPeeringDB } = require("../services/peeringdb"); +const { validateRPKIWithCache } = require("../services/rpki"); +const { cacheGet, cacheSet, CACHE_TTL_DEFAULT } = require("../caches/response-caches"); + +// Compare endpoint: /api/compare?asn1=X&asn2=Y +async function handleCompare(req, res, url) { + const asn1 = (url.searchParams.get("asn1") || "").replace(/[^0-9]/g, ""); + const asn2 = (url.searchParams.get("asn2") || "").replace(/[^0-9]/g, ""); + if (!asn1 || !asn2) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Need asn1 and asn2 parameters" })); + } + + const compareCacheKey = "compare:" + asn1 + ":" + asn2; + const compareCached = cacheGet(compareCacheKey); + if (compareCached) { + res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" }); + return res.end(JSON.stringify(compareCached)); + } + const start = Date.now(); + try { + // ALL calls in parallel — single batch + // Phase 1: Get PDB net objects + RIPE data + const [pdb1, pdb2, nb1Data, nb2Data, pfx1Data, pfx2Data] = await Promise.all([ + fetchPeeringDB("/net?asn=" + asn1), + fetchPeeringDB("/net?asn=" + asn2), + fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn1, { timeout: 8000 }), + fetchRipeStatCached("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn2, { timeout: 8000 }), + fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn1, { timeout: 8000 }), + fetchRipeStatCached("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn2, { timeout: 8000 }), + ]); + + const net1 = pdb1?.data?.[0] || {}; + const net2 = pdb2?.data?.[0] || {}; + const netId1 = net1.id; + const netId2 = net2.id; + + // Phase 2: IX + Facility using net_id (Bug 1 fix: netfac requires net_id, not asn) + const ixFacPromises = []; + ixFacPromises.push(netId1 ? fetchPeeringDB("/netixlan?net_id=" + netId1) : Promise.resolve(null)); + ixFacPromises.push(netId2 ? fetchPeeringDB("/netixlan?net_id=" + netId2) : Promise.resolve(null)); + ixFacPromises.push(netId1 ? fetchPeeringDB("/netfac?net_id=" + netId1) : Promise.resolve(null)); + ixFacPromises.push(netId2 ? fetchPeeringDB("/netfac?net_id=" + netId2) : Promise.resolve(null)); + const [ix1Data, ix2Data, fac1Data, fac2Data] = await Promise.all(ixFacPromises); + + const ix1Set = new Set((ix1Data?.data || []).map((ix) => ix.ix_id)); + const ix2Set = new Set((ix2Data?.data || []).map((ix) => ix.ix_id)); + const ix1Names = {}; + (ix1Data?.data || []).forEach((ix) => (ix1Names[ix.ix_id] = ix.name)); + const ix2Names = {}; + (ix2Data?.data || []).forEach((ix) => (ix2Names[ix.ix_id] = ix.name)); + + const commonIX = [...ix1Set].filter((id) => ix2Set.has(id)).map((id) => ({ ix_id: id, name: ix1Names[id] || ix2Names[id] || "" })); + const only1IX = [...ix1Set].filter((id) => !ix2Set.has(id)).map((id) => ({ ix_id: id, name: ix1Names[id] || "" })); + const only2IX = [...ix2Set].filter((id) => !ix1Set.has(id)).map((id) => ({ ix_id: id, name: ix2Names[id] || "" })); + + const fac1Set = new Set((fac1Data?.data || []).map((f) => f.fac_id)); + const fac2Set = new Set((fac2Data?.data || []).map((f) => f.fac_id)); + const fac1Names = {}; + (fac1Data?.data || []).forEach((f) => (fac1Names[f.fac_id] = f.name)); + const fac2Names = {}; + (fac2Data?.data || []).forEach((f) => (fac2Names[f.fac_id] = f.name)); + + const commonFac = [...fac1Set].filter((id) => fac2Set.has(id)).map((id) => ({ fac_id: id, name: fac1Names[id] || fac2Names[id] || "" })); + + const nb1 = (nb1Data?.data?.neighbours || []).filter((n) => n.type === "left"); + const nb2 = (nb2Data?.data?.neighbours || []).filter((n) => n.type === "left"); + const up1Set = new Set(nb1.map((n) => n.asn)); + const up2Set = new Set(nb2.map((n) => n.asn)); + const nb1Map = {}; + nb1.forEach((n) => (nb1Map[n.asn] = n.as_name || "")); + const nb2Map = {}; + nb2.forEach((n) => (nb2Map[n.asn] = n.as_name || "")); + + const commonUpstreams = [...up1Set] + .filter((a) => up2Set.has(a)) + .map((a) => ({ asn: a, name: nb1Map[a] || nb2Map[a] || "" })); + + // ---- Threat Intelligence Enrichment for Common Upstreams ---- + const threatMap = {}; + const threatPromises = commonUpstreams.slice(0, 50).map(async (n) => { + try { + const asNum = String(n.asn); + const threat = await localDb.getThreatIntel(asNum); + if (threat) { + threatMap[n.asn] = { + threat_level: threat.threat_level, + confidence_score: threat.confidence_score, + source: threat.source, + }; + } + } catch (e) { + console.error(`[Compare Threat Lookup] Error checking ASN ${n.asn}:`, e.message); + } + }); + await Promise.race([ + Promise.all(threatPromises), + new Promise(r => setTimeout(r, 2000)), + ]); + + // Attach threat status to upstream objects + commonUpstreams.forEach((n) => { + if (threatMap[n.asn]) { + n.threat_level = threatMap[n.asn].threat_level; + n.threat_confidence = threatMap[n.asn].confidence_score; + n.threat_source = threatMap[n.asn].source; + } + }); + + // Resolve names + RPKI sample (max 3+3 prefixes) all in parallel with 5s timeout + const pfx1 = (pfx1Data?.data?.prefixes || []).slice(0, 3).map((p) => p.prefix); + const pfx2 = (pfx2Data?.data?.prefixes || []).slice(0, 3).map((p) => p.prefix); + const [, rpki1Results, rpki2Results] = await Promise.race([ + Promise.all([ + commonUpstreams.length > 0 ? Promise.all(commonUpstreams.map(n => + fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 }) + .then(r => { if (r?.data?.holder) n.name = r.data.holder; }) + .catch(() => {}) + )) : Promise.resolve([]), + Promise.all(pfx1.map((p) => validateRPKIWithCache(asn1, p))), + Promise.all(pfx2.map((p) => validateRPKIWithCache(asn2, p))), + ]), + new Promise(r => setTimeout(() => r([[], [], []]), 5000)), + ]); + + const rpki1Valid = rpki1Results.filter((r) => r.status === "valid").length; + const rpki2Valid = rpki2Results.filter((r) => r.status === "valid").length; + const rpki1Pct = rpki1Results.length > 0 ? Math.round((rpki1Valid / rpki1Results.length) * 100) : 0; + const rpki2Pct = rpki2Results.length > 0 ? Math.round((rpki2Valid / rpki2Results.length) * 100) : 0; + + const duration = Date.now() - start; + const compareResult = { + meta: { duration_ms: duration, timestamp: new Date().toISOString() }, + asn1: { + asn: parseInt(asn1), + name: net1.name || "Unknown", + ix_count: ix1Set.size, + fac_count: fac1Set.size, + upstream_count: up1Set.size, + rpki_coverage: rpki1Pct, + }, + asn2: { + asn: parseInt(asn2), + name: net2.name || "Unknown", + ix_count: ix2Set.size, + fac_count: fac2Set.size, + upstream_count: up2Set.size, + rpki_coverage: rpki2Pct, + }, + common_ixps: commonIX, + only_asn1_ixps: only1IX, + only_asn2_ixps: only2IX, + common_facilities: commonFac, + common_upstreams: commonUpstreams, + rpki_comparison: { + asn1_coverage: rpki1Pct, + asn2_coverage: rpki2Pct, + asn1_checked: rpki1Results.length, + asn2_checked: rpki2Results.length, + better: rpki1Pct > rpki2Pct ? "AS" + asn1 : rpki2Pct > rpki1Pct ? "AS" + asn2 : "equal", + }, + }; + cacheSet(compareCacheKey, compareResult, CACHE_TTL_DEFAULT); + res.end(JSON.stringify(compareResult, null, 2)); + } catch (err) { + res.writeHead(500); + res.end(JSON.stringify({ error: "Compare failed", message: err.message })); + } + return; + +} + +module.exports = { handleCompare }; diff --git a/server/routes/enrich.js b/server/routes/enrich.js new file mode 100644 index 0000000..e6f80c7 --- /dev/null +++ b/server/routes/enrich.js @@ -0,0 +1,125 @@ +const http = require("http"); +const https = require("https"); +const { fetchJSON } = require("../services/http-helpers"); + +// Feature 28b: Company enrichment via Wikipedia + website meta scraping +async function handleEnrich(req, res, url) { + res.setHeader("Content-Type", "application/json"); + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Cache-Control", "no-store"); + const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); + const companyName = (url.searchParams.get("name") || "").trim(); + const website = (url.searchParams.get("website") || "").trim(); + const UA_SCRAPE = "Mozilla/5.0 (compatible; PeerCortex/1.0; +https://peercortex.org)"; + + let description = null; + let wikiUrl = null; + + try { + // 1. Direct Wikipedia lookup by company name + if (companyName) { + const nameLower = companyName.toLowerCase(); + const isRelevant = (title, extract) => { + const titleLower = (title || "").toLowerCase(); + const extractLower = (extract || "").toLowerCase(); + const nameWords = nameLower.replace(/\s+(gmbh|ag|ltd|inc|llc|bv|sa|sas|oy|ab)\s*$/i, "").split(/\s+/).filter(w => w.length > 3); + const titleMatch = nameWords.some(w => titleLower.includes(w)); + const netTerms = ["internet", "network", "isp", "hosting", "provider", "transit", "data center", "datacenter", "telecommunications", "telecom", "bandwidth", "peering", "routing", "autonomous system", "colocation", "colo", "fiber", "optical", "transceiver"]; + const hasNetContext = netTerms.some(t => extractLower.includes(t)); + return titleMatch && (hasNetContext || titleMatch); + }; + + // Direct title lookup — try full name first, then first word as fallback + const cleanName = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim(); + const firstName = cleanName.split(/\s+/)[0]; + const namesToTry = cleanName === firstName ? [cleanName] : [cleanName, firstName]; + for (const tryName of namesToTry) { + const wikiDirect = await fetchJSON( + "https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(tryName), + { timeout: 5000 } + ); + if (wikiDirect && wikiDirect.type === "disambiguation") continue; // skip disambiguation pages + if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30 && isRelevant(wikiDirect.title, wikiDirect.extract)) { + description = wikiDirect.extract.replace(/\s+/g, " ").trim().slice(0, 300); + wikiUrl = wikiDirect.content_urls && wikiDirect.content_urls.desktop && wikiDirect.content_urls.desktop.page; + break; + } + } + + // 2. Wikipedia search if direct lookup didn't match + if (!description) { + const searchQuery = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC)$/i, "") + " internet service provider"; + const searchData = await fetchJSON( + "https://en.wikipedia.org/w/api.php?action=opensearch&format=json&search=" + encodeURIComponent(searchQuery) + "&limit=3", + { timeout: 5000 } + ); + if (searchData && Array.isArray(searchData) && searchData[1] && searchData[1].length > 0) { + const topTitle = searchData[1][0]; + const wikiSearch = await fetchJSON( + "https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(topTitle), + { timeout: 5000 } + ); + if (wikiSearch && wikiSearch.extract && wikiSearch.extract.length > 30) { + if (isRelevant(wikiSearch.title, wikiSearch.extract)) { + description = wikiSearch.extract.replace(/\s+/g, " ").trim().slice(0, 300); + wikiUrl = wikiSearch.content_urls && wikiSearch.content_urls.desktop && wikiSearch.content_urls.desktop.page; + } + } + } + } + } + + // 3. Fallback: scrape website meta description (follows up to 3 redirects) + function fetchPage(pageUrl, hops) { + if (hops <= 0) return Promise.resolve(null); + return new Promise((resolve) => { + const mod = pageUrl.startsWith("https") ? https : http; + const req = mod.get(pageUrl, { headers: { "User-Agent": UA_SCRAPE }, timeout: 6000 }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); // drain to free socket + const next = res.headers.location.startsWith("http") ? res.headers.location : new URL(res.headers.location, pageUrl).href; + return resolve(fetchPage(next, hops - 1)); + } + let data = ""; + res.on("data", (c) => { data += c; if (data.length > 40000) { req.destroy(); resolve(data); } }); + res.on("end", () => resolve(data)); + }); + req.on("error", () => resolve(null)); + req.on("timeout", () => { req.destroy(); resolve(null); }); + }); + } + if (!description && website) { + let wsUrl = website; + if (!wsUrl.startsWith("http")) wsUrl = "https://" + wsUrl; + const aboutUrl = wsUrl.replace(/\/$/, "") + "/about"; + const tryUrls = [aboutUrl, wsUrl]; + for (const tryUrl of tryUrls) { + const resp = await fetchPage(tryUrl, 3); + if (!resp) continue; + const metaPatterns = [ + /]+name=["']description["'][^>]+content=["']([^"']{20,500})["']/i, + /]+content=["']([^"']{20,500})["'][^>]+name=["']description["']/i, + /]+property=["']og:description["'][^>]+content=["']([^"']{20,500})["']/i, + /]+content=["']([^"']{20,500})["'][^>]+property=["']og:description["']/i, + ]; + let matched = false; + for (const pat of metaPatterns) { + const m = resp.match(pat); + if (m && m[1]) { + description = m[1].replace(/ |✓|&/g, " ").replace(/\s+/g, " ").trim().slice(0, 300); + matched = true; + break; + } + } + if (matched) break; + } + } + } catch (_e) { + // Return whatever we have + } + + return res.end(JSON.stringify({ asn: rawAsn, description, wiki_url: wikiUrl })); + +} + +module.exports = { handleEnrich }; diff --git a/server/routes/ix-detail.js b/server/routes/ix-detail.js new file mode 100644 index 0000000..d55f8be --- /dev/null +++ b/server/routes/ix-detail.js @@ -0,0 +1,59 @@ +const { fetchPeeringDB } = require("../services/peeringdb"); + +// IX Detail endpoint: /api/ix/detail?ix_id=X +async function handleIxDetail(req, res, url) { + const ixId = (url.searchParams.get("ix_id") || "").replace(/[^0-9]/g, ""); + if (!ixId) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing ix_id parameter" })); + } + const start = Date.now(); + try { + const [ixData, ixlanData] = await Promise.all([ + fetchPeeringDB("/ix/" + ixId), + fetchPeeringDB("/ixlan?ix_id=" + ixId), + ]); + + const ix = ixData?.data?.[0] || {}; + const ixlans = ixlanData?.data || []; + const ixlanId = ixlans.length > 0 ? ixlans[0].id : null; + + let members = []; + if (ixlanId) { + const netixlanData = await fetchPeeringDB("/netixlan?ixlan_id=" + ixlanId); + members = (netixlanData?.data || []).map(m => ({ + asn: m.asn, + name: m.name || "", + speed_mbps: m.speed || 0, + speed_display: (m.speed || 0) >= 1000 ? ((m.speed || 0) / 1000) + " Gbps" : (m.speed || 0) + " Mbps", + ipv4: m.ipaddr4 || null, + ipv6: m.ipaddr6 || null, + })); + } + + // Sort by speed desc for top members + const sorted = members.slice().sort((a, b) => b.speed_mbps - a.speed_mbps); + + const duration = Date.now() - start; + return res.end(JSON.stringify({ + meta: { duration_ms: duration, timestamp: new Date().toISOString() }, + ix: { + id: parseInt(ixId), + name: ix.name || "", + city: ix.city || "", + country: ix.country || "", + website: ix.website || "", + peeringdb_url: "https://www.peeringdb.com/ix/" + ixId, + }, + total_members: members.length, + top_members_by_speed: sorted.slice(0, 20), + all_members: sorted, + }, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "IX detail failed", message: err.message })); + } + +} + +module.exports = { handleIxDetail }; diff --git a/server/routes/peers-find.js b/server/routes/peers-find.js new file mode 100644 index 0000000..d9a6bef --- /dev/null +++ b/server/routes/peers-find.js @@ -0,0 +1,106 @@ +const { fetchPeeringDB } = require("../services/peeringdb"); + +// Peer Matching endpoint: /api/peers/find?ix=NAME&policy=open&min_speed=10000 +async function handlePeersFind(req, res, url) { + const ixName = url.searchParams.get("ix") || ""; + const policy = url.searchParams.get("policy") || ""; + const minSpeed = parseInt(url.searchParams.get("min_speed") || "0"); + const netType = url.searchParams.get("type") || ""; + + if (!ixName) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing ix parameter (IX name)" })); + } + + const start = Date.now(); + try { + // Search for IX by name + const ixSearch = await fetchPeeringDB("/ix?name__contains=" + encodeURIComponent(ixName)); + const ixResults = ixSearch?.data || []; + if (ixResults.length === 0) { + return res.end(JSON.stringify({ error: "No IX found matching: " + ixName, matches: [] })); + } + + // Use first matching IX + const ix = ixResults[0]; + const ixId = ix.id; + + // Get ixlan for this IX + const ixlanData = await fetchPeeringDB("/ixlan?ix_id=" + ixId); + const ixlans = ixlanData?.data || []; + if (ixlans.length === 0) { + return res.end(JSON.stringify({ ix: { id: ixId, name: ix.name }, matches: [] })); + } + + const ixlanId = ixlans[0].id; + + // Get all networks at this IX + const netixlanData = await fetchPeeringDB("/netixlan?ixlan_id=" + ixlanId); + const netixlans = netixlanData?.data || []; + + // Get unique net_ids + const netIds = [...new Set(netixlans.map(n => n.net_id))]; + + // Fetch network details in batches + const networks = []; + const batchSize = 20; + for (let i = 0; i < Math.min(netIds.length, 200); i += batchSize) { + const batch = netIds.slice(i, i + batchSize); + const batchResults = await Promise.all( + batch.map(nid => fetchPeeringDB("/net/" + nid)) + ); + batchResults.forEach(r => { + if (r?.data?.[0]) networks.push(r.data[0]); + }); + } + + // Filter and rank + let filtered = networks.map(net => { + const nix = netixlans.filter(n => n.net_id === net.id); + const maxSpeed = Math.max(...nix.map(n => n.speed || 0)); + return { + asn: net.asn, + name: net.name || "", + policy: net.policy_general || "", + type: net.info_type || "", + speed_mbps: maxSpeed, + speed_gbps: maxSpeed >= 1000 ? (maxSpeed / 1000) + " Gbps" : maxSpeed + " Mbps", + traffic: net.info_traffic || "", + website: net.website || "", + peeringdb_id: net.id, + ipv4: nix[0]?.ipaddr4 || null, + ipv6: nix[0]?.ipaddr6 || null, + }; + }); + + // Apply filters + if (policy) { + filtered = filtered.filter(n => n.policy.toLowerCase().includes(policy.toLowerCase())); + } + if (minSpeed > 0) { + filtered = filtered.filter(n => n.speed_mbps >= minSpeed); + } + if (netType) { + filtered = filtered.filter(n => n.type.toLowerCase().includes(netType.toLowerCase())); + } + + // Sort by speed desc + filtered.sort((a, b) => b.speed_mbps - a.speed_mbps); + + // Also find common IXPs for each match (check if they share other IXPs) + const duration = Date.now() - start; + return res.end(JSON.stringify({ + meta: { duration_ms: duration, timestamp: new Date().toISOString() }, + ix: { id: ixId, name: ix.name, ixlan_id: ixlanId }, + total_members: netixlans.length, + filtered_count: filtered.length, + matches: filtered.slice(0, 50), + }, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "Peer matching failed", message: err.message })); + } + +} + +module.exports = { handlePeersFind }; diff --git a/server/routes/prefix-detail.js b/server/routes/prefix-detail.js new file mode 100644 index 0000000..74d1011 --- /dev/null +++ b/server/routes/prefix-detail.js @@ -0,0 +1,77 @@ +const { fetchRipeStatCached } = require("../services/ripe-stat"); +const { fetchBgproutesVisibility } = require("../services/http-helpers"); +const { validateRPKIWithCache } = require("../services/rpki"); + +const BGPROUTES_API_KEY = process.env.BGPROUTES_API_KEY || ""; + +// Prefix Detail endpoint: /api/prefix/detail?prefix=X +async function handlePrefixDetail(req, res, url) { + const prefix = url.searchParams.get("prefix") || ""; + if (!prefix) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing prefix parameter" })); + } + const start = Date.now(); + try { + const [routingStatus, visibility] = await Promise.all([ + fetchRipeStatCached("https://stat.ripe.net/data/routing-status/data.json?resource=" + encodeURIComponent(prefix)), + fetchRipeStatCached("https://stat.ripe.net/data/visibility/data.json?resource=" + encodeURIComponent(prefix)), + ]); + + const origins = routingStatus?.data?.origins || []; + const firstSeen = routingStatus?.data?.first_seen?.time || null; + + // RPKI validation: use local PostgreSQL database (sub-10ms, zero external API calls) + let rpkiStatus = "unknown"; + let rpkiRoas = []; + const originAsn = origins.length > 0 ? origins[0].asn : null; + if (originAsn) { + try { + const localRpki = await validateRPKIWithCache(originAsn, prefix); + rpkiStatus = localRpki.status; + rpkiRoas = new Array(localRpki.validating_roas); // count only, no detail + } catch (e) { + console.error("[Prefix Detail] RPKI validation error:", e.message); + rpkiStatus = "unknown"; + rpkiRoas = []; + } + } + var visData = visibility?.data?.visibilities || []; + var risPeersSeeingIt = visData.length > 0 ? visData.filter(v => v.ris_peers_seeing > 0).length : 0; + var visibilitySource = "ripe_stat"; + // bgproutes.io fallback if RIPE Stat visibility returned no data + if (visData.length === 0 && BGPROUTES_API_KEY) { + var bgprVis = await fetchBgproutesVisibility(prefix); + if (bgprVis && bgprVis.vps_seeing > 0) { + risPeersSeeingIt = bgprVis.vps_seeing; + visData = []; // keep empty, use risPeersSeeingIt + visibilitySource = "bgproutes.io"; + } + } + + // Try to get IRR data + let irrStatus = "unknown"; + try { + const whoisData = await fetchRipeStatCached("https://stat.ripe.net/data/whois/data.json?resource=" + encodeURIComponent(prefix)); + const records = whoisData?.data?.records || []; + if (records.length > 0) irrStatus = "found"; + } catch(_e) {} + + const duration = Date.now() - start; + return res.end(JSON.stringify({ + meta: { duration_ms: duration, timestamp: new Date().toISOString() }, + prefix: prefix, + origins: origins.map(o => ({ asn: o.asn, prefix: o.prefix })), + rpki: { status: rpkiStatus, validating_roas: rpkiRoas.length }, + irr_status: irrStatus, + visibility: { ris_peers_seeing: risPeersSeeingIt, total_probes: visData.length || risPeersSeeingIt, source: visibilitySource }, + first_seen: firstSeen, + }, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "Prefix detail failed", message: err.message })); + } + +} + +module.exports = { handlePrefixDetail }; diff --git a/server/routes/quick-ix.js b/server/routes/quick-ix.js new file mode 100644 index 0000000..5647a0a --- /dev/null +++ b/server/routes/quick-ix.js @@ -0,0 +1,48 @@ +const { fetchJSON } = require("../services/http-helpers"); +const { fetchRipeStatCached } = require("../services/ripe-stat"); +const { quickIxCacheGet, quickIxCacheSet } = require("../caches/response-caches"); + +// Quick-IX endpoint: /api/quick-ix?asn=X +// Lightweight: only IX connections from PeeringDB, 1h cached +// Used by Peering Recommendations to avoid 20x full lookups +async function handleQuickIx(req, res, url) { + const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); + if (!rawAsn) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing asn parameter" })); + } + const cached = quickIxCacheGet(rawAsn); + if (cached !== undefined) { + res.writeHead(200, { "Content-Type": "application/json" }); + return res.end(JSON.stringify(cached)); + } + try { + const [pdbNetData, pdbIxlanData] = await Promise.all([ + fetchJSON("https://www.peeringdb.com/api/net?asn=" + rawAsn + "&depth=0", { timeout: 5000 }).catch(() => null), + fetchJSON("https://www.peeringdb.com/api/netixlan?asn=" + rawAsn + "&limit=100", { timeout: 6000 }).catch(() => null), + ]); + const netName = pdbNetData?.data?.[0]?.name || ""; + const ixConnections = []; + if (pdbIxlanData && pdbIxlanData.data) { + pdbIxlanData.data.forEach((row) => { + ixConnections.push({ ix_id: row.ixlan_id, name: row.name || "", speed: row.speed || 0 }); + }); + } + // Fall back to RIPE Stat if PeeringDB returns nothing + if (ixConnections.length === 0) { + const rsStat = await fetchRipeStatCached("https://stat.ripe.net/data/ixs/data.json?resource=AS" + rawAsn, { timeout: 5000 }).catch(() => null); + const ixs = rsStat?.data?.ixs || []; + ixs.forEach((ix) => ixConnections.push({ ix_id: ix.ixp_id || 0, name: ix.name || "", speed: 0 })); + } + const result = { asn: parseInt(rawAsn), name: netName, ix_connections: ixConnections }; + quickIxCacheSet(rawAsn, result); + res.writeHead(200, { "Content-Type": "application/json" }); + return res.end(JSON.stringify(result)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "quick-ix lookup failed", message: err.message })); + } + +} + +module.exports = { handleQuickIx }; diff --git a/server/routes/relationships.js b/server/routes/relationships.js new file mode 100644 index 0000000..869e025 --- /dev/null +++ b/server/routes/relationships.js @@ -0,0 +1,110 @@ +const localDb = require("../../local-db-client"); +const { fetchRipeStatCached } = require("../services/ripe-stat"); +const { cacheGet, cacheSet } = require("../caches/response-caches"); + +// AS Relationships endpoint: /api/relationships?asn=X +// Returns upstream providers, downstream customers, and peers with resolved +// names. Based on RIPE Stat asn-neighbours. +async function handleRelationships(req, res, url) { + const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); + if (!rawAsn) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" })); + } + const cacheKey = "relationships:" + rawAsn; + const cached = cacheGet(cacheKey); + if (cached) { + res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" }); + return res.end(JSON.stringify(cached)); + } + const start = Date.now(); + try { + const neighbourData = await fetchRipeStatCached( + "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1", + { timeout: 8000 } + ); + const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || []; + const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {}; + + const upstreams = neighbours.filter(n => n.type === "left").sort((a,b) => (b.power||0)-(a.power||0)); + const downstreams = neighbours.filter(n => n.type === "right").sort((a,b) => (b.power||0)-(a.power||0)); + const peers = neighbours.filter(n => n.type === "uncertain").sort((a,b) => (b.power||0)-(a.power||0)); + + // Resolve AS names for top entries (upstreams + downstreams all, top 20 peers) + const toResolve = [...upstreams, ...downstreams, ...peers.slice(0, 20)]; + const resolvedNames = {}; + await Promise.race([ + Promise.all(toResolve.map(n => + fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 }) + .then(r => { if (r && r.data && r.data.holder) resolvedNames[n.asn] = r.data.holder; }) + .catch(() => {}) + )), + new Promise(r => setTimeout(r, 5000)), + ]); + + // ---- Threat Intelligence Enrichment ---- + // Enrich neighbors with threat status from local threat_intel table + const threatMap = {}; + const allNeighborsForThreat = [...upstreams, ...downstreams, ...peers]; + const threatPromises = allNeighborsForThreat.slice(0, 100).map(async (n) => { + try { + const asNum = String(n.asn); + const threat = await localDb.getThreatIntel(asNum); + if (threat) { + threatMap[n.asn] = { + threat_level: threat.threat_level, + confidence_score: threat.confidence_score, + source: threat.source, + }; + } + } catch (e) { + console.error(`[Relationships Threat Lookup] Error checking ASN ${n.asn}:`, e.message); + } + }); + await Promise.race([ + Promise.all(threatPromises), + new Promise(r => setTimeout(r, 3000)), + ]); + + const fmt = n => ({ + asn: n.asn, + name: resolvedNames[n.asn] || "", + power: n.power || 0, + v4_peers: n.v4_peers || 0, + v6_peers: n.v6_peers || 0, + threat_level: threatMap[n.asn]?.threat_level || null, + threat_confidence: threatMap[n.asn]?.confidence_score || null, + threat_source: threatMap[n.asn]?.source || null, + }); + + const result = { + asn: parseInt(rawAsn), + query_time: new Date().toISOString(), + duration_ms: Date.now() - start, + counts: { + upstreams: counts.left || upstreams.length, + downstreams: counts.right || downstreams.length, + peers_total: counts.unique || peers.length, + uncertain: counts.uncertain || peers.length, + }, + upstreams: upstreams.map(fmt), + downstreams: downstreams.map(fmt), + peers: peers.slice(0, 50).map(fmt), + methodology: "RIPE Stat asn-neighbours API. left=upstream providers (carry your traffic), right=downstream customers (you carry their traffic), uncertain=lateral peers. Sorted by power score (number of prefixes seen via this relationship).", + source_url: "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn, + }; + + // A neighbourData fetch failure (not a genuine zero-neighbour ASN) would + // otherwise get cached as "0 upstreams/downstreams/peers" for the full 10min -- + // match the existing /api/validate precedent (90s TTL for suspicious zero counts). + cacheSet(cacheKey, result, neighbourData ? 10 * 60 * 1000 : 90 * 1000); + res.writeHead(200, { "Content-Type": "application/json" }); + return res.end(JSON.stringify(result, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "Relationships lookup failed", message: err.message })); + } + +} + +module.exports = { handleRelationships }; diff --git a/server/routes/topology.js b/server/routes/topology.js new file mode 100644 index 0000000..7f661a8 --- /dev/null +++ b/server/routes/topology.js @@ -0,0 +1,59 @@ +const localDb = require("../../local-db-client"); +const { fetchTopology } = require("../topology"); + +// Feature 25: Topology endpoint +async function handleTopology(req, res, url) { + const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, ""); + const depth = parseInt(url.searchParams.get("depth") || "2") || 2; + if (!rawAsn) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" })); + } + const start = Date.now(); + try { + const topology = await fetchTopology(parseInt(rawAsn), depth); + + // ---- Threat Intelligence Enrichment for Topology Nodes ---- + const threatMap = {}; + const threatPromises = topology.nodes.slice(0, 100).map(async (node) => { + try { + const asNum = String(node.asn); + const threat = await localDb.getThreatIntel(asNum); + if (threat) { + threatMap[node.asn] = { + threat_level: threat.threat_level, + confidence_score: threat.confidence_score, + source: threat.source, + }; + } + } catch (e) { + console.error(`[Topology Threat Lookup] Error checking ASN ${node.asn}:`, e.message); + } + }); + await Promise.race([ + Promise.all(threatPromises), + new Promise(r => setTimeout(r, 3000)), + ]); + + // Attach threat status to node objects + topology.nodes.forEach((node) => { + if (threatMap[node.asn]) { + node.threat_level = threatMap[node.asn].threat_level; + node.threat_confidence = threatMap[node.asn].confidence_score; + node.threat_source = threatMap[node.asn].source; + } + }); + + topology.meta = { + query: "AS" + rawAsn, depth: depth, duration_ms: Date.now() - start, + timestamp: new Date().toISOString(), node_count: topology.nodes.length, edge_count: topology.edges.length, + }; + return res.end(JSON.stringify(topology, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "Topology query failed", message: err.message })); + } + +} + +module.exports = { handleTopology }; diff --git a/server/routes/whois.js b/server/routes/whois.js new file mode 100644 index 0000000..43ad972 --- /dev/null +++ b/server/routes/whois.js @@ -0,0 +1,26 @@ +const { fetchWhois } = require("../whois"); + +// Feature 27: WHOIS endpoint +async function handleWhoisRoute(req, res, url) { + const resource = url.searchParams.get("resource") || ""; + if (!resource) { + res.writeHead(400); + return res.end(JSON.stringify({ error: "Missing resource parameter (ASN, prefix, or domain)" })); + } + const start = Date.now(); + try { + const whoisResult = await fetchWhois(resource); + if (!whoisResult || typeof whoisResult !== "object") { + res.writeHead(503); + return res.end(JSON.stringify({ error: "WHOIS data temporarily unavailable" })); + } + whoisResult.meta = { duration_ms: Date.now() - start, timestamp: new Date().toISOString() }; + return res.end(JSON.stringify(whoisResult, null, 2)); + } catch (err) { + res.writeHead(500); + return res.end(JSON.stringify({ error: "WHOIS lookup failed", message: err.message })); + } + +} + +module.exports = { handleWhoisRoute };