From e63723c2b000d247862196fc7b25367ec7716f51 Mon Sep 17 00:00:00 2001
From: Rene Fichtmueller <renefichtmueller@MacBook-Pro-von-Rene.local>
Date: Sat, 28 Mar 2026 10:54:39 +1300
Subject: [PATCH] =?UTF-8?q?fix:=20reliable=20data=20=E2=80=94=20retry=20Pe?=
 =?UTF-8?q?eringDB/RIPE=20Stat,=20limit=3D1000=20for=20IX,=20fallback=20wh?=
 =?UTF-8?q?en=20netId=3Dnull?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add fetchPeeringDBWithRetry: 1 retry with 1.5s delay on null response
- Add fetchJSONWithRetry: 1 retry for RIPE Stat prefixes + neighbours
- Log HTTP 429 from PeeringDB instead of silently swallowing it
- Add &limit=1000 to netixlan/netfac queries (prevents truncation at 250)
- Fall back to asn= / local_asn= queries when PeeringDB net lookup fails
  (previously: netId=null → IX=0, fac=0 for ~22 ASNs)
---
 server.js | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/server.js b/server.js
index 12460e5..c98a5cc 100644
--- a/server.js
+++ b/server.js
@@ -279,6 +279,22 @@ function fetchPeeringDB(path, options) {
   return fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } });
 }
 
+// PeeringDB fetch with one retry on failure (handles transient rate-limits)
+async function fetchPeeringDBWithRetry(path, options) {
+  const result = await fetchPeeringDB(path, options);
+  if (result !== null) return result;
+  await new Promise(r => setTimeout(r, 1500));
+  return fetchPeeringDB(path, options);
+}
+
+// Generic JSON fetch with one retry — for sources that occasionally fail under load (RIPE Stat, Atlas)
+async function fetchJSONWithRetry(url, options) {
+  const result = await fetchJSON(url, options);
+  if (result !== null) return result;
+  await new Promise(r => setTimeout(r, 1000));
+  return fetchJSON(url, options);
+}
+
 // bgproutes.io visibility fallback helper
 // Queries the RIB endpoint to estimate prefix visibility across vantage points
 function fetchBgproutesVisibility(prefix) {
@@ -340,6 +356,10 @@ function fetchJSON(url, options) {
         res.on("data", (chunk) => (data += chunk));
         res.on("end", () => {
           clearTimeout(timer);
+          if (res.statusCode === 429) {
+            console.warn("[PDB] Rate limited (429):", url.substring(0, 80));
+            return resolve(null);
+          }
           try {
             resolve(JSON.parse(data));
           } catch (_e) {
@@ -2055,22 +2075,31 @@ const server = http.createServer(async (req, res) => {
 
     try {
       // Phase 0: Get PDB net first (fast, <1s) to get net_id for IX/Fac queries
-      const pdbNet = await fetchPeeringDB("/net?asn=" + asn);
+      // Use retry to handle transient rate-limits
+      const pdbNet = await fetchPeeringDBWithRetry("/net?asn=" + asn);
       const net = pdbNet?.data?.[0] || {};
       const netId = net.id;
 
       // Phase 1: ALL calls in parallel — RIPE Stat + PDB IX/Fac + Atlas + bgp.he.net
+      // IX/Fac: use net_id when available (canonical), fall back to asn/local_asn filter
+      // &limit=1000 prevents truncation for large networks (default PeeringDB limit is 250)
+      const ixQuery = netId
+        ? "/netixlan?net_id=" + netId + "&limit=1000"
+        : "/netixlan?asn=" + asn + "&limit=1000";
+      const facQuery = netId
+        ? "/netfac?net_id=" + netId + "&limit=1000"
+        : "/netfac?local_asn=" + asn + "&limit=1000";
       const promises = [
-        fetchJSON("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }),
-        fetchJSON("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }),
+        fetchJSONWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }),
+        fetchJSONWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }),
         fetchJSON("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
         fetchJSON("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn),
         fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),
         fetchBgpHeNet(asn),
         fetchJSON("https://stat.ripe.net/data/visibility/data.json?resource=AS" + asn, { timeout: 30000 }),
         fetchJSON("https://stat.ripe.net/data/prefix-size-distribution/data.json?resource=AS" + asn),
-        netId ? fetchPeeringDB("/netixlan?net_id=" + netId) : Promise.resolve(null),
-        netId ? fetchPeeringDB("/netfac?net_id=" + netId) : Promise.resolve(null),
+        fetchPeeringDBWithRetry(ixQuery),
+        fetchPeeringDBWithRetry(facQuery),
       ];
       const [prefixData, neighbourData, overviewData, rirData, atlasProbeData, bgpHeData, visibilityData, prefixSizeData, ixlanData, facData] = await Promise.all(promises);