From 98b5cb1843c307f36419f2846f38dac053efef8a Mon Sep 17 00:00:00 2001
From: Rene Fichtmueller <renefichtmueller@MacBook-Pro-von-Rene.local>
Date: Sat, 28 Mar 2026 18:26:22 +1300
Subject: [PATCH] fix: prevent rate-limit 0-values under concurrent load
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

server.js: fetchPeeringDBWithRetry now does 3 attempts with exponential
backoff (2s, 5s) instead of 1 retry at 1.5s. Under audit load (9+
concurrent PDB requests), the longer delays let rate limits clear.

audit.py: stagger ASN submissions by 2s so PeerCortex's internal PDB
requests don't all fire simultaneously. Nightly audit takes ~8min
instead of 5min — acceptable for a midnight cron job.
---
 audit/audit.py |  8 +++++++-
 server.js      | 14 +++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/audit/audit.py b/audit/audit.py
index 9d17f3e..cc76070 100644
--- a/audit/audit.py
+++ b/audit/audit.py
@@ -414,7 +414,13 @@ def main():
 
     results = []
     with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
-        futures = {pool.submit(_audit_asn, asn): asn for asn in batch}
+        # Stagger submissions by 2s so PeerCortex's internal PDB requests
+        # don't all fire simultaneously (9+ concurrent PDB calls → rate limit).
+        futures = {}
+        for idx, asn in enumerate(batch):
+            if idx > 0:
+                time.sleep(2)
+            futures[pool.submit(_audit_asn, asn)] = asn
         for i, future in enumerate(as_completed(futures), 1):
             asn = futures[future]
             try:
diff --git a/server.js b/server.js
index bf4b63d..96fc6c6 100644
--- a/server.js
+++ b/server.js
@@ -279,12 +279,16 @@ function fetchPeeringDB(path, options) {
   return fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } });
 }
 
-// PeeringDB fetch with one retry on failure (handles transient rate-limits)
+// PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load).
+// Up to 3 attempts: immediate → 2s → 5s. Returns null only after all attempts exhausted.
 async function fetchPeeringDBWithRetry(path, options) {
-  const result = await fetchPeeringDB(path, options);
-  if (result !== null) return result;
-  await new Promise(r => setTimeout(r, 1500));
-  return fetchPeeringDB(path, options);
+  const delays = [2000, 5000];
+  let result = await fetchPeeringDB(path, options);
+  for (let i = 0; i < delays.length && result === null; i++) {
+    await new Promise(r => setTimeout(r, delays[i]));
+    result = await fetchPeeringDB(path, options);
+  }
+  return result;
 }
 
 // Generic JSON fetch with one retry — for sources that occasionally fail under load (RIPE Stat, Atlas)