fix: prevent rate-limit 0-values under concurrent load

server.js: fetchPeeringDBWithRetry now does 3 attempts with exponential
backoff (2s, 5s) instead of 1 retry at 1.5s. Under audit load (9+
concurrent PDB requests), the longer delays let rate limits clear.

audit.py: stagger ASN submissions by 2s so PeerCortex's internal PDB
requests don't all fire simultaneously. Nightly audit takes ~8min
instead of 5min — acceptable for a midnight cron job.
This commit is contained in:
Rene Fichtmueller 2026-03-28 18:26:22 +13:00
parent a9ee94466e
commit 98b5cb1843
2 changed files with 16 additions and 6 deletions

View File

@ -414,7 +414,13 @@ def main():
results = []
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
futures = {pool.submit(_audit_asn, asn): asn for asn in batch}
# Stagger submissions by 2s so PeerCortex's internal PDB requests
# don't all fire simultaneously (9+ concurrent PDB calls → rate limit).
futures = {}
for idx, asn in enumerate(batch):
if idx > 0:
time.sleep(2)
futures[pool.submit(_audit_asn, asn)] = asn
for i, future in enumerate(as_completed(futures), 1):
asn = futures[future]
try:

View File

@ -279,12 +279,16 @@ function fetchPeeringDB(path, options) {
return fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } });
}
// PeeringDB fetch with one retry on failure (handles transient rate-limits)
// PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load).
// Up to 3 attempts: immediate → 2s → 5s. Returns null only after all attempts exhausted.
async function fetchPeeringDBWithRetry(path, options) {
const result = await fetchPeeringDB(path, options);
if (result !== null) return result;
await new Promise(r => setTimeout(r, 1500));
return fetchPeeringDB(path, options);
const delays = [2000, 5000];
let result = await fetchPeeringDB(path, options);
for (let i = 0; i < delays.length && result === null; i++) {
await new Promise(r => setTimeout(r, delays[i]));
result = await fetchPeeringDB(path, options);
}
return result;
}
// Generic JSON fetch with one retry — for sources that occasionally fail under load (RIPE Stat, Atlas)