fix: prevent rate-limit 0-values under concurrent load
server.js: fetchPeeringDBWithRetry now does 3 attempts with exponential backoff (2s, 5s) instead of 1 retry at 1.5s. Under audit load (9+ concurrent PDB requests), the longer delays let rate limits clear. audit.py: stagger ASN submissions by 2s so PeerCortex's internal PDB requests don't all fire simultaneously. Nightly audit takes ~8min instead of 5min — acceptable for a midnight cron job.
This commit is contained in:
parent
a9ee94466e
commit
98b5cb1843
@ -414,7 +414,13 @@ def main():
|
||||
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
||||
futures = {pool.submit(_audit_asn, asn): asn for asn in batch}
|
||||
# Stagger submissions by 2s so PeerCortex's internal PDB requests
|
||||
# don't all fire simultaneously (9+ concurrent PDB calls → rate limit).
|
||||
futures = {}
|
||||
for idx, asn in enumerate(batch):
|
||||
if idx > 0:
|
||||
time.sleep(2)
|
||||
futures[pool.submit(_audit_asn, asn)] = asn
|
||||
for i, future in enumerate(as_completed(futures), 1):
|
||||
asn = futures[future]
|
||||
try:
|
||||
|
||||
14
server.js
14
server.js
@ -279,12 +279,16 @@ function fetchPeeringDB(path, options) {
|
||||
return fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } });
|
||||
}
|
||||
|
||||
// PeeringDB fetch with one retry on failure (handles transient rate-limits)
|
||||
// PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load).
|
||||
// Up to 3 attempts: immediate → 2s → 5s. Returns null only after all attempts exhausted.
|
||||
async function fetchPeeringDBWithRetry(path, options) {
|
||||
const result = await fetchPeeringDB(path, options);
|
||||
if (result !== null) return result;
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
return fetchPeeringDB(path, options);
|
||||
const delays = [2000, 5000];
|
||||
let result = await fetchPeeringDB(path, options);
|
||||
for (let i = 0; i < delays.length && result === null; i++) {
|
||||
await new Promise(r => setTimeout(r, delays[i]));
|
||||
result = await fetchPeeringDB(path, options);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Generic JSON fetch with one retry — for sources that occasionally fail under load (RIPE Stat, Atlas)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user