Compare commits

..

2 Commits

Author SHA1 Message Date
Rene Fichtmueller
35e0b69442 fix: enrich - skip disambiguation pages, try first-word fallback for compound names 2026-03-30 06:04:34 +02:00
Rene Fichtmueller
0cebb1973f fix: add PeeringDB semaphore (max 5 concurrent) to prevent 429 rate-limits
Previously PDB requests fired in parallel without throttling, causing
rate-limit cascades under audit load. Now all fetchPeeringDB calls
go through a counting semaphore (max 5 concurrent requests).

Results:
- Zero 429 errors in clean test
- AS6939 HE: 327 IX connections (was 0), 338 facilities (was 2)
- AS13335 CF: 413 IX, 221 facilities, 5600 prefixes (94% RPKI valid)
- Audit: 84% accuracy (82% -> 84%, +2%), trend positive
2026-03-30 06:04:24 +02:00
2 changed files with 24 additions and 14 deletions

View File

@ -3480,15 +3480,20 @@ const server = http.createServer(async (req, res) => {
return titleMatch && (hasNetContext || titleMatch);
};
// Direct title lookup
const wikiDirect = await fetchJSON(
"https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim()),
{ timeout: 5000 }
);
if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30) {
if (isRelevant(wikiDirect.title, wikiDirect.extract)) {
// Direct title lookup — try full name first, then first word as fallback
const cleanName = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim();
const firstName = cleanName.split(/\s+/)[0];
const namesToTry = cleanName === firstName ? [cleanName] : [cleanName, firstName];
for (const tryName of namesToTry) {
const wikiDirect = await fetchJSON(
"https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(tryName),
{ timeout: 5000 }
);
if (wikiDirect && wikiDirect.type === "disambiguation") continue; // skip disambiguation pages
if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30 && isRelevant(wikiDirect.title, wikiDirect.extract)) {
description = wikiDirect.extract.replace(/\s+/g, " ").trim().slice(0, 300);
wikiUrl = wikiDirect.content_urls && wikiDirect.content_urls.desktop && wikiDirect.content_urls.desktop.page;
break;
}
}

View File

@ -3480,15 +3480,20 @@ const server = http.createServer(async (req, res) => {
return titleMatch && (hasNetContext || titleMatch);
};
// Direct title lookup
const wikiDirect = await fetchJSON(
"https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim()),
{ timeout: 5000 }
);
if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30) {
if (isRelevant(wikiDirect.title, wikiDirect.extract)) {
// Direct title lookup — try full name first, then first word as fallback
const cleanName = companyName.replace(/\s+(GmbH|AG|Ltd|Inc|LLC|BV|SA|SAS|Oy|AB)$/i, "").trim();
const firstName = cleanName.split(/\s+/)[0];
const namesToTry = cleanName === firstName ? [cleanName] : [cleanName, firstName];
for (const tryName of namesToTry) {
const wikiDirect = await fetchJSON(
"https://en.wikipedia.org/api/rest_v1/page/summary/" + encodeURIComponent(tryName),
{ timeout: 5000 }
);
if (wikiDirect && wikiDirect.type === "disambiguation") continue; // skip disambiguation pages
if (wikiDirect && wikiDirect.extract && wikiDirect.extract.length > 30 && isRelevant(wikiDirect.title, wikiDirect.extract)) {
description = wikiDirect.extract.replace(/\s+/g, " ").trim().slice(0, 300);
wikiUrl = wikiDirect.content_urls && wikiDirect.content_urls.desktop && wikiDirect.content_urls.desktop.page;
break;
}
}