Compare commits

..

4 Commits

Author SHA1 Message Date
Rene Fichtmueller
6fb0eb86af feat: add local PeeringDB SQLite integration via peeringdb-py
- Install better-sqlite3 for zero-latency local queries
- queryPeeringDBLocal() handles all major PDB API paths locally:
  /net?asn=X, /netixlan, /netfac, /fac?id__in=, /ixfac, /ix, /ixlan
- fetchPeeringDB() now tries local SQLite first, falls back to live API
- Eliminates rate limits and reduces P99 response times dramatically
- Local DB synced daily at 03:48 via peeringdb-py cron on Erik
- Graceful fallback: if SQLite missing/corrupt, live API used transparently
2026-03-30 21:49:51 +02:00
Rene Fichtmueller
96b6ef2d4a feat: MANRS HTML scraping, AS relationships endpoint, rebrand to ASN News
- MANRS: replace broken Observatory API with public participants page scraping
  (www.manrs.org/netops/participants/), 24h cache, returns pass/fail with member count
- /api/validate: add 'relationships' field (upstreams/downstreams/top_peers)
  sourced from RIPE Stat asn-neighbours, no extra API calls needed
- /api/relationships?asn=X: new dedicated endpoint with resolved AS names,
  full upstream/downstream/peer lists sorted by power score, 10min cache
- editorial: rebrand 'The ASN Newspaper' → 'The ASN News' across index-editorial.html
2026-03-30 21:23:42 +02:00
Rene Fichtmueller
8f51f32dc3 fix: never cache null responses + increase RIPE Stat timeout for large carriers
Root cause of neighbour=0 for large carriers (AS9002, AS3491, AS12956):
1. RIPE Stat asn-neighbours returns 5000+ entries for Tier-1 carriers,
   exceeding the 30s timeout → fetchJSON returns null
2. null was cached in ripeStatCache for 15 minutes (the endpoint TTL)
3. All subsequent requests hit the null cache → perpetual 0 neighbours

Fixes:
- Never cache null results in ripeStatCache (only successful responses)
- Never persist null entries to disk cache
- Increase RIPE Stat timeout from 30s to 45s for prefix/neighbour queries
- Increase RIPE Stat semaphore from 10 to 15 concurrent requests

Verified: AS9002 up=146 down=2702, AS3491 up=90 down=710
2026-03-30 07:58:24 +02:00
Rene Fichtmueller
9bc1292bac fix: add rate-limiting semaphores to audit script
The audit script was flooding RIPE Stat and PeeringDB with unthrottled
parallel requests, causing 429 rate-limits that resulted in auth=0
false negatives (inflating the failure count).

Changes:
- Added threading.Semaphore for RIPE Stat (max 3) and PeeringDB (max 2)
- Added retry logic to _fetch_ripe (was fire-and-forget)
- Increased PDB retries from 2 to 3 with longer backoff (2s, 4s, 6s)
- Increased ASN stagger from 2s to 3s

Results: Accuracy 84% -> 87% (trend: 77% -> 87%, +10%)
2026-03-30 07:22:09 +02:00
6 changed files with 1552 additions and 55 deletions

11
.claude/launch.json Normal file
View File

@ -0,0 +1,11 @@
{
"version": "0.0.1",
"configurations": [
{
"name": "peercortex-redesign",
"runtimeExecutable": "npx",
"runtimeArgs": ["serve", "-p", "8902", "public"],
"port": 8902
}
]
}

View File

@ -12,7 +12,7 @@ Reports dir: /opt/peercortex-app/audit/reports/YYYY-MM-DD.json
Latest text: /opt/peercortex-app/audit/latest_report.txt Latest text: /opt/peercortex-app/audit/latest_report.txt
""" """
import json, os, sys, time, datetime, urllib.request, urllib.error import json, os, sys, time, datetime, urllib.request, urllib.error, threading
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path from pathlib import Path
@ -89,6 +89,13 @@ SEED_ASNS = [
1, 64512, 65000, 0, 4294967295, 1, 64512, 65000, 0, 4294967295,
] ]
# ─── Rate-limiting semaphores ────────────────────────────────────────────────
# Prevents 429 errors from RIPE Stat and PeeringDB by limiting concurrent requests.
# Without this, the audit floods both APIs and gets rate-limited, causing auth=0
# false negatives that inflate the failure count.
_ripe_sem = threading.Semaphore(3) # max 3 concurrent RIPE Stat requests
_pdb_sem = threading.Semaphore(2) # max 2 concurrent PeeringDB requests
# ─── HTTP helpers ───────────────────────────────────────────────────────────── # ─── HTTP helpers ─────────────────────────────────────────────────────────────
def _fetch(url, timeout=30, headers=None): def _fetch(url, timeout=30, headers=None):
"""GET url → parsed JSON dict, or None on any error.""" """GET url → parsed JSON dict, or None on any error."""
@ -101,24 +108,39 @@ def _fetch(url, timeout=30, headers=None):
except Exception: except Exception:
return None return None
def _fetch_pdb(path, timeout=30, retries=2): def _fetch_pdb(path, timeout=30, retries=3):
"""Fetch PeeringDB with API key and retry on 429 / failures.""" """Fetch PeeringDB with API key, semaphore throttling, and retry on 429."""
headers = {} headers = {}
if PEERINGDB_KEY: if PEERINGDB_KEY:
headers["Authorization"] = "Api-Key " + PEERINGDB_KEY headers["Authorization"] = "Api-Key " + PEERINGDB_KEY
url = PDB_BASE + path url = PDB_BASE + path
for attempt in range(retries + 1): for attempt in range(retries + 1):
_pdb_sem.acquire()
try:
result = _fetch(url, timeout=timeout, headers=headers) result = _fetch(url, timeout=timeout, headers=headers)
finally:
_pdb_sem.release()
if result is not None:
return result
if attempt < retries:
time.sleep(2.0 * (attempt + 1)) # 2s, 4s, 6s backoff
return None
def _fetch_ripe(endpoint, asn, timeout=30, retries=2):
"""Fetch RIPE Stat with semaphore throttling and retry on failure."""
url = f"{RIPE_BASE}/{endpoint}/data.json?resource=AS{asn}"
for attempt in range(retries + 1):
_ripe_sem.acquire()
try:
result = _fetch(url, timeout=timeout)
finally:
_ripe_sem.release()
if result is not None: if result is not None:
return result return result
if attempt < retries: if attempt < retries:
time.sleep(1.5 * (attempt + 1)) # 1.5s, 3s backoff time.sleep(1.5 * (attempt + 1)) # 1.5s, 3s backoff
return None return None
def _fetch_ripe(endpoint, asn, timeout=30):
url = f"{RIPE_BASE}/{endpoint}/data.json?resource=AS{asn}"
return _fetch(url, timeout=timeout)
def _fetch_pc(asn, timeout=90): def _fetch_pc(asn, timeout=90):
return _fetch(f"{PEERCORTEX_URL}/api/lookup?asn={asn}", timeout=timeout) return _fetch(f"{PEERCORTEX_URL}/api/lookup?asn={asn}", timeout=timeout)
@ -186,8 +208,7 @@ def _fetch_auth(asn):
net = ((pdb_net or {}).get("data") or [{}])[0] net = ((pdb_net or {}).get("data") or [{}])[0]
net_id = net.get("id") net_id = net.get("id")
# RIPE Stat (run in parallel via threads) # RIPE Stat + PDB IX/Fac (run in parallel via threads, throttled by semaphores)
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=4) as pool: with ThreadPoolExecutor(max_workers=4) as pool:
f_pfx = pool.submit(_fetch_ripe, "announced-prefixes", asn, TIMEOUT_AUTH) f_pfx = pool.submit(_fetch_ripe, "announced-prefixes", asn, TIMEOUT_AUTH)
f_nb = pool.submit(_fetch_ripe, "asn-neighbours", asn, TIMEOUT_AUTH) f_nb = pool.submit(_fetch_ripe, "asn-neighbours", asn, TIMEOUT_AUTH)
@ -414,12 +435,13 @@ def main():
results = [] results = []
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
# Stagger submissions by 2s so PeerCortex's internal PDB requests # Stagger submissions by 3s so PeerCortex's internal PDB requests
# don't all fire simultaneously (9+ concurrent PDB calls → rate limit). # don't all fire simultaneously (semaphore limits concurrent API calls,
# but staggering avoids burst pressure on rate-limit windows).
futures = {} futures = {}
for idx, asn in enumerate(batch): for idx, asn in enumerate(batch):
if idx > 0: if idx > 0:
time.sleep(2) time.sleep(3)
futures[pool.submit(_audit_asn, asn)] = asn futures[pool.submit(_audit_asn, asn)] = asn
for i, future in enumerate(as_completed(futures), 1): for i, future in enumerate(as_completed(futures), 1):
asn = futures[future] asn = futures[future]

View File

@ -445,7 +445,7 @@ class Semaphore {
if (this.queue.length > 0) { this.current++; this.queue.shift()(); } if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
} }
} }
const ripeStatSemaphore = new Semaphore(10); const ripeStatSemaphore = new Semaphore(15);
// Cached + throttled RIPE Stat fetch // Cached + throttled RIPE Stat fetch
async function fetchRipeStatCached(url, options) { async function fetchRipeStatCached(url, options) {
@ -474,11 +474,14 @@ async function fetchRipeStatCached(url, options) {
const result = await fetchJSON(url, options); const result = await fetchJSON(url, options);
// Store in cache (evict oldest if full) // Only cache successful results — never cache null (failed/rate-limited responses)
// Caching null causes cascading failures: retry hits cache, returns null again
if (result !== null) {
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
ripeStatCache.delete(ripeStatCache.keys().next().value); ripeStatCache.delete(ripeStatCache.keys().next().value);
} }
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
}
return result; return result;
} finally { } finally {
ripeStatSemaphore.release(); ripeStatSemaphore.release();
@ -489,17 +492,19 @@ async function fetchRipeStatCached(url, options) {
async function fetchRipeStatCachedWithRetry(url, options) { async function fetchRipeStatCachedWithRetry(url, options) {
const result = await fetchRipeStatCached(url, options); const result = await fetchRipeStatCached(url, options);
if (result !== null) return result; if (result !== null) return result;
await new Promise(r => setTimeout(r, 1000)); await new Promise(r => setTimeout(r, 1500));
return fetchRipeStatCached(url, options); return fetchRipeStatCached(url, options);
} }
// RIPE Stat cache disk persistence // RIPE Stat cache disk persistence (skip null entries)
function saveRipeStatCacheToDisk(filePath) { function saveRipeStatCacheToDisk(filePath) {
try { try {
const obj = {}; const obj = {};
for (const [k, v] of ripeStatCache) obj[k] = v; for (const [k, v] of ripeStatCache) {
if (v.data !== null) obj[k] = v;
}
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
} catch (e) { } catch (e) {
console.warn("[RIPE-CACHE] Disk save failed:", e.message); console.warn("[RIPE-CACHE] Disk save failed:", e.message);
} }
@ -2591,8 +2596,8 @@ const server = http.createServer(async (req, res) => {
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
const promises = [ const promises = [
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }),
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }),
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn),
fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),

File diff suppressed because it is too large Load Diff

View File

@ -3,7 +3,7 @@
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>PeerCortex — The ASN Newspaper</title> <title>PeerCortex — The ASN News</title>
<link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,700;0,800;0,900;1,400&family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet"> <link href="https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,700;0,800;0,900;1,400&family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
@ -337,9 +337,9 @@ a:hover{color:var(--purple)}
<div class="ed-masthead-top"> <div class="ed-masthead-top">
<div> <div>
<div class="ed-logo">PeerCortex<sup>β</sup></div> <div class="ed-logo">PeerCortex<sup>β</sup></div>
<div class="ed-tagline">The ASN Newspaper</div> <div class="ed-tagline">The ASN News</div>
</div> </div>
<div class="ed-masthead-meta">The ASN Newspaper<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div> <div class="ed-masthead-meta">The ASN News<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div>
</div> </div>
<hr class="ed-rule-h"> <hr class="ed-rule-h">
<nav class="ed-nav"> <nav class="ed-nav">
@ -677,7 +677,7 @@ a:hover{color:var(--purple)}
<!-- FOOTER --> <!-- FOOTER -->
<footer class="ed-footer"> <footer class="ed-footer">
<div class="ed-footer-name">PeerCortex — The ASN Newspaper</div> <div class="ed-footer-name">PeerCortex — The ASN News</div>
<nav class="ed-footer-links"> <nav class="ed-footer-links">
<a href="https://www.peeringdb.com" target="_blank">PeeringDB</a> <a href="https://www.peeringdb.com" target="_blank">PeeringDB</a>
<a href="https://stat.ripe.net" target="_blank">RIPE Stat</a> <a href="https://stat.ripe.net" target="_blank">RIPE Stat</a>

378
server.js
View File

@ -26,6 +26,170 @@ const BGPROUTES_API_URL = process.env.BGPROUTES_API_URL || "https://api.bgproute
const PEERINGDB_API_KEY = process.env.PEERINGDB_API_KEY || ""; const PEERINGDB_API_KEY = process.env.PEERINGDB_API_KEY || "";
const PEERINGDB_API_URL = process.env.PEERINGDB_API_URL || "https://www.peeringdb.com/api"; const PEERINGDB_API_URL = process.env.PEERINGDB_API_URL || "https://www.peeringdb.com/api";
// ── Local PeeringDB SQLite (peeringdb-py sync, refreshed daily by cron) ──────
const PEERINGDB_LOCAL_PATH = process.env.PEERINGDB_LOCAL_PATH || "/opt/peeringdb-data/peeringdb.sqlite3";
let _pdbLocal = null;
function getPdbLocal() {
if (_pdbLocal) return _pdbLocal;
try {
const BetterSqlite3 = require("better-sqlite3");
if (!fs.existsSync(PEERINGDB_LOCAL_PATH)) return null;
_pdbLocal = new BetterSqlite3(PEERINGDB_LOCAL_PATH, { readonly: true, fileMustExist: true });
console.log("[PeeringDB-local] SQLite opened:", PEERINGDB_LOCAL_PATH);
return _pdbLocal;
} catch (e) {
console.warn("[PeeringDB-local] Could not open SQLite:", e.message);
return null;
}
}
// Map API path → SQLite result in { data: [...] } format, emulating the live PDB REST API.
function queryPeeringDBLocal(path) {
const db = getPdbLocal();
if (!db) return null;
try {
// /net?asn=X
const netAsnMatch = path.match(/^\/net\?asn=(\d+)/);
if (netAsnMatch) {
const rows = db.prepare(
"SELECT n.*, o.name AS org_name FROM peeringdb_network n " +
"LEFT JOIN peeringdb_organization o ON n.org_id = o.id " +
"WHERE n.asn = ? AND n.status = 'ok'"
).all(parseInt(netAsnMatch[1]));
return { data: rows };
}
// /net?status=ok&depth=0 (coverage endpoint — all networks)
if (path === "/net?status=ok&depth=0" || path.startsWith("/net?status=ok")) {
const rows = db.prepare(
"SELECT id, asn, name, aka, website, info_prefixes4, info_prefixes6, " +
"info_type, info_traffic, info_unicast, info_ipv6, policy_general, org_id " +
"FROM peeringdb_network WHERE status = 'ok' ORDER BY asn"
).all();
return { data: rows };
}
// /netixlan?net_id=X&limit=... or /netixlan?asn=X&limit=...
const netixlanNetId = path.match(/\/netixlan\?net_id=(\d+)/);
if (netixlanNetId) {
const rows = db.prepare(
"SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " +
"ni.operational, ni.bfd_support, il.id AS ixlan_id, " +
"ix.id AS ix_id, ix.name, ix.city, ix.country " +
"FROM peeringdb_network_ixlan ni " +
"LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " +
"LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " +
"WHERE ni.net_id = ? AND ni.status = 'ok'"
).all(parseInt(netixlanNetId[1]));
return { data: rows };
}
const netixlanAsn = path.match(/\/netixlan\?asn=(\d+)/);
if (netixlanAsn) {
const rows = db.prepare(
"SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " +
"ni.operational, ni.bfd_support, il.id AS ixlan_id, " +
"ix.id AS ix_id, ix.name, ix.city, ix.country " +
"FROM peeringdb_network_ixlan ni " +
"LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " +
"LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " +
"WHERE ni.asn = ? AND ni.status = 'ok'"
).all(parseInt(netixlanAsn[1]));
return { data: rows };
}
// /netixlan?ixlan_id=X
const netixlanIxlanId = path.match(/\/netixlan\?ixlan_id=(\d+)/);
if (netixlanIxlanId) {
const rows = db.prepare(
"SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " +
"n.name AS net_name " +
"FROM peeringdb_network_ixlan ni " +
"LEFT JOIN peeringdb_network n ON ni.net_id = n.id " +
"WHERE ni.ixlan_id = ? AND ni.status = 'ok'"
).all(parseInt(netixlanIxlanId[1]));
return { data: rows };
}
// /netfac?net_id=X
const netfacNetId = path.match(/\/netfac\?net_id=(\d+)/);
if (netfacNetId) {
const rows = db.prepare(
"SELECT nf.id, nf.net_id, f.id AS fac_id, f.name, f.city, f.state, " +
"f.country, f.latitude, f.longitude, f.website " +
"FROM peeringdb_network_facility nf " +
"LEFT JOIN peeringdb_facility f ON nf.fac_id = f.id " +
"WHERE nf.net_id = ? AND nf.status = 'ok'"
).all(parseInt(netfacNetId[1]));
return { data: rows };
}
// /fac?id__in=X,Y,Z&fields=...
const facIdIn = path.match(/\/fac\?id__in=([\d,]+)/);
if (facIdIn) {
const ids = facIdIn[1].split(",").map(Number).filter(Boolean);
if (ids.length === 0) return { data: [] };
const placeholders = ids.map(() => "?").join(",");
const rows = db.prepare(
"SELECT id, name, city, country, latitude, longitude, website " +
"FROM peeringdb_facility WHERE id IN (" + placeholders + ") AND status = 'ok'"
).all(...ids);
return { data: rows };
}
// /ixfac?ix_id__in=X,Y,Z
const ixfacIxIdIn = path.match(/\/ixfac\?ix_id__in=([\d,]+)/);
if (ixfacIxIdIn) {
const ids = ixfacIxIdIn[1].split(",").map(Number).filter(Boolean);
if (ids.length === 0) return { data: [] };
const placeholders = ids.map(() => "?").join(",");
const rows = db.prepare(
"SELECT ixf.id, ixf.ix_id, ixf.fac_id, f.latitude, f.longitude, f.city, f.country " +
"FROM peeringdb_ix_facility ixf " +
"LEFT JOIN peeringdb_facility f ON ixf.fac_id = f.id " +
"WHERE ixf.ix_id IN (" + placeholders + ") AND ixf.status = 'ok'"
).all(...ids);
return { data: rows };
}
// /ix?name__contains=X
const ixNameContains = path.match(/\/ix\?name__contains=([^&]+)/);
if (ixNameContains) {
const term = "%" + decodeURIComponent(ixNameContains[1]) + "%";
const rows = db.prepare(
"SELECT id, name, name_long, city, country, website, region_continent " +
"FROM peeringdb_ix WHERE (name LIKE ? OR name_long LIKE ?) AND status = 'ok' LIMIT 20"
).all(term, term);
return { data: rows };
}
// /ixlan?ix_id=X
const ixlanIxId = path.match(/\/ixlan\?ix_id=(\d+)/);
if (ixlanIxId) {
const rows = db.prepare(
"SELECT id, ix_id, name, rs_asn, arp_sponge, mtu FROM peeringdb_ixlan " +
"WHERE ix_id = ? AND status = 'ok'"
).all(parseInt(ixlanIxId[1]));
return { data: rows };
}
// /net/X (single network by PDB id)
const netById = path.match(/^\/net\/(\d+)$/);
if (netById) {
const row = db.prepare(
"SELECT n.*, o.name AS org_name FROM peeringdb_network n " +
"LEFT JOIN peeringdb_organization o ON n.org_id = o.id " +
"WHERE n.id = ? AND n.status = 'ok'"
).get(parseInt(netById[1]));
return row ? { data: [row] } : { data: [] };
}
return null; // path not handled locally — fall through to live API
} catch (e) {
console.warn("[PeeringDB-local] Query error for", path, ":", e.message);
return null;
}
}
const FEEDBACK_TOKEN = process.env.FEEDBACK_TOKEN || "changeme-set-in-env"; const FEEDBACK_TOKEN = process.env.FEEDBACK_TOKEN || "changeme-set-in-env";
const FEEDBACK_FILE = "/opt/peercortex-app/feedback.json"; const FEEDBACK_FILE = "/opt/peercortex-app/feedback.json";
@ -87,6 +251,69 @@ const CACHE_TTL_ASPA = 4 * 60 * 60 * 1000; // 4 hours
const CACHE_TTL_NEWS = 10 * 60 * 1000; // 10 minutes const CACHE_TTL_NEWS = 10 * 60 * 1000; // 10 minutes
const CACHE_TTL_DEFAULT = 5 * 60 * 1000; // 5 minutes const CACHE_TTL_DEFAULT = 5 * 60 * 1000; // 5 minutes
// ============================================================
// MANRS Participants Cache (scraped from public HTML page, 24h TTL)
// ============================================================
let manrsAsnSet = null; // Set<string> of member ASNs
let manrsLastFetch = 0;
let manrsFetching = false;
const MANRS_CACHE_TTL = 24 * 60 * 60 * 1000; // 24 hours
async function ensureManrsCache() {
const now = Date.now();
if (manrsAsnSet && (now - manrsLastFetch) < MANRS_CACHE_TTL) return;
if (manrsFetching) {
// Wait up to 8s for in-progress fetch
for (let i = 0; i < 80; i++) {
await new Promise(r => setTimeout(r, 100));
if (manrsAsnSet) return;
}
return;
}
manrsFetching = true;
try {
const html = await new Promise((resolve, reject) => {
const opts = { hostname: "www.manrs.org", path: "/netops/participants/", method: "GET", timeout: 15000,
headers: { "User-Agent": UA, "Accept": "text/html" } };
const req = https.request(opts, res => {
let body = "";
res.on("data", d => { body += d; });
res.on("end", () => resolve(body));
});
req.on("error", reject);
req.on("timeout", () => { req.destroy(); reject(new Error("MANRS fetch timeout")); });
req.end();
});
// Extract ASNs from <td class="asns">267490</td> — may contain multiple space-separated ASNs
const set = new Set();
const re = /<td[^>]*class="asns"[^>]*>([\d\s,]+)<\/td>/gi;
let m;
while ((m = re.exec(html)) !== null) {
m[1].split(/[\s,]+/).forEach(a => { const n = a.trim(); if (n) set.add(n); });
}
if (set.size > 0) {
manrsAsnSet = set;
manrsLastFetch = Date.now();
console.log("[MANRS] Loaded " + set.size + " participant ASNs from manrs.org");
}
} catch (e) {
console.warn("[MANRS] Failed to fetch participants:", e.message);
} finally {
manrsFetching = false;
}
}
function checkManrsMembership(asn) {
if (!manrsAsnSet) return { status: "info", participant: "unknown", message: "MANRS data not yet loaded", note: "https://www.manrs.org/netops/participants/" };
const isMember = manrsAsnSet.has(String(asn));
return {
status: isMember ? "pass" : "fail",
participant: isMember,
member_count: manrsAsnSet.size,
note: isMember ? "Confirmed MANRS Network Operator participant" : "Not listed as MANRS participant — https://www.manrs.org/beamanrs/",
};
}
// ============================================================ // ============================================================
// Infrastructure overlay caches // Infrastructure overlay caches
let subCableCache = null; // TeleGeography submarine cables (24h) let subCableCache = null; // TeleGeography submarine cables (24h)
@ -445,7 +672,7 @@ class Semaphore {
if (this.queue.length > 0) { this.current++; this.queue.shift()(); } if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
} }
} }
const ripeStatSemaphore = new Semaphore(10); const ripeStatSemaphore = new Semaphore(15);
// Cached + throttled RIPE Stat fetch // Cached + throttled RIPE Stat fetch
async function fetchRipeStatCached(url, options) { async function fetchRipeStatCached(url, options) {
@ -474,11 +701,14 @@ async function fetchRipeStatCached(url, options) {
const result = await fetchJSON(url, options); const result = await fetchJSON(url, options);
// Store in cache (evict oldest if full) // Only cache successful results — never cache null (failed/rate-limited responses)
// Caching null causes cascading failures: retry hits cache, returns null again
if (result !== null) {
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) { if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
ripeStatCache.delete(ripeStatCache.keys().next().value); ripeStatCache.delete(ripeStatCache.keys().next().value);
} }
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() }); ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
}
return result; return result;
} finally { } finally {
ripeStatSemaphore.release(); ripeStatSemaphore.release();
@ -489,17 +719,19 @@ async function fetchRipeStatCached(url, options) {
async function fetchRipeStatCachedWithRetry(url, options) { async function fetchRipeStatCachedWithRetry(url, options) {
const result = await fetchRipeStatCached(url, options); const result = await fetchRipeStatCached(url, options);
if (result !== null) return result; if (result !== null) return result;
await new Promise(r => setTimeout(r, 1000)); await new Promise(r => setTimeout(r, 1500));
return fetchRipeStatCached(url, options); return fetchRipeStatCached(url, options);
} }
// RIPE Stat cache disk persistence // RIPE Stat cache disk persistence (skip null entries)
function saveRipeStatCacheToDisk(filePath) { function saveRipeStatCacheToDisk(filePath) {
try { try {
const obj = {}; const obj = {};
for (const [k, v] of ripeStatCache) obj[k] = v; for (const [k, v] of ripeStatCache) {
if (v.data !== null) obj[k] = v;
}
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj })); fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
console.log("[RIPE-CACHE] Saved " + ripeStatCache.size + " entries to disk"); console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
} catch (e) { } catch (e) {
console.warn("[RIPE-CACHE] Disk save failed:", e.message); console.warn("[RIPE-CACHE] Disk save failed:", e.message);
} }
@ -604,8 +836,13 @@ function lookupAspaFromRpki(asn) {
// PeeringDB semaphore — limits concurrent PDB requests to avoid 429 rate-limits // PeeringDB semaphore — limits concurrent PDB requests to avoid 429 rate-limits
const pdbSemaphore = new Semaphore(5); const pdbSemaphore = new Semaphore(5);
// PeeringDB authenticated fetch helper (throttled via semaphore) // PeeringDB authenticated fetch helper — tries local SQLite first, falls back to live API
async function fetchPeeringDB(path, options) { async function fetchPeeringDB(path, options) {
// Try local SQLite (instant, no rate-limits) — skip large "all networks" calls to live API
const localResult = queryPeeringDBLocal(path);
if (localResult !== null) return localResult;
// Fallback: live PeeringDB API (throttled via semaphore)
const url = PEERINGDB_API_URL + path; const url = PEERINGDB_API_URL + path;
const headers = { "User-Agent": UA }; const headers = { "User-Agent": UA };
if (PEERINGDB_API_KEY) { if (PEERINGDB_API_KEY) {
@ -2244,20 +2481,10 @@ const server = http.createServer(async (req, res) => {
return { status: listedPrefixes.length === 0 ? "pass" : "fail", checked: results.length, listed_prefixes: listedPrefixes }; return { status: listedPrefixes.length === 0 ? "pass" : "fail", checked: results.length, listed_prefixes: listedPrefixes };
}).catch(function(e) { return { status: "error", error: String(e) }; }); }).catch(function(e) { return { status: "error", error: String(e) }; });
// 16. MANRS Compliance (observatory API requires auth — use fallback indicators) // 16. MANRS Compliance — scraped from public participants list (24h cache)
validationPromises.manrs = fetchJSON("https://observatory.manrs.org/api/v2/asn/" + rawAsn + "/conformance", { timeout: 5000 }).then(function(data) { validationPromises.manrs = ensureManrsCache().then(function() {
if (!data || data.error || data.detail === "Authentication credentials were not provided.") { return checkManrsMembership(rawAsn);
// API unavailable — check MANRS indicators: RPKI ROA + IRR objects as proxy }).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable: " + e.message, note: "https://www.manrs.org/netops/participants/" }; });
var hasRoa = samplePrefixes.length > 0; // will be checked by RPKI validation
var hasIrr = !!(net.irr_as_set);
if (hasRoa && hasIrr) {
return { status: "info", participant: "unknown", message: "MANRS Observatory API requires authentication — cannot verify membership. Network has ROA + IRR objects (positive indicators).", note: "Unable to verify — MANRS API requires auth. Check https://observatory.manrs.org/asn/" + rawAsn };
}
return { status: "info", participant: "unknown", message: "Unable to verify MANRS membership (API requires authentication)", note: "Check manually: https://observatory.manrs.org/asn/" + rawAsn };
}
var score = data.conformance_score || data.score || 0;
return { status: score >= 50 ? "pass" : "warning", participant: true, score: score, details: data };
}).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable", note: "https://observatory.manrs.org/asn/" + rawAsn }; });
// 17. Reverse DNS Coverage (sample up to 20 prefixes for better coverage) // 17. Reverse DNS Coverage (sample up to 20 prefixes for better coverage)
var rdnsSampleSize = Math.min(20, samplePrefixes.length); var rdnsSampleSize = Math.min(20, samplePrefixes.length);
@ -2532,6 +2759,24 @@ const server = http.createServer(async (req, res) => {
var healthScore = totalWeight > 0 ? Math.round((earnedScore / totalWeight) * 100) : 0; var healthScore = totalWeight > 0 ? Math.round((earnedScore / totalWeight) * 100) : 0;
var duration = Date.now() - start; var duration = Date.now() - start;
// Build relationships from neighbour data
var relNeighbours = neighbourData && neighbourData.data && neighbourData.data.neighbour_counts
? neighbourData.data.neighbour_counts : {};
var relList = neighbourData && neighbourData.data && neighbourData.data.neighbours
? neighbourData.data.neighbours : [];
var relUpstreams = relList.filter(function(n) { return n.type === "left"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 20)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
var relDownstreams = relList.filter(function(n) { return n.type === "right"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 20)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
var relPeers = relList.filter(function(n) { return n.type === "uncertain"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 30)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
return res.end( return res.end(
JSON.stringify( JSON.stringify(
{ {
@ -2541,6 +2786,14 @@ const server = http.createServer(async (req, res) => {
health_score: healthScore, health_score: healthScore,
score_breakdown: checkResults, score_breakdown: checkResults,
validations: validations, validations: validations,
relationships: {
counts: { upstreams: relNeighbours.left || relUpstreams.length, downstreams: relNeighbours.right || relDownstreams.length, peers: relNeighbours.unique || relPeers.length, uncertain: relNeighbours.uncertain || 0 },
upstreams: relUpstreams,
downstreams: relDownstreams,
top_peers: relPeers,
source: "RIPE Stat asn-neighbours",
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
},
}, },
null, null,
2 2
@ -2591,8 +2844,8 @@ const server = http.createServer(async (req, res) => {
let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null; let cachedFac = netId ? pdbSourceCache.get("netfac", String(netId)) : null;
const promises = [ const promises = [
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 30000 }), fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS" + asn, { timeout: 45000 }),
fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 30000 }), fetchRipeStatCachedWithRetry("https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + asn, { timeout: 45000 }),
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + asn),
fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn), fetchRipeStatCached("https://stat.ripe.net/data/rir-stats-country/data.json?resource=AS" + asn),
fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"), fetchJSON("https://atlas.ripe.net/api/v2/probes/?asn_v4=" + asn + "&page_size=500"),
@ -3031,6 +3284,82 @@ const server = http.createServer(async (req, res) => {
return; return;
} }
// ============================================================
// AS Relationships endpoint: /api/relationships?asn=X
// Returns upstream providers, downstream customers, and peers
// with resolved names. Based on RIPE Stat asn-neighbours.
// ============================================================
if (reqPath === "/api/relationships") {
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
if (!rawAsn) {
res.writeHead(400);
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
}
const cacheKey = "relationships:" + rawAsn;
const cached = cacheGet(cacheKey);
if (cached) {
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
return res.end(JSON.stringify(cached));
}
const start = Date.now();
try {
const neighbourData = await fetchRipeStatCached(
"https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1",
{ timeout: 30000 }
);
const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || [];
const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {};
const upstreams = neighbours.filter(n => n.type === "left").sort((a,b) => (b.power||0)-(a.power||0));
const downstreams = neighbours.filter(n => n.type === "right").sort((a,b) => (b.power||0)-(a.power||0));
const peers = neighbours.filter(n => n.type === "uncertain").sort((a,b) => (b.power||0)-(a.power||0));
// Resolve AS names for top entries (upstreams + downstreams all, top 20 peers)
const toResolve = [...upstreams, ...downstreams, ...peers.slice(0, 20)];
const resolvedNames = {};
await Promise.race([
Promise.all(toResolve.map(n =>
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 })
.then(r => { if (r && r.data && r.data.holder) resolvedNames[n.asn] = r.data.holder; })
.catch(() => {})
)),
new Promise(r => setTimeout(r, 5000)),
]);
const fmt = n => ({
asn: n.asn,
name: resolvedNames[n.asn] || "",
power: n.power || 0,
v4_peers: n.v4_peers || 0,
v6_peers: n.v6_peers || 0,
});
const result = {
asn: parseInt(rawAsn),
query_time: new Date().toISOString(),
duration_ms: Date.now() - start,
counts: {
upstreams: counts.left || upstreams.length,
downstreams: counts.right || downstreams.length,
peers_total: counts.unique || peers.length,
uncertain: counts.uncertain || peers.length,
},
upstreams: upstreams.map(fmt),
downstreams: downstreams.map(fmt),
peers: peers.slice(0, 50).map(fmt),
methodology: "RIPE Stat asn-neighbours API. left=upstream providers (carry your traffic), right=downstream customers (you carry their traffic), uncertain=lateral peers. Sorted by power score (number of prefixes seen via this relationship).",
source_url: "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn,
};
cacheSet(cacheKey, result, 10 * 60 * 1000); // 10 min cache
res.writeHead(200, { "Content-Type": "application/json" });
return res.end(JSON.stringify(result, null, 2));
} catch (err) {
res.writeHead(500);
return res.end(JSON.stringify({ error: "Relationships lookup failed", message: err.message }));
}
}
// ============================================================ // ============================================================
// Compare endpoint: /api/compare?asn1=X&asn2=Y // Compare endpoint: /api/compare?asn1=X&asn2=Y
// ============================================================ // ============================================================
@ -3783,7 +4112,8 @@ roaStore.loadFromDisk("/opt/peercortex-app/.roa-cache.json");
pdbSourceCache.loadFromDisk("/opt/peercortex-app/.pdb-source-cache.json"); pdbSourceCache.loadFromDisk("/opt/peercortex-app/.pdb-source-cache.json");
loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json"); loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json");
// Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries // Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries + MANRS participants
ensureManrsCache(); // fire-and-forget, 24h cache
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => { Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
server.listen(PORT, "0.0.0.0", () => { server.listen(PORT, "0.0.0.0", () => {
console.log("PeerCortex v0.6.0 running on http://0.0.0.0:" + PORT); console.log("PeerCortex v0.6.0 running on http://0.0.0.0:" + PORT);