refactor: extract aspa-adoption/tracker.js, services/ripe-stat.js, services/rpki.js
Real dependency graph was denser than the initial plan assumed: - recordAspaAdoptionSnapshot needed atlasState (atlas-probes.js) and rpkiAspaMap (rpki.js) -- threaded rpkiAspaMap through as an explicit parameter instead of a rpki.js<->tracker.js circular require. - atlasProbeCache-style reassignment problem recurred for rpkiAspaLastFetch (a `let`, not a mutated object) -- exposed via a getRpkiAspaLastFetch() getter, same fix pattern as atlas-probes.js's atlasState wrapper. - ripe-stat.js's master fetchRipeStatCached dispatcher lives ~600 lines away from its own cache/semaphore/fallback internals in the original file; kept together in one module per the extraction plan (splitting these apart previously caused the duplicate-declaration bug fixed earlier this session). Also drops confirmed-dead subCableCache/globalFacCache (declared, never read). Verified via smoke-test harness (28/28 match). server.js: 5622 -> 5018 lines.
This commit is contained in:
Rene Fichtmueller
parent
63ace5d87a
commit
3ce478953a
674
server.js
674
server.js
@ -266,293 +266,12 @@ const {
|
|||||||
const { webhookSignature, deliverWebhook, notifyWebhooks } = require("./server/hijack-monitoring/webhooks");
|
const { webhookSignature, deliverWebhook, notifyWebhooks } = require("./server/hijack-monitoring/webhooks");
|
||||||
const { classifyHijackSeverity, checkHijacksForAsn, runHijackCheck } = require("./server/hijack-monitoring/monitor");
|
const { classifyHijackSeverity, checkHijacksForAsn, runHijackCheck } = require("./server/hijack-monitoring/monitor");
|
||||||
|
|
||||||
const ASPA_ADOPTION_FILE = DATA_DIR + '/aspa-adoption-history.json';
|
const {
|
||||||
|
aspaAdoptionDailyHistory,
|
||||||
// ============================================================
|
recordAspaAdoptionSnapshot,
|
||||||
// FEATURE 3: ASPA Adoption Tracker
|
getAdoptionTrend,
|
||||||
// ============================================================
|
buildAspaAdoptionDashboard,
|
||||||
|
} = require("./server/aspa-adoption/tracker");
|
||||||
/**
|
|
||||||
* Load persisted ASPA adoption history from disk.
|
|
||||||
* Returns array of snapshot objects (up to 365 entries).
|
|
||||||
*/
|
|
||||||
function loadAspaAdoptionHistory() {
|
|
||||||
try {
|
|
||||||
const raw = JSON.parse(fs.readFileSync(ASPA_ADOPTION_FILE, 'utf8'));
|
|
||||||
return Array.isArray(raw) ? raw : [];
|
|
||||||
} catch(_) { return []; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Save ASPA adoption history to disk (keep last 365 entries).
|
|
||||||
*/
|
|
||||||
function saveAspaAdoptionHistory(history) {
|
|
||||||
try {
|
|
||||||
const trimmed = history.slice(-365);
|
|
||||||
fs.writeFileSync(ASPA_ADOPTION_FILE, JSON.stringify(trimmed, null, 2), 'utf8');
|
|
||||||
} catch(e) {
|
|
||||||
console.error('[ASPA-ADOPT] Save failed:', e.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** In-memory adoption history (persisted on each new snapshot) */
|
|
||||||
let aspaAdoptionDailyHistory = loadAspaAdoptionHistory();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Take a new adoption snapshot and persist it.
|
|
||||||
* Called whenever the RPKI feed is refreshed.
|
|
||||||
* @param {number} aspaCount — total ASPA objects in rpkiAspaMap
|
|
||||||
* @param {number} roaCount — total ROAs in roaStore
|
|
||||||
*/
|
|
||||||
function recordAspaAdoptionSnapshot(aspaCount, roaCount) {
|
|
||||||
const now = new Date();
|
|
||||||
const date = now.toISOString().slice(0, 10); // "YYYY-MM-DD"
|
|
||||||
|
|
||||||
// Compute how many Atlas-visible ASNs have ASPA
|
|
||||||
let atlasTotal = 0;
|
|
||||||
let atlasWithAspa = 0;
|
|
||||||
if (atlasState.cache?.asns_with_probes) {
|
|
||||||
atlasTotal = atlasState.cache.asns_with_probes.length;
|
|
||||||
// rpkiAspaMap is keyed by integer ASN
|
|
||||||
atlasWithAspa = atlasState.cache.asns_with_probes.filter(a => rpkiAspaMap.has(Number(a))).length;
|
|
||||||
}
|
|
||||||
|
|
||||||
const coveragePct = atlasTotal > 0
|
|
||||||
? Math.round((atlasWithAspa / atlasTotal) * 10000) / 100 // 2 decimal places
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
// Compute delta from previous snapshot
|
|
||||||
const prev = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1];
|
|
||||||
const delta = prev ? aspaCount - prev.aspa_objects : 0;
|
|
||||||
|
|
||||||
const snapshot = {
|
|
||||||
date,
|
|
||||||
ts: now.getTime(),
|
|
||||||
aspa_objects: aspaCount,
|
|
||||||
roa_count: roaCount,
|
|
||||||
atlas_asns_total: atlasTotal,
|
|
||||||
atlas_asns_with_aspa: atlasWithAspa,
|
|
||||||
coverage_pct_atlas: coveragePct,
|
|
||||||
aspa_delta: delta,
|
|
||||||
method: 'rpki_cloudflare_feed',
|
|
||||||
};
|
|
||||||
|
|
||||||
// Only store one snapshot per date (overwrite if same day)
|
|
||||||
const existingIdx = aspaAdoptionDailyHistory.findIndex(s => s.date === date);
|
|
||||||
if (existingIdx >= 0) {
|
|
||||||
aspaAdoptionDailyHistory[existingIdx] = snapshot;
|
|
||||||
} else {
|
|
||||||
aspaAdoptionDailyHistory.push(snapshot);
|
|
||||||
}
|
|
||||||
saveAspaAdoptionHistory(aspaAdoptionDailyHistory);
|
|
||||||
console.log(`[ASPA-ADOPT] Snapshot ${date}: ${aspaCount} objects, ${coveragePct}% Atlas coverage (${atlasWithAspa}/${atlasTotal})`);
|
|
||||||
return snapshot;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get adoption trend for a given period string ('7d', '30d', '90d', '1y').
|
|
||||||
*/
|
|
||||||
function getAdoptionTrend(period) {
|
|
||||||
const days = period === '7d' ? 7 : period === '30d' ? 30 : period === '90d' ? 90 : 365;
|
|
||||||
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
|
||||||
return aspaAdoptionDailyHistory.filter(s => s.ts >= cutoff);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build the ASPA adoption dashboard HTML.
|
|
||||||
*/
|
|
||||||
function buildAspaAdoptionDashboard() {
|
|
||||||
const latest = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1];
|
|
||||||
const trend30 = getAdoptionTrend('30d');
|
|
||||||
const trend7 = getAdoptionTrend('7d');
|
|
||||||
|
|
||||||
// Forecast: simple linear regression over last 30 days
|
|
||||||
function linearForecast(data, daysAhead) {
|
|
||||||
if (data.length < 2) return null;
|
|
||||||
const n = data.length;
|
|
||||||
const xs = data.map((_, i) => i);
|
|
||||||
const ys = data.map(d => d.coverage_pct_atlas);
|
|
||||||
const xMean = xs.reduce((a, b) => a + b, 0) / n;
|
|
||||||
const yMean = ys.reduce((a, b) => a + b, 0) / n;
|
|
||||||
const num = xs.reduce((s, x, i) => s + (x - xMean) * (ys[i] - yMean), 0);
|
|
||||||
const den = xs.reduce((s, x) => s + (x - xMean) ** 2, 0);
|
|
||||||
if (den === 0) return yMean;
|
|
||||||
const slope = num / den;
|
|
||||||
return Math.max(0, Math.min(100, yMean + slope * (n - 1 + daysAhead)));
|
|
||||||
}
|
|
||||||
|
|
||||||
const forecast90d = linearForecast(trend30, 90);
|
|
||||||
const change7d = trend7.length >= 2
|
|
||||||
? (trend7[trend7.length-1].aspa_objects - trend7[0].aspa_objects)
|
|
||||||
: 0;
|
|
||||||
const trendLabels = trend30.map(d => d.date);
|
|
||||||
const trendValues = trend30.map(d => d.coverage_pct_atlas);
|
|
||||||
const asnValues = trend30.map(d => d.aspa_objects);
|
|
||||||
|
|
||||||
const coverageNow = latest?.coverage_pct_atlas ?? 0;
|
|
||||||
const aspaNow = latest?.aspa_objects ?? 0;
|
|
||||||
const atlasTotal = latest?.atlas_asns_total ?? 0;
|
|
||||||
|
|
||||||
return `<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8"/>
|
|
||||||
<meta name="viewport" content="width=device-width,initial-scale=1"/>
|
|
||||||
<title>ASPA Adoption Tracker · PeerCortex</title>
|
|
||||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
|
||||||
<style>
|
|
||||||
*,*::before,*::after{box-sizing:border-box;margin:0;padding:0}
|
|
||||||
body{font-family:-apple-system,'Segoe UI',system-ui,sans-serif;background:#f5f3ff;color:#1e1b4b;min-height:100vh}
|
|
||||||
nav{background:#4f46e5;color:#fff;padding:14px 24px;display:flex;align-items:center;gap:12px}
|
|
||||||
nav .logo{font-size:16px;font-weight:700;letter-spacing:-0.5px}
|
|
||||||
nav .logo span{opacity:0.7;font-weight:400}
|
|
||||||
nav a{color:rgba(255,255,255,0.8);text-decoration:none;font-size:13px;margin-left:16px}
|
|
||||||
nav a:hover{color:#fff}
|
|
||||||
.container{max-width:1100px;margin:0 auto;padding:24px 20px}
|
|
||||||
h1{font-size:22px;font-weight:700;color:#3730a3;margin-bottom:4px}
|
|
||||||
.subtitle{font-size:13px;color:#6b7280;margin-bottom:24px}
|
|
||||||
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:16px;margin-bottom:28px}
|
|
||||||
.card{background:#fff;border:1px solid #e0e7ff;border-radius:12px;padding:18px 20px;box-shadow:0 1px 3px rgba(79,70,229,.08)}
|
|
||||||
.card .label{font-size:10px;color:#6b7280;text-transform:uppercase;letter-spacing:0.7px;margin-bottom:6px}
|
|
||||||
.card .value{font-size:28px;font-weight:700;color:#4f46e5;line-height:1}
|
|
||||||
.card .unit{font-size:11px;color:#9ca3af;margin-top:4px}
|
|
||||||
.card.green .value{color:#16a34a}
|
|
||||||
.card.amber .value{color:#d97706}
|
|
||||||
.chart-wrap{background:#fff;border:1px solid #e0e7ff;border-radius:12px;padding:20px;margin-bottom:24px;box-shadow:0 1px 3px rgba(79,70,229,.08)}
|
|
||||||
.chart-wrap h2{font-size:14px;font-weight:600;color:#3730a3;margin-bottom:16px}
|
|
||||||
.chart-row{display:grid;grid-template-columns:2fr 1fr;gap:20px;margin-bottom:24px}
|
|
||||||
.export-bar{display:flex;gap:10px;margin-bottom:20px;flex-wrap:wrap}
|
|
||||||
.btn{padding:8px 16px;border-radius:8px;font-size:12px;font-weight:600;cursor:pointer;text-decoration:none;border:none}
|
|
||||||
.btn-primary{background:#4f46e5;color:#fff}
|
|
||||||
.btn-outline{background:#fff;color:#4f46e5;border:1.5px solid #4f46e5}
|
|
||||||
.btn:hover{opacity:0.9}
|
|
||||||
.forecast-box{background:linear-gradient(135deg,#4f46e5,#6366f1);color:#fff;border-radius:12px;padding:18px 22px}
|
|
||||||
.forecast-box .label{font-size:10px;opacity:0.8;text-transform:uppercase;letter-spacing:0.7px;margin-bottom:6px}
|
|
||||||
.forecast-box .value{font-size:32px;font-weight:700;line-height:1}
|
|
||||||
.forecast-box .unit{font-size:11px;opacity:0.7;margin-top:4px}
|
|
||||||
.meta{font-size:11px;color:#9ca3af;margin-top:24px;text-align:center}
|
|
||||||
@media(max-width:700px){.chart-row{grid-template-columns:1fr}}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<nav>
|
|
||||||
<span class="logo">⬡ PeerCortex <span>Network Intelligence</span></span>
|
|
||||||
<a href="/">Dashboard</a>
|
|
||||||
<a href="/aspa-adoption">ASPA Tracker</a>
|
|
||||||
</nav>
|
|
||||||
<div class="container">
|
|
||||||
<h1>ASPA Adoption Tracker</h1>
|
|
||||||
<p class="subtitle">Global ASPA deployment trends · Powered by RPKI Cloudflare feed · Updated every ~4 hours</p>
|
|
||||||
|
|
||||||
<div class="grid">
|
|
||||||
<div class="card ${coverageNow >= 15 ? 'green' : 'amber'}">
|
|
||||||
<div class="label">Atlas Coverage</div>
|
|
||||||
<div class="value">${coverageNow.toFixed(1)}%</div>
|
|
||||||
<div class="unit">of Atlas-visible ASNs</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="label">ASPA Objects</div>
|
|
||||||
<div class="value">${aspaNow.toLocaleString()}</div>
|
|
||||||
<div class="unit">total in RPKI</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="label">Atlas ASNs</div>
|
|
||||||
<div class="value">${atlasTotal.toLocaleString()}</div>
|
|
||||||
<div class="unit">sampled denominator</div>
|
|
||||||
</div>
|
|
||||||
<div class="card ${change7d > 0 ? 'green' : ''}">
|
|
||||||
<div class="label">7-Day Delta</div>
|
|
||||||
<div class="value">${change7d > 0 ? '+' : ''}${change7d}</div>
|
|
||||||
<div class="unit">new ASPA objects</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="label">Data Points</div>
|
|
||||||
<div class="value">${aspaAdoptionDailyHistory.length}</div>
|
|
||||||
<div class="unit">history snapshots</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="export-bar">
|
|
||||||
<a class="btn btn-primary" href="/api/aspa-adoption-stats/export?format=json&period=30d">↓ JSON (30d)</a>
|
|
||||||
<a class="btn btn-outline" href="/api/aspa-adoption-stats/export?format=csv&period=30d">↓ CSV (30d)</a>
|
|
||||||
<a class="btn btn-outline" href="/api/aspa-adoption-stats/export?format=csv&period=1y">↓ CSV (1y)</a>
|
|
||||||
<a class="btn btn-outline" href="/api/export/pdf?asn=0&format=report" style="display:none">PDF Report</a>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="chart-row">
|
|
||||||
<div class="chart-wrap">
|
|
||||||
<h2>Atlas Coverage % (last 30 days)</h2>
|
|
||||||
<canvas id="coverageChart" height="120"></canvas>
|
|
||||||
</div>
|
|
||||||
<div class="forecast-box">
|
|
||||||
<div class="label">90-Day Forecast</div>
|
|
||||||
<div class="value">${forecast90d !== null ? forecast90d.toFixed(1) + '%' : '—'}</div>
|
|
||||||
<div class="unit">linear projection from 30-day trend</div>
|
|
||||||
<div style="margin-top:16px;font-size:11px;opacity:0.8">
|
|
||||||
Based on ${trend30.length} data points.<br/>
|
|
||||||
Current: ${coverageNow.toFixed(1)}%
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="chart-wrap">
|
|
||||||
<h2>Total ASPA Objects in RPKI (last 30 days)</h2>
|
|
||||||
<canvas id="asnChart" height="80"></canvas>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="meta">
|
|
||||||
Last snapshot: ${latest?.date ?? 'none'} · Denominator: ${atlasTotal.toLocaleString()} Atlas ASNs ·
|
|
||||||
<a href="/api/aspa-adoption-stats?period=30d" style="color:#4f46e5">JSON API</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
const labels = ${JSON.stringify(trendLabels)};
|
|
||||||
const covData = ${JSON.stringify(trendValues)};
|
|
||||||
const asnData = ${JSON.stringify(asnValues)};
|
|
||||||
|
|
||||||
const commonOpts = {
|
|
||||||
responsive: true,
|
|
||||||
plugins: { legend: { display: false } },
|
|
||||||
scales: {
|
|
||||||
x: { grid: { color: '#f0eeff' }, ticks: { maxRotation: 45, font: { size: 10 } } },
|
|
||||||
y: { grid: { color: '#f0eeff' }, ticks: { font: { size: 10 } }, beginAtZero: false }
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
new Chart(document.getElementById('coverageChart'), {
|
|
||||||
type: 'line',
|
|
||||||
data: {
|
|
||||||
labels,
|
|
||||||
datasets: [{
|
|
||||||
data: covData,
|
|
||||||
borderColor: '#4f46e5',
|
|
||||||
backgroundColor: 'rgba(79,70,229,0.08)',
|
|
||||||
fill: true,
|
|
||||||
tension: 0.3,
|
|
||||||
pointRadius: 3,
|
|
||||||
}]
|
|
||||||
},
|
|
||||||
options: { ...commonOpts, scales: { ...commonOpts.scales, y: { ...commonOpts.scales.y, ticks: { callback: v => v + '%', font: { size: 10 } } } } }
|
|
||||||
});
|
|
||||||
|
|
||||||
new Chart(document.getElementById('asnChart'), {
|
|
||||||
type: 'bar',
|
|
||||||
data: {
|
|
||||||
labels,
|
|
||||||
datasets: [{
|
|
||||||
data: asnData,
|
|
||||||
backgroundColor: 'rgba(79,70,229,0.6)',
|
|
||||||
borderColor: '#4f46e5',
|
|
||||||
borderWidth: 1,
|
|
||||||
}]
|
|
||||||
},
|
|
||||||
options: commonOpts
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// FEATURE 4: IPv6 Adoption per RIR
|
// FEATURE 4: IPv6 Adoption per RIR
|
||||||
@ -1348,224 +1067,32 @@ const {
|
|||||||
const { ensureManrsCache, checkManrsMembership } = require("./server/services/manrs");
|
const { ensureManrsCache, checkManrsMembership } = require("./server/services/manrs");
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// Infrastructure overlay caches
|
// subCableCache/globalFacCache removed 2026-07-16: confirmed dead code,
|
||||||
let subCableCache = null; // TeleGeography submarine cables (24h)
|
// never read anywhere in the file.
|
||||||
let globalFacCache = null; // PeeringDB global facilities (24h)
|
|
||||||
|
|
||||||
// RPKI ASPA + ROA Cache from Cloudflare RPKI JSON feed
|
|
||||||
// ============================================================
|
|
||||||
const rpkiAspaMap = new Map(); // customer_asid -> Set<provider_asn>
|
|
||||||
let rpkiAspaLastFetch = 0;
|
|
||||||
let rpkiAspaFetching = false;
|
|
||||||
|
|
||||||
const { roaStore } = require("./server/caches/roa-store");
|
const { roaStore } = require("./server/caches/roa-store");
|
||||||
|
|
||||||
|
const {
|
||||||
|
rpkiAspaMap,
|
||||||
|
getRpkiAspaLastFetch,
|
||||||
|
fetchRpkiAspaFeed,
|
||||||
|
ensureAspaCache,
|
||||||
|
lookupAspaFromRpki,
|
||||||
|
validateRPKIWithCache,
|
||||||
|
fetchRipeRpkiValidator,
|
||||||
|
crossCheckRpki,
|
||||||
|
} = require("./server/services/rpki");
|
||||||
|
|
||||||
const { pdbSourceCache } = require("./server/caches/pdb-source-cache");
|
const { pdbSourceCache } = require("./server/caches/pdb-source-cache");
|
||||||
|
const {
|
||||||
// ============================================================
|
ripeStatCache,
|
||||||
// RIPE Stat Source Cache + Semaphore (L2)
|
fetchRipeStatCached,
|
||||||
// Prevents 429 rate-limiting by throttling + caching responses
|
fetchRipeStatCachedFromApi,
|
||||||
// ============================================================
|
fetchRipeStatCachedWithRetry,
|
||||||
const ripeStatCache = new Map(); // key: "endpoint:resource" → {data, ts}
|
saveRipeStatCacheToDisk,
|
||||||
const RIPE_STAT_CACHE_MAX = 2000;
|
loadRipeStatCacheFromDisk,
|
||||||
const RIPE_STAT_TTL = {
|
Semaphore,
|
||||||
"announced-prefixes": 15 * 60 * 1000,
|
} = require("./server/services/ripe-stat");
|
||||||
"asn-neighbours": 15 * 60 * 1000,
|
|
||||||
"as-overview": 60 * 60 * 1000,
|
|
||||||
"rir-stats-country": 24 * 60 * 60 * 1000,
|
|
||||||
"visibility": 15 * 60 * 1000,
|
|
||||||
"prefix-size-distribution": 60 * 60 * 1000,
|
|
||||||
"abuse-contact-finder": 24 * 60 * 60 * 1000,
|
|
||||||
"blocklist": 60 * 60 * 1000,
|
|
||||||
"reverse-dns-consistency": 60 * 60 * 1000,
|
|
||||||
"routing-status": 15 * 60 * 1000,
|
|
||||||
"bgp-updates": 15 * 60 * 1000,
|
|
||||||
"maxmind-geo-lite-pfx": 24 * 60 * 60 * 1000,
|
|
||||||
"looking-glass": 15 * 60 * 1000,
|
|
||||||
"whois": 24 * 60 * 60 * 1000,
|
|
||||||
"rpki-validation": 6 * 60 * 60 * 1000,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Counting semaphore — limits concurrent RIPE Stat requests
|
|
||||||
class Semaphore {
|
|
||||||
constructor(max) { this.max = max; this.current = 0; this.queue = []; }
|
|
||||||
acquire() {
|
|
||||||
if (this.current < this.max) { this.current++; return Promise.resolve(); }
|
|
||||||
return new Promise((resolve) => this.queue.push(resolve));
|
|
||||||
}
|
|
||||||
release() {
|
|
||||||
this.current--;
|
|
||||||
if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const ripeStatSemaphore = new Semaphore(15);
|
|
||||||
|
|
||||||
// Cached + throttled RIPE Stat fetch
|
|
||||||
// Renamed from fetchRipeStatCached 2026-07-16: a second function with that exact
|
|
||||||
// name was added ~150 lines down (the local-DB-routing wrapper) during the Postgres
|
|
||||||
// refactor. In JS, two `function` declarations sharing a name silently collapse to
|
|
||||||
// the LAST one in source order -- the real RIPE Stat fetch+cache+throttle
|
|
||||||
// implementation below was completely shadowed, and every call to
|
|
||||||
// fetchRipeStatCached() for any endpoint NOT in {announced-prefixes, asn-neighbours,
|
|
||||||
// as-overview, visibility, prefix-size-distribution} was unconditionally returning
|
|
||||||
// null instead of ever reaching this code. That silently broke blocklist,
|
|
||||||
// abuse-contact-finder, bgp-updates, routing-status, looking-glass, whois,
|
|
||||||
// reverse-dns-consistency, maxmind-geo-lite-pfx, and ixs lookups -- see the router's
|
|
||||||
// fallback call to this function for the fix.
|
|
||||||
async function fetchRipeStatCachedFromApi(url, options) {
|
|
||||||
// Extract endpoint name from URL for TTL lookup
|
|
||||||
const match = url.match(/\/data\/([^/]+)\//);
|
|
||||||
const endpoint = match ? match[1] : "default";
|
|
||||||
const resourceMatch = url.match(/resource=([^&]+)/);
|
|
||||||
const resource = resourceMatch ? resourceMatch[1] : url;
|
|
||||||
const cacheKey = endpoint + ":" + resource;
|
|
||||||
|
|
||||||
// Check cache
|
|
||||||
const cached = ripeStatCache.get(cacheKey);
|
|
||||||
const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000;
|
|
||||||
if (cached && (Date.now() - cached.ts) < ttl) {
|
|
||||||
return cached.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Throttle via semaphore
|
|
||||||
await ripeStatSemaphore.acquire();
|
|
||||||
try {
|
|
||||||
// Double-check cache after acquiring semaphore (another request may have filled it)
|
|
||||||
const cached2 = ripeStatCache.get(cacheKey);
|
|
||||||
if (cached2 && (Date.now() - cached2.ts) < ttl) {
|
|
||||||
return cached2.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await fetchJSON(url, options);
|
|
||||||
|
|
||||||
// Only cache successful results — never cache null (failed/rate-limited responses)
|
|
||||||
// Caching null causes cascading failures: retry hits cache, returns null again
|
|
||||||
if (result !== null) {
|
|
||||||
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
|
||||||
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
|
||||||
}
|
|
||||||
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
} finally {
|
|
||||||
ripeStatSemaphore.release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cached + throttled RIPE Stat with one retry on failure
|
|
||||||
async function fetchRipeStatCachedWithRetry(url, options) {
|
|
||||||
const result = await fetchRipeStatCached(url, options);
|
|
||||||
if (result !== null) return result;
|
|
||||||
await new Promise(r => setTimeout(r, 1500));
|
|
||||||
return fetchRipeStatCached(url, options);
|
|
||||||
}
|
|
||||||
|
|
||||||
// RIPE Stat cache disk persistence (skip null entries)
|
|
||||||
function saveRipeStatCacheToDisk(filePath) {
|
|
||||||
try {
|
|
||||||
const obj = {};
|
|
||||||
for (const [k, v] of ripeStatCache) {
|
|
||||||
if (v.data !== null) obj[k] = v;
|
|
||||||
}
|
|
||||||
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
|
|
||||||
console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
|
|
||||||
} catch (e) {
|
|
||||||
console.warn("[RIPE-CACHE] Disk save failed:", e.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function loadRipeStatCacheFromDisk(filePath) {
|
|
||||||
try {
|
|
||||||
if (!fs.existsSync(filePath)) return false;
|
|
||||||
const raw = fs.readFileSync(filePath, "utf8");
|
|
||||||
const data = JSON.parse(raw);
|
|
||||||
const now = Date.now();
|
|
||||||
for (const [k, v] of Object.entries(data.entries || {})) {
|
|
||||||
const match = k.match(/^([^:]+):/);
|
|
||||||
const endpoint = match ? match[1] : "default";
|
|
||||||
const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000;
|
|
||||||
if (now - v.ts < ttl) {
|
|
||||||
ripeStatCache.set(k, v);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log("[RIPE-CACHE] Loaded " + ripeStatCache.size + " entries from disk");
|
|
||||||
return true;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn("[RIPE-CACHE] Disk load failed:", e.message);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function fetchRpkiAspaFeed() {
|
|
||||||
if (rpkiAspaFetching) return Promise.resolve();
|
|
||||||
rpkiAspaFetching = true;
|
|
||||||
console.log("[RPKI] Fetching Cloudflare RPKI feed (ASPA + ROA)...");
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
const options = {
|
|
||||||
headers: { "User-Agent": UA },
|
|
||||||
timeout: 120000,
|
|
||||||
};
|
|
||||||
https.get("https://rpki.cloudflare.com/rpki.json", options, (res) => {
|
|
||||||
let data = "";
|
|
||||||
res.on("data", (chunk) => (data += chunk));
|
|
||||||
res.on("end", () => {
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(data);
|
|
||||||
|
|
||||||
// Load ASPA objects
|
|
||||||
const aspas = parsed.aspas || [];
|
|
||||||
rpkiAspaMap.clear();
|
|
||||||
aspas.forEach((a) => {
|
|
||||||
const customerAsid = Number(a.customer_asid);
|
|
||||||
const providers = (a.providers || []).map(Number);
|
|
||||||
rpkiAspaMap.set(customerAsid, new Set(providers));
|
|
||||||
});
|
|
||||||
|
|
||||||
// Load ROA objects into local store (eliminates RIPE Stat per-prefix calls)
|
|
||||||
const roas = parsed.roas || [];
|
|
||||||
roaStore.build(roas);
|
|
||||||
roaStore.saveToDisk("/opt/peercortex-app/.roa-cache.json");
|
|
||||||
|
|
||||||
// Track ASPA adoption — persist to disk + update in-memory trend
|
|
||||||
recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count);
|
|
||||||
|
|
||||||
rpkiAspaLastFetch = Date.now();
|
|
||||||
console.log("[RPKI] Loaded " + rpkiAspaMap.size + " ASPA objects + " + roaStore.count + " ROAs from Cloudflare feed");
|
|
||||||
} catch (e) {
|
|
||||||
console.error("[RPKI] Failed to parse RPKI feed:", e.message);
|
|
||||||
}
|
|
||||||
rpkiAspaFetching = false;
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
}).on("error", (e) => {
|
|
||||||
console.error("[RPKI] Fetch failed:", e.message);
|
|
||||||
rpkiAspaFetching = false;
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure ASPA + ROA cache is fresh
|
|
||||||
async function ensureAspaCache() {
|
|
||||||
if (Date.now() - rpkiAspaLastFetch > 4 * 60 * 60 * 1000) {
|
|
||||||
await fetchRpkiAspaFeed();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lookup ASPA object for a given ASN from the RPKI feed cache. feedLoaded
|
|
||||||
// distinguishes "the feed loaded fine and this ASN genuinely has no ASPA object"
|
|
||||||
// from "the Cloudflare feed fetch has never succeeded, so we don't actually know" --
|
|
||||||
// rpkiAspaLastFetch is only set on a successful parse (see fetchRpkiAspaFeed), so
|
|
||||||
// 0 means every attempt so far has failed and exists:false here is not trustworthy.
|
|
||||||
function lookupAspaFromRpki(asn) {
|
|
||||||
const asnNum = Number(asn);
|
|
||||||
const feedLoaded = rpkiAspaLastFetch > 0;
|
|
||||||
if (rpkiAspaMap.has(asnNum)) {
|
|
||||||
const providers = rpkiAspaMap.get(asnNum);
|
|
||||||
return { exists: true, providers: [...providers].sort((a, b) => a - b), feedLoaded };
|
|
||||||
}
|
|
||||||
return { exists: false, providers: [], feedLoaded };
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1634,71 +1161,6 @@ async function resolveASNames(providers) {
|
|||||||
return providers;
|
return providers;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── MASTER RIPE Stat API wrapper (Local-first, zero external API calls) ──
|
|
||||||
// Analyzes RIPE Stat URL and dispatches to appropriate localDb function
|
|
||||||
async function fetchRipeStatCached(url, options = {}) {
|
|
||||||
try {
|
|
||||||
// Detect which RIPE Stat endpoint this is and call local DB
|
|
||||||
if (url.includes('/announced-prefixes/')) {
|
|
||||||
const asnMatch = url.match(/resource=AS(\d+)/);
|
|
||||||
if (asnMatch) return await localDb.getRipeStatAnnouncedPrefixes(parseInt(asnMatch[1]));
|
|
||||||
}
|
|
||||||
if (url.includes('/asn-neighbours/')) {
|
|
||||||
const asnMatch = url.match(/resource=AS(\d+)/);
|
|
||||||
if (asnMatch) return await localDb.getRipeStatAsnNeighbours(parseInt(asnMatch[1]));
|
|
||||||
}
|
|
||||||
if (url.includes('/as-overview/')) {
|
|
||||||
const asnMatch = url.match(/resource=AS(\d+)/);
|
|
||||||
if (asnMatch) return await localDb.getRipeStatAsOverview(parseInt(asnMatch[1]));
|
|
||||||
}
|
|
||||||
if (url.includes('/visibility/')) {
|
|
||||||
const asnMatch = url.match(/resource=AS(\d+)/);
|
|
||||||
if (asnMatch) return await localDb.getRipeStatVisibility(parseInt(asnMatch[1]));
|
|
||||||
}
|
|
||||||
if (url.includes('/prefix-size-distribution/')) {
|
|
||||||
const asnMatch = url.match(/resource=AS(\d+)/);
|
|
||||||
if (asnMatch) return await localDb.getRipeStatPrefixSizeDistribution(parseInt(asnMatch[1]));
|
|
||||||
}
|
|
||||||
|
|
||||||
// For every other RIPE Stat endpoint (not mirrored into the local Postgres DB) --
|
|
||||||
// e.g. blocklist, abuse-contact-finder, bgp-updates, routing-status, looking-glass,
|
|
||||||
// whois, reverse-dns-consistency, maxmind-geo-lite-pfx, ixs -- fall through to the
|
|
||||||
// real RIPE Stat API with its own cache/throttle/never-cache-null protections,
|
|
||||||
// instead of returning null unconditionally (see fetchRipeStatCachedFromApi's
|
|
||||||
// header comment for how this silently broke ~9 checks until 2026-07-16).
|
|
||||||
return fetchRipeStatCachedFromApi(url, options);
|
|
||||||
} catch (e) {
|
|
||||||
console.error("[fetchRipeStatCached] Error:", e.message);
|
|
||||||
return Promise.resolve(null);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RPKI per-prefix validation — uses local ROA store (instant, no API calls)
|
|
||||||
// Falls back to RIPE Stat only if ROA store is not yet loaded (cold start)
|
|
||||||
// Validate RPKI for a prefix — uses local PostgreSQL database (sub-10ms, zero external API calls)
|
|
||||||
// Returns: { prefix, status: "valid"|"invalid"|"not_found"|"unavailable", validating_roas: N }
|
|
||||||
// "not_found" means the DB was queried and genuinely has no covering ROA -- a real result.
|
|
||||||
// "unavailable" means the DB query itself failed (connection/timeout/error) -- we don't know.
|
|
||||||
// Conflating these used to make DB hiccups masquerade as "no ROA published" (a real, worse-
|
|
||||||
// sounding finding). Callers computing coverage percentages should exclude "unavailable".
|
|
||||||
async function validateRPKIWithCache(asn, prefix) {
|
|
||||||
try {
|
|
||||||
const result = await localDb.validateRpki(prefix, asn);
|
|
||||||
if (result.status === 'valid') {
|
|
||||||
return { prefix, status: "valid", validating_roas: 1 };
|
|
||||||
} else if (result.status === 'invalid') {
|
|
||||||
return { prefix, status: "invalid", validating_roas: 1 };
|
|
||||||
} else if (result.status === 'unknown') {
|
|
||||||
return { prefix, status: "unavailable", validating_roas: 0 };
|
|
||||||
} else {
|
|
||||||
// 'not-found': genuinely no covering ROA
|
|
||||||
return { prefix, status: "not_found", validating_roas: 0 };
|
|
||||||
}
|
|
||||||
} catch (_e) {
|
|
||||||
console.error("[RPKI] Error validating " + prefix + ":", _e.message);
|
|
||||||
return { prefix, status: "unavailable", validating_roas: 0 };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
@ -1713,83 +1175,6 @@ const {
|
|||||||
} = require("./server/aspa-verification/engine");
|
} = require("./server/aspa-verification/engine");
|
||||||
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// Feature 30: RIPE NCC RPKI Validator cross-check (max 5 prefixes)
|
|
||||||
// ============================================================
|
|
||||||
async function fetchRipeRpkiValidator(asn, prefix) {
|
|
||||||
try {
|
|
||||||
const encoded = encodeURIComponent(prefix);
|
|
||||||
const url = "https://rpki-validator.ripe.net/api/v1/validity/AS" + asn + "/" + encoded;
|
|
||||||
const result = await fetchJSON(url, { timeout: 5000 });
|
|
||||||
if (result && result.validated_route) {
|
|
||||||
return {
|
|
||||||
prefix: prefix,
|
|
||||||
validity: result.validated_route.validity || {},
|
|
||||||
state: (result.validated_route.validity && result.validated_route.validity.state) || "unknown",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { prefix: prefix, state: "unknown", error: "no_data" };
|
|
||||||
} catch (_e) {
|
|
||||||
return { prefix: prefix, state: "error", error: "timeout_or_unavailable" };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cross-check a sample of prefixes against RIPE RPKI Validator (max 5, in parallel).
|
|
||||||
// agreement_pct only counts pairs where BOTH sources actually returned a real
|
|
||||||
// verdict -- a zero-sample or all-RIPE-lookups-failed result reports agreement_pct:
|
|
||||||
// null (not a fabricated 100), since nothing was actually cross-checked.
|
|
||||||
async function crossCheckRpki(asn, prefixes, localResults) {
|
|
||||||
const sample = prefixes.slice(0, 5);
|
|
||||||
if (sample.length === 0) return { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 };
|
|
||||||
|
|
||||||
const ripeResults = await Promise.all(
|
|
||||||
sample.map((pfx) => fetchRipeRpkiValidator(asn, pfx))
|
|
||||||
);
|
|
||||||
|
|
||||||
const localMap = new Map();
|
|
||||||
for (const lr of localResults) {
|
|
||||||
localMap.set(lr.prefix, lr.status);
|
|
||||||
}
|
|
||||||
|
|
||||||
let cloudflareValid = 0;
|
|
||||||
let ripeValid = 0;
|
|
||||||
let agreements = 0;
|
|
||||||
let comparable = 0;
|
|
||||||
const disagreements = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < sample.length; i++) {
|
|
||||||
const pfx = sample[i];
|
|
||||||
const cfStatus = localMap.get(pfx) || "not_found";
|
|
||||||
const ripeState = ripeResults[i].state;
|
|
||||||
|
|
||||||
const cfIsValid = cfStatus === "valid";
|
|
||||||
const ripeIsValid = ripeState === "valid" || ripeState === "VALID";
|
|
||||||
|
|
||||||
if (cfIsValid) cloudflareValid++;
|
|
||||||
if (ripeIsValid) ripeValid++;
|
|
||||||
|
|
||||||
// A failed/unknown RIPE lookup isn't a comparison at all -- exclude it from
|
|
||||||
// the agreement percentage instead of counting it as a silent "agreement".
|
|
||||||
if (ripeState === "error" || ripeState === "unknown") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
comparable++;
|
|
||||||
if (cfIsValid === ripeIsValid) {
|
|
||||||
agreements++;
|
|
||||||
} else {
|
|
||||||
disagreements.push({
|
|
||||||
prefix: pfx,
|
|
||||||
cloudflare: cfStatus,
|
|
||||||
ripe: ripeState,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const agreementPct = comparable > 0 ? Math.round((agreements / comparable) * 100) : null;
|
|
||||||
return { cloudflare_valid: cloudflareValid, ripe_valid: ripeValid, agreement_pct: agreementPct, disagreements: disagreements, sample_size: sample.length, comparable_size: comparable };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// Feature 24: bgp.he.net Integration
|
// Feature 24: bgp.he.net Integration
|
||||||
// ============================================================
|
// ============================================================
|
||||||
@ -2401,6 +1786,7 @@ const server = http.createServer(async (req, res) => {
|
|||||||
// Health endpoint — extended with cache status, ASPA metrics, and local DB stats
|
// Health endpoint — extended with cache status, ASPA metrics, and local DB stats
|
||||||
if (reqPath === "/api/health") {
|
if (reqPath === "/api/health") {
|
||||||
const mem = process.memoryUsage();
|
const mem = process.memoryUsage();
|
||||||
|
const rpkiAspaLastFetch = getRpkiAspaLastFetch();
|
||||||
const aspaAge = rpkiAspaLastFetch ? Math.floor((Date.now() - rpkiAspaLastFetch) / 60000) : -1;
|
const aspaAge = rpkiAspaLastFetch ? Math.floor((Date.now() - rpkiAspaLastFetch) / 60000) : -1;
|
||||||
const pdbTotal = pdbSourceCache.hits + pdbSourceCache.misses;
|
const pdbTotal = pdbSourceCache.hits + pdbSourceCache.misses;
|
||||||
|
|
||||||
@ -5542,7 +4928,7 @@ loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json");
|
|||||||
ensureManrsCache(); // fire-and-forget, 24h cache
|
ensureManrsCache(); // fire-and-forget, 24h cache
|
||||||
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
|
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
|
||||||
// Re-record ASPA adoption snapshot now that Atlas data is fully loaded
|
// Re-record ASPA adoption snapshot now that Atlas data is fully loaded
|
||||||
recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count);
|
recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count, rpkiAspaMap);
|
||||||
|
|
||||||
server.listen(PORT, "0.0.0.0", () => {
|
server.listen(PORT, "0.0.0.0", () => {
|
||||||
console.log("PeerCortex v0.6.5 running on http://0.0.0.0:" + PORT);
|
console.log("PeerCortex v0.6.5 running on http://0.0.0.0:" + PORT);
|
||||||
|
|||||||
291
server/aspa-adoption/tracker.js
Normal file
291
server/aspa-adoption/tracker.js
Normal file
@ -0,0 +1,291 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const { DATA_DIR } = require("../hijack-monitoring/store");
|
||||||
|
const { atlasState } = require("../atlas-probes");
|
||||||
|
|
||||||
|
const ASPA_ADOPTION_FILE = DATA_DIR + '/aspa-adoption-history.json';
|
||||||
|
|
||||||
|
// Load persisted ASPA adoption history from disk.
|
||||||
|
// Returns array of snapshot objects (up to 365 entries).
|
||||||
|
function loadAspaAdoptionHistory() {
|
||||||
|
try {
|
||||||
|
const raw = JSON.parse(fs.readFileSync(ASPA_ADOPTION_FILE, 'utf8'));
|
||||||
|
return Array.isArray(raw) ? raw : [];
|
||||||
|
} catch(_) { return []; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save ASPA adoption history to disk (keep last 365 entries).
|
||||||
|
function saveAspaAdoptionHistory(history) {
|
||||||
|
try {
|
||||||
|
const trimmed = history.slice(-365);
|
||||||
|
fs.writeFileSync(ASPA_ADOPTION_FILE, JSON.stringify(trimmed, null, 2), 'utf8');
|
||||||
|
} catch(e) {
|
||||||
|
console.error('[ASPA-ADOPT] Save failed:', e.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// In-memory adoption history (persisted on each new snapshot). Mutated
|
||||||
|
// in place (push / index-assign), never reassigned -- safe to export as a
|
||||||
|
// live array reference, same pattern as roaStore/pdbSourceCache.
|
||||||
|
let aspaAdoptionDailyHistory = loadAspaAdoptionHistory();
|
||||||
|
|
||||||
|
// Take a new adoption snapshot and persist it. Called whenever the RPKI feed
|
||||||
|
// is refreshed.
|
||||||
|
// @param {number} aspaCount — total ASPA objects in rpkiAspaMap
|
||||||
|
// @param {number} roaCount — total ROAs in roaStore
|
||||||
|
// @param {Map} rpkiAspaMap — customer_asid -> Set<provider_asn>, owned by
|
||||||
|
// server/services/rpki.js; passed in explicitly (rather than required here)
|
||||||
|
// to avoid a circular require between this module and rpki.js.
|
||||||
|
function recordAspaAdoptionSnapshot(aspaCount, roaCount, rpkiAspaMap) {
|
||||||
|
const now = new Date();
|
||||||
|
const date = now.toISOString().slice(0, 10); // "YYYY-MM-DD"
|
||||||
|
|
||||||
|
// Compute how many Atlas-visible ASNs have ASPA
|
||||||
|
let atlasTotal = 0;
|
||||||
|
let atlasWithAspa = 0;
|
||||||
|
if (atlasState.cache?.asns_with_probes) {
|
||||||
|
atlasTotal = atlasState.cache.asns_with_probes.length;
|
||||||
|
// rpkiAspaMap is keyed by integer ASN
|
||||||
|
atlasWithAspa = atlasState.cache.asns_with_probes.filter(a => rpkiAspaMap.has(Number(a))).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
const coveragePct = atlasTotal > 0
|
||||||
|
? Math.round((atlasWithAspa / atlasTotal) * 10000) / 100 // 2 decimal places
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
// Compute delta from previous snapshot
|
||||||
|
const prev = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1];
|
||||||
|
const delta = prev ? aspaCount - prev.aspa_objects : 0;
|
||||||
|
|
||||||
|
const snapshot = {
|
||||||
|
date,
|
||||||
|
ts: now.getTime(),
|
||||||
|
aspa_objects: aspaCount,
|
||||||
|
roa_count: roaCount,
|
||||||
|
atlas_asns_total: atlasTotal,
|
||||||
|
atlas_asns_with_aspa: atlasWithAspa,
|
||||||
|
coverage_pct_atlas: coveragePct,
|
||||||
|
aspa_delta: delta,
|
||||||
|
method: 'rpki_cloudflare_feed',
|
||||||
|
};
|
||||||
|
|
||||||
|
// Only store one snapshot per date (overwrite if same day)
|
||||||
|
const existingIdx = aspaAdoptionDailyHistory.findIndex(s => s.date === date);
|
||||||
|
if (existingIdx >= 0) {
|
||||||
|
aspaAdoptionDailyHistory[existingIdx] = snapshot;
|
||||||
|
} else {
|
||||||
|
aspaAdoptionDailyHistory.push(snapshot);
|
||||||
|
}
|
||||||
|
saveAspaAdoptionHistory(aspaAdoptionDailyHistory);
|
||||||
|
console.log(`[ASPA-ADOPT] Snapshot ${date}: ${aspaCount} objects, ${coveragePct}% Atlas coverage (${atlasWithAspa}/${atlasTotal})`);
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get adoption trend for a given period string ('7d', '30d', '90d', '1y').
|
||||||
|
function getAdoptionTrend(period) {
|
||||||
|
const days = period === '7d' ? 7 : period === '30d' ? 30 : period === '90d' ? 90 : 365;
|
||||||
|
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
return aspaAdoptionDailyHistory.filter(s => s.ts >= cutoff);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the ASPA adoption dashboard HTML.
|
||||||
|
function buildAspaAdoptionDashboard() {
|
||||||
|
const latest = aspaAdoptionDailyHistory[aspaAdoptionDailyHistory.length - 1];
|
||||||
|
const trend30 = getAdoptionTrend('30d');
|
||||||
|
const trend7 = getAdoptionTrend('7d');
|
||||||
|
|
||||||
|
// Forecast: simple linear regression over last 30 days
|
||||||
|
function linearForecast(data, daysAhead) {
|
||||||
|
if (data.length < 2) return null;
|
||||||
|
const n = data.length;
|
||||||
|
const xs = data.map((_, i) => i);
|
||||||
|
const ys = data.map(d => d.coverage_pct_atlas);
|
||||||
|
const xMean = xs.reduce((a, b) => a + b, 0) / n;
|
||||||
|
const yMean = ys.reduce((a, b) => a + b, 0) / n;
|
||||||
|
const num = xs.reduce((s, x, i) => s + (x - xMean) * (ys[i] - yMean), 0);
|
||||||
|
const den = xs.reduce((s, x) => s + (x - xMean) ** 2, 0);
|
||||||
|
if (den === 0) return yMean;
|
||||||
|
const slope = num / den;
|
||||||
|
return Math.max(0, Math.min(100, yMean + slope * (n - 1 + daysAhead)));
|
||||||
|
}
|
||||||
|
|
||||||
|
const forecast90d = linearForecast(trend30, 90);
|
||||||
|
const change7d = trend7.length >= 2
|
||||||
|
? (trend7[trend7.length-1].aspa_objects - trend7[0].aspa_objects)
|
||||||
|
: 0;
|
||||||
|
const trendLabels = trend30.map(d => d.date);
|
||||||
|
const trendValues = trend30.map(d => d.coverage_pct_atlas);
|
||||||
|
const asnValues = trend30.map(d => d.aspa_objects);
|
||||||
|
|
||||||
|
const coverageNow = latest?.coverage_pct_atlas ?? 0;
|
||||||
|
const aspaNow = latest?.aspa_objects ?? 0;
|
||||||
|
const atlasTotal = latest?.atlas_asns_total ?? 0;
|
||||||
|
|
||||||
|
return `<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8"/>
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1"/>
|
||||||
|
<title>ASPA Adoption Tracker · PeerCortex</title>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
||||||
|
<style>
|
||||||
|
*,*::before,*::after{box-sizing:border-box;margin:0;padding:0}
|
||||||
|
body{font-family:-apple-system,'Segoe UI',system-ui,sans-serif;background:#f5f3ff;color:#1e1b4b;min-height:100vh}
|
||||||
|
nav{background:#4f46e5;color:#fff;padding:14px 24px;display:flex;align-items:center;gap:12px}
|
||||||
|
nav .logo{font-size:16px;font-weight:700;letter-spacing:-0.5px}
|
||||||
|
nav .logo span{opacity:0.7;font-weight:400}
|
||||||
|
nav a{color:rgba(255,255,255,0.8);text-decoration:none;font-size:13px;margin-left:16px}
|
||||||
|
nav a:hover{color:#fff}
|
||||||
|
.container{max-width:1100px;margin:0 auto;padding:24px 20px}
|
||||||
|
h1{font-size:22px;font-weight:700;color:#3730a3;margin-bottom:4px}
|
||||||
|
.subtitle{font-size:13px;color:#6b7280;margin-bottom:24px}
|
||||||
|
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:16px;margin-bottom:28px}
|
||||||
|
.card{background:#fff;border:1px solid #e0e7ff;border-radius:12px;padding:18px 20px;box-shadow:0 1px 3px rgba(79,70,229,.08)}
|
||||||
|
.card .label{font-size:10px;color:#6b7280;text-transform:uppercase;letter-spacing:0.7px;margin-bottom:6px}
|
||||||
|
.card .value{font-size:28px;font-weight:700;color:#4f46e5;line-height:1}
|
||||||
|
.card .unit{font-size:11px;color:#9ca3af;margin-top:4px}
|
||||||
|
.card.green .value{color:#16a34a}
|
||||||
|
.card.amber .value{color:#d97706}
|
||||||
|
.chart-wrap{background:#fff;border:1px solid #e0e7ff;border-radius:12px;padding:20px;margin-bottom:24px;box-shadow:0 1px 3px rgba(79,70,229,.08)}
|
||||||
|
.chart-wrap h2{font-size:14px;font-weight:600;color:#3730a3;margin-bottom:16px}
|
||||||
|
.chart-row{display:grid;grid-template-columns:2fr 1fr;gap:20px;margin-bottom:24px}
|
||||||
|
.export-bar{display:flex;gap:10px;margin-bottom:20px;flex-wrap:wrap}
|
||||||
|
.btn{padding:8px 16px;border-radius:8px;font-size:12px;font-weight:600;cursor:pointer;text-decoration:none;border:none}
|
||||||
|
.btn-primary{background:#4f46e5;color:#fff}
|
||||||
|
.btn-outline{background:#fff;color:#4f46e5;border:1.5px solid #4f46e5}
|
||||||
|
.btn:hover{opacity:0.9}
|
||||||
|
.forecast-box{background:linear-gradient(135deg,#4f46e5,#6366f1);color:#fff;border-radius:12px;padding:18px 22px}
|
||||||
|
.forecast-box .label{font-size:10px;opacity:0.8;text-transform:uppercase;letter-spacing:0.7px;margin-bottom:6px}
|
||||||
|
.forecast-box .value{font-size:32px;font-weight:700;line-height:1}
|
||||||
|
.forecast-box .unit{font-size:11px;opacity:0.7;margin-top:4px}
|
||||||
|
.meta{font-size:11px;color:#9ca3af;margin-top:24px;text-align:center}
|
||||||
|
@media(max-width:700px){.chart-row{grid-template-columns:1fr}}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<nav>
|
||||||
|
<span class="logo">⬡ PeerCortex <span>Network Intelligence</span></span>
|
||||||
|
<a href="/">Dashboard</a>
|
||||||
|
<a href="/aspa-adoption">ASPA Tracker</a>
|
||||||
|
</nav>
|
||||||
|
<div class="container">
|
||||||
|
<h1>ASPA Adoption Tracker</h1>
|
||||||
|
<p class="subtitle">Global ASPA deployment trends · Powered by RPKI Cloudflare feed · Updated every ~4 hours</p>
|
||||||
|
|
||||||
|
<div class="grid">
|
||||||
|
<div class="card ${coverageNow >= 15 ? 'green' : 'amber'}">
|
||||||
|
<div class="label">Atlas Coverage</div>
|
||||||
|
<div class="value">${coverageNow.toFixed(1)}%</div>
|
||||||
|
<div class="unit">of Atlas-visible ASNs</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="label">ASPA Objects</div>
|
||||||
|
<div class="value">${aspaNow.toLocaleString()}</div>
|
||||||
|
<div class="unit">total in RPKI</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="label">Atlas ASNs</div>
|
||||||
|
<div class="value">${atlasTotal.toLocaleString()}</div>
|
||||||
|
<div class="unit">sampled denominator</div>
|
||||||
|
</div>
|
||||||
|
<div class="card ${change7d > 0 ? 'green' : ''}">
|
||||||
|
<div class="label">7-Day Delta</div>
|
||||||
|
<div class="value">${change7d > 0 ? '+' : ''}${change7d}</div>
|
||||||
|
<div class="unit">new ASPA objects</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="label">Data Points</div>
|
||||||
|
<div class="value">${aspaAdoptionDailyHistory.length}</div>
|
||||||
|
<div class="unit">history snapshots</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="export-bar">
|
||||||
|
<a class="btn btn-primary" href="/api/aspa-adoption-stats/export?format=json&period=30d">↓ JSON (30d)</a>
|
||||||
|
<a class="btn btn-outline" href="/api/aspa-adoption-stats/export?format=csv&period=30d">↓ CSV (30d)</a>
|
||||||
|
<a class="btn btn-outline" href="/api/aspa-adoption-stats/export?format=csv&period=1y">↓ CSV (1y)</a>
|
||||||
|
<a class="btn btn-outline" href="/api/export/pdf?asn=0&format=report" style="display:none">PDF Report</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="chart-row">
|
||||||
|
<div class="chart-wrap">
|
||||||
|
<h2>Atlas Coverage % (last 30 days)</h2>
|
||||||
|
<canvas id="coverageChart" height="120"></canvas>
|
||||||
|
</div>
|
||||||
|
<div class="forecast-box">
|
||||||
|
<div class="label">90-Day Forecast</div>
|
||||||
|
<div class="value">${forecast90d !== null ? forecast90d.toFixed(1) + '%' : '—'}</div>
|
||||||
|
<div class="unit">linear projection from 30-day trend</div>
|
||||||
|
<div style="margin-top:16px;font-size:11px;opacity:0.8">
|
||||||
|
Based on ${trend30.length} data points.<br/>
|
||||||
|
Current: ${coverageNow.toFixed(1)}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="chart-wrap">
|
||||||
|
<h2>Total ASPA Objects in RPKI (last 30 days)</h2>
|
||||||
|
<canvas id="asnChart" height="80"></canvas>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="meta">
|
||||||
|
Last snapshot: ${latest?.date ?? 'none'} · Denominator: ${atlasTotal.toLocaleString()} Atlas ASNs ·
|
||||||
|
<a href="/api/aspa-adoption-stats?period=30d" style="color:#4f46e5">JSON API</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const labels = ${JSON.stringify(trendLabels)};
|
||||||
|
const covData = ${JSON.stringify(trendValues)};
|
||||||
|
const asnData = ${JSON.stringify(asnValues)};
|
||||||
|
|
||||||
|
const commonOpts = {
|
||||||
|
responsive: true,
|
||||||
|
plugins: { legend: { display: false } },
|
||||||
|
scales: {
|
||||||
|
x: { grid: { color: '#f0eeff' }, ticks: { maxRotation: 45, font: { size: 10 } } },
|
||||||
|
y: { grid: { color: '#f0eeff' }, ticks: { font: { size: 10 } }, beginAtZero: false }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
new Chart(document.getElementById('coverageChart'), {
|
||||||
|
type: 'line',
|
||||||
|
data: {
|
||||||
|
labels,
|
||||||
|
datasets: [{
|
||||||
|
data: covData,
|
||||||
|
borderColor: '#4f46e5',
|
||||||
|
backgroundColor: 'rgba(79,70,229,0.08)',
|
||||||
|
fill: true,
|
||||||
|
tension: 0.3,
|
||||||
|
pointRadius: 3,
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
options: { ...commonOpts, scales: { ...commonOpts.scales, y: { ...commonOpts.scales.y, ticks: { callback: v => v + '%', font: { size: 10 } } } } }
|
||||||
|
});
|
||||||
|
|
||||||
|
new Chart(document.getElementById('asnChart'), {
|
||||||
|
type: 'bar',
|
||||||
|
data: {
|
||||||
|
labels,
|
||||||
|
datasets: [{
|
||||||
|
data: asnData,
|
||||||
|
backgroundColor: 'rgba(79,70,229,0.6)',
|
||||||
|
borderColor: '#4f46e5',
|
||||||
|
borderWidth: 1,
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
options: commonOpts
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
aspaAdoptionDailyHistory,
|
||||||
|
loadAspaAdoptionHistory,
|
||||||
|
saveAspaAdoptionHistory,
|
||||||
|
recordAspaAdoptionSnapshot,
|
||||||
|
getAdoptionTrend,
|
||||||
|
buildAspaAdoptionDashboard,
|
||||||
|
};
|
||||||
184
server/services/ripe-stat.js
Normal file
184
server/services/ripe-stat.js
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const { fetchJSON } = require("./http-helpers");
|
||||||
|
const localDb = require("../../local-db-client");
|
||||||
|
|
||||||
|
// RIPE Stat Source Cache + Semaphore (L2)
|
||||||
|
// Prevents 429 rate-limiting by throttling + caching responses
|
||||||
|
const ripeStatCache = new Map(); // key: "endpoint:resource" → {data, ts}
|
||||||
|
const RIPE_STAT_CACHE_MAX = 2000;
|
||||||
|
const RIPE_STAT_TTL = {
|
||||||
|
"announced-prefixes": 15 * 60 * 1000,
|
||||||
|
"asn-neighbours": 15 * 60 * 1000,
|
||||||
|
"as-overview": 60 * 60 * 1000,
|
||||||
|
"rir-stats-country": 24 * 60 * 60 * 1000,
|
||||||
|
"visibility": 15 * 60 * 1000,
|
||||||
|
"prefix-size-distribution": 60 * 60 * 1000,
|
||||||
|
"abuse-contact-finder": 24 * 60 * 60 * 1000,
|
||||||
|
"blocklist": 60 * 60 * 1000,
|
||||||
|
"reverse-dns-consistency": 60 * 60 * 1000,
|
||||||
|
"routing-status": 15 * 60 * 1000,
|
||||||
|
"bgp-updates": 15 * 60 * 1000,
|
||||||
|
"maxmind-geo-lite-pfx": 24 * 60 * 60 * 1000,
|
||||||
|
"looking-glass": 15 * 60 * 1000,
|
||||||
|
"whois": 24 * 60 * 60 * 1000,
|
||||||
|
"rpki-validation": 6 * 60 * 60 * 1000,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Counting semaphore — limits concurrent RIPE Stat requests
|
||||||
|
class Semaphore {
|
||||||
|
constructor(max) { this.max = max; this.current = 0; this.queue = []; }
|
||||||
|
acquire() {
|
||||||
|
if (this.current < this.max) { this.current++; return Promise.resolve(); }
|
||||||
|
return new Promise((resolve) => this.queue.push(resolve));
|
||||||
|
}
|
||||||
|
release() {
|
||||||
|
this.current--;
|
||||||
|
if (this.queue.length > 0) { this.current++; this.queue.shift()(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const ripeStatSemaphore = new Semaphore(15);
|
||||||
|
|
||||||
|
// Cached + throttled RIPE Stat fetch
|
||||||
|
// Renamed from fetchRipeStatCached 2026-07-16: a second function with that exact
|
||||||
|
// name was added ~150 lines down (the local-DB-routing wrapper) during the Postgres
|
||||||
|
// refactor. In JS, two `function` declarations sharing a name silently collapse to
|
||||||
|
// the LAST one in source order -- the real RIPE Stat fetch+cache+throttle
|
||||||
|
// implementation below was completely shadowed, and every call to
|
||||||
|
// fetchRipeStatCached() for any endpoint NOT in {announced-prefixes, asn-neighbours,
|
||||||
|
// as-overview, visibility, prefix-size-distribution} was unconditionally returning
|
||||||
|
// null instead of ever reaching this code. That silently broke blocklist,
|
||||||
|
// abuse-contact-finder, bgp-updates, routing-status, looking-glass, whois,
|
||||||
|
// reverse-dns-consistency, maxmind-geo-lite-pfx, and ixs lookups -- see the router's
|
||||||
|
// fallback call to this function for the fix.
|
||||||
|
async function fetchRipeStatCachedFromApi(url, options) {
|
||||||
|
// Extract endpoint name from URL for TTL lookup
|
||||||
|
const match = url.match(/\/data\/([^/]+)\//);
|
||||||
|
const endpoint = match ? match[1] : "default";
|
||||||
|
const resourceMatch = url.match(/resource=([^&]+)/);
|
||||||
|
const resource = resourceMatch ? resourceMatch[1] : url;
|
||||||
|
const cacheKey = endpoint + ":" + resource;
|
||||||
|
|
||||||
|
// Check cache
|
||||||
|
const cached = ripeStatCache.get(cacheKey);
|
||||||
|
const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000;
|
||||||
|
if (cached && (Date.now() - cached.ts) < ttl) {
|
||||||
|
return cached.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Throttle via semaphore
|
||||||
|
await ripeStatSemaphore.acquire();
|
||||||
|
try {
|
||||||
|
// Double-check cache after acquiring semaphore (another request may have filled it)
|
||||||
|
const cached2 = ripeStatCache.get(cacheKey);
|
||||||
|
if (cached2 && (Date.now() - cached2.ts) < ttl) {
|
||||||
|
return cached2.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await fetchJSON(url, options);
|
||||||
|
|
||||||
|
// Only cache successful results — never cache null (failed/rate-limited responses)
|
||||||
|
// Caching null causes cascading failures: retry hits cache, returns null again
|
||||||
|
if (result !== null) {
|
||||||
|
if (ripeStatCache.size >= RIPE_STAT_CACHE_MAX) {
|
||||||
|
ripeStatCache.delete(ripeStatCache.keys().next().value);
|
||||||
|
}
|
||||||
|
ripeStatCache.set(cacheKey, { data: result, ts: Date.now() });
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
ripeStatSemaphore.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cached + throttled RIPE Stat with one retry on failure
|
||||||
|
async function fetchRipeStatCachedWithRetry(url, options) {
|
||||||
|
const result = await fetchRipeStatCached(url, options);
|
||||||
|
if (result !== null) return result;
|
||||||
|
await new Promise(r => setTimeout(r, 1500));
|
||||||
|
return fetchRipeStatCached(url, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
// RIPE Stat cache disk persistence (skip null entries)
|
||||||
|
function saveRipeStatCacheToDisk(filePath) {
|
||||||
|
try {
|
||||||
|
const obj = {};
|
||||||
|
for (const [k, v] of ripeStatCache) {
|
||||||
|
if (v.data !== null) obj[k] = v;
|
||||||
|
}
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify({ ts: Date.now(), entries: obj }));
|
||||||
|
console.log("[RIPE-CACHE] Saved " + Object.keys(obj).length + " entries to disk");
|
||||||
|
} catch (e) {
|
||||||
|
console.warn("[RIPE-CACHE] Disk save failed:", e.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadRipeStatCacheFromDisk(filePath) {
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(filePath)) return false;
|
||||||
|
const raw = fs.readFileSync(filePath, "utf8");
|
||||||
|
const data = JSON.parse(raw);
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [k, v] of Object.entries(data.entries || {})) {
|
||||||
|
const match = k.match(/^([^:]+):/);
|
||||||
|
const endpoint = match ? match[1] : "default";
|
||||||
|
const ttl = RIPE_STAT_TTL[endpoint] || 15 * 60 * 1000;
|
||||||
|
if (now - v.ts < ttl) {
|
||||||
|
ripeStatCache.set(k, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log("[RIPE-CACHE] Loaded " + ripeStatCache.size + " entries from disk");
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
console.warn("[RIPE-CACHE] Disk load failed:", e.message);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MASTER RIPE Stat API wrapper (Local-first, zero external API calls).
|
||||||
|
// Analyzes RIPE Stat URL and dispatches to appropriate localDb function.
|
||||||
|
async function fetchRipeStatCached(url, options = {}) {
|
||||||
|
try {
|
||||||
|
// Detect which RIPE Stat endpoint this is and call local DB
|
||||||
|
if (url.includes('/announced-prefixes/')) {
|
||||||
|
const asnMatch = url.match(/resource=AS(\d+)/);
|
||||||
|
if (asnMatch) return await localDb.getRipeStatAnnouncedPrefixes(parseInt(asnMatch[1]));
|
||||||
|
}
|
||||||
|
if (url.includes('/asn-neighbours/')) {
|
||||||
|
const asnMatch = url.match(/resource=AS(\d+)/);
|
||||||
|
if (asnMatch) return await localDb.getRipeStatAsnNeighbours(parseInt(asnMatch[1]));
|
||||||
|
}
|
||||||
|
if (url.includes('/as-overview/')) {
|
||||||
|
const asnMatch = url.match(/resource=AS(\d+)/);
|
||||||
|
if (asnMatch) return await localDb.getRipeStatAsOverview(parseInt(asnMatch[1]));
|
||||||
|
}
|
||||||
|
if (url.includes('/visibility/')) {
|
||||||
|
const asnMatch = url.match(/resource=AS(\d+)/);
|
||||||
|
if (asnMatch) return await localDb.getRipeStatVisibility(parseInt(asnMatch[1]));
|
||||||
|
}
|
||||||
|
if (url.includes('/prefix-size-distribution/')) {
|
||||||
|
const asnMatch = url.match(/resource=AS(\d+)/);
|
||||||
|
if (asnMatch) return await localDb.getRipeStatPrefixSizeDistribution(parseInt(asnMatch[1]));
|
||||||
|
}
|
||||||
|
|
||||||
|
// For every other RIPE Stat endpoint (not mirrored into the local Postgres DB) --
|
||||||
|
// e.g. blocklist, abuse-contact-finder, bgp-updates, routing-status, looking-glass,
|
||||||
|
// whois, reverse-dns-consistency, maxmind-geo-lite-pfx, ixs -- fall through to the
|
||||||
|
// real RIPE Stat API with its own cache/throttle/never-cache-null protections,
|
||||||
|
// instead of returning null unconditionally (see fetchRipeStatCachedFromApi's
|
||||||
|
// header comment for how this silently broke ~9 checks until 2026-07-16).
|
||||||
|
return fetchRipeStatCachedFromApi(url, options);
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[fetchRipeStatCached] Error:", e.message);
|
||||||
|
return Promise.resolve(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
ripeStatCache,
|
||||||
|
fetchRipeStatCached,
|
||||||
|
fetchRipeStatCachedFromApi,
|
||||||
|
fetchRipeStatCachedWithRetry,
|
||||||
|
saveRipeStatCacheToDisk,
|
||||||
|
loadRipeStatCacheFromDisk,
|
||||||
|
Semaphore,
|
||||||
|
};
|
||||||
200
server/services/rpki.js
Normal file
200
server/services/rpki.js
Normal file
@ -0,0 +1,200 @@
|
|||||||
|
const https = require("https");
|
||||||
|
const localDb = require("../../local-db-client");
|
||||||
|
const { UA } = require("../data/constants");
|
||||||
|
const { fetchJSON } = require("./http-helpers");
|
||||||
|
const { roaStore } = require("../caches/roa-store");
|
||||||
|
const { recordAspaAdoptionSnapshot } = require("../aspa-adoption/tracker");
|
||||||
|
|
||||||
|
// RPKI ASPA + ROA Cache from Cloudflare RPKI JSON feed
|
||||||
|
const rpkiAspaMap = new Map(); // customer_asid -> Set<provider_asn>
|
||||||
|
let rpkiAspaLastFetch = 0;
|
||||||
|
let rpkiAspaFetching = false;
|
||||||
|
|
||||||
|
function fetchRpkiAspaFeed() {
|
||||||
|
if (rpkiAspaFetching) return Promise.resolve();
|
||||||
|
rpkiAspaFetching = true;
|
||||||
|
console.log("[RPKI] Fetching Cloudflare RPKI feed (ASPA + ROA)...");
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const options = {
|
||||||
|
headers: { "User-Agent": UA },
|
||||||
|
timeout: 120000,
|
||||||
|
};
|
||||||
|
https.get("https://rpki.cloudflare.com/rpki.json", options, (res) => {
|
||||||
|
let data = "";
|
||||||
|
res.on("data", (chunk) => (data += chunk));
|
||||||
|
res.on("end", () => {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(data);
|
||||||
|
|
||||||
|
// Load ASPA objects
|
||||||
|
const aspas = parsed.aspas || [];
|
||||||
|
rpkiAspaMap.clear();
|
||||||
|
aspas.forEach((a) => {
|
||||||
|
const customerAsid = Number(a.customer_asid);
|
||||||
|
const providers = (a.providers || []).map(Number);
|
||||||
|
rpkiAspaMap.set(customerAsid, new Set(providers));
|
||||||
|
});
|
||||||
|
|
||||||
|
// Load ROA objects into local store (eliminates RIPE Stat per-prefix calls)
|
||||||
|
const roas = parsed.roas || [];
|
||||||
|
roaStore.build(roas);
|
||||||
|
roaStore.saveToDisk("/opt/peercortex-app/.roa-cache.json");
|
||||||
|
|
||||||
|
// Track ASPA adoption — persist to disk + update in-memory trend
|
||||||
|
recordAspaAdoptionSnapshot(rpkiAspaMap.size, roaStore.count, rpkiAspaMap);
|
||||||
|
|
||||||
|
rpkiAspaLastFetch = Date.now();
|
||||||
|
console.log("[RPKI] Loaded " + rpkiAspaMap.size + " ASPA objects + " + roaStore.count + " ROAs from Cloudflare feed");
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[RPKI] Failed to parse RPKI feed:", e.message);
|
||||||
|
}
|
||||||
|
rpkiAspaFetching = false;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
}).on("error", (e) => {
|
||||||
|
console.error("[RPKI] Fetch failed:", e.message);
|
||||||
|
rpkiAspaFetching = false;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure ASPA + ROA cache is fresh
|
||||||
|
async function ensureAspaCache() {
|
||||||
|
if (Date.now() - rpkiAspaLastFetch > 4 * 60 * 60 * 1000) {
|
||||||
|
await fetchRpkiAspaFeed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lookup ASPA object for a given ASN from the RPKI feed cache. feedLoaded
|
||||||
|
// distinguishes "the feed loaded fine and this ASN genuinely has no ASPA object"
|
||||||
|
// from "the Cloudflare feed fetch has never succeeded, so we don't actually know" --
|
||||||
|
// rpkiAspaLastFetch is only set on a successful parse (see fetchRpkiAspaFeed), so
|
||||||
|
// 0 means every attempt so far has failed and exists:false here is not trustworthy.
|
||||||
|
function lookupAspaFromRpki(asn) {
|
||||||
|
const asnNum = Number(asn);
|
||||||
|
const feedLoaded = rpkiAspaLastFetch > 0;
|
||||||
|
if (rpkiAspaMap.has(asnNum)) {
|
||||||
|
const providers = rpkiAspaMap.get(asnNum);
|
||||||
|
return { exists: true, providers: [...providers].sort((a, b) => a - b), feedLoaded };
|
||||||
|
}
|
||||||
|
return { exists: false, providers: [], feedLoaded };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate RPKI for a prefix — uses local PostgreSQL database (sub-10ms, zero external API calls)
|
||||||
|
// Returns: { prefix, status: "valid"|"invalid"|"not_found"|"unavailable", validating_roas: N }
|
||||||
|
// "not_found" means the DB was queried and genuinely has no covering ROA -- a real result.
|
||||||
|
// "unavailable" means the DB query itself failed (connection/timeout/error) -- we don't know.
|
||||||
|
// Conflating these used to make DB hiccups masquerade as "no ROA published" (a real, worse-
|
||||||
|
// sounding finding). Callers computing coverage percentages should exclude "unavailable".
|
||||||
|
async function validateRPKIWithCache(asn, prefix) {
|
||||||
|
try {
|
||||||
|
const result = await localDb.validateRpki(prefix, asn);
|
||||||
|
if (result.status === 'valid') {
|
||||||
|
return { prefix, status: "valid", validating_roas: 1 };
|
||||||
|
} else if (result.status === 'invalid') {
|
||||||
|
return { prefix, status: "invalid", validating_roas: 1 };
|
||||||
|
} else if (result.status === 'unknown') {
|
||||||
|
return { prefix, status: "unavailable", validating_roas: 0 };
|
||||||
|
} else {
|
||||||
|
// 'not-found': genuinely no covering ROA
|
||||||
|
return { prefix, status: "not_found", validating_roas: 0 };
|
||||||
|
}
|
||||||
|
} catch (_e) {
|
||||||
|
console.error("[RPKI] Error validating " + prefix + ":", _e.message);
|
||||||
|
return { prefix, status: "unavailable", validating_roas: 0 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Feature 30: RIPE NCC RPKI Validator cross-check (max 5 prefixes)
|
||||||
|
async function fetchRipeRpkiValidator(asn, prefix) {
|
||||||
|
try {
|
||||||
|
const encoded = encodeURIComponent(prefix);
|
||||||
|
const url = "https://rpki-validator.ripe.net/api/v1/validity/AS" + asn + "/" + encoded;
|
||||||
|
const result = await fetchJSON(url, { timeout: 5000 });
|
||||||
|
if (result && result.validated_route) {
|
||||||
|
return {
|
||||||
|
prefix: prefix,
|
||||||
|
validity: result.validated_route.validity || {},
|
||||||
|
state: (result.validated_route.validity && result.validated_route.validity.state) || "unknown",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { prefix: prefix, state: "unknown", error: "no_data" };
|
||||||
|
} catch (_e) {
|
||||||
|
return { prefix: prefix, state: "error", error: "timeout_or_unavailable" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cross-check a sample of prefixes against RIPE RPKI Validator (max 5, in parallel).
|
||||||
|
// agreement_pct only counts pairs where BOTH sources actually returned a real
|
||||||
|
// verdict -- a zero-sample or all-RIPE-lookups-failed result reports agreement_pct:
|
||||||
|
// null (not a fabricated 100), since nothing was actually cross-checked.
|
||||||
|
async function crossCheckRpki(asn, prefixes, localResults) {
|
||||||
|
const sample = prefixes.slice(0, 5);
|
||||||
|
if (sample.length === 0) return { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: null, disagreements: [], sample_size: 0, comparable_size: 0 };
|
||||||
|
|
||||||
|
const ripeResults = await Promise.all(
|
||||||
|
sample.map((pfx) => fetchRipeRpkiValidator(asn, pfx))
|
||||||
|
);
|
||||||
|
|
||||||
|
const localMap = new Map();
|
||||||
|
for (const lr of localResults) {
|
||||||
|
localMap.set(lr.prefix, lr.status);
|
||||||
|
}
|
||||||
|
|
||||||
|
let cloudflareValid = 0;
|
||||||
|
let ripeValid = 0;
|
||||||
|
let agreements = 0;
|
||||||
|
let comparable = 0;
|
||||||
|
const disagreements = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < sample.length; i++) {
|
||||||
|
const pfx = sample[i];
|
||||||
|
const cfStatus = localMap.get(pfx) || "not_found";
|
||||||
|
const ripeState = ripeResults[i].state;
|
||||||
|
|
||||||
|
const cfIsValid = cfStatus === "valid";
|
||||||
|
const ripeIsValid = ripeState === "valid" || ripeState === "VALID";
|
||||||
|
|
||||||
|
if (cfIsValid) cloudflareValid++;
|
||||||
|
if (ripeIsValid) ripeValid++;
|
||||||
|
|
||||||
|
// A failed/unknown RIPE lookup isn't a comparison at all -- exclude it from
|
||||||
|
// the agreement percentage instead of counting it as a silent "agreement".
|
||||||
|
if (ripeState === "error" || ripeState === "unknown") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
comparable++;
|
||||||
|
if (cfIsValid === ripeIsValid) {
|
||||||
|
agreements++;
|
||||||
|
} else {
|
||||||
|
disagreements.push({
|
||||||
|
prefix: pfx,
|
||||||
|
cloudflare: cfStatus,
|
||||||
|
ripe: ripeState,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const agreementPct = comparable > 0 ? Math.round((agreements / comparable) * 100) : null;
|
||||||
|
return { cloudflare_valid: cloudflareValid, ripe_valid: ripeValid, agreement_pct: agreementPct, disagreements: disagreements, sample_size: sample.length, comparable_size: comparable };
|
||||||
|
}
|
||||||
|
|
||||||
|
// rpkiAspaLastFetch is a reassigned `let`, not a mutated object -- a bare
|
||||||
|
// export would only capture its value (0) at require time. Expose it via a
|
||||||
|
// getter so callers (e.g. /api/health) always see the current value.
|
||||||
|
function getRpkiAspaLastFetch() {
|
||||||
|
return rpkiAspaLastFetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
rpkiAspaMap,
|
||||||
|
getRpkiAspaLastFetch,
|
||||||
|
fetchRpkiAspaFeed,
|
||||||
|
ensureAspaCache,
|
||||||
|
lookupAspaFromRpki,
|
||||||
|
validateRPKIWithCache,
|
||||||
|
fetchRipeRpkiValidator,
|
||||||
|
crossCheckRpki,
|
||||||
|
};
|
||||||
Loading…
x
Reference in New Issue
Block a user