feat: MANRS HTML scraping, AS relationships endpoint, rebrand to ASN News
- MANRS: replace broken Observatory API with public participants page scraping (www.manrs.org/netops/participants/), 24h cache, returns pass/fail with member count - /api/validate: add 'relationships' field (upstreams/downstreams/top_peers) sourced from RIPE Stat asn-neighbours, no extra API calls needed - /api/relationships?asn=X: new dedicated endpoint with resolved AS names, full upstream/downstream/peer lists sorted by power score, 10min cache - editorial: rebrand 'The ASN Newspaper' → 'The ASN News' across index-editorial.html
This commit is contained in:
Rene Fichtmueller
parent
8f51f32dc3
commit
96b6ef2d4a
11
.claude/launch.json
Normal file
11
.claude/launch.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": "0.0.1",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "peercortex-redesign",
|
||||
"runtimeExecutable": "npx",
|
||||
"runtimeArgs": ["serve", "-p", "8902", "public"],
|
||||
"port": 8902
|
||||
}
|
||||
]
|
||||
}
|
||||
1129
public/design-draft-editorial.html
Normal file
1129
public/design-draft-editorial.html
Normal file
File diff suppressed because it is too large
Load Diff
@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>PeerCortex — The ASN Newspaper</title>
|
||||
<title>PeerCortex — The ASN News</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,700;0,800;0,900;1,400&family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
||||
@ -337,9 +337,9 @@ a:hover{color:var(--purple)}
|
||||
<div class="ed-masthead-top">
|
||||
<div>
|
||||
<div class="ed-logo">PeerCortex<sup>β</sup></div>
|
||||
<div class="ed-tagline">The ASN Newspaper</div>
|
||||
<div class="ed-tagline">The ASN News</div>
|
||||
</div>
|
||||
<div class="ed-masthead-meta">The ASN Newspaper<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div>
|
||||
<div class="ed-masthead-meta">The ASN News<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div>
|
||||
</div>
|
||||
<hr class="ed-rule-h">
|
||||
<nav class="ed-nav">
|
||||
@ -677,7 +677,7 @@ a:hover{color:var(--purple)}
|
||||
|
||||
<!-- FOOTER -->
|
||||
<footer class="ed-footer">
|
||||
<div class="ed-footer-name">PeerCortex — The ASN Newspaper</div>
|
||||
<div class="ed-footer-name">PeerCortex — The ASN News</div>
|
||||
<nav class="ed-footer-links">
|
||||
<a href="https://www.peeringdb.com" target="_blank">PeeringDB</a>
|
||||
<a href="https://stat.ripe.net" target="_blank">RIPE Stat</a>
|
||||
|
||||
186
server.js
186
server.js
@ -87,6 +87,69 @@ const CACHE_TTL_ASPA = 4 * 60 * 60 * 1000; // 4 hours
|
||||
const CACHE_TTL_NEWS = 10 * 60 * 1000; // 10 minutes
|
||||
const CACHE_TTL_DEFAULT = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
// ============================================================
|
||||
// MANRS Participants Cache (scraped from public HTML page, 24h TTL)
|
||||
// ============================================================
|
||||
let manrsAsnSet = null; // Set<string> of member ASNs
|
||||
let manrsLastFetch = 0;
|
||||
let manrsFetching = false;
|
||||
const MANRS_CACHE_TTL = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
async function ensureManrsCache() {
|
||||
const now = Date.now();
|
||||
if (manrsAsnSet && (now - manrsLastFetch) < MANRS_CACHE_TTL) return;
|
||||
if (manrsFetching) {
|
||||
// Wait up to 8s for in-progress fetch
|
||||
for (let i = 0; i < 80; i++) {
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
if (manrsAsnSet) return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
manrsFetching = true;
|
||||
try {
|
||||
const html = await new Promise((resolve, reject) => {
|
||||
const opts = { hostname: "www.manrs.org", path: "/netops/participants/", method: "GET", timeout: 15000,
|
||||
headers: { "User-Agent": UA, "Accept": "text/html" } };
|
||||
const req = https.request(opts, res => {
|
||||
let body = "";
|
||||
res.on("data", d => { body += d; });
|
||||
res.on("end", () => resolve(body));
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.on("timeout", () => { req.destroy(); reject(new Error("MANRS fetch timeout")); });
|
||||
req.end();
|
||||
});
|
||||
// Extract ASNs from <td class="asns">267490</td> — may contain multiple space-separated ASNs
|
||||
const set = new Set();
|
||||
const re = /<td[^>]*class="asns"[^>]*>([\d\s,]+)<\/td>/gi;
|
||||
let m;
|
||||
while ((m = re.exec(html)) !== null) {
|
||||
m[1].split(/[\s,]+/).forEach(a => { const n = a.trim(); if (n) set.add(n); });
|
||||
}
|
||||
if (set.size > 0) {
|
||||
manrsAsnSet = set;
|
||||
manrsLastFetch = Date.now();
|
||||
console.log("[MANRS] Loaded " + set.size + " participant ASNs from manrs.org");
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[MANRS] Failed to fetch participants:", e.message);
|
||||
} finally {
|
||||
manrsFetching = false;
|
||||
}
|
||||
}
|
||||
|
||||
function checkManrsMembership(asn) {
|
||||
if (!manrsAsnSet) return { status: "info", participant: "unknown", message: "MANRS data not yet loaded", note: "https://www.manrs.org/netops/participants/" };
|
||||
const isMember = manrsAsnSet.has(String(asn));
|
||||
return {
|
||||
status: isMember ? "pass" : "fail",
|
||||
participant: isMember,
|
||||
member_count: manrsAsnSet.size,
|
||||
note: isMember ? "Confirmed MANRS Network Operator participant" : "Not listed as MANRS participant — https://www.manrs.org/beamanrs/",
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Infrastructure overlay caches
|
||||
let subCableCache = null; // TeleGeography submarine cables (24h)
|
||||
@ -2249,20 +2312,10 @@ const server = http.createServer(async (req, res) => {
|
||||
return { status: listedPrefixes.length === 0 ? "pass" : "fail", checked: results.length, listed_prefixes: listedPrefixes };
|
||||
}).catch(function(e) { return { status: "error", error: String(e) }; });
|
||||
|
||||
// 16. MANRS Compliance (observatory API requires auth — use fallback indicators)
|
||||
validationPromises.manrs = fetchJSON("https://observatory.manrs.org/api/v2/asn/" + rawAsn + "/conformance", { timeout: 5000 }).then(function(data) {
|
||||
if (!data || data.error || data.detail === "Authentication credentials were not provided.") {
|
||||
// API unavailable — check MANRS indicators: RPKI ROA + IRR objects as proxy
|
||||
var hasRoa = samplePrefixes.length > 0; // will be checked by RPKI validation
|
||||
var hasIrr = !!(net.irr_as_set);
|
||||
if (hasRoa && hasIrr) {
|
||||
return { status: "info", participant: "unknown", message: "MANRS Observatory API requires authentication — cannot verify membership. Network has ROA + IRR objects (positive indicators).", note: "Unable to verify — MANRS API requires auth. Check https://observatory.manrs.org/asn/" + rawAsn };
|
||||
}
|
||||
return { status: "info", participant: "unknown", message: "Unable to verify MANRS membership (API requires authentication)", note: "Check manually: https://observatory.manrs.org/asn/" + rawAsn };
|
||||
}
|
||||
var score = data.conformance_score || data.score || 0;
|
||||
return { status: score >= 50 ? "pass" : "warning", participant: true, score: score, details: data };
|
||||
}).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable", note: "https://observatory.manrs.org/asn/" + rawAsn }; });
|
||||
// 16. MANRS Compliance — scraped from public participants list (24h cache)
|
||||
validationPromises.manrs = ensureManrsCache().then(function() {
|
||||
return checkManrsMembership(rawAsn);
|
||||
}).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable: " + e.message, note: "https://www.manrs.org/netops/participants/" }; });
|
||||
|
||||
// 17. Reverse DNS Coverage (sample up to 20 prefixes for better coverage)
|
||||
var rdnsSampleSize = Math.min(20, samplePrefixes.length);
|
||||
@ -2537,6 +2590,24 @@ const server = http.createServer(async (req, res) => {
|
||||
var healthScore = totalWeight > 0 ? Math.round((earnedScore / totalWeight) * 100) : 0;
|
||||
var duration = Date.now() - start;
|
||||
|
||||
// Build relationships from neighbour data
|
||||
var relNeighbours = neighbourData && neighbourData.data && neighbourData.data.neighbour_counts
|
||||
? neighbourData.data.neighbour_counts : {};
|
||||
var relList = neighbourData && neighbourData.data && neighbourData.data.neighbours
|
||||
? neighbourData.data.neighbours : [];
|
||||
var relUpstreams = relList.filter(function(n) { return n.type === "left"; })
|
||||
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
|
||||
.slice(0, 20)
|
||||
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
|
||||
var relDownstreams = relList.filter(function(n) { return n.type === "right"; })
|
||||
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
|
||||
.slice(0, 20)
|
||||
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
|
||||
var relPeers = relList.filter(function(n) { return n.type === "uncertain"; })
|
||||
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
|
||||
.slice(0, 30)
|
||||
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
|
||||
|
||||
return res.end(
|
||||
JSON.stringify(
|
||||
{
|
||||
@ -2546,6 +2617,14 @@ const server = http.createServer(async (req, res) => {
|
||||
health_score: healthScore,
|
||||
score_breakdown: checkResults,
|
||||
validations: validations,
|
||||
relationships: {
|
||||
counts: { upstreams: relNeighbours.left || relUpstreams.length, downstreams: relNeighbours.right || relDownstreams.length, peers: relNeighbours.unique || relPeers.length, uncertain: relNeighbours.uncertain || 0 },
|
||||
upstreams: relUpstreams,
|
||||
downstreams: relDownstreams,
|
||||
top_peers: relPeers,
|
||||
source: "RIPE Stat asn-neighbours",
|
||||
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
|
||||
},
|
||||
},
|
||||
null,
|
||||
2
|
||||
@ -3036,6 +3115,82 @@ const server = http.createServer(async (req, res) => {
|
||||
return;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// AS Relationships endpoint: /api/relationships?asn=X
|
||||
// Returns upstream providers, downstream customers, and peers
|
||||
// with resolved names. Based on RIPE Stat asn-neighbours.
|
||||
// ============================================================
|
||||
if (reqPath === "/api/relationships") {
|
||||
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
|
||||
if (!rawAsn) {
|
||||
res.writeHead(400);
|
||||
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
|
||||
}
|
||||
const cacheKey = "relationships:" + rawAsn;
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) {
|
||||
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
|
||||
return res.end(JSON.stringify(cached));
|
||||
}
|
||||
const start = Date.now();
|
||||
try {
|
||||
const neighbourData = await fetchRipeStatCached(
|
||||
"https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1",
|
||||
{ timeout: 30000 }
|
||||
);
|
||||
const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || [];
|
||||
const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {};
|
||||
|
||||
const upstreams = neighbours.filter(n => n.type === "left").sort((a,b) => (b.power||0)-(a.power||0));
|
||||
const downstreams = neighbours.filter(n => n.type === "right").sort((a,b) => (b.power||0)-(a.power||0));
|
||||
const peers = neighbours.filter(n => n.type === "uncertain").sort((a,b) => (b.power||0)-(a.power||0));
|
||||
|
||||
// Resolve AS names for top entries (upstreams + downstreams all, top 20 peers)
|
||||
const toResolve = [...upstreams, ...downstreams, ...peers.slice(0, 20)];
|
||||
const resolvedNames = {};
|
||||
await Promise.race([
|
||||
Promise.all(toResolve.map(n =>
|
||||
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 })
|
||||
.then(r => { if (r && r.data && r.data.holder) resolvedNames[n.asn] = r.data.holder; })
|
||||
.catch(() => {})
|
||||
)),
|
||||
new Promise(r => setTimeout(r, 5000)),
|
||||
]);
|
||||
|
||||
const fmt = n => ({
|
||||
asn: n.asn,
|
||||
name: resolvedNames[n.asn] || "",
|
||||
power: n.power || 0,
|
||||
v4_peers: n.v4_peers || 0,
|
||||
v6_peers: n.v6_peers || 0,
|
||||
});
|
||||
|
||||
const result = {
|
||||
asn: parseInt(rawAsn),
|
||||
query_time: new Date().toISOString(),
|
||||
duration_ms: Date.now() - start,
|
||||
counts: {
|
||||
upstreams: counts.left || upstreams.length,
|
||||
downstreams: counts.right || downstreams.length,
|
||||
peers_total: counts.unique || peers.length,
|
||||
uncertain: counts.uncertain || peers.length,
|
||||
},
|
||||
upstreams: upstreams.map(fmt),
|
||||
downstreams: downstreams.map(fmt),
|
||||
peers: peers.slice(0, 50).map(fmt),
|
||||
methodology: "RIPE Stat asn-neighbours API. left=upstream providers (carry your traffic), right=downstream customers (you carry their traffic), uncertain=lateral peers. Sorted by power score (number of prefixes seen via this relationship).",
|
||||
source_url: "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn,
|
||||
};
|
||||
|
||||
cacheSet(cacheKey, result, 10 * 60 * 1000); // 10 min cache
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
return res.end(JSON.stringify(result, null, 2));
|
||||
} catch (err) {
|
||||
res.writeHead(500);
|
||||
return res.end(JSON.stringify({ error: "Relationships lookup failed", message: err.message }));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Compare endpoint: /api/compare?asn1=X&asn2=Y
|
||||
// ============================================================
|
||||
@ -3788,7 +3943,8 @@ roaStore.loadFromDisk("/opt/peercortex-app/.roa-cache.json");
|
||||
pdbSourceCache.loadFromDisk("/opt/peercortex-app/.pdb-source-cache.json");
|
||||
loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json");
|
||||
|
||||
// Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries
|
||||
// Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries + MANRS participants
|
||||
ensureManrsCache(); // fire-and-forget, 24h cache
|
||||
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
|
||||
server.listen(PORT, "0.0.0.0", () => {
|
||||
console.log("PeerCortex v0.6.0 running on http://0.0.0.0:" + PORT);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user