rene/PeerCortex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: MANRS HTML scraping, AS relationships endpoint, rebrand to ASN News

Browse Source 
- MANRS: replace broken Observatory API with public participants page scraping
  (www.manrs.org/netops/participants/), 24h cache, returns pass/fail with member count
- /api/validate: add 'relationships' field (upstreams/downstreams/top_peers)
  sourced from RIPE Stat asn-neighbours, no extra API calls needed
- /api/relationships?asn=X: new dedicated endpoint with resolved AS names,
  full upstream/downstream/peer lists sorted by power score, 10min cache
- editorial: rebrand 'The ASN Newspaper' → 'The ASN News' across index-editorial.html
This commit is contained in:
Rene Fichtmueller 2026-03-30 21:23:42 +02:00
parent 8f51f32dc3
commit 96b6ef2d4a
4 changed files with 1315 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.claude/launch.json Normal file
View File

@ -0,0 +1,11 @@
{
"version": "0.0.1",
"configurations": [
{
"name": "peercortex-redesign",
"runtimeExecutable": "npx",
"runtimeArgs": ["serve", "-p", "8902", "public"],
"port": 8902
}
]
}

1129
public/design-draft-editorial.html Normal file
View File

File diff suppressed because it is too large Load Diff

8
public/index-editorial.html
View File

@ -3,7 +3,7 @@
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>PeerCortex — The ASN Newspaper</title> <title>PeerCortex — The ASN News</title>
<link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,700;0,800;0,900;1,400&family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet"> <link href="https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,700;0,800;0,900;1,400&family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
@ -337,9 +337,9 @@ a:hover{color:var(--purple)}
<div class="ed-masthead-top"> <div class="ed-masthead-top">
<div> <div>
<div class="ed-logo">PeerCortex<sup>β</sup></div> <div class="ed-logo">PeerCortex<sup>β</sup></div>
<div class="ed-tagline">The ASN Newspaper</div> <div class="ed-tagline">The ASN News</div>
</div> </div>
<div class="ed-masthead-meta">The ASN Newspaper<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div> <div class="ed-masthead-meta">The ASN News<br><span style="font-family:var(--mono)">v2.peercortex.org · routing intelligence</span></div>
</div> </div>
<hr class="ed-rule-h"> <hr class="ed-rule-h">
<nav class="ed-nav"> <nav class="ed-nav">
@ -677,7 +677,7 @@ a:hover{color:var(--purple)}
<!-- FOOTER --> <!-- FOOTER -->
<footer class="ed-footer"> <footer class="ed-footer">
<div class="ed-footer-name">PeerCortex — The ASN Newspaper</div> <div class="ed-footer-name">PeerCortex — The ASN News</div>
<nav class="ed-footer-links"> <nav class="ed-footer-links">
<a href="https://www.peeringdb.com" target="_blank">PeeringDB</a> <a href="https://www.peeringdb.com" target="_blank">PeeringDB</a>
<a href="https://stat.ripe.net" target="_blank">RIPE Stat</a> <a href="https://stat.ripe.net" target="_blank">RIPE Stat</a>

186
server.js
View File

@ -87,6 +87,69 @@ const CACHE_TTL_ASPA = 4 * 60 * 60 * 1000; // 4 hours
const CACHE_TTL_NEWS = 10 * 60 * 1000; // 10 minutes const CACHE_TTL_NEWS = 10 * 60 * 1000; // 10 minutes
const CACHE_TTL_DEFAULT = 5 * 60 * 1000; // 5 minutes const CACHE_TTL_DEFAULT = 5 * 60 * 1000; // 5 minutes
// ============================================================
// MANRS Participants Cache (scraped from public HTML page, 24h TTL)
// ============================================================
let manrsAsnSet = null; // Set<string> of member ASNs
let manrsLastFetch = 0;
let manrsFetching = false;
const MANRS_CACHE_TTL = 24 * 60 * 60 * 1000; // 24 hours
async function ensureManrsCache() {
const now = Date.now();
if (manrsAsnSet && (now - manrsLastFetch) < MANRS_CACHE_TTL) return;
if (manrsFetching) {
// Wait up to 8s for in-progress fetch
for (let i = 0; i < 80; i++) {
await new Promise(r => setTimeout(r, 100));
if (manrsAsnSet) return;
}
return;
}
manrsFetching = true;
try {
const html = await new Promise((resolve, reject) => {
const opts = { hostname: "www.manrs.org", path: "/netops/participants/", method: "GET", timeout: 15000,
headers: { "User-Agent": UA, "Accept": "text/html" } };
const req = https.request(opts, res => {
let body = "";
res.on("data", d => { body += d; });
res.on("end", () => resolve(body));
});
req.on("error", reject);
req.on("timeout", () => { req.destroy(); reject(new Error("MANRS fetch timeout")); });
req.end();
});
// Extract ASNs from <td class="asns">267490</td> — may contain multiple space-separated ASNs
const set = new Set();
const re = /<td[^>]*class="asns"[^>]*>([\d\s,]+)<\/td>/gi;
let m;
while ((m = re.exec(html)) !== null) {
m[1].split(/[\s,]+/).forEach(a => { const n = a.trim(); if (n) set.add(n); });
}
if (set.size > 0) {
manrsAsnSet = set;
manrsLastFetch = Date.now();
console.log("[MANRS] Loaded " + set.size + " participant ASNs from manrs.org");
}
} catch (e) {
console.warn("[MANRS] Failed to fetch participants:", e.message);
} finally {
manrsFetching = false;
}
}
function checkManrsMembership(asn) {
if (!manrsAsnSet) return { status: "info", participant: "unknown", message: "MANRS data not yet loaded", note: "https://www.manrs.org/netops/participants/" };
const isMember = manrsAsnSet.has(String(asn));
return {
status: isMember ? "pass" : "fail",
participant: isMember,
member_count: manrsAsnSet.size,
note: isMember ? "Confirmed MANRS Network Operator participant" : "Not listed as MANRS participant — https://www.manrs.org/beamanrs/",
};
}
// ============================================================ // ============================================================
// Infrastructure overlay caches // Infrastructure overlay caches
let subCableCache = null; // TeleGeography submarine cables (24h) let subCableCache = null; // TeleGeography submarine cables (24h)
@ -2249,20 +2312,10 @@ const server = http.createServer(async (req, res) => {
return { status: listedPrefixes.length === 0 ? "pass" : "fail", checked: results.length, listed_prefixes: listedPrefixes }; return { status: listedPrefixes.length === 0 ? "pass" : "fail", checked: results.length, listed_prefixes: listedPrefixes };
}).catch(function(e) { return { status: "error", error: String(e) }; }); }).catch(function(e) { return { status: "error", error: String(e) }; });
// 16. MANRS Compliance (observatory API requires auth — use fallback indicators) // 16. MANRS Compliance — scraped from public participants list (24h cache)
validationPromises.manrs = fetchJSON("https://observatory.manrs.org/api/v2/asn/" + rawAsn + "/conformance", { timeout: 5000 }).then(function(data) { validationPromises.manrs = ensureManrsCache().then(function() {
if (!data || data.error || data.detail === "Authentication credentials were not provided.") { return checkManrsMembership(rawAsn);
// API unavailable — check MANRS indicators: RPKI ROA + IRR objects as proxy }).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable: " + e.message, note: "https://www.manrs.org/netops/participants/" }; });
var hasRoa = samplePrefixes.length > 0; // will be checked by RPKI validation
var hasIrr = !!(net.irr_as_set);
if (hasRoa && hasIrr) {
return { status: "info", participant: "unknown", message: "MANRS Observatory API requires authentication — cannot verify membership. Network has ROA + IRR objects (positive indicators).", note: "Unable to verify — MANRS API requires auth. Check https://observatory.manrs.org/asn/" + rawAsn };
}
return { status: "info", participant: "unknown", message: "Unable to verify MANRS membership (API requires authentication)", note: "Check manually: https://observatory.manrs.org/asn/" + rawAsn };
}
var score = data.conformance_score || data.score || 0;
return { status: score >= 50 ? "pass" : "warning", participant: true, score: score, details: data };
}).catch(function(e) { return { status: "info", participant: "unknown", message: "MANRS check unavailable", note: "https://observatory.manrs.org/asn/" + rawAsn }; });
// 17. Reverse DNS Coverage (sample up to 20 prefixes for better coverage) // 17. Reverse DNS Coverage (sample up to 20 prefixes for better coverage)
var rdnsSampleSize = Math.min(20, samplePrefixes.length); var rdnsSampleSize = Math.min(20, samplePrefixes.length);
@ -2537,6 +2590,24 @@ const server = http.createServer(async (req, res) => {
var healthScore = totalWeight > 0 ? Math.round((earnedScore / totalWeight) * 100) : 0; var healthScore = totalWeight > 0 ? Math.round((earnedScore / totalWeight) * 100) : 0;
var duration = Date.now() - start; var duration = Date.now() - start;
// Build relationships from neighbour data
var relNeighbours = neighbourData && neighbourData.data && neighbourData.data.neighbour_counts
? neighbourData.data.neighbour_counts : {};
var relList = neighbourData && neighbourData.data && neighbourData.data.neighbours
? neighbourData.data.neighbours : [];
var relUpstreams = relList.filter(function(n) { return n.type === "left"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 20)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
var relDownstreams = relList.filter(function(n) { return n.type === "right"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 20)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
var relPeers = relList.filter(function(n) { return n.type === "uncertain"; })
.sort(function(a, b) { return (b.power || 0) - (a.power || 0); })
.slice(0, 30)
.map(function(n) { return { asn: n.asn, power: n.power || 0, v4_peers: n.v4_peers || 0, v6_peers: n.v6_peers || 0 }; });
return res.end( return res.end(
JSON.stringify( JSON.stringify(
{ {
@ -2546,6 +2617,14 @@ const server = http.createServer(async (req, res) => {
health_score: healthScore, health_score: healthScore,
score_breakdown: checkResults, score_breakdown: checkResults,
validations: validations, validations: validations,
relationships: {
counts: { upstreams: relNeighbours.left || relUpstreams.length, downstreams: relNeighbours.right || relDownstreams.length, peers: relNeighbours.unique || relPeers.length, uncertain: relNeighbours.uncertain || 0 },
upstreams: relUpstreams,
downstreams: relDownstreams,
top_peers: relPeers,
source: "RIPE Stat asn-neighbours",
note: "left=upstream providers, right=downstream customers, uncertain=peers. Sorted by power score.",
},
}, },
null, null,
2 2
@ -3036,6 +3115,82 @@ const server = http.createServer(async (req, res) => {
return; return;
} }
// ============================================================
// AS Relationships endpoint: /api/relationships?asn=X
// Returns upstream providers, downstream customers, and peers
// with resolved names. Based on RIPE Stat asn-neighbours.
// ============================================================
if (reqPath === "/api/relationships") {
const rawAsn = (url.searchParams.get("asn") || "").replace(/[^0-9]/g, "");
if (!rawAsn) {
res.writeHead(400);
return res.end(JSON.stringify({ error: "Missing or invalid ASN parameter" }));
}
const cacheKey = "relationships:" + rawAsn;
const cached = cacheGet(cacheKey);
if (cached) {
res.writeHead(200, { "Content-Type": "application/json", "X-Cache": "HIT" });
return res.end(JSON.stringify(cached));
}
const start = Date.now();
try {
const neighbourData = await fetchRipeStatCached(
"https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn + "&lod=1",
{ timeout: 30000 }
);
const neighbours = (neighbourData && neighbourData.data && neighbourData.data.neighbours) || [];
const counts = (neighbourData && neighbourData.data && neighbourData.data.neighbour_counts) || {};
const upstreams = neighbours.filter(n => n.type === "left").sort((a,b) => (b.power||0)-(a.power||0));
const downstreams = neighbours.filter(n => n.type === "right").sort((a,b) => (b.power||0)-(a.power||0));
const peers = neighbours.filter(n => n.type === "uncertain").sort((a,b) => (b.power||0)-(a.power||0));
// Resolve AS names for top entries (upstreams + downstreams all, top 20 peers)
const toResolve = [...upstreams, ...downstreams, ...peers.slice(0, 20)];
const resolvedNames = {};
await Promise.race([
Promise.all(toResolve.map(n =>
fetchRipeStatCached("https://stat.ripe.net/data/as-overview/data.json?resource=AS" + n.asn, { timeout: 3000 })
.then(r => { if (r && r.data && r.data.holder) resolvedNames[n.asn] = r.data.holder; })
.catch(() => {})
)),
new Promise(r => setTimeout(r, 5000)),
]);
const fmt = n => ({
asn: n.asn,
name: resolvedNames[n.asn] || "",
power: n.power || 0,
v4_peers: n.v4_peers || 0,
v6_peers: n.v6_peers || 0,
});
const result = {
asn: parseInt(rawAsn),
query_time: new Date().toISOString(),
duration_ms: Date.now() - start,
counts: {
upstreams: counts.left || upstreams.length,
downstreams: counts.right || downstreams.length,
peers_total: counts.unique || peers.length,
uncertain: counts.uncertain || peers.length,
},
upstreams: upstreams.map(fmt),
downstreams: downstreams.map(fmt),
peers: peers.slice(0, 50).map(fmt),
methodology: "RIPE Stat asn-neighbours API. left=upstream providers (carry your traffic), right=downstream customers (you carry their traffic), uncertain=lateral peers. Sorted by power score (number of prefixes seen via this relationship).",
source_url: "https://stat.ripe.net/data/asn-neighbours/data.json?resource=AS" + rawAsn,
};
cacheSet(cacheKey, result, 10 * 60 * 1000); // 10 min cache
res.writeHead(200, { "Content-Type": "application/json" });
return res.end(JSON.stringify(result, null, 2));
} catch (err) {
res.writeHead(500);
return res.end(JSON.stringify({ error: "Relationships lookup failed", message: err.message }));
}
}
// ============================================================ // ============================================================
// Compare endpoint: /api/compare?asn1=X&asn2=Y // Compare endpoint: /api/compare?asn1=X&asn2=Y
// ============================================================ // ============================================================
@ -3788,7 +3943,8 @@ roaStore.loadFromDisk("/opt/peercortex-app/.roa-cache.json");
pdbSourceCache.loadFromDisk("/opt/peercortex-app/.pdb-source-cache.json"); pdbSourceCache.loadFromDisk("/opt/peercortex-app/.pdb-source-cache.json");
loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json"); loadRipeStatCacheFromDisk("/opt/peercortex-app/.ripe-stat-cache.json");
// Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries // Phase 1: Fetch fresh RPKI feed (ASPA + ROA) + Atlas probes + PDB org countries + MANRS participants
ensureManrsCache(); // fire-and-forget, 24h cache
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => { Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
server.listen(PORT, "0.0.0.0", () => { server.listen(PORT, "0.0.0.0", () => {
console.log("PeerCortex v0.6.0 running on http://0.0.0.0:" + PORT); console.log("PeerCortex v0.6.0 running on http://0.0.0.0:" + PORT);