feat: Lia's Paradise country data fix + file upload
- Fix org→country mapping: pre-cache 20k+ PeeringDB orgs at startup - 22k networks now have country codes (was 0 before) - Add file upload: CSV/TXT/PDF/XLS/DOC → ASN extraction → probe check - Export file results as printable PDF
This commit is contained in:
Rene Fichtmueller
parent
41af8be7f4
commit
3adc34c42b
@ -80,6 +80,22 @@ body{font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;background:
|
|||||||
<button class="export-btn" onclick="exportPDF()">Export PDF</button>
|
<button class="export-btn" onclick="exportPDF()">Export PDF</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div style="max-width:1400px;margin:.75rem auto;padding:0 1.5rem">
|
||||||
|
<div style="background:var(--card);border:1px solid var(--border);border-radius:12px;padding:1rem 1.5rem">
|
||||||
|
<div style="display:flex;align-items:center;gap:1rem;flex-wrap:wrap">
|
||||||
|
<div style="font-size:.8rem;font-weight:600;color:var(--pink)">FILE LOOKUP</div>
|
||||||
|
<div style="font-size:.75rem;color:var(--dim)">Upload a file with ASNs or company names — we'll check probe coverage for each</div>
|
||||||
|
<div style="flex:1"></div>
|
||||||
|
<label style="padding:.45rem 1rem;border-radius:8px;border:1px solid var(--pink);background:transparent;color:var(--pink);font-size:.8rem;font-weight:600;cursor:pointer;transition:all .2s;display:inline-flex;align-items:center;gap:.4rem" onmouseenter="this.style.background='var(--pink)';this.style.color='var(--bg)'" onmouseleave="this.style.background='transparent';this.style.color='var(--pink)'">
|
||||||
|
<span>Upload File</span>
|
||||||
|
<input type="file" id="fileUpload" accept=".csv,.txt,.pdf,.xls,.xlsx,.doc,.docx" style="display:none" onchange="handleFileUpload(this)">
|
||||||
|
</label>
|
||||||
|
<span id="fileStatus" style="font-size:.7rem;color:var(--dim)"></span>
|
||||||
|
</div>
|
||||||
|
<div id="fileResults" class="hidden" style="margin-top:1rem"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="stats-bar" id="statsBar"></div>
|
<div class="stats-bar" id="statsBar"></div>
|
||||||
|
|
||||||
<div id="summaryGrid" class="summary-grid" style="max-width:1400px;margin:1rem auto;padding:0 1.5rem"></div>
|
<div id="summaryGrid" class="summary-grid" style="max-width:1400px;margin:1rem auto;padding:0 1.5rem"></div>
|
||||||
@ -319,6 +335,152 @@ function exportPDF() {
|
|||||||
w.print();
|
w.print();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// File Upload: Parse ASNs/company names from uploaded files
|
||||||
|
// ============================================================
|
||||||
|
function handleFileUpload(input) {
|
||||||
|
var file = input.files[0];
|
||||||
|
if (!file) return;
|
||||||
|
var status = $('fileStatus');
|
||||||
|
status.textContent = 'Processing ' + file.name + '...';
|
||||||
|
status.style.color = 'var(--cyan)';
|
||||||
|
|
||||||
|
var ext = file.name.split('.').pop().toLowerCase();
|
||||||
|
|
||||||
|
if (ext === 'csv' || ext === 'txt') {
|
||||||
|
file.text().then(function(text) { processFileText(text, file.name); });
|
||||||
|
} else if (ext === 'pdf' || ext === 'doc' || ext === 'docx' || ext === 'xls' || ext === 'xlsx') {
|
||||||
|
// For binary formats, upload to server for parsing
|
||||||
|
var reader = new FileReader();
|
||||||
|
reader.onload = function(e) {
|
||||||
|
var base64 = e.target.result.split(',')[1];
|
||||||
|
fetch('/api/lia/parse-file', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ filename: file.name, data: base64 })
|
||||||
|
}).then(function(r) { return r.json(); }).then(function(d) {
|
||||||
|
if (d.error) {
|
||||||
|
status.textContent = 'Error: ' + d.error;
|
||||||
|
status.style.color = 'var(--red)';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
processFileText(d.text || '', file.name);
|
||||||
|
}).catch(function(err) {
|
||||||
|
// Fallback: try reading as text
|
||||||
|
file.text().then(function(text) { processFileText(text, file.name); }).catch(function() {
|
||||||
|
status.textContent = 'Cannot parse ' + ext.toUpperCase() + ' files client-side. Use CSV or TXT.';
|
||||||
|
status.style.color = 'var(--red)';
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
reader.readAsDataURL(file);
|
||||||
|
} else {
|
||||||
|
status.textContent = 'Unsupported file type: ' + ext;
|
||||||
|
status.style.color = 'var(--red)';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function processFileText(text, filename) {
|
||||||
|
var status = $('fileStatus');
|
||||||
|
|
||||||
|
// Extract ASNs (AS12345 or just numbers that look like ASNs)
|
||||||
|
var asnMatches = text.match(/\bAS?(\d{3,7})\b/gi) || [];
|
||||||
|
var asns = new Set();
|
||||||
|
asnMatches.forEach(function(m) {
|
||||||
|
var num = parseInt(m.replace(/^AS/i, ''));
|
||||||
|
if (num >= 100 && num <= 9999999) asns.add(num);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Also try to match company names against our loaded data
|
||||||
|
var nameMatches = [];
|
||||||
|
if (allData.length > 0) {
|
||||||
|
var lines = text.split(/[\n\r,;]+/).map(function(l) { return l.trim().toLowerCase(); }).filter(function(l) { return l.length > 2; });
|
||||||
|
lines.forEach(function(line) {
|
||||||
|
allData.forEach(function(n) {
|
||||||
|
if (n.name && n.name.toLowerCase().indexOf(line) >= 0 && line.length > 4) {
|
||||||
|
asns.add(n.asn);
|
||||||
|
}
|
||||||
|
if (line.indexOf(n.name ? n.name.toLowerCase() : '###') >= 0 && n.name && n.name.length > 4) {
|
||||||
|
asns.add(n.asn);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (asns.size === 0) {
|
||||||
|
status.textContent = 'No ASNs or matching networks found in ' + filename;
|
||||||
|
status.style.color = 'var(--orange)';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match against our data
|
||||||
|
var results = [];
|
||||||
|
var asnArr = Array.from(asns);
|
||||||
|
var dataMap = {};
|
||||||
|
allData.forEach(function(n) { dataMap[n.asn] = n; });
|
||||||
|
|
||||||
|
asnArr.forEach(function(asn) {
|
||||||
|
var n = dataMap[asn];
|
||||||
|
results.push({
|
||||||
|
asn: asn,
|
||||||
|
name: n ? n.name : '(Unknown)',
|
||||||
|
country: n ? n.country : '',
|
||||||
|
info_type: n ? n.info_type : '',
|
||||||
|
has_probe: n ? n.has_probe : false,
|
||||||
|
in_peeringdb: !!n,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
results.sort(function(a, b) { return (a.has_probe ? 1 : 0) - (b.has_probe ? 1 : 0); });
|
||||||
|
|
||||||
|
var withProbe = results.filter(function(r) { return r.has_probe; }).length;
|
||||||
|
var noProbe = results.length - withProbe;
|
||||||
|
|
||||||
|
status.innerHTML = '<span style="color:var(--green)">' + withProbe + ' with probe</span> · <span style="color:var(--red)">' + noProbe + ' without probe</span> · ' + results.length + ' total from ' + escHtml(filename);
|
||||||
|
|
||||||
|
var h = '<div style="display:flex;gap:.5rem;margin-bottom:.75rem;flex-wrap:wrap">';
|
||||||
|
h += '<button class="export-btn" onclick="exportFileResults()" style="font-size:.7rem;padding:.3rem .8rem">Export Results as PDF</button>';
|
||||||
|
h += '</div>';
|
||||||
|
h += '<div class="asn-grid">';
|
||||||
|
results.forEach(function(r) {
|
||||||
|
h += '<div class="asn-row" onclick="window.open(\'/?asn=' + r.asn + '\',\'_blank\')">';
|
||||||
|
h += '<span class="asn-num">AS' + r.asn + '</span>';
|
||||||
|
h += '<span class="asn-name">' + escHtml(r.name) + '</span>';
|
||||||
|
if (r.country) h += '<span style="font-size:.65rem;color:var(--dim)">' + flag(r.country) + ' ' + r.country + '</span>';
|
||||||
|
if (r.info_type) h += '<span class="asn-type">' + escHtml(r.info_type) + '</span>';
|
||||||
|
h += r.has_probe ? '<span class="has-probe">\u2714 Probe</span>' : '<span class="no-probe">\u2718 No Probe</span>';
|
||||||
|
if (!r.in_peeringdb) h += '<span style="font-size:.6rem;color:var(--dim)">(not in PeeringDB)</span>';
|
||||||
|
h += '</div>';
|
||||||
|
});
|
||||||
|
h += '</div>';
|
||||||
|
|
||||||
|
$('fileResults').innerHTML = h;
|
||||||
|
$('fileResults').classList.remove('hidden');
|
||||||
|
|
||||||
|
// Store for export
|
||||||
|
window._fileResults = results;
|
||||||
|
window._fileName = filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
function exportFileResults() {
|
||||||
|
if (!window._fileResults) return;
|
||||||
|
var results = window._fileResults;
|
||||||
|
var w = window.open('', '_blank');
|
||||||
|
var html = '<!DOCTYPE html><html><head><title>Atlas Probe Check — ' + escHtml(window._fileName) + '</title>';
|
||||||
|
html += '<style>body{font-family:Arial,sans-serif;margin:2rem;color:#1a1a2e}h1{color:#5b21b6;font-size:1.3rem}table{width:100%;border-collapse:collapse;margin:1rem 0}th,td{text-align:left;padding:.4rem .6rem;border-bottom:1px solid #e2e8f0;font-size:.85rem}th{background:#f8fafc;font-weight:600}.yes{color:#16a34a;font-weight:700}.no{color:#dc2626;font-weight:700}.meta{color:#64748b;font-size:.8rem}</style></head><body>';
|
||||||
|
html += '<h1>Atlas Probe Coverage Check</h1>';
|
||||||
|
html += '<div class="meta">Source: ' + escHtml(window._fileName) + ' | Generated: ' + new Date().toISOString().split('T')[0] + ' | peercortex.org/lia</div>';
|
||||||
|
html += '<table><thead><tr><th>ASN</th><th>Name</th><th>Country</th><th>Type</th><th>Atlas Probe</th></tr></thead><tbody>';
|
||||||
|
results.forEach(function(r) {
|
||||||
|
html += '<tr><td>AS' + r.asn + '</td><td>' + escHtml(r.name) + '</td><td>' + (r.country || '-') + '</td><td>' + (r.info_type || '-') + '</td>';
|
||||||
|
html += '<td class="' + (r.has_probe ? 'yes' : 'no') + '">' + (r.has_probe ? 'YES' : 'NO') + '</td></tr>';
|
||||||
|
});
|
||||||
|
html += '</tbody></table></body></html>';
|
||||||
|
w.document.write(html);
|
||||||
|
w.document.close();
|
||||||
|
w.print();
|
||||||
|
}
|
||||||
|
|
||||||
// Boot
|
// Boot
|
||||||
loadData();
|
loadData();
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
150
deploy/server.js
150
deploy/server.js
@ -737,101 +737,40 @@ const server = http.createServer(async (req, res) => {
|
|||||||
if (liaCached) return res.end(liaCached);
|
if (liaCached) return res.end(liaCached);
|
||||||
|
|
||||||
// Fetch PeeringDB network list (all networks with status "ok")
|
// Fetch PeeringDB network list (all networks with status "ok")
|
||||||
|
// Use pre-cached org→country map (loaded at startup, 16MB response cached in memory)
|
||||||
fetchPeeringDB("/net?status=ok&depth=0").then(function(pdbData) {
|
fetchPeeringDB("/net?status=ok&depth=0").then(function(pdbData) {
|
||||||
if (!pdbData || !pdbData.data) {
|
if (!pdbData || !pdbData.data) {
|
||||||
return res.end(JSON.stringify({ error: "Could not fetch PeeringDB networks" }));
|
return res.end(JSON.stringify({ error: "Could not fetch PeeringDB networks" }));
|
||||||
}
|
}
|
||||||
|
|
||||||
var probeAsns = new Set(atlasProbeCache.asns_with_probes || []);
|
var probeAsns = new Set(atlasProbeCache.asns_with_probes || []);
|
||||||
// Country name lookup
|
|
||||||
var countryNames = {};
|
|
||||||
try { countryNames = require("./country-names.json"); } catch(_e) { /* optional */ }
|
|
||||||
|
|
||||||
var networks = pdbData.data.map(function(n) {
|
var enriched = pdbData.data.map(function(n) {
|
||||||
|
var org = pdbOrgCountryMap.get(n.org_id) || {};
|
||||||
|
var cc = org.country || "";
|
||||||
return {
|
return {
|
||||||
asn: n.asn,
|
asn: n.asn,
|
||||||
name: n.name || "",
|
name: n.name || "",
|
||||||
country: n.info_prefixes4 > 0 || n.info_prefixes6 > 0 ? "" : "", // PeeringDB doesn't have country directly on net
|
org_name: org.name || "",
|
||||||
|
country: cc,
|
||||||
|
country_name: cc,
|
||||||
info_type: n.info_type || "",
|
info_type: n.info_type || "",
|
||||||
has_probe: probeAsns.has(n.asn),
|
has_probe: probeAsns.has(n.asn),
|
||||||
};
|
};
|
||||||
|
}).filter(function(n) { return n.asn > 0 && n.country; });
|
||||||
|
|
||||||
|
var result = JSON.stringify({
|
||||||
|
networks: enriched,
|
||||||
|
total: enriched.length,
|
||||||
|
with_probes: enriched.filter(function(n) { return n.has_probe; }).length,
|
||||||
|
without_probes: enriched.filter(function(n) { return !n.has_probe; }).length,
|
||||||
|
atlas_unique_asns: probeAsns.size,
|
||||||
|
org_countries_loaded: pdbOrgCountryMap.size,
|
||||||
|
fetched_at: new Date().toISOString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
// We need countries — fetch from RIPE Stat for each unique ASN is too slow.
|
cacheSet(liaCacheKey, result, 30 * 60 * 1000);
|
||||||
// Instead, use the Atlas byCountry data to enrich. For PeeringDB, we need netfac or netixlan for country.
|
res.end(result);
|
||||||
// Better approach: Use PeeringDB org country. Fetch with depth=1 to get org.
|
|
||||||
// But that's too heavy (50MB+). Instead, use a separate PeeringDB call for orgs.
|
|
||||||
// Pragmatic: Fetch net with depth=1 but limit fields
|
|
||||||
// Actually the simplest: PeeringDB net API doesn't expose country directly.
|
|
||||||
// Use the ix_count/fac_count fields and the "org" for country.
|
|
||||||
// Let's just add a second call for orgs.
|
|
||||||
|
|
||||||
// Simplest approach: use RIPE Stat resource-overview for country from ASN prefix
|
|
||||||
// But that's per-ASN. Instead, build country from Atlas probes data.
|
|
||||||
// Atlas byCountry has {CC: {asnSet}} — we can reverse-map ASN→country from there.
|
|
||||||
|
|
||||||
// Build ASN→country from atlas data
|
|
||||||
// We stored asnSet in byCountry but only in the internal function.
|
|
||||||
// atlasProbeCache.by_country has {CC: {total, connected, asn_count}} — no ASN list!
|
|
||||||
// We need to store ASN→country mapping. Let's add it.
|
|
||||||
|
|
||||||
// For now, return without country and let frontend handle it via RIR mapping
|
|
||||||
// Actually: PeeringDB net objects have no country, but we can batch-fetch orgs.
|
|
||||||
// The org object has country. Let's do net?depth=1 but that's huge.
|
|
||||||
// Compromise: Get first 5000 networks and their org_id, then batch-fetch orgs.
|
|
||||||
|
|
||||||
// PRAGMATIC FIX: Use net?depth=0 + a separate org fetch
|
|
||||||
// PeeringDB org API: /org?status=ok&limit=0 returns all orgs with country.
|
|
||||||
|
|
||||||
fetchPeeringDB("/org?status=ok&depth=0").then(function(orgData) {
|
|
||||||
// Build org_id → country map
|
|
||||||
var orgCountry = {};
|
|
||||||
if (orgData && orgData.data) {
|
|
||||||
orgData.data.forEach(function(o) {
|
|
||||||
orgCountry[o.id] = { country: o.country || "", name: o.name || "" };
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enrich networks with org country
|
|
||||||
var enriched = pdbData.data.map(function(n) {
|
|
||||||
var org = orgCountry[n.org_id] || {};
|
|
||||||
var cc = org.country || "";
|
|
||||||
return {
|
|
||||||
asn: n.asn,
|
|
||||||
name: n.name || "",
|
|
||||||
org_name: org.name || "",
|
|
||||||
country: cc,
|
|
||||||
country_name: cc, // frontend will display full name from its own mapping
|
|
||||||
info_type: n.info_type || "",
|
|
||||||
has_probe: probeAsns.has(n.asn),
|
|
||||||
};
|
|
||||||
}).filter(function(n) { return n.asn > 0; });
|
|
||||||
|
|
||||||
var result = JSON.stringify({
|
|
||||||
networks: enriched,
|
|
||||||
total: enriched.length,
|
|
||||||
with_probes: enriched.filter(function(n) { return n.has_probe; }).length,
|
|
||||||
without_probes: enriched.filter(function(n) { return !n.has_probe; }).length,
|
|
||||||
atlas_unique_asns: probeAsns.size,
|
|
||||||
fetched_at: new Date().toISOString(),
|
|
||||||
});
|
|
||||||
|
|
||||||
cacheSet(liaCacheKey, result, 30 * 60 * 1000); // 30 min cache
|
|
||||||
res.end(result);
|
|
||||||
}).catch(function(e) {
|
|
||||||
// If org fetch fails, return without country
|
|
||||||
var result = JSON.stringify({
|
|
||||||
networks: networks,
|
|
||||||
total: networks.length,
|
|
||||||
with_probes: networks.filter(function(n) { return n.has_probe; }).length,
|
|
||||||
without_probes: networks.filter(function(n) { return !n.has_probe; }).length,
|
|
||||||
atlas_unique_asns: probeAsns.size,
|
|
||||||
error_note: "Country data unavailable: " + e.message,
|
|
||||||
fetched_at: new Date().toISOString(),
|
|
||||||
});
|
|
||||||
cacheSet(liaCacheKey, result, 5 * 60 * 1000);
|
|
||||||
res.end(result);
|
|
||||||
});
|
|
||||||
}).catch(function(e) {
|
}).catch(function(e) {
|
||||||
res.end(JSON.stringify({ error: "PeeringDB fetch failed: " + e.message }));
|
res.end(JSON.stringify({ error: "PeeringDB fetch failed: " + e.message }));
|
||||||
});
|
});
|
||||||
@ -2447,10 +2386,59 @@ function fetchAllAtlasProbes() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// PeeringDB Org → Country Cache (for Lia's Paradise)
|
||||||
|
// ============================================================
|
||||||
|
let pdbOrgCountryMap = new Map(); // org_id → { country, name }
|
||||||
|
|
||||||
|
function fetchPdbOrgCountries() {
|
||||||
|
console.log("[PDB-ORG] Fetching PeeringDB org countries...");
|
||||||
|
return new Promise(function(resolve) {
|
||||||
|
// Use raw https to handle the large 16MB response with streaming
|
||||||
|
var chunks = [];
|
||||||
|
var req = require("https").get("https://www.peeringdb.com/api/org?status=ok&depth=0", {
|
||||||
|
headers: {
|
||||||
|
"User-Agent": UA,
|
||||||
|
"Authorization": PEERINGDB_API_KEY ? "Api-Key " + PEERINGDB_API_KEY : undefined,
|
||||||
|
},
|
||||||
|
timeout: 120000,
|
||||||
|
}, function(res) {
|
||||||
|
res.on("data", function(chunk) { chunks.push(chunk); });
|
||||||
|
res.on("end", function() {
|
||||||
|
try {
|
||||||
|
var body = Buffer.concat(chunks).toString("utf8");
|
||||||
|
var data = JSON.parse(body);
|
||||||
|
if (data && data.data) {
|
||||||
|
pdbOrgCountryMap = new Map();
|
||||||
|
data.data.forEach(function(o) {
|
||||||
|
if (o.id && o.country) {
|
||||||
|
pdbOrgCountryMap.set(o.id, { country: o.country, name: o.name || "" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log("[PDB-ORG] Loaded " + pdbOrgCountryMap.size + " org→country mappings");
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[PDB-ORG] Parse error:", e.message);
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
req.on("error", function(e) {
|
||||||
|
console.error("[PDB-ORG] Fetch error:", e.message);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
req.on("timeout", function() {
|
||||||
|
console.error("[PDB-ORG] Timeout after 120s");
|
||||||
|
req.destroy();
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const PORT = process.env.PORT || 3101;
|
const PORT = process.env.PORT || 3101;
|
||||||
|
|
||||||
// Fetch RPKI ASPA feed at startup and refresh every 10 minutes
|
// Fetch RPKI ASPA feed at startup and refresh every 10 minutes
|
||||||
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes()]).then(() => {
|
Promise.all([fetchRpkiAspaFeed(), fetchAllAtlasProbes(), fetchPdbOrgCountries()]).then(() => {
|
||||||
server.listen(PORT, "0.0.0.0", () => {
|
server.listen(PORT, "0.0.0.0", () => {
|
||||||
console.log("PeerCortex v0.4.0 running on http://0.0.0.0:" + PORT);
|
console.log("PeerCortex v0.4.0 running on http://0.0.0.0:" + PORT);
|
||||||
console.log("bgproutes.io API key: " + (BGPROUTES_API_KEY ? "configured" : "NOT configured"));
|
console.log("bgproutes.io API key: " + (BGPROUTES_API_KEY ? "configured" : "NOT configured"));
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user