From 96950992dfa900538928f83ccf1881cc9e18ec69 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Mon, 30 Mar 2026 05:42:38 +0200 Subject: [PATCH] feat: add company enrichment, ASPA timeout guard, map side panel, OIM telecoms - /api/enrich: Wikipedia + website meta scraping with redirect following - ASPA /api/aspa: 18s hard timeout guard + 8s per-call limit - WHOIS: defensive null check - Map: replace popups with left side panel - Map: OIM Telecoms fiber layer (OpenInfraMap vector tiles) - Map layer toggles: fix source-exists early-return bug - Provider graph: fix text colors for light background - Network Health: defensive HTML response check --- deploy/server.js | 45 ++++--- public/index-editorial.html | 238 +++++++++++++++++++++++++++++------- server.js | 45 ++++--- 3 files changed, 256 insertions(+), 72 deletions(-) diff --git a/deploy/server.js b/deploy/server.js index d287b6f..dc9fb18 100644 --- a/deploy/server.js +++ b/deploy/server.js @@ -601,14 +601,22 @@ function lookupAspaFromRpki(asn) { -// PeeringDB authenticated fetch helper -function fetchPeeringDB(path, options) { +// PeeringDB semaphore — limits concurrent PDB requests to avoid 429 rate-limits +const pdbSemaphore = new Semaphore(5); + +// PeeringDB authenticated fetch helper (throttled via semaphore) +async function fetchPeeringDB(path, options) { const url = PEERINGDB_API_URL + path; const headers = { "User-Agent": UA }; if (PEERINGDB_API_KEY) { headers["Authorization"] = "Api-Key " + PEERINGDB_API_KEY; } - return fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } }); + await pdbSemaphore.acquire(); + try { + return await fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } }); + } finally { + pdbSemaphore.release(); + } } // PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load). @@ -3507,23 +3515,32 @@ const server = http.createServer(async (req, res) => { } } - // 3. Fallback: scrape website meta description + // 3. Fallback: scrape website meta description (follows up to 3 redirects) + function fetchPage(pageUrl, hops) { + if (hops <= 0) return Promise.resolve(null); + return new Promise((resolve) => { + const mod = pageUrl.startsWith("https") ? https : http; + const req = mod.get(pageUrl, { headers: { "User-Agent": UA_SCRAPE }, timeout: 6000 }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); // drain to free socket + const next = res.headers.location.startsWith("http") ? res.headers.location : new URL(res.headers.location, pageUrl).href; + return resolve(fetchPage(next, hops - 1)); + } + let data = ""; + res.on("data", (c) => { data += c; if (data.length > 40000) { req.destroy(); resolve(data); } }); + res.on("end", () => resolve(data)); + }); + req.on("error", () => resolve(null)); + req.on("timeout", () => { req.destroy(); resolve(null); }); + }); + } if (!description && website) { let wsUrl = website; if (!wsUrl.startsWith("http")) wsUrl = "https://" + wsUrl; const aboutUrl = wsUrl.replace(/\/$/, "") + "/about"; const tryUrls = [aboutUrl, wsUrl]; for (const tryUrl of tryUrls) { - const resp = await new Promise((resolve) => { - const mod = tryUrl.startsWith("https") ? https : http; - const req = mod.get(tryUrl, { headers: { "User-Agent": UA_SCRAPE }, timeout: 5000 }, (res) => { - let data = ""; - res.on("data", (c) => { data += c; if (data.length > 40000) { req.destroy(); resolve(data); } }); - res.on("end", () => resolve(data)); - }); - req.on("error", () => resolve(null)); - req.on("timeout", () => { req.destroy(); resolve(null); }); - }); + const resp = await fetchPage(tryUrl, 3); if (!resp) continue; const metaPatterns = [ /]+name=["']description["'][^>]+content=["']([^"']{20,500})["']/i, diff --git a/public/index-editorial.html b/public/index-editorial.html index ec623c4..aeba8ca 100644 --- a/public/index-editorial.html +++ b/public/index-editorial.html @@ -508,14 +508,22 @@ a:hover{color:var(--purple)} + -
+
+
+
Click point for details
+
+
+
+
● IXP ● Datacenter/Facility ▬ Submarine Cable ○ Global Datacenter (PeeringDB) + ▬ OIM Fiber/Telecoms
@@ -930,6 +938,7 @@ async function doLookup() { if (d.ix_presence && d.ix_presence.connections) loadPeeringRecommendations(currentAsn, d.ix_presence.connections, d); // Load ASPA and bgproutes.io data asynchronously + loadOverviewEnrichment(raw, d.network ? d.network.name : '', d.network ? d.network.website : ''); loadHealthReport(raw); loadAspaData(raw); loadAspaVerifyData(raw); @@ -1164,6 +1173,9 @@ function renderDashboard(d) { if (n.org_name) ov += '
' + escHtml(n.org_name) + '
'; if (n.notes) ov += '
' + escHtml(n.notes) + '
'; + // Enrichment placeholder — filled async by loadOverviewEnrichment() + ov += '
'; + ov += '
'; if (n.peeringdb_id) ov += 'PeeringDB'; ov += 'bgp.he.net'; @@ -1322,13 +1334,14 @@ function renderDashboard(d) { // ============================================================ // Global Infrastructure Map (MapLibre GL) -// Layers: ASN PoPs | Submarine Cables | Global Datacenters +// Layers: ASN PoPs | Submarine Cables | Global Datacenters | OIM Telecoms // ============================================================ var _pcMap = null; var _pcMapData = null; // current ASN data -var _mapLayers = { pops: true, cables: false, globalFacs: false }; +var _mapLayers = { pops: true, cables: false, globalFacs: false, telecoms: false }; var _cablesLoaded = false; var _globalFacsLoaded = false; +var _telecomsLoaded = false; function toggleMapLayer(layer, btn) { _mapLayers[layer] = !_mapLayers[layer]; @@ -1350,14 +1363,25 @@ function toggleMapLayer(layer, btn) { } else if (layer === 'globalFacs') { if (active && !_globalFacsLoaded) { _loadGlobalFacs(); return; } if (_pcMap.getLayer('global-facs')) _pcMap.setLayoutProperty('global-facs', 'visibility', active ? 'visible' : 'none'); + } else if (layer === 'telecoms') { + if (active && !_telecomsLoaded) { _loadTelecoms(); return; } + ['oim-telecoms-line', 'oim-telecoms-dc'].forEach(function(id) { + if (_pcMap.getLayer(id)) _pcMap.setLayoutProperty(id, 'visibility', active ? 'visible' : 'none'); + }); } } function _layerColor(l) { - return l === 'pops' ? '#7dcfff' : l === 'cables' ? '#9ece6a' : '#bb9af7'; + if (l === 'pops') return '#7dcfff'; + if (l === 'cables') return '#9ece6a'; + if (l === 'globalFacs') return '#bb9af7'; + return '#f7ae54'; // telecoms } function _layerRgb(l) { - return l === 'pops' ? '125,207,255' : l === 'cables' ? '158,206,106' : '187,154,247'; + if (l === 'pops') return '125,207,255'; + if (l === 'cables') return '158,206,106'; + if (l === 'globalFacs') return '187,154,247'; + return '247,174,84'; // telecoms } function _showMapLoader(show) { @@ -1365,31 +1389,49 @@ function _showMapLoader(show) { if (el) el.style.display = show ? 'inline' : 'none'; } +function _showMapPanel(html) { + var el = document.getElementById('mapSidePanelContent'); + if (el) el.innerHTML = html; +} + +function _mapPanelItem(label, color, title, subtitle) { + return '
' + + '
' + label + '
' + + '
' + title + '
' + + (subtitle ? '
' + subtitle + '
' : '') + + '
'; +} + function _loadCables() { _showMapLoader(true); fetch('/api/submarine-cables').then(function(r) { return r.json(); }).then(function(geo) { if (!_pcMap) return; _cablesLoaded = true; _showMapLoader(false); - if (_pcMap.getSource('cables')) return; + if (_pcMap.getSource('cables')) { + ['cables-glow','cables-line'].forEach(function(id) { + if (_pcMap.getLayer(id)) _pcMap.setLayoutProperty(id, 'visibility', 'visible'); + }); + return; + } + var before = _pcMap.getLayer('pops-fac') ? 'pops-fac' : undefined; _pcMap.addSource('cables', { type: 'geojson', data: geo }); _pcMap.addLayer({ id: 'cables-glow', type: 'line', source: 'cables', layout: { 'line-cap': 'round', visibility: 'visible' }, - paint: { 'line-color': '#9ece6a', 'line-width': 4, 'line-opacity': 0.08 } - }, 'pops-fac'); + paint: { 'line-color': '#9ece6a', 'line-width': 4, 'line-opacity': 0.12 } + }, before); _pcMap.addLayer({ id: 'cables-line', type: 'line', source: 'cables', layout: { 'line-cap': 'round', visibility: 'visible' }, - paint: { 'line-color': '#9ece6a', 'line-width': 1.2, 'line-opacity': 0.55 } - }, 'pops-fac'); + paint: { 'line-color': '#9ece6a', 'line-width': 1.5, 'line-opacity': 0.7 } + }, before); _pcMap.on('click', 'cables-line', function(e) { var props = e.features[0].properties; - new maplibregl.Popup({ closeButton: false, className: 'pc-popup' }) - .setLngLat(e.lngLat) - .setHTML('' + (props.name || 'Submarine Cable') + '' + - (props.color ? '
Color: ' + props.color + '' : '')) - .addTo(_pcMap); + _showMapPanel(_mapPanelItem('Submarine Cable', '#9ece6a', + escHtml(props.name || 'Unnamed Cable'), + props.color ? 'Color code: ' + escHtml(props.color) : null + )); }); _pcMap.on('mouseenter', 'cables-line', function() { _pcMap.getCanvas().style.cursor = 'pointer'; }); _pcMap.on('mouseleave', 'cables-line', function() { _pcMap.getCanvas().style.cursor = ''; }); @@ -1407,24 +1449,111 @@ function _loadGlobalFacs() { return { type: 'Feature', geometry: { type: 'Point', coordinates: [f.lng, f.lat] }, properties: { name: f.name, city: f.city, country: f.country } }; }); + var before = _pcMap.getLayer('pops-fac') ? 'pops-fac' : undefined; _pcMap.addSource('global-facs', { type: 'geojson', data: { type: 'FeatureCollection', features: features } }); _pcMap.addLayer({ id: 'global-facs', type: 'circle', source: 'global-facs', layout: { visibility: 'visible' }, - paint: { 'circle-radius': 3, 'circle-color': '#bb9af7', 'circle-opacity': 0.5, 'circle-stroke-width': 0.5, 'circle-stroke-color': '#bb9af7' } - }, 'pops-fac'); + paint: { 'circle-radius': 3, 'circle-color': '#bb9af7', 'circle-opacity': 0.65, 'circle-stroke-width': 0.5, 'circle-stroke-color': '#bb9af7' } + }, before); _pcMap.on('click', 'global-facs', function(e) { var p = e.features[0].properties; - new maplibregl.Popup({ closeButton: false, className: 'pc-popup' }) - .setLngLat(e.lngLat) - .setHTML('' + p.name + '
' + (p.city || '') + (p.country ? ', ' + p.country : '') + '') - .addTo(_pcMap); + _showMapPanel(_mapPanelItem('Datacenter (PeeringDB)', '#bb9af7', + escHtml(p.name || 'Unnamed'), + [(p.city || ''), (p.country || '')].filter(Boolean).join(', ') || null + )); }); _pcMap.on('mouseenter', 'global-facs', function() { _pcMap.getCanvas().style.cursor = 'pointer'; }); _pcMap.on('mouseleave', 'global-facs', function() { _pcMap.getCanvas().style.cursor = ''; }); }).catch(function() { _showMapLoader(false); _globalFacsLoaded = false; }); } +function _loadTelecoms() { + if (!_pcMap) return; + _telecomsLoaded = true; + _showMapLoader(true); + + // Add OpenInfraMap telecoms vector source + if (!_pcMap.getSource('oim-telecoms')) { + _pcMap.addSource('oim-telecoms', { + type: 'vector', + tiles: ['https://openinframap.org/map/telecoms/{z}/{x}/{y}.pbf'], + maxzoom: 17, + attribution: 'Open Infrastructure Map' + }); + } + + var before = _pcMap.getLayer('pops-fac') ? 'pops-fac' : undefined; + + try { + // Glasfaser-Leitungen (solid, gut sichtbar ab Zoom 2) + if (!_pcMap.getLayer('oim-telecoms-line')) { + _pcMap.addLayer({ + id: 'oim-telecoms-line', + type: 'line', + source: 'oim-telecoms', + 'source-layer': 'telecoms_communication_line', + minzoom: 2, + layout: { visibility: 'visible', 'line-cap': 'round', 'line-join': 'round' }, + paint: { + 'line-color': '#f7ae54', + 'line-width': ['interpolate', ['linear'], ['zoom'], 2, 1, 6, 1.5, 10, 2.5, 14, 4], + 'line-opacity': 0.8 + } + }, before); + } + } catch(e) { console.warn('OIM telecoms line layer error:', e); } + + try { + // Datacenters als Kreise (ab Zoom 4) + if (!_pcMap.getLayer('oim-telecoms-dc')) { + _pcMap.addLayer({ + id: 'oim-telecoms-dc', + type: 'circle', + source: 'oim-telecoms', + 'source-layer': 'telecoms_data_center_point', + minzoom: 4, + layout: { visibility: 'visible' }, + paint: { + 'circle-radius': ['interpolate', ['linear'], ['zoom'], 4, 3, 8, 5, 12, 8], + 'circle-color': '#f7ae54', + 'circle-opacity': 0.85, + 'circle-stroke-width': 1.5, + 'circle-stroke-color': '#1a1a1a', + 'circle-stroke-opacity': 0.6 + } + }, before); + } + } catch(e) { console.warn('OIM telecoms dc layer error:', e); } + + if (_pcMap.getLayer('oim-telecoms-line')) { + _pcMap.on('click', 'oim-telecoms-line', function(e) { + var p = e.features[0].properties; + _showMapPanel(_mapPanelItem('OIM Glasfaser-Leitung', '#f7ae54', + escHtml(p.name || p.operator || 'Unnamed cable'), + [p.type, p.location].filter(Boolean).join(' · ') || null + )); + }); + _pcMap.on('mouseenter', 'oim-telecoms-line', function() { _pcMap.getCanvas().style.cursor = 'pointer'; }); + _pcMap.on('mouseleave', 'oim-telecoms-line', function() { _pcMap.getCanvas().style.cursor = ''; }); + } + + if (_pcMap.getLayer('oim-telecoms-dc')) { + + _pcMap.on('click', 'oim-telecoms-dc', function(e) { + var p = e.features[0].properties; + _showMapPanel(_mapPanelItem('OIM Telecoms', '#f7ae54', + escHtml(p.name || p.operator || 'Unnamed facility'), + p.type ? p.type.replace(/_/g,' ') : null + )); + }); + _pcMap.on('mouseenter', 'oim-telecoms-dc', function() { _pcMap.getCanvas().style.cursor = 'pointer'; }); + _pcMap.on('mouseleave', 'oim-telecoms-dc', function() { _pcMap.getCanvas().style.cursor = ''; }); + } + + _showMapLoader(false); +} + function renderNetworkMap(d) { var mapCard = document.getElementById('mapCard'); var mapDiv = document.getElementById('networkMap'); @@ -1461,7 +1590,7 @@ function renderNetworkMap(d) { mapCard.style.display = 'block'; // Destroy previous map - if (_pcMap) { _pcMap.remove(); _pcMap = null; _cablesLoaded = false; _globalFacsLoaded = false; } + if (_pcMap) { _pcMap.remove(); _pcMap = null; _cablesLoaded = false; _globalFacsLoaded = false; _telecomsLoaded = false; } setTimeout(function() { _pcMap = new maplibregl.Map({ @@ -1499,14 +1628,10 @@ function renderNetworkMap(d) { var p = e.features[0].properties; var color = p.type === 'ix' ? '#ff9e64' : '#7dcfff'; var label = p.type === 'ix' ? 'IXP' : 'Datacenter'; - new maplibregl.Popup({ closeButton: false, className: 'pc-popup' }) - .setLngLat(e.lngLat) - .setHTML('
' + - '' + label + '
' + - '' + p.name + '' + - (p.detail ? '
' + p.detail + '' : '') + - '
') - .addTo(_pcMap); + _showMapPanel(_mapPanelItem(label, color, + escHtml(p.name || ''), + p.detail ? escHtml(p.detail) : null + )); }); _pcMap.on('mouseenter', layerId, function() { _pcMap.getCanvas().style.cursor = 'pointer'; }); _pcMap.on('mouseleave', layerId, function() { _pcMap.getCanvas().style.cursor = ''; }); @@ -1521,6 +1646,7 @@ function renderNetworkMap(d) { // Restore active overlay layers if (_mapLayers.cables && !_cablesLoaded) _loadCables(); if (_mapLayers.globalFacs && !_globalFacsLoaded) _loadGlobalFacs(); + if (_mapLayers.telecoms && !_telecomsLoaded) _loadTelecoms(); }); }, 50); } @@ -2142,10 +2268,10 @@ function renderProviderGraph(asn, providers) { function provCard(p, color, label) { var n = escHtml(p.name || ''); var f = p.frequency_pct ? p.frequency_pct.toFixed(0) + '%' : ''; - return '
' + - '
' + label + '
' + - '
AS' + p.asn + '
' + n + '
' + - (f ? '
' + f + '
' : '') + + return '
' + + '
' + label + '
' + + '
AS' + p.asn + '
' + n + '
' + + (f ? '
' + f + '
' : '') + '
'; } @@ -2154,7 +2280,7 @@ function renderProviderGraph(asn, providers) { function section(items, color, title, label, limit) { if (!items.length) return ''; - var s = '
' + title + ' (' + items.length + ')
'; + var s = '
' + title + ' (' + items.length + ')
'; s += '
'; var show = items.slice(0, limit); show.forEach(function(p) { s += provCard(p, color, label); }); @@ -2174,7 +2300,7 @@ function renderProviderGraph(asn, providers) { h += section(transit, '#60a5fa', 'Transit Providers', 'TR', 12); h += section(peers, '#4ade80', 'IX / Peers', 'IX', 12); - h += '
' + providers.length + ' providers total (Tier 1: ' + tier1.length + ' \u00b7 Transit: ' + transit.length + ' \u00b7 IX/Peer: ' + peers.length + ')
'; + h += '
' + providers.length + ' providers total  ·  Tier 1: ' + tier1.length + '  ·  Transit: ' + transit.length + '  ·  IX/Peer: ' + peers.length + '
'; $('providerGraphContent').innerHTML = h; } @@ -2516,18 +2642,42 @@ $('asnInput').addEventListener('keydown', function(e) { } })(); +async function loadOverviewEnrichment(asn, name, website) { + var el = document.getElementById('overviewEnrich'); + if (!el) return; + try { + var url = '/api/enrich?asn=' + asn + '&name=' + encodeURIComponent(name || ''); + if (website) url += '&website=' + encodeURIComponent(website); + var resp = await fetch(url); + if (!resp.ok) return; + var text = await resp.text(); + if (!text || text[0] === '<') return; + var d = JSON.parse(text); + if (!d.description) return; + var h = '
'; + h += 'About'; + h += escHtml(d.description); + if (d.wiki_url) h += ' Wikipedia →'; + h += '
'; + el.innerHTML = h; + } catch(e) { /* silently fail */ } +} + async function loadHealthReport(asn) { $('healthContent').innerHTML = '
Running comprehensive validation (13 checks)...
'; try { var resp = await fetch('/api/validate?asn=' + asn); - var d = await resp.json(); + if (!resp.ok) { $('healthContent').innerHTML = '
Health report unavailable (server ' + resp.status + ')
'; return; } + var text = await resp.text(); + if (!text || text[0] === '<') { $('healthContent').innerHTML = '
Health report temporarily unavailable
'; return; } + var d = JSON.parse(text); if (d.error) { $('healthContent').innerHTML = '
Validation failed: ' + escHtml(d.error) + '
'; return; } renderHealthReport(d); } catch (e) { - $('healthContent').innerHTML = '
Validation failed: ' + escHtml(e.message) + '
'; + $('healthContent').innerHTML = '
Health report temporarily unavailable
'; } } @@ -3149,15 +3299,15 @@ function termKeydown(e) { + onmouseover="this.style.opacity='1';this.style.borderColor='rgba(0,255,65,.7)';this.style.boxShadow='0 4px 24px rgba(0,0,0,.5),0 0 32px rgba(0,255,65,.2)'" + onmouseout="this.style.opacity='0.25';this.style.borderColor='rgba(0,255,65,.35)';this.style.boxShadow='0 4px 20px rgba(0,0,0,.4),0 0 20px rgba(0,255,65,.07)'" + style="position:fixed;bottom:24px;right:24px;width:52px;height:52px;opacity:0.25;background:rgba(10,10,10,.88);backdrop-filter:blur(14px);-webkit-backdrop-filter:blur(14px);border:1px solid rgba(0,255,65,.35);border-radius:6px;color:#00ff41;font-family:'IBM Plex Mono','Courier New',monospace;font-size:.82rem;font-weight:600;letter-spacing:.04em;cursor:pointer;z-index:9999;display:flex;align-items:center;justify-content:center;transition:border-color .2s,box-shadow .2s,opacity .4s ease;box-shadow:0 4px 20px rgba(0,0,0,.4),0 0 20px rgba(0,255,65,.07)">$_