From 13c5152bf9bf7f79137c31fa18d92641fa308754 Mon Sep 17 00:00:00 2001
From: Rene Fichtmueller <renefichtmueller@MacBook-Pro-von-Rene.local>
Date: Fri, 27 Mar 2026 10:22:10 +1300
Subject: [PATCH] feat: multi-source data validation with confidence scoring

- RPKI cross-check: Cloudflare RPKI feed + RIPE NCC Validator API (5 sample prefixes)
- Prefix cross-check: RIPE Stat vs bgp.he.net count comparison
- Neighbour cross-check: RIPE Stat vs bgp.he.net peer data
- Data Quality badge in dashboard (High/Medium/Low confidence)
- Hover tooltip: "Data Quality Report" with per-source agreement breakdown
- Added BETA tag to site header and version string (v0.5.0-beta)
- All UI text in English
---
 public/index.html | 141 +++++++++++++++++++++++++++++++++++++++-
 server.js         | 160 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 295 insertions(+), 6 deletions(-)

diff --git a/public/index.html b/public/index.html
index fa07b1b..d5bb780 100644
--- a/public/index.html
+++ b/public/index.html
@@ -100,7 +100,19 @@ a{color:var(--blue);text-decoration:none;transition:color .2s}a:hover{color:var(
 @keyframes shimmer{0%{background-position:200% 0}100%{background-position:-200% 0}}
 .skeleton.h2{height:2rem;width:60%}.skeleton.h3{height:1.2rem;width:40%}.skeleton.wide{width:100%}.skeleton.med{width:70%}
 
-.meta-bar{text-align:center;font-size:.75rem;color:var(--dim);margin-top:1rem;padding:0 1.5rem}
+.meta-bar{text-align:center;font-size:.75rem;color:var(--dim);margin-top:1rem;padding:0 1.5rem;display:flex;align-items:center;justify-content:center;gap:.75rem;flex-wrap:wrap}
+.dq-badge{display:inline-flex;align-items:center;gap:.35rem;padding:.25rem .65rem;border-radius:6px;font-size:.7rem;font-weight:600;cursor:help;position:relative;transition:all .2s}
+.dq-badge.high{background:rgba(158,206,106,.15);color:var(--green);border:1px solid rgba(158,206,106,.3)}
+.dq-badge.medium{background:rgba(255,158,100,.15);color:var(--orange);border:1px solid rgba(255,158,100,.3)}
+.dq-badge.low{background:rgba(247,118,142,.15);color:var(--red);border:1px solid rgba(247,118,142,.3)}
+.dq-badge svg{width:12px;height:12px}
+.dq-tooltip{display:none;position:absolute;bottom:calc(100% + 8px);left:50%;transform:translateX(-50%);background:var(--card);border:1px solid var(--border-light);border-radius:8px;padding:.75rem;font-size:.7rem;font-weight:400;color:var(--text-dim);min-width:280px;max-width:360px;z-index:100;box-shadow:0 4px 12px rgba(0,0,0,.3);text-align:left;line-height:1.5}
+.dq-badge:hover .dq-tooltip{display:block}
+.dq-tooltip-row{display:flex;justify-content:space-between;padding:.2rem 0;border-bottom:1px solid rgba(42,43,61,.5)}
+.dq-tooltip-row:last-child{border-bottom:none}
+.dq-tooltip-label{color:var(--muted)}
+.dq-tooltip-value{font-weight:600}
+.dq-tooltip-value.agree{color:var(--green)}.dq-tooltip-value.warn{color:var(--orange)}.dq-tooltip-value.bad{color:var(--red)}.dq-tooltip-value.na{color:var(--dim)}
 
 .footer{text-align:center;padding:2rem 1.5rem;color:var(--dim);font-size:.75rem;border-top:1px solid var(--border);margin-top:2rem}
 .footer a{color:var(--muted)}
@@ -267,7 +279,7 @@ a{color:var(--blue);text-decoration:none;transition:color .2s}a:hover{color:var(
       <line x1="18" y1="13" x2="25" y2="22" stroke="#565f89" stroke-width="1.5"/>
       <line x1="13" y1="24" x2="23" y2="24" stroke="#565f89" stroke-width="1.5"/>
     </svg>
-    <div><h1>PeerCortex</h1><span>Network Intelligence Dashboard v0.3</span></div>
+    <div><h1>PeerCortex <span style="font-size:.45em;background:linear-gradient(135deg,#f59e0b,#ef4444);color:#fff;padding:2px 10px;border-radius:4px;vertical-align:middle;letter-spacing:.08em;font-weight:700">BETA</span></h1><span>Network Intelligence Dashboard v0.5.0-beta</span></div>
   </div>
   <nav class="quick-links">
     <a href="https://github.com/peercortex/peercortex" target="_blank">GitHub</a>
@@ -667,6 +679,129 @@ function renderAuditList(containerId, list, sortBy, type) {
   document.getElementById(containerId).innerHTML = h;
 }
 
+function renderMetaBar(d) {
+  var bar = $('metaBar');
+  // Clear previous content
+  while (bar.firstChild) bar.removeChild(bar.firstChild);
+
+  // Meta text span
+  var metaSpan = document.createElement('span');
+  metaSpan.textContent = 'Query: ' + d.meta.query + ' | Duration: ' + d.meta.duration_ms + 'ms | RPKI checked: ' + d.meta.rpki_prefixes_checked + '/' + d.meta.total_prefixes + ' prefixes | ' + d.meta.timestamp;
+  bar.appendChild(metaSpan);
+
+  // Data Quality badge
+  var dq = d.data_quality;
+  if (dq) {
+    var conf = dq.overall_confidence || 'high';
+    var badge = document.createElement('span');
+    badge.className = 'dq-badge ' + conf;
+
+    // Shield icon
+    var svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
+    svg.setAttribute('viewBox', '0 0 24 24');
+    svg.setAttribute('fill', 'none');
+    svg.setAttribute('stroke', 'currentColor');
+    svg.setAttribute('stroke-width', '2');
+    var path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
+    path.setAttribute('d', 'M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z');
+    svg.appendChild(path);
+    badge.appendChild(svg);
+
+    var label = document.createElement('span');
+    var confLabel = conf === 'high' ? 'High Confidence' : conf === 'medium' ? 'Medium Confidence' : 'Low Confidence';
+    label.textContent = confLabel;
+    badge.appendChild(label);
+
+    // Tooltip
+    var tooltip = document.createElement('div');
+    tooltip.className = 'dq-tooltip';
+
+    var title = document.createElement('div');
+    title.style.cssText = 'font-weight:600;color:var(--white);margin-bottom:.5rem;font-size:.75rem';
+    title.textContent = 'Data Quality Report';
+    tooltip.appendChild(title);
+
+    var sources = document.createElement('div');
+    sources.style.cssText = 'margin-bottom:.5rem;color:var(--dim);font-size:.65rem';
+    sources.textContent = 'Sources Queried: ' + (dq.sources_queried || []).join(', ');
+    tooltip.appendChild(sources);
+
+    // Cross-Checks header
+    var checksHeader = document.createElement('div');
+    checksHeader.style.cssText = 'font-weight:600;color:var(--white);margin-bottom:.3rem;margin-top:.3rem;font-size:.7rem';
+    checksHeader.textContent = 'Cross-Checks';
+    tooltip.appendChild(checksHeader);
+
+    var checks = dq.cross_checks || {};
+    var checkNames = [['rpki', 'RPKI Validation'], ['prefixes', 'Prefix Count'], ['neighbours', 'Neighbours']];
+    var totalAgreement = 0;
+    var totalChecks = 0;
+    for (var ci = 0; ci < checkNames.length; ci++) {
+      var key = checkNames[ci][0];
+      var name = checkNames[ci][1];
+      var check = checks[key] || {};
+      var row = document.createElement('div');
+      row.className = 'dq-tooltip-row';
+
+      var rowLabel = document.createElement('span');
+      rowLabel.className = 'dq-tooltip-label';
+      rowLabel.textContent = name + ' (' + (check.sources || 0) + ' sources)';
+      row.appendChild(rowLabel);
+
+      var rowVal = document.createElement('span');
+      var pct = check.agreement_pct;
+      if (pct == null) {
+        rowVal.className = 'dq-tooltip-value na';
+        rowVal.textContent = 'N/A';
+      } else {
+        rowVal.className = 'dq-tooltip-value ' + (pct > 90 ? 'agree' : pct >= 70 ? 'warn' : 'bad');
+        rowVal.textContent = 'Agreement: ' + pct + '%';
+        totalAgreement += pct;
+        totalChecks++;
+      }
+      row.appendChild(rowVal);
+      tooltip.appendChild(row);
+    }
+
+    // Disagreements Found detail
+    var rpkiCheck = checks.rpki || {};
+    if (rpkiCheck.disagreements && rpkiCheck.disagreements.length > 0) {
+      var disDiv = document.createElement('div');
+      disDiv.style.cssText = 'margin-top:.4rem;font-size:.6rem;color:var(--orange)';
+      disDiv.textContent = 'Disagreements Found: ' + rpkiCheck.disagreements.map(function(dd) { return dd.prefix; }).join(', ');
+      tooltip.appendChild(disDiv);
+    }
+
+    // Prefix note
+    var pfxCheck = checks.prefixes || {};
+    if (pfxCheck.note) {
+      var noteDiv = document.createElement('div');
+      noteDiv.style.cssText = 'margin-top:.3rem;font-size:.6rem;color:var(--dim)';
+      noteDiv.textContent = pfxCheck.note;
+      tooltip.appendChild(noteDiv);
+    }
+
+    // Overall note
+    var avgAgreement = totalChecks > 0 ? totalAgreement / totalChecks : 100;
+    var overallNote = document.createElement('div');
+    overallNote.style.cssText = 'margin-top:.4rem;font-size:.6rem;font-style:italic;color:var(--muted)';
+    if (avgAgreement > 90) {
+      overallNote.textContent = 'All sources agree';
+      overallNote.style.color = 'var(--green)';
+    } else if (avgAgreement >= 70) {
+      overallNote.textContent = 'Minor discrepancies between sources';
+      overallNote.style.color = 'var(--orange)';
+    } else {
+      overallNote.textContent = 'Significant data disagreements detected';
+      overallNote.style.color = 'var(--red)';
+    }
+    tooltip.appendChild(overallNote);
+
+    badge.appendChild(tooltip);
+    bar.appendChild(badge);
+  }
+}
+
 async function doLookup() {
   const raw = $('asnInput').value.trim().replace(/[^0-9]/g, '');
   if (!raw) return;
@@ -692,7 +827,7 @@ async function doLookup() {
     renderDashboard(d);
     $('skeleton').classList.add('hidden');
     $('dashboard').classList.remove('hidden');
-    $('metaBar').textContent = 'Query: ' + d.meta.query + ' | Duration: ' + d.meta.duration_ms + 'ms | RPKI checked: ' + d.meta.rpki_prefixes_checked + '/' + d.meta.total_prefixes + ' prefixes | ' + d.meta.timestamp;
+    renderMetaBar(d);
 
     history.replaceState(null, '', '?asn=' + raw);
     saveToHistory(raw, d.network ? d.network.name : 'AS' + raw);
diff --git a/server.js b/server.js
index 1124fe1..363f5b7 100644
--- a/server.js
+++ b/server.js
@@ -619,6 +619,78 @@ function calculateAspaReadinessScore(params) {
 }
 
 
+// ============================================================
+// Feature 30: RIPE NCC RPKI Validator cross-check (max 5 prefixes)
+// ============================================================
+async function fetchRipeRpkiValidator(asn, prefix) {
+  try {
+    const encoded = encodeURIComponent(prefix);
+    const url = "https://rpki-validator.ripe.net/api/v1/validity/AS" + asn + "/" + encoded;
+    const result = await fetchJSON(url, { timeout: 5000 });
+    if (result && result.validated_route) {
+      return {
+        prefix: prefix,
+        validity: result.validated_route.validity || {},
+        state: (result.validated_route.validity && result.validated_route.validity.state) || "unknown",
+      };
+    }
+    return { prefix: prefix, state: "unknown", error: "no_data" };
+  } catch (_e) {
+    return { prefix: prefix, state: "error", error: "timeout_or_unavailable" };
+  }
+}
+
+// Cross-check a sample of prefixes against RIPE RPKI Validator (max 5, in parallel)
+async function crossCheckRpki(asn, prefixes, localResults) {
+  const sample = prefixes.slice(0, 5);
+  if (sample.length === 0) return { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: 100, disagreements: [], sample_size: 0 };
+
+  const ripeResults = await Promise.all(
+    sample.map((pfx) => fetchRipeRpkiValidator(asn, pfx))
+  );
+
+  const localMap = new Map();
+  for (const lr of localResults) {
+    localMap.set(lr.prefix, lr.status);
+  }
+
+  let cloudflareValid = 0;
+  let ripeValid = 0;
+  let agreements = 0;
+  const disagreements = [];
+
+  for (let i = 0; i < sample.length; i++) {
+    const pfx = sample[i];
+    const cfStatus = localMap.get(pfx) || "not_found";
+    const ripeState = ripeResults[i].state;
+
+    const cfIsValid = cfStatus === "valid";
+    const ripeIsValid = ripeState === "valid" || ripeState === "VALID";
+
+    if (cfIsValid) cloudflareValid++;
+    if (ripeIsValid) ripeValid++;
+
+    // Skip comparison if RIPE returned error/unknown
+    if (ripeState === "error" || ripeState === "unknown") {
+      agreements++; // Don't count failed lookups as disagreements
+      continue;
+    }
+
+    if (cfIsValid === ripeIsValid) {
+      agreements++;
+    } else {
+      disagreements.push({
+        prefix: pfx,
+        cloudflare: cfStatus,
+        ripe: ripeState,
+      });
+    }
+  }
+
+  const agreementPct = sample.length > 0 ? Math.round((agreements / sample.length) * 100) : 100;
+  return { cloudflare_valid: cloudflareValid, ripe_valid: ripeValid, agreement_pct: agreementPct, disagreements: disagreements, sample_size: sample.length };
+}
+
 // ============================================================
 // Feature 24: bgp.he.net Integration
 // ============================================================
@@ -983,7 +1055,7 @@ const server = http.createServer(async (req, res) => {
       JSON.stringify({
         status: "ok",
         service: "PeerCortex",
-        version: "0.5.0",
+        version: "0.5.0-beta",
         timestamp: new Date().toISOString(),
         uptime_seconds: Math.floor(process.uptime()),
         bgproutes_configured: !!BGPROUTES_API_KEY,
@@ -2077,13 +2149,88 @@ const server = http.createServer(async (req, res) => {
         };
       })();
 
+      // ============================================================
+      // Multi-source cross-checks (run in parallel, non-blocking)
+      // ============================================================
+      let rpkiCrossCheck = { cloudflare_valid: 0, ripe_valid: 0, agreement_pct: 100, disagreements: [], sample_size: 0 };
+      let prefixCrossCheck = { ripe_stat: prefixes.length, bgp_he_net: null, agreement: null, note: "" };
+      let neighbourCrossCheck = { ripe_stat_total: neighbours.length, bgp_he_net_total: null };
+
+      try {
+        // RPKI cross-check: sample up to 5 prefixes against RIPE Validator (with 8s total timeout)
+        const rpkiCrossPromise = crossCheckRpki(asn, allPrefixes, rpkiStatuses);
+        const rpkiCrossResult = await Promise.race([
+          rpkiCrossPromise,
+          new Promise((resolve) => setTimeout(() => resolve(null), 8000)),
+        ]);
+        if (rpkiCrossResult) rpkiCrossCheck = rpkiCrossResult;
+      } catch (_e) { /* cross-check failed, keep defaults */ }
+
+      // Prefix count cross-check: compare RIPE Stat vs bgp.he.net
+      if (bgpHeData) {
+        const heV4 = bgpHeData.prefixes_v4 || 0;
+        const heV6 = bgpHeData.prefixes_v6 || 0;
+        const heTotal = heV4 + heV6;
+        if (heTotal > 0) {
+          prefixCrossCheck.bgp_he_net = heTotal;
+          const ripeStat = prefixes.length;
+          if (ripeStat > 0 && heTotal > 0) {
+            const ratio = Math.min(ripeStat, heTotal) / Math.max(ripeStat, heTotal);
+            prefixCrossCheck.agreement = ratio >= 0.9;
+            const diff = Math.abs(ripeStat - heTotal);
+            prefixCrossCheck.note = diff === 0
+              ? "Exact match"
+              : "Difference of " + diff + " prefixes (" + Math.round((1 - ratio) * 100) + "% divergence)";
+          }
+        } else {
+          prefixCrossCheck.note = "bgp.he.net prefix count unavailable";
+        }
+
+        // Neighbour cross-check: compare RIPE Stat vs bgp.he.net peer_count
+        if (bgpHeData.peer_count != null) {
+          neighbourCrossCheck.bgp_he_net_total = bgpHeData.peer_count;
+        }
+      } else {
+        prefixCrossCheck.note = "bgp.he.net data unavailable";
+      }
+
+      // Compute overall data quality
+      const crossCheckScores = [];
+      // RPKI agreement
+      crossCheckScores.push(rpkiCrossCheck.agreement_pct);
+      // Prefix agreement: convert to percentage
+      if (prefixCrossCheck.bgp_he_net != null && prefixes.length > 0) {
+        const pfxRatio = Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net);
+        crossCheckScores.push(Math.round(pfxRatio * 100));
+      }
+      // Neighbour agreement
+      if (neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0) {
+        const nbrRatio = Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total);
+        crossCheckScores.push(Math.round(nbrRatio * 100));
+      }
+      const avgAgreement = crossCheckScores.length > 0
+        ? Math.round(crossCheckScores.reduce((a, b) => a + b, 0) / crossCheckScores.length)
+        : 100;
+      const overallConfidence = avgAgreement > 90 ? "high" : avgAgreement >= 70 ? "medium" : "low";
+
+      const dataQuality = {
+        sources_queried: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator"],
+        cross_checks: {
+          rpki: { sources: 2, agreement_pct: rpkiCrossCheck.agreement_pct, sample_size: rpkiCrossCheck.sample_size, disagreements: rpkiCrossCheck.disagreements },
+          prefixes: { sources: 2, agreement_pct: prefixCrossCheck.bgp_he_net != null ? Math.round((Math.min(prefixes.length, prefixCrossCheck.bgp_he_net) / Math.max(prefixes.length, prefixCrossCheck.bgp_he_net || 1)) * 100) : null, ripe_stat: prefixCrossCheck.ripe_stat, bgp_he_net: prefixCrossCheck.bgp_he_net, note: prefixCrossCheck.note },
+          neighbours: { sources: 2, agreement_pct: neighbourCrossCheck.bgp_he_net_total != null && neighbours.length > 0 ? Math.round((Math.min(neighbours.length, neighbourCrossCheck.bgp_he_net_total) / Math.max(neighbours.length, neighbourCrossCheck.bgp_he_net_total)) * 100) : null, ripe_stat_total: neighbourCrossCheck.ripe_stat_total, bgp_he_net_total: neighbourCrossCheck.bgp_he_net_total },
+        },
+        overall_confidence: overallConfidence,
+        overall_agreement_pct: avgAgreement,
+      };
+
       const result = {
         meta: {
           service: "PeerCortex",
-          version: "0.5.0",
+          version: "0.5.0-beta",
           query: "AS" + asn,
           duration_ms: duration,
-          sources: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "Route Views"],
+          sources: ["PeeringDB", "RIPE Stat", "bgp.he.net", "Cloudflare RPKI", "RIPE RPKI Validator", "Route Views"],
           timestamp: new Date().toISOString(),
           rpki_prefixes_checked: rpkiTotal,
           total_prefixes: prefixes.length,
@@ -2111,6 +2258,7 @@ const server = http.createServer(async (req, res) => {
           ipv4: prefixes.filter((p) => !p.prefix.includes(":")).length,
           ipv6: prefixes.filter((p) => p.prefix.includes(":")).length,
           list: prefixes.map((p) => p.prefix),
+          cross_check: prefixCrossCheck,
         },
         rpki: {
           coverage_percent: rpkiCoverage,
@@ -2119,6 +2267,7 @@ const server = http.createServer(async (req, res) => {
           not_found: rpkiNotFound,
           checked: rpkiTotal,
           details: rpkiStatuses,
+          cross_check: rpkiCrossCheck,
         },
         neighbours: {
           total: neighbours.length,
@@ -2128,6 +2277,7 @@ const server = http.createServer(async (req, res) => {
           upstreams: upstreams.slice(0, 20),
           downstreams: downstreams.slice(0, 20),
           peers: peers.slice(0, 20),
+          cross_check: neighbourCrossCheck,
         },
         ix_presence: {
           total_connections: ixConnections.length,
@@ -2155,8 +2305,12 @@ const server = http.createServer(async (req, res) => {
             description: p.description || "",
           })),
         },
+        data_quality: dataQuality,
       };
 
+      // Update duration to include cross-check time
+      result.meta.duration_ms = Date.now() - start;
+
       cacheSet(cacheKey, result, CACHE_TTL_LOOKUP);
       res.end(JSON.stringify(result, null, 2));
     } catch (err) {