diff --git a/server.js b/server.js index 1ec0b1e..73fc846 100644 --- a/server.js +++ b/server.js @@ -70,172 +70,15 @@ try { console.log('[bio-rd] RIS client not available (bio-rd-client.js missing or gRPC not installed)'); } -const PEERINGDB_API_KEY = process.env.PEERINGDB_API_KEY || ""; -const PEERINGDB_API_URL = process.env.PEERINGDB_API_URL || "https://www.peeringdb.com/api"; - -// ── Local PeeringDB SQLite (peeringdb-py sync, refreshed daily by cron) ────── -const PEERINGDB_LOCAL_PATH = process.env.PEERINGDB_LOCAL_PATH || "/opt/peeringdb-data/peeringdb.sqlite3"; -let _pdbLocal = null; -function getPdbLocal() { - if (_pdbLocal) return _pdbLocal; - try { - const BetterSqlite3 = require("better-sqlite3"); - if (!fs.existsSync(PEERINGDB_LOCAL_PATH)) return null; - _pdbLocal = new BetterSqlite3(PEERINGDB_LOCAL_PATH, { readonly: true, fileMustExist: true }); - console.log("[PeeringDB-local] SQLite opened:", PEERINGDB_LOCAL_PATH); - return _pdbLocal; - } catch (e) { - console.warn("[PeeringDB-local] Could not open SQLite:", e.message); - return null; - } -} - -// Map API path → SQLite result in { data: [...] } format, emulating the live PDB REST API. -function queryPeeringDBLocal(path) { - const db = getPdbLocal(); - if (!db) return null; - try { - // /net?asn=X - const netAsnMatch = path.match(/^\/net\?asn=(\d+)/); - if (netAsnMatch) { - const rows = db.prepare( - "SELECT n.*, o.name AS org_name FROM peeringdb_network n " + - "LEFT JOIN peeringdb_organization o ON n.org_id = o.id " + - "WHERE n.asn = ? AND n.status = 'ok'" - ).all(parseInt(netAsnMatch[1])); - return { data: rows }; - } - - // /net?status=ok&depth=0 (coverage endpoint — all networks) - if (path === "/net?status=ok&depth=0" || path.startsWith("/net?status=ok")) { - const rows = db.prepare( - "SELECT id, asn, name, aka, website, info_prefixes4, info_prefixes6, " + - "info_type, info_traffic, info_unicast, info_ipv6, policy_general, org_id " + - "FROM peeringdb_network WHERE status = 'ok' ORDER BY asn" - ).all(); - return { data: rows }; - } - - // /netixlan?net_id=X&limit=... or /netixlan?asn=X&limit=... - const netixlanNetId = path.match(/\/netixlan\?net_id=(\d+)/); - if (netixlanNetId) { - const rows = db.prepare( - "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + - "ni.operational, ni.bfd_support, il.id AS ixlan_id, " + - "ix.id AS ix_id, ix.name, ix.city, ix.country " + - "FROM peeringdb_network_ixlan ni " + - "LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " + - "LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " + - "WHERE ni.net_id = ? AND ni.status = 'ok'" - ).all(parseInt(netixlanNetId[1])); - return { data: rows }; - } - const netixlanAsn = path.match(/\/netixlan\?asn=(\d+)/); - if (netixlanAsn) { - const rows = db.prepare( - "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + - "ni.operational, ni.bfd_support, il.id AS ixlan_id, " + - "ix.id AS ix_id, ix.name, ix.city, ix.country " + - "FROM peeringdb_network_ixlan ni " + - "LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " + - "LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " + - "WHERE ni.asn = ? AND ni.status = 'ok'" - ).all(parseInt(netixlanAsn[1])); - return { data: rows }; - } - - // /netixlan?ixlan_id=X - const netixlanIxlanId = path.match(/\/netixlan\?ixlan_id=(\d+)/); - if (netixlanIxlanId) { - const rows = db.prepare( - "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + - "n.name AS net_name " + - "FROM peeringdb_network_ixlan ni " + - "LEFT JOIN peeringdb_network n ON ni.net_id = n.id " + - "WHERE ni.ixlan_id = ? AND ni.status = 'ok'" - ).all(parseInt(netixlanIxlanId[1])); - return { data: rows }; - } - - // /netfac?net_id=X - const netfacNetId = path.match(/\/netfac\?net_id=(\d+)/); - if (netfacNetId) { - const rows = db.prepare( - "SELECT nf.id, nf.net_id, f.id AS fac_id, f.name, f.city, f.state, " + - "f.country, f.latitude, f.longitude, f.website " + - "FROM peeringdb_network_facility nf " + - "LEFT JOIN peeringdb_facility f ON nf.fac_id = f.id " + - "WHERE nf.net_id = ? AND nf.status = 'ok'" - ).all(parseInt(netfacNetId[1])); - return { data: rows }; - } - - // /fac?id__in=X,Y,Z&fields=... - const facIdIn = path.match(/\/fac\?id__in=([\d,]+)/); - if (facIdIn) { - const ids = facIdIn[1].split(",").map(Number).filter(Boolean); - if (ids.length === 0) return { data: [] }; - const placeholders = ids.map(() => "?").join(","); - const rows = db.prepare( - "SELECT id, name, city, country, latitude, longitude, website " + - "FROM peeringdb_facility WHERE id IN (" + placeholders + ") AND status = 'ok'" - ).all(...ids); - return { data: rows }; - } - - // /ixfac?ix_id__in=X,Y,Z - const ixfacIxIdIn = path.match(/\/ixfac\?ix_id__in=([\d,]+)/); - if (ixfacIxIdIn) { - const ids = ixfacIxIdIn[1].split(",").map(Number).filter(Boolean); - if (ids.length === 0) return { data: [] }; - const placeholders = ids.map(() => "?").join(","); - const rows = db.prepare( - "SELECT ixf.id, ixf.ix_id, ixf.fac_id, f.latitude, f.longitude, f.city, f.country " + - "FROM peeringdb_ix_facility ixf " + - "LEFT JOIN peeringdb_facility f ON ixf.fac_id = f.id " + - "WHERE ixf.ix_id IN (" + placeholders + ") AND ixf.status = 'ok'" - ).all(...ids); - return { data: rows }; - } - - // /ix?name__contains=X - const ixNameContains = path.match(/\/ix\?name__contains=([^&]+)/); - if (ixNameContains) { - const term = "%" + decodeURIComponent(ixNameContains[1]) + "%"; - const rows = db.prepare( - "SELECT id, name, name_long, city, country, website, region_continent " + - "FROM peeringdb_ix WHERE (name LIKE ? OR name_long LIKE ?) AND status = 'ok' LIMIT 20" - ).all(term, term); - return { data: rows }; - } - - // /ixlan?ix_id=X - const ixlanIxId = path.match(/\/ixlan\?ix_id=(\d+)/); - if (ixlanIxId) { - const rows = db.prepare( - "SELECT id, ix_id, name, rs_asn, arp_sponge, mtu FROM peeringdb_ixlan " + - "WHERE ix_id = ? AND status = 'ok'" - ).all(parseInt(ixlanIxId[1])); - return { data: rows }; - } - - // /net/X (single network by PDB id) - const netById = path.match(/^\/net\/(\d+)$/); - if (netById) { - const row = db.prepare( - "SELECT n.*, o.name AS org_name FROM peeringdb_network n " + - "LEFT JOIN peeringdb_organization o ON n.org_id = o.id " + - "WHERE n.id = ? AND n.status = 'ok'" - ).get(parseInt(netById[1])); - return row ? { data: [row] } : { data: [] }; - } - - return null; // path not handled locally — fall through to live API - } catch (e) { - console.warn("[PeeringDB-local] Query error for", path, ":", e.message); - return null; - } -} +const { + PEERINGDB_API_KEY, + PEERINGDB_API_URL, + getPdbLocal, + queryPeeringDBLocal, + pdbSemaphore, + fetchPeeringDB, + fetchPeeringDBWithRetry, +} = require("./server/services/peeringdb"); const FEEDBACK_TOKEN = process.env.FEEDBACK_TOKEN || "changeme-set-in-env"; const FEEDBACK_FILE = "/opt/peercortex-app/feedback.json"; @@ -1095,42 +938,6 @@ const { } = require("./server/services/ripe-stat"); - -// PeeringDB semaphore — limits concurrent PDB requests to avoid 429 rate-limits -const pdbSemaphore = new Semaphore(5); - -// PeeringDB authenticated fetch helper — tries local SQLite first, falls back to live API -async function fetchPeeringDB(path, options) { - // Try local SQLite (instant, no rate-limits) — skip large "all networks" calls to live API - const localResult = queryPeeringDBLocal(path); - if (localResult !== null) return localResult; - - // Fallback: live PeeringDB API (throttled via semaphore) - const url = PEERINGDB_API_URL + path; - const headers = { "User-Agent": UA }; - if (PEERINGDB_API_KEY) { - headers["Authorization"] = "Api-Key " + PEERINGDB_API_KEY; - } - await pdbSemaphore.acquire(); - try { - return await fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } }); - } finally { - pdbSemaphore.release(); - } -} - -// PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load). -// Up to 3 attempts: immediate → 2s → 5s. Returns null only after all attempts exhausted. -async function fetchPeeringDBWithRetry(path, options) { - const delays = [2000, 5000]; - let result = await fetchPeeringDB(path, options); - for (let i = 0; i < delays.length && result === null; i++) { - await new Promise(r => setTimeout(r, delays[i])); - result = await fetchPeeringDB(path, options); - } - return result; -} - const { fetchJSON, fetchJSONWithRetry, diff --git a/server/services/peeringdb.js b/server/services/peeringdb.js new file mode 100644 index 0000000..0797731 --- /dev/null +++ b/server/services/peeringdb.js @@ -0,0 +1,216 @@ +const fs = require("fs"); +const { UA } = require("../data/constants"); +const { fetchJSON } = require("./http-helpers"); +const { Semaphore } = require("./ripe-stat"); + +const PEERINGDB_API_KEY = process.env.PEERINGDB_API_KEY || ""; +const PEERINGDB_API_URL = process.env.PEERINGDB_API_URL || "https://www.peeringdb.com/api"; + +// Local PeeringDB SQLite (peeringdb-py sync, refreshed daily by cron) +const PEERINGDB_LOCAL_PATH = process.env.PEERINGDB_LOCAL_PATH || "/opt/peeringdb-data/peeringdb.sqlite3"; +let _pdbLocal = null; +function getPdbLocal() { + if (_pdbLocal) return _pdbLocal; + try { + const BetterSqlite3 = require("better-sqlite3"); + if (!fs.existsSync(PEERINGDB_LOCAL_PATH)) return null; + _pdbLocal = new BetterSqlite3(PEERINGDB_LOCAL_PATH, { readonly: true, fileMustExist: true }); + console.log("[PeeringDB-local] SQLite opened:", PEERINGDB_LOCAL_PATH); + return _pdbLocal; + } catch (e) { + console.warn("[PeeringDB-local] Could not open SQLite:", e.message); + return null; + } +} + +// Map API path → SQLite result in { data: [...] } format, emulating the live PDB REST API. +function queryPeeringDBLocal(path) { + const db = getPdbLocal(); + if (!db) return null; + try { + // /net?asn=X + const netAsnMatch = path.match(/^\/net\?asn=(\d+)/); + if (netAsnMatch) { + const rows = db.prepare( + "SELECT n.*, o.name AS org_name FROM peeringdb_network n " + + "LEFT JOIN peeringdb_organization o ON n.org_id = o.id " + + "WHERE n.asn = ? AND n.status = 'ok'" + ).all(parseInt(netAsnMatch[1])); + return { data: rows }; + } + + // /net?status=ok&depth=0 (coverage endpoint — all networks) + if (path === "/net?status=ok&depth=0" || path.startsWith("/net?status=ok")) { + const rows = db.prepare( + "SELECT id, asn, name, aka, website, info_prefixes4, info_prefixes6, " + + "info_type, info_traffic, info_unicast, info_ipv6, policy_general, org_id " + + "FROM peeringdb_network WHERE status = 'ok' ORDER BY asn" + ).all(); + return { data: rows }; + } + + // /netixlan?net_id=X&limit=... or /netixlan?asn=X&limit=... + const netixlanNetId = path.match(/\/netixlan\?net_id=(\d+)/); + if (netixlanNetId) { + const rows = db.prepare( + "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + + "ni.operational, ni.bfd_support, il.id AS ixlan_id, " + + "ix.id AS ix_id, ix.name, ix.city, ix.country " + + "FROM peeringdb_network_ixlan ni " + + "LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " + + "LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " + + "WHERE ni.net_id = ? AND ni.status = 'ok'" + ).all(parseInt(netixlanNetId[1])); + return { data: rows }; + } + const netixlanAsn = path.match(/\/netixlan\?asn=(\d+)/); + if (netixlanAsn) { + const rows = db.prepare( + "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + + "ni.operational, ni.bfd_support, il.id AS ixlan_id, " + + "ix.id AS ix_id, ix.name, ix.city, ix.country " + + "FROM peeringdb_network_ixlan ni " + + "LEFT JOIN peeringdb_ixlan il ON ni.ixlan_id = il.id " + + "LEFT JOIN peeringdb_ix ix ON il.ix_id = ix.id " + + "WHERE ni.asn = ? AND ni.status = 'ok'" + ).all(parseInt(netixlanAsn[1])); + return { data: rows }; + } + + // /netixlan?ixlan_id=X + const netixlanIxlanId = path.match(/\/netixlan\?ixlan_id=(\d+)/); + if (netixlanIxlanId) { + const rows = db.prepare( + "SELECT ni.id, ni.net_id, ni.asn, ni.speed, ni.ipaddr4, ni.ipaddr6, ni.is_rs_peer, " + + "n.name AS net_name " + + "FROM peeringdb_network_ixlan ni " + + "LEFT JOIN peeringdb_network n ON ni.net_id = n.id " + + "WHERE ni.ixlan_id = ? AND ni.status = 'ok'" + ).all(parseInt(netixlanIxlanId[1])); + return { data: rows }; + } + + // /netfac?net_id=X + const netfacNetId = path.match(/\/netfac\?net_id=(\d+)/); + if (netfacNetId) { + const rows = db.prepare( + "SELECT nf.id, nf.net_id, f.id AS fac_id, f.name, f.city, f.state, " + + "f.country, f.latitude, f.longitude, f.website " + + "FROM peeringdb_network_facility nf " + + "LEFT JOIN peeringdb_facility f ON nf.fac_id = f.id " + + "WHERE nf.net_id = ? AND nf.status = 'ok'" + ).all(parseInt(netfacNetId[1])); + return { data: rows }; + } + + // /fac?id__in=X,Y,Z&fields=... + const facIdIn = path.match(/\/fac\?id__in=([\d,]+)/); + if (facIdIn) { + const ids = facIdIn[1].split(",").map(Number).filter(Boolean); + if (ids.length === 0) return { data: [] }; + const placeholders = ids.map(() => "?").join(","); + const rows = db.prepare( + "SELECT id, name, city, country, latitude, longitude, website " + + "FROM peeringdb_facility WHERE id IN (" + placeholders + ") AND status = 'ok'" + ).all(...ids); + return { data: rows }; + } + + // /ixfac?ix_id__in=X,Y,Z + const ixfacIxIdIn = path.match(/\/ixfac\?ix_id__in=([\d,]+)/); + if (ixfacIxIdIn) { + const ids = ixfacIxIdIn[1].split(",").map(Number).filter(Boolean); + if (ids.length === 0) return { data: [] }; + const placeholders = ids.map(() => "?").join(","); + const rows = db.prepare( + "SELECT ixf.id, ixf.ix_id, ixf.fac_id, f.latitude, f.longitude, f.city, f.country " + + "FROM peeringdb_ix_facility ixf " + + "LEFT JOIN peeringdb_facility f ON ixf.fac_id = f.id " + + "WHERE ixf.ix_id IN (" + placeholders + ") AND ixf.status = 'ok'" + ).all(...ids); + return { data: rows }; + } + + // /ix?name__contains=X + const ixNameContains = path.match(/\/ix\?name__contains=([^&]+)/); + if (ixNameContains) { + const term = "%" + decodeURIComponent(ixNameContains[1]) + "%"; + const rows = db.prepare( + "SELECT id, name, name_long, city, country, website, region_continent " + + "FROM peeringdb_ix WHERE (name LIKE ? OR name_long LIKE ?) AND status = 'ok' LIMIT 20" + ).all(term, term); + return { data: rows }; + } + + // /ixlan?ix_id=X + const ixlanIxId = path.match(/\/ixlan\?ix_id=(\d+)/); + if (ixlanIxId) { + const rows = db.prepare( + "SELECT id, ix_id, name, rs_asn, arp_sponge, mtu FROM peeringdb_ixlan " + + "WHERE ix_id = ? AND status = 'ok'" + ).all(parseInt(ixlanIxId[1])); + return { data: rows }; + } + + // /net/X (single network by PDB id) + const netById = path.match(/^\/net\/(\d+)$/); + if (netById) { + const row = db.prepare( + "SELECT n.*, o.name AS org_name FROM peeringdb_network n " + + "LEFT JOIN peeringdb_organization o ON n.org_id = o.id " + + "WHERE n.id = ? AND n.status = 'ok'" + ).get(parseInt(netById[1])); + return row ? { data: [row] } : { data: [] }; + } + + return null; // path not handled locally — fall through to live API + } catch (e) { + console.warn("[PeeringDB-local] Query error for", path, ":", e.message); + return null; + } +} + +// PeeringDB semaphore — limits concurrent PDB requests to avoid 429 rate-limits +const pdbSemaphore = new Semaphore(5); + +// PeeringDB authenticated fetch helper — tries local SQLite first, falls back to live API +async function fetchPeeringDB(path, options) { + // Try local SQLite (instant, no rate-limits) — skip large "all networks" calls to live API + const localResult = queryPeeringDBLocal(path); + if (localResult !== null) return localResult; + + // Fallback: live PeeringDB API (throttled via semaphore) + const url = PEERINGDB_API_URL + path; + const headers = { "User-Agent": UA }; + if (PEERINGDB_API_KEY) { + headers["Authorization"] = "Api-Key " + PEERINGDB_API_KEY; + } + await pdbSemaphore.acquire(); + try { + return await fetchJSON(url, { ...options, headers: { ...(options && options.headers || {}), ...headers } }); + } finally { + pdbSemaphore.release(); + } +} + +// PeeringDB fetch with exponential backoff retries (handles rate-limits under concurrent load). +// Up to 3 attempts: immediate → 2s → 5s. Returns null only after all attempts exhausted. +async function fetchPeeringDBWithRetry(path, options) { + const delays = [2000, 5000]; + let result = await fetchPeeringDB(path, options); + for (let i = 0; i < delays.length && result === null; i++) { + await new Promise(r => setTimeout(r, delays[i])); + result = await fetchPeeringDB(path, options); + } + return result; +} + +module.exports = { + PEERINGDB_API_KEY, + PEERINGDB_API_URL, + getPdbLocal, + queryPeeringDBLocal, + pdbSemaphore, + fetchPeeringDB, + fetchPeeringDBWithRetry, +};