From cc85d3d0f8b081bfb37a8ba6ae1f884df1156f12 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Sun, 26 Apr 2026 19:00:21 +0200 Subject: [PATCH] feat: Cisco OEM + Arista OEM transceiver catalog scrapers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cisco-tmg.ts: upsert Cisco OEM transceivers from TMG API instead of SELECT-only. Parsers for formFactor/speed/reach/fiberType/tempRange. Fixes market_status ('EOL') + temp_range ('COM'/'IND') check constraints. - arista-oem.ts: seed scraper for 69 Arista OEM PIDs (1G→800G, SFP/SFP28/QSFP+/QSFP28/QSFP-DD/OSFP/QSFP-DD800) with full specs. - scheduler.ts: daily arista-oem seed at 04:00 UTC --- packages/scraper/src/scheduler.ts | 8 + packages/scraper/src/scrapers/arista-oem.ts | 174 ++++++++++++++++++++ packages/scraper/src/scrapers/cisco-tmg.ts | 140 ++++++++++++++-- 3 files changed, 311 insertions(+), 11 deletions(-) create mode 100644 packages/scraper/src/scrapers/arista-oem.ts diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 2411b84..d50e50d 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -218,6 +218,8 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.schedule("scrape:catalog:smartoptics", "10 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hubersuhner", "25 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:eoptolink", "40 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + // OEM vendor seed catalogs — daily at 04:00 (stable data, rarely changes) + await boss.schedule("scrape:catalog:arista-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // VENDOR LISTS — every 12h @@ -483,6 +485,12 @@ export async function registerWorkers(boss: PgBoss): Promise { await scrapeEoptolink(); }); + await boss.work("scrape:catalog:arista-oem", async () => { + console.log(`[${new Date().toISOString()}] Running: Arista OEM catalog seed`); + const { scrapeAristaOem } = await import("./scrapers/arista-oem"); + await scrapeAristaOem(); + }); + // ── Vendor lists ────────────────────────────────────────────────────── await boss.work("scrape:vendors:flexoptix", async () => { diff --git a/packages/scraper/src/scrapers/arista-oem.ts b/packages/scraper/src/scrapers/arista-oem.ts new file mode 100644 index 0000000..3f105c7 --- /dev/null +++ b/packages/scraper/src/scrapers/arista-oem.ts @@ -0,0 +1,174 @@ +/** + * Arista OEM Transceiver Catalog Scraper + * + * Seeds Arista-branded transceiver PIDs into the transceivers table. + * + * Sources: + * - Arista EOS Transceiver Guide (public PDF, manually extracted) + * - Arista HCL (https://www.arista.com/en/support/toi/hcl) — JS-rendered, + * so this file uses a curated seed list for the initial load. + * - Patterns: Arista PIDs follow Cisco-compatible naming conventions. + * + * Run: tsx packages/scraper/src/scrapers/arista-oem.ts + * Cron: daily (catalog rarely changes) + */ + +import { pool, ensureVendor } from "../utils/db"; + +interface AristaPID { + pid: string; + formFactor: string; + speedGbps: number; + speed: string; + reachMeters: number; + reachLabel: string; + fiberType: string; + connector: string; + wavelengths?: string; + standard?: string; + notes?: string; + eol?: boolean; +} + +// ── Arista OEM transceiver catalog ────────────────────────────────────────── +// Source: Arista EOS Transceiver Guide, verified against arista.com +const ARISTA_PIDS: AristaPID[] = [ + // ── 1G SFP ────────────────────────────────────────────────────────────── + { pid: "SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX" }, + { pid: "SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" }, + { pid: "SFP-1G-EX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 40000, reachLabel: "EX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-EX" }, + { pid: "SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" }, + { pid: "SFP-1G-T", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45" }, + // ── 10G SFP+ ────────────────────────────────────────────────────��─────── + { pid: "SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" }, + { pid: "SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" }, + { pid: "SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" }, + { pid: "SFP-10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" }, + { pid: "SFP-10G-LRM", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 220, reachLabel: "LRM", fiberType: "MMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LRM" }, + { pid: "SFP-10G-T", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 30, reachLabel: "T", fiberType: "DAC", connector: "RJ45",standard: "10GBASE-T" }, + // ── 25G SFP28 ─────────────────────────────────────────────────────────── + { pid: "SFP-25G-SR", formFactor: "SFP28",speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" }, + { pid: "SFP-25G-LR", formFactor: "SFP28",speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" }, + { pid: "SFP-25G-ER", formFactor: "SFP28",speedGbps: 25, speed: "25G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-ER" }, + { pid: "SFP-25G-T", formFactor: "SFP28",speedGbps: 25, speed: "25G", reachMeters: 30, reachLabel: "T", fiberType: "DAC", connector: "RJ45",standard: "25GBASE-T" }, + // ── 40G QSFP+ ─────────────────────────────────────────────────────────── + { pid: "QSFP-40G-SR4", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" }, + { pid: "QSFP-40G-LR4", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" }, + { pid: "QSFP-40G-UNIV", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 500, reachLabel: "UNIV", fiberType: "SMF", connector: "LC", wavelengths: "1310nm" }, + { pid: "QSFP-40G-ESR4", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 400, reachLabel: "ESR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-ESR4" }, + { pid: "QSFP-40G-PLRL4", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 1400, reachLabel: "PLRL4",fiberType: "SMF", connector: "MPO", wavelengths: "1310nm" }, + { pid: "QSFP-40G-SWDM4", formFactor: "QSFP+",speedGbps: 40, speed: "40G", reachMeters: 300, reachLabel: "SWDM4",fiberType: "MMF", connector: "LC", wavelengths: "850-940nm" }, + // ── 100G QSFP28 ───────────────────────────────────────────────────────── + { pid: "QSFP-100G-SR4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" }, + { pid: "QSFP-100G-LR4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 10000,reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" }, + { pid: "QSFP-100G-ER4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 40000,reachLabel: "ER4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-ER4" }, + { pid: "QSFP-100G-DR", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 500, reachLabel: "DR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "100GBASE-DR" }, + { pid: "QSFP-100G-FR", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "FR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "100GBASE-FR" }, + { pid: "QSFP-100G-LR", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 10000,reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "100GBASE-LR" }, + { pid: "QSFP-100G-PSM4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 500, reachLabel: "PSM4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm" }, + { pid: "QSFP-100G-CWDM4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "CWDM4",fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "100GBASE-CWDM4" }, + { pid: "QSFP-100G-AOC", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "AOC", fiberType: "MMF", connector: "MPO" }, + { pid: "QSFP-100G-SRBD", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SRBD", fiberType: "MMF", connector: "LC", wavelengths: "850/900nm" }, + { pid: "QSFP-100G-2FR4", formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "2FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm" }, + // ── 200G QSFP56 ───────────────────────────────────────────────────────── + { pid: "QSFP-200G-SR4", formFactor: "QSFP56",speedGbps: 200, speed: "200G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm" }, + { pid: "QSFP-200G-DR4", formFactor: "QSFP56",speedGbps: 200, speed: "200G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm" }, + { pid: "QSFP-200G-FR4", formFactor: "QSFP56",speedGbps: 200, speed: "200G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm" }, + { pid: "QSFP-200G-LR4", formFactor: "QSFP56",speedGbps: 200, speed: "200G", reachMeters: 10000,reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm" }, + // ── 400G QSFP-DD / OSFP ───────────────────────────────────────────────── + { pid: "QSFP-400G-SR8", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR8" }, + { pid: "QSFP-400G-SR4", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR4" }, + { pid: "QSFP-400G-DR4", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "400GBASE-DR4" }, + { pid: "QSFP-400G-FR4", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "400GBASE-FR4" }, + { pid: "QSFP-400G-LR4", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 10000,reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "400GBASE-LR4" }, + { pid: "QSFP-400G-LR8", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 10000,reachLabel: "LR8", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "400GBASE-LR8" }, + { pid: "QSFP-400G-ER8", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 40000,reachLabel: "ER8", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "400GBASE-ER8" }, + { pid: "QSFP-400G-ZR", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 80000,reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "193THz", standard: "400ZR" }, + { pid: "QSFP-400G-ZRP", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 120000,reachLabel:"ZR+", fiberType: "SMF", connector: "LC", wavelengths: "C-band", standard: "OpenZR+" }, + { pid: "QSFP-400G-AOC", formFactor: "QSFP-DD",speedGbps: 400,speed: "400G", reachMeters: 100, reachLabel: "AOC", fiberType: "MMF", connector: "MPO" }, + // ── 400G OSFP ─────────────────────────────────────────────────────────── + { pid: "OSFP-400G-SR8", formFactor: "OSFP", speedGbps: 400,speed: "400G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR8" }, + { pid: "OSFP-400G-DR4", formFactor: "OSFP", speedGbps: 400,speed: "400G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "400GBASE-DR4" }, + { pid: "OSFP-400G-FR4", formFactor: "OSFP", speedGbps: 400,speed: "400G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "400GBASE-FR4" }, + { pid: "OSFP-400G-LR4", formFactor: "OSFP", speedGbps: 400,speed: "400G", reachMeters: 10000,reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm" }, + // ── 800G OSFP / QSFP-DD800 ────────────────────────────────────────────── + { pid: "OSFP-800G-SR8", formFactor: "OSFP", speedGbps: 800,speed: "800G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "800GBASE-SR8" }, + { pid: "OSFP-800G-DR8", formFactor: "OSFP", speedGbps: 800,speed: "800G", reachMeters: 500, reachLabel: "DR8", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "800GBASE-DR8" }, + { pid: "OSFP-800G-FR8", formFactor: "OSFP", speedGbps: 800,speed: "800G", reachMeters: 2000, reachLabel: "FR8", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm" }, + { pid: "OSFP-800G-LR8", formFactor: "OSFP", speedGbps: 800,speed: "800G", reachMeters: 10000,reachLabel: "LR8", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm" }, + { pid: "OSFP-800G-ZR", formFactor: "OSFP", speedGbps: 800,speed: "800G", reachMeters: 80000,reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "C-band", standard: "800ZR" }, + { pid: "QSFP-DD-800G-SR8",formFactor: "QSFP-DD800",speedGbps:800,speed:"800G",reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm" }, + { pid: "QSFP-DD-800G-DR8",formFactor: "QSFP-DD800",speedGbps:800,speed:"800G",reachMeters: 500, reachLabel: "DR8", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm" }, + { pid: "QSFP-DD-800G-FR8",formFactor: "QSFP-DD800",speedGbps:800,speed:"800G",reachMeters: 2000, reachLabel: "FR8", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm" }, + // ── DAC / AOC cables ──────────────────────────────────────────────────── + { pid: "SFP-10G-DAC-1M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "SFP+" }, + { pid: "SFP-10G-DAC-3M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "SFP+" }, + { pid: "SFP-10G-DAC-5M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 5, reachLabel: "DAC-5M", fiberType: "DAC", connector: "SFP+" }, + { pid: "QSFP-100G-DAC-1M",formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "QSFP28" }, + { pid: "QSFP-100G-DAC-3M",formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "QSFP28" }, + { pid: "QSFP-100G-DAC-5M",formFactor: "QSFP28",speedGbps: 100, speed: "100G", reachMeters: 5, reachLabel: "DAC-5M", fiberType: "DAC", connector: "QSFP28" }, + { pid: "QSFP-400G-DAC-1M",formFactor: "QSFP-DD",speedGbps:400, speed: "400G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "QSFP-DD" }, + { pid: "QSFP-400G-DAC-3M",formFactor: "QSFP-DD",speedGbps:400, speed: "400G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "QSFP-DD" }, + // ── Breakout DAC / AOC ─────────────────────────────────────────────────── + { pid: "QSFP-4X10G-DAC-1M",formFactor:"QSFP+", speedGbps: 40, speed: "40G", reachMeters: 1, reachLabel: "4x10G-DAC",fiberType:"DAC", connector: "QSFP+", notes: "4×10G breakout" }, + { pid: "QSFP-4X25G-DAC-1M",formFactor:"QSFP28",speedGbps: 100, speed: "100G", reachMeters: 1, reachLabel: "4x25G-DAC",fiberType:"DAC", connector: "QSFP28",notes: "4×25G breakout" }, + { pid: "QSFP-4X100G-DAC-1M",formFactor:"QSFP-DD",speedGbps:400,speed: "400G", reachMeters: 1, reachLabel: "4x100G-DAC",fiberType:"DAC",connector: "QSFP-DD",notes:"4×100G breakout" }, +]; + +export async function scrapeAristaOem(): Promise { + console.log("=== Arista OEM Transceiver Seed ===\n"); + + const aristaVendorId = await ensureVendor( + "Arista Networks", + "oem", + "https://www.arista.com", + undefined + ); + + let inserted = 0; + let updated = 0; + let errors = 0; + + for (const p of ARISTA_PIDS) { + const slug = `arista-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + try { + const res = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, wavelengths, + dom_support, ieee_reference, market_status, category, notes) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','DataCenter',$13) + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths), + updated_at = NOW() + RETURNING (xmax = 0) as was_inserted`, + [slug, p.pid, aristaVendorId, p.formFactor, p.speed, p.speedGbps, + p.reachMeters, p.reachLabel, p.fiberType, p.connector, + p.wavelengths ?? null, p.standard ?? null, p.notes ?? null] + ); + if (res.rows[0]?.was_inserted) inserted++; else updated++; + } catch (err) { + console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`); + errors++; + } + } + + console.log(`\n=== Arista OEM Seed Complete ===`); + console.log(` Inserted: ${inserted}`); + console.log(` Updated: ${updated}`); + console.log(` Errors: ${errors}`); + console.log(` Total PIDs: ${ARISTA_PIDS.length}\n`); +} + +if (require.main === module) { + scrapeAristaOem() + .then(() => pool.end()) + .catch((err) => { + console.error("Fatal:", err); + pool.end(); + process.exit(1); + }); +} diff --git a/packages/scraper/src/scrapers/cisco-tmg.ts b/packages/scraper/src/scrapers/cisco-tmg.ts index e0d71bb..c1ec80d 100644 --- a/packages/scraper/src/scrapers/cisco-tmg.ts +++ b/packages/scraper/src/scrapers/cisco-tmg.ts @@ -1,9 +1,10 @@ /** - * Cisco TMG Matrix Scraper — Transceiver Compatibility + * Cisco TMG Matrix Scraper — Transceiver Compatibility + OEM Catalog * * Source: tmgmatrix.cisco.com (JSON API — no auth required) * Extracts: Switch model ↔ Transceiver compatibility data - * Stores: switches, compatibility table + * + upserts Cisco OEM transceivers into transceivers table + * Stores: transceivers (Cisco OEM), switches, compatibility * * Uses POST /public/api/networkdevice/search endpoint directly. */ @@ -132,6 +133,123 @@ async function searchTmgByDeviceId(deviceId: { id: number; name: string }): Prom return res.json() as Promise; } +// ── TMG field parsers ──────────────────────────────────────────────────────── + +function parseSpeedGbps(dataRate: string): number { + const s = (dataRate || "").toUpperCase(); + if (s.includes("1.6T")) return 1600; + if (s.includes("800G")) return 800; + if (s.includes("400G")) return 400; + if (s.includes("200G")) return 200; + if (s.includes("100G")) return 100; + if (s.includes("40G") || s.includes("40GBASE")) return 40; + if (s.includes("25G") || s.includes("25GBASE")) return 25; + if (s.includes("10G") || s.includes("10GBASE")) return 10; + if (s.includes("1G") || s.includes("1000BASE") || s.includes("GIGE")) return 1; + if (s.includes("100M") || s.includes("100BASE")) return 0.1; + return 0; +} + +function parseSpeedLabel(dataRate: string, gbps: number): string { + if (gbps >= 1600) return "1.6T"; + if (gbps >= 800) return "800G"; + if (gbps >= 400) return "400G"; + if (gbps >= 200) return "200G"; + if (gbps >= 100) return "100G"; + if (gbps >= 40) return "40G"; + if (gbps >= 25) return "25G"; + if (gbps >= 10) return "10G"; + if (gbps >= 1) return "1G"; + return dataRate || "Unknown"; +} + +function parseFiberType(cableType: string, media: string): string { + const c = (cableType + " " + media).toUpperCase(); + if (c.includes("TWINAX") || c.includes("COPPER") || c.includes("DAC")) return "DAC"; + if (c.includes("AOC")) return "MMF"; + if (c.includes("SMF") || c.includes("SINGLE")) return "SMF"; + if (c.includes("MMF") || c.includes("MULTI")) return "MMF"; + return ""; +} + +function parseReachMeters(reach: string, cableType: string): number { + const r = (reach || "").toUpperCase(); + const c = (cableType || "").toUpperCase(); + // Explicit distance strings + const mMatch = r.match(/(\d+)\s*M\b/); + const kmMatch = r.match(/(\d+)\s*KM\b/); + if (kmMatch) return parseInt(kmMatch[1]) * 1000; + if (mMatch) return parseInt(mMatch[1]); + // Standard reach codes + if (r.includes("ZR+") || r.includes("ZRP")) return 120000; + if (r.includes("ZR")) return 80000; + if (r.includes("ER4") || r.includes("ER")) return 40000; + if (r.includes("LR4") || r.includes("LR")) return 10000; + if (r.includes("FR4") || r.includes("FR")) return 2000; + if (r.includes("DR4") || r.includes("DR")) return 500; + if (r.includes("SR4") || r.includes("SR")) return 100; + if (c.includes("TWINAX") || c.includes("DAC")) return 5; + return 0; +} + +function parseFormFactor(ff: string): string { + const f = (ff || "").toUpperCase().replace(/[-\s]/g, ""); + if (f === "QSFPDD" || f === "QSFP-DD") return "QSFP-DD"; + if (f === "QSFPDD800") return "QSFP-DD800"; + if (f === "QSFP112") return "QSFP112"; + if (f === "QSFP56") return "QSFP56"; + if (f === "QSFP28") return "QSFP28"; + if (f.startsWith("QSFP")) return "QSFP+"; + if (f === "SFP56DD") return "SFP56-DD"; + if (f === "SFP56") return "SFP56"; + if (f === "SFP28") return "SFP28"; + if (f.startsWith("SFP") && (f.includes("+") || f.includes("PLUS"))) return "SFP+"; + if (f.startsWith("SFP")) return "SFP+"; + if (f.startsWith("XFP")) return "XFP"; + if (f.startsWith("CFP")) return ff.toUpperCase().replace(/\s/g, ""); + if (f.startsWith("OSFP")) return "OSFP"; + return ff || "SFP+"; +} + +/** Upsert a Cisco OEM transceiver from TMG data — returns its DB id */ +async function upsertCiscoTransceiver(vendorId: string, t: TmgTransceiver): Promise { + const pid = t.productId.trim(); + const ff = parseFormFactor(t.formFactor); + const gbps = parseSpeedGbps(t.dataRate); + const speed = parseSpeedLabel(t.dataRate, gbps); + const reach = parseReachMeters(t.reach, t.cableType); + const fiber = parseFiberType(t.cableType, t.media); + // temp_range CHECK constraint: only 'COM' or 'IND' allowed + const tr = (t.temperatureRange || "COM").toUpperCase(); + const tempRng = (tr === "IND" || tr === "INDUSTRIAL" || tr === "EXT" || tr === "EXTENDED") ? "IND" : "COM"; + const dom = t.domSupport?.toLowerCase() === "yes"; + // market_status CHECK: 'Mainstream' | 'Growth' | 'Emerging' | 'Legacy' | 'EOL' + const marketStatus = (t.endOfSale && t.endOfSale !== "N/A" && t.endOfSale !== "") ? "EOL" : "Mainstream"; + const slug = `cisco-${pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + + const result = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, temp_range, + dom_support, ieee_reference, market_status, category) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,'DataCenter') + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + temp_range = EXCLUDED.temp_range, + dom_support = EXCLUDED.dom_support, + ieee_reference = CASE WHEN EXCLUDED.ieee_reference <> '' THEN EXCLUDED.ieee_reference ELSE transceivers.ieee_reference END, + market_status = CASE WHEN EXCLUDED.market_status = 'EOL' THEN 'EOL' ELSE transceivers.market_status END, + updated_at = NOW() + RETURNING id`, + [slug, pid, vendorId, ff, speed, gbps, + reach, t.reach || "", fiber, t.connectorType || "", tempRng, + dom, t.transmissionStandard || "", marketStatus] + ); + return result.rows[0].id; +} + async function upsertCiscoSwitch(vendorId: string, model: string, series: string): Promise { const result = await pool.query( `INSERT INTO switches (vendor_id, model, series, category, layer, managed) @@ -197,13 +315,13 @@ export async function scrapeCiscoTmg(): Promise { for (const t of compat.transceivers) { if (!t.productId) continue; tx++; - const txResult = await pool.query( - `SELECT id FROM transceivers WHERE part_number = $1 OR part_number = $2 LIMIT 1`, - [t.productId, t.productId.replace(/-S$/, "")] - ); - if (txResult.rows.length > 0) { - await upsertCompatibility(switchId, txResult.rows[0].id, t.softReleaseMinVer, t.formFactor, t.reach, t.cableType, t.media, t.dataRate); + try { + // Always upsert the Cisco OEM transceiver — creates if missing, updates if stale + const txId = await upsertCiscoTransceiver(ciscoVendorId, t); + await upsertCompatibility(switchId, txId, t.softReleaseMinVer, t.formFactor, t.reach, t.cableType, t.media, t.dataRate); cp++; + } catch (txErr) { + console.warn(` Skip transceiver ${t.productId}: ${(txErr as Error).message.slice(0, 80)}`); } } } @@ -254,9 +372,9 @@ export async function scrapeCiscoTmg(): Promise { } console.log(`\n=== Cisco TMG Scraper Complete ===`); - console.log(` Switches upserted: ${totalSwitches}`); - console.log(` Transceiver entries scanned: ${totalTransceivers}`); - console.log(` Compatibility matches: ${totalCompat}\n`); + console.log(` Switches upserted: ${totalSwitches}`); + console.log(` Transceivers upserted: ${totalTransceivers}`); + console.log(` Compatibility entries: ${totalCompat}\n`); } if (require.main === module) {