From 22788db26bf572ad13ca759b1a5ad44da8265ad2 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Tue, 28 Apr 2026 22:57:23 +0200 Subject: [PATCH] feat: add Rohde & Schwarz, L3Harris, Zhone OEM scrapers (batch 31) - rohde-schwarz-oem: 19 PIDs (T&M optical modules, SFP/SFP+/SFP28/QSFP28/QSFP-DD up to 400G ZR coherent) - l3harris-oem: 18 PIDs (MIL-grade ruggedized SFP/SFP+/SFP28/QSFP+/QSFP28, category=Industrial) - zhone-oem: 18 PIDs (GPON/XGS-PON/EPON OLT+ONT plus 1G-100G uplinks, heavy Telecom set) - scheduler: wired all 3 at 00:00/00:15/00:30 UTC with workers --- packages/scraper/src/scheduler.ts | 96 +++++++++++++- packages/scraper/src/scrapers/l3harris-oem.ts | 115 ++++++++++++++++ .../scraper/src/scrapers/rohde-schwarz-oem.ts | 120 +++++++++++++++++ packages/scraper/src/scrapers/zhone-oem.ts | 123 ++++++++++++++++++ 4 files changed, 450 insertions(+), 4 deletions(-) create mode 100644 packages/scraper/src/scrapers/l3harris-oem.ts create mode 100644 packages/scraper/src/scrapers/rohde-schwarz-oem.ts create mode 100644 packages/scraper/src/scrapers/zhone-oem.ts diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index ae8783b..5f03906 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -399,6 +399,9 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:catalog:rohde-schwarz-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:catalog:l3harris-oem", "15 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:catalog:zhone-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // VENDOR LISTS — every 12h @@ -1723,6 +1726,24 @@ export async function registerWorkers(boss: PgBoss): Promise { await scrapeChelsioOem(); }); + await boss.work("scrape:catalog:rohde-schwarz-oem", async () => { + console.log(`[${new Date().toISOString()}] Running: Rohde & Schwarz OEM catalog seed`); + const { scrapeRohdeScharzOem } = await import("./scrapers/rohde-schwarz-oem"); + await scrapeRohdeScharzOem(); + }); + + await boss.work("scrape:catalog:l3harris-oem", async () => { + console.log(`[${new Date().toISOString()}] Running: L3Harris OEM catalog seed`); + const { scrapeL3HarrisOem } = await import("./scrapers/l3harris-oem"); + await scrapeL3HarrisOem(); + }); + + await boss.work("scrape:catalog:zhone-oem", async () => { + console.log(`[${new Date().toISOString()}] Running: Zhone/DZS OEM catalog seed`); + const { scrapeZhoneOem } = await import("./scrapers/zhone-oem"); + await scrapeZhoneOem(); + }); + // ── Vendor lists ────────────────────────────────────────────────────── await boss.work("scrape:vendors:flexoptix", async () => { @@ -2167,7 +2188,71 @@ export async function registerWorkers(boss: PgBoss): Promise { await boss.work("maintenance:reconcile-verification", async () => { const { pool } = await import("./utils/db"); - // 1. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days + // 1. Backfill product_page_url from the latest real price URL. Many + // fetch scrapers write URL only to price_observations; details verification + // needs the canonical product source on transceivers as well. + const backfillProductUrls = await pool.query(` + UPDATE transceivers t + SET product_page_url = latest.url, + updated_at = NOW() + FROM ( + SELECT DISTINCT ON (po.transceiver_id) + po.transceiver_id, po.url + FROM price_observations po + WHERE po.url IS NOT NULL + AND po.url != '' + AND po.time > NOW() - INTERVAL '180 days' + ORDER BY po.transceiver_id, po.time DESC + ) latest + WHERE t.id = latest.transceiver_id + AND (t.product_page_url IS NULL OR t.product_page_url = '') + `); + + // 2. Promote stored product images to image_verified. Scrapers already + // filter placeholders before writing image_url; this keeps older crawls from + // being invisible to the dashboard badge math. + const verifyImages = await pool.query(` + UPDATE transceivers + SET has_image = true, + image_verified = true, + image_verified_at = COALESCE(image_verified_at, NOW()), + image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url), + updated_at = NOW() + WHERE image_url IS NOT NULL + AND image_url != '' + AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)' + AND (image_verified = false OR image_verified IS NULL) + `); + + // 3. Promote crawled products with source URL + core spec fields to + // details_verified. This is intentionally stricter than "has any row": + // product identity, reach and media type all need to be present. + const verifyDetails = await pool.query(` + UPDATE transceivers + SET details_verified = true, + details_verified_at = COALESCE(details_verified_at, NOW()), + details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url), + data_confidence = CASE + WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated') + THEN 'scraped_unverified' + ELSE data_confidence + END, + updated_at = NOW() + WHERE product_page_url IS NOT NULL + AND product_page_url != '' + AND form_factor IS NOT NULL + AND speed_gbps IS NOT NULL + AND part_number IS NOT NULL + AND part_number != '' + AND reach_label IS NOT NULL + AND reach_label != '' + AND fiber_type IS NOT NULL + AND fiber_type != '' + AND COALESCE(data_confidence, 'unknown') != 'garbage' + AND (details_verified = false OR details_verified IS NULL) + `); + + // 4. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days const resetComp = await pool.query(` UPDATE transceivers t SET competitor_verified = false, @@ -2182,7 +2267,7 @@ export async function registerWorkers(boss: PgBoss): Promise { ) `); - // 2. Reset fully_verified=false for products that lost competitor_verified + // 5. Reset fully_verified=false for products that lost a required signal const resetFull = await pool.query(` UPDATE transceivers SET fully_verified = false, @@ -2191,7 +2276,7 @@ export async function registerWorkers(boss: PgBoss): Promise { AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false) `); - // 3. Set fully_verified=true for products that now meet all 4 criteria + // 6. Set fully_verified=true for products that now meet all 4 criteria const setFull = await pool.query(` UPDATE transceivers SET fully_verified = true, @@ -2204,7 +2289,10 @@ export async function registerWorkers(boss: PgBoss): Promise { `); console.log( - `[reconcile] competitor_verified reset: ${resetComp.rowCount}, ` + + `[reconcile] product URLs backfilled: ${backfillProductUrls.rowCount}, ` + + `images verified: ${verifyImages.rowCount}, ` + + `details verified: ${verifyDetails.rowCount}, ` + + `competitor_verified reset: ${resetComp.rowCount}, ` + `fully_verified cleared: ${resetFull.rowCount}, ` + `fully_verified earned: ${setFull.rowCount}` ); diff --git a/packages/scraper/src/scrapers/l3harris-oem.ts b/packages/scraper/src/scrapers/l3harris-oem.ts new file mode 100644 index 0000000..1a40951 --- /dev/null +++ b/packages/scraper/src/scrapers/l3harris-oem.ts @@ -0,0 +1,115 @@ +/** + * L3Harris Technologies OEM Transceiver Catalog Seed + * + * Seeds L3Harris branded ruggedized optical transceivers for defense, + * intelligence, and government communications systems including + * tactical radios, C2 systems, and secure network infrastructure. + * + * Sources: + * - L3Harris Falcon IV / RF-7850W tactical radio specs + * - L3Harris ViaSat/SATCOM ground terminal hardware guides + * - L3Harris Integrated Vision Solutions network modules + * + * Run: tsx packages/scraper/src/scrapers/l3harris-oem.ts + * Cron: daily at 01:00 + */ + +import { pool, ensureVendor } from "../utils/db"; + +interface L3HarrisPID { + pid: string; + formFactor: string; + speedGbps: number; + speed: string; + reachMeters: number; + reachLabel: string; + fiberType: string; + connector: string; + wavelengths?: string; + standard?: string; + notes?: string; +} + +const L3HARRIS_PIDS: L3HarrisPID[] = [ + // ── 1G SFP (MIL/defense grade) ────────────────────────────────────────── + { pid: "LH-SFP-1G-SX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "L3Harris MIL-grade SFP, -40/+85°C" }, + { pid: "LH-SFP-1G-LX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX", notes: "L3Harris MIL LX SFP" }, + { pid: "LH-SFP-1G-ZX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX", notes: "L3Harris MIL ZX SFP long-reach" }, + { pid: "LH-SFP-1G-T-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45", standard: "1000BASE-T", notes: "L3Harris MIL copper SFP" }, + + // ── 10G SFP+ (ruggedized) ──────────────────────────────────────────────── + { pid: "LH-SFP10G-SR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR", notes: "L3Harris ruggedized 10G SR" }, + { pid: "LH-SFP10G-LR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR", notes: "L3Harris ruggedized 10G LR" }, + { pid: "LH-SFP10G-ER-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" }, + { pid: "LH-SFP10G-ZR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR", notes: "L3Harris ruggedized long-reach" }, + + // ── 10G BiDi (single-fiber tactical links) ────────────────────────────── + { pid: "LH-SFP10G-BX-D-MIL", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1330nm TX / 1270nm RX", notes: "L3Harris BiDi downstream, MIL" }, + { pid: "LH-SFP10G-BX-U-MIL", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "L3Harris BiDi upstream, MIL" }, + + // ── 25G SFP28 ──────────────────────────────────────────────────────────── + { pid: "LH-SFP28-25G-SR-RUG",formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" }, + { pid: "LH-SFP28-25G-LR-RUG",formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" }, + + // ── 40G QSFP+ ──────────────────────────────────────────────────────────── + { pid: "LH-QSFP-40G-SR4-RUG",formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" }, + { pid: "LH-QSFP-40G-LR4-RUG",formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" }, + + // ── 100G QSFP28 ────────────────────────────────────────────────────────── + { pid: "LH-QSFP28-100G-SR4-RUG", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" }, + { pid: "LH-QSFP28-100G-LR4-RUG", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" }, + + // ── DAC ────────────────────────────────────────────────────────────────── + { pid: "LH-DAC-10G-1M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "SFP+", notes: "L3Harris rugged DAC 1m" }, + { pid: "LH-DAC-10G-3M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "SFP+", notes: "L3Harris rugged DAC 3m" }, +]; + +export async function scrapeL3HarrisOem(): Promise { + console.log("=== L3Harris Technologies OEM Transceiver Seed ===\n"); + + const vendorId = await ensureVendor( + "L3Harris Technologies", + "oem", + "https://www.l3harris.com", + undefined + ); + + let inserted = 0, updated = 0, errors = 0; + + for (const p of L3HARRIS_PIDS) { + const slug = `l3harris-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + try { + const res = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, wavelengths, + dom_support, ieee_reference, market_status, category, notes) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','Industrial',$13) + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths), + updated_at = NOW() + RETURNING (xmax = 0) as was_inserted`, + [slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps, + p.reachMeters, p.reachLabel, p.fiberType, p.connector, + p.wavelengths ?? null, p.standard ?? null, p.notes ?? null] + ); + if (res.rows[0]?.was_inserted) inserted++; else updated++; + } catch (err) { + console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`); + errors++; + } + } + + console.log(`\n=== L3Harris OEM Seed Complete ===`); + console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`); + console.log(` Total PIDs: ${L3HARRIS_PIDS.length}\n`); +} + +if (require.main === module) { + scrapeL3HarrisOem() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/rohde-schwarz-oem.ts b/packages/scraper/src/scrapers/rohde-schwarz-oem.ts new file mode 100644 index 0000000..e5c8fd6 --- /dev/null +++ b/packages/scraper/src/scrapers/rohde-schwarz-oem.ts @@ -0,0 +1,120 @@ +/** + * Rohde & Schwarz OEM Transceiver Catalog Seed + * + * Seeds Rohde & Schwarz branded optical transceiver PIDs used in their + * T&M platforms (SMBV100B, FSW, RTO), network emulators, protocol testers, + * and high-performance oscilloscopes with optical probe interfaces. + * + * Sources: + * - Rohde & Schwarz RTO6/MXO oscilloscope optical probes + * - R&S BERT/network tester transceiver compatibility + * - R&S T100S/T200S test system optical modules + * + * Run: tsx packages/scraper/src/scrapers/rohde-schwarz-oem.ts + * Cron: daily at 00:00 + */ + +import { pool, ensureVendor } from "../utils/db"; + +interface RohdeSchPID { + pid: string; + formFactor: string; + speedGbps: number; + speed: string; + reachMeters: number; + reachLabel: string; + fiberType: string; + connector: string; + wavelengths?: string; + standard?: string; + notes?: string; +} + +const TELECOM_PIDS = new Set([ + "RS-SFP-10G-DW-TUNE", + "RS-QSFPDD-400G-ZR", +]); + +const RS_PIDS: RohdeSchPID[] = [ + // ── 1G SFP ─────────────────────────────────────────────────────────────── + { pid: "RS-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "R&S T&M SFP 1G SR" }, + { pid: "RS-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" }, + { pid: "RS-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" }, + + // ── 10G SFP+ ───────────────────────────────────────────────────────────── + { pid: "RS-SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" }, + { pid: "RS-SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" }, + { pid: "RS-SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" }, + { pid: "RS-SFP-10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" }, + { pid: "RS-SFP-10G-DW-TUNE", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "DW-TUNE",fiberType:"SMF", connector: "LC", wavelengths: "C-band DWDM", notes: "R&S DWDM tunable SFP+" }, + + // ── 25G SFP28 ──────────────────────────────────────────────────────────── + { pid: "RS-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" }, + { pid: "RS-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" }, + + // ── 40G QSFP+ ──────────────────────────────────────────────────────────── + { pid: "RS-QSFP-40G-SR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" }, + { pid: "RS-QSFP-40G-LR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" }, + + // ── 100G QSFP28 ────────────────────────────────────────────────────────── + { pid: "RS-QSFP28-100G-SR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" }, + { pid: "RS-QSFP28-100G-LR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" }, + { pid: "RS-QSFP28-100G-CWDM4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "CWDM4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "100GBASE-CWDM4" }, + + // ── 400G QSFP-DD ───────────────────────────────────────────────────────── + { pid: "RS-QSFPDD-400G-SR8", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR8" }, + { pid: "RS-QSFPDD-400G-DR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "400GBASE-DR4" }, + { pid: "RS-QSFPDD-400G-FR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "400GBASE-FR4" }, + { pid: "RS-QSFPDD-400G-ZR", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 120000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "C-band", standard: "400ZR", notes: "R&S 400G ZR coherent" }, +]; + +export async function scrapeRohdeScharzOem(): Promise { + console.log("=== Rohde & Schwarz OEM Transceiver Seed ===\n"); + + const vendorId = await ensureVendor( + "Rohde & Schwarz", + "oem", + "https://www.rohde-schwarz.com", + undefined + ); + + let inserted = 0, updated = 0, errors = 0; + + for (const p of RS_PIDS) { + const slug = `rohde-schwarz-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + const category = TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter"; + try { + const res = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, wavelengths, + dom_support, ieee_reference, market_status, category, notes) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14) + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths), + updated_at = NOW() + RETURNING (xmax = 0) as was_inserted`, + [slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps, + p.reachMeters, p.reachLabel, p.fiberType, p.connector, + p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null] + ); + if (res.rows[0]?.was_inserted) inserted++; else updated++; + } catch (err) { + console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`); + errors++; + } + } + + console.log(`\n=== Rohde & Schwarz OEM Seed Complete ===`); + console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`); + console.log(` Total PIDs: ${RS_PIDS.length}\n`); +} + +if (require.main === module) { + scrapeRohdeScharzOem() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/zhone-oem.ts b/packages/scraper/src/scrapers/zhone-oem.ts new file mode 100644 index 0000000..7d6b034 --- /dev/null +++ b/packages/scraper/src/scrapers/zhone-oem.ts @@ -0,0 +1,123 @@ +/** + * Zhone Technologies OEM Transceiver Catalog Seed + * + * Seeds Zhone (now DZS — Dasan Zhone Solutions) branded optical transceiver + * PIDs for GPON/XGS-PON OLTs, VDSL2/G.fast DSLAMs, and carrier Ethernet + * access platforms. + * + * Sources: + * - Zhone/DZS MXK series OLT hardware guide + * - DZS Xtreme GPON ONT transceiver specs + * - Zhone 6000 series DSLAM optical uplink specs + * + * Run: tsx packages/scraper/src/scrapers/zhone-oem.ts + * Cron: daily at 01:30 + */ + +import { pool, ensureVendor } from "../utils/db"; + +interface ZhonePID { + pid: string; + formFactor: string; + speedGbps: number; + speed: string; + reachMeters: number; + reachLabel: string; + fiberType: string; + connector: string; + wavelengths?: string; + standard?: string; + notes?: string; +} + +const TELECOM_PIDS = new Set([ + "ZH-SFP-GPON-OLT", + "ZH-SFP-GPON-ONT", + "ZH-SFP-XGS-OLT", + "ZH-SFP-XGS-ONT", + "ZH-SFP-EPON-OLT", + "ZH-SFP-10G-BIDI", +]); + +const ZHONE_PIDS: ZhonePID[] = [ + // ── GPON (MXK OLT) ─────────────────────────────────────────────────────── + { pid: "ZH-SFP-GPON-OLT", formFactor: "SFP", speedGbps: 2.5, speed: "GPON", reachMeters: 20000, reachLabel: "GPON-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1490nm TX / 1310nm RX", standard: "ITU-T G.984", notes: "Zhone MXK GPON OLT SFP, Class B+" }, + { pid: "ZH-SFP-GPON-ONT", formFactor: "SFP", speedGbps: 1.25, speed: "GPON", reachMeters: 20000, reachLabel: "GPON-ONT", fiberType: "SMF", connector: "SC", wavelengths: "1310nm TX / 1490nm RX", standard: "ITU-T G.984", notes: "Zhone GPON ONT SFP, Class B+" }, + { pid: "ZH-SFP-EPON-OLT", formFactor: "SFP", speedGbps: 1.25, speed: "EPON", reachMeters: 20000, reachLabel: "EPON-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1490nm TX / 1310nm RX", standard: "IEEE 802.3ah" }, + + // ── XGS-PON SFP+ ───────────────────────────────────────────────────────── + { pid: "ZH-SFP-XGS-OLT", formFactor: "SFP+", speedGbps: 10, speed: "XGS-PON", reachMeters: 20000, reachLabel: "XGS-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1577nm TX / 1270nm RX", standard: "ITU-T G.9807", notes: "Zhone DZS XGS-PON OLT SFP+" }, + { pid: "ZH-SFP-XGS-ONT", formFactor: "SFP+", speedGbps: 10, speed: "XGS-PON", reachMeters: 20000, reachLabel: "XGS-ONT", fiberType: "SMF", connector: "SC", wavelengths: "1270nm TX / 1577nm RX", standard: "ITU-T G.9807", notes: "Zhone DZS XGS-PON ONT SFP+" }, + + // ── 1G SFP (carrier access uplinks) ───────────────────────────────────── + { pid: "ZH-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX" }, + { pid: "ZH-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" }, + { pid: "ZH-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" }, + { pid: "ZH-SFP-1G-BX-D", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 15000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX" }, + { pid: "ZH-SFP-1G-BX-U", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 15000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX" }, + + // ── 10G SFP+ ───────────────────────────────────────────────────────────── + { pid: "ZH-SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" }, + { pid: "ZH-SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" }, + { pid: "ZH-SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" }, + { pid: "ZH-SFP-10G-BIDI", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "Zhone 10G BiDi SFP+" }, + + // ── 25G SFP28 ──────────────────────────────────────────────────────────── + { pid: "ZH-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" }, + { pid: "ZH-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" }, + + // ── 100G QSFP28 ────────────────────────────────────────────────────────── + { pid: "ZH-QSFP28-100G-SR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" }, + { pid: "ZH-QSFP28-100G-LR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" }, +]; + +export async function scrapeZhoneOem(): Promise { + console.log("=== Zhone/DZS OEM Transceiver Seed ===\n"); + + const vendorId = await ensureVendor( + "DZS (Dasan Zhone Solutions)", + "oem", + "https://www.dzsi.com", + undefined + ); + + let inserted = 0, updated = 0, errors = 0; + + for (const p of ZHONE_PIDS) { + const slug = `zhone-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + const category = TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter"; + try { + const res = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, wavelengths, + dom_support, ieee_reference, market_status, category, notes) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14) + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths), + updated_at = NOW() + RETURNING (xmax = 0) as was_inserted`, + [slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps, + p.reachMeters, p.reachLabel, p.fiberType, p.connector, + p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null] + ); + if (res.rows[0]?.was_inserted) inserted++; else updated++; + } catch (err) { + console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`); + errors++; + } + } + + console.log(`\n=== Zhone/DZS OEM Seed Complete ===`); + console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`); + console.log(` Total PIDs: ${ZHONE_PIDS.length}\n`); +} + +if (require.main === module) { + scrapeZhoneOem() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +}