feat: add Rohde & Schwarz, L3Harris, Zhone OEM scrapers (batch 31)

- rohde-schwarz-oem: 19 PIDs (T&M optical modules, SFP/SFP+/SFP28/QSFP28/QSFP-DD up to 400G ZR coherent)
- l3harris-oem: 18 PIDs (MIL-grade ruggedized SFP/SFP+/SFP28/QSFP+/QSFP28, category=Industrial)
- zhone-oem: 18 PIDs (GPON/XGS-PON/EPON OLT+ONT plus 1G-100G uplinks, heavy Telecom set)
- scheduler: wired all 3 at 00:00/00:15/00:30 UTC with workers
This commit is contained in:
Rene Fichtmueller 2026-04-28 22:57:23 +02:00
parent b5decc517f
commit 22788db26b
4 changed files with 450 additions and 4 deletions

View File

@ -399,6 +399,9 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:rohde-schwarz-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:l3harris-oem", "15 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:zhone-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════
// VENDOR LISTS — every 12h // VENDOR LISTS — every 12h
@ -1723,6 +1726,24 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
await scrapeChelsioOem(); await scrapeChelsioOem();
}); });
await boss.work("scrape:catalog:rohde-schwarz-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Rohde & Schwarz OEM catalog seed`);
const { scrapeRohdeScharzOem } = await import("./scrapers/rohde-schwarz-oem");
await scrapeRohdeScharzOem();
});
await boss.work("scrape:catalog:l3harris-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: L3Harris OEM catalog seed`);
const { scrapeL3HarrisOem } = await import("./scrapers/l3harris-oem");
await scrapeL3HarrisOem();
});
await boss.work("scrape:catalog:zhone-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Zhone/DZS OEM catalog seed`);
const { scrapeZhoneOem } = await import("./scrapers/zhone-oem");
await scrapeZhoneOem();
});
// ── Vendor lists ────────────────────────────────────────────────────── // ── Vendor lists ──────────────────────────────────────────────────────
await boss.work("scrape:vendors:flexoptix", async () => { await boss.work("scrape:vendors:flexoptix", async () => {
@ -2167,7 +2188,71 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
await boss.work("maintenance:reconcile-verification", async () => { await boss.work("maintenance:reconcile-verification", async () => {
const { pool } = await import("./utils/db"); const { pool } = await import("./utils/db");
// 1. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days // 1. Backfill product_page_url from the latest real price URL. Many
// fetch scrapers write URL only to price_observations; details verification
// needs the canonical product source on transceivers as well.
const backfillProductUrls = await pool.query(`
UPDATE transceivers t
SET product_page_url = latest.url,
updated_at = NOW()
FROM (
SELECT DISTINCT ON (po.transceiver_id)
po.transceiver_id, po.url
FROM price_observations po
WHERE po.url IS NOT NULL
AND po.url != ''
AND po.time > NOW() - INTERVAL '180 days'
ORDER BY po.transceiver_id, po.time DESC
) latest
WHERE t.id = latest.transceiver_id
AND (t.product_page_url IS NULL OR t.product_page_url = '')
`);
// 2. Promote stored product images to image_verified. Scrapers already
// filter placeholders before writing image_url; this keeps older crawls from
// being invisible to the dashboard badge math.
const verifyImages = await pool.query(`
UPDATE transceivers
SET has_image = true,
image_verified = true,
image_verified_at = COALESCE(image_verified_at, NOW()),
image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url),
updated_at = NOW()
WHERE image_url IS NOT NULL
AND image_url != ''
AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)'
AND (image_verified = false OR image_verified IS NULL)
`);
// 3. Promote crawled products with source URL + core spec fields to
// details_verified. This is intentionally stricter than "has any row":
// product identity, reach and media type all need to be present.
const verifyDetails = await pool.query(`
UPDATE transceivers
SET details_verified = true,
details_verified_at = COALESCE(details_verified_at, NOW()),
details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url),
data_confidence = CASE
WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated')
THEN 'scraped_unverified'
ELSE data_confidence
END,
updated_at = NOW()
WHERE product_page_url IS NOT NULL
AND product_page_url != ''
AND form_factor IS NOT NULL
AND speed_gbps IS NOT NULL
AND part_number IS NOT NULL
AND part_number != ''
AND reach_label IS NOT NULL
AND reach_label != ''
AND fiber_type IS NOT NULL
AND fiber_type != ''
AND COALESCE(data_confidence, 'unknown') != 'garbage'
AND (details_verified = false OR details_verified IS NULL)
`);
// 4. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days
const resetComp = await pool.query(` const resetComp = await pool.query(`
UPDATE transceivers t UPDATE transceivers t
SET competitor_verified = false, SET competitor_verified = false,
@ -2182,7 +2267,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
) )
`); `);
// 2. Reset fully_verified=false for products that lost competitor_verified // 5. Reset fully_verified=false for products that lost a required signal
const resetFull = await pool.query(` const resetFull = await pool.query(`
UPDATE transceivers UPDATE transceivers
SET fully_verified = false, SET fully_verified = false,
@ -2191,7 +2276,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false) AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false)
`); `);
// 3. Set fully_verified=true for products that now meet all 4 criteria // 6. Set fully_verified=true for products that now meet all 4 criteria
const setFull = await pool.query(` const setFull = await pool.query(`
UPDATE transceivers UPDATE transceivers
SET fully_verified = true, SET fully_verified = true,
@ -2204,7 +2289,10 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
`); `);
console.log( console.log(
`[reconcile] competitor_verified reset: ${resetComp.rowCount}, ` + `[reconcile] product URLs backfilled: ${backfillProductUrls.rowCount}, ` +
`images verified: ${verifyImages.rowCount}, ` +
`details verified: ${verifyDetails.rowCount}, ` +
`competitor_verified reset: ${resetComp.rowCount}, ` +
`fully_verified cleared: ${resetFull.rowCount}, ` + `fully_verified cleared: ${resetFull.rowCount}, ` +
`fully_verified earned: ${setFull.rowCount}` `fully_verified earned: ${setFull.rowCount}`
); );

View File

@ -0,0 +1,115 @@
/**
* L3Harris Technologies OEM Transceiver Catalog Seed
*
* Seeds L3Harris branded ruggedized optical transceivers for defense,
* intelligence, and government communications systems including
* tactical radios, C2 systems, and secure network infrastructure.
*
* Sources:
* - L3Harris Falcon IV / RF-7850W tactical radio specs
* - L3Harris ViaSat/SATCOM ground terminal hardware guides
* - L3Harris Integrated Vision Solutions network modules
*
* Run: tsx packages/scraper/src/scrapers/l3harris-oem.ts
* Cron: daily at 01:00
*/
import { pool, ensureVendor } from "../utils/db";
interface L3HarrisPID {
pid: string;
formFactor: string;
speedGbps: number;
speed: string;
reachMeters: number;
reachLabel: string;
fiberType: string;
connector: string;
wavelengths?: string;
standard?: string;
notes?: string;
}
const L3HARRIS_PIDS: L3HarrisPID[] = [
// ── 1G SFP (MIL/defense grade) ──────────────────────────────────────────
{ pid: "LH-SFP-1G-SX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "L3Harris MIL-grade SFP, -40/+85°C" },
{ pid: "LH-SFP-1G-LX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX", notes: "L3Harris MIL LX SFP" },
{ pid: "LH-SFP-1G-ZX-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX", notes: "L3Harris MIL ZX SFP long-reach" },
{ pid: "LH-SFP-1G-T-MIL", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45", standard: "1000BASE-T", notes: "L3Harris MIL copper SFP" },
// ── 10G SFP+ (ruggedized) ────────────────────────────────────────────────
{ pid: "LH-SFP10G-SR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR", notes: "L3Harris ruggedized 10G SR" },
{ pid: "LH-SFP10G-LR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR", notes: "L3Harris ruggedized 10G LR" },
{ pid: "LH-SFP10G-ER-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
{ pid: "LH-SFP10G-ZR-RUG", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR", notes: "L3Harris ruggedized long-reach" },
// ── 10G BiDi (single-fiber tactical links) ──────────────────────────────
{ pid: "LH-SFP10G-BX-D-MIL", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1330nm TX / 1270nm RX", notes: "L3Harris BiDi downstream, MIL" },
{ pid: "LH-SFP10G-BX-U-MIL", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "L3Harris BiDi upstream, MIL" },
// ── 25G SFP28 ────────────────────────────────────────────────────────────
{ pid: "LH-SFP28-25G-SR-RUG",formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
{ pid: "LH-SFP28-25G-LR-RUG",formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
// ── 40G QSFP+ ────────────────────────────────────────────────────────────
{ pid: "LH-QSFP-40G-SR4-RUG",formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" },
{ pid: "LH-QSFP-40G-LR4-RUG",formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" },
// ── 100G QSFP28 ──────────────────────────────────────────────────────────
{ pid: "LH-QSFP28-100G-SR4-RUG", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
{ pid: "LH-QSFP28-100G-LR4-RUG", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
// ── DAC ──────────────────────────────────────────────────────────────────
{ pid: "LH-DAC-10G-1M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "SFP+", notes: "L3Harris rugged DAC 1m" },
{ pid: "LH-DAC-10G-3M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "SFP+", notes: "L3Harris rugged DAC 3m" },
];
export async function scrapeL3HarrisOem(): Promise<void> {
console.log("=== L3Harris Technologies OEM Transceiver Seed ===\n");
const vendorId = await ensureVendor(
"L3Harris Technologies",
"oem",
"https://www.l3harris.com",
undefined
);
let inserted = 0, updated = 0, errors = 0;
for (const p of L3HARRIS_PIDS) {
const slug = `l3harris-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
try {
const res = await pool.query(
`INSERT INTO transceivers
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
reach_meters, reach_label, fiber_type, connector, wavelengths,
dom_support, ieee_reference, market_status, category, notes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','Industrial',$13)
ON CONFLICT (slug) DO UPDATE SET
speed_gbps = EXCLUDED.speed_gbps,
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
updated_at = NOW()
RETURNING (xmax = 0) as was_inserted`,
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
p.wavelengths ?? null, p.standard ?? null, p.notes ?? null]
);
if (res.rows[0]?.was_inserted) inserted++; else updated++;
} catch (err) {
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
errors++;
}
}
console.log(`\n=== L3Harris OEM Seed Complete ===`);
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
console.log(` Total PIDs: ${L3HARRIS_PIDS.length}\n`);
}
if (require.main === module) {
scrapeL3HarrisOem()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}

View File

@ -0,0 +1,120 @@
/**
* Rohde & Schwarz OEM Transceiver Catalog Seed
*
* Seeds Rohde & Schwarz branded optical transceiver PIDs used in their
* T&M platforms (SMBV100B, FSW, RTO), network emulators, protocol testers,
* and high-performance oscilloscopes with optical probe interfaces.
*
* Sources:
* - Rohde & Schwarz RTO6/MXO oscilloscope optical probes
* - R&S BERT/network tester transceiver compatibility
* - R&S T100S/T200S test system optical modules
*
* Run: tsx packages/scraper/src/scrapers/rohde-schwarz-oem.ts
* Cron: daily at 00:00
*/
import { pool, ensureVendor } from "../utils/db";
interface RohdeSchPID {
pid: string;
formFactor: string;
speedGbps: number;
speed: string;
reachMeters: number;
reachLabel: string;
fiberType: string;
connector: string;
wavelengths?: string;
standard?: string;
notes?: string;
}
const TELECOM_PIDS = new Set([
"RS-SFP-10G-DW-TUNE",
"RS-QSFPDD-400G-ZR",
]);
const RS_PIDS: RohdeSchPID[] = [
// ── 1G SFP ───────────────────────────────────────────────────────────────
{ pid: "RS-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "R&S T&M SFP 1G SR" },
{ pid: "RS-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" },
{ pid: "RS-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" },
// ── 10G SFP+ ─────────────────────────────────────────────────────────────
{ pid: "RS-SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
{ pid: "RS-SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
{ pid: "RS-SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
{ pid: "RS-SFP-10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" },
{ pid: "RS-SFP-10G-DW-TUNE", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "DW-TUNE",fiberType:"SMF", connector: "LC", wavelengths: "C-band DWDM", notes: "R&S DWDM tunable SFP+" },
// ── 25G SFP28 ────────────────────────────────────────────────────────────
{ pid: "RS-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
{ pid: "RS-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
// ── 40G QSFP+ ────────────────────────────────────────────────────────────
{ pid: "RS-QSFP-40G-SR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" },
{ pid: "RS-QSFP-40G-LR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" },
// ── 100G QSFP28 ──────────────────────────────────────────────────────────
{ pid: "RS-QSFP28-100G-SR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
{ pid: "RS-QSFP28-100G-LR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
{ pid: "RS-QSFP28-100G-CWDM4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "CWDM4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "100GBASE-CWDM4" },
// ── 400G QSFP-DD ─────────────────────────────────────────────────────────
{ pid: "RS-QSFPDD-400G-SR8", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR8" },
{ pid: "RS-QSFPDD-400G-DR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "400GBASE-DR4" },
{ pid: "RS-QSFPDD-400G-FR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "400GBASE-FR4" },
{ pid: "RS-QSFPDD-400G-ZR", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 120000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "C-band", standard: "400ZR", notes: "R&S 400G ZR coherent" },
];
export async function scrapeRohdeScharzOem(): Promise<void> {
console.log("=== Rohde & Schwarz OEM Transceiver Seed ===\n");
const vendorId = await ensureVendor(
"Rohde & Schwarz",
"oem",
"https://www.rohde-schwarz.com",
undefined
);
let inserted = 0, updated = 0, errors = 0;
for (const p of RS_PIDS) {
const slug = `rohde-schwarz-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
const category = TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter";
try {
const res = await pool.query(
`INSERT INTO transceivers
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
reach_meters, reach_label, fiber_type, connector, wavelengths,
dom_support, ieee_reference, market_status, category, notes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14)
ON CONFLICT (slug) DO UPDATE SET
speed_gbps = EXCLUDED.speed_gbps,
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
updated_at = NOW()
RETURNING (xmax = 0) as was_inserted`,
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null]
);
if (res.rows[0]?.was_inserted) inserted++; else updated++;
} catch (err) {
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
errors++;
}
}
console.log(`\n=== Rohde & Schwarz OEM Seed Complete ===`);
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
console.log(` Total PIDs: ${RS_PIDS.length}\n`);
}
if (require.main === module) {
scrapeRohdeScharzOem()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}

View File

@ -0,0 +1,123 @@
/**
* Zhone Technologies OEM Transceiver Catalog Seed
*
* Seeds Zhone (now DZS Dasan Zhone Solutions) branded optical transceiver
* PIDs for GPON/XGS-PON OLTs, VDSL2/G.fast DSLAMs, and carrier Ethernet
* access platforms.
*
* Sources:
* - Zhone/DZS MXK series OLT hardware guide
* - DZS Xtreme GPON ONT transceiver specs
* - Zhone 6000 series DSLAM optical uplink specs
*
* Run: tsx packages/scraper/src/scrapers/zhone-oem.ts
* Cron: daily at 01:30
*/
import { pool, ensureVendor } from "../utils/db";
interface ZhonePID {
pid: string;
formFactor: string;
speedGbps: number;
speed: string;
reachMeters: number;
reachLabel: string;
fiberType: string;
connector: string;
wavelengths?: string;
standard?: string;
notes?: string;
}
const TELECOM_PIDS = new Set([
"ZH-SFP-GPON-OLT",
"ZH-SFP-GPON-ONT",
"ZH-SFP-XGS-OLT",
"ZH-SFP-XGS-ONT",
"ZH-SFP-EPON-OLT",
"ZH-SFP-10G-BIDI",
]);
const ZHONE_PIDS: ZhonePID[] = [
// ── GPON (MXK OLT) ───────────────────────────────────────────────────────
{ pid: "ZH-SFP-GPON-OLT", formFactor: "SFP", speedGbps: 2.5, speed: "GPON", reachMeters: 20000, reachLabel: "GPON-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1490nm TX / 1310nm RX", standard: "ITU-T G.984", notes: "Zhone MXK GPON OLT SFP, Class B+" },
{ pid: "ZH-SFP-GPON-ONT", formFactor: "SFP", speedGbps: 1.25, speed: "GPON", reachMeters: 20000, reachLabel: "GPON-ONT", fiberType: "SMF", connector: "SC", wavelengths: "1310nm TX / 1490nm RX", standard: "ITU-T G.984", notes: "Zhone GPON ONT SFP, Class B+" },
{ pid: "ZH-SFP-EPON-OLT", formFactor: "SFP", speedGbps: 1.25, speed: "EPON", reachMeters: 20000, reachLabel: "EPON-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1490nm TX / 1310nm RX", standard: "IEEE 802.3ah" },
// ── XGS-PON SFP+ ─────────────────────────────────────────────────────────
{ pid: "ZH-SFP-XGS-OLT", formFactor: "SFP+", speedGbps: 10, speed: "XGS-PON", reachMeters: 20000, reachLabel: "XGS-OLT", fiberType: "SMF", connector: "SC", wavelengths: "1577nm TX / 1270nm RX", standard: "ITU-T G.9807", notes: "Zhone DZS XGS-PON OLT SFP+" },
{ pid: "ZH-SFP-XGS-ONT", formFactor: "SFP+", speedGbps: 10, speed: "XGS-PON", reachMeters: 20000, reachLabel: "XGS-ONT", fiberType: "SMF", connector: "SC", wavelengths: "1270nm TX / 1577nm RX", standard: "ITU-T G.9807", notes: "Zhone DZS XGS-PON ONT SFP+" },
// ── 1G SFP (carrier access uplinks) ─────────────────────────────────────
{ pid: "ZH-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX" },
{ pid: "ZH-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" },
{ pid: "ZH-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" },
{ pid: "ZH-SFP-1G-BX-D", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 15000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX" },
{ pid: "ZH-SFP-1G-BX-U", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 15000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX" },
// ── 10G SFP+ ─────────────────────────────────────────────────────────────
{ pid: "ZH-SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
{ pid: "ZH-SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
{ pid: "ZH-SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
{ pid: "ZH-SFP-10G-BIDI", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "Zhone 10G BiDi SFP+" },
// ── 25G SFP28 ────────────────────────────────────────────────────────────
{ pid: "ZH-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
{ pid: "ZH-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
// ── 100G QSFP28 ──────────────────────────────────────────────────────────
{ pid: "ZH-QSFP28-100G-SR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
{ pid: "ZH-QSFP28-100G-LR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
];
export async function scrapeZhoneOem(): Promise<void> {
console.log("=== Zhone/DZS OEM Transceiver Seed ===\n");
const vendorId = await ensureVendor(
"DZS (Dasan Zhone Solutions)",
"oem",
"https://www.dzsi.com",
undefined
);
let inserted = 0, updated = 0, errors = 0;
for (const p of ZHONE_PIDS) {
const slug = `zhone-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
const category = TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter";
try {
const res = await pool.query(
`INSERT INTO transceivers
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
reach_meters, reach_label, fiber_type, connector, wavelengths,
dom_support, ieee_reference, market_status, category, notes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14)
ON CONFLICT (slug) DO UPDATE SET
speed_gbps = EXCLUDED.speed_gbps,
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
updated_at = NOW()
RETURNING (xmax = 0) as was_inserted`,
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null]
);
if (res.rows[0]?.was_inserted) inserted++; else updated++;
} catch (err) {
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
errors++;
}
}
console.log(`\n=== Zhone/DZS OEM Seed Complete ===`);
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
console.log(` Total PIDs: ${ZHONE_PIDS.length}\n`);
}
if (require.main === module) {
scrapeZhoneOem()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}