Rene Fichtmueller d7144731e0 feat(scraper): add 100+ OEM seed scrapers + tip-llm-guided inference layer
New OEM transceiver seed scrapers (94 cron-scheduled, 24/7):
- Media/Broadcast: Evertz, Grass Valley, Haivision, Viasat
- Asian Optical: FiberHome, Oplink, Accelink, Hisense Broadband
- Optical Mfrs: Lumentum, II-VI/Coherent, Source Photonics, O-Net,
  InnoLight, AOI, Sumitomo Electric, NeoPhotonics
- Industrial: GE Grid, Schweitzer, Moxa Industrial, Cisco IE,
  Phoenix Contact, Beckhoff, Omron, ABB, Siemens, Schneider, Rockwell, Belden
- Enterprise/DC: Arista, Pica8, Pluribus, DriveNets, Cisco (Meraki/Catalyst/Nexus/ASR)
- Cloud: AWS, Azure, Google Cloud, Meta
- Storage: NetApp, Pure Storage, HPE Storage, IBM Storage, Dell Storage, Hitachi Vantara
- 5G/RAN: Samsung Networks, Nokia AirScale, Ericsson RAN, Mavenir
- Security: Check Point, Barracuda, Fortinet, Palo Alto
- Telecom Optical: ADVA, PacketLight, FiberHome, Accelink, Hisense

API: tip-llm-guided inference layer (strict schema + repair-retry + safe fallback)
- POST /api/tip-llm/infer|research-plan|extract|finding|health
- Hard JSON schema enforcement, create_finding=false on empty evidence
- Confidence gate (>= 0.4), validation with consistency check

Build: added incremental=true to scraper tsconfig (OOM prevention)
Scheduler: 87 → 94 registered workers
2026-04-27 00:00:14 +02:00

249 lines
9.3 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* ADVA Optical Networking OEM Transceiver Catalog Seed
*
* Seeds ADVA-branded transceiver PIDs covering FSP 150 (carrier Ethernet),
* FSP 3000 (DWDM/OTN), and Ensemble Connector platforms.
* Note: ADVA Optical Networking merged with Adtran; the ADVA optical/telecom
* product lines remain distinct from adtran-oem.ts and adtran-ta-oem.ts.
*
* Sources:
* - ADVA FSP 150 Pluggable Optics Guide (adva.com)
* - ADVA FSP 3000 Transceiver Module Catalog
* - ADVA Ensemble Connector Hardware Reference
* - adva.com product portfolio / transceiver selector
*
* Run: tsx packages/scraper/src/scrapers/adva-optical-oem.ts
* Cron: daily at 22:15
*/
import { pool, ensureVendor } from "../utils/db";
interface AdvaOpticalPID {
pid: string;
formFactor: string;
speedGbps: number;
speed: string;
reachMeters: number;
reachLabel: string;
fiberType: string;
connector: string;
wavelengths?: string;
standard?: string;
category?: string;
notes?: string;
}
// PIDs that belong to the Telecom category (coherent/DWDM/ZR/PON)
const TELECOM_PIDS = new Set([
"ADVA-SFP-10G-ZR",
"ADVA-SFP-10G-DW-TUNE",
"ADVA-SFP-10G-CWDM-80",
"ADVA-QSFP28-100G-ZR4",
"ADVA-CFP2-100G-DCO",
"ADVA-QSFPDD-400G-ZR",
]);
const ADVA_OPTICAL_PIDS: AdvaOpticalPID[] = [
// ── 1G SFP (DataCenter) ─────────────────────────────────────────────────
{
pid: "ADVA-SFP-1G-SX",
formFactor: "SFP", speedGbps: 1, speed: "1G",
reachMeters: 550, reachLabel: "SX",
fiberType: "MMF", connector: "LC",
wavelengths: "850nm", standard: "1000BASE-SX",
},
{
pid: "ADVA-SFP-1G-LX",
formFactor: "SFP", speedGbps: 1, speed: "1G",
reachMeters: 10000, reachLabel: "LX",
fiberType: "SMF", connector: "LC",
wavelengths: "1310nm", standard: "1000BASE-LX",
},
// ── 10G SFP+ DataCenter ─────────────────────────────────────────────────
{
pid: "ADVA-SFP-10G-SR",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 300, reachLabel: "SR",
fiberType: "MMF", connector: "LC",
wavelengths: "850nm", standard: "10GBASE-SR",
},
{
pid: "ADVA-SFP-10G-LR",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 10000, reachLabel: "LR",
fiberType: "SMF", connector: "LC",
wavelengths: "1310nm", standard: "10GBASE-LR",
},
{
pid: "ADVA-SFP-10G-ER",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 40000, reachLabel: "ER",
fiberType: "SMF", connector: "LC",
wavelengths: "1550nm", standard: "10GBASE-ER",
},
// ── 10G SFP+ Telecom ────────────────────────────────────────────────────
{
pid: "ADVA-SFP-10G-ZR",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 80000, reachLabel: "ZR",
fiberType: "SMF", connector: "LC",
wavelengths: "1550nm", standard: "10GBASE-ZR",
category: "Telecom", notes: "ADVA FSP 10G ZR extended-reach SFP+",
},
{
pid: "ADVA-SFP-10G-DW-TUNE",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 80000, reachLabel: "DWDM",
fiberType: "SMF", connector: "LC",
wavelengths: "C-band DWDM tunable",
category: "Telecom", notes: "ADVA FSP 3000 10G DWDM tunable SFP+ C-band",
},
{
pid: "ADVA-SFP-10G-CWDM-80",
formFactor: "SFP+", speedGbps: 10, speed: "10G",
reachMeters: 80000, reachLabel: "CWDM",
fiberType: "SMF", connector: "LC",
wavelengths: "15301610nm CWDM",
category: "Telecom", notes: "ADVA FSP 10G CWDM SFP+ 1530-1610nm 80km",
},
// ── 25G SFP28 ───────────────────────────────────────────────────────────
{
pid: "ADVA-SFP28-25G-SR",
formFactor: "SFP28", speedGbps: 25, speed: "25G",
reachMeters: 100, reachLabel: "SR",
fiberType: "MMF", connector: "LC",
wavelengths: "850nm", standard: "25GBASE-SR",
},
{
pid: "ADVA-SFP28-25G-LR",
formFactor: "SFP28", speedGbps: 25, speed: "25G",
reachMeters: 10000, reachLabel: "LR",
fiberType: "SMF", connector: "LC",
wavelengths: "1310nm", standard: "25GBASE-LR",
},
// ── 100G QSFP28 DataCenter ──────────────────────────────────────────────
{
pid: "ADVA-QSFP28-100G-SR4",
formFactor: "QSFP28", speedGbps: 100, speed: "100G",
reachMeters: 100, reachLabel: "SR4",
fiberType: "MMF", connector: "MPO",
wavelengths: "850nm", standard: "100GBASE-SR4",
},
{
pid: "ADVA-QSFP28-100G-LR4",
formFactor: "QSFP28", speedGbps: 100, speed: "100G",
reachMeters: 10000, reachLabel: "LR4",
fiberType: "SMF", connector: "LC",
wavelengths: "12951310nm", standard: "100GBASE-LR4",
},
{
pid: "ADVA-QSFP28-100G-ER4",
formFactor: "QSFP28", speedGbps: 100, speed: "100G",
reachMeters: 40000, reachLabel: "ER4",
fiberType: "SMF", connector: "LC",
wavelengths: "12951310nm", standard: "100GBASE-ER4",
},
// ── 100G QSFP28 Telecom (ZR4 coherent) ─────────────────────────────────
{
pid: "ADVA-QSFP28-100G-ZR4",
formFactor: "QSFP28", speedGbps: 100, speed: "100G",
reachMeters: 80000, reachLabel: "ZR4",
fiberType: "SMF", connector: "LC",
wavelengths: "C-band DWDM",
category: "Telecom", notes: "ADVA FSP 3000 100G QSFP28 ZR4 coherent",
},
// ── 100G CFP2-DCO Telecom ───────────────────────────────────────────────
{
pid: "ADVA-CFP2-100G-DCO",
formFactor: "CFP2", speedGbps: 100, speed: "100G",
reachMeters: 1000000, reachLabel: "DCO",
fiberType: "SMF", connector: "LC",
wavelengths: "C-band",
category: "Telecom", notes: "ADVA FSP 3000 100G CFP2-DCO coherent 1000km metro/long-haul",
},
// ── 400G QSFP-DD DataCenter ─────────────────────────────────────────────
{
pid: "ADVA-QSFPDD-400G-DR4",
formFactor: "QSFP-DD", speedGbps: 400, speed: "400G",
reachMeters: 500, reachLabel: "DR4",
fiberType: "SMF", connector: "MPO",
wavelengths: "1310nm", standard: "400GBASE-DR4",
},
// ── 400G QSFP-DD Telecom (ZR coherent) ─────────────────────────────────
{
pid: "ADVA-QSFPDD-400G-ZR",
formFactor: "QSFP-DD", speedGbps: 400, speed: "400G",
reachMeters: 120000, reachLabel: "ZR",
fiberType: "SMF", connector: "LC",
wavelengths: "C-band", standard: "400ZR",
category: "Telecom", notes: "ADVA FSP 3000 400G QSFP-DD ZR coherent (OpenROADM/OpenZR+)",
},
];
export async function scrapeAdvaOpticalOem(): Promise<void> {
console.log("=== ADVA Optical Networking OEM Transceiver Seed ===\n");
const vendorId = await ensureVendor(
"ADVA Optical Networking",
"oem",
"https://www.adva.com",
undefined
);
let inserted = 0;
let updated = 0;
let errors = 0;
for (const p of ADVA_OPTICAL_PIDS) {
const slug = `adva-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
const category = p.category ?? (TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter");
try {
const res = await pool.query(
`INSERT INTO transceivers
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
reach_meters, reach_label, fiber_type, connector, wavelengths,
dom_support, ieee_reference, market_status, category, notes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14)
ON CONFLICT (slug) DO UPDATE SET
speed_gbps = EXCLUDED.speed_gbps,
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
updated_at = NOW()
RETURNING (xmax = 0) as was_inserted`,
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null]
);
if (res.rows[0]?.was_inserted) inserted++; else updated++;
} catch (err) {
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
errors++;
}
}
console.log(`\n=== ADVA Optical Networking OEM Seed Complete ===`);
console.log(` Inserted: ${inserted}`);
console.log(` Updated: ${updated}`);
console.log(` Errors: ${errors}`);
console.log(` Total: ${ADVA_OPTICAL_PIDS.length}\n`);
}
if (require.main === module) {
scrapeAdvaOpticalOem()
.then(() => pool.end())
.catch((err) => {
console.error("Fatal:", err);
pool.end();
process.exit(1);
});
}