feat: add Black Box, Radiflow, DragonWave, Teledyne LeCroy OEM scrapers (batch 33)
- black-box-oem: 19 PIDs (enterprise LAN SFP/SFP+/SFP28/QSFP28 + BiDi + DAC) - radiflow-oem: 17 PIDs (OT/ICS security, 100M-100G incl. substation BiDi, category=Industrial) - dragonwave-oem: 17 PIDs (microwave backhaul fiber uplinks 100M-100G, market_status=Legacy) - teledyne-lecroy-oem: 18 PIDs (T&M oscilloscopes/analyzers SFP+-QSFP-DD up to 400G ZR) - scheduler: wired all 4 at 01:45/02:00/02:15/02:30 UTC
This commit is contained in:
Rene Fichtmueller
parent
7f59f445b6
commit
1023b24fd0
@ -406,6 +406,10 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
|||||||
await boss.schedule("scrape:catalog:tektronix-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
await boss.schedule("scrape:catalog:tektronix-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
await boss.schedule("scrape:catalog:clearfield-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
await boss.schedule("scrape:catalog:clearfield-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
await boss.schedule("scrape:catalog:lanner-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
await boss.schedule("scrape:catalog:lanner-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
|
await boss.schedule("scrape:catalog:black-box-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
|
await boss.schedule("scrape:catalog:radiflow-oem", "0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
|
await boss.schedule("scrape:catalog:dragonwave-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
|
await boss.schedule("scrape:catalog:teledyne-lecroy-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════
|
||||||
// VENDOR LISTS — every 12h
|
// VENDOR LISTS — every 12h
|
||||||
@ -1772,6 +1776,30 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
|||||||
await scrapeLannerOem();
|
await scrapeLannerOem();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await boss.work("scrape:catalog:black-box-oem", async () => {
|
||||||
|
console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`);
|
||||||
|
const { scrapeBlackBoxOem } = await import("./scrapers/black-box-oem");
|
||||||
|
await scrapeBlackBoxOem();
|
||||||
|
});
|
||||||
|
|
||||||
|
await boss.work("scrape:catalog:radiflow-oem", async () => {
|
||||||
|
console.log(`[${new Date().toISOString()}] Running: Radiflow OEM catalog seed`);
|
||||||
|
const { scrapeRadiflowOem } = await import("./scrapers/radiflow-oem");
|
||||||
|
await scrapeRadiflowOem();
|
||||||
|
});
|
||||||
|
|
||||||
|
await boss.work("scrape:catalog:dragonwave-oem", async () => {
|
||||||
|
console.log(`[${new Date().toISOString()}] Running: DragonWave OEM catalog seed`);
|
||||||
|
const { scrapeDragonwaveOem } = await import("./scrapers/dragonwave-oem");
|
||||||
|
await scrapeDragonwaveOem();
|
||||||
|
});
|
||||||
|
|
||||||
|
await boss.work("scrape:catalog:teledyne-lecroy-oem", async () => {
|
||||||
|
console.log(`[${new Date().toISOString()}] Running: Teledyne LeCroy OEM catalog seed`);
|
||||||
|
const { scrapeTeledyneLeCroyOem } = await import("./scrapers/teledyne-lecroy-oem");
|
||||||
|
await scrapeTeledyneLeCroyOem();
|
||||||
|
});
|
||||||
|
|
||||||
// ── Vendor lists ──────────────────────────────────────────────────────
|
// ── Vendor lists ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
await boss.work("scrape:vendors:flexoptix", async () => {
|
await boss.work("scrape:vendors:flexoptix", async () => {
|
||||||
|
|||||||
112
packages/scraper/src/scrapers/black-box-oem.ts
Normal file
112
packages/scraper/src/scrapers/black-box-oem.ts
Normal file
@ -0,0 +1,112 @@
|
|||||||
|
/**
|
||||||
|
* Black Box Corporation OEM Transceiver Catalog Seed
|
||||||
|
*
|
||||||
|
* Seeds Black Box branded SFP/SFP+/QSFP28 optical transceivers and media
|
||||||
|
* converters for enterprise LAN extension, fiber-to-copper conversion,
|
||||||
|
* and industrial managed switches (LGB5124A, LGC5210A series).
|
||||||
|
*
|
||||||
|
* Sources:
|
||||||
|
* - Black Box LGC fiber SFP compatibility guide
|
||||||
|
* - Black Box LGB managed switch SFP specs
|
||||||
|
* - Black Box industrial fiber converter catalog
|
||||||
|
*
|
||||||
|
* Run: tsx packages/scraper/src/scrapers/black-box-oem.ts
|
||||||
|
* Cron: daily at 01:45
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { pool, ensureVendor } from "../utils/db";
|
||||||
|
|
||||||
|
interface BlackBoxPID {
|
||||||
|
pid: string;
|
||||||
|
formFactor: string;
|
||||||
|
speedGbps: number;
|
||||||
|
speed: string;
|
||||||
|
reachMeters: number;
|
||||||
|
reachLabel: string;
|
||||||
|
fiberType: string;
|
||||||
|
connector: string;
|
||||||
|
wavelengths?: string;
|
||||||
|
standard?: string;
|
||||||
|
notes?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const BB_PIDS: BlackBoxPID[] = [
|
||||||
|
// ── 1G SFP ───────────────────────────────────────────────────────────
|
||||||
|
{ pid: "BB-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "Black Box 1G SX SFP" },
|
||||||
|
{ pid: "BB-SFP-1G-LX10", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX10", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX", notes: "Black Box 1G LX 10km" },
|
||||||
|
{ pid: "BB-SFP-1G-LX40", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 40000, reachLabel: "LX40", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", notes: "Black Box 1G LX 40km" },
|
||||||
|
{ pid: "BB-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" },
|
||||||
|
{ pid: "BB-SFP-1G-BX-D", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX", notes: "Black Box BiDi downstream" },
|
||||||
|
{ pid: "BB-SFP-1G-BX-U", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX", notes: "Black Box BiDi upstream" },
|
||||||
|
{ pid: "BB-SFP-1G-T", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45", standard: "1000BASE-T" },
|
||||||
|
|
||||||
|
// ── 10G SFP+ ─────────────────────────────────────────────────────────
|
||||||
|
{ pid: "BB-SFP10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
|
||||||
|
{ pid: "BB-SFP10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
|
||||||
|
{ pid: "BB-SFP10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
|
||||||
|
{ pid: "BB-SFP10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" },
|
||||||
|
{ pid: "BB-SFP10G-BX-D", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1330nm TX / 1270nm RX" },
|
||||||
|
{ pid: "BB-SFP10G-BX-U", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX" },
|
||||||
|
|
||||||
|
// ── 25G SFP28 ────────────────────────────────────────────────────────
|
||||||
|
{ pid: "BB-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
|
||||||
|
{ pid: "BB-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
|
||||||
|
|
||||||
|
// ── 100G QSFP28 ──────────────────────────────────────────────────────
|
||||||
|
{ pid: "BB-QSFP28-100G-SR4", formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
|
||||||
|
{ pid: "BB-QSFP28-100G-LR4", formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
|
||||||
|
|
||||||
|
// ── DAC ──────────────────────────────────────────────────────────────
|
||||||
|
{ pid: "BB-DAC-10G-1M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 1, reachLabel: "DAC-1M", fiberType: "DAC", connector: "SFP+", notes: "Black Box DAC 1m" },
|
||||||
|
{ pid: "BB-DAC-10G-3M", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 3, reachLabel: "DAC-3M", fiberType: "DAC", connector: "SFP+", notes: "Black Box DAC 3m" },
|
||||||
|
];
|
||||||
|
|
||||||
|
export async function scrapeBlackBoxOem(): Promise<void> {
|
||||||
|
console.log("=== Black Box Corporation OEM Transceiver Seed ===\n");
|
||||||
|
|
||||||
|
const vendorId = await ensureVendor(
|
||||||
|
"Black Box Corporation",
|
||||||
|
"oem",
|
||||||
|
"https://www.blackbox.com",
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
let inserted = 0, updated = 0, errors = 0;
|
||||||
|
|
||||||
|
for (const p of BB_PIDS) {
|
||||||
|
const slug = `black-box-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
|
||||||
|
try {
|
||||||
|
const res = await pool.query(
|
||||||
|
`INSERT INTO transceivers
|
||||||
|
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
|
||||||
|
reach_meters, reach_label, fiber_type, connector, wavelengths,
|
||||||
|
dom_support, ieee_reference, market_status, category, notes)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','DataCenter',$13)
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET
|
||||||
|
speed_gbps = EXCLUDED.speed_gbps,
|
||||||
|
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
|
||||||
|
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
|
||||||
|
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) as was_inserted`,
|
||||||
|
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
|
||||||
|
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
|
||||||
|
p.wavelengths ?? null, p.standard ?? null, p.notes ?? null]
|
||||||
|
);
|
||||||
|
if (res.rows[0]?.was_inserted) inserted++; else updated++;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n=== Black Box OEM Seed Complete ===`);
|
||||||
|
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
|
||||||
|
console.log(` Total PIDs: ${BB_PIDS.length}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
scrapeBlackBoxOem()
|
||||||
|
.then(() => pool.end())
|
||||||
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
||||||
|
}
|
||||||
110
packages/scraper/src/scrapers/dragonwave-oem.ts
Normal file
110
packages/scraper/src/scrapers/dragonwave-oem.ts
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
/**
|
||||||
|
* DragonWave OEM Transceiver Catalog Seed
|
||||||
|
*
|
||||||
|
* Seeds DragonWave (now Cambium Networks microwave portfolio) branded optical
|
||||||
|
* transceiver PIDs for their Horizon Compact, Horizon Duo, and Avenue Link
|
||||||
|
* microwave backhaul platforms with Ethernet fiber uplink modules.
|
||||||
|
*
|
||||||
|
* Sources:
|
||||||
|
* - DragonWave Horizon Compact+ hardware installation guide
|
||||||
|
* - DragonWave Avenue Link fiber interconnect specs
|
||||||
|
* - Cambium/DragonWave XMS platform SFP compatibility
|
||||||
|
*
|
||||||
|
* Run: tsx packages/scraper/src/scrapers/dragonwave-oem.ts
|
||||||
|
* Cron: daily at 02:15
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { pool, ensureVendor } from "../utils/db";
|
||||||
|
|
||||||
|
interface DragonwavePID {
|
||||||
|
pid: string;
|
||||||
|
formFactor: string;
|
||||||
|
speedGbps: number;
|
||||||
|
speed: string;
|
||||||
|
reachMeters: number;
|
||||||
|
reachLabel: string;
|
||||||
|
fiberType: string;
|
||||||
|
connector: string;
|
||||||
|
wavelengths?: string;
|
||||||
|
standard?: string;
|
||||||
|
notes?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DW_PIDS: DragonwavePID[] = [
|
||||||
|
// ── 100M SFP (Horizon Compact legacy) ────────────────────────────────
|
||||||
|
{ pid: "DW-SFP-100M-FX", formFactor: "SFP", speedGbps: 0.1, speed: "100M", reachMeters: 2000, reachLabel: "FX", fiberType: "MMF", connector: "LC", wavelengths: "1310nm", standard: "100BASE-FX", notes: "DragonWave Horizon 100M FX" },
|
||||||
|
{ pid: "DW-SFP-100M-LX", formFactor: "SFP", speedGbps: 0.1, speed: "100M", reachMeters: 15000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", notes: "DragonWave Horizon 100M LX" },
|
||||||
|
|
||||||
|
// ── 1G SFP (Horizon Duo / Avenue Link) ──────────────────────────────
|
||||||
|
{ pid: "DW-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "DragonWave 1G SX" },
|
||||||
|
{ pid: "DW-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" },
|
||||||
|
{ pid: "DW-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" },
|
||||||
|
{ pid: "DW-SFP-1G-BX-D", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX", notes: "DragonWave BiDi DS" },
|
||||||
|
{ pid: "DW-SFP-1G-BX-U", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX", notes: "DragonWave BiDi US" },
|
||||||
|
|
||||||
|
// ── 10G SFP+ (Avenue 10G backhaul uplink) ───────────────────────────
|
||||||
|
{ pid: "DW-SFP10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
|
||||||
|
{ pid: "DW-SFP10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
|
||||||
|
{ pid: "DW-SFP10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
|
||||||
|
{ pid: "DW-SFP10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" },
|
||||||
|
{ pid: "DW-SFP10G-BX-D", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1330nm TX / 1270nm RX", notes: "DragonWave 10G BiDi DS" },
|
||||||
|
{ pid: "DW-SFP10G-BX-U", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "DragonWave 10G BiDi US" },
|
||||||
|
|
||||||
|
// ── 25G SFP28 ────────────────────────────────────────────────────────
|
||||||
|
{ pid: "DW-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
|
||||||
|
{ pid: "DW-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
|
||||||
|
|
||||||
|
// ── 100G QSFP28 ──────────────────────────────────────────────────────
|
||||||
|
{ pid: "DW-QSFP28-100G-SR4", formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
|
||||||
|
{ pid: "DW-QSFP28-100G-LR4", formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
|
||||||
|
];
|
||||||
|
|
||||||
|
export async function scrapeDragonwaveOem(): Promise<void> {
|
||||||
|
console.log("=== DragonWave OEM Transceiver Seed ===\n");
|
||||||
|
|
||||||
|
const vendorId = await ensureVendor(
|
||||||
|
"DragonWave",
|
||||||
|
"oem",
|
||||||
|
"https://www.dragonwave.com",
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
let inserted = 0, updated = 0, errors = 0;
|
||||||
|
|
||||||
|
for (const p of DW_PIDS) {
|
||||||
|
const slug = `dragonwave-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
|
||||||
|
try {
|
||||||
|
const res = await pool.query(
|
||||||
|
`INSERT INTO transceivers
|
||||||
|
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
|
||||||
|
reach_meters, reach_label, fiber_type, connector, wavelengths,
|
||||||
|
dom_support, ieee_reference, market_status, category, notes)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Legacy','DataCenter',$13)
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET
|
||||||
|
speed_gbps = EXCLUDED.speed_gbps,
|
||||||
|
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
|
||||||
|
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
|
||||||
|
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) as was_inserted`,
|
||||||
|
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
|
||||||
|
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
|
||||||
|
p.wavelengths ?? null, p.standard ?? null, p.notes ?? null]
|
||||||
|
);
|
||||||
|
if (res.rows[0]?.was_inserted) inserted++; else updated++;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n=== DragonWave OEM Seed Complete ===`);
|
||||||
|
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
|
||||||
|
console.log(` Total PIDs: ${DW_PIDS.length}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
scrapeDragonwaveOem()
|
||||||
|
.then(() => pool.end())
|
||||||
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
||||||
|
}
|
||||||
111
packages/scraper/src/scrapers/radiflow-oem.ts
Normal file
111
packages/scraper/src/scrapers/radiflow-oem.ts
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
/**
|
||||||
|
* Radiflow OEM Transceiver Catalog Seed
|
||||||
|
*
|
||||||
|
* Seeds Radiflow branded optical transceiver PIDs for their OT/ICS
|
||||||
|
* cybersecurity platforms (iSID, iSAP), industrial network monitoring
|
||||||
|
* sensors, and secure remote access gateways used in critical
|
||||||
|
* infrastructure (power, water, oil & gas).
|
||||||
|
*
|
||||||
|
* Sources:
|
||||||
|
* - Radiflow iSID ICS intrusion detection platform specs
|
||||||
|
* - Radiflow iSAP industrial secure remote access guide
|
||||||
|
* - Radiflow 3180 industrial switch transceiver compatibility
|
||||||
|
*
|
||||||
|
* Run: tsx packages/scraper/src/scrapers/radiflow-oem.ts
|
||||||
|
* Cron: daily at 02:00
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { pool, ensureVendor } from "../utils/db";
|
||||||
|
|
||||||
|
interface RadioflowPID {
|
||||||
|
pid: string;
|
||||||
|
formFactor: string;
|
||||||
|
speedGbps: number;
|
||||||
|
speed: string;
|
||||||
|
reachMeters: number;
|
||||||
|
reachLabel: string;
|
||||||
|
fiberType: string;
|
||||||
|
connector: string;
|
||||||
|
wavelengths?: string;
|
||||||
|
standard?: string;
|
||||||
|
notes?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RF_PIDS: RadioflowPID[] = [
|
||||||
|
// ── 100M SFP (ICS/SCADA sensor links) ───────────────────────────────
|
||||||
|
{ pid: "RFL-SFP-100M-FX", formFactor: "SFP", speedGbps: 0.1, speed: "100M", reachMeters: 2000, reachLabel: "FX", fiberType: "MMF", connector: "LC", wavelengths: "1310nm", standard: "100BASE-FX", notes: "Radiflow ICS 100M SFP" },
|
||||||
|
{ pid: "RFL-SFP-100M-LX", formFactor: "SFP", speedGbps: 0.1, speed: "100M", reachMeters: 20000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", notes: "Radiflow ICS 100M LX 20km" },
|
||||||
|
|
||||||
|
// ── 1G SFP (industrial grade) ────────────────────────────────────────
|
||||||
|
{ pid: "RFL-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "Radiflow iSID SFP" },
|
||||||
|
{ pid: "RFL-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" },
|
||||||
|
{ pid: "RFL-SFP-1G-LX40", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 40000, reachLabel: "LX40", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", notes: "Radiflow substation 40km link" },
|
||||||
|
{ pid: "RFL-SFP-1G-ZX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 80000, reachLabel: "ZX", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "1000BASE-ZX" },
|
||||||
|
{ pid: "RFL-SFP-1G-BX-D", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX", notes: "Radiflow BiDi downstream" },
|
||||||
|
{ pid: "RFL-SFP-1G-BX-U", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 20000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX", notes: "Radiflow BiDi upstream" },
|
||||||
|
{ pid: "RFL-SFP-1G-T", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45", standard: "1000BASE-T" },
|
||||||
|
|
||||||
|
// ── 10G SFP+ ─────────────────────────────────────────────────────────
|
||||||
|
{ pid: "RFL-SFP10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
|
||||||
|
{ pid: "RFL-SFP10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
|
||||||
|
{ pid: "RFL-SFP10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
|
||||||
|
{ pid: "RFL-SFP10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" },
|
||||||
|
|
||||||
|
// ── 10G BiDi (single-fiber substation links) ─────────────────────────
|
||||||
|
{ pid: "RFL-SFP10G-BX-D", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-D", fiberType: "SMF", connector: "LC", wavelengths: "1330nm TX / 1270nm RX", notes: "Radiflow BiDi 10G downstream" },
|
||||||
|
{ pid: "RFL-SFP10G-BX-U", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BX-U", fiberType: "SMF", connector: "LC", wavelengths: "1270nm TX / 1330nm RX", notes: "Radiflow BiDi 10G upstream" },
|
||||||
|
|
||||||
|
// ── 100G QSFP28 ──────────────────────────────────────────────────────
|
||||||
|
{ pid: "RFL-QSFP28-100G-SR4",formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
|
||||||
|
{ pid: "RFL-QSFP28-100G-LR4",formFactor: "QSFP28",speedGbps: 100,speed: "100G",reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
|
||||||
|
];
|
||||||
|
|
||||||
|
export async function scrapeRadiflowOem(): Promise<void> {
|
||||||
|
console.log("=== Radiflow OEM Transceiver Seed ===\n");
|
||||||
|
|
||||||
|
const vendorId = await ensureVendor(
|
||||||
|
"Radiflow",
|
||||||
|
"oem",
|
||||||
|
"https://www.radiflow.com",
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
let inserted = 0, updated = 0, errors = 0;
|
||||||
|
|
||||||
|
for (const p of RF_PIDS) {
|
||||||
|
const slug = `radiflow-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
|
||||||
|
try {
|
||||||
|
const res = await pool.query(
|
||||||
|
`INSERT INTO transceivers
|
||||||
|
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
|
||||||
|
reach_meters, reach_label, fiber_type, connector, wavelengths,
|
||||||
|
dom_support, ieee_reference, market_status, category, notes)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','Industrial',$13)
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET
|
||||||
|
speed_gbps = EXCLUDED.speed_gbps,
|
||||||
|
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
|
||||||
|
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
|
||||||
|
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) as was_inserted`,
|
||||||
|
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
|
||||||
|
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
|
||||||
|
p.wavelengths ?? null, p.standard ?? null, p.notes ?? null]
|
||||||
|
);
|
||||||
|
if (res.rows[0]?.was_inserted) inserted++; else updated++;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n=== Radiflow OEM Seed Complete ===`);
|
||||||
|
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
|
||||||
|
console.log(` Total PIDs: ${RF_PIDS.length}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
scrapeRadiflowOem()
|
||||||
|
.then(() => pool.end())
|
||||||
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
||||||
|
}
|
||||||
120
packages/scraper/src/scrapers/teledyne-lecroy-oem.ts
Normal file
120
packages/scraper/src/scrapers/teledyne-lecroy-oem.ts
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
/**
|
||||||
|
* Teledyne LeCroy OEM Transceiver Catalog Seed
|
||||||
|
*
|
||||||
|
* Seeds Teledyne LeCroy branded optical transceiver PIDs used in their
|
||||||
|
* WavePro / WaveRunner / SDA oscilloscopes with optical probes, Protocol
|
||||||
|
* Analyzers (Voyager, Catapult), BERT test instruments, and high-speed
|
||||||
|
* serial data analysis platforms.
|
||||||
|
*
|
||||||
|
* Sources:
|
||||||
|
* - Teledyne LeCroy WavePro HD optical probe module specs
|
||||||
|
* - Teledyne LeCroy Voyager M310P protocol analyzer hardware guide
|
||||||
|
* - Teledyne LeCroy QPHY-400G-SR8 test system optical interface
|
||||||
|
*
|
||||||
|
* Run: tsx packages/scraper/src/scrapers/teledyne-lecroy-oem.ts
|
||||||
|
* Cron: daily at 02:30
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { pool, ensureVendor } from "../utils/db";
|
||||||
|
|
||||||
|
interface TeledynePID {
|
||||||
|
pid: string;
|
||||||
|
formFactor: string;
|
||||||
|
speedGbps: number;
|
||||||
|
speed: string;
|
||||||
|
reachMeters: number;
|
||||||
|
reachLabel: string;
|
||||||
|
fiberType: string;
|
||||||
|
connector: string;
|
||||||
|
wavelengths?: string;
|
||||||
|
standard?: string;
|
||||||
|
notes?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const TELECOM_PIDS = new Set([
|
||||||
|
"TDY-SFP-10G-DWDM-TUNE",
|
||||||
|
"TDY-QSFPDD-400G-ZR",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const TDY_PIDS: TeledynePID[] = [
|
||||||
|
// ── 1G SFP ───────────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX", notes: "Teledyne LeCroy T&M 1G SX" },
|
||||||
|
{ pid: "TDY-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" },
|
||||||
|
|
||||||
|
// ── 10G SFP+ ─────────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-SFP-10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" },
|
||||||
|
{ pid: "TDY-SFP-10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" },
|
||||||
|
{ pid: "TDY-SFP-10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" },
|
||||||
|
{ pid: "TDY-SFP-10G-ZR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ZR" },
|
||||||
|
{ pid: "TDY-SFP-10G-DWDM-TUNE",formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 80000, reachLabel: "DWDM", fiberType: "SMF", connector: "LC", wavelengths: "C-band DWDM", notes: "Teledyne LeCroy tunable DWDM SFP+" },
|
||||||
|
|
||||||
|
// ── 25G SFP28 ────────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 100, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" },
|
||||||
|
{ pid: "TDY-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" },
|
||||||
|
|
||||||
|
// ── 40G QSFP+ ────────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-QSFP-40G-SR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4", notes: "Teledyne LeCroy protocol analyzer" },
|
||||||
|
{ pid: "TDY-QSFP-40G-LR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" },
|
||||||
|
|
||||||
|
// ── 100G QSFP28 ──────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-QSFP28-100G-SR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 100, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "100GBASE-SR4" },
|
||||||
|
{ pid: "TDY-QSFP28-100G-LR4", formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1295-1310nm", standard: "100GBASE-LR4" },
|
||||||
|
{ pid: "TDY-QSFP28-100G-CWDM4",formFactor: "QSFP28", speedGbps: 100, speed: "100G", reachMeters: 2000, reachLabel: "CWDM4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "100GBASE-CWDM4" },
|
||||||
|
|
||||||
|
// ── 400G QSFP-DD ─────────────────────────────────────────────────────
|
||||||
|
{ pid: "TDY-QSFPDD-400G-SR8", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 100, reachLabel: "SR8", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "400GBASE-SR8", notes: "Teledyne LeCroy QPHY-400G-SR8" },
|
||||||
|
{ pid: "TDY-QSFPDD-400G-DR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 500, reachLabel: "DR4", fiberType: "SMF", connector: "MPO", wavelengths: "1310nm", standard: "400GBASE-DR4" },
|
||||||
|
{ pid: "TDY-QSFPDD-400G-FR4", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 2000, reachLabel: "FR4", fiberType: "SMF", connector: "LC", wavelengths: "1271-1331nm", standard: "400GBASE-FR4" },
|
||||||
|
{ pid: "TDY-QSFPDD-400G-ZR", formFactor: "QSFP-DD",speedGbps: 400, speed: "400G", reachMeters: 120000, reachLabel: "ZR", fiberType: "SMF", connector: "LC", wavelengths: "C-band", standard: "400ZR", notes: "Teledyne LeCroy 400G coherent ZR" },
|
||||||
|
];
|
||||||
|
|
||||||
|
export async function scrapeTeledyneLeCroyOem(): Promise<void> {
|
||||||
|
console.log("=== Teledyne LeCroy OEM Transceiver Seed ===\n");
|
||||||
|
|
||||||
|
const vendorId = await ensureVendor(
|
||||||
|
"Teledyne LeCroy",
|
||||||
|
"oem",
|
||||||
|
"https://www.teledynelecroy.com",
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
let inserted = 0, updated = 0, errors = 0;
|
||||||
|
|
||||||
|
for (const p of TDY_PIDS) {
|
||||||
|
const slug = `teledyne-lecroy-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
|
||||||
|
const category = TELECOM_PIDS.has(p.pid) ? "Telecom" : "DataCenter";
|
||||||
|
try {
|
||||||
|
const res = await pool.query(
|
||||||
|
`INSERT INTO transceivers
|
||||||
|
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
|
||||||
|
reach_meters, reach_label, fiber_type, connector, wavelengths,
|
||||||
|
dom_support, ieee_reference, market_status, category, notes)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream',$13,$14)
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET
|
||||||
|
speed_gbps = EXCLUDED.speed_gbps,
|
||||||
|
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
|
||||||
|
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
|
||||||
|
wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) as was_inserted`,
|
||||||
|
[slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
|
||||||
|
p.reachMeters, p.reachLabel, p.fiberType, p.connector,
|
||||||
|
p.wavelengths ?? null, p.standard ?? null, category, p.notes ?? null]
|
||||||
|
);
|
||||||
|
if (res.rows[0]?.was_inserted) inserted++; else updated++;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n=== Teledyne LeCroy OEM Seed Complete ===`);
|
||||||
|
console.log(` Inserted: ${inserted}, Updated: ${updated}, Errors: ${errors}`);
|
||||||
|
console.log(` Total PIDs: ${TDY_PIDS.length}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
scrapeTeledyneLeCroyOem()
|
||||||
|
.then(() => pool.end())
|
||||||
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user