diff --git a/.gitignore b/.gitignore index 12768c5..04c1ac4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ ecosystem.config.js assets/images/ assets/datasheets/ assets/manuals/ + +# Crawlee runtime artifacts +storage/ +storage-fs/ +.crawlee/ diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 7e56bf2..22b8609 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -700,8 +700,8 @@ export async function registerWorkers(boss: PgBoss): Promise { console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set"); return; } - const { scrapeDigikey } = await import("./scrapers/digikey"); - await scrapeDigikey(); + const { scrapeMouser } = await import("./scrapers/mouser"); + await scrapeMouser(); }); // ── Health monitor ────────────────────────────────────────────────────── diff --git a/packages/scraper/src/scrapers/digikey.ts b/packages/scraper/src/scrapers/mouser.ts similarity index 97% rename from packages/scraper/src/scrapers/digikey.ts rename to packages/scraper/src/scrapers/mouser.ts index b5b6de4..2ca2079 100644 --- a/packages/scraper/src/scrapers/digikey.ts +++ b/packages/scraper/src/scrapers/mouser.ts @@ -10,9 +10,8 @@ * * Rate limit: 30 req/min on free tier → 2s delay between requests * - * Note: This file is intentionally named digikey.ts (task origin) but uses - * Mouser as the actual source since DigiKey + Arrow both require Playwright - * to bypass Cloudflare/Akamai. Mouser's free API returns the same data. + * Note: DigiKey + Arrow both require Playwright to bypass Cloudflare/Akamai. + * Mouser offers a free REST API with the same data — hence this implementation. */ import { pool, ensureVendor, upsertPriceObservation } from "../utils/db"; @@ -166,7 +165,7 @@ async function searchMouser(partNumber: string): Promise { // ── Main ────────────────────────────────────────────────────────────────────── -export async function scrapeDigikey(): Promise { +export async function scrapeMouser(): Promise { console.log("=== OEM Reference Price Scraper (Mouser Electronics API) ===\n"); if (!MOUSER_API_KEY) { @@ -282,7 +281,7 @@ export async function scrapeDigikey(): Promise { // ── CLI ─────────────────────────────────────────────────────────────────────── if (require.main === module) { - scrapeDigikey() + scrapeMouser() .then(() => pool.end()) .catch((err: unknown) => { console.error("Fatal:", err); diff --git a/packages/scraper/src/scrapers/switch-seed-smb.ts b/packages/scraper/src/scrapers/switch-seed-smb.ts deleted file mode 100644 index aa9e55e..0000000 --- a/packages/scraper/src/scrapers/switch-seed-smb.ts +++ /dev/null @@ -1,373 +0,0 @@ -/** - * SMB & Campus Switch Seed — Small/Medium Business switches - * - * Fills the gap between enterprise DataCenter switches and real-world SMB deployments. - * These are the switches Flexoptix customers actually ask "what transceiver fits?". - * - * Sources: Public datasheets, vendor product pages. - * Vendors: Cisco SG/SX/CBS, HP/HPE 1820/1920/2530, Ubiquiti UniFi, Netgear M4300, - * MikroTik CRS/CSS, TP-Link TL-SG, D-Link DGS, Zyxel XGS - */ -import { pool, ensureVendor } from "../utils/db"; - -interface SwitchSeed { - vendor: string; - vendorType: string; - vendorWebsite: string; - model: string; - series: string; - category: "Campus" | "Edge" | "Industrial" | "DataCenter"; - layer: "L2" | "L3" | "L2/L3"; - portsConfig: Record; - totalPorts: number; - maxSpeedGbps: number; - uplinkSpeedGbps?: number; - switchingCapacityTbps?: number; - rackUnits?: number; - maxPowerW?: number; - poeSupport?: string; - stackingSupport?: boolean; - bgpSupport?: boolean; - lifecycleStatus?: string; - tags?: string[]; -} - -// ═══════════════════════════════════════════════════════ -// CISCO SMALL BUSINESS (SG/SX/CBS Series) -// ═══════════════════════════════════════════════════════ -const CISCO_SMB: SwitchSeed[] = [ - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "SG350-28", series: "SG350", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 56, - stackingSupport: false, bgpSupport: false, - tags: ["smb", "sfp-uplink", "L3", "soho"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "SG350-28P", series: "SG350", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45_PoE": 24, "1G_SFP": 4 }, totalPorts: 28, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 195, - poeSupport: "PoE+", stackingSupport: false, bgpSupport: false, - tags: ["smb", "poe-plus", "sfp-uplink", "L3"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "SG350-52", series: "SG350", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.104, rackUnits: 1, maxPowerW: 88, - stackingSupport: false, bgpSupport: false, - tags: ["smb", "sfp-uplink", "L3", "48-port"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "SG550X-24MP", series: "SG550X", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45_PoE": 24, "10G_SFP+": 4 }, totalPorts: 28, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.128, rackUnits: 1, maxPowerW: 740, - poeSupport: "PoE+", stackingSupport: true, bgpSupport: false, - tags: ["smb", "10g-uplink", "sfp-plus", "stacking", "poe"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "SG550X-48MP", series: "SG550X", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45_PoE": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 1000, - poeSupport: "PoE+", stackingSupport: true, bgpSupport: false, - tags: ["smb", "10g-uplink", "sfp-plus", "stacking", "poe"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "CBS350-24T-4G", series: "CBS350", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 56, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-uplink", "current-gen", "CBS"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "CBS350-48T-4X", series: "CBS350", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 88, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "10g-uplink", "sfp-plus", "CBS", "current-gen"], - }, - { - vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com", - model: "CBS350-8MGP-2X", series: "CBS350", category: "Edge", layer: "L3", - portsConfig: { "1G_RJ45_PoE": 8, "10G_SFP+": 2 }, totalPorts: 10, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.04, maxPowerW: 240, - poeSupport: "PoE+ 2.5G", stackingSupport: false, lifecycleStatus: "Active", - tags: ["smb", "10g-uplink", "multi-gig", "CBS"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// HPE / ARUBA (1820/1920/2530/2540 Series) -// ═══════════════════════════════════════════════════════ -const HPE_ARUBA: SwitchSeed[] = [ - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL816A", series: "Aruba 1820-24G", category: "Campus", layer: "L2", - portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 19, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-uplink", "aruba", "1820-series"], - }, - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL817A", series: "Aruba 1820-48G", category: "Campus", layer: "L2", - portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.104, rackUnits: 1, maxPowerW: 36, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-uplink", "aruba", "1820-series", "48-port"], - }, - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL003A", series: "Aruba 2530-48G", category: "Campus", layer: "L2", - portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52, - maxSpeedGbps: 1, uplinkSpeedGbps: 1, - switchingCapacityTbps: 0.104, rackUnits: 1, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-uplink", "aruba", "2530-series"], - }, - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL356A", series: "Aruba 2540-48G-4SFP+", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "10g-uplink", "sfp-plus", "aruba", "2540-series"], - }, - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL322A", series: "Aruba 2930F-24G-4SFP+", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.128, rackUnits: 1, - stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active", - tags: ["campus", "10g-uplink", "sfp-plus", "stacking", "aruba"], - }, - { - vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com", - model: "JL481A", series: "Aruba 2930F-48G-4SFP+", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, - stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active", - tags: ["campus", "10g-uplink", "sfp-plus", "stacking", "aruba"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// UBIQUITI UNIFI -// ═══════════════════════════════════════════════════════ -const UBIQUITI: SwitchSeed[] = [ - { - vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com", - model: "USW-Pro-24", series: "UniFi USW-Pro", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "10G_SFP+": 2 }, totalPorts: 26, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.052, rackUnits: 1, maxPowerW: 32, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "unifi", "prosumer"], - }, - { - vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com", - model: "USW-Pro-48", series: "UniFi USW-Pro", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 50, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "unifi", "prosumer", "48-port"], - }, - { - vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com", - model: "USW-Pro-Aggregation", series: "UniFi USW-Pro-Agg", category: "Campus", layer: "L3", - portsConfig: { "10G_SFP+": 20, "25G_SFP28": 4 }, totalPorts: 28, - maxSpeedGbps: 25, uplinkSpeedGbps: 25, - switchingCapacityTbps: 0.456, rackUnits: 1, maxPowerW: 65, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["campus", "aggregation", "sfp28", "sfp-plus", "unifi"], - }, - { - vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com", - model: "USW-EnterpriseXG-24", series: "UniFi USW-EnterpriseXG", category: "Campus", layer: "L3", - portsConfig: { "10G_RJ45": 20, "25G_SFP28": 4 }, totalPorts: 24, - maxSpeedGbps: 25, uplinkSpeedGbps: 25, - switchingCapacityTbps: 0.456, rackUnits: 1, maxPowerW: 280, - poeSupport: "PoE++ 2.5G/5G/10G", stackingSupport: false, lifecycleStatus: "Active", - tags: ["campus", "multi-gig", "sfp28", "unifi", "10g-rj45"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// MIKROTIK (CRS/CCR Series) -// ═══════════════════════════════════════════════════════ -const MIKROTIK: SwitchSeed[] = [ - { - vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com", - model: "CRS326-24G-2S+RM", series: "CRS326", category: "Campus", layer: "L2/L3", - portsConfig: { "1G_RJ45": 24, "10G_SFP+": 2 }, totalPorts: 26, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.052, rackUnits: 1, maxPowerW: 52, - stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "mikrotik", "routeros"], - }, - { - vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com", - model: "CRS354-48G-4S+2Q+RM", series: "CRS354", category: "Campus", layer: "L2/L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4, "40G_QSFP+": 2 }, totalPorts: 54, - maxSpeedGbps: 40, uplinkSpeedGbps: 40, - switchingCapacityTbps: 0.336, rackUnits: 1, maxPowerW: 98, - stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active", - tags: ["campus", "sfp-plus", "qsfp", "mikrotik", "48-port"], - }, - { - vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com", - model: "CRS504-4XQ-IN", series: "CRS504", category: "Campus", layer: "L2/L3", - portsConfig: { "100G_QSFP28": 4 }, totalPorts: 4, - maxSpeedGbps: 100, uplinkSpeedGbps: 100, - switchingCapacityTbps: 0.8, maxPowerW: 60, - stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active", - tags: ["campus", "100g", "qsfp28", "mikrotik", "compact"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// NETGEAR (M4300/M4500 Series) -// ═══════════════════════════════════════════════════════ -const NETGEAR: SwitchSeed[] = [ - { - vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com", - model: "M4300-28G", series: "M4300", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.128, rackUnits: 1, - stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "netgear", "stacking"], - }, - { - vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com", - model: "M4300-52G", series: "M4300", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.176, rackUnits: 1, - stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "netgear", "stacking", "48-port"], - }, - { - vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com", - model: "M4500-32F", series: "M4500", category: "Campus", layer: "L3", - portsConfig: { "100G_QSFP28": 8, "25G_SFP28": 24 }, totalPorts: 32, - maxSpeedGbps: 100, uplinkSpeedGbps: 100, - switchingCapacityTbps: 3.2, rackUnits: 1, - stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active", - tags: ["campus", "100g", "25g", "sfp28", "qsfp28", "netgear"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// ZYXEL (XGS/XS Series) -// ═══════════════════════════════════════════════════════ -const ZYXEL: SwitchSeed[] = [ - { - vendor: "Zyxel", vendorType: "oem", vendorWebsite: "https://www.zyxel.com", - model: "XGS1930-28", series: "XGS1930", category: "Campus", layer: "L3", - portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.128, rackUnits: 1, - stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active", - tags: ["smb", "sfp-plus", "zyxel"], - }, - { - vendor: "Zyxel", vendorType: "oem", vendorWebsite: "https://www.zyxel.com", - model: "XGS2210-28HP", series: "XGS2210", category: "Campus", layer: "L2", - portsConfig: { "1G_RJ45_PoE": 24, "10G_SFP+": 4 }, totalPorts: 28, - maxSpeedGbps: 10, uplinkSpeedGbps: 10, - switchingCapacityTbps: 0.128, rackUnits: 1, maxPowerW: 400, - poeSupport: "PoE+", stackingSupport: false, lifecycleStatus: "Active", - tags: ["smb", "poe", "sfp-plus", "zyxel"], - }, -]; - -// ═══════════════════════════════════════════════════════ -// SEED RUNNER -// ═══════════════════════════════════════════════════════ - -async function upsertSwitch(seed: SwitchSeed): Promise { - const vendorId = await ensureVendor(seed.vendor, seed.vendorType as any, seed.vendorWebsite); - - await pool.query( - `INSERT INTO switches ( - vendor_id, model, series, category, layer, - ports_config, total_ports, max_speed_gbps, uplink_speed_gbps, - switching_capacity_tbps, rack_units, max_power_w, - poe_support, stacking_support, bgp_support, - lifecycle_status, tags - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) - ON CONFLICT (vendor_id, model) DO UPDATE SET - series = EXCLUDED.series, - category = EXCLUDED.category, - ports_config = EXCLUDED.ports_config, - total_ports = EXCLUDED.total_ports, - max_speed_gbps = EXCLUDED.max_speed_gbps, - uplink_speed_gbps = EXCLUDED.uplink_speed_gbps, - switching_capacity_tbps = EXCLUDED.switching_capacity_tbps, - poe_support = EXCLUDED.poe_support, - stacking_support = EXCLUDED.stacking_support, - bgp_support = EXCLUDED.bgp_support, - lifecycle_status = EXCLUDED.lifecycle_status, - tags = EXCLUDED.tags`, - [ - vendorId, - seed.model, - seed.series, - seed.category, - seed.layer, - JSON.stringify(seed.portsConfig), - seed.totalPorts, - seed.maxSpeedGbps, - seed.uplinkSpeedGbps ?? null, - seed.switchingCapacityTbps ?? null, - seed.rackUnits ?? null, - seed.maxPowerW ?? null, - seed.poeSupport ?? null, - seed.stackingSupport ?? false, - seed.bgpSupport ?? false, - seed.lifecycleStatus ?? "Active", - seed.tags ?? [], - ] - ); -} - -export async function seedSmbSwitches(): Promise { - const all = [...CISCO_SMB, ...HPE_ARUBA, ...UBIQUITI, ...MIKROTIK, ...NETGEAR, ...ZYXEL]; - console.log(`[smb-seed] Seeding ${all.length} SMB/campus switches...`); - let inserted = 0; - for (const s of all) { - try { - await upsertSwitch(s); - inserted++; - } catch (err) { - console.error(`[smb-seed] Error seeding ${s.model}:`, err); - } - } - console.log(`[smb-seed] Done: ${inserted}/${all.length} switches upserted.`); -} - -// Allow running directly -if (require.main === module) { - seedSmbSwitches().then(() => process.exit(0)).catch((e) => { console.error(e); process.exit(1); }); -}