chore: cleanup — rename digikey→mouser, remove orphan files, gitignore Crawlee artifacts

- Rename scrapers/digikey.ts → scrapers/mouser.ts: export scrapeMouser()
  (file was Mouser API implementation mislabeled from task origin)
- Fix scheduler.ts mouser-oem worker: import scrapeMouser from ./scrapers/mouser
- Delete switch-seed-smb.ts (unreferenced, no CLI flag, no scheduler job)
- Add storage/, storage-fs/, .crawlee/ to .gitignore (Crawlee runtime artifacts)
This commit is contained in:
Rene Fichtmueller 2026-04-18 01:09:10 +02:00
parent 1c8dec52c9
commit 2b770aa1a9
4 changed files with 11 additions and 380 deletions

5
.gitignore vendored
View File

@ -12,3 +12,8 @@ ecosystem.config.js
assets/images/ assets/images/
assets/datasheets/ assets/datasheets/
assets/manuals/ assets/manuals/
# Crawlee runtime artifacts
storage/
storage-fs/
.crawlee/

View File

@ -700,8 +700,8 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set"); console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set");
return; return;
} }
const { scrapeDigikey } = await import("./scrapers/digikey"); const { scrapeMouser } = await import("./scrapers/mouser");
await scrapeDigikey(); await scrapeMouser();
}); });
// ── Health monitor ────────────────────────────────────────────────────── // ── Health monitor ──────────────────────────────────────────────────────

View File

@ -10,9 +10,8 @@
* *
* Rate limit: 30 req/min on free tier 2s delay between requests * Rate limit: 30 req/min on free tier 2s delay between requests
* *
* Note: This file is intentionally named digikey.ts (task origin) but uses * Note: DigiKey + Arrow both require Playwright to bypass Cloudflare/Akamai.
* Mouser as the actual source since DigiKey + Arrow both require Playwright * Mouser offers a free REST API with the same data hence this implementation.
* to bypass Cloudflare/Akamai. Mouser's free API returns the same data.
*/ */
import { pool, ensureVendor, upsertPriceObservation } from "../utils/db"; import { pool, ensureVendor, upsertPriceObservation } from "../utils/db";
@ -166,7 +165,7 @@ async function searchMouser(partNumber: string): Promise<MouserPart | null> {
// ── Main ────────────────────────────────────────────────────────────────────── // ── Main ──────────────────────────────────────────────────────────────────────
export async function scrapeDigikey(): Promise<void> { export async function scrapeMouser(): Promise<void> {
console.log("=== OEM Reference Price Scraper (Mouser Electronics API) ===\n"); console.log("=== OEM Reference Price Scraper (Mouser Electronics API) ===\n");
if (!MOUSER_API_KEY) { if (!MOUSER_API_KEY) {
@ -282,7 +281,7 @@ export async function scrapeDigikey(): Promise<void> {
// ── CLI ─────────────────────────────────────────────────────────────────────── // ── CLI ───────────────────────────────────────────────────────────────────────
if (require.main === module) { if (require.main === module) {
scrapeDigikey() scrapeMouser()
.then(() => pool.end()) .then(() => pool.end())
.catch((err: unknown) => { .catch((err: unknown) => {
console.error("Fatal:", err); console.error("Fatal:", err);

View File

@ -1,373 +0,0 @@
/**
* SMB & Campus Switch Seed Small/Medium Business switches
*
* Fills the gap between enterprise DataCenter switches and real-world SMB deployments.
* These are the switches Flexoptix customers actually ask "what transceiver fits?".
*
* Sources: Public datasheets, vendor product pages.
* Vendors: Cisco SG/SX/CBS, HP/HPE 1820/1920/2530, Ubiquiti UniFi, Netgear M4300,
* MikroTik CRS/CSS, TP-Link TL-SG, D-Link DGS, Zyxel XGS
*/
import { pool, ensureVendor } from "../utils/db";
interface SwitchSeed {
vendor: string;
vendorType: string;
vendorWebsite: string;
model: string;
series: string;
category: "Campus" | "Edge" | "Industrial" | "DataCenter";
layer: "L2" | "L3" | "L2/L3";
portsConfig: Record<string, number>;
totalPorts: number;
maxSpeedGbps: number;
uplinkSpeedGbps?: number;
switchingCapacityTbps?: number;
rackUnits?: number;
maxPowerW?: number;
poeSupport?: string;
stackingSupport?: boolean;
bgpSupport?: boolean;
lifecycleStatus?: string;
tags?: string[];
}
// ═══════════════════════════════════════════════════════
// CISCO SMALL BUSINESS (SG/SX/CBS Series)
// ═══════════════════════════════════════════════════════
const CISCO_SMB: SwitchSeed[] = [
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "SG350-28", series: "SG350", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 56,
stackingSupport: false, bgpSupport: false,
tags: ["smb", "sfp-uplink", "L3", "soho"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "SG350-28P", series: "SG350", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45_PoE": 24, "1G_SFP": 4 }, totalPorts: 28,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 195,
poeSupport: "PoE+", stackingSupport: false, bgpSupport: false,
tags: ["smb", "poe-plus", "sfp-uplink", "L3"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "SG350-52", series: "SG350", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.104, rackUnits: 1, maxPowerW: 88,
stackingSupport: false, bgpSupport: false,
tags: ["smb", "sfp-uplink", "L3", "48-port"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "SG550X-24MP", series: "SG550X", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45_PoE": 24, "10G_SFP+": 4 }, totalPorts: 28,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.128, rackUnits: 1, maxPowerW: 740,
poeSupport: "PoE+", stackingSupport: true, bgpSupport: false,
tags: ["smb", "10g-uplink", "sfp-plus", "stacking", "poe"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "SG550X-48MP", series: "SG550X", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45_PoE": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 1000,
poeSupport: "PoE+", stackingSupport: true, bgpSupport: false,
tags: ["smb", "10g-uplink", "sfp-plus", "stacking", "poe"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "CBS350-24T-4G", series: "CBS350", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 56,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-uplink", "current-gen", "CBS"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "CBS350-48T-4X", series: "CBS350", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 88,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "10g-uplink", "sfp-plus", "CBS", "current-gen"],
},
{
vendor: "Cisco Systems", vendorType: "oem", vendorWebsite: "https://www.cisco.com",
model: "CBS350-8MGP-2X", series: "CBS350", category: "Edge", layer: "L3",
portsConfig: { "1G_RJ45_PoE": 8, "10G_SFP+": 2 }, totalPorts: 10,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.04, maxPowerW: 240,
poeSupport: "PoE+ 2.5G", stackingSupport: false, lifecycleStatus: "Active",
tags: ["smb", "10g-uplink", "multi-gig", "CBS"],
},
];
// ═══════════════════════════════════════════════════════
// HPE / ARUBA (1820/1920/2530/2540 Series)
// ═══════════════════════════════════════════════════════
const HPE_ARUBA: SwitchSeed[] = [
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL816A", series: "Aruba 1820-24G", category: "Campus", layer: "L2",
portsConfig: { "1G_RJ45": 24, "1G_SFP": 4 }, totalPorts: 28,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.056, rackUnits: 1, maxPowerW: 19,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-uplink", "aruba", "1820-series"],
},
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL817A", series: "Aruba 1820-48G", category: "Campus", layer: "L2",
portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.104, rackUnits: 1, maxPowerW: 36,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-uplink", "aruba", "1820-series", "48-port"],
},
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL003A", series: "Aruba 2530-48G", category: "Campus", layer: "L2",
portsConfig: { "1G_RJ45": 48, "1G_SFP": 4 }, totalPorts: 52,
maxSpeedGbps: 1, uplinkSpeedGbps: 1,
switchingCapacityTbps: 0.104, rackUnits: 1,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-uplink", "aruba", "2530-series"],
},
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL356A", series: "Aruba 2540-48G-4SFP+", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "10g-uplink", "sfp-plus", "aruba", "2540-series"],
},
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL322A", series: "Aruba 2930F-24G-4SFP+", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.128, rackUnits: 1,
stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active",
tags: ["campus", "10g-uplink", "sfp-plus", "stacking", "aruba"],
},
{
vendor: "HPE / Aruba", vendorType: "oem", vendorWebsite: "https://www.arubanetworks.com",
model: "JL481A", series: "Aruba 2930F-48G-4SFP+", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1,
stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active",
tags: ["campus", "10g-uplink", "sfp-plus", "stacking", "aruba"],
},
];
// ═══════════════════════════════════════════════════════
// UBIQUITI UNIFI
// ═══════════════════════════════════════════════════════
const UBIQUITI: SwitchSeed[] = [
{
vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com",
model: "USW-Pro-24", series: "UniFi USW-Pro", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "10G_SFP+": 2 }, totalPorts: 26,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.052, rackUnits: 1, maxPowerW: 32,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "unifi", "prosumer"],
},
{
vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com",
model: "USW-Pro-48", series: "UniFi USW-Pro", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1, maxPowerW: 50,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "unifi", "prosumer", "48-port"],
},
{
vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com",
model: "USW-Pro-Aggregation", series: "UniFi USW-Pro-Agg", category: "Campus", layer: "L3",
portsConfig: { "10G_SFP+": 20, "25G_SFP28": 4 }, totalPorts: 28,
maxSpeedGbps: 25, uplinkSpeedGbps: 25,
switchingCapacityTbps: 0.456, rackUnits: 1, maxPowerW: 65,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["campus", "aggregation", "sfp28", "sfp-plus", "unifi"],
},
{
vendor: "Ubiquiti", vendorType: "oem", vendorWebsite: "https://www.ui.com",
model: "USW-EnterpriseXG-24", series: "UniFi USW-EnterpriseXG", category: "Campus", layer: "L3",
portsConfig: { "10G_RJ45": 20, "25G_SFP28": 4 }, totalPorts: 24,
maxSpeedGbps: 25, uplinkSpeedGbps: 25,
switchingCapacityTbps: 0.456, rackUnits: 1, maxPowerW: 280,
poeSupport: "PoE++ 2.5G/5G/10G", stackingSupport: false, lifecycleStatus: "Active",
tags: ["campus", "multi-gig", "sfp28", "unifi", "10g-rj45"],
},
];
// ═══════════════════════════════════════════════════════
// MIKROTIK (CRS/CCR Series)
// ═══════════════════════════════════════════════════════
const MIKROTIK: SwitchSeed[] = [
{
vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com",
model: "CRS326-24G-2S+RM", series: "CRS326", category: "Campus", layer: "L2/L3",
portsConfig: { "1G_RJ45": 24, "10G_SFP+": 2 }, totalPorts: 26,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.052, rackUnits: 1, maxPowerW: 52,
stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "mikrotik", "routeros"],
},
{
vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com",
model: "CRS354-48G-4S+2Q+RM", series: "CRS354", category: "Campus", layer: "L2/L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4, "40G_QSFP+": 2 }, totalPorts: 54,
maxSpeedGbps: 40, uplinkSpeedGbps: 40,
switchingCapacityTbps: 0.336, rackUnits: 1, maxPowerW: 98,
stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active",
tags: ["campus", "sfp-plus", "qsfp", "mikrotik", "48-port"],
},
{
vendor: "MikroTik", vendorType: "oem", vendorWebsite: "https://mikrotik.com",
model: "CRS504-4XQ-IN", series: "CRS504", category: "Campus", layer: "L2/L3",
portsConfig: { "100G_QSFP28": 4 }, totalPorts: 4,
maxSpeedGbps: 100, uplinkSpeedGbps: 100,
switchingCapacityTbps: 0.8, maxPowerW: 60,
stackingSupport: false, bgpSupport: true, lifecycleStatus: "Active",
tags: ["campus", "100g", "qsfp28", "mikrotik", "compact"],
},
];
// ═══════════════════════════════════════════════════════
// NETGEAR (M4300/M4500 Series)
// ═══════════════════════════════════════════════════════
const NETGEAR: SwitchSeed[] = [
{
vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com",
model: "M4300-28G", series: "M4300", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.128, rackUnits: 1,
stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "netgear", "stacking"],
},
{
vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com",
model: "M4300-52G", series: "M4300", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 48, "10G_SFP+": 4 }, totalPorts: 52,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.176, rackUnits: 1,
stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "netgear", "stacking", "48-port"],
},
{
vendor: "Netgear", vendorType: "oem", vendorWebsite: "https://www.netgear.com",
model: "M4500-32F", series: "M4500", category: "Campus", layer: "L3",
portsConfig: { "100G_QSFP28": 8, "25G_SFP28": 24 }, totalPorts: 32,
maxSpeedGbps: 100, uplinkSpeedGbps: 100,
switchingCapacityTbps: 3.2, rackUnits: 1,
stackingSupport: true, bgpSupport: true, lifecycleStatus: "Active",
tags: ["campus", "100g", "25g", "sfp28", "qsfp28", "netgear"],
},
];
// ═══════════════════════════════════════════════════════
// ZYXEL (XGS/XS Series)
// ═══════════════════════════════════════════════════════
const ZYXEL: SwitchSeed[] = [
{
vendor: "Zyxel", vendorType: "oem", vendorWebsite: "https://www.zyxel.com",
model: "XGS1930-28", series: "XGS1930", category: "Campus", layer: "L3",
portsConfig: { "1G_RJ45": 24, "10G_SFP+": 4 }, totalPorts: 28,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.128, rackUnits: 1,
stackingSupport: false, bgpSupport: false, lifecycleStatus: "Active",
tags: ["smb", "sfp-plus", "zyxel"],
},
{
vendor: "Zyxel", vendorType: "oem", vendorWebsite: "https://www.zyxel.com",
model: "XGS2210-28HP", series: "XGS2210", category: "Campus", layer: "L2",
portsConfig: { "1G_RJ45_PoE": 24, "10G_SFP+": 4 }, totalPorts: 28,
maxSpeedGbps: 10, uplinkSpeedGbps: 10,
switchingCapacityTbps: 0.128, rackUnits: 1, maxPowerW: 400,
poeSupport: "PoE+", stackingSupport: false, lifecycleStatus: "Active",
tags: ["smb", "poe", "sfp-plus", "zyxel"],
},
];
// ═══════════════════════════════════════════════════════
// SEED RUNNER
// ═══════════════════════════════════════════════════════
async function upsertSwitch(seed: SwitchSeed): Promise<void> {
const vendorId = await ensureVendor(seed.vendor, seed.vendorType as any, seed.vendorWebsite);
await pool.query(
`INSERT INTO switches (
vendor_id, model, series, category, layer,
ports_config, total_ports, max_speed_gbps, uplink_speed_gbps,
switching_capacity_tbps, rack_units, max_power_w,
poe_support, stacking_support, bgp_support,
lifecycle_status, tags
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
ON CONFLICT (vendor_id, model) DO UPDATE SET
series = EXCLUDED.series,
category = EXCLUDED.category,
ports_config = EXCLUDED.ports_config,
total_ports = EXCLUDED.total_ports,
max_speed_gbps = EXCLUDED.max_speed_gbps,
uplink_speed_gbps = EXCLUDED.uplink_speed_gbps,
switching_capacity_tbps = EXCLUDED.switching_capacity_tbps,
poe_support = EXCLUDED.poe_support,
stacking_support = EXCLUDED.stacking_support,
bgp_support = EXCLUDED.bgp_support,
lifecycle_status = EXCLUDED.lifecycle_status,
tags = EXCLUDED.tags`,
[
vendorId,
seed.model,
seed.series,
seed.category,
seed.layer,
JSON.stringify(seed.portsConfig),
seed.totalPorts,
seed.maxSpeedGbps,
seed.uplinkSpeedGbps ?? null,
seed.switchingCapacityTbps ?? null,
seed.rackUnits ?? null,
seed.maxPowerW ?? null,
seed.poeSupport ?? null,
seed.stackingSupport ?? false,
seed.bgpSupport ?? false,
seed.lifecycleStatus ?? "Active",
seed.tags ?? [],
]
);
}
export async function seedSmbSwitches(): Promise<void> {
const all = [...CISCO_SMB, ...HPE_ARUBA, ...UBIQUITI, ...MIKROTIK, ...NETGEAR, ...ZYXEL];
console.log(`[smb-seed] Seeding ${all.length} SMB/campus switches...`);
let inserted = 0;
for (const s of all) {
try {
await upsertSwitch(s);
inserted++;
} catch (err) {
console.error(`[smb-seed] Error seeding ${s.model}:`, err);
}
}
console.log(`[smb-seed] Done: ${inserted}/${all.length} switches upserted.`);
}
// Allow running directly
if (require.main === module) {
seedSmbSwitches().then(() => process.exit(0)).catch((e) => { console.error(e); process.exit(1); });
}