/** * Switch Product Assets Scraper — Images, Datasheets, Manuals * * Scrapes product pages from all major switch vendors to collect: * - Product images (hero shots) * - Datasheet PDFs * - Installation/Configuration guides * - CLI references * - Quick start guides * * Usage: * tsx src/index.ts --switch-assets — Scrape all vendor assets * tsx src/index.ts --switch-assets --vendor cisco — Scrape single vendor */ import { pool } from "../utils/db"; import { downloadSwitchImage, downloadSwitchDatasheet, downloadSwitchManual, setSwitchProductPage, setVendorDocUrls, } from "../utils/assets"; // ═══════════════════════════════════════════════════════ // Vendor-specific URL patterns for product pages, images, datasheets // ═══════════════════════════════════════════════════════ interface VendorAssetConfig { vendorName: string; docsPortal: string; datasheetLibrary: string; supportPortal: string; imageCdn?: string; /** Map model patterns → { productPage, imageUrl, datasheetUrl, manualUrls } */ modelAssets: ModelAssetMap[]; } interface ModelAssetMap { modelPattern: RegExp; productPage: (model: string) => string; imageUrl: (model: string) => string | null; datasheetUrl: (model: string) => string | null; manualUrls?: (model: string) => Array<{ url: string; title: string; type: string }>; } // ═══════════════════════════════════════════════════════ // CISCO // ═══════════════════════════════════════════════════════ const CISCO_CONFIG: VendorAssetConfig = { vendorName: "Cisco Systems", docsPortal: "https://www.cisco.com/c/en/us/support/index.html", datasheetLibrary: "https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/datasheet-listing.html", supportPortal: "https://www.cisco.com/c/en/us/support/index.html", imageCdn: "https://www.cisco.com/c/dam/en/us/products/collateral/switches/", modelAssets: [ { // Nexus 9500 modular modelPattern: /^N9K-C95/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`, imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-729404.docx/_jcr_content/renditions/datasheet-c78-729404_0.jpg`, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-729404.pdf`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-installation-and-configuration-guides-list.html`, title: "Nexus 9000 Configuration Guides", type: "manual" }, ], }, { // Nexus 9300-FX/FX2/FX3 fixed modelPattern: /^N9K-C93\d{2,4}.*FX/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`, imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-744052.docx/_jcr_content/renditions/datasheet-c78-744052_0.png`, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-744052.pdf`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/interfaces/cisco-nexus-9000-series-nx-os-interfaces-configuration-guide-104x.html`, title: "NX-OS Interfaces Configuration Guide", type: "manual" }, { url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/fundamentals/cisco-nexus-9000-series-nx-os-fundamentals-configuration-guide-104x.html`, title: "NX-OS Fundamentals Configuration Guide", type: "manual" }, ], }, { // Nexus 9300-GX/GX2 (newer) modelPattern: /^N9K-C93\d{2,4}.*GX/, productPage: () => `https://www.cisco.com/site/us/en/products/networking/cloud-networking-switches/nexus-9000-switches/index.html`, imageUrl: () => `https://www.cisco.com/content/dam/cisco-cdc/site/us/en/images/networking/nexus9000-switching-nx9364gx2a-530x280.png`, datasheetUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/networking/switches/nexus-9000-series-switches/nexus-9300-gx2-series-fixed-switches-ds.pdf`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-installation-and-configuration-guides-list.html`, title: "Nexus 9000 Configuration Guides", type: "manual" }, ], }, { // All other Nexus 9000 modelPattern: /^N9K-|^N3K-/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`, imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-736967.docx/_jcr_content/renditions/datasheet-c78-736967_0.jpg`, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-736967.pdf`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/interfaces/cisco-nexus-9000-series-nx-os-interfaces-configuration-guide-104x.html`, title: "NX-OS Interfaces Configuration Guide", type: "manual" }, ], }, { // Catalyst 9300 modelPattern: /^C93/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9300-series-switches/index.html`, imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/catalyst-9300-series-switches/nb-06-cat9300-ser-data-sheet-cte-en.docx/_jcr_content/renditions/nb-06-cat9300-ser-data-sheet-cte-en_0.png`, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9300-series-switches/nb-06-cat9300-ser-data-sheet-cte-en.html`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9300-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9300 Configuration Guides", type: "manual" }, ], }, { // Catalyst 9200 modelPattern: /^C92/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9200-series-switches/index.html`, imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/catalyst-9200-series-switches/nb-06-cat9200-ser-data-sheet-cte-en.docx/_jcr_content/renditions/nb-06-cat9200-ser-data-sheet-cte-en_0.png`, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9200-series-switches/nb-06-cat9200-ser-data-sheet-cte-en.html`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9200-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9200 Configuration Guides", type: "manual" }, ], }, { // Catalyst 9400 modelPattern: /^C94/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9400-series-switches/index.html`, imageUrl: () => null, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9400-series-switches/nb-06-cat9400-ser-data-sheet-cte-en.html`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9400-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9400 Configuration Guides", type: "manual" }, ], }, { // Catalyst 9500 modelPattern: /^C95/, productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9500-series-switches/index.html`, imageUrl: () => null, datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9500-series-switches/nb-06-cat9500-ser-data-sheet-cte-en.html`, manualUrls: () => [ { url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9500-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9500 Configuration Guides", type: "manual" }, ], }, { // NCS / Cisco 8000 modelPattern: /^NCS-|^8[01]\d{2}/, productPage: (m) => `https://www.cisco.com/c/en/us/products/routers/network-convergence-system-5500-series/index.html`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // ARISTA // ═══════════════════════════════════════════════════════ const ARISTA_CONFIG: VendorAssetConfig = { vendorName: "Arista Networks", docsPortal: "https://www.arista.com/en/support/product-documentation", datasheetLibrary: "https://www.arista.com/en/products", supportPortal: "https://www.arista.com/en/support/customer-support", imageCdn: "https://www.arista.com/assets/images/", modelAssets: [ { // Arista 7000 series — verified URL patterns // Datasheets: /assets/data/pdf/Datasheets/{SeriesName}-Datasheet.pdf (naming inconsistent) // Images: /assets/images/product/{series}-stack-200w200h.png or /assets/images/article/{model}-Right.png // QSGs: /assets/data/pdf/qsg/qsg-books/QS_{series}_{formfactor}_{gen}.pdf modelPattern: /^7[0-9]/, productPage: (m) => { const series = m.match(/^(7\d{3,4}[A-Z]*\d*)/)?.[1] || m; return `https://www.arista.com/en/products/${series.toLowerCase().replace(/[^a-z0-9]/g, "")}-series`; }, imageUrl: (m) => { // Try article-level images (higher res): {model}-Right.png const model = m.match(/^(\d{4}[A-Z]+\d*)/)?.[1] || m.split("-")[0]; return `https://www.arista.com/assets/images/article/${model}-Right.png`; }, datasheetUrl: (m) => { // Arista naming: {series}-Datasheet.pdf or {series}-Data-Sheet.pdf const series = m.match(/^(7\d{3,4}[A-Z]*\d*)/)?.[1]; return series ? `https://www.arista.com/assets/data/pdf/Datasheets/${series}-Datasheet.pdf` : null; }, manualUrls: (m) => { const series = m.match(/^(7\d{3,4})/)?.[1] || ""; return [ { url: `https://www.arista.com/assets/data/pdf/qsg/qsg-books/QS_${series}_1RU_Gen3.pdf`, title: `${m} Quick Start Guide`, type: "quick_start" }, ]; }, }, { // Arista 5000 (Campus) modelPattern: /^5[0-9]/, productPage: () => `https://www.arista.com/en/products/arista-5000-series`, imageUrl: () => null, datasheetUrl: () => `https://www.arista.com/assets/data/pdf/Datasheets/Arista-5000-Datasheet.pdf`, manualUrls: () => [], }, ], }; // ═══════════════════════════════════════════════════════ // JUNIPER // ═══════════════════════════════════════════════════════ const JUNIPER_CONFIG: VendorAssetConfig = { vendorName: "Juniper Networks", docsPortal: "https://www.juniper.net/documentation/", datasheetLibrary: "https://www.juniper.net/us/en/products.html", supportPortal: "https://supportportal.juniper.net/", imageCdn: "https://www.juniper.net/content/dam/www/assets/images/", modelAssets: [ { // Juniper switches — verified URL patterns // Datasheets: /content/dam/www/assets/datasheets/us/en/switches/{name}-datasheet.pdf // Images: /content/dam/www/assets/images/us/en/products/switches/{series}/{filename}.png // HW guides: /documentation/us/en/hardware/{model}/{model}.pdf // QSGs: /documentation/us/en/quick-start/hardware/qsg/{model}/{model}.pdf modelPattern: /^QFX|^EX|^MX|^PTX|^SRX|^ACX/, productPage: (m) => { const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || ""; const seriesNum = m.match(/^[A-Z]+(\d+)/)?.[1] || ""; return `https://www.juniper.net/us/en/products/switches/${seriesPrefix}-series/${seriesPrefix}${seriesNum}.html`; }, imageUrl: (m) => { const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || ""; const category = (seriesPrefix === "mx" || seriesPrefix === "ptx" || seriesPrefix === "acx") ? "routers" : "switches"; return `https://www.juniper.net/content/dam/www/assets/images/us/en/products/${category}/${seriesPrefix}-series/${seriesPrefix}-family-21DEC20.png`; }, datasheetUrl: (m) => { const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || ""; const category = (seriesPrefix === "mx" || seriesPrefix === "ptx" || seriesPrefix === "acx") ? "routers" : "switches"; // Juniper datasheet naming: {full-series-name}-datasheet.pdf or {product-description}-datasheet.pdf const model = m.match(/^([A-Z]+\d+)/)?.[1]?.toLowerCase() || ""; return `https://www.juniper.net/content/dam/www/assets/datasheets/us/en/${category}/${model}-ethernet-switch-datasheet.pdf`; }, manualUrls: (m) => { const model = m.toLowerCase().replace(/\s+/g, ""); return [ { url: `https://www.juniper.net/documentation/us/en/hardware/${model}/${model}.pdf`, title: `${m} Hardware Guide`, type: "manual" }, { url: `https://www.juniper.net/documentation/us/en/quick-start/hardware/qsg/${model}/${model}.pdf`, title: `${m} Quick Start Guide`, type: "quick_start" }, ]; }, }, ], }; // ═══════════════════════════════════════════════════════ // NOKIA // ═══════════════════════════════════════════════════════ const NOKIA_CONFIG: VendorAssetConfig = { vendorName: "Nokia", docsPortal: "https://documentation.nokia.com/", datasheetLibrary: "https://www.nokia.com/networks/products/", supportPortal: "https://customer.nokia.com/support/s/", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://www.nokia.com/networks/products/${m.toLowerCase().replace(/\s+/g, "-")}/`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // FORTINET // ═══════════════════════════════════════════════════════ const FORTINET_CONFIG: VendorAssetConfig = { vendorName: "Fortinet", docsPortal: "https://docs.fortinet.com/", datasheetLibrary: "https://www.fortinet.com/products/switches", supportPortal: "https://support.fortinet.com/", modelAssets: [ { // FortiSwitch product pages — use CheerioCrawler for image extraction // Datasheets at /content/dam/fortinet/assets/data-sheets/ modelPattern: /^FortiSwitch/, productPage: (m) => { const num = m.match(/(\d+[A-Z]*)/)?.[1]?.toLowerCase() || ""; return `https://www.fortinet.com/products/switches/fortiswitch-${num}`; }, imageUrl: () => null, // extracted from product page by crawler datasheetUrl: (m) => { // Fortinet DAM paths: fortiswitch-{series}00-series.pdf or FortiSwitch-{model}.pdf const series = m.match(/FortiSwitch[- ]*(\d)/)?.[1]; if (series) return `https://www.fortinet.com/content/dam/fortinet/assets/data-sheets/fortiswitch-${series}00-series.pdf`; return null; }, manualUrls: () => [ { url: `https://docs.fortinet.com/document/fortiswitch/latest/administration-guide`, title: "FortiSwitch Administration Guide", type: "manual" }, { url: `https://docs.fortinet.com/document/fortiswitch/latest/hardware-guide`, title: "FortiSwitch Hardware Guide", type: "installation_guide" }, ], }, ], }; // ═══════════════════════════════════════════════════════ // MIKROTIK // ═══════════════════════════════════════════════════════ const MIKROTIK_CONFIG: VendorAssetConfig = { vendorName: "MikroTik", docsPortal: "https://help.mikrotik.com/docs/", datasheetLibrary: "https://mikrotik.com/products", supportPortal: "https://help.mikrotik.com/", modelAssets: [ { // MikroTik uses underscored model names in URLs // Images & PDFs use unpredictable numeric IDs — must scrape product page // CheerioCrawler handles this via parseMikroTikPage() modelPattern: /^CRS|^CCR/, productPage: (m) => `https://mikrotik.com/product/${m.replace(/\s+/g, "_")}`, imageUrl: () => null, // extracted from product page by crawler datasheetUrl: () => null, // extracted from product page by crawler manualUrls: (m) => [ { url: `https://help.mikrotik.com/docs/display/UM/${m.replace(/\s+/g, "+")}`, title: `${m} User Manual`, type: "manual" }, ], }, ], }; // ═══════════════════════════════════════════════════════ // HIRSCHMANN // ═══════════════════════════════════════════════════════ const HIRSCHMANN_CONFIG: VendorAssetConfig = { vendorName: "Hirschmann", docsPortal: "https://catalog.belden.com/index.cfm", datasheetLibrary: "https://catalog.belden.com/index.cfm", supportPortal: "https://www.belden.com/support", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://catalog.belden.com/techdata/en/${m.replace(/\s+/g, "_")}_en.html`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // Additional vendor stubs (HPE, Dell, Extreme, Huawei, etc.) // ═══════════════════════════════════════════════════════ const HPE_CONFIG: VendorAssetConfig = { vendorName: "HPE / Aruba", docsPortal: "https://www.arubanetworks.com/techdocs/", datasheetLibrary: "https://www.arubanetworks.com/products/switches/", supportPortal: "https://asp.arubanetworks.com/", modelAssets: [ { modelPattern: /^CX/, productPage: (m) => `https://www.arubanetworks.com/products/switches/cx-${m.match(/CX\s*(\d+)/)?.[1] || ""}-series/`, imageUrl: (m) => `https://www.arubanetworks.com/wp-content/uploads/aruba-cx-${m.match(/CX\s*(\d+)/)?.[1] || ""}.png`, datasheetUrl: (m) => `https://www.arubanetworks.com/assets/ds/DS_CX${m.match(/CX\s*(\d+)/)?.[1] || ""}.pdf`, manualUrls: (m) => [ { url: `https://www.arubanetworks.com/techdocs/CX/`, title: `Aruba CX ${m} Configuration Guide`, type: "manual" }, ], }, ], }; const DELL_CONFIG: VendorAssetConfig = { vendorName: "Dell Technologies", docsPortal: "https://www.dell.com/support/", datasheetLibrary: "https://www.dell.com/en-us/lp/networking", supportPortal: "https://www.dell.com/support/home/", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://www.dell.com/en-us/shop/networking/cp/${m.toLowerCase().replace(/[^a-z0-9]/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; const EXTREME_CONFIG: VendorAssetConfig = { vendorName: "Extreme Networks", docsPortal: "https://extremeportal.force.com/ExtrArticleDetail", datasheetLibrary: "https://www.extremenetworks.com/products/", supportPortal: "https://extremeportal.force.com/", modelAssets: [ { modelPattern: /^X|^SLX|^VDX|^5[0-9]/, productPage: (m) => `https://www.extremenetworks.com/product/${m.toLowerCase().replace(/[^a-z0-9]/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; const MOXA_CONFIG: VendorAssetConfig = { vendorName: "Moxa", docsPortal: "https://www.moxa.com/en/support/support", datasheetLibrary: "https://www.moxa.com/en/products/industrial-network-infrastructure", supportPortal: "https://www.moxa.com/en/support/", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://www.moxa.com/en/products/industrial-network-infrastructure/${m.toLowerCase()}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; const SIEMENS_CONFIG: VendorAssetConfig = { vendorName: "Siemens", docsPortal: "https://support.industry.siemens.com/", datasheetLibrary: "https://mall.industry.siemens.com/", supportPortal: "https://support.industry.siemens.com/", modelAssets: [ { modelPattern: /^SCALANCE/, productPage: (m) => `https://www.siemens.com/global/en/products/automation/industrial-communication/industrial-ethernet/${m.toLowerCase().replace(/\s+/g, "-")}.html`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // D-LINK // ═══════════════════════════════════════════════════════ const DLINK_CONFIG: VendorAssetConfig = { vendorName: "D-Link", docsPortal: "https://www.dlink.com/en/support", datasheetLibrary: "https://www.dlink.com/en/products/switches", supportPortal: "https://www.dlink.com/en/support", modelAssets: [ { modelPattern: /^D[GMXW]S/, productPage: (m) => `https://www.dlink.com/en/products/${m.toLowerCase()}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // ALCATEL-LUCENT ENTERPRISE // ═══════════════════════════════════════════════════════ const ALE_CONFIG: VendorAssetConfig = { vendorName: "Alcatel-Lucent Enterprise", docsPortal: "https://www.al-enterprise.com/en-us/documentation", datasheetLibrary: "https://www.al-enterprise.com/en-us/products/switches", supportPortal: "https://myportal.al-enterprise.com/", modelAssets: [ { modelPattern: /^OmniSwitch/, productPage: (m) => { const num = m.match(/(\d{4})/)?.[1] || ""; return `https://www.al-enterprise.com/en-us/products/switches/omniswitch-${num}`; }, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // BROCADE / BROADCOM // ═══════════════════════════════════════════════════════ const BROCADE_CONFIG: VendorAssetConfig = { vendorName: "Brocade", docsPortal: "https://www.broadcom.com/support/fibre-channel-networking", datasheetLibrary: "https://www.broadcom.com/products/fibre-channel-networking", supportPortal: "https://www.broadcom.com/support", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://www.broadcom.com/products/fibre-channel-networking/switches/${m.toLowerCase().replace(/\s+/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // CIENA // ═══════════════════════════════════════════════════════ const CIENA_CONFIG: VendorAssetConfig = { vendorName: "Ciena", docsPortal: "https://www.ciena.com/insights/resources", datasheetLibrary: "https://www.ciena.com/products", supportPortal: "https://www.ciena.com/support", modelAssets: [ { modelPattern: /./, productPage: (m) => `https://www.ciena.com/products/${m.toLowerCase().replace(/\s+/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // H3C // ═══════════════════════════════════════════════════════ const H3C_CONFIG: VendorAssetConfig = { vendorName: "H3C", docsPortal: "https://www.h3c.com/en/Support/Resource_Center/", datasheetLibrary: "https://www.h3c.com/en/Products_And_Solution/Networking/", supportPortal: "https://www.h3c.com/en/Support/", modelAssets: [ { modelPattern: /^S/, productPage: (m) => `https://www.h3c.com/en/Products_And_Solution/Networking/Switches/${m}/`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // GIGAMON // ═══════════════════════════════════════════════════════ const GIGAMON_CONFIG: VendorAssetConfig = { vendorName: "Gigamon", docsPortal: "https://docs.gigamon.com/", datasheetLibrary: "https://www.gigamon.com/products", supportPortal: "https://community.gigamon.com/", modelAssets: [ { modelPattern: /^GigaVUE/, productPage: (m) => `https://www.gigamon.com/products/access-traffic/${m.toLowerCase().replace(/\s+/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // ═══════════════════════════════════════════════════════ // RUCKUS / COMMSCOPE // ═══════════════════════════════════════════════════════ const RUCKUS_CONFIG: VendorAssetConfig = { vendorName: "Ruckus", docsPortal: "https://support.ruckuswireless.com/", datasheetLibrary: "https://www.commscope.com/ruckus/products/", supportPortal: "https://support.ruckuswireless.com/", modelAssets: [ { modelPattern: /^ICX/, productPage: (m) => `https://www.commscope.com/ruckus/products/switches/${m.toLowerCase().replace(/\s+/g, "-")}`, imageUrl: () => null, datasheetUrl: () => null, }, ], }; // All vendor configs const VENDOR_CONFIGS: VendorAssetConfig[] = [ CISCO_CONFIG, ARISTA_CONFIG, JUNIPER_CONFIG, NOKIA_CONFIG, FORTINET_CONFIG, MIKROTIK_CONFIG, HIRSCHMANN_CONFIG, HPE_CONFIG, DELL_CONFIG, EXTREME_CONFIG, MOXA_CONFIG, SIEMENS_CONFIG, DLINK_CONFIG, ALE_CONFIG, BROCADE_CONFIG, CIENA_CONFIG, H3C_CONFIG, GIGAMON_CONFIG, RUCKUS_CONFIG, ]; function findVendorConfig(vendorName: string): VendorAssetConfig | undefined { return VENDOR_CONFIGS.find((c) => vendorName.toLowerCase().includes(c.vendorName.toLowerCase().split(" ")[0].toLowerCase()) ); } // ═══════════════════════════════════════════════════════ // Main scraper function // ═══════════════════════════════════════════════════════ export async function scrapeSwitchAssets(targetVendor?: string): Promise { console.log("=== Switch Product Assets Scraper ===\n"); // Get all switches that haven't had assets scraped const vendorFilter = targetVendor ? `AND v.name ILIKE '%${targetVendor}%'` : ""; const result = await pool.query(` SELECT sw.id, sw.model, sw.series, v.name as vendor_name, v.id as vendor_id FROM switches sw JOIN vendors v ON sw.vendor_id = v.id WHERE sw.assets_scraped_at IS NULL ${vendorFilter} ORDER BY v.name, sw.model `); console.log(`Found ${result.rows.length} switches without assets${targetVendor ? ` (vendor: ${targetVendor})` : ""}.\n`); let images = 0; let datasheets = 0; let manuals = 0; let productPages = 0; for (const row of result.rows) { const config = findVendorConfig(row.vendor_name); if (!config) { console.log(` [SKIP] No config for vendor: ${row.vendor_name}`); continue; } // Find matching model asset map const assetMap = config.modelAssets.find((m) => m.modelPattern.test(row.model)); if (!assetMap) continue; console.log(` ${row.vendor_name} ${row.model}:`); // 1. Set product page URL const productPageUrl = assetMap.productPage(row.model); if (productPageUrl) { await setSwitchProductPage(row.id, productPageUrl); productPages++; console.log(` ✓ Product page`); } // 2. Download image const imageUrl = assetMap.imageUrl(row.model); if (imageUrl) { const ok = await downloadSwitchImage(row.id, imageUrl, row.vendor_name, row.model); if (ok) { images++; console.log(` ✓ Image downloaded`); } } // 3. Download datasheet const datasheetUrl = assetMap.datasheetUrl(row.model); if (datasheetUrl) { const ok = await downloadSwitchDatasheet( row.id, row.vendor_id, datasheetUrl, `${row.model} Datasheet`, row.vendor_name, row.model ); if (ok) { datasheets++; console.log(` ✓ Datasheet downloaded`); } } // 4. Download manuals const manualList = assetMap.manualUrls?.(row.model) || []; for (const manual of manualList) { const ok = await downloadSwitchManual( row.id, row.vendor_id, manual.url, manual.title, manual.type, row.vendor_name, row.model ); if (ok) { manuals++; console.log(` ✓ ${manual.type}: ${manual.title}`); } } // Rate limiting: 500ms between vendors await new Promise((r) => setTimeout(r, 500)); } // Update vendor doc portal URLs for (const config of VENDOR_CONFIGS) { const vendorResult = await pool.query(`SELECT id FROM vendors WHERE name ILIKE $1`, [`%${config.vendorName.split(" ")[0]}%`]); if (vendorResult.rows.length > 0) { await setVendorDocUrls(vendorResult.rows[0].id, { docsPortal: config.docsPortal, datasheetLibrary: config.datasheetLibrary, supportPortal: config.supportPortal, imageCdn: config.imageCdn, }); } } console.log(`\n=== Assets Complete ===`); console.log(` Product pages: ${productPages}`); console.log(` Images: ${images}`); console.log(` Datasheets: ${datasheets}`); console.log(` Manuals: ${manuals}`); } if (require.main === module) { const vendor = process.argv.find((a) => a.startsWith("--vendor="))?.split("=")[1]; scrapeSwitchAssets(vendor) .then(() => pool.end()) .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); }