683 lines
32 KiB
TypeScript
683 lines
32 KiB
TypeScript
/**
|
|
* Switch Product Assets Scraper — Images, Datasheets, Manuals
|
|
*
|
|
* Scrapes product pages from all major switch vendors to collect:
|
|
* - Product images (hero shots)
|
|
* - Datasheet PDFs
|
|
* - Installation/Configuration guides
|
|
* - CLI references
|
|
* - Quick start guides
|
|
*
|
|
* Usage:
|
|
* tsx src/index.ts --switch-assets — Scrape all vendor assets
|
|
* tsx src/index.ts --switch-assets --vendor cisco — Scrape single vendor
|
|
*/
|
|
import { pool } from "../utils/db";
|
|
import {
|
|
downloadSwitchImage,
|
|
downloadSwitchDatasheet,
|
|
downloadSwitchManual,
|
|
setSwitchProductPage,
|
|
setVendorDocUrls,
|
|
} from "../utils/assets";
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Vendor-specific URL patterns for product pages, images, datasheets
|
|
// ═══════════════════════════════════════════════════════
|
|
interface VendorAssetConfig {
|
|
vendorName: string;
|
|
docsPortal: string;
|
|
datasheetLibrary: string;
|
|
supportPortal: string;
|
|
imageCdn?: string;
|
|
/** Map model patterns → { productPage, imageUrl, datasheetUrl, manualUrls } */
|
|
modelAssets: ModelAssetMap[];
|
|
}
|
|
|
|
interface ModelAssetMap {
|
|
modelPattern: RegExp;
|
|
productPage: (model: string) => string;
|
|
imageUrl: (model: string) => string | null;
|
|
datasheetUrl: (model: string) => string | null;
|
|
manualUrls?: (model: string) => Array<{ url: string; title: string; type: string }>;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// CISCO
|
|
// ═══════════════════════════════════════════════════════
|
|
const CISCO_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Cisco Systems",
|
|
docsPortal: "https://www.cisco.com/c/en/us/support/index.html",
|
|
datasheetLibrary: "https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/datasheet-listing.html",
|
|
supportPortal: "https://www.cisco.com/c/en/us/support/index.html",
|
|
imageCdn: "https://www.cisco.com/c/dam/en/us/products/collateral/switches/",
|
|
modelAssets: [
|
|
{
|
|
// Nexus 9500 modular
|
|
modelPattern: /^N9K-C95/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-729404.docx/_jcr_content/renditions/datasheet-c78-729404_0.jpg`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-729404.pdf`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-installation-and-configuration-guides-list.html`, title: "Nexus 9000 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Nexus 9300-FX/FX2/FX3 fixed
|
|
modelPattern: /^N9K-C93\d{2,4}.*FX/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-744052.docx/_jcr_content/renditions/datasheet-c78-744052_0.png`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-744052.pdf`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/interfaces/cisco-nexus-9000-series-nx-os-interfaces-configuration-guide-104x.html`, title: "NX-OS Interfaces Configuration Guide", type: "manual" },
|
|
{ url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/fundamentals/cisco-nexus-9000-series-nx-os-fundamentals-configuration-guide-104x.html`, title: "NX-OS Fundamentals Configuration Guide", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Nexus 9300-GX/GX2 (newer)
|
|
modelPattern: /^N9K-C93\d{2,4}.*GX/,
|
|
productPage: () => `https://www.cisco.com/site/us/en/products/networking/cloud-networking-switches/nexus-9000-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/content/dam/cisco-cdc/site/us/en/images/networking/nexus9000-switching-nx9364gx2a-530x280.png`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/networking/switches/nexus-9000-series-switches/nexus-9300-gx2-series-fixed-switches-ds.pdf`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-installation-and-configuration-guides-list.html`, title: "Nexus 9000 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// All other Nexus 9000
|
|
modelPattern: /^N9K-|^N3K-/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/nexus-9000-series-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-736967.docx/_jcr_content/renditions/datasheet-c78-736967_0.jpg`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/datasheet-c78-736967.pdf`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/configuration/interfaces/cisco-nexus-9000-series-nx-os-interfaces-configuration-guide-104x.html`, title: "NX-OS Interfaces Configuration Guide", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Catalyst 9300
|
|
modelPattern: /^C93/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9300-series-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/catalyst-9300-series-switches/nb-06-cat9300-ser-data-sheet-cte-en.docx/_jcr_content/renditions/nb-06-cat9300-ser-data-sheet-cte-en_0.png`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9300-series-switches/nb-06-cat9300-ser-data-sheet-cte-en.html`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9300-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9300 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Catalyst 9200
|
|
modelPattern: /^C92/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9200-series-switches/index.html`,
|
|
imageUrl: () => `https://www.cisco.com/c/dam/en/us/products/collateral/switches/catalyst-9200-series-switches/nb-06-cat9200-ser-data-sheet-cte-en.docx/_jcr_content/renditions/nb-06-cat9200-ser-data-sheet-cte-en_0.png`,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9200-series-switches/nb-06-cat9200-ser-data-sheet-cte-en.html`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9200-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9200 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Catalyst 9400
|
|
modelPattern: /^C94/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9400-series-switches/index.html`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9400-series-switches/nb-06-cat9400-ser-data-sheet-cte-en.html`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9400-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9400 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// Catalyst 9500
|
|
modelPattern: /^C95/,
|
|
productPage: () => `https://www.cisco.com/c/en/us/products/switches/catalyst-9500-series-switches/index.html`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => `https://www.cisco.com/c/en/us/products/collateral/switches/catalyst-9500-series-switches/nb-06-cat9500-ser-data-sheet-cte-en.html`,
|
|
manualUrls: () => [
|
|
{ url: `https://www.cisco.com/c/en/us/support/switches/catalyst-9500-series-switches/products-installation-and-configuration-guides-list.html`, title: "Catalyst 9500 Configuration Guides", type: "manual" },
|
|
],
|
|
},
|
|
{
|
|
// NCS / Cisco 8000
|
|
modelPattern: /^NCS-|^8[01]\d{2}/,
|
|
productPage: (m) => `https://www.cisco.com/c/en/us/products/routers/network-convergence-system-5500-series/index.html`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// ARISTA
|
|
// ═══════════════════════════════════════════════════════
|
|
const ARISTA_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Arista Networks",
|
|
docsPortal: "https://www.arista.com/en/support/product-documentation",
|
|
datasheetLibrary: "https://www.arista.com/en/products",
|
|
supportPortal: "https://www.arista.com/en/support/customer-support",
|
|
imageCdn: "https://www.arista.com/assets/images/",
|
|
modelAssets: [
|
|
{
|
|
// Arista 7000 series — verified URL patterns
|
|
// Datasheets: /assets/data/pdf/Datasheets/{SeriesName}-Datasheet.pdf (naming inconsistent)
|
|
// Images: /assets/images/product/{series}-stack-200w200h.png or /assets/images/article/{model}-Right.png
|
|
// QSGs: /assets/data/pdf/qsg/qsg-books/QS_{series}_{formfactor}_{gen}.pdf
|
|
modelPattern: /^7[0-9]/,
|
|
productPage: (m) => {
|
|
const series = m.match(/^(7\d{3,4}[A-Z]*\d*)/)?.[1] || m;
|
|
return `https://www.arista.com/en/products/${series.toLowerCase().replace(/[^a-z0-9]/g, "")}-series`;
|
|
},
|
|
imageUrl: (m) => {
|
|
// Try article-level images (higher res): {model}-Right.png
|
|
const model = m.match(/^(\d{4}[A-Z]+\d*)/)?.[1] || m.split("-")[0];
|
|
return `https://www.arista.com/assets/images/article/${model}-Right.png`;
|
|
},
|
|
datasheetUrl: (m) => {
|
|
// Arista naming: {series}-Datasheet.pdf or {series}-Data-Sheet.pdf
|
|
const series = m.match(/^(7\d{3,4}[A-Z]*\d*)/)?.[1];
|
|
return series ? `https://www.arista.com/assets/data/pdf/Datasheets/${series}-Datasheet.pdf` : null;
|
|
},
|
|
manualUrls: (m) => {
|
|
const series = m.match(/^(7\d{3,4})/)?.[1] || "";
|
|
return [
|
|
{ url: `https://www.arista.com/assets/data/pdf/qsg/qsg-books/QS_${series}_1RU_Gen3.pdf`, title: `${m} Quick Start Guide`, type: "quick_start" },
|
|
];
|
|
},
|
|
},
|
|
{
|
|
// Arista 5000 (Campus)
|
|
modelPattern: /^5[0-9]/,
|
|
productPage: () => `https://www.arista.com/en/products/arista-5000-series`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => `https://www.arista.com/assets/data/pdf/Datasheets/Arista-5000-Datasheet.pdf`,
|
|
manualUrls: () => [],
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// JUNIPER
|
|
// ═══════════════════════════════════════════════════════
|
|
const JUNIPER_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Juniper Networks",
|
|
docsPortal: "https://www.juniper.net/documentation/",
|
|
datasheetLibrary: "https://www.juniper.net/us/en/products.html",
|
|
supportPortal: "https://supportportal.juniper.net/",
|
|
imageCdn: "https://www.juniper.net/content/dam/www/assets/images/",
|
|
modelAssets: [
|
|
{
|
|
// Juniper switches — verified URL patterns
|
|
// Datasheets: /content/dam/www/assets/datasheets/us/en/switches/{name}-datasheet.pdf
|
|
// Images: /content/dam/www/assets/images/us/en/products/switches/{series}/{filename}.png
|
|
// HW guides: /documentation/us/en/hardware/{model}/{model}.pdf
|
|
// QSGs: /documentation/us/en/quick-start/hardware/qsg/{model}/{model}.pdf
|
|
modelPattern: /^QFX|^EX|^MX|^PTX|^SRX|^ACX/,
|
|
productPage: (m) => {
|
|
const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || "";
|
|
const seriesNum = m.match(/^[A-Z]+(\d+)/)?.[1] || "";
|
|
return `https://www.juniper.net/us/en/products/switches/${seriesPrefix}-series/${seriesPrefix}${seriesNum}.html`;
|
|
},
|
|
imageUrl: (m) => {
|
|
const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || "";
|
|
const category = (seriesPrefix === "mx" || seriesPrefix === "ptx" || seriesPrefix === "acx") ? "routers" : "switches";
|
|
return `https://www.juniper.net/content/dam/www/assets/images/us/en/products/${category}/${seriesPrefix}-series/${seriesPrefix}-family-21DEC20.png`;
|
|
},
|
|
datasheetUrl: (m) => {
|
|
const seriesPrefix = m.match(/^([A-Z]+)/)?.[1]?.toLowerCase() || "";
|
|
const category = (seriesPrefix === "mx" || seriesPrefix === "ptx" || seriesPrefix === "acx") ? "routers" : "switches";
|
|
// Juniper datasheet naming: {full-series-name}-datasheet.pdf or {product-description}-datasheet.pdf
|
|
const model = m.match(/^([A-Z]+\d+)/)?.[1]?.toLowerCase() || "";
|
|
return `https://www.juniper.net/content/dam/www/assets/datasheets/us/en/${category}/${model}-ethernet-switch-datasheet.pdf`;
|
|
},
|
|
manualUrls: (m) => {
|
|
const model = m.toLowerCase().replace(/\s+/g, "");
|
|
return [
|
|
{ url: `https://www.juniper.net/documentation/us/en/hardware/${model}/${model}.pdf`, title: `${m} Hardware Guide`, type: "manual" },
|
|
{ url: `https://www.juniper.net/documentation/us/en/quick-start/hardware/qsg/${model}/${model}.pdf`, title: `${m} Quick Start Guide`, type: "quick_start" },
|
|
];
|
|
},
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// NOKIA
|
|
// ═══════════════════════════════════════════════════════
|
|
const NOKIA_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Nokia",
|
|
docsPortal: "https://documentation.nokia.com/",
|
|
datasheetLibrary: "https://www.nokia.com/networks/products/",
|
|
supportPortal: "https://customer.nokia.com/support/s/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://www.nokia.com/networks/products/${m.toLowerCase().replace(/\s+/g, "-")}/`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// FORTINET
|
|
// ═══════════════════════════════════════════════════════
|
|
const FORTINET_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Fortinet",
|
|
docsPortal: "https://docs.fortinet.com/",
|
|
datasheetLibrary: "https://www.fortinet.com/products/switches",
|
|
supportPortal: "https://support.fortinet.com/",
|
|
modelAssets: [
|
|
{
|
|
// FortiSwitch product pages — use CheerioCrawler for image extraction
|
|
// Datasheets at /content/dam/fortinet/assets/data-sheets/
|
|
modelPattern: /^FortiSwitch/,
|
|
productPage: (m) => {
|
|
const num = m.match(/(\d+[A-Z]*)/)?.[1]?.toLowerCase() || "";
|
|
return `https://www.fortinet.com/products/switches/fortiswitch-${num}`;
|
|
},
|
|
imageUrl: () => null, // extracted from product page by crawler
|
|
datasheetUrl: (m) => {
|
|
// Fortinet DAM paths: fortiswitch-{series}00-series.pdf or FortiSwitch-{model}.pdf
|
|
const series = m.match(/FortiSwitch[- ]*(\d)/)?.[1];
|
|
if (series) return `https://www.fortinet.com/content/dam/fortinet/assets/data-sheets/fortiswitch-${series}00-series.pdf`;
|
|
return null;
|
|
},
|
|
manualUrls: () => [
|
|
{ url: `https://docs.fortinet.com/document/fortiswitch/latest/administration-guide`, title: "FortiSwitch Administration Guide", type: "manual" },
|
|
{ url: `https://docs.fortinet.com/document/fortiswitch/latest/hardware-guide`, title: "FortiSwitch Hardware Guide", type: "installation_guide" },
|
|
],
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// MIKROTIK
|
|
// ═══════════════════════════════════════════════════════
|
|
const MIKROTIK_CONFIG: VendorAssetConfig = {
|
|
vendorName: "MikroTik",
|
|
docsPortal: "https://help.mikrotik.com/docs/",
|
|
datasheetLibrary: "https://mikrotik.com/products",
|
|
supportPortal: "https://help.mikrotik.com/",
|
|
modelAssets: [
|
|
{
|
|
// MikroTik uses underscored model names in URLs
|
|
// Images & PDFs use unpredictable numeric IDs — must scrape product page
|
|
// CheerioCrawler handles this via parseMikroTikPage()
|
|
modelPattern: /^CRS|^CCR/,
|
|
productPage: (m) => `https://mikrotik.com/product/${m.replace(/\s+/g, "_")}`,
|
|
imageUrl: () => null, // extracted from product page by crawler
|
|
datasheetUrl: () => null, // extracted from product page by crawler
|
|
manualUrls: (m) => [
|
|
{ url: `https://help.mikrotik.com/docs/display/UM/${m.replace(/\s+/g, "+")}`, title: `${m} User Manual`, type: "manual" },
|
|
],
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// HIRSCHMANN
|
|
// ═══════════════════════════════════════════════════════
|
|
const HIRSCHMANN_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Hirschmann",
|
|
docsPortal: "https://catalog.belden.com/index.cfm",
|
|
datasheetLibrary: "https://catalog.belden.com/index.cfm",
|
|
supportPortal: "https://www.belden.com/support",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://catalog.belden.com/techdata/en/${m.replace(/\s+/g, "_")}_en.html`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Additional vendor stubs (HPE, Dell, Extreme, Huawei, etc.)
|
|
// ═══════════════════════════════════════════════════════
|
|
const HPE_CONFIG: VendorAssetConfig = {
|
|
vendorName: "HPE / Aruba",
|
|
docsPortal: "https://www.arubanetworks.com/techdocs/",
|
|
datasheetLibrary: "https://www.arubanetworks.com/products/switches/",
|
|
supportPortal: "https://asp.arubanetworks.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^CX/,
|
|
productPage: (m) => `https://www.arubanetworks.com/products/switches/cx-${m.match(/CX\s*(\d+)/)?.[1] || ""}-series/`,
|
|
imageUrl: (m) => `https://www.arubanetworks.com/wp-content/uploads/aruba-cx-${m.match(/CX\s*(\d+)/)?.[1] || ""}.png`,
|
|
datasheetUrl: (m) => `https://www.arubanetworks.com/assets/ds/DS_CX${m.match(/CX\s*(\d+)/)?.[1] || ""}.pdf`,
|
|
manualUrls: (m) => [
|
|
{ url: `https://www.arubanetworks.com/techdocs/CX/`, title: `Aruba CX ${m} Configuration Guide`, type: "manual" },
|
|
],
|
|
},
|
|
],
|
|
};
|
|
|
|
const DELL_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Dell Technologies",
|
|
docsPortal: "https://www.dell.com/support/",
|
|
datasheetLibrary: "https://www.dell.com/en-us/lp/networking",
|
|
supportPortal: "https://www.dell.com/support/home/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://www.dell.com/en-us/shop/networking/cp/${m.toLowerCase().replace(/[^a-z0-9]/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
const EXTREME_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Extreme Networks",
|
|
docsPortal: "https://extremeportal.force.com/ExtrArticleDetail",
|
|
datasheetLibrary: "https://www.extremenetworks.com/products/",
|
|
supportPortal: "https://extremeportal.force.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^X|^SLX|^VDX|^5[0-9]/,
|
|
productPage: (m) => `https://www.extremenetworks.com/product/${m.toLowerCase().replace(/[^a-z0-9]/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
const MOXA_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Moxa",
|
|
docsPortal: "https://www.moxa.com/en/support/support",
|
|
datasheetLibrary: "https://www.moxa.com/en/products/industrial-network-infrastructure",
|
|
supportPortal: "https://www.moxa.com/en/support/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://www.moxa.com/en/products/industrial-network-infrastructure/${m.toLowerCase()}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
const SIEMENS_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Siemens",
|
|
docsPortal: "https://support.industry.siemens.com/",
|
|
datasheetLibrary: "https://mall.industry.siemens.com/",
|
|
supportPortal: "https://support.industry.siemens.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^SCALANCE/,
|
|
productPage: (m) => `https://www.siemens.com/global/en/products/automation/industrial-communication/industrial-ethernet/${m.toLowerCase().replace(/\s+/g, "-")}.html`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// D-LINK
|
|
// ═══════════════════════════════════════════════════════
|
|
const DLINK_CONFIG: VendorAssetConfig = {
|
|
vendorName: "D-Link",
|
|
docsPortal: "https://www.dlink.com/en/support",
|
|
datasheetLibrary: "https://www.dlink.com/en/products/switches",
|
|
supportPortal: "https://www.dlink.com/en/support",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^D[GMXW]S/,
|
|
productPage: (m) => `https://www.dlink.com/en/products/${m.toLowerCase()}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// ALCATEL-LUCENT ENTERPRISE
|
|
// ═══════════════════════════════════════════════════════
|
|
const ALE_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Alcatel-Lucent Enterprise",
|
|
docsPortal: "https://www.al-enterprise.com/en-us/documentation",
|
|
datasheetLibrary: "https://www.al-enterprise.com/en-us/products/switches",
|
|
supportPortal: "https://myportal.al-enterprise.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^OmniSwitch/,
|
|
productPage: (m) => {
|
|
const num = m.match(/(\d{4})/)?.[1] || "";
|
|
return `https://www.al-enterprise.com/en-us/products/switches/omniswitch-${num}`;
|
|
},
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// BROCADE / BROADCOM
|
|
// ═══════════════════════════════════════════════════════
|
|
const BROCADE_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Brocade",
|
|
docsPortal: "https://www.broadcom.com/support/fibre-channel-networking",
|
|
datasheetLibrary: "https://www.broadcom.com/products/fibre-channel-networking",
|
|
supportPortal: "https://www.broadcom.com/support",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://www.broadcom.com/products/fibre-channel-networking/switches/${m.toLowerCase().replace(/\s+/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// CIENA
|
|
// ═══════════════════════════════════════════════════════
|
|
const CIENA_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Ciena",
|
|
docsPortal: "https://www.ciena.com/insights/resources",
|
|
datasheetLibrary: "https://www.ciena.com/products",
|
|
supportPortal: "https://www.ciena.com/support",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /./,
|
|
productPage: (m) => `https://www.ciena.com/products/${m.toLowerCase().replace(/\s+/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// H3C
|
|
// ═══════════════════════════════════════════════════════
|
|
const H3C_CONFIG: VendorAssetConfig = {
|
|
vendorName: "H3C",
|
|
docsPortal: "https://www.h3c.com/en/Support/Resource_Center/",
|
|
datasheetLibrary: "https://www.h3c.com/en/Products_And_Solution/Networking/",
|
|
supportPortal: "https://www.h3c.com/en/Support/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^S/,
|
|
productPage: (m) => `https://www.h3c.com/en/Products_And_Solution/Networking/Switches/${m}/`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// GIGAMON
|
|
// ═══════════════════════════════════════════════════════
|
|
const GIGAMON_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Gigamon",
|
|
docsPortal: "https://docs.gigamon.com/",
|
|
datasheetLibrary: "https://www.gigamon.com/products",
|
|
supportPortal: "https://community.gigamon.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^GigaVUE/,
|
|
productPage: (m) => `https://www.gigamon.com/products/access-traffic/${m.toLowerCase().replace(/\s+/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// RUCKUS / COMMSCOPE
|
|
// ═══════════════════════════════════════════════════════
|
|
const RUCKUS_CONFIG: VendorAssetConfig = {
|
|
vendorName: "Ruckus",
|
|
docsPortal: "https://support.ruckuswireless.com/",
|
|
datasheetLibrary: "https://www.commscope.com/ruckus/products/",
|
|
supportPortal: "https://support.ruckuswireless.com/",
|
|
modelAssets: [
|
|
{
|
|
modelPattern: /^ICX/,
|
|
productPage: (m) => `https://www.commscope.com/ruckus/products/switches/${m.toLowerCase().replace(/\s+/g, "-")}`,
|
|
imageUrl: () => null,
|
|
datasheetUrl: () => null,
|
|
},
|
|
],
|
|
};
|
|
|
|
// All vendor configs
|
|
const VENDOR_CONFIGS: VendorAssetConfig[] = [
|
|
CISCO_CONFIG,
|
|
ARISTA_CONFIG,
|
|
JUNIPER_CONFIG,
|
|
NOKIA_CONFIG,
|
|
FORTINET_CONFIG,
|
|
MIKROTIK_CONFIG,
|
|
HIRSCHMANN_CONFIG,
|
|
HPE_CONFIG,
|
|
DELL_CONFIG,
|
|
EXTREME_CONFIG,
|
|
MOXA_CONFIG,
|
|
SIEMENS_CONFIG,
|
|
DLINK_CONFIG,
|
|
ALE_CONFIG,
|
|
BROCADE_CONFIG,
|
|
CIENA_CONFIG,
|
|
H3C_CONFIG,
|
|
GIGAMON_CONFIG,
|
|
RUCKUS_CONFIG,
|
|
];
|
|
|
|
function findVendorConfig(vendorName: string): VendorAssetConfig | undefined {
|
|
return VENDOR_CONFIGS.find((c) =>
|
|
vendorName.toLowerCase().includes(c.vendorName.toLowerCase().split(" ")[0].toLowerCase())
|
|
);
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Main scraper function
|
|
// ═══════════════════════════════════════════════════════
|
|
export async function scrapeSwitchAssets(targetVendor?: string): Promise<void> {
|
|
console.log("=== Switch Product Assets Scraper ===\n");
|
|
|
|
// Get all switches that haven't had assets scraped
|
|
const vendorFilter = targetVendor
|
|
? `AND v.name ILIKE '%${targetVendor}%'`
|
|
: "";
|
|
|
|
const result = await pool.query(`
|
|
SELECT sw.id, sw.model, sw.series, v.name as vendor_name, v.id as vendor_id
|
|
FROM switches sw
|
|
JOIN vendors v ON sw.vendor_id = v.id
|
|
WHERE sw.assets_scraped_at IS NULL ${vendorFilter}
|
|
ORDER BY v.name, sw.model
|
|
`);
|
|
|
|
console.log(`Found ${result.rows.length} switches without assets${targetVendor ? ` (vendor: ${targetVendor})` : ""}.\n`);
|
|
|
|
let images = 0;
|
|
let datasheets = 0;
|
|
let manuals = 0;
|
|
let productPages = 0;
|
|
|
|
for (const row of result.rows) {
|
|
const config = findVendorConfig(row.vendor_name);
|
|
if (!config) {
|
|
console.log(` [SKIP] No config for vendor: ${row.vendor_name}`);
|
|
continue;
|
|
}
|
|
|
|
// Find matching model asset map
|
|
const assetMap = config.modelAssets.find((m) => m.modelPattern.test(row.model));
|
|
if (!assetMap) continue;
|
|
|
|
console.log(` ${row.vendor_name} ${row.model}:`);
|
|
|
|
// 1. Set product page URL
|
|
const productPageUrl = assetMap.productPage(row.model);
|
|
if (productPageUrl) {
|
|
await setSwitchProductPage(row.id, productPageUrl);
|
|
productPages++;
|
|
console.log(` ✓ Product page`);
|
|
}
|
|
|
|
// 2. Download image
|
|
const imageUrl = assetMap.imageUrl(row.model);
|
|
if (imageUrl) {
|
|
const ok = await downloadSwitchImage(row.id, imageUrl, row.vendor_name, row.model);
|
|
if (ok) {
|
|
images++;
|
|
console.log(` ✓ Image downloaded`);
|
|
}
|
|
}
|
|
|
|
// 3. Download datasheet
|
|
const datasheetUrl = assetMap.datasheetUrl(row.model);
|
|
if (datasheetUrl) {
|
|
const ok = await downloadSwitchDatasheet(
|
|
row.id, row.vendor_id, datasheetUrl,
|
|
`${row.model} Datasheet`, row.vendor_name, row.model
|
|
);
|
|
if (ok) {
|
|
datasheets++;
|
|
console.log(` ✓ Datasheet downloaded`);
|
|
}
|
|
}
|
|
|
|
// 4. Download manuals
|
|
const manualList = assetMap.manualUrls?.(row.model) || [];
|
|
for (const manual of manualList) {
|
|
const ok = await downloadSwitchManual(
|
|
row.id, row.vendor_id, manual.url,
|
|
manual.title, manual.type, row.vendor_name, row.model
|
|
);
|
|
if (ok) {
|
|
manuals++;
|
|
console.log(` ✓ ${manual.type}: ${manual.title}`);
|
|
}
|
|
}
|
|
|
|
// Rate limiting: 500ms between vendors
|
|
await new Promise((r) => setTimeout(r, 500));
|
|
}
|
|
|
|
// Update vendor doc portal URLs
|
|
for (const config of VENDOR_CONFIGS) {
|
|
const vendorResult = await pool.query(`SELECT id FROM vendors WHERE name ILIKE $1`, [`%${config.vendorName.split(" ")[0]}%`]);
|
|
if (vendorResult.rows.length > 0) {
|
|
await setVendorDocUrls(vendorResult.rows[0].id, {
|
|
docsPortal: config.docsPortal,
|
|
datasheetLibrary: config.datasheetLibrary,
|
|
supportPortal: config.supportPortal,
|
|
imageCdn: config.imageCdn,
|
|
});
|
|
}
|
|
}
|
|
|
|
console.log(`\n=== Assets Complete ===`);
|
|
console.log(` Product pages: ${productPages}`);
|
|
console.log(` Images: ${images}`);
|
|
console.log(` Datasheets: ${datasheets}`);
|
|
console.log(` Manuals: ${manuals}`);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
const vendor = process.argv.find((a) => a.startsWith("--vendor="))?.split("=")[1];
|
|
scrapeSwitchAssets(vendor)
|
|
.then(() => pool.end())
|
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
|
}
|