/** * Flexoptix Switch Compatibility Scraper * * Populates the `compatibility` table by querying Flexoptix search/product API * with each switch model number. Flexoptix names compatible transceivers like * "Cisco N9K-C9364C compatible QSFP28 100GBASE-LR4" — so searching by switch * model returns all transceivers that are explicitly marked compatible. * * Strategy: * 1. For each switch in DB, query Flexoptix AJAX search * 2. Parse returned transceiver names for form-factor and vendor hints * 3. Match against transceivers table (Flexoptix vendor) * 4. Insert into compatibility with status = 'compatible' * * Fallback (when search returns nothing): * Form-factor matching — for each port type in switch.ports_config, * find Flexoptix transceivers matching that form factor. * Used as spec_match rather than vendor_compat — shown in dashboard * as "by form factor" rather than "explicitly verified". * * Rate limit: 1 req/1.5sec. */ import { pool } from "../utils/db"; const BASE = "https://www.flexoptix.net"; const HEADERS = { "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; internal-flexoptix)", Accept: "application/json, text/html", }; function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } // ── Ports-config → form factor map ───────────────────────────────────────── const PORT_KEY_TO_FORM_FACTOR: Array<[RegExp, string]> = [ [/QSFP-DD/i, "QSFP-DD"], [/QSFP-DD800/i, "QSFP-DD800"], [/OSFP224/i, "OSFP224"], [/OSFP/i, "OSFP"], [/QSFP28/i, "QSFP28"], [/QSFP\+/i, "QSFP+"], [/QSFP/i, "QSFP+"], [/SFP28/i, "SFP28"], [/SFP\+/i, "SFP+"], [/SFP/i, "SFP"], [/CFP2/i, "CFP2"], [/CFP4/i, "CFP4"], [/CFP/i, "CFP"], ]; function portsConfigToFormFactors(portsConfig: Record): string[] { const factors = new Set(); for (const key of Object.keys(portsConfig)) { for (const [pattern, ff] of PORT_KEY_TO_FORM_FACTOR) { if (pattern.test(key)) { factors.add(ff); break; } } } return [...factors]; } // ── Flexoptix search API ──────────────────────────────────────────────────── interface FlexoptixSuggestion { name: string; url: string; sku?: string; image?: string; price?: string; } async function searchFlexoptix(query: string): Promise { const url = `${BASE}/en/search/ajax/suggest/?q=${encodeURIComponent(query)}`; try { const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(15000), }); if (!resp.ok) return []; const data = await resp.json() as unknown; // Magento suggest returns array or object with products key if (Array.isArray(data)) return data as FlexoptixSuggestion[]; const obj = data as Record; if (obj["products"] && Array.isArray(obj["products"])) { return obj["products"] as FlexoptixSuggestion[]; } return []; } catch { return []; } } // ── Main scraper ──────────────────────────────────────────────────────────── export async function scrapeFlexoptixCompatibility(): Promise { console.log("=== Flexoptix Compatibility Scraper ===\n"); // Get all switches const { rows: switches } = await pool.query<{ id: string; model: string; vendor_name: string; ports_config: Record | null; }>( `SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config FROM switches sw JOIN vendors v ON v.id = sw.vendor_id ORDER BY sw.max_speed_gbps DESC`, ); console.log(` ${switches.length} switches to process\n`); // Get Flexoptix vendor ID const { rows: [flexoptixVendor] } = await pool.query<{ id: string }>( `SELECT id FROM vendors WHERE UPPER(name) LIKE '%FLEXOPTIX%' LIMIT 1`, ); if (!flexoptixVendor) { console.error(" [FAIL] Flexoptix vendor not found in DB — run flexoptix-catalog scraper first"); return; } const flexoptixVendorId = flexoptixVendor.id; let totalCompatAdded = 0; let totalSwitchesProcessed = 0; for (const sw of switches) { console.log(`\n[${sw.vendor_name}] ${sw.model}`); const addedForSwitch: string[] = []; // ── Strategy 1: Search Flexoptix by switch model ────────────────────── await sleep(1500); const suggestions = await searchFlexoptix(sw.model); const matchedBySku = new Set(); if (suggestions.length > 0) { console.log(` Search: ${suggestions.length} results`); for (const suggestion of suggestions) { if (!suggestion.sku && !suggestion.name) continue; const sku = suggestion.sku?.trim(); const name = suggestion.name?.trim(); if (!sku && !name) continue; // Find matching transceiver in DB let txRow: { id: string } | undefined; if (sku) { const r = await pool.query<{ id: string }>( `SELECT id FROM transceivers WHERE vendor_id = $1 AND (part_number = $2 OR slug = $3) LIMIT 1`, [flexoptixVendorId, sku, sku.toLowerCase().replace(/[^a-z0-9]/g, "-")], ); txRow = r.rows[0]; } if (!txRow && name) { // Try matching by name similarity to part_number const r = await pool.query<{ id: string }>( `SELECT id FROM transceivers WHERE vendor_id = $1 AND (part_number ILIKE $2 OR standard_name ILIKE $3) LIMIT 1`, [flexoptixVendorId, `%${name.slice(0, 20)}%`, `%${name.slice(0, 20)}%`], ); txRow = r.rows[0]; } if (txRow) { matchedBySku.add(txRow.id); await upsertCompat(sw.id, txRow.id, "Flexoptix", "vendor_compat"); addedForSwitch.push(txRow.id); } } console.log(` Vendor-compat matches: ${addedForSwitch.length}`); } // ── Strategy 2: Form-factor fallback ───────────────────────────────── if (sw.ports_config) { const formFactors = portsConfigToFormFactors(sw.ports_config); console.log(` Form factors: ${formFactors.join(", ")}`); for (const ff of formFactors) { const { rows: txRows } = await pool.query<{ id: string }>( `SELECT t.id FROM transceivers t WHERE t.vendor_id = $1 AND t.form_factor = $2 AND NOT EXISTS ( SELECT 1 FROM compatibility c WHERE c.switch_id = $3 AND c.transceiver_id = t.id )`, [flexoptixVendorId, ff, sw.id], ); for (const tx of txRows) { if (matchedBySku.has(tx.id)) continue; // already added via search await upsertCompat(sw.id, tx.id, "Form Factor Match", "spec_match"); addedForSwitch.push(tx.id); } if (txRows.length > 0) { console.log(` Form-factor ${ff}: +${txRows.length} transceivers`); } } } totalCompatAdded += addedForSwitch.length; totalSwitchesProcessed++; console.log(` → Added ${addedForSwitch.length} compat entries`); } console.log(`\n=== Flexoptix Compatibility Scraper Complete ===`); console.log(` Switches processed: ${totalSwitchesProcessed}`); console.log(` Compatibility entries added: ${totalCompatAdded}`); } async function upsertCompat( switchId: string, transceiverId: string, verifiedBy: string, verificationMethod: string, ): Promise { await pool.query( `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, source_url) VALUES ($1, $2, $3, $4, 'compatible', 'https://www.flexoptix.net') ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET verified_by = CASE WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verified_by ELSE compatibility.verified_by END, verification_method = CASE WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verification_method ELSE compatibility.verification_method END`, [switchId, transceiverId, verifiedBy, verificationMethod], ); } if (require.main === module) { scrapeFlexoptixCompatibility() .then(() => pool.end()) .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); }