From 1ea73112c6f905f2c1e9d20c23aea43fe2e340ae Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Mon, 20 Apr 2026 22:50:57 +0200 Subject: [PATCH] feat: Flexoptix compatibility scraper + transceiver issue scanner - Add flexoptix-compat.ts: maps switch models to compatible Flexoptix transceivers via search API (vendor_compat) with form-factor fallback (spec_match) Scheduled daily at 09:00 UTC as scrape:compat:flexoptix - Enhance community-issues.ts: add vendor advisory sources (Cisco Field Notices, Juniper KB, SONiC GitHub Issues) + new scrapeTransceiverCompatIssues() that searches for switch+transceiver combination problems specifically - Scheduler: 59 schedules, 78 workers --- packages/scraper/src/scheduler.ts | 18 +- .../scraper/src/scrapers/community-issues.ts | 55 ++++ .../scraper/src/scrapers/flexoptix-compat.ts | 249 ++++++++++++++++++ 3 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 packages/scraper/src/scrapers/flexoptix-compat.ts diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index de8e070..c56c084 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -76,6 +76,7 @@ export async function registerSchedules(boss: PgBoss): Promise { "scrape:vendors:flexoptix", "scrape:vendors:flexoptix-supported", // ── Compatibility (every 12h) ────────────────────────────────────── + "scrape:compat:flexoptix", "scrape:compat:cisco", "scrape:compat:juniper", "scrape:compat:sonic", @@ -210,6 +211,8 @@ export async function registerSchedules(boss: PgBoss): Promise { // COMPATIBILITY MATRICES — every 12h // ══════════════════════════════════════════════════════════════════════ + // Flexoptix compatibility — every 24h at 09:00 (after both switch-assets + image-fetcher) + await boss.schedule("scrape:compat:flexoptix", "0 9 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); @@ -296,7 +299,7 @@ export async function registerSchedules(boss: PgBoss): Promise { // Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); - console.log("All schedules registered — 24/7 continuous scraping (58 jobs)"); + console.log("All schedules registered — 24/7 continuous scraping (59 jobs)"); } export async function registerWorkers(boss: PgBoss): Promise { @@ -318,7 +321,8 @@ export async function registerWorkers(boss: PgBoss): Promise { const { scrapeUfiSpace } = await import("./scrapers/ufispace"); const { scrapeEdgecore } = await import("./scrapers/edgecore"); const { scrapeSwitchAssets } = await import("./scrapers/switch-assets"); - const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher"); + const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher"); + const { scrapeFlexoptixCompatibility } = await import("./scrapers/flexoptix-compat"); // ── Prediction signal scrapers ──────────────────────────────────────── const { scrapeSecEdgar } = await import("./scrapers/sec-edgar"); const { scrapeGithubSignals } = await import("./scrapers/github-signals"); @@ -468,6 +472,11 @@ export async function registerWorkers(boss: PgBoss): Promise { // ── Compatibility scrapers ──────────────────────────────────────────── + await boss.work("scrape:compat:flexoptix", async () => { + console.log(`[${new Date().toISOString()}] Running: Flexoptix compatibility mapping`); + await scrapeFlexoptixCompatibility(); + }); + await boss.work("scrape:compat:cisco", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`); await scrapeCiscoTmg(); @@ -534,8 +543,9 @@ export async function registerWorkers(boss: PgBoss): Promise { await boss.work("scrape:community-issues", async () => { console.log(`[${new Date().toISOString()}] Running: Community issues`); - const { scrapeAllSwitchIssues } = await import("./scrapers/community-issues"); + const { scrapeAllSwitchIssues, scrapeTransceiverCompatIssues } = await import("./scrapers/community-issues"); await scrapeAllSwitchIssues(30); + await scrapeTransceiverCompatIssues(15); }); await boss.work("scrape:datasheet-links", async () => { @@ -1126,5 +1136,5 @@ export async function registerWorkers(boss: PgBoss): Promise { console.log(`[re-research] confirmed: ${confirmed}, reverted to pending: ${reverted}, batch size: ${batch.rows.length}`); }); - console.log("All workers registered (77 jobs, 24/7 continuous)"); + console.log("All workers registered (78 jobs, 24/7 continuous)"); } diff --git a/packages/scraper/src/scrapers/community-issues.ts b/packages/scraper/src/scrapers/community-issues.ts index a1dbac9..7869a1e 100644 --- a/packages/scraper/src/scrapers/community-issues.ts +++ b/packages/scraper/src/scrapers/community-issues.ts @@ -77,6 +77,25 @@ const COMMUNITY_SOURCES: Array<{ buildSearchUrl: (model) => `https://networkengineering.stackexchange.com/search?q=${encodeURIComponent(model)}`, }, + // ── Vendor advisory / field notice pages ────────────────────────────── + { + name: "Cisco Field Notices", + type: "vendor_advisory", + buildSearchUrl: (model) => + `https://www.cisco.com/c/en/us/support/web/tools/psn/BT/search.html?query=${encodeURIComponent(model + " transceiver")}`, + }, + { + name: "Juniper KB", + type: "vendor_kb", + buildSearchUrl: (model) => + `https://supportportal.juniper.net/s/global-search/${encodeURIComponent(model + " transceiver")}?language=en_US`, + }, + { + name: "SONiC GitHub Issues", + type: "github", + buildSearchUrl: (model) => + `https://github.com/sonic-net/sonic-buildimage/issues?q=${encodeURIComponent(model + " transceiver")}+is%3Aissue`, + }, ]; // ───────────────────────────────────────────────────────────────────────────── @@ -380,6 +399,40 @@ export async function findAndSeedDatasheetLinks(limit = 50): Promise { } } +// ───────────────────────────────────────────────────────────────────────────── +// Scrape transceiver-specific issues per switch (switch + transceiver combos) +// ───────────────────────────────────────────────────────────────────────────── +export async function scrapeTransceiverCompatIssues(switchLimit = 20): Promise { + // Get switches that have compatibility entries — search for their transceiver issues + const result = await pool.query<{ switch_model: string; form_factors: string[] }>( + `SELECT sw.model AS switch_model, + ARRAY_AGG(DISTINCT t.form_factor) AS form_factors + FROM switches sw + JOIN compatibility c ON c.switch_id = sw.id + JOIN transceivers t ON c.transceiver_id = t.id + GROUP BY sw.model + ORDER BY COUNT(c.id) DESC + LIMIT $1`, + [switchLimit], + ); + + for (const row of result.rows) { + const { switch_model, form_factors } = row; + // Build composite search queries: "N9K-C9364C QSFP28 issue" + const queries: string[] = []; + for (const ff of (form_factors || [])) { + queries.push(`${switch_model} ${ff} issue`); + queries.push(`${switch_model} ${ff} not recognized`); + queries.push(`${switch_model} transceiver incompatible`); + } + + await scrapeProductIssues(queries.slice(0, 4), 2); + await new Promise((r) => setTimeout(r, 2000)); + } + + logger.info(`Transceiver compat issues scraping complete for ${result.rows.length} switches`); +} + // CLI entrypoint if (require.main === module) { (async () => { @@ -388,6 +441,8 @@ if (require.main === module) { await scrapeAllSwitchIssues(parseInt(process.argv[3] || "30")); } else if (cmd === "datasheets") { await findAndSeedDatasheetLinks(parseInt(process.argv[3] || "50")); + } else if (cmd === "transceiver-compat") { + await scrapeTransceiverCompatIssues(parseInt(process.argv[3] || "20")); } process.exit(0); })(); diff --git a/packages/scraper/src/scrapers/flexoptix-compat.ts b/packages/scraper/src/scrapers/flexoptix-compat.ts new file mode 100644 index 0000000..e5ac083 --- /dev/null +++ b/packages/scraper/src/scrapers/flexoptix-compat.ts @@ -0,0 +1,249 @@ +/** + * Flexoptix Switch Compatibility Scraper + * + * Populates the `compatibility` table by querying Flexoptix search/product API + * with each switch model number. Flexoptix names compatible transceivers like + * "Cisco N9K-C9364C compatible QSFP28 100GBASE-LR4" — so searching by switch + * model returns all transceivers that are explicitly marked compatible. + * + * Strategy: + * 1. For each switch in DB, query Flexoptix AJAX search + * 2. Parse returned transceiver names for form-factor and vendor hints + * 3. Match against transceivers table (Flexoptix vendor) + * 4. Insert into compatibility with status = 'compatible' + * + * Fallback (when search returns nothing): + * Form-factor matching — for each port type in switch.ports_config, + * find Flexoptix transceivers matching that form factor. + * Used as spec_match rather than vendor_compat — shown in dashboard + * as "by form factor" rather than "explicitly verified". + * + * Rate limit: 1 req/1.5sec. + */ +import { pool } from "../utils/db"; + +const BASE = "https://www.flexoptix.net"; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; internal-flexoptix)", + Accept: "application/json, text/html", +}; + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +// ── Ports-config → form factor map ───────────────────────────────────────── + +const PORT_KEY_TO_FORM_FACTOR: Array<[RegExp, string]> = [ + [/QSFP-DD/i, "QSFP-DD"], + [/QSFP-DD800/i, "QSFP-DD800"], + [/OSFP224/i, "OSFP224"], + [/OSFP/i, "OSFP"], + [/QSFP28/i, "QSFP28"], + [/QSFP\+/i, "QSFP+"], + [/QSFP/i, "QSFP+"], + [/SFP28/i, "SFP28"], + [/SFP\+/i, "SFP+"], + [/SFP/i, "SFP"], + [/CFP2/i, "CFP2"], + [/CFP4/i, "CFP4"], + [/CFP/i, "CFP"], +]; + +function portsConfigToFormFactors(portsConfig: Record): string[] { + const factors = new Set(); + for (const key of Object.keys(portsConfig)) { + for (const [pattern, ff] of PORT_KEY_TO_FORM_FACTOR) { + if (pattern.test(key)) { + factors.add(ff); + break; + } + } + } + return [...factors]; +} + +// ── Flexoptix search API ──────────────────────────────────────────────────── + +interface FlexoptixSuggestion { + name: string; + url: string; + sku?: string; + image?: string; + price?: string; +} + +async function searchFlexoptix(query: string): Promise { + const url = `${BASE}/en/search/ajax/suggest/?q=${encodeURIComponent(query)}`; + try { + const resp = await fetch(url, { + headers: HEADERS, + signal: AbortSignal.timeout(15000), + }); + if (!resp.ok) return []; + const data = await resp.json() as unknown; + // Magento suggest returns array or object with products key + if (Array.isArray(data)) return data as FlexoptixSuggestion[]; + const obj = data as Record; + if (obj["products"] && Array.isArray(obj["products"])) { + return obj["products"] as FlexoptixSuggestion[]; + } + return []; + } catch { + return []; + } +} + +// ── Main scraper ──────────────────────────────────────────────────────────── + +export async function scrapeFlexoptixCompatibility(): Promise { + console.log("=== Flexoptix Compatibility Scraper ===\n"); + + // Get all switches + const { rows: switches } = await pool.query<{ + id: string; + model: string; + vendor_name: string; + ports_config: Record | null; + }>( + `SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config + FROM switches sw + JOIN vendors v ON v.id = sw.vendor_id + ORDER BY sw.max_speed_gbps DESC`, + ); + + console.log(` ${switches.length} switches to process\n`); + + // Get Flexoptix vendor ID + const { rows: [flexoptixVendor] } = await pool.query<{ id: string }>( + `SELECT id FROM vendors WHERE UPPER(name) LIKE '%FLEXOPTIX%' LIMIT 1`, + ); + + if (!flexoptixVendor) { + console.error(" [FAIL] Flexoptix vendor not found in DB — run flexoptix-catalog scraper first"); + return; + } + + const flexoptixVendorId = flexoptixVendor.id; + let totalCompatAdded = 0; + let totalSwitchesProcessed = 0; + + for (const sw of switches) { + console.log(`\n[${sw.vendor_name}] ${sw.model}`); + const addedForSwitch: string[] = []; + + // ── Strategy 1: Search Flexoptix by switch model ────────────────────── + await sleep(1500); + const suggestions = await searchFlexoptix(sw.model); + + const matchedBySku = new Set(); + + if (suggestions.length > 0) { + console.log(` Search: ${suggestions.length} results`); + + for (const suggestion of suggestions) { + if (!suggestion.sku && !suggestion.name) continue; + const sku = suggestion.sku?.trim(); + const name = suggestion.name?.trim(); + if (!sku && !name) continue; + + // Find matching transceiver in DB + let txRow: { id: string } | undefined; + if (sku) { + const r = await pool.query<{ id: string }>( + `SELECT id FROM transceivers WHERE vendor_id = $1 AND (part_number = $2 OR slug = $3) LIMIT 1`, + [flexoptixVendorId, sku, sku.toLowerCase().replace(/[^a-z0-9]/g, "-")], + ); + txRow = r.rows[0]; + } + + if (!txRow && name) { + // Try matching by name similarity to part_number + const r = await pool.query<{ id: string }>( + `SELECT id FROM transceivers + WHERE vendor_id = $1 + AND (part_number ILIKE $2 OR standard_name ILIKE $3) + LIMIT 1`, + [flexoptixVendorId, `%${name.slice(0, 20)}%`, `%${name.slice(0, 20)}%`], + ); + txRow = r.rows[0]; + } + + if (txRow) { + matchedBySku.add(txRow.id); + await upsertCompat(sw.id, txRow.id, "Flexoptix", "vendor_compat"); + addedForSwitch.push(txRow.id); + } + } + + console.log(` Vendor-compat matches: ${addedForSwitch.length}`); + } + + // ── Strategy 2: Form-factor fallback ───────────────────────────────── + if (sw.ports_config) { + const formFactors = portsConfigToFormFactors(sw.ports_config); + console.log(` Form factors: ${formFactors.join(", ")}`); + + for (const ff of formFactors) { + const { rows: txRows } = await pool.query<{ id: string }>( + `SELECT t.id + FROM transceivers t + WHERE t.vendor_id = $1 + AND t.form_factor = $2 + AND NOT EXISTS ( + SELECT 1 FROM compatibility c + WHERE c.switch_id = $3 AND c.transceiver_id = t.id + )`, + [flexoptixVendorId, ff, sw.id], + ); + + for (const tx of txRows) { + if (matchedBySku.has(tx.id)) continue; // already added via search + await upsertCompat(sw.id, tx.id, "Form Factor Match", "spec_match"); + addedForSwitch.push(tx.id); + } + + if (txRows.length > 0) { + console.log(` Form-factor ${ff}: +${txRows.length} transceivers`); + } + } + } + + totalCompatAdded += addedForSwitch.length; + totalSwitchesProcessed++; + + console.log(` → Added ${addedForSwitch.length} compat entries`); + } + + console.log(`\n=== Flexoptix Compatibility Scraper Complete ===`); + console.log(` Switches processed: ${totalSwitchesProcessed}`); + console.log(` Compatibility entries added: ${totalCompatAdded}`); +} + +async function upsertCompat( + switchId: string, + transceiverId: string, + verifiedBy: string, + verificationMethod: string, +): Promise { + await pool.query( + `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, source_url) + VALUES ($1, $2, $3, $4, 'compatible', 'https://www.flexoptix.net') + ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET + verified_by = CASE + WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verified_by + ELSE compatibility.verified_by + END, + verification_method = CASE + WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verification_method + ELSE compatibility.verification_method + END`, + [switchId, transceiverId, verifiedBy, verificationMethod], + ); +} + +if (require.main === module) { + scrapeFlexoptixCompatibility() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +}