feat: Flexoptix compatibility scraper + transceiver issue scanner
- Add flexoptix-compat.ts: maps switch models to compatible Flexoptix transceivers via search API (vendor_compat) with form-factor fallback (spec_match) Scheduled daily at 09:00 UTC as scrape:compat:flexoptix - Enhance community-issues.ts: add vendor advisory sources (Cisco Field Notices, Juniper KB, SONiC GitHub Issues) + new scrapeTransceiverCompatIssues() that searches for switch+transceiver combination problems specifically - Scheduler: 59 schedules, 78 workers
This commit is contained in:
Rene Fichtmueller
parent
a0a7a97d83
commit
4bf5c95824
@ -76,6 +76,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
"scrape:vendors:flexoptix",
|
||||
"scrape:vendors:flexoptix-supported",
|
||||
// ── Compatibility (every 12h) ──────────────────────────────────────
|
||||
"scrape:compat:flexoptix",
|
||||
"scrape:compat:cisco",
|
||||
"scrape:compat:juniper",
|
||||
"scrape:compat:sonic",
|
||||
@ -210,6 +211,8 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
// COMPATIBILITY MATRICES — every 12h
|
||||
// ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
// Flexoptix compatibility — every 24h at 09:00 (after both switch-assets + image-fetcher)
|
||||
await boss.schedule("scrape:compat:flexoptix", "0 9 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||||
await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
@ -296,7 +299,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
// Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run
|
||||
await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||||
|
||||
console.log("All schedules registered — 24/7 continuous scraping (58 jobs)");
|
||||
console.log("All schedules registered — 24/7 continuous scraping (59 jobs)");
|
||||
}
|
||||
|
||||
export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
@ -319,6 +322,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
const { scrapeEdgecore } = await import("./scrapers/edgecore");
|
||||
const { scrapeSwitchAssets } = await import("./scrapers/switch-assets");
|
||||
const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher");
|
||||
const { scrapeFlexoptixCompatibility } = await import("./scrapers/flexoptix-compat");
|
||||
// ── Prediction signal scrapers ────────────────────────────────────────
|
||||
const { scrapeSecEdgar } = await import("./scrapers/sec-edgar");
|
||||
const { scrapeGithubSignals } = await import("./scrapers/github-signals");
|
||||
@ -468,6 +472,11 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
|
||||
// ── Compatibility scrapers ────────────────────────────────────────────
|
||||
|
||||
await boss.work("scrape:compat:flexoptix", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Flexoptix compatibility mapping`);
|
||||
await scrapeFlexoptixCompatibility();
|
||||
});
|
||||
|
||||
await boss.work("scrape:compat:cisco", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`);
|
||||
await scrapeCiscoTmg();
|
||||
@ -534,8 +543,9 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
|
||||
await boss.work("scrape:community-issues", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Community issues`);
|
||||
const { scrapeAllSwitchIssues } = await import("./scrapers/community-issues");
|
||||
const { scrapeAllSwitchIssues, scrapeTransceiverCompatIssues } = await import("./scrapers/community-issues");
|
||||
await scrapeAllSwitchIssues(30);
|
||||
await scrapeTransceiverCompatIssues(15);
|
||||
});
|
||||
|
||||
await boss.work("scrape:datasheet-links", async () => {
|
||||
@ -1126,5 +1136,5 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
console.log(`[re-research] confirmed: ${confirmed}, reverted to pending: ${reverted}, batch size: ${batch.rows.length}`);
|
||||
});
|
||||
|
||||
console.log("All workers registered (77 jobs, 24/7 continuous)");
|
||||
console.log("All workers registered (78 jobs, 24/7 continuous)");
|
||||
}
|
||||
|
||||
@ -77,6 +77,25 @@ const COMMUNITY_SOURCES: Array<{
|
||||
buildSearchUrl: (model) =>
|
||||
`https://networkengineering.stackexchange.com/search?q=${encodeURIComponent(model)}`,
|
||||
},
|
||||
// ── Vendor advisory / field notice pages ──────────────────────────────
|
||||
{
|
||||
name: "Cisco Field Notices",
|
||||
type: "vendor_advisory",
|
||||
buildSearchUrl: (model) =>
|
||||
`https://www.cisco.com/c/en/us/support/web/tools/psn/BT/search.html?query=${encodeURIComponent(model + " transceiver")}`,
|
||||
},
|
||||
{
|
||||
name: "Juniper KB",
|
||||
type: "vendor_kb",
|
||||
buildSearchUrl: (model) =>
|
||||
`https://supportportal.juniper.net/s/global-search/${encodeURIComponent(model + " transceiver")}?language=en_US`,
|
||||
},
|
||||
{
|
||||
name: "SONiC GitHub Issues",
|
||||
type: "github",
|
||||
buildSearchUrl: (model) =>
|
||||
`https://github.com/sonic-net/sonic-buildimage/issues?q=${encodeURIComponent(model + " transceiver")}+is%3Aissue`,
|
||||
},
|
||||
];
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -380,6 +399,40 @@ export async function findAndSeedDatasheetLinks(limit = 50): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Scrape transceiver-specific issues per switch (switch + transceiver combos)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
export async function scrapeTransceiverCompatIssues(switchLimit = 20): Promise<void> {
|
||||
// Get switches that have compatibility entries — search for their transceiver issues
|
||||
const result = await pool.query<{ switch_model: string; form_factors: string[] }>(
|
||||
`SELECT sw.model AS switch_model,
|
||||
ARRAY_AGG(DISTINCT t.form_factor) AS form_factors
|
||||
FROM switches sw
|
||||
JOIN compatibility c ON c.switch_id = sw.id
|
||||
JOIN transceivers t ON c.transceiver_id = t.id
|
||||
GROUP BY sw.model
|
||||
ORDER BY COUNT(c.id) DESC
|
||||
LIMIT $1`,
|
||||
[switchLimit],
|
||||
);
|
||||
|
||||
for (const row of result.rows) {
|
||||
const { switch_model, form_factors } = row;
|
||||
// Build composite search queries: "N9K-C9364C QSFP28 issue"
|
||||
const queries: string[] = [];
|
||||
for (const ff of (form_factors || [])) {
|
||||
queries.push(`${switch_model} ${ff} issue`);
|
||||
queries.push(`${switch_model} ${ff} not recognized`);
|
||||
queries.push(`${switch_model} transceiver incompatible`);
|
||||
}
|
||||
|
||||
await scrapeProductIssues(queries.slice(0, 4), 2);
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
|
||||
logger.info(`Transceiver compat issues scraping complete for ${result.rows.length} switches`);
|
||||
}
|
||||
|
||||
// CLI entrypoint
|
||||
if (require.main === module) {
|
||||
(async () => {
|
||||
@ -388,6 +441,8 @@ if (require.main === module) {
|
||||
await scrapeAllSwitchIssues(parseInt(process.argv[3] || "30"));
|
||||
} else if (cmd === "datasheets") {
|
||||
await findAndSeedDatasheetLinks(parseInt(process.argv[3] || "50"));
|
||||
} else if (cmd === "transceiver-compat") {
|
||||
await scrapeTransceiverCompatIssues(parseInt(process.argv[3] || "20"));
|
||||
}
|
||||
process.exit(0);
|
||||
})();
|
||||
|
||||
249
packages/scraper/src/scrapers/flexoptix-compat.ts
Normal file
249
packages/scraper/src/scrapers/flexoptix-compat.ts
Normal file
@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Flexoptix Switch Compatibility Scraper
|
||||
*
|
||||
* Populates the `compatibility` table by querying Flexoptix search/product API
|
||||
* with each switch model number. Flexoptix names compatible transceivers like
|
||||
* "Cisco N9K-C9364C compatible QSFP28 100GBASE-LR4" — so searching by switch
|
||||
* model returns all transceivers that are explicitly marked compatible.
|
||||
*
|
||||
* Strategy:
|
||||
* 1. For each switch in DB, query Flexoptix AJAX search
|
||||
* 2. Parse returned transceiver names for form-factor and vendor hints
|
||||
* 3. Match against transceivers table (Flexoptix vendor)
|
||||
* 4. Insert into compatibility with status = 'compatible'
|
||||
*
|
||||
* Fallback (when search returns nothing):
|
||||
* Form-factor matching — for each port type in switch.ports_config,
|
||||
* find Flexoptix transceivers matching that form factor.
|
||||
* Used as spec_match rather than vendor_compat — shown in dashboard
|
||||
* as "by form factor" rather than "explicitly verified".
|
||||
*
|
||||
* Rate limit: 1 req/1.5sec.
|
||||
*/
|
||||
import { pool } from "../utils/db";
|
||||
|
||||
const BASE = "https://www.flexoptix.net";
|
||||
const HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; internal-flexoptix)",
|
||||
Accept: "application/json, text/html",
|
||||
};
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
// ── Ports-config → form factor map ─────────────────────────────────────────
|
||||
|
||||
const PORT_KEY_TO_FORM_FACTOR: Array<[RegExp, string]> = [
|
||||
[/QSFP-DD/i, "QSFP-DD"],
|
||||
[/QSFP-DD800/i, "QSFP-DD800"],
|
||||
[/OSFP224/i, "OSFP224"],
|
||||
[/OSFP/i, "OSFP"],
|
||||
[/QSFP28/i, "QSFP28"],
|
||||
[/QSFP\+/i, "QSFP+"],
|
||||
[/QSFP/i, "QSFP+"],
|
||||
[/SFP28/i, "SFP28"],
|
||||
[/SFP\+/i, "SFP+"],
|
||||
[/SFP/i, "SFP"],
|
||||
[/CFP2/i, "CFP2"],
|
||||
[/CFP4/i, "CFP4"],
|
||||
[/CFP/i, "CFP"],
|
||||
];
|
||||
|
||||
function portsConfigToFormFactors(portsConfig: Record<string, number>): string[] {
|
||||
const factors = new Set<string>();
|
||||
for (const key of Object.keys(portsConfig)) {
|
||||
for (const [pattern, ff] of PORT_KEY_TO_FORM_FACTOR) {
|
||||
if (pattern.test(key)) {
|
||||
factors.add(ff);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...factors];
|
||||
}
|
||||
|
||||
// ── Flexoptix search API ────────────────────────────────────────────────────
|
||||
|
||||
interface FlexoptixSuggestion {
|
||||
name: string;
|
||||
url: string;
|
||||
sku?: string;
|
||||
image?: string;
|
||||
price?: string;
|
||||
}
|
||||
|
||||
async function searchFlexoptix(query: string): Promise<FlexoptixSuggestion[]> {
|
||||
const url = `${BASE}/en/search/ajax/suggest/?q=${encodeURIComponent(query)}`;
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: HEADERS,
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json() as unknown;
|
||||
// Magento suggest returns array or object with products key
|
||||
if (Array.isArray(data)) return data as FlexoptixSuggestion[];
|
||||
const obj = data as Record<string, unknown>;
|
||||
if (obj["products"] && Array.isArray(obj["products"])) {
|
||||
return obj["products"] as FlexoptixSuggestion[];
|
||||
}
|
||||
return [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main scraper ────────────────────────────────────────────────────────────
|
||||
|
||||
export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
console.log("=== Flexoptix Compatibility Scraper ===\n");
|
||||
|
||||
// Get all switches
|
||||
const { rows: switches } = await pool.query<{
|
||||
id: string;
|
||||
model: string;
|
||||
vendor_name: string;
|
||||
ports_config: Record<string, number> | null;
|
||||
}>(
|
||||
`SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config
|
||||
FROM switches sw
|
||||
JOIN vendors v ON v.id = sw.vendor_id
|
||||
ORDER BY sw.max_speed_gbps DESC`,
|
||||
);
|
||||
|
||||
console.log(` ${switches.length} switches to process\n`);
|
||||
|
||||
// Get Flexoptix vendor ID
|
||||
const { rows: [flexoptixVendor] } = await pool.query<{ id: string }>(
|
||||
`SELECT id FROM vendors WHERE UPPER(name) LIKE '%FLEXOPTIX%' LIMIT 1`,
|
||||
);
|
||||
|
||||
if (!flexoptixVendor) {
|
||||
console.error(" [FAIL] Flexoptix vendor not found in DB — run flexoptix-catalog scraper first");
|
||||
return;
|
||||
}
|
||||
|
||||
const flexoptixVendorId = flexoptixVendor.id;
|
||||
let totalCompatAdded = 0;
|
||||
let totalSwitchesProcessed = 0;
|
||||
|
||||
for (const sw of switches) {
|
||||
console.log(`\n[${sw.vendor_name}] ${sw.model}`);
|
||||
const addedForSwitch: string[] = [];
|
||||
|
||||
// ── Strategy 1: Search Flexoptix by switch model ──────────────────────
|
||||
await sleep(1500);
|
||||
const suggestions = await searchFlexoptix(sw.model);
|
||||
|
||||
const matchedBySku = new Set<string>();
|
||||
|
||||
if (suggestions.length > 0) {
|
||||
console.log(` Search: ${suggestions.length} results`);
|
||||
|
||||
for (const suggestion of suggestions) {
|
||||
if (!suggestion.sku && !suggestion.name) continue;
|
||||
const sku = suggestion.sku?.trim();
|
||||
const name = suggestion.name?.trim();
|
||||
if (!sku && !name) continue;
|
||||
|
||||
// Find matching transceiver in DB
|
||||
let txRow: { id: string } | undefined;
|
||||
if (sku) {
|
||||
const r = await pool.query<{ id: string }>(
|
||||
`SELECT id FROM transceivers WHERE vendor_id = $1 AND (part_number = $2 OR slug = $3) LIMIT 1`,
|
||||
[flexoptixVendorId, sku, sku.toLowerCase().replace(/[^a-z0-9]/g, "-")],
|
||||
);
|
||||
txRow = r.rows[0];
|
||||
}
|
||||
|
||||
if (!txRow && name) {
|
||||
// Try matching by name similarity to part_number
|
||||
const r = await pool.query<{ id: string }>(
|
||||
`SELECT id FROM transceivers
|
||||
WHERE vendor_id = $1
|
||||
AND (part_number ILIKE $2 OR standard_name ILIKE $3)
|
||||
LIMIT 1`,
|
||||
[flexoptixVendorId, `%${name.slice(0, 20)}%`, `%${name.slice(0, 20)}%`],
|
||||
);
|
||||
txRow = r.rows[0];
|
||||
}
|
||||
|
||||
if (txRow) {
|
||||
matchedBySku.add(txRow.id);
|
||||
await upsertCompat(sw.id, txRow.id, "Flexoptix", "vendor_compat");
|
||||
addedForSwitch.push(txRow.id);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Vendor-compat matches: ${addedForSwitch.length}`);
|
||||
}
|
||||
|
||||
// ── Strategy 2: Form-factor fallback ─────────────────────────────────
|
||||
if (sw.ports_config) {
|
||||
const formFactors = portsConfigToFormFactors(sw.ports_config);
|
||||
console.log(` Form factors: ${formFactors.join(", ")}`);
|
||||
|
||||
for (const ff of formFactors) {
|
||||
const { rows: txRows } = await pool.query<{ id: string }>(
|
||||
`SELECT t.id
|
||||
FROM transceivers t
|
||||
WHERE t.vendor_id = $1
|
||||
AND t.form_factor = $2
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM compatibility c
|
||||
WHERE c.switch_id = $3 AND c.transceiver_id = t.id
|
||||
)`,
|
||||
[flexoptixVendorId, ff, sw.id],
|
||||
);
|
||||
|
||||
for (const tx of txRows) {
|
||||
if (matchedBySku.has(tx.id)) continue; // already added via search
|
||||
await upsertCompat(sw.id, tx.id, "Form Factor Match", "spec_match");
|
||||
addedForSwitch.push(tx.id);
|
||||
}
|
||||
|
||||
if (txRows.length > 0) {
|
||||
console.log(` Form-factor ${ff}: +${txRows.length} transceivers`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
totalCompatAdded += addedForSwitch.length;
|
||||
totalSwitchesProcessed++;
|
||||
|
||||
console.log(` → Added ${addedForSwitch.length} compat entries`);
|
||||
}
|
||||
|
||||
console.log(`\n=== Flexoptix Compatibility Scraper Complete ===`);
|
||||
console.log(` Switches processed: ${totalSwitchesProcessed}`);
|
||||
console.log(` Compatibility entries added: ${totalCompatAdded}`);
|
||||
}
|
||||
|
||||
async function upsertCompat(
|
||||
switchId: string,
|
||||
transceiverId: string,
|
||||
verifiedBy: string,
|
||||
verificationMethod: string,
|
||||
): Promise<void> {
|
||||
await pool.query(
|
||||
`INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, source_url)
|
||||
VALUES ($1, $2, $3, $4, 'compatible', 'https://www.flexoptix.net')
|
||||
ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET
|
||||
verified_by = CASE
|
||||
WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verified_by
|
||||
ELSE compatibility.verified_by
|
||||
END,
|
||||
verification_method = CASE
|
||||
WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verification_method
|
||||
ELSE compatibility.verification_method
|
||||
END`,
|
||||
[switchId, transceiverId, verifiedBy, verificationMethod],
|
||||
);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
scrapeFlexoptixCompatibility()
|
||||
.then(() => pool.end())
|
||||
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user