diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 14c11f9..7d7dac8 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -5,6 +5,15 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA --- +{"d":"2026-04-01","t":"FEAT","m":"Product Intelligence Layer (migration 020): product_issues table (forum/community bugs), condition+marketplace on price_observations, features JSONB on switches+transceivers"} +{"d":"2026-04-01","t":"FEAT","m":"eBay Enricher: scrapes eBay.de for switch/transceiver listings — extracts features, description, refurbished prices, images — nightly via pg-boss"} +{"d":"2026-04-01","t":"FEAT","m":"Community Issues Scraper: extracts known bugs/incompatibilities from Reddit, ServeTheHome, Arista Community, Cisco Community, NetworkEngineering SE"} +{"d":"2026-04-01","t":"DATA","m":"7 pre-seeded community issues: Arista QSFP28 EOS compatibility, Cisco SFP DOM bug, Juniper QFX5120 config tip, SG350 SFP speed limit, MikroTik CRS326 QoS, DCS-7800R3 QSA, UniFi third-party warning"} +{"d":"2026-04-01","t":"FEAT","m":"API: GET /api/switches/:id/issues — known community issues with severity, tags, source links; GET /api/switches/:id/documents — official datasheets+manuals"} +{"d":"2026-04-01","t":"UI","m":"Switch detail modal: shows features array from DB, description, eBay refurbished price, known issues with severity color coding, datasheets with download links"} +{"d":"2026-04-01","t":"FEAT","m":"Datasheet Finder: discovers and links official PDF datasheets/manuals from vendor sites (Arista, Cisco, Juniper, HPE) for existing switches"} +{"d":"2026-04-01","t":"DATA","m":"SMB/campus switch seed: 26 models across Cisco SG/CBS 350/550/CBS350, HPE Aruba 1820/2530/2930F, Ubiquiti UniFi Pro/Aggregation, MikroTik CRS326/354/504, Netgear M4300/M4500, Zyxel XGS"} +{"d":"2026-04-01","t":"FIX","m":"forecast.ts: fixed fiveYearProjection accessor (hype.forecast.fiveYearProjection[n] instead of hype.forecast[n])"} {"d":"2026-04-01","t":"FEAT","m":"Procurement Intelligence Engine (WS0c): stock_snapshots, abc_classification, reorder_signals, product_lifecycle_events, market_intelligence tables"} {"d":"2026-04-01","t":"FEAT","m":"Crawler LLM: Ollama-based two-stage extractor (page type detection + structured product extraction) with vendor profiles for 7 vendors"} {"d":"2026-04-01","t":"FEAT","m":"ABC classification: dynamic A/B/C turnover scoring from price observations, compatibility breadth, vendor count — computed daily"} diff --git a/packages/api/src/db/queries.ts b/packages/api/src/db/queries.ts index 2f657eb..6def7d8 100644 --- a/packages/api/src/db/queries.ts +++ b/packages/api/src/db/queries.ts @@ -228,16 +228,54 @@ export async function getSwitchById(id: string) { export async function getSwitchDocuments(switchId: string) { const result = await pool.query( - `SELECT pd.*, v.name as vendor_name - FROM product_documents pd - LEFT JOIN vendors v ON pd.vendor_id = v.id - WHERE pd.switch_id = $1 - ORDER BY pd.doc_type, pd.title`, + `SELECT id, doc_type, title, source_url, download_url, language, version, is_official, vendor_doc_id, page_count, file_size_bytes, r2_key, created_at + FROM product_documents + WHERE switch_id = $1 + ORDER BY is_official DESC, doc_type, title`, [switchId] ); return result.rows; } +export async function getSwitchIssues(switchId: string) { + const result = await pool.query( + `SELECT id, source_type, source_name, source_url, title, summary, severity, + issue_tags, affected_firmware, fix_firmware, date_reported, is_resolved, confidence + FROM product_issues + WHERE switch_id = $1 + ORDER BY + CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END, + date_reported DESC NULLS LAST`, + [switchId] + ); + return result.rows; +} + +export async function getTransceiverIssues(transceiverI: string) { + const result = await pool.query( + `SELECT id, source_type, source_name, source_url, title, summary, severity, + issue_tags, affected_firmware, fix_firmware, date_reported, is_resolved, confidence + FROM product_issues + WHERE transceiver_id = $1 + ORDER BY + CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END, + date_reported DESC NULLS LAST`, + [transceiverI] + ); + return result.rows; +} + +export async function getTransceiverDocuments(transceiverI: string) { + const result = await pool.query( + `SELECT id, doc_type, title, source_url, download_url, language, version, is_official, vendor_doc_id, page_count, file_size_bytes, r2_key, created_at + FROM product_documents + WHERE transceiver_id = $1 + ORDER BY is_official DESC, doc_type, title`, + [transceiverI] + ); + return result.rows; +} + export async function getCompatibleTransceivers(switchId: string) { const result = await pool.query( `SELECT t.*, c.status, c.verified_by, c.notes as compat_notes, diff --git a/packages/api/src/routes/switches.ts b/packages/api/src/routes/switches.ts index f6c3d3f..d481f62 100644 --- a/packages/api/src/routes/switches.ts +++ b/packages/api/src/routes/switches.ts @@ -1,5 +1,5 @@ import { Router, Request, Response } from "express"; -import { searchSwitches, getSwitchById, getCompatibleTransceivers, getSwitchDocuments } from "../db/queries"; +import { searchSwitches, getSwitchById, getCompatibleTransceivers, getSwitchDocuments, getSwitchIssues } from "../db/queries"; export const switchRouter = Router(); @@ -52,6 +52,17 @@ switchRouter.get("/:id/documents", async (req: Request, res: Response) => { } }); +// GET /api/switches/:id/issues — Known bugs, incompatibilities from community sources +switchRouter.get("/:id/issues", async (req: Request, res: Response) => { + try { + const issues = await getSwitchIssues(String(req.params.id)); + res.json({ success: true, data: issues, total: issues.length }); + } catch (err) { + console.error("Get switch issues error:", err); + res.status(500).json({ success: false, error: "Internal server error" }); + } +}); + // GET /api/switches/:id/compatibility — Compatible transceivers for a switch switchRouter.get("/:id/compatibility", async (req: Request, res: Response) => { try { diff --git a/packages/dashboard/index.html b/packages/dashboard/index.html index 7fcc92a..b6ec5ca 100644 --- a/packages/dashboard/index.html +++ b/packages/dashboard/index.html @@ -2465,16 +2465,34 @@ async function openSwitchDetail(id) { h += '
Features
'; var features = []; - if (s.vxlan_support) features.push('VXLAN'); - if (s.evpn_support) features.push('EVPN'); - if (s.bgp_support) features.push('BGP'); - if (s.mpls_support) features.push('MPLS'); - if (s.openconfig_support) features.push('OpenConfig'); - if (s.sonic_compatible) features.push('SONiC'); - if (s.macsec_support) features.push('MACsec'); - if (s.stacking_support) features.push('Stacking'); + // Use JSONB features array from DB if populated, fall back to boolean flags + if (s.features && Array.isArray(s.features) && s.features.length > 0) { + features = s.features; + } else { + if (s.vxlan_support) features.push('VXLAN'); + if (s.evpn_support) features.push('EVPN'); + if (s.bgp_support) features.push('BGP'); + if (s.mpls_support) features.push('MPLS'); + if (s.openconfig_support) features.push('OpenConfig'); + if (s.sonic_compatible) features.push('SONiC'); + if (s.macsec_support) features.push('MACsec'); + if (s.stacking_support) features.push('Stacking'); + } h += '
' + features.map(function(f) { return '' + esc(f) + ''; }).join('') + (features.length === 0 ? 'None listed' : '') + '
'; + // Description + if (s.description) { + h += '
Description
'; + h += '
' + esc(s.description) + '
'; + } + + // eBay Refurbished Price + if (s.ebay_refurb_price_usd) { + h += '
Market Pricing
'; + h += '
Refurbished (eBay)€' + parseFloat(s.ebay_refurb_price_usd).toFixed(0) + ' (incl. warranty, market)
'; + if (s.msrp_usd) h += '
List Price (MSRP)$' + parseFloat(s.msrp_usd).toFixed(0) + '
'; + } + if (s.ports_config && Object.keys(s.ports_config).length > 0) { h += '
Port Configuration
'; Object.keys(s.ports_config).forEach(function(k) { @@ -2482,6 +2500,14 @@ async function openSwitchDetail(id) { }); } + // Known Issues placeholder (loaded async) + h += ''; + h += '
'; + + // Documents placeholder (loaded async) + h += ''; + h += '
'; + var links = []; if (s.product_page_url) links.push('Product Page'); if (s.datasheet_url) links.push('Datasheet'); @@ -2516,6 +2542,72 @@ async function openSwitchDetail(id) { buildDOM(el('panel-content'), h); + // Async: load known issues + api('/api/switches/' + id + '/issues').then(function(idata) { + var issues = idata.data || []; + if (issues.length === 0) return; + var hdr = document.getElementById('sw-issues-hdr-' + id); + var cnt = document.getElementById('sw-issues-cnt-' + id); + var body = document.getElementById('sw-issues-body-' + id); + if (!hdr || !body) return; + hdr.style.display = ''; + if (cnt) cnt.textContent = issues.length; + var ih = ''; + var severityColors = { critical: '#ff4d4d', warning: '#f59e0b', info: '#6b7280' }; + var severityIcons = { critical: '🔴', warning: '⚠️', info: 'ℹ️' }; + issues.forEach(function(issue) { + var col = severityColors[issue.severity] || '#6b7280'; + var icon = severityIcons[issue.severity] || 'ℹ️'; + ih += '
'; + ih += '
' + icon + ' ' + esc(issue.title) + '
'; + if (issue.summary) ih += '
' + esc(issue.summary) + '
'; + var meta = []; + if (issue.source_name) meta.push('' + esc(issue.source_name) + ' ↗'); + if (issue.affected_firmware) meta.push('Affects: ' + esc(issue.affected_firmware) + ''); + if (issue.fix_firmware) meta.push('Fixed in: ' + esc(issue.fix_firmware) + ''); + if (issue.is_resolved) meta.push('✓ Resolved'); + if (issue.issue_tags && issue.issue_tags.length) { + issue.issue_tags.forEach(function(tag) { meta.push('' + esc(tag) + ''); }); + } + if (meta.length) ih += '
' + meta.join('') + '
'; + ih += '
'; + }); + buildDOM(body, ih); + }).catch(function() {}); + + // Async: load datasheets & manuals + api('/api/switches/' + id + '/documents').then(function(ddata) { + var docs = ddata.data || []; + if (docs.length === 0) return; + var hdr = document.getElementById('sw-docs-hdr-' + id); + var cnt = document.getElementById('sw-docs-cnt-' + id); + var body = document.getElementById('sw-docs-body-' + id); + if (!hdr || !body) return; + hdr.style.display = ''; + if (cnt) cnt.textContent = docs.length; + var docTypeLabels = { datasheet: '📄 Datasheet', user_guide: '📖 Manual', release_notes: '📋 Release Notes', app_note: '📝 App Note', product_page: '🌐 Product Page', config_guide: '⚙️ Config Guide' }; + var docTypeColors = { datasheet: '#ff6600', user_guide: '#4287f5', release_notes: '#22c55e', app_note: '#f59e0b', product_page: '#8b5cf6', config_guide: '#06b6d4' }; + var dh = '
'; + docs.forEach(function(doc) { + var url = doc.download_url || doc.source_url; + var label = docTypeLabels[doc.doc_type] || doc.doc_type; + var col = docTypeColors[doc.doc_type] || 'var(--accent)'; + var officialBadge = doc.is_official ? 'Official' : ''; + var langBadge = doc.language && doc.language !== 'en' ? '' + esc(doc.language.toUpperCase()) + '' : ''; + dh += '
'; + if (url) { + dh += '' + label + ''; + } else { + dh += '' + label + ''; + } + dh += '' + esc(doc.title) + ''; + dh += officialBadge + langBadge; + dh += '
'; + }); + dh += '
'; + buildDOM(body, dh); + }).catch(function() {}); + api('/api/switches/' + id + '/compatibility').then(function(cdata) { var txList = cdata.data || cdata.transceivers || []; if (txList.length === 0) return; diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 3b90a69..c93e736 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -70,6 +70,10 @@ export async function registerSchedules(boss: PgBoss): Promise { "scrape:market-intel", "compute:abc", "compute:reorder-signals", + "enrich:ebay-switches", + "enrich:ebay-transceivers", + "scrape:community-issues", + "scrape:datasheet-links", ]; for (const q of queues) { await boss.createQueue(q).catch(() => { /* already exists */ }); @@ -161,6 +165,30 @@ export async function registerSchedules(boss: PgBoss): Promise { expireInSeconds: 600, }); + // eBay switch enrichment: features, descriptions, refurb prices (nightly at 1am) + await boss.schedule("enrich:ebay-switches", "0 1 * * *", {}, { + retryLimit: 1, + expireInSeconds: 7200, + }); + + // eBay transceiver pricing with condition (nightly at 2am) + await boss.schedule("enrich:ebay-transceivers", "0 2 * * *", {}, { + retryLimit: 1, + expireInSeconds: 7200, + }); + + // Community issues scraping: Reddit/forums for known bugs (weekly on Sunday 4am) + await boss.schedule("scrape:community-issues", "0 4 * * 0", {}, { + retryLimit: 1, + expireInSeconds: 3600, + }); + + // Datasheet link discovery (weekly on Monday 6am) + await boss.schedule("scrape:datasheet-links", "0 6 * * 1", {}, { + retryLimit: 1, + expireInSeconds: 3600, + }); + console.log("All schedules registered"); } @@ -247,5 +275,29 @@ export async function registerWorkers(boss: PgBoss): Promise { await computeReorderSignals(); }); + await boss.work("enrich:ebay-switches", async (_job) => { + console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`); + const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher"); + await withIsolatedStorage("ebay-switches", () => enrichSwitchesFromEbay(30)); + }); + + await boss.work("enrich:ebay-transceivers", async (_job) => { + console.log(`[${new Date().toISOString()}] Running: eBay transceiver pricing`); + const { enrichTransceiversFromEbay } = await import("./scrapers/ebay-enricher"); + await withIsolatedStorage("ebay-transceivers", () => enrichTransceiversFromEbay(100)); + }); + + await boss.work("scrape:community-issues", async (_job) => { + console.log(`[${new Date().toISOString()}] Running: Community issues scraping`); + const { scrapeAllSwitchIssues } = await import("./scrapers/community-issues"); + await withIsolatedStorage("community-issues", () => scrapeAllSwitchIssues(30)); + }); + + await boss.work("scrape:datasheet-links", async (_job) => { + console.log(`[${new Date().toISOString()}] Running: Datasheet link discovery`); + const { findAndSeedDatasheetLinks } = await import("./scrapers/community-issues"); + await findAndSeedDatasheetLinks(50); + }); + console.log("All workers registered"); } diff --git a/packages/scraper/src/scrapers/community-issues.ts b/packages/scraper/src/scrapers/community-issues.ts new file mode 100644 index 0000000..a88a3fb --- /dev/null +++ b/packages/scraper/src/scrapers/community-issues.ts @@ -0,0 +1,391 @@ +/** + * Community Issues Scraper + * + * Scrapes known issues, bugs, incompatibilities from: + * - Reddit r/networking, r/homelab, r/sysadmin + * - ServeTheHome forums + * - Arista Community / EOS Central + * - Cisco Community + * - Juniper Community + * - NetworkEngineering StackExchange + * - GitHub Issues (for SONiC, OpenConfig, etc.) + * + * Uses Crawler LLM to extract structured issue data. + */ + +import { CheerioCrawler, RequestQueue } from "crawlee"; +import { extractMarketIntel } from "../crawler-llm/core"; +import { db as pool } from "../utils/db"; +import { logger } from "../utils/logger"; + +interface ExtractedIssue { + productModel: string; + title: string; + summary: string; + severity: "info" | "warning" | "critical"; + issueTags: string[]; + affectedFirmware: string | null; + fixFirmware: string | null; + dateReported: string | null; + isResolved: boolean; + confidence: number; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Search URL builders per source +// ───────────────────────────────────────────────────────────────────────────── + +const COMMUNITY_SOURCES: Array<{ + name: string; + type: string; + buildSearchUrl: (model: string) => string; +}> = [ + { + name: "Reddit r/networking", + type: "reddit", + buildSearchUrl: (model) => + `https://www.reddit.com/r/networking/search/?q=${encodeURIComponent(model + " issue")}&sort=relevance&t=all`, + }, + { + name: "Reddit r/homelab", + type: "reddit", + buildSearchUrl: (model) => + `https://www.reddit.com/r/homelab/search/?q=${encodeURIComponent(model)}&sort=relevance&t=all`, + }, + { + name: "ServeTheHome", + type: "forum", + buildSearchUrl: (model) => + `https://forums.servethehome.com/index.php?search/1/?q=${encodeURIComponent(model)}&t=post&c[users]=&o=date`, + }, + { + name: "Arista Community", + type: "vendor_kb", + buildSearchUrl: (model) => + `https://eos.arista.com/?s=${encodeURIComponent(model)}`, + }, + { + name: "Cisco Community", + type: "vendor_kb", + buildSearchUrl: (model) => + `https://community.cisco.com/t5/forums/searchpage/tab/message?q=${encodeURIComponent(model + " transceiver issue")}&collapse_discussion=true`, + }, + { + name: "NetworkEngineering SE", + type: "forum", + buildSearchUrl: (model) => + `https://networkengineering.stackexchange.com/search?q=${encodeURIComponent(model)}`, + }, +]; + +// ───────────────────────────────────────────────────────────────────────────── +// Determine severity from extracted intel +// ───────────────────────────────────────────────────────────────────────────── +function determineSeverity(text: string): "info" | "warning" | "critical" { + const lower = text.toLowerCase(); + if ( + lower.includes("security") || + lower.includes("vulnerability") || + lower.includes("cve") || + lower.includes("crash") || + lower.includes("data loss") || + lower.includes("critical") + ) return "critical"; + + if ( + lower.includes("not working") || + lower.includes("incompatib") || + lower.includes("failure") || + lower.includes("not recognized") || + lower.includes("port down") || + lower.includes("bug") || + lower.includes("workaround") + ) return "warning"; + + return "info"; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Extract issue tags from text +// ───────────────────────────────────────────────────────────────────────────── +function extractIssueTags(text: string): string[] { + const lower = text.toLowerCase(); + const tags: string[] = []; + const tagMap: [RegExp, string][] = [ + [/firmware|eos|junos|nxos|iosxe/, "firmware"], + [/interop|compatibility|third.party/, "interop"], + [/macsec|encryption|security/, "macsec"], + [/temperature|thermal|overheating/, "thermal"], + [/dom|digital optical|ddm/, "dom"], + [/breakout|split|qsa|adapter/, "breakout"], + [/sfp\+?|qsfp|osfp|cfp/, "transceiver"], + [/vxlan|evpn|bgp|ospf/, "routing"], + [/poe|power/, "poe"], + [/stacking|lag|lacp/, "stacking"], + [/memory|buffer|overflow/, "memory"], + [/driver|module|kernel/, "driver"], + [/snmp|telemetry|monitoring/, "monitoring"], + [/latency|performance|throughput/, "performance"], + ]; + for (const [pattern, tag] of tagMap) { + if (pattern.test(lower)) tags.push(tag); + } + return [...new Set(tags)]; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Save extracted issues to DB +// ───────────────────────────────────────────────────────────────────────────── +async function saveIssue( + issue: ExtractedIssue, + sourceUrl: string, + sourceName: string, + sourceType: string +): Promise { + // Find product ID + const switchResult = await pool.query( + `SELECT id FROM switches WHERE model ILIKE $1 OR model ILIKE '%' || $2 || '%' LIMIT 1`, + [issue.productModel, issue.productModel.split("-")[0]] + ); + const transceiverResult = await pool.query( + `SELECT id FROM transceivers WHERE part_number ILIKE $1 OR slug ILIKE $2 LIMIT 1`, + [issue.productModel, issue.productModel.toLowerCase().replace(/\s+/g, "-")] + ); + + const switchId = switchResult.rows[0]?.id || null; + const transceiverI = transceiverResult.rows[0]?.id || null; + + if (!switchId && !transceiverI) { + // Unknown product — still store with model name for future lookup + logger.debug(`Issue for unknown product: ${issue.productModel}`); + } + + await pool.query( + `INSERT INTO product_issues ( + switch_id, transceiver_id, product_model, + source_type, source_name, source_url, + title, summary, severity, issue_tags, + affected_firmware, fix_firmware, + date_reported, is_resolved, confidence + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) + ON CONFLICT DO NOTHING`, + [ + switchId, transceiverI, issue.productModel, + sourceType, sourceName, sourceUrl, + issue.title, issue.summary, issue.severity, issue.issueTags, + issue.affectedFirmware, issue.fixFirmware, + issue.dateReported, issue.isResolved, issue.confidence, + ] + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Main: scrape community issues for given switch/transceiver models +// ───────────────────────────────────────────────────────────────────────────── +export async function scrapeProductIssues( + models: string[], + sourceLimit = 3 +): Promise { + const queue = await RequestQueue.open("community-issues"); + + // Add search requests for each model × source combination + for (const model of models) { + const sources = COMMUNITY_SOURCES.slice(0, sourceLimit); + for (const source of sources) { + await queue.addRequest({ + url: source.buildSearchUrl(model), + userData: { model, sourceName: source.name, sourceType: source.type }, + uniqueKey: `${source.name}-${model}`, + }); + } + } + + const crawler = new CheerioCrawler({ + requestQueue: queue, + maxConcurrency: 2, + requestHandlerTimeoutSecs: 30, + navigationTimeoutSecs: 20, + + async requestHandler({ request, $, response }) { + const { model, sourceName, sourceType } = request.userData as { + model: string; sourceName: string; sourceType: string; + }; + + // Extract text content for LLM analysis + // Remove nav, scripts, ads for cleaner input + $("nav, script, style, .ad, #sidebar, footer, header").remove(); + const pageText = $("body").text().replace(/\s+/g, " ").substring(0, 8000); + + if (pageText.length < 100) return; + + // Use Crawler LLM market intel extractor to find issues + const prompt = `You are analyzing a networking community forum/search results page. +Find any reports of problems, bugs, incompatibilities, or issues specifically about the networking device "${model}". + +For each issue found, extract: +- title: brief description of the issue +- summary: 1-2 sentence explanation including cause and workaround if mentioned +- severity: "critical" (security/crash/data loss), "warning" (functional problem, workaround needed), or "info" (minor/cosmetic) +- affectedFirmware: firmware version where issue occurs (or null) +- fixFirmware: firmware version where it's fixed (or null) +- isResolved: true/false +- tags: array of relevant tags from: firmware, interop, thermal, dom, breakout, performance, security, config + +Page text: ${pageText} + +Return valid JSON array: [{"title":"...","summary":"...","severity":"...","affectedFirmware":null,"fixFirmware":null,"isResolved":false,"tags":[]}] +If no issues found, return []`; + + try { + const intelResult = await extractMarketIntel(pageText, request.url, sourceName); + + if (intelResult && intelResult.title) { + const issue: ExtractedIssue = { + productModel: model, + title: intelResult.title.substring(0, 200), + summary: intelResult.description?.substring(0, 500) || "", + severity: determineSeverity(intelResult.description || intelResult.title), + issueTags: extractIssueTags(`${intelResult.title} ${intelResult.description}`), + affectedFirmware: null, + fixFirmware: null, + dateReported: intelResult.publishedDate || null, + isResolved: false, + confidence: intelResult.confidence || 0.6, + }; + + await saveIssue(issue, request.url, sourceName, sourceType); + logger.info(`Issue saved: ${model} — ${issue.title.substring(0, 60)}`); + } + } catch (err) { + logger.warn(`Issue extraction failed for ${model} from ${sourceName}`, { err }); + } + }, + + failedRequestHandler: ({ request, error }) => { + logger.warn(`Community scraper failed: ${request.url}`, { error }); + }, + }); + + await crawler.run(); + logger.info(`Community issues scraping complete for ${models.length} models`); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scrape issues for all switches in DB +// ───────────────────────────────────────────────────────────────────────────── +export async function scrapeAllSwitchIssues(limit = 30): Promise { + const result = await pool.query<{ model: string }>( + `SELECT sw.model FROM switches sw + WHERE NOT EXISTS ( + SELECT 1 FROM product_issues pi WHERE pi.product_model = sw.model + ) + ORDER BY sw.max_speed_gbps DESC + LIMIT $1`, + [limit] + ); + + const models = result.rows.map(r => r.model); + if (models.length === 0) { + logger.info("All switches already have issue data"); + return; + } + + logger.info(`Scraping community issues for ${models.length} switches`); + await scrapeProductIssues(models, 2); // 2 sources per switch to avoid rate limits +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scrape datasheet links for switches/transceivers +// ───────────────────────────────────────────────────────────────────────────── + +interface DatasheetSource { + vendor: string; + pattern: (model: string) => string | null; +} + +const DATASHEET_SOURCES: DatasheetSource[] = [ + { + vendor: "Arista", + pattern: (model) => { + const series = model.match(/DCS-(\d+)/)?.[1]; + if (!series) return null; + return `https://www.arista.com/en/products/fixedconfiguration/${series.toLowerCase()}`; + }, + }, + { + vendor: "Cisco", + pattern: (model) => { + const lower = model.toLowerCase().replace(/\s+/g, "-"); + return `https://www.cisco.com/c/en/us/products/collateral/switches/search.html?q=${encodeURIComponent(model)}`; + }, + }, + { + vendor: "Juniper", + pattern: (model) => { + const series = model.split("-")[0]?.toLowerCase(); + if (!series) return null; + return `https://www.juniper.net/documentation/product/${series}.html`; + }, + }, + { + vendor: "HPE Aruba", + pattern: (model) => + `https://h20195.www2.hpe.com/v2/getpdf.aspx/a00${model.replace(/[^a-z0-9]/gi, "").toLowerCase()}.pdf`, + }, +]; + +export async function findAndSeedDatasheetLinks(limit = 50): Promise { + const result = await pool.query<{ id: string; model: string; vendor_name: string }>( + `SELECT sw.id, sw.model, v.name AS vendor_name + FROM switches sw + JOIN vendors v ON sw.vendor_id = v.id + WHERE NOT EXISTS ( + SELECT 1 FROM product_documents pd WHERE pd.switch_id = sw.id + ) + LIMIT $1`, + [limit] + ); + + for (const sw of result.rows) { + for (const source of DATASHEET_SOURCES) { + if (!sw.vendor_name.toLowerCase().includes(source.vendor.toLowerCase())) continue; + + const url = source.pattern(sw.model); + if (!url) continue; + + try { + // Check if URL is accessible (simple HEAD request) + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + const resp = await fetch(url, { method: "HEAD", signal: controller.signal }); + clearTimeout(timeout); + + if (resp.ok) { + const docType = url.includes(".pdf") ? "datasheet" : "product_page"; + await pool.query( + `INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language) + VALUES ($1, $2, $3, $4, TRUE, 'en') + ON CONFLICT DO NOTHING`, + [sw.id, docType, `${sw.vendor_name} ${sw.model} ${docType.replace("_", " ")}`, url] + ); + logger.info(`✓ Doc linked: ${sw.model} → ${url}`); + } + } catch { + // URL not accessible — skip silently + } + } + } +} + +// CLI entrypoint +if (require.main === module) { + (async () => { + const cmd = process.argv[2] || "issues"; + if (cmd === "issues") { + await scrapeAllSwitchIssues(parseInt(process.argv[3] || "30")); + } else if (cmd === "datasheets") { + await findAndSeedDatasheetLinks(parseInt(process.argv[3] || "50")); + } + process.exit(0); + })(); +} diff --git a/packages/scraper/src/scrapers/ebay-enricher.ts b/packages/scraper/src/scrapers/ebay-enricher.ts new file mode 100644 index 0000000..99999c4 --- /dev/null +++ b/packages/scraper/src/scrapers/ebay-enricher.ts @@ -0,0 +1,450 @@ +/** + * eBay Product Enricher + * + * Searches eBay for switch/transceiver models to extract: + * - Product description & features + * - Refurbished/used prices + * - Product images + * - Technical specs from listing descriptions + * + * Uses CheerioCrawler + Crawler LLM for structured extraction. + */ + +import { CheerioCrawler, RequestQueue } from "crawlee"; +import { scrapeWithLLM } from "../crawler-llm/core"; +import { db } from "../utils/db"; +import { logger } from "../utils/logger"; + +interface EbayListing { + title: string; + price: number; + currency: string; + condition: "new" | "refurbished" | "used"; + seller: string; + warrantyMonths: number | null; + imageUrl: string | null; + listingUrl: string; + itemId: string; + description: string; + features: string[]; + specs: Record; +} + +interface EnrichResult { + model: string; + listings: EbayListing[]; + bestRefurbPrice: number | null; + bestNewPrice: number | null; + features: string[]; + description: string; + imageUrl: string | null; +} + +// eBay search URL for .de (EUR pricing, covers DE/EU market) +function buildSearchUrl(query: string, page = 1): string { + const encoded = encodeURIComponent(query); + const offset = (page - 1) * 50; + return `https://www.ebay.de/sch/i.html?_nkw=${encoded}&_sop=15&LH_ItemCondition=3000%7C1500%7C1000&_ipg=50&_pgn=${page}&_stpos=0&_from=R40`; +} + +// Parse eBay condition string to our condition type +function parseCondition(condStr: string): "new" | "refurbished" | "used" { + const lower = condStr.toLowerCase(); + if (lower.includes("neu") || lower.includes("new")) return "new"; + if (lower.includes("refurb") || lower.includes("überholt") || lower.includes("generalüber")) return "refurbished"; + return "used"; +} + +// Extract warranty months from listing title/description +function extractWarranty(text: string): number | null { + const patterns = [ + /(\d+)\s*[-–]?\s*month\s*warrant/i, + /(\d+)\s*[-–]?\s*monat\s*gewähr/i, + /(\d+)\s*[-–]?\s*year\s*warrant/i, + /(\d+)\s*[-–]?\s*jahr\s*gewähr/i, + ]; + for (const pattern of patterns) { + const match = text.match(pattern); + if (match && match[1]) { + const num = parseInt(match[1]); + return pattern.source.includes("year") || pattern.source.includes("jahr") ? num * 12 : num; + } + } + return null; +} + +async function parseSearchResults($: cheerio.CheerioAPI, baseUrl: string): Promise> { + const items: Array<{ title: string; url: string; price: string; condition: string; imageUrl: string }> = []; + + $(".s-item").each((_, el) => { + const titleEl = $(el).find(".s-item__title"); + const priceEl = $(el).find(".s-item__price"); + const condEl = $(el).find(".SECONDARY_INFO"); + const linkEl = $(el).find(".s-item__link"); + const imgEl = $(el).find(".s-item__image-img"); + + const title = titleEl.text().trim(); + const price = priceEl.text().trim(); + const condition = condEl.text().trim(); + const url = linkEl.attr("href") || ""; + const imageUrl = imgEl.attr("src") || imgEl.attr("data-src") || ""; + + if (title && url && !title.toLowerCase().includes("shop on ebay")) { + items.push({ title, url, price, condition, imageUrl }); + } + }); + + return items; +} + +async function enrichSwitchFromEbay(switchId: string, model: string): Promise { + const result: EnrichResult = { + model, + listings: [], + bestRefurbPrice: null, + bestNewPrice: null, + features: [], + description: "", + imageUrl: null, + }; + + const queue = await RequestQueue.open(`ebay-${switchId.substring(0, 8)}`); + await queue.addRequest({ url: buildSearchUrl(model), userData: { model, phase: "search" } }); + + const crawler = new CheerioCrawler({ + requestQueue: queue, + maxRequestsPerCrawl: 5, + requestHandlerTimeoutSecs: 30, + + async requestHandler({ request, $, crawler }) { + const { phase, model } = request.userData as { phase: string; model: string }; + + if (phase === "search") { + const items = await parseSearchResults($, request.url); + + // Take up to 3 most relevant listings + const relevant = items.filter(item => + item.title.toLowerCase().includes(model.toLowerCase().split("-")[0]?.toLowerCase() ?? "") + ).slice(0, 3); + + for (const item of relevant) { + if (item.url && item.url.startsWith("http")) { + await crawler.addRequests([{ + url: item.url.split("?")[0]!, + userData: { + phase: "listing", + model, + priceStr: item.price, + conditionStr: item.condition, + imageUrl: item.imageUrl, + title: item.title, + }, + }]); + } + } + } else if (phase === "listing") { + const { title, priceStr, conditionStr, imageUrl: searchImageUrl, model } = request.userData as { + title: string; priceStr: string; conditionStr: string; imageUrl: string; model: string; + }; + + // Use Crawler LLM to extract structured data from listing page + const html = $.html(); + const extracted = await scrapeWithLLM(html, request.url, { + vendorSlug: "ebay", + extractType: "stock", + }); + + // Parse price from string (handle EUR format "1.234,56 EUR") + const priceClean = priceStr.replace(/[^\d,.-]/g, "").replace(".", "").replace(",", "."); + const price = parseFloat(priceClean) || 0; + + const condition = parseCondition(conditionStr); + const warranty = extractWarranty(title); + + // Extract image from listing page (higher quality than search thumbnail) + const listingImage = $(".ux-image-carousel-item img").first().attr("src") + || $(".img img").first().attr("src") + || searchImageUrl; + + // Extract features from item specifics table + const features: string[] = []; + $(".ux-labels-values").each((_, el) => { + const label = $(el).find(".ux-labels-values__labels").text().trim(); + const value = $(el).find(".ux-labels-values__values").text().trim(); + if (label && value && value !== "Siehe Anzeige") { + features.push(`${label}: ${value}`); + } + }); + + // Extract description + const description = extracted?.description + || $(".ux-textspans--BOLD").first().text().trim() + || ""; + + const listing: EbayListing = { + title, + price, + currency: "EUR", + condition, + seller: $(".ux-seller-section__item--seller a").text().trim() || "unknown", + warrantyMonths: warranty, + imageUrl: listingImage || null, + listingUrl: request.url, + itemId: request.url.match(/\/itm\/(\d+)/)?.[1] || "", + description, + features, + specs: {}, + }; + + result.listings.push(listing); + + // Track best prices + if (price > 0) { + if (condition === "refurbished" || condition === "used") { + if (!result.bestRefurbPrice || price < result.bestRefurbPrice) { + result.bestRefurbPrice = price; + } + } else if (condition === "new") { + if (!result.bestNewPrice || price < result.bestNewPrice) { + result.bestNewPrice = price; + } + } + } + + // Collect features for switch enrichment + if (features.length > 0 && result.features.length === 0) { + result.features = features.slice(0, 10); + } + + // Use best image + if (!result.imageUrl && listingImage) { + result.imageUrl = listingImage; + } + + // Use first good description + if (!result.description && description.length > 50) { + result.description = description.substring(0, 500); + } + } + }, + failedRequestHandler: ({ request, error }) => { + logger.warn(`eBay enricher failed for ${request.url}: ${error}`); + }, + }); + + try { + await crawler.run(); + } catch (err) { + logger.error("eBay crawler run error", { err, model }); + } + + return result.listings.length > 0 ? result : null; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Save enrichment results to DB +// ───────────────────────────────────────────────────────────────────────────── +async function saveEnrichment(switchId: string, result: EnrichResult): Promise { + const { db: pool } = await import("../utils/db"); + + // Update switch: features, description, refurb price, image + const updateFields: string[] = ["ebay_enriched_at = NOW()"]; + const params: unknown[] = []; + let idx = 1; + + if (result.features.length > 0) { + updateFields.push(`features = $${idx}::jsonb`); + params.push(JSON.stringify(result.features)); + idx++; + } + + if (result.description) { + updateFields.push(`description = COALESCE(description, $${idx})`); + params.push(result.description); + idx++; + } + + if (result.bestRefurbPrice) { + updateFields.push(`ebay_refurb_price_usd = $${idx}`); + params.push(result.bestRefurbPrice); + idx++; + } + + if (result.imageUrl && result.imageUrl.startsWith("http")) { + // Only set image_url if not already set + updateFields.push(`image_url = COALESCE(NULLIF(image_url, ''), $${idx})`); + params.push(result.imageUrl); + idx++; + } + + params.push(switchId); + await pool.query( + `UPDATE switches SET ${updateFields.join(", ")} WHERE id = $${idx}`, + params + ); + + // Find eBay vendor ID (create if needed) + const ebayVendorResult = await pool.query( + `INSERT INTO vendors (name, slug, type, website_url) + VALUES ('eBay Marketplace', 'ebay', 'marketplace', 'https://www.ebay.de') + ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name + RETURNING id` + ); + const ebayVendorId = ebayVendorResult.rows[0]?.id; + if (!ebayVendorId) return; + + // For each listing that has a price, we need a transceiver_id or we skip + // (price_observations requires transceiver_id — for switches we'll use a different approach later) + // For now, just log the refurb price data + logger.info("eBay enrichment saved", { + model: result.model, + listingsCount: result.listings.length, + bestRefurb: result.bestRefurbPrice, + featuresCount: result.features.length, + hasImage: !!result.imageUrl, + }); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Main: enrich switches that haven't been enriched yet +// ───────────────────────────────────────────────────────────────────────────── +export async function enrichSwitchesFromEbay(limit = 20): Promise { + const { db: pool } = await import("../utils/db"); + + const switches = await pool.query<{ id: string; model: string; vendor_name: string }>( + `SELECT sw.id, sw.model, v.name AS vendor_name + FROM switches sw + JOIN vendors v ON sw.vendor_id = v.id + WHERE sw.ebay_enriched_at IS NULL + AND sw.max_speed_gbps >= 10 + ORDER BY sw.max_speed_gbps DESC, sw.created_at ASC + LIMIT $1`, + [limit] + ); + + logger.info(`eBay enricher: processing ${switches.rows.length} switches`); + + for (const sw of switches.rows) { + logger.info(`Enriching ${sw.model} from eBay...`); + try { + const result = await enrichSwitchFromEbay(sw.id, sw.model); + if (result) { + await saveEnrichment(sw.id, result); + logger.info(`✓ ${sw.model}: ${result.listings.length} listings, refurb €${result.bestRefurbPrice}`); + } else { + // Mark as tried even if no results + await pool.query("UPDATE switches SET ebay_enriched_at = NOW() WHERE id = $1", [sw.id]); + logger.info(`○ ${sw.model}: no eBay listings found`); + } + } catch (err) { + logger.error(`✗ ${sw.model}: enrichment failed`, { err }); + } + + // Rate limiting — be polite to eBay + await new Promise(r => setTimeout(r, 3000 + Math.random() * 2000)); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Enrich transceivers from eBay (price observations with condition) +// ───────────────────────────────────────────────────────────────────────────── +export async function enrichTransceiversFromEbay(limit = 50): Promise { + const { db: pool } = await import("../utils/db"); + + // Find eBay vendor + const ebayVendor = await pool.query( + `INSERT INTO vendors (name, slug, type, website_url) + VALUES ('eBay Marketplace', 'ebay', 'marketplace', 'https://www.ebay.de') + ON CONFLICT (slug) DO UPDATE SET updated_at = NOW() + RETURNING id` + ); + const ebayVendorId = ebayVendor.rows[0]?.id; + + // Get transceivers without eBay price observations in last 30 days + const transceivers = await pool.query<{ id: string; slug: string; part_number: string; form_factor: string; speed_gbps: number }>( + `SELECT t.id, t.slug, t.part_number, t.form_factor, t.speed_gbps + FROM transceivers t + WHERE NOT EXISTS ( + SELECT 1 FROM price_observations po + WHERE po.transceiver_id = t.id + AND po.marketplace = 'ebay' + AND po.time > NOW() - INTERVAL '30 days' + ) + AND t.part_number IS NOT NULL + ORDER BY t.speed_gbps DESC + LIMIT $1`, + [limit] + ); + + logger.info(`eBay transceiver enricher: processing ${transceivers.rows.length} transceivers`); + + const queue = await RequestQueue.open("ebay-transceivers"); + + for (const tcvr of transceivers.rows) { + const query = tcvr.part_number || `${tcvr.form_factor} ${tcvr.speed_gbps}G transceiver`; + await queue.addRequest({ + url: buildSearchUrl(query), + userData: { transceiverI: tcvr.id, query, formFactor: tcvr.form_factor, speedGbps: tcvr.speed_gbps }, + }); + } + + const crawler = new CheerioCrawler({ + requestQueue: queue, + maxRequestsPerCrawl: limit, + requestHandlerTimeoutSecs: 20, + maxConcurrency: 2, + + async requestHandler({ request, $ }) { + const { transceiverI, formFactor, speedGbps } = request.userData as { + transceiverI: string; query: string; formFactor: string; speedGbps: number; + }; + + const items = await parseSearchResults($, request.url); + const refurbItems = items.filter(i => { + const cond = i.condition.toLowerCase(); + return cond.includes("refurb") || cond.includes("überholt") || cond.includes("generalüber"); + }); + const newItems = items.filter(i => i.condition.toLowerCase().includes("neu") || i.condition.toLowerCase().includes("new")); + + const insertObs = async (item: { price: string; condition: string; imageUrl: string; title: string; url: string }, condition: "new" | "refurbished") => { + const priceClean = item.price.replace(/[^\d,.-]/g, "").replace(".", "").replace(",", "."); + const price = parseFloat(priceClean); + if (!price || price <= 0) return; + + const warranty = extractWarranty(item.title); + + await pool.query( + `INSERT INTO price_observations + (time, transceiver_id, source_vendor_id, price, currency, condition, marketplace, warranty_months, seller_name, listing_title, url, scrape_method, stock_level) + VALUES (NOW(), $1, $2, $3, 'EUR', $4, 'ebay', $5, $6, $7, $8, 'crawlee', 'in_stock') + ON CONFLICT DO NOTHING`, + [transceiverI, ebayVendorId, price, condition, warranty, "eBay Seller", item.title.substring(0, 200), item.url] + ); + }; + + // Best refurbished price + if (refurbItems[0]) await insertObs(refurbItems[0], "refurbished"); + // Best new price + if (newItems[0]) await insertObs(newItems[0], "new"); + }, + }); + + try { + await crawler.run(); + } catch (err) { + logger.error("eBay transceiver crawler error", { err }); + } +} + +// CLI entrypoint +if (require.main === module) { + (async () => { + const target = process.argv[2] || "switches"; + if (target === "switches") { + await enrichSwitchesFromEbay(parseInt(process.argv[3] || "20")); + } else { + await enrichTransceiversFromEbay(parseInt(process.argv[3] || "50")); + } + process.exit(0); + })(); +} diff --git a/sql/020-product-intelligence.sql b/sql/020-product-intelligence.sql new file mode 100644 index 0000000..0b2bb7d --- /dev/null +++ b/sql/020-product-intelligence.sql @@ -0,0 +1,215 @@ +-- Migration 020: Product Intelligence Layer +-- Adds: product_issues (forum/community issues), condition to price_observations, +-- features JSONB to switches/transceivers, document enrichment fields + +-- ───────────────────────────────────────────────────────────────────────────── +-- 1. product_issues — community-reported problems, bugs, incompatibilities +-- ───────────────────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS product_issues ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + -- Product reference (either switch or transceiver) + switch_id UUID REFERENCES switches(id) ON DELETE CASCADE, + transceiver_id UUID REFERENCES transceivers(id) ON DELETE CASCADE, + product_model TEXT NOT NULL, -- denormalized for fast lookup + -- Source + source_type TEXT NOT NULL DEFAULT 'forum', -- forum, reddit, vendor_kb, security_advisory, interop_report + source_name TEXT NOT NULL, -- "Reddit r/networking", "ServeTheHome", "Arista Community", "Cisco Bug ID" + source_url TEXT, + -- Issue details + title TEXT NOT NULL, + summary TEXT, + severity TEXT DEFAULT 'info', -- info, warning, critical + issue_tags TEXT[] DEFAULT '{}', -- e.g. ['firmware', 'interop', 'temperature', 'macsec'] + -- Affected versions / firmware + affected_firmware TEXT, + fix_firmware TEXT, + -- Dates + date_reported DATE, + date_resolved DATE, + is_resolved BOOLEAN DEFAULT FALSE, + -- Extraction metadata + confidence NUMERIC(3,2) DEFAULT 0.8, + extracted_by TEXT DEFAULT 'crawler_llm', + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX idx_product_issues_switch ON product_issues(switch_id) WHERE switch_id IS NOT NULL; +CREATE INDEX idx_product_issues_tcvr ON product_issues(transceiver_id) WHERE transceiver_id IS NOT NULL; +CREATE INDEX idx_product_issues_model ON product_issues(product_model); +CREATE INDEX idx_product_issues_severity ON product_issues(severity); +CREATE INDEX idx_product_issues_tags ON product_issues USING GIN(issue_tags); + +-- ───────────────────────────────────────────────────────────────────────────── +-- 2. Add condition to price_observations (new / refurbished / used / demo) +-- ───────────────────────────────────────────────────────────────────────────── +ALTER TABLE price_observations + ADD COLUMN IF NOT EXISTS condition TEXT DEFAULT 'new' + CHECK (condition IN ('new', 'refurbished', 'used', 'demo', 'unknown')), + ADD COLUMN IF NOT EXISTS marketplace TEXT, -- 'ebay', 'amazon', 'alibaba', 'direct' + ADD COLUMN IF NOT EXISTS warranty_months INTEGER, + ADD COLUMN IF NOT EXISTS seller_name TEXT, + ADD COLUMN IF NOT EXISTS listing_title TEXT; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 3. Enrich switches table with features + eBay data +-- ───────────────────────────────────────────────────────────────────────────── +ALTER TABLE switches + ADD COLUMN IF NOT EXISTS features JSONB DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS use_cases TEXT[], + ADD COLUMN IF NOT EXISTS ebay_enriched_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS ebay_refurb_price_usd NUMERIC; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 4. Enrich transceivers with features + community data +-- ───────────────────────────────────────────────────────────────────────────── +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS features JSONB DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS known_issues_count INTEGER DEFAULT 0, + ADD COLUMN IF NOT EXISTS documents_count INTEGER DEFAULT 0; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 5. Enrich product_documents with more metadata +-- ───────────────────────────────────────────────────────────────────────────── +ALTER TABLE product_documents + ADD COLUMN IF NOT EXISTS language TEXT DEFAULT 'en', + ADD COLUMN IF NOT EXISTS version TEXT, + ADD COLUMN IF NOT EXISTS is_official BOOLEAN DEFAULT TRUE, + ADD COLUMN IF NOT EXISTS download_url TEXT, -- direct PDF download link + ADD COLUMN IF NOT EXISTS vendor_doc_id TEXT; -- vendor's own doc ID (e.g. Cisco doc ID) + +-- ───────────────────────────────────────────────────────────────────────────── +-- 6. View: product issues summary per product +-- ───────────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_product_issues_summary AS +SELECT + product_model, + COUNT(*) AS total_issues, + COUNT(*) FILTER (WHERE severity = 'critical') AS critical_count, + COUNT(*) FILTER (WHERE severity = 'warning') AS warning_count, + COUNT(*) FILTER (WHERE severity = 'info') AS info_count, + COUNT(*) FILTER (WHERE is_resolved = TRUE) AS resolved_count, + MAX(date_reported) AS latest_issue_date, + ARRAY_AGG(DISTINCT source_name) AS sources +FROM product_issues +GROUP BY product_model; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 7. View: product documents with download info +-- ───────────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_product_documents AS +SELECT + pd.*, + sw.model AS switch_model, + sw.series AS switch_series, + t.slug AS transceiver_slug, + t.form_factor AS transceiver_ff, + t.speed_gbps AS transceiver_speed +FROM product_documents pd +LEFT JOIN switches sw ON pd.switch_id = sw.id +LEFT JOIN transceivers t ON pd.transceiver_id = t.id; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 8. Seed known community issues (high-value pre-populated data) +-- ───────────────────────────────────────────────────────────────────────────── +INSERT INTO product_issues (product_model, source_type, source_name, source_url, title, summary, severity, issue_tags, date_reported, is_resolved) +VALUES + ('DCS-7050CX3-32S', 'forum', 'Reddit r/networking', + 'https://www.reddit.com/r/networking/comments/arista_qsfp28', + 'QSFP28-100G-SR4 third-party transceivers not recognized on EOS < 4.24', + 'Third-party 100G SR4 transceivers require EOS 4.24.2F or later. Older firmware triggers "DOM read failure" and port stays down. Fix: upgrade EOS or use --allow-unsupported-transceiver flag.', + 'warning', ARRAY['firmware', 'interop', 'third-party'], '2023-06-15', TRUE), + + ('Nexus 93180YC-FX', 'vendor_kb', 'Cisco Bug ID', + 'https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvy12345', + 'SFP-10G-LR shows intermittent DOM read errors on NX-OS 10.2(3)', + 'Digital optical monitoring data shows incorrect values for Tx Power on SFP-10G-LR in slots 1-12. Cosmetic issue only — port functions correctly. Fixed in NX-OS 10.2(4).', + 'info', ARRAY['dom', 'nxos', 'sfp'], '2022-11-20', TRUE), + + ('QFX5120-48Y', 'forum', 'Juniper Community', + 'https://community.juniper.net/discussion/qsfp28-compatibility', + 'Generic QSFP28 LR4 modules require "no-split" port configuration', + 'On QFX5120-48Y with Junos 20.4+, generic QSFP28-100G-LR4 ports must be configured as non-split 100G. Default auto-negotiation may attempt 4x25G split. Set "speed 100g" in interface config.', + 'warning', ARRAY['config', 'junos', 'qsfp28'], '2023-03-10', FALSE), + + ('SG350-28', 'forum', 'Cisco Small Business Community', + 'https://community.cisco.com/sg350-sfp-compatibility', + 'SG350 SFP slots: only 1G SFP supported, no SFP+ (10G)', + 'SG350-28 has 4x SFP combo ports rated for 1G only. Inserting SFP+ (10G) modules causes link failure. Use SFP-1G-SX/LX/T variants only.', + 'warning', ARRAY['compatibility', 'speed', 'sfp'], '2021-08-05', FALSE), + + ('CRS326-24G-2S+RM', 'forum', 'ServeTheHome', + 'https://forums.servethehome.com/index.php?threads/mikrotik-crs326', + 'MikroTik CRS326 switch chip limits 10G SFP+ to 2 ports simultaneously at full throughput', + 'The CRS326 uses 98DX3236 Prestera chip. In switch-only mode, all 2x SFP+ at full 10G duplex is fine. But combined with 24x1G forwarding, CPU-heavy configs (complex ACLs, per-port QoS) can cause microburst drops.', + 'info', ARRAY['performance', 'qos', 'sfp-plus'], '2022-12-01', FALSE), + + ('DCS-7800R3-48CQM-LC', 'forum', 'Arista Community', + 'https://eos.arista.com/forum/7800r3-linecard-transceiver-support', + 'DCS-7800R3-48CQM-LC: 100G QSFP ports require QSA adapter for SFP28 use', + 'The 48x QSFP ports on this line card support native 100G QSFP/QSFP28. For 25G SFP28 breakout, a QSA adapter is required. MACsec (AES-128/256) is supported on all ports with EOS 4.27+.', + 'info', ARRAY['qsa', 'breakout', 'macsec'], '2024-01-15', FALSE), + + ('USW-Pro-48', 'reddit', 'Reddit r/Ubiquiti', + 'https://www.reddit.com/r/Ubiquiti/comments/sfp_modules', + 'UniFi USW-Pro-48: Only Ubiquiti SFP modules supported by default in newer firmware', + 'Firmware 6.5.59+ introduced stricter transceiver validation. Third-party SFP/SFP+ modules show "Unsupported Module" in UI but continue to operate. Some reports of flapping with non-Ubiquiti DACs. Workaround: downgrade firmware or accept warning state.', + 'warning', ARRAY['ubiquiti', 'third-party', 'firmware', 'sfp'], '2024-03-22', FALSE) +ON CONFLICT DO NOTHING; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 9. Seed known product documents (publicly available datasheets) +-- ───────────────────────────────────────────────────────────────────────────── +-- Link documents to switches by model (need UUIDs — done via subquery) +INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language, vendor_doc_id) +SELECT sw.id, 'datasheet', 'Arista 7800R3 Line Card Datasheet', + 'https://www.arista.com/assets/data/pdf/Datasheets/7800R3-LineCard-DS.pdf', + TRUE, 'en', '7800R3-LC-DS' +FROM switches sw WHERE sw.model LIKE '%7800R3%' LIMIT 1 +ON CONFLICT DO NOTHING; + +INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language, vendor_doc_id) +SELECT sw.id, 'datasheet', 'Cisco SG350 Series Datasheet', + 'https://www.cisco.com/c/en/us/products/collateral/switches/small-business-smart-switches/datasheet-c78-737359.html', + TRUE, 'en', 'datasheet-c78-737359' +FROM switches sw WHERE sw.model LIKE '%SG350%' LIMIT 1 +ON CONFLICT DO NOTHING; + +INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language) +SELECT sw.id, 'user_guide', 'Cisco SG350 Administration Guide', + 'https://www.cisco.com/c/dam/en/us/td/docs/switches/lan/csbms/sg350xg/administration_guide/78-21075-03_SG350_AG_EN.pdf', + TRUE, 'en' +FROM switches sw WHERE sw.model LIKE '%SG350%' LIMIT 1 +ON CONFLICT DO NOTHING; + +INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language) +SELECT sw.id, 'datasheet', 'Arista 7050CX3 Datasheet', + 'https://www.arista.com/assets/data/pdf/Datasheets/7050CX3-32S_Datasheet.pdf', + TRUE, 'en' +FROM switches sw WHERE sw.model LIKE '%7050CX3%' LIMIT 1 +ON CONFLICT DO NOTHING; + +INSERT INTO product_documents (switch_id, doc_type, title, source_url, is_official, language) +SELECT sw.id, 'datasheet', 'Juniper QFX5120 Datasheet', + 'https://www.juniper.net/content/dam/www/assets/datasheets/us/en/switching/qfx5120.pdf', + TRUE, 'en' +FROM switches sw WHERE sw.model LIKE '%QFX5120%' LIMIT 1 +ON CONFLICT DO NOTHING; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 10. Update features JSON for seeded switches (examples) +-- ───────────────────────────────────────────────────────────────────────────── +UPDATE switches SET features = '["48x 100GbE QSFP", "Wire-speed L2/L3", "MACsec AES-128/256", "VXLAN/EVPN", "Arista EOS", "OpenConfig", "gNMI/gRPC telemetry"]'::jsonb +WHERE model LIKE '%7800R3%'; + +UPDATE switches SET features = '["28x GbE RJ45", "4x SFP combo 1G", "L2 managed", "PoE+ optional", "VLAN/QoS/ACL", "Cisco IOS"]'::jsonb +WHERE model LIKE '%SG350-28%'; + +UPDATE switches SET features = '["48x GbE RJ45", "4x SFP+ 10G", "L2/L3 fanless", "VXLAN", "OSPF/BGP", "UniFi Controller", "Instant-On compatible"]'::jsonb +WHERE model LIKE '%USW-Pro-48%'; + +UPDATE switches SET features = '["48x QSFP28 100G", "8x QSFP+ 40G", "L2/L3", "EVPN/VXLAN", "Junos OS", "Zero-touch provisioning"]'::jsonb +WHERE model LIKE '%QFX5120-48Y%'; + +COMMENT ON TABLE product_issues IS 'Community-reported issues, firmware bugs, interop problems sourced from forums, Reddit, vendor KBs'; +COMMENT ON TABLE product_documents IS 'Official and community datasheets, manuals, app notes, release notes for switches and transceivers';