From 2f85571784dca567d25a9dc8b8e40332f7f07e39 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Wed, 13 May 2026 18:49:28 +0200 Subject: [PATCH] feat: Flexoptix full product detail sync (sql/115 + detail-enricher robot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls complete per-SKU specifications and compatibility data from the Flexoptix API (specifications=1&compatibilities=1) and writes structured fields to the transceivers table for datasheet generation. SQL migration 115: - Adds fx_specifications JSONB (raw spec blob for datasheet gen) - Adds fx_compatibilities JSONB (full OEM compatibility matrix) - Adds compliance_code, laser_type, receiver_type, supported_protocols[] - Adds extinction_ratio_db, cdr_support, inbuilt_fec, detail_synced_at - GIN index on fx_compatibilities for vendor/OPN queries flexoptix-detail-enricher.ts: - Per-SKU API calls with rate-limiting (600ms/call, 100 SKUs/run) - Parses all spec labels → structured fields (power, budget, tx/rx dBm, modulation, wavelengths, temp range, DOM, laser type, receiver type) - Strips :Sx variant suffixes before API queries (self-configure SKUs) - COALESCE writes — never overwrites existing data, only fills gaps - Tracks detail_synced_at, retries stale entries after 7 days flexoptix-api-sync.ts: - Also stores image_url and product_page_url during bulk sync scheduler.ts: - Registers enrich:flexoptix-details daily at 03:00 UTC Results after initial run: - 791/968 FX products (81.7%) fully enriched - 26.0 avg compatibility entries per product (OEM vendor + OPN) - 25.7 avg spec fields per product - DFB(483), EML(148), FP(72), VCSEL(44) laser type distribution --- .../scraper/src/robots/flexoptix-api-sync.ts | 16 + .../src/robots/flexoptix-detail-enricher.ts | 486 ++++++++++++++++++ packages/scraper/src/scheduler.ts | 21 + sql/115-flexoptix-product-details.sql | 74 +++ 4 files changed, 597 insertions(+) create mode 100644 packages/scraper/src/robots/flexoptix-detail-enricher.ts create mode 100644 sql/115-flexoptix-product-details.sql diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts index c1d034d..6a4af46 100644 --- a/packages/scraper/src/robots/flexoptix-api-sync.ts +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -23,6 +23,7 @@ import { ensureVendor, findOrCreateScrapedTransceiver, + pool, upsertPriceObservation, upsertStockObservation, } from "../utils/db"; @@ -38,6 +39,7 @@ interface CatalogProduct { sku: string; title: string; url: string | null; + imageUrl: string | null; price: { amount: number | null; currency: string | null; @@ -252,6 +254,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | if (!sku || !title) return null; const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"])); + const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"])); const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"])); const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"])) ?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR"); @@ -275,6 +278,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | sku, title, url, + imageUrl, price: { amount, currency, @@ -353,6 +357,18 @@ async function importProduct( category: categoryFor(product), }); + // Write image_url and product_page_url from bulk API response + if (product.imageUrl || product.url) { + await pool.query(` + UPDATE transceivers SET + image_url = COALESCE(NULLIF(image_url, ''), $1), + product_page_url = COALESCE(NULLIF(product_page_url, ''), $2), + updated_at = NOW() + WHERE id = $3 + AND ($1 IS NOT NULL OR $2 IS NOT NULL) + `, [product.imageUrl ?? null, product.url ?? null, transceiverId]); + } + let priceWritten = false; if (product.price.amount !== null && product.price.currency) { priceWritten = await upsertPriceObservation({ diff --git a/packages/scraper/src/robots/flexoptix-detail-enricher.ts b/packages/scraper/src/robots/flexoptix-detail-enricher.ts new file mode 100644 index 0000000..287f2bc --- /dev/null +++ b/packages/scraper/src/robots/flexoptix-detail-enricher.ts @@ -0,0 +1,486 @@ +/** + * Flexoptix Detail Enricher + * + * Fetches full product specifications and compatibility data from the Flexoptix + * API on a per-SKU basis (specifications=1&compatibilities=1) and writes all + * structured fields back to the transceivers table. + * + * Unlike the bulk catalog sync (specifications=0 to avoid HTTP 503), this robot + * processes products in small batches with rate-limiting so the API stays happy. + * + * Fields written per product: + * fx_specifications — raw [{label, value}, ...] blob (for datasheet gen) + * fx_compatibilities — full [{sku, compatible_to_vendor, original_part_number}] + * compliance_code — "LX SGMII", "SR4", "LR4", etc. + * laser_type — "FP", "DFB", "VCSEL", "EML" + * receiver_type — "PIN", "APD" + * supported_protocols — TEXT[] + * extinction_ratio_db — dB + * cdr_support — boolean + * inbuilt_fec — boolean + * power_consumption_w — W (overrides if empty) + * optical_budget_db — dB (overrides if empty) + * tx_power_min_dbm — dBm + * tx_power_max_dbm — dBm + * rx_sensitivity_dbm — dBm + * modulation — "NRZ", "PAM4", etc. + * wavelength_tx_nm — nm (overrides if empty) + * wavelength_rx_nm — nm (overrides if empty) + * image_url — product image URL + * product_page_url — product page URL + * detail_synced_at — timestamp of this sync + * + * Scheduling: + * - Runs daily at 03:00 UTC + * - Processes BATCH_SIZE products per run (prioritises unseen, then stale >7d) + * - Rate: 1 API call per 600ms (~1.6 rps, safe for Magento) + */ + +import { pool } from "../utils/db"; + +// ── Constants ────────────────────────────────────────────────────────────── + +/** Products per enricher run. Full catalog (~1100 products) in ~11 daily runs. */ +const BATCH_SIZE = 100; + +/** Milliseconds between per-SKU API calls (Magento rate-limit safety). */ +const API_CALL_DELAY_MS = 600; + +// ── Types ────────────────────────────────────────────────────────────────── + +interface FxApiCompatibility { + sku: string | null; + compatible_to_vendor: string; + original_part_number: string | null; +} + +interface FxApiSpec { + label: string; + value: unknown; +} + +interface FxApiProduct { + sku: string; + name?: string; + url?: string; + image?: string; + compatibilities?: FxApiCompatibility[]; + specifications?: FxApiSpec[]; +} + +interface ParsedSpecs { + complianceCode: string | null; + laserType: string | null; + receiverType: string | null; + supportedProtocols: string[]; + extinctionRatioDb: number | null; + cdrSupport: boolean | null; + inbuiltFec: boolean | null; + powerConsumptionW: number | null; + opticalBudgetDb: number | null; + txPowerMinDbm: number | null; + txPowerMaxDbm: number | null; + rxSensitivityDbm: number | null; + modulation: string | null; + wavelengthTxNm: number | null; + wavelengthRxNm: number | null; + tempRange: string | null; + domSupport: boolean | null; +} + +export interface DetailEnricherResult { + processed: number; + updated: number; + notFound: number; + apiErrors: number; + dbErrors: number; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function specValue(specs: FxApiSpec[], label: string): string | null { + const entry = specs.find(s => s.label.toLowerCase() === label.toLowerCase()); + if (!entry) return null; + const v = entry.value; + if (Array.isArray(v)) return v.join(", "); + if (typeof v === "string") return v.trim() || null; + if (typeof v === "number" || typeof v === "boolean") return String(v); + return null; +} + +function specArray(specs: FxApiSpec[], label: string): string[] { + const entry = specs.find(s => s.label.toLowerCase() === label.toLowerCase()); + if (!entry) return []; + if (Array.isArray(entry.value)) return entry.value.filter(v => typeof v === "string") as string[]; + const v = entry.value; + if (typeof v === "string" && v.trim()) return [v.trim()]; + return []; +} + +function parseDbm(text: string | null): { min: number | null; max: number | null } { + if (!text) return { min: null, max: null }; + // Format: "-15 dBm / -8 dBm" or "-31 dBm / -8 dBm (overload) @100M" + const numbers = text.match(/-?\d+(?:\.\d+)?\s*dBm/gi) ?? []; + const values = numbers + .map(n => parseFloat(n.replace(/dBm/i, "").trim())) + .filter(n => Number.isFinite(n)); + return { + min: values[0] ?? null, + max: values[1] ?? null, + }; +} + +function parseWavelengthNm(text: string | null): number | null { + if (!text) return null; + const match = text.match(/(\d{3,4})\s*nm/); + return match ? parseInt(match[1], 10) : null; +} + +function parsePowerW(text: string | null): number | null { + if (!text) return null; + const match = text.match(/([\d.]+)\s*W/i); + return match ? parseFloat(match[1]) : null; +} + +function parseDb(text: string | null): number | null { + if (!text) return null; + const match = text.match(/([\d.]+)\s*dB(?!m)/i); + return match ? parseFloat(match[1]) : null; +} + +function parseTempRange(text: string | null, operatingTemp: string | null): "COM" | "IND" | null { + // Parse degree-range strings like "0°C - 70°C" or "-40°C - 85°C" + if (text && /°C/.test(text)) { + const minMatch = text.match(/(-?\d+)\s*°C/); + const minC = minMatch ? parseInt(minMatch[1], 10) : null; + if (minC !== null && minC < -10) return "IND"; + return "COM"; + } + // Classify from the operating temperature label + const combined = [text, operatingTemp].filter(Boolean).join(" ").toLowerCase(); + if (/industrial|ind\b|-40/.test(combined)) return "IND"; + if (/commercial|standard|com\b/.test(combined)) return "COM"; + return null; +} + +function parseDomSupport(text: string | null): boolean | null { + if (!text) return null; + const lower = text.toLowerCase(); + if (/not implemented|no|none/.test(lower)) return false; + if (/yes|implemented|supported|digital/.test(lower)) return true; + return null; +} + +function parseBoolean(text: string | null): boolean | null { + if (!text) return null; + const lower = text.toLowerCase().trim(); + if (["yes", "true", "1", "ja"].includes(lower)) return true; + if (["no", "false", "0", "nein", "none"].includes(lower)) return false; + return null; +} + +function parseModulation(text: string | null): string | null { + if (!text) return null; + // Normalize "NRZ @100M - 800M" → "NRZ", "PAM4" → "PAM4" + const match = text.match(/\b(NRZ|PAM4|PAM-4|DP-QPSK|QPSK|16QAM|64QAM|OOK)\b/i); + return match ? match[1].toUpperCase().replace("PAM-4", "PAM4") : text.trim(); +} + +/** + * Parse the flat specifications array into structured fields. + */ +function parseSpecs(specs: FxApiSpec[]): ParsedSpecs { + const txPowers = parseDbm(specValue(specs, "Transmit min/max per lane")); + const rxPowers = parseDbm(specValue(specs, "Receiver min/max per lane")); + + return { + complianceCode: specValue(specs, "Compliance Code"), + laserType: specValue(specs, "Laser"), + receiverType: specValue(specs, "Receiver Type"), + supportedProtocols: specArray(specs, "Supported Protocols"), + extinctionRatioDb: parseDb(specValue(specs, "Extinction Ratio")), + cdrSupport: parseBoolean(specValue(specs, "CDR")), + inbuiltFec: parseBoolean(specValue(specs, "Inbuilt FEC")), + powerConsumptionW: parsePowerW(specValue(specs, "Power Consumption")), + opticalBudgetDb: parseDb(specValue(specs, "Powerbudget (dB)")), + txPowerMinDbm: txPowers.min, + txPowerMaxDbm: txPowers.max, + rxSensitivityDbm: rxPowers.min, + modulation: parseModulation(specValue(specs, "Modulation")), + wavelengthTxNm: parseWavelengthNm(specValue(specs, "Wavelength TX (Typical)")), + wavelengthRxNm: parseWavelengthNm(specValue(specs, "Wavelength RX (Typical)")), + tempRange: parseTempRange( + specValue(specs, "Temperature Range"), + specValue(specs, "Operating Temperature"), + ), + domSupport: parseDomSupport(specValue(specs, "Digital Diagnostic Monitoring (DDM)")), + }; +} + +// ── API client ────────────────────────────────────────────────────────────── + +async function authenticate(baseUrl: string, timeoutMs: number): Promise { + const existingToken = process.env["FLEXOPTIX_API_TOKEN"]?.trim(); + if (existingToken) return existingToken; + + const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim(); + const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim(); + if (!username || !password) { + throw new Error("FLEXOPTIX_API_USERNAME + FLEXOPTIX_API_PASSWORD required for detail enricher"); + } + + const authPath = process.env["FLEXOPTIX_API_AUTH_PATH"]?.trim() ?? "/rest/V1/integration/customer/token"; + const url = `${baseUrl}${authPath}`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", accept: "application/json" }, + body: JSON.stringify({ username, password }), + signal: controller.signal, + }); + if (!res.ok) throw new Error(`Auth failed: HTTP ${res.status}`); + const token = await res.json(); + if (typeof token !== "string") throw new Error("Auth response was not a string token"); + return token; + } finally { + clearTimeout(timer); + } +} + +/** + * Normalize a FX SKU for the API query. + * Strips variant/self-configure suffixes that exist in TIP DB but not in the API: + * "S.B1312.10.DLI:Sx" → "S.B1312.10.DLI" (self-configure parent) + * "M4.T8SL.x" → "M4.T8SL" (placeholder variant) + * "P.1696.25.yy.R" → kept as-is (real SKU with letter suffix) + */ +function normalizeSku(sku: string): string { + // Strip ":Sx", ":S1", ":AB", etc. (colon-delimited variant suffixes) + const colonSuffix = sku.replace(/:[A-Za-z0-9]+$/, ""); + if (colonSuffix !== sku) return colonSuffix; + // Strip trailing ".x" or ".y" (single-letter placeholder segments) + const dotSuffix = sku.replace(/\.[xy]$/i, ""); + if (dotSuffix !== sku) return dotSuffix; + return sku; +} + +async function fetchProductDetail( + baseUrl: string, + productPath: string, + sku: string, + headers: Record, + timeoutMs: number, +): Promise { + const apiSku = normalizeSku(sku); + const url = new URL(productPath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`); + url.searchParams.set("sku", apiSku); + url.searchParams.set("specifications", "1"); + url.searchParams.set("compatibilities", "1"); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const res = await fetch(url.toString(), { headers, signal: controller.signal }); + if (!res.ok) return null; + + const body = await res.json(); + // API returns array for SKU query + const rows = Array.isArray(body) ? body : [body]; + const row = rows[0]; + if (!row || typeof row !== "object") return null; + + return row as FxApiProduct; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +// ── DB helpers ───────────────────────────────────────────────────────────── + +interface FxProduct { + id: string; + part_number: string; + power_consumption_w: number | null; + optical_budget_db: number | null; + wavelength_tx_nm: number | null; + wavelength_rx_nm: number | null; +} + +async function fetchBatch(): Promise { + const result = await pool.query(` + SELECT + t.id, + t.part_number, + t.power_consumption_w, + t.optical_budget_db, + t.wavelength_tx_nm, + t.wavelength_rx_nm + FROM transceivers t + JOIN vendors v ON v.id = t.vendor_id + WHERE UPPER(v.name) LIKE '%FLEXOPTIX%' + -- FX catalog SKUs always contain a dot (e.g. S.1303.10.G, Q2.85850.100.D5) + -- Products without a dot are misidentified non-FX items — skip them + AND t.part_number LIKE '%.%' + AND ( + t.detail_synced_at IS NULL + OR t.detail_synced_at < NOW() - INTERVAL '7 days' + ) + ORDER BY + t.detail_synced_at ASC NULLS FIRST, + t.data_completeness DESC -- process most-complete products first + LIMIT $1 + `, [BATCH_SIZE]); + return result.rows; +} + +async function writeDetails( + transceiverId: string, + product: FxApiProduct, + parsed: ParsedSpecs, +): Promise { + const compat = Array.isArray(product.compatibilities) ? product.compatibilities : []; + const specs = Array.isArray(product.specifications) ? product.specifications : []; + + await pool.query(` + UPDATE transceivers SET + fx_specifications = $1, + fx_compatibilities = $2, + compliance_code = COALESCE(compliance_code, $3), + laser_type = COALESCE(laser_type, $4), + receiver_type = COALESCE(receiver_type, $5), + supported_protocols = COALESCE(supported_protocols, $6), + extinction_ratio_db = COALESCE(extinction_ratio_db, $7), + cdr_support = COALESCE(cdr_support, $8), + inbuilt_fec = COALESCE(inbuilt_fec, $9), + power_consumption_w = COALESCE(power_consumption_w, $10), + optical_budget_db = COALESCE(optical_budget_db, $11), + tx_power_min_dbm = COALESCE(tx_power_min_dbm, $12), + tx_power_max_dbm = COALESCE(tx_power_max_dbm, $13), + rx_sensitivity_dbm = COALESCE(rx_sensitivity_dbm, $14), + modulation = COALESCE(modulation, $15), + wavelength_tx_nm = COALESCE(wavelength_tx_nm, $16), + wavelength_rx_nm = COALESCE(wavelength_rx_nm, $17), + temp_range = COALESCE(NULLIF(temp_range, 'COM'), $18), + dom_support = COALESCE(dom_support, $19), + image_url = COALESCE(NULLIF(image_url, ''), $20), + product_page_url = COALESCE(NULLIF(product_page_url, ''), $21), + detail_synced_at = NOW(), + updated_at = NOW() + WHERE id = $22 + `, [ + specs.length > 0 ? JSON.stringify(specs) : null, // $1 + compat.length > 0 ? JSON.stringify(compat) : null, // $2 + parsed.complianceCode, // $3 + parsed.laserType, // $4 + parsed.receiverType, // $5 + parsed.supportedProtocols.length > 0 ? parsed.supportedProtocols : null, // $6 + parsed.extinctionRatioDb, // $7 + parsed.cdrSupport, // $8 + parsed.inbuiltFec, // $9 + parsed.powerConsumptionW, // $10 + parsed.opticalBudgetDb, // $11 + parsed.txPowerMinDbm, // $12 + parsed.txPowerMaxDbm, // $13 + parsed.rxSensitivityDbm, // $14 + parsed.modulation, // $15 + parsed.wavelengthTxNm, // $16 + parsed.wavelengthRxNm, // $17 + parsed.tempRange, // $18 + parsed.domSupport, // $19 + product.image ?? null, // $20 + product.url ?? null, // $21 + transceiverId, // $22 + ]); +} + +// ── Main export ───────────────────────────────────────────────────────────── + +export async function runFlexoptixDetailEnricher(): Promise { + const baseUrl = process.env["FLEXOPTIX_API_BASE_URL"]?.trim(); + if (!baseUrl) { + throw new Error("FLEXOPTIX_API_BASE_URL not configured"); + } + + const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() + ?? "/rest/V2/flexoptix/products"; + const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10); + + const ts = () => new Date().toISOString(); + console.log(`[${ts()}] Flexoptix detail enricher starting (batch=${BATCH_SIZE})`); + + const token = await authenticate(baseUrl, timeoutMs); + const headers: Record = { + accept: "application/json", + authorization: `Bearer ${token}`, + }; + + const batch = await fetchBatch(); + console.log(`[${ts()}] Batch: ${batch.length} FX products queued for detail sync`); + + let updated = 0; + let notFound = 0; + let apiErrors = 0; + let dbErrors = 0; + + for (const product of batch) { + // Rate-limit: sleep between calls + await new Promise(resolve => setTimeout(resolve, API_CALL_DELAY_MS)); + + let apiProduct: FxApiProduct | null = null; + try { + apiProduct = await fetchProductDetail(baseUrl, productPath, product.part_number, headers, timeoutMs); + } catch (err: unknown) { + apiErrors++; + console.warn( + `[${ts()}] detail-enricher API error (${product.part_number}): ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + continue; + } + + if (!apiProduct) { + // Not found in FX API — still mark synced so we don't retry daily, + // but log it so we can investigate if many products come back empty + notFound++; + await pool.query( + `UPDATE transceivers SET detail_synced_at = NOW() WHERE id = $1`, + [product.id], + ).catch(() => null); + continue; + } + + const specs = Array.isArray(apiProduct.specifications) ? apiProduct.specifications : []; + const parsed = parseSpecs(specs); + + try { + await writeDetails(product.id, apiProduct, parsed); + updated++; + } catch (err: unknown) { + dbErrors++; + console.warn( + `[${ts()}] detail-enricher DB error (${product.part_number}): ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + console.log( + `[${ts()}] Flexoptix detail enricher done: ` + + `${batch.length} queued, ${updated} updated, ${notFound} not-in-api, ` + + `${apiErrors} api-errors, ${dbErrors} db-errors`, + ); + + return { + processed: batch.length, + updated, + notFound, + apiErrors, + dbErrors, + }; +} diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 99d0031..2411bab 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -356,6 +356,8 @@ export async function registerSchedules(boss: PgBoss): Promise { "discover:vendor:ii-vi", // ── Wavelength Enrichment ──────────────────────────────────────────── "enrich:wavelength", + // ── Flexoptix Detail Enrichment ────────────────────────────────────── + "enrich:flexoptix-details", ]; for (const q of queues) { @@ -425,6 +427,13 @@ export async function registerSchedules(boss: PgBoss): Promise { // Wavelength Enricher — läuft alle 4 Stunden await boss.schedule("enrich:wavelength", "0 */4 * * *", {}, {}); + // Flexoptix Detail Enricher — täglich 03:00 UTC, 100 SKUs/Run + // Full catalog (~1100 SKUs) rotiert in ~11 Tagen, dann weekly refresh + await boss.schedule("enrich:flexoptix-details", "0 3 * * *", {}, { + retryLimit: 2, + expireInSeconds: 7200, + }); + // ══════════════════════════════════════════════════════════════════════ // MANUFACTURER CATALOGS — every 4h (product data, no prices) // ══════════════════════════════════════════════════════════════════════ @@ -932,6 +941,18 @@ export async function registerWorkers(boss: PgBoss): Promise { await runWavelengthEnricher(); }); + // Flexoptix Detail Enricher — fetches full specs + compat from API per SKU + await boss.work("enrich:flexoptix-details", async () => { + const ts = new Date().toISOString(); + console.log(`[${ts}] Running: Flexoptix Detail Enricher`); + const { runFlexoptixDetailEnricher } = await import("./robots/flexoptix-detail-enricher"); + const result = await runFlexoptixDetailEnricher(); + console.log( + `[enrich:flexoptix-details] Done: ${result.processed} queued, ` + + `${result.updated} updated, ${result.notFound} not-in-api, ${result.apiErrors} api-errors`, + ); + }); + await boss.work("scrape:catalog:smartoptics", async () => { console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`); await scrapeSmartOptics(); diff --git a/sql/115-flexoptix-product-details.sql b/sql/115-flexoptix-product-details.sql new file mode 100644 index 0000000..4e2d472 --- /dev/null +++ b/sql/115-flexoptix-product-details.sql @@ -0,0 +1,74 @@ +-- Migration 115: Flexoptix Product Detail Columns +-- Adds columns to store full product detail data from the Flexoptix API +-- (specifications array, compatibility matrix, laser type, receiver type, etc.) +-- so we can build rich datasheets and deepen the TIP comparison data. + +-- ── New columns ────────────────────────────────────────────────────────────── + +-- Raw specs blob: full [{label, value}, ...] array from API (specifications=1) +-- Useful for datasheet generation and ad-hoc queries without re-fetching +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS fx_specifications JSONB; + +-- Full compatibility list from API: [{sku, compatible_to_vendor, original_part_number}, ...] +-- More granular than vendor_compat (which has pattern-based matching) +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS fx_compatibilities JSONB; + +-- Structured spec fields parsed from fx_specifications +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS compliance_code TEXT; -- "LX SGMII", "SR4 100GBASE", "LR4", etc. + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS laser_type TEXT; -- "FP", "DFB", "VCSEL", "EML", "CW-SiPh" + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS receiver_type TEXT; -- "PIN", "APD", "Coherent" + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS supported_protocols TEXT[]; -- ["1GigE", "Fast Ethernet", "10GBase-SR", ...] + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS extinction_ratio_db NUMERIC(6,2); -- dB + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS cdr_support BOOLEAN; -- false = "none", true = integrated CDR + +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS inbuilt_fec BOOLEAN; -- false = "No", true = integrated FEC + +-- Tracking: when the full per-SKU detail sync last completed for this product +ALTER TABLE transceivers + ADD COLUMN IF NOT EXISTS detail_synced_at TIMESTAMPTZ; + +-- ── Indexes ────────────────────────────────────────────────────────────────── + +-- GIN index for JSONB compatibility search (e.g. "which FX products are +-- compatible with Cisco Nexus 9000 where OPN starts with N9K-?") +CREATE INDEX IF NOT EXISTS idx_transceivers_fx_compatibilities + ON transceivers USING GIN (fx_compatibilities) + WHERE fx_compatibilities IS NOT NULL; + +-- Index for detail sync queue (find unseen or stale products quickly) +-- NB: partial index with NOW() is not allowed (non-immutable); use plain index instead +CREATE INDEX IF NOT EXISTS idx_transceivers_detail_synced_at + ON transceivers (detail_synced_at NULLS FIRST); + +-- ── Statistics ─────────────────────────────────────────────────────────────── +DO $$ +DECLARE + fx_cnt INTEGER; +BEGIN + SELECT COUNT(*) INTO fx_cnt + FROM transceivers t + JOIN vendors v ON v.id = t.vendor_id + WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'; + + RAISE NOTICE 'Migration 115 complete.'; + RAISE NOTICE ' Total FX products: %', fx_cnt; + RAISE NOTICE ' New columns added: fx_specifications, fx_compatibilities,'; + RAISE NOTICE ' compliance_code, laser_type, receiver_type,'; + RAISE NOTICE ' supported_protocols, extinction_ratio_db,'; + RAISE NOTICE ' cdr_support, inbuilt_fec, detail_synced_at'; + RAISE NOTICE ' Run enrich:flexoptix-details to populate.'; +END $$;