From 73ad10f1a6396a5f3f26449bd445b6156c3e88d6 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 11:39:38 +0000 Subject: [PATCH 1/6] fix(stock): 7-day heartbeat write bypasses dedup for stale observations upsertStockObservation dedup guard was preventing writes for unchanged stock data (e.g. ATGBICS all-OOS for 55d), causing v_scraper_health to mark sources as dead even when scrapers run normally. Skip dedup when last observation is >7 days old so health view stays current. --- packages/scraper/src/utils/db.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index d29b56f..903ca68 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -417,9 +417,11 @@ export async function upsertStockObservation(params: { return false; } - // Compare against the last observation to avoid duplicate writes + // Compare against the last observation to avoid duplicate writes. + // Bypass dedup if last observation is older than 7 days (heartbeat to keep health view alive). + const STOCK_REFRESH_DAYS = 7; const lastObs = await pool.query( - `SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available + `SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available, time FROM stock_observations WHERE transceiver_id = $1 AND source_vendor_id = $2 ORDER BY time DESC LIMIT 1`, @@ -428,13 +430,17 @@ export async function upsertStockObservation(params: { if (lastObs.rows.length > 0) { const r = lastObs.rows[0]; - const unchanged = - (r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) && - (r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) && - (r.backorder_qty ?? null) === (params.backorderQty ?? null) && - (r.units_sold ?? null) === (params.unitsSold ?? null) && - (r.quantity_available ?? null) === (params.quantityAvailable ?? null); - if (unchanged) return false; + const ageMs = Date.now() - new Date(r.time).getTime(); + const ageDays = ageMs / (1000 * 60 * 60 * 24); + if (ageDays < STOCK_REFRESH_DAYS) { + const unchanged = + (r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) && + (r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) && + (r.backorder_qty ?? null) === (params.backorderQty ?? null) && + (r.units_sold ?? null) === (params.unitsSold ?? null) && + (r.quantity_available ?? null) === (params.quantityAvailable ?? null); + if (unchanged) return false; + } } const inStock = From 3ebb55264701a15140504c745f4f265c2c56279b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 11:58:09 +0000 Subject: [PATCH 2/6] fix(stock): repair Flexoptix stock write path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs blocked all Flexoptix stock_observations: 1. db.ts: compatible_brands null param needs ::text[] cast; price_currency needs ::bpchar cast in stock_observations INSERT 2. flexoptix-api-sync.ts: UPDATE transceivers used $1 in WHERE IS NOT NULL clause without explicit type cast — PostgreSQL cant infer type of untyped null params in WHERE context. Fixed with $1::text / $2::text casts. Result: 346 stock observations written on first run after fix. --- packages/scraper/src/robots/flexoptix-api-sync.ts | 6 +++--- packages/scraper/src/utils/db.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts index cb5c1ce..f180dcb 100644 --- a/packages/scraper/src/robots/flexoptix-api-sync.ts +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -361,11 +361,11 @@ async function importProduct( if (product.imageUrl || product.url) { await pool.query(` UPDATE transceivers SET - image_url = COALESCE(NULLIF(image_url, ''), $1), - product_page_url = COALESCE(NULLIF(product_page_url, ''), $2), + image_url = COALESCE(NULLIF(image_url, ''), $1::text), + product_page_url = COALESCE(NULLIF(product_page_url, ''), $2::text), updated_at = NOW() WHERE id = $3 - AND ($1 IS NOT NULL OR $2 IS NOT NULL) + AND ($1::text IS NOT NULL OR $2::text IS NOT NULL) `, [product.imageUrl ?? null, product.url ?? null, transceiverId]); } diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index 903ca68..bc2b76a 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -461,8 +461,8 @@ export async function upsertStockObservation(params: { $5, $6::date, $7, $8::date, $9, $10::date, - $11, $12, $13, $14, $15, - $16, $17, $18, $19 + $11, $12, $13::text[], $14, $15, + $16, $17::bpchar, $18, $19 )`, [ params.transceiverId, From 91d19e152d69a30b5c9412487036afa5b0de300c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 12:15:43 +0000 Subject: [PATCH 3/6] fix(equivalences): resolve wavelength column mismatch + reach-tier substitution policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the equivalence review-queue backlog: spec-matcher read transceivers.wavelengths (text), but the enricher wrote wavelength_tx_nm (int) — 27,908 rows had the wavelength in the wrong column, and tx_nm itself defaulted incorrectly to 1310nm in ~22% of disagreements with the authoritative part-derived value. sql/131: backfill + sync trigger from wavelength_tx_nm into wavelengths. sql/132: correct authority (bare-number wavelengths, not tx_nm) + format normalization + valid promotion on the full spec-matcher gate. sql/133: one-directional reach-tier substitution (higher reach may satisfy a lower-reach requirement, never the reverse; ZR/coherent excluded — real link-budget/receiver-overload risk, not a simple tier comparison). Tagged as its own status (compatible_superset), not merged into auto_approved. --- sql/131-wavelength-column-sync.sql | 60 ++++++++++++++++++ sql/132-wavelength-authority-normalize.sql | 74 ++++++++++++++++++++++ sql/133-reach-substitution-policy.sql | 46 ++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 sql/131-wavelength-column-sync.sql create mode 100644 sql/132-wavelength-authority-normalize.sql create mode 100644 sql/133-reach-substitution-policy.sql diff --git a/sql/131-wavelength-column-sync.sql b/sql/131-wavelength-column-sync.sql new file mode 100644 index 0000000..19c8d6f --- /dev/null +++ b/sql/131-wavelength-column-sync.sql @@ -0,0 +1,60 @@ +-- sql/131 — Wavelength column sync (root-cause fix for equivalence review backlog) +-- +-- Root cause: the enricher populates the structured integer column +-- transceivers.wavelength_tx_nm, but the equivalence spec-matcher reads the TEXT +-- column transceivers.wavelengths via tip_extract_wavelength_nm(). 27,908 rows had +-- wavelength_tx_nm set while wavelengths was NULL, so the matcher could not confirm +-- wavelength on either side and dumped ~3,155 genuine equivalences into manual review. +-- +-- Fix (no fabrication — copies existing structured data into the format the matcher +-- expects, and keeps it in sync going forward): +-- A) one-off backfill of transceivers.wavelengths from wavelength_tx_nm +-- B) durable BEFORE-trigger so future ingests stay in sync +-- C) promote pending -> auto_approved ONLY where the FULL spec-matcher criteria now +-- agree (form_factor + speed + fiber_type + reach-tier + wavelength +/-10nm). +-- Everything with a genuine reach-tier / fiber / wavelength difference stays pending. + +BEGIN; + +-- A) one-off backfill (existing structured data -> text column the matcher reads) +UPDATE transceivers + SET wavelengths = wavelength_tx_nm || 'nm' + WHERE (wavelengths IS NULL OR wavelengths = '') + AND wavelength_tx_nm IS NOT NULL; + +-- B) durable sync trigger for future inserts/updates +CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$ +BEGIN + IF (NEW.wavelengths IS NULL OR NEW.wavelengths = '') + AND NEW.wavelength_tx_nm IS NOT NULL THEN + NEW.wavelengths := NEW.wavelength_tx_nm || 'nm'; + END IF; + RETURN NEW; +END; +$fn$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_sync_wavelengths ON transceivers; +CREATE TRIGGER trg_sync_wavelengths + BEFORE INSERT OR UPDATE ON transceivers + FOR EACH ROW EXECUTE FUNCTION tip_sync_wavelengths_from_tx(); + +-- C) valid promotion: pending -> auto_approved where full spec now confirmed +UPDATE transceiver_equivalences e + SET status = 'auto_approved', + confidence = GREATEST(e.confidence, 0.85), + match_notes = COALESCE(e.match_notes,'') || ' | sql/131: full spec confirmed after wavelength column sync', + updated_at = now() + FROM transceivers fx, transceivers cx + WHERE e.flexoptix_id = fx.id + AND e.competitor_id = cx.id + AND e.status = 'pending' + AND fx.form_factor = cx.form_factor + AND fx.speed_gbps = cx.speed_gbps + AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type + AND fx.reach_meters >= 10 AND cx.reach_meters >= 10 + AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters) + AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL + AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL + AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10; + +COMMIT; diff --git a/sql/132-wavelength-authority-normalize.sql b/sql/132-wavelength-authority-normalize.sql new file mode 100644 index 0000000..2665bda --- /dev/null +++ b/sql/132-wavelength-authority-normalize.sql @@ -0,0 +1,74 @@ +-- sql/132 — Wavelength authority correction + format normalization +-- +-- Follow-up to sql/131. Discovery: transceivers.wavelengths holds the part-derived +-- wavelength as a BARE number ("1470") for ~16.8k rows, which tip_extract_wavelength_nm() +-- cannot read (it requires the "nm" suffix). Meanwhile wavelength_tx_nm disagrees with +-- that authoritative bare number 33% of the time, and 64% of those disagreements are a +-- systematic tx_nm=1310 enricher default. So the bare-number column is authoritative; +-- tx_nm is not. This migration prefers the authoritative source. +-- +-- 1) normalize authoritative bare-number wavelengths -> "NNNnm" (format only, no value change) +-- 2) clean "N/A" text -> NULL +-- 3) correct the few CWDM rows sql/131 backfilled with a wrong tx_nm default, using the +-- deterministic CWDM wavelength in the part number +-- 4) re-promote pending -> auto_approved where the FULL spec gate now agrees on +-- authoritative data (gate already protects against tx_nm noise: both sides must +-- agree within +/-10nm, so a wrong 1310 default can never auto-approve a real 1470). + +BEGIN; + +-- 1) normalize authoritative bare-number wavelengths (preserve value, add unit) +UPDATE transceivers + SET wavelengths = wavelengths || 'nm' + WHERE wavelengths ~ '^\d{3,4}$'; + +-- 2) non-values -> NULL (N/A, -, empty, etc.) +UPDATE transceivers + SET wavelengths = NULL + WHERE wavelengths IS NOT NULL + AND tip_extract_wavelength_nm(wavelengths) IS NULL + AND wavelengths !~ '\d'; + +-- 3) correct CWDM rows that sql/131 filled from a wrong tx_nm default: +-- use the deterministic CWDM wavelength encoded in the part number +UPDATE transceivers + SET wavelengths = (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1] || 'nm' + WHERE wavelengths = wavelength_tx_nm || 'nm' + AND part_number ~ '(12[6-9]|13[0-9]|14[0-9]|15[0-9]|16[01])0' + AND (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1]::int <> wavelength_tx_nm; + +-- 3b) upgrade the sync trigger so future ingests self-normalize: +-- prefer an authoritative bare-number wavelength (add "nm"), fall back to tx_nm only +-- when no textual wavelength is present at all. +CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$ +BEGIN + IF NEW.wavelengths ~ '^\d{3,4}$' THEN + NEW.wavelengths := NEW.wavelengths || 'nm'; + ELSIF (NEW.wavelengths IS NULL OR NEW.wavelengths = '' OR tip_extract_wavelength_nm(NEW.wavelengths) IS NULL) + AND NEW.wavelength_tx_nm IS NOT NULL THEN + NEW.wavelengths := NEW.wavelength_tx_nm || 'nm'; + END IF; + RETURN NEW; +END; +$fn$ LANGUAGE plpgsql; + +-- 4) valid promotion on corrected/authoritative data (same full spec gate as spec-matcher) +UPDATE transceiver_equivalences e + SET status = 'auto_approved', + confidence = GREATEST(e.confidence, 0.85), + match_notes = COALESCE(e.match_notes,'') || ' | sql/132: full spec confirmed on authoritative wavelength', + updated_at = now() + FROM transceivers fx, transceivers cx + WHERE e.flexoptix_id = fx.id + AND e.competitor_id = cx.id + AND e.status = 'pending' + AND fx.form_factor = cx.form_factor + AND fx.speed_gbps = cx.speed_gbps + AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type + AND fx.reach_meters >= 10 AND cx.reach_meters >= 10 + AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters) + AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL + AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL + AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10; + +COMMIT; diff --git a/sql/133-reach-substitution-policy.sql b/sql/133-reach-substitution-policy.sql new file mode 100644 index 0000000..71ab77c --- /dev/null +++ b/sql/133-reach-substitution-policy.sql @@ -0,0 +1,46 @@ +-- sql/133 — One-directional reach-tier substitution policy +-- +-- Decision (Rene, 2026-07-07): a higher-reach optic may satisfy a lower-reach +-- requirement (e.g. a 40km ER can serve where a 10km LR is asked for), never the +-- reverse — the shorter-reach part cannot close the link budget on a longer run. +-- ZR/coherent/amplified tiers are explicitly EXCLUDED from this automatic rule: +-- those can overload a short-haul receiver without an attenuator, which is a real +-- power-budget calculation, not a simple tier comparison. Those stay in manual review. +-- +-- Result is tagged as its own status (compatible_superset), never merged into +-- auto_approved/exact-spec matches, so downstream consumers (Fossy/Beo/Pulso, pricing) +-- can distinguish "identical spec" from "safe higher-spec substitute" — they usually +-- differ in price and are not a like-for-like swap. + +BEGIN; + +-- widen the status check (additive — existing values still valid) +ALTER TABLE transceiver_equivalences DROP CONSTRAINT transceiver_equivalences_status_check; +ALTER TABLE transceiver_equivalences ADD CONSTRAINT transceiver_equivalences_status_check + CHECK (status::text = ANY (ARRAY['pending','approved','rejected','auto_approved','compatible_superset']::text[])); + +UPDATE transceiver_equivalences e + SET status = 'compatible_superset', + confidence = GREATEST(e.confidence, 0.75), + match_notes = COALESCE(e.match_notes,'') || ' | sql/133: ' || + CASE WHEN fx.reach_meters > cx.reach_meters + THEN 'flexoptix ('||tip_reach_tier(fx.reach_meters)||') covers competitor requirement ('||tip_reach_tier(cx.reach_meters)||')' + ELSE 'competitor ('||tip_reach_tier(cx.reach_meters)||') covers flexoptix requirement ('||tip_reach_tier(fx.reach_meters)||')' + END || ' — reach-tier substitute, not identical spec', + updated_at = now() + FROM transceivers fx, transceivers cx + WHERE e.flexoptix_id = fx.id + AND e.competitor_id = cx.id + AND e.status = 'pending' + AND fx.form_factor = cx.form_factor + AND fx.speed_gbps = cx.speed_gbps + AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type + AND fx.reach_meters >= 10 AND cx.reach_meters >= 10 + AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL + AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL + AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10 + AND tip_reach_tier(fx.reach_meters) <> tip_reach_tier(cx.reach_meters) + AND tip_reach_tier(fx.reach_meters) <> 'ZR' + AND tip_reach_tier(cx.reach_meters) <> 'ZR'; + +COMMIT; From 764f16d68f11b58305753331c37e9ee9705fc52b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 12:22:33 +0000 Subject: [PATCH 4/6] feat(flexoptix): capture self_configure_price + use API type field - Add two-phase sync: bulk catalog fetch then targeted batch enrichment (50 SKUs/call, ~10 extra calls) to retrieve self_configure_price field which is absent from paginated responses - Write self_configure_price as flexoptix-selfconfigure marketplace rows in price_observations (343/346 products enriched on first run) - Add list_price support (flexoptix-msrp marketplace) for future use - Use API-provided type field (Transceiver/Cables/Accessories) in categoryFor() instead of inferring from title alone - Add productType field to CatalogProduct interface --- .../scraper/src/robots/flexoptix-api-sync.ts | 113 +++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts index f180dcb..4ca7a2a 100644 --- a/packages/scraper/src/robots/flexoptix-api-sync.ts +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -40,8 +40,11 @@ interface CatalogProduct { title: string; url: string | null; imageUrl: string | null; + productType: string | null; price: { amount: number | null; + listAmount: number | null; + selfConfigureAmount: number | null; currency: string | null; source: "api" | "missing"; fetchedAt: string; @@ -255,7 +258,10 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"])); const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"])); + const productType = asString(pick(flat, ["type", "producttype", "moduletype"])); const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"])); + const listAmount = asNumber(pick(flat, ["listprice", "msrp", "rrp", "recommendedprice"])); + const selfConfigureAmount = asNumber(pick(flat, ["selfconfigureprice", "selfconfigprice"])); const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"])) ?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR"); const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"])); @@ -277,10 +283,13 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | fetchedAt, sku, title, + productType, url, imageUrl, price: { amount, + listAmount: (listAmount !== null && listAmount !== amount) ? listAmount : null, + selfConfigureAmount, currency, source: amount === null ? "missing" : "api", fetchedAt, @@ -332,6 +341,8 @@ function speedLabel(speedGbps: number | null): string | undefined { } function categoryFor(product: CatalogProduct): string { + const apiType = product.productType?.toLowerCase() ?? ""; + if (apiType.includes("cable")) return "Cable"; const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase(); if (/\bdac\b|direct attach|copper/.test(text)) return "DAC"; if (/\baoc\b|active optical/.test(text)) return "AOC"; @@ -389,6 +400,48 @@ async function importProduct( }); } + if (product.price.selfConfigureAmount !== null && product.price.currency) { + await upsertPriceObservation({ + transceiverId, + sourceVendorId: vendorId, + price: product.price.selfConfigureAmount, + currency: product.price.currency, + marketplace: "flexoptix-selfconfigure", + stockLevel: product.stock.status ?? "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + url: product.url ?? undefined, + contentHash: contentHash({ + source: product.source, + sku: product.sku, + price: product.price.selfConfigureAmount, + currency: product.price.currency, + marketplace: "flexoptix-selfconfigure", + fetchedAt: product.price.fetchedAt, + }), + }); + } + + if (product.price.listAmount !== null && product.price.currency) { + await upsertPriceObservation({ + transceiverId, + sourceVendorId: vendorId, + price: product.price.listAmount, + currency: product.price.currency, + marketplace: "flexoptix-msrp", + stockLevel: product.stock.status ?? "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + url: product.url ?? undefined, + contentHash: contentHash({ + source: product.source, + sku: product.sku, + price: product.price.listAmount, + currency: product.price.currency, + marketplace: "flexoptix-msrp", + fetchedAt: product.price.fetchedAt, + }), + }); + } + const stockWritten = await upsertStockObservation({ transceiverId, sourceVendorId: vendorId, @@ -492,6 +545,46 @@ function extractRows(payload: unknown): JsonRecord[] { // ── Main export ──────────────────────────────────────────────────────────── + +// ── Self-configure price enrichment ─────────────────────────────────────── +// The bulk paginated endpoint omits self_configure_price. +// A targeted sku[] fetch returns it. Batch 50 SKUs per call (~10 calls total). + +async function fetchSelfConfigurePrices( + skus: string[], + baseUrl: string, + headers: Record, + currency: string, + timeoutMs: number, +): Promise> { + const BATCH_SIZE = 50; + const priceMap = new Map(); + const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products"; + + for (let i = 0; i < skus.length; i += BATCH_SIZE) { + const batch = skus.slice(i, i + BATCH_SIZE); + const url = buildUrl(baseUrl, productPath); + url.searchParams.set("currency", currency); + batch.forEach(sku => url.searchParams.append("sku[]", sku)); + + try { + const payload = await fetchJson(url, { headers }, timeoutMs); + for (const row of extractRows(payload)) { + if (!isRecord(row)) continue; + const sku = asString(row["sku"] ?? row["SKU"]); + const flat = flatLookup(row); + const scPrice = asNumber(flat["selfconfigureprice"] ?? flat["selfconfigprice"] ?? null); + if (sku && scPrice !== null) priceMap.set(sku, scPrice); + } + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.warn(`[${new Date().toISOString()}] Self-configure enrichment batch failed: ${msg.slice(0, 80)}`); + } + } + + return priceMap; +} + export async function syncFlexoptixCatalog(): Promise { const { baseUrl, username, password, token } = validateEnv(); const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10); @@ -517,7 +610,25 @@ export async function syncFlexoptixCatalog(): Promise { const importable = products.filter(canImportProduct); const skipped = products.length - importable.length; - console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`); + // Enrich importable products with self_configure_price (not in bulk response) + const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR"; + const scPrices = await fetchSelfConfigurePrices( + importable.map(p => p.sku), + baseUrl, + headers, + currency, + timeoutMs, + ); + let scEnriched = 0; + for (const product of importable) { + const scPrice = scPrices.get(product.sku); + if (scPrice !== undefined) { + product.price.selfConfigureAmount = scPrice; + scEnriched++; + } + } + + console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped} | self-configure enriched: ${scEnriched}`); const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net"); From eb2888cb0e2494a72c535be3753dc2299a2f24db Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 14:25:21 +0000 Subject: [PATCH 5/6] feat(flexoptix): populate flexoptix_product_map from API compatibility data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse Flexoptix API compat entries (compatible_to_vendor + original_part_number) and batch-upsert into flexoptix_product_map during each catalog sync. - Fix extractCompatibility() to recognise compatibleToVendor key from API - Add batchUpsertProductMap(): 500-row batch INSERTs with ON CONFLICT update - Add writeProductMapEntries() per product, called after price/stock writes - Track mapWrites in FlexoptixSyncResult and log in completion message First sync: 9254 upserts → 2903 unique rows, 95 vendors, 343 Flexoptix SKUs. Example: CWDM-SFP10G-1330 (Cisco) → P.1696.10.D.I (EUR 150.77, SFP+ 10G). --- .../scraper/src/robots/flexoptix-api-sync.ts | 74 ++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts index 4ca7a2a..28129f1 100644 --- a/packages/scraper/src/robots/flexoptix-api-sync.ts +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -81,6 +81,7 @@ export interface FlexoptixSyncResult { skipped: number; priceWrites: number; stockWrites: number; + mapWrites: number; } // ── Generic helpers ──────────────────────────────────────────────────────── @@ -239,7 +240,7 @@ function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : []; return rows.flatMap(entry => { const flat = flatLookup(entry); - const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"])); + const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor", "compatibleToVendor"])); if (!vendor) return []; return [{ vendor, @@ -362,7 +363,7 @@ async function importProduct( speedGbps: product.optics.speedGbps ?? undefined, speed: speedLabel(product.optics.speedGbps), reachMeters: product.optics.reachM ?? undefined, - reachLabel: reachLabel(product.optics.reachM), + reachLabel: reachLabel(product.optics.reachM) ?? undefined, fiberType: product.optics.fiberType ?? undefined, wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`, category: categoryFor(product), @@ -456,6 +457,22 @@ async function importProduct( return { priceWritten, stockWritten }; } +async function writeProductMapEntries(product: CatalogProduct): Promise { + const entries = product.compatibility + .filter(c => c.coding !== null && c.vendor !== null) + .map(c => ({ + oemPartNumber: c.coding!, + oemVendor: c.vendor, + flexoptixSku: product.sku, + priceEur: product.price.amount, + formFactor: product.optics.formFactor, + speedGbps: product.optics.speedGbps, + reachLabel: reachLabel(product.optics.reachM) ?? null, + fiberType: product.optics.fiberType, + })); + return batchUpsertProductMap(entries); +} + // ── API client ───────────────────────────────────────────────────────────── function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } { @@ -585,6 +602,54 @@ async function fetchSelfConfigurePrices( return priceMap; } + +// ── Product map batch upsert ───────────────────────────────────────────────── +// Writes OEM vendor coding entries into flexoptix_product_map. +// One row per (oem_part_number, oem_vendor) pair — ON CONFLICT updates specs. + +async function batchUpsertProductMap(entries: Array<{ + oemPartNumber: string; + oemVendor: string; + flexoptixSku: string; + priceEur: number | null; + formFactor: string | null; + speedGbps: number | null; + reachLabel: string | null; + fiberType: string | null; +}>): Promise { + if (entries.length === 0) return 0; + const BATCH_SIZE = 500; + let written = 0; + for (let i = 0; i < entries.length; i += BATCH_SIZE) { + const chunk = entries.slice(i, i + BATCH_SIZE); + const placeholders: string[] = []; + const params: unknown[] = []; + let n = 1; + for (const e of chunk) { + placeholders.push(`($${n},$${n+1},$${n+2},$${n+3},$${n+4},$${n+5},$${n+6},$${n+7},'exact',NOW())`); + params.push(e.oemPartNumber, e.oemVendor, e.flexoptixSku, e.priceEur, e.formFactor, e.speedGbps, e.reachLabel, e.fiberType); + n += 8; + } + await pool.query(` + INSERT INTO flexoptix_product_map + (oem_part_number, oem_vendor, flexoptix_sku, flexoptix_price_eur, + form_factor, speed_gbps, reach_label, fiber_type, match_type, last_verified) + VALUES ${placeholders.join(',')} + ON CONFLICT (oem_part_number, oem_vendor) DO UPDATE SET + flexoptix_sku = EXCLUDED.flexoptix_sku, + flexoptix_price_eur = EXCLUDED.flexoptix_price_eur, + form_factor = EXCLUDED.form_factor, + speed_gbps = EXCLUDED.speed_gbps, + reach_label = EXCLUDED.reach_label, + fiber_type = EXCLUDED.fiber_type, + last_verified = EXCLUDED.last_verified, + updated_at = NOW() + `, params); + written += chunk.length; + } + return written; +} + export async function syncFlexoptixCatalog(): Promise { const { baseUrl, username, password, token } = validateEnv(); const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10); @@ -634,19 +699,21 @@ export async function syncFlexoptixCatalog(): Promise { let priceWrites = 0; let stockWrites = 0; + let mapWrites = 0; for (const product of importable) { try { const result = await importProduct(product, vendorId); if (result.priceWritten) priceWrites++; if (result.stockWritten) stockWrites++; + mapWrites += await writeProductMapEntries(product); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`); } } - console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`); + console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes, ${mapWrites} product-map upserts`); return { fetched: rawRows.length, @@ -654,5 +721,6 @@ export async function syncFlexoptixCatalog(): Promise { skipped, priceWrites, stockWrites, + mapWrites, }; } From 443c1f08c693e5203470f84586adf54db6073cd2 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 7 Jul 2026 14:44:57 +0000 Subject: [PATCH 6/6] security: exclude flexoptix-marketplace data from all public API endpoints Public price-comparison and hot-topics routes previously had no filter for the flexoptix/flexoptix-selfconfigure/flexoptix-msrp marketplaces. Flexoptix API data is customer-confidential and must not leave the system. Adds marketplace NOT LIKE flexoptix% filter to: - /api/price-comparison (top-50 CTE) - /api/price-comparison/summary (both CTEs) - /api/price-comparison/:sku (per-vendor subquery) - /api/hot-topics price_drops query (vendor name filter) --- packages/api/src/routes/hot-topics.ts | 1 + packages/api/src/routes/price-comparison.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/packages/api/src/routes/hot-topics.ts b/packages/api/src/routes/hot-topics.ts index 0d6e229..ba9c4d6 100644 --- a/packages/api/src/routes/hot-topics.ts +++ b/packages/api/src/routes/hot-topics.ts @@ -50,6 +50,7 @@ hotTopicsRouter.get("/", async (req, res) => { JOIN vendors v ON pc.vendor_id = v.id JOIN transceivers t ON pc.transceiver_id = t.id WHERE pc.delta_pct < -10 AND pc.detected_at > NOW() - INTERVAL '14 days' + AND v.name != 'Flexoptix' ORDER BY pc.delta_pct ASC LIMIT 5 `).catch(() => ({ rows: [] })); diff --git a/packages/api/src/routes/price-comparison.ts b/packages/api/src/routes/price-comparison.ts index 57b9673..805b137 100644 --- a/packages/api/src/routes/price-comparison.ts +++ b/packages/api/src/routes/price-comparison.ts @@ -36,6 +36,7 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => { po.price, po.currency FROM price_observations po + WHERE po.marketplace NOT LIKE 'flexoptix%' ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC ) SELECT @@ -55,6 +56,7 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => { po.price, po.currency FROM price_observations po + WHERE po.marketplace NOT LIKE 'flexoptix%' ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC ) SELECT @@ -110,6 +112,7 @@ priceComparisonRouter.get("/", async (req: Request, res: Response) => { po.price, po.currency FROM price_observations po + WHERE po.marketplace NOT LIKE 'flexoptix%' ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC ) SELECT @@ -216,6 +219,7 @@ priceComparisonRouter.get("/:sku", async (req: Request, res: Response) => { po.time FROM price_observations po WHERE po.transceiver_id = $1 + AND po.marketplace NOT LIKE 'flexoptix%' ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC ) po JOIN vendors v ON v.id = po.source_vendor_id