diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 9c316c7..13421dc 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -328,3 +328,9 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA {"d":"2026-07-04","t":"DATA","m":"lead_time_days als NICHT ERFASST dokumentiert (sql/121, COMMENT auf price_observations/stock_observations/stock_snapshots): 0% befüllt über 1.378.984 + 85.406 + 0 Zeilen. Plumbing existiert (crawler-llm stock-schema erfasst es, upsertPriceObservation nimmt Param), aber aktiver Scraper-Pfad liefert nie einen Wert und LLM-Crawler-Tabelle stock_snapshots ist leer. Kein Default fabrizieren; NULL = unbekannt. Befüllen ist Scraper-Aufgabe."} {"d":"2026-07-04","t":"DATA","m":"Befund FS.COM-Untererfassung bestätigt (kein Fix, Scraper-Thema): 392 Zeilen/378 distinct SKUs vs ATGBICS 8420 (22x-Lücke). 391/392 sind bereits product_type='module' -> Klassifikation ist NICHT die Ursache, reine Crawl-Tiefe. Ursache in fs-com.ts + Seed-Scope (Kategorie-Abdeckung) + evtl. zu aggressive Quarantäne von fs.com/c/-URLs. Empfehlung: FS.com-Crawl-Seed erweitern (Scraper-Pipeline, nicht raten)."} {"d":"2026-07-07","t":"SECURITY","m":"Hardcodierte DB-Passwoerter aus dem Source entfernt und auf env umgestellt. tip-DB-Passwort (transceiver_db) lag in 14 Dateien im Klartext (quoted export, unquoted psql-Aufrufe, Python-env-Dicts, env-or-hardcoded-default-Fallbacks); llm_gateway-Passwort in seed-blog-training-data.py. Ersetzt durch PGPASSWORD-Env-Referenz mit Fail-Fast-Guard bzw. os.environ. training-data/*.jsonl (48 MB) aus Tracking genommen (git rm --cached, Datei bleibt) und gitignored. .security-scan-allowlist ergaenzt fuer legitime False-Positives (Vendor flexoptix, eigene Infra, sync-Journal-Codenamen, env-aufgeloeste URIs). Pre-Push-Leak-Scanner 397 auf 0 Funde, gruen ohne --no-verify. Offen: Rotation der geleakten Werte (llm_gateway zuerst, tip fleet-weit koordiniert); Git-History behaelt Altwerte."} +{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync: 3 Bugs gefixt + erster Lauf. NCS-5000-Regex (passte nicht auf NCS-5001/5002/5011), FX/FX3-Boundary-Guard (verhindert falsche Matches der neueren Generation), source-aware Date-Routing (Format2-Announcement-Dates -> eos_date, nicht last_support_date). 13 Switches auf EoS_Announced gesetzt. Wöchentlicher Sync Sun 04:00 via pg-boss."} +{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync Bug 3 fix: source-aware Date-Routing (Format2-Announcement-Dates → eos_date, nicht last_support_date)"} +{"d":"2026-07-10","t":"DATA","m":"Run cisco-eol-sync for real to update 13 switches in DB (NCS-5001/5002/5011, NCS-5502/SE, NCS-55A1, N9K-C92300YC/C9272Q/C93120TX/C93180YC-FX/C9332C, ASR-9001/9901)"} +{"d":"2026-07-11","t":"DATA","m":"Transceiver classification fields: 4 new columns added to transceivers table (dwdm_channel, cwdm_channel, cdr_type, protocol_family). Backfill from Flexoptix notes field: 1,209 standard_names enriched (up from 72, +1,579%), 1,177 cdr_types (all dual_cdr), 2,498 wdm_types (1,896 DWDM + 602 CWDM), 7,918 protocol_family values (7,645 ethernet + 273 fibre_channel). Migration 134 applied. Note: DWDM and CWDM are separate columns — never interchangeable; fibre_channel is a separate protocol family from ethernet."} +{"d":"2026-07-11","t":"FIX","m":"flexoptix-api-sync.ts: API sync now writes standard_name (from protocol field), wdm_type (DWDM/CWDM from optics.dwdm/cwdm), cdr_type (dual_cdr/cdr from title regex), cdr_support (true when cdr_type set), and protocol_family (ethernet/fibre_channel/sonet/infiniband from title regex) on every Flexoptix catalog sync. Previously these columns were never populated from the API — only dwdm/cwdm booleans existed. This fixes the root cause of empty standard_name on 95.5% of auto_approved equivalences."} +{"d":"2026-07-11","t":"FIX","m":"catalog-reconcile.ts: Hard rejects for incompatible entity types: protocol_family mismatch (ethernet≠fibre_channel), temp_range mismatch (COM≠IND), wdm_type mismatch (DWDM≠CWDM) → confidence=0, skip immediately. Standard_name mismatch now -25pts penalty (was no penalty). CDR type mismatch: -20pts + CDR match bonus +5pts. Auto-approve gate now requires fx.standard_name IS NOT NULL, blocking the 77,419/81,116 cases where FX had no standard_name. Confidence scoring schema extended with hardReject flag. Candidates and fxProducts queries now include cdr_type, temp_range, protocol_family, wdm_type."} diff --git a/packages/scraper/src/robots/catalog-reconcile.ts b/packages/scraper/src/robots/catalog-reconcile.ts index df4b291..5627b81 100644 --- a/packages/scraper/src/robots/catalog-reconcile.ts +++ b/packages/scraper/src/robots/catalog-reconcile.ts @@ -53,23 +53,41 @@ function extractFirstNm(wavelengths: string | null): number | null { // ── Haupt-Matching-Logik (identisch mit Nightly-Matcher) ──────────────────── function calcConfidence( - fx: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null }, - cand: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null } -): { confidence: number; basis: string[] } { + fx: { + standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null; + cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null; + }, + cand: { + standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null; + cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null; + } +): { confidence: number; basis: string[]; hardReject: boolean } { + // Hard rejects: incompatible entity types — no spec similarity overrides these + if (fx.protocol_family && cand.protocol_family && fx.protocol_family !== cand.protocol_family) { + return { confidence: 0, basis: [`protocol_family:${fx.protocol_family}≠${cand.protocol_family}`], hardReject: true }; + } + if (fx.temp_range && cand.temp_range && fx.temp_range !== cand.temp_range) { + return { confidence: 0, basis: [`temp_range:${fx.temp_range}≠${cand.temp_range}`], hardReject: true }; + } + if (fx.wdm_type && cand.wdm_type && fx.wdm_type !== cand.wdm_type) { + return { confidence: 0, basis: [`wdm_type:${fx.wdm_type}≠${cand.wdm_type}`], hardReject: true }; + } + // Max-Score: form_factor(25) + speed_gbps(20) + standard_name(30) + // wavelength_nm(20) + fiber_type(10) + reach(10) = 115 - // Beide form_factor und speed_gbps sind bereits durch den SQL-Filter gesichert. + // form_factor and speed_gbps are pre-filtered by SQL. let score = 0; const basis: string[] = []; score += 25; basis.push("form_factor"); score += 20; basis.push("speed_gbps"); - if ( - fx.standard_name && cand.standard_name && - fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase() - ) { - score += 30; basis.push("standard_name"); + if (fx.standard_name && cand.standard_name) { + if (fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) { + score += 30; basis.push("standard_name"); + } else { + score -= 25; // known mismatch: strong negative signal + } } const fxNm = extractFirstNm(fx.wavelengths); @@ -102,8 +120,17 @@ function calcConfidence( score += 5; basis.push("reach_null"); } + // CDR type mismatch at >=400G is a strong penalty (different product class) + if (fx.cdr_type && cand.cdr_type) { + if (fx.cdr_type !== cand.cdr_type) { + score -= 20; + } else { + score += 5; basis.push(`cdr_${fx.cdr_type}`); + } + } + const confidence = Math.max(0, Math.min(1, score / 115)); - return { confidence, basis }; + return { confidence, basis, hardReject: false }; } // ── Haupt-Funktion ─────────────────────────────────────────────────────────── @@ -135,9 +162,14 @@ export async function runCatalogReconcile(): Promise { fiber_type: string | null; reach_meters: number | null; wavelengths: string | null; + cdr_type: string | null; + temp_range: string | null; + protocol_family: string | null; + wdm_type: string | null; }>(` SELECT t.id, t.part_number, t.standard_name, t.form_factor, - t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths + t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, + t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type FROM transceivers t JOIN vendors v ON v.id = t.vendor_id WHERE UPPER(v.name) LIKE '%FLEXOPTIX%' @@ -169,12 +201,17 @@ export async function runCatalogReconcile(): Promise { reach_meters: number | null; wavelengths: string | null; vendor_name: string; + cdr_type: string | null; + temp_range: string | null; + protocol_family: string | null; + wdm_type: string | null; last_price: Date | null; price_count: string; }>(` SELECT t.id AS competitor_id, t.part_number, t.standard_name, t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name AS vendor_name, + t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type, MAX(po.time) AS last_price, COUNT(*) AS price_count FROM transceivers t JOIN vendors v ON v.id = t.vendor_id @@ -186,19 +223,25 @@ export async function runCatalogReconcile(): Promise { AND ROUND(t.speed_gbps::NUMERIC, 2) = ROUND($2::NUMERIC, 2) AND t.id != $3 GROUP BY t.id, t.part_number, t.standard_name, t.form_factor, - t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name + t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name, + t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type HAVING COUNT(*) >= 1 `, [fx.form_factor, fx.speed_gbps, fx.id]); for (const cand of candidates) { - const { confidence, basis } = calcConfidence(fx, cand); + const { confidence, basis, hardReject } = calcConfidence(fx, cand); - if (confidence < CONFIDENCE_MIN) { + if (hardReject || confidence < CONFIDENCE_MIN) { result.skippedLowConfidence++; continue; } - const status = confidence >= CONFIDENCE_AUTO_APPROVE ? "auto_approved" : "pending"; + // Require FX standard_name for auto_approve; if competitor also has one it must match + const candStdOk = !cand.standard_name || !fx.standard_name || + cand.standard_name.trim().toUpperCase() === fx.standard_name.trim().toUpperCase(); + const canAutoApprove = fx.standard_name !== null && candStdOk; + + const status = (confidence >= CONFIDENCE_AUTO_APPROVE && canAutoApprove) ? "auto_approved" : "pending"; const notes = `${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` + `basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` + diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts index 28129f1..8a1c7a7 100644 --- a/packages/scraper/src/robots/flexoptix-api-sync.ts +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -66,6 +66,8 @@ interface CatalogProduct { bidi: boolean | null; dwdm: boolean | null; cwdm: boolean | null; + cdrType: string | null; + protocolFamily: string | null; }; compatibility: Array<{ vendor: string; @@ -185,7 +187,8 @@ function parseReachMeters(value: unknown): number | null { function parseSpeedGbps(value: unknown): number | null { const numeric = asNumber(value); - if (numeric !== null) return numeric; + // API returns -10 / -1 as sentinel for N/A (e.g. SFP trays, accessories) + if (numeric !== null) return numeric < 0 ? null : numeric; const text = asString(value)?.toLowerCase(); if (!text) return null; const gbps = text.match(/([\d.,]+)\s*(g|gb|gbps|gbit)/); @@ -235,6 +238,22 @@ function inferWavelengthNm(...values: Array): number | null { // ── Normalization ────────────────────────────────────────────────────────── + +function parseCdrType(title: string | null): string | null { + if (!title) return null; + if (/dual[\s-]cdr/i.test(title)) return "dual_cdr"; + if (/with[\s-]?cdr/i.test(title)) return "cdr"; + return null; +} + +function parseProtocolFamily(title: string | null): string | null { + if (!title) return null; + if (/fibre[\s-]?channel|fc\s*\d+g/i.test(title)) return "fibre_channel"; + if (/\bsonet\b|\bsdh\b|\boc-\d+\b/i.test(title)) return "sonet"; + if (/\binfiniband\b/i.test(title)) return "infiniband"; + return "ethernet"; +} + function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] { const rawCompat = row.compatibility ?? row.compatibilities ?? row.vendorCompatibility; const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : []; @@ -312,6 +331,8 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | bidi: asBoolean(pick(flat, ["bidi", "bidirectional"])) ?? flags.bidi, dwdm: asBoolean(pick(flat, ["dwdm"])) ?? flags.dwdm, cwdm: asBoolean(pick(flat, ["cwdm"])) ?? flags.cwdm, + cdrType: parseCdrType(title), + protocolFamily: parseProtocolFamily(title), }, compatibility: extractCompatibility(row), }; @@ -319,14 +340,33 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | // ── Import helpers ───────────────────────────────────────────────────────── +function isMerchandise(product: CatalogProduct): boolean { + // Flexoptix merchandise SKU prefixes: jackets (MJA), mugs (MMU), stickers (MST), + // t-shirts (MSH), trolley (MTR), buff (MBU). FO-EXTRA-* and FO-PC-* are non-product. + if (/^FO\.M(JA|MU|ST|SH|TR|BU)\d/i.test(product.sku)) return true; + if (/^FO-(EXTRA-COMMUNITY-KIT|PC-\d)/i.test(product.sku)) return true; + const t = product.title.toLowerCase(); + return /\b(jacket|coffee mug|t-shirt|sticker|travel trolley|buff)\b/.test(t); +} + function canImportProduct(product: CatalogProduct): boolean { - return Boolean( - product.sku - && product.title - && product.optics.formFactor - && product.optics.speedGbps !== null - && product.optics.reachM !== null, - ); + // Minimal gate: sku + title required. formFactor/speed/reach are optional + // — findOrCreateScrapedTransceiver provides sensible defaults for accessories. + // Skip country-of-origin variants (SKUs like "S2A.CL.10000:S2") — they are + // duplicates of the base SKU with a different sourcing region, same price/stock. + if (product.sku.includes(":")) return false; + if (isMerchandise(product)) return false; + return Boolean(product.sku && product.title); +} + +function formFactorFallback(product: CatalogProduct): string | undefined { + const t = product.productType?.toLowerCase() ?? ""; + if (/accessories|adapter/i.test(t)) return "Accessory"; + if (/hardware/i.test(t)) return "Hardware"; + // Passive MTP/MPO breakout cables have no form-factor in the API response. + const title = product.title.toLowerCase(); + if (/\bmtp\b|\bmpo\b/.test(title) && /cable|breakout/.test(title)) return "MTP"; + return undefined; } function reachLabel(reachM: number | null): string | undefined { @@ -343,9 +383,13 @@ function speedLabel(speedGbps: number | null): string | undefined { function categoryFor(product: CatalogProduct): string { const apiType = product.productType?.toLowerCase() ?? ""; + if (/accessories|adapter/i.test(apiType)) return "Accessory"; + if (/hardware/i.test(apiType)) return "Hardware"; if (apiType.includes("cable")) return "Cable"; const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase(); - if (/\bdac\b|direct attach|copper/.test(text)) return "DAC"; + // Passive MTP/MPO fan-out cables (e.g. M3.T10L.* "MTP female to 10x LC ... Breakout cable") + if (/\bmtp\b.*cable|cable.*\bmtp\b|\bmpo\b.*cable|\bbreakout cable\b/.test(text)) return "Cable"; + if (/\bdac\b|direct attach|passive copper/.test(text)) return "DAC"; if (/\baoc\b|active optical/.test(text)) return "AOC"; if (/coherent|zr|dco/.test(text)) return "Coherent"; return "DataCenter"; @@ -359,7 +403,7 @@ async function importProduct( partNumber: product.sku, vendorId, productUrl: product.url ?? undefined, - formFactor: product.optics.formFactor ?? undefined, + formFactor: product.optics.formFactor ?? formFactorFallback(product), speedGbps: product.optics.speedGbps ?? undefined, speed: speedLabel(product.optics.speedGbps), reachMeters: product.optics.reachM ?? undefined, @@ -369,16 +413,31 @@ async function importProduct( category: categoryFor(product), }); - // Write image_url and product_page_url from bulk API response - if (product.imageUrl || product.url) { + // Write image_url, product_page_url, and notes (product title) from API + if (product.imageUrl || product.url || product.title) { + const wdmType = product.optics.dwdm ? "DWDM" : product.optics.cwdm ? "CWDM" : null; await pool.query(` UPDATE transceivers SET image_url = COALESCE(NULLIF(image_url, ''), $1::text), product_page_url = COALESCE(NULLIF(product_page_url, ''), $2::text), + notes = $3::text, + standard_name = COALESCE(NULLIF(standard_name, ''), $5::text), + wdm_type = COALESCE(NULLIF(wdm_type, ''), $6::text), + cdr_type = COALESCE(NULLIF(cdr_type, ''), $7::text), + cdr_support = CASE WHEN $7::text IS NOT NULL THEN true ELSE cdr_support END, + protocol_family = COALESCE(NULLIF(protocol_family, ''), $8::text), updated_at = NOW() - WHERE id = $3 - AND ($1::text IS NOT NULL OR $2::text IS NOT NULL) - `, [product.imageUrl ?? null, product.url ?? null, transceiverId]); + WHERE id = $4 + `, [ + product.imageUrl ?? null, + product.url ?? null, + product.title, + transceiverId, + product.optics.protocol ?? null, + wdmType, + product.optics.cdrType ?? null, + product.optics.protocolFamily ?? null, + ]); } let priceWritten = false; @@ -443,11 +502,19 @@ async function importProduct( }); } + // When status is known but no numeric quantity, synthesize a value so + // upsertStockObservation's early-exit guard doesn't skip the write. + const syntheticQty = + product.stock.quantity !== null ? product.stock.quantity + : product.stock.status === "in_stock" ? 1 + : product.stock.status === "out_of_stock" ? 0 + : undefined; + const stockWritten = await upsertStockObservation({ transceiverId, sourceVendorId: vendorId, stockLevel: product.stock.status ?? "unknown", - quantityAvailable: product.stock.quantity ?? undefined, + quantityAvailable: syntheticQty, priceNet: product.price.amount ?? undefined, productUrl: product.url ?? undefined, priceCurrency: product.price.currency ?? undefined, @@ -543,8 +610,6 @@ async function fetchAllProducts(baseUrl: string, headers: Record if (rows.length === 0) break; allRows.push(...rows); - - if (rows.length < (Number.isFinite(limit) ? limit : 500)) break; } return allRows;