fix(scraper): FiberMall URL schema + price parser + Flexoptix EUR comma bug

FiberMall:
- Correct /store-XXXXX-name.htm category URLs (was /c/xxx/ → HTTP 404)
- Parser: split on new_proList_mainListLi, price from data-price on
  currency_price span — fix 0.00 false-match from SKU variant items
- Also scrape SKU brand variant links from .sku_item divs
- Result: 3,410 prices now in DB (was 0)

Flexoptix:
- Fix extractPrice regex for EUR thousand-separator format
  (2,921.60 EUR was parsed as 2 EUR)
- Add OSFP224 / 1.6T search queries (4 new, form factor was missing)
- Fix O.138HG2.C.05 stale price 3009.60→2921.60 EUR

Schema: competitor_verified + competitor_verified_at columns
added via ALTER TABLE (were referenced in code but missing in DB)

CHANGELOG: added 6 entries for 2026-04-12

CHANGELOG_PENDING.md

@@ -5,6 +5,17 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA
 
 ---
 
+{"d":"2026-04-12","t":"FIX","m":"FiberMall scraper: URL schema corrected — wrong /c/1g-sfp-transceiver/ paths (HTTP 404) replaced with actual /store-XXXXX-name.htm category URLs discovered via homepage navigation scrape"}
+{"d":"2026-04-12","t":"FIX","m":"FiberMall parser: product card split on new_proList_mainListLi (Vue.js SSR), price extracted from <span class=currency_price data-price=X.XX> — fixed false-match on data-price=0.00 from SKU variant items that appears before real price in each card"}
+{"d":"2026-04-12","t":"FIX","m":"FiberMall: also scrapes SKU brand variants from .sku_item divs within each product group (Cisco/Arista/Juniper compatible versions listed per product)"}
+{"d":"2026-04-12","t":"FIX","m":"Flexoptix price parsing: EUR text regex /([\d.]+)\s*EUR/ matched only digits before thousand separator (2,921.60 EUR → 2 EUR) — fixed to /([\d,]+\.?\d*)\s*EUR/ with comma strip; affects all Flexoptix prices >999 EUR"}
+{"d":"2026-04-12","t":"FIX","m":"Flexoptix catalog: O.138HG2.C.05 (1.6T OSFP224 2x DR4) price corrected 3009.60→2921.60 EUR (stale since 2026-04-09, Flexoptix.net shows FLEXBOX price 2921.60 via data-price-amount attribute)"}
+{"d":"2026-04-12","t":"FEAT","m":"Flexoptix catalog: 4 new search queries added — OSFP224 1.6T, OSFP224, 1.6T DR4, 1.6T transceiver — covers new 1.6T form factor previously missing entirely from catalog scraper"}
+{"d":"2026-04-12","t":"FIX","m":"Schema: competitor_verified + competitor_verified_at columns added to transceivers table (ALTER TABLE) — were referenced in db.ts upsertPriceObservation but not in schema, causing price writes to fail silently for all competitor vendors (FiberMall, QSFPTEK etc.)"}
+{"d":"2026-04-11","t":"FEAT","m":"Scraper coverage expansion: 3 new scrapers added — FiberMall (fibermall.com, USD), Vcelink (vcelink.com, USD, Shopify), OpticsBay (opticsbay.com, USD, WooCommerce) — all wired into scheduler and Pi fleet"}
+{"d":"2026-04-11","t":"FIX","m":"QSFPTEK scraper fully rewritten: site migrated from OpenCart to custom Java/Spring+Vue — old /c/*.html paths 404, now uses /mall/commodity/list API with attribute-based data rate filtering; 8 attribute IDs for 1G/10G/25G/40G/100G/200G/400G/800G"}
+{"d":"2026-04-11","t":"INFRA","m":"Scheduler: 61 workers total, 53 cron schedules — FiberMall/Vcelink/OpticsBay added at :03, :07, :57 past even hours"}
+{"d":"2026-04-09","t":"FEAT","m":"Price anomaly detection: PRICE_BOUNDS per form-factor in db.ts upsertPriceObservation — prices outside [min,max] USD range silently rejected to prevent garbage data (e.g. SFP+ [4, 5000], OSFP224 [200, 60000])"}
 {"d":"2026-04-09","t":"UI","m":"Dashboard: LLM panel redesigned for light theme readability; LLM model selector added to Blog Engine tab"}
 {"d":"2026-04-09","t":"INFRA","m":"Pi Starlink proxy-agent: scraper routes selected lightweight scrapers exclusively to Pi worker fleet via SOCKS5 — no Playwright traffic on Pi nodes"}
 {"d":"2026-04-09","t":"DATA","m":"800G standards deep enrichment: migration 033 — IEEE 802.3df, OIF 800G IA, 800G MSA, OSFP MSA, QSFP-DD800 MSA with links, status, timeline"}

packages/scraper/src/scrapers/fibermall.ts

@@ -1,37 +1,41 @@
 /**
  * FiberMall Scraper — Chinese compatible transceiver vendor
  *
- * fibermall.com — custom PHP/REST shop, USD pricing.
+ * fibermall.com — custom Vue.js/PHP shop, USD pricing.
- * Large catalog: 1G–400G, SFP/SFP+/QSFP28/QSFP-DD/OSFP.
+ * Large catalog: 1G–800G, SFP/SFP+/QSFP28/QSFP-DD/OSFP.
- * Pagination via ?page=N. Rate limited: 1 req/2sec.
+ * Rate limited: 1 req/2sec.
  *
- * FiberMall (Shenzhen FiberMall Technology Co.) offers DAC/AOC + optics,
+ * URL schema (discovered 2026-04-11):
- * transparent USD prices, no login required.
+ *   Category pages: /store-XXXXX-name.htm
+ *   Product pages:  /sale-XXXXXX-name.htm
+ *   Pagination:     /store-XXXXX-name.htm?page=N
+ *   Product list:   CSS class "new_proList_mainListLi"
  */
 import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
 import { contentHash } from "../utils/hash";
 
 const BASE = "https://www.fibermall.com";
 const HEADERS = {
-  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
   Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
   "Accept-Language": "en-US,en;q=0.9",
+  Referer: "https://www.fibermall.com/",
 };
 
 const MAX_PAGES = 30;
 
+// Discovered via homepage navigation scrape 2026-04-11
+// Format: /store-XXXXX-description.htm
 const CATEGORIES = [
-  { path: "/c/1g-sfp-transceiver/", formFactor: "SFP",     speed: "1G",   speedGbps: 1   },
+  { path: "/store-17147-sfp-transceivers.htm",        formFactor: "SFP",     speed: "1G",   speedGbps: 1   },
-  { path: "/c/10g-sfp-transceiver/", formFactor: "SFP+",   speed: "10G",  speedGbps: 10  },
+  { path: "/store-17014-10g-sfp.htm",                 formFactor: "SFP+",    speed: "10G",  speedGbps: 10  },
-  { path: "/c/25g-sfp28-transceiver/", formFactor: "SFP28",speed: "25G",  speedGbps: 25  },
+  { path: "/store-17012-25g-sfp28.htm",               formFactor: "SFP28",   speed: "25G",  speedGbps: 25  },
-  { path: "/c/40g-qsfp-transceiver/", formFactor: "QSFP+", speed: "40G",  speedGbps: 40  },
+  { path: "/store-16652-40g-qsfp.htm",                formFactor: "QSFP+",   speed: "40G",  speedGbps: 40  },
-  { path: "/c/100g-qsfp28-transceiver/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
+  { path: "/store-16528-100g-qsfp28.htm",             formFactor: "QSFP28",  speed: "100G", speedGbps: 100 },
-  { path: "/c/200g-qsfp56-transceiver/", formFactor: "QSFP56", speed: "200G", speedGbps: 200 },
+  { path: "/store-20654-200g-qsfp56-qsfp-dd.htm",     formFactor: "QSFP56",  speed: "200G", speedGbps: 200 },
-  { path: "/c/400g-qsfp-dd-transceiver/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
+  { path: "/store-20656-400g-qsfp-dd.htm",            formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
-  { path: "/c/800g-osfp-transceiver/", formFactor: "OSFP", speed: "800G", speedGbps: 800 },
+  { path: "/store-21972-800g-qsfp-dd-osfp.htm",       formFactor: "OSFP",    speed: "800G", speedGbps: 800 },
-  { path: "/c/dac-cable/",             formFactor: "DAC",  speed: "10G",  speedGbps: 10  },
+  { path: "/store-16527-dac-aoc-acc-aec-cables.htm",  formFactor: "DAC",     speed: "10G",  speedGbps: 10  },
-  { path: "/c/aoc-cable/",             formFactor: "AOC",  speed: "10G",  speedGbps: 10  },
-  { path: "/c/optical-transceivers/",  formFactor: "SFP+", speed: "10G",  speedGbps: 10  },
 ];
 
 interface Product {
@@ -93,29 +97,37 @@ function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product
   const seen = new Set<string>();
   const collapsed = html.replace(/\s+/g, " ");
 
-  // Strategy 1: fibermall product card (class="product-item" or similar)
+  // FiberMall HTML structure (SSR, confirmed 2026-04-11):
-  for (const m of collapsed.matchAll(/<(?:li|div|article)[^>]+class="[^"]*(?:product-item|product-thumb|goods-item|pro-item)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div|article)>/gi)) {
+  //   <li class="new_proList_mainListLi">
-    const card = m[1];
+  //     <a href="/sale-XXXXXX-name.htm" title="Full Name">...</a>
-    const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]+)"/i) ||
+  //     <span class="currency_price" data-price="12.00">12.00</span>
-                     card.match(/href="(\/[a-z0-9][^"?#]{5,})"/i);
+  //   </li>
-    if (!urlMatch) continue;
+  //
-    const url = urlMatch[1].startsWith("http") ? urlMatch[1] : BASE + urlMatch[1];
+  // Each <li> is a product GROUP with SKU variants inside .sku_item divs.
-    if (seen.has(url) || !/fibermall\.com|\/product|\/p\//i.test(url)) continue;
+  // The main product URL has a `title` attribute; sku variants do not.
+  // Price: data-price="X.XX" on <span class="currency_price">
+
+  // Split by <li class="new_proList_mainListLi"
+  const liParts = collapsed.split(/(?=<li class="new_proList_mainListLi")/);
+
+  for (const card of liParts) {
+    if (!card.includes('new_proList_mainListLi')) continue;
+
+    // Price: from <span class="currency_price" data-price="X.XX">
+    // Must target specifically currency_price span — SKU items have data-price="0.00"
+    const priceM = card.match(/class="currency_price"[^>]*data-price="([\d.]+)"/i) ||
+                   card.match(/data-price="([1-9][\d]*\.?\d{0,2})"/); // skip 0.00
+    const price = priceM ? parseFloat(priceM[1]) : undefined;
+
+    // Main product link: first <a href="/sale-..."> with title attribute
+    const mainLinkM = card.match(/href="(\/sale-\d+[^"?#]*\.htm)"[^>]*title="([^"]{8,})"/i);
+    if (mainLinkM) {
+      const url = BASE + mainLinkM[1];
+      const name = mainLinkM[2].trim().replace(/&amp;/g, "&").replace(/&#\d+;/g, "").replace(/\s+/g, " ");
+      if (!seen.has(url) && name.length >= 5) {
         seen.add(url);
-
-    const nameMatch = card.match(/<(?:h[23456]|p)[^>]+class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]{8,})<\//i) ||
-                      card.match(/title="([^"]{10,})"/i) ||
-                      card.match(/<a[^>]*>([^<]{10,})<\/a>/i);
-    if (!nameMatch) continue;
-    const name = nameMatch[1].trim().replace(/&amp;/g, "&").replace(/&#\d+;/g, "");
-    if (name.length < 5) continue;
-
-    const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/);
-    const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
-
         const reach = detectReach(name);
         const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);
-
         products.push({
           partNumber, name, url,
           price: price && price > 0 && price < 100000 ? price : undefined,
@@ -124,24 +136,17 @@ function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product
           fiberType: detectFiber(name), wavelength: detectWavelength(name),
         });
       }
+    }
 
-  // Strategy 2: generic product link scan
+    // Also collect SKU variant links from .sku_item (no title attribute, use link text)
-  if (products.length === 0) {
+    for (const m of card.matchAll(/class="sku_item[^"]*"[^>]*>\s*<a href="(\/sale-\d+[^"?#]*\.htm)"[^>]*>([^<]{5,})<\/a>/gi)) {
-    for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]{10,})"[^>]*>([^<]{10,})</gi)) {
+      const url = BASE + m[1];
-      const url = m[1];
       const name = m[2].trim().replace(/&amp;/g, "&");
-      if (seen.has(url) || name.length < 8) continue;
+      if (seen.has(url) || name.length < 4) continue;
-      if (!/transceiver|sfp|qsfp|osfp|dac|aoc|optic/i.test(name)) continue;
       seen.add(url);
-
-      const idx = collapsed.indexOf(url);
-      const ctx = collapsed.slice(Math.max(0, idx - 300), idx + 600);
-      const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
-      const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
       const reach = detectReach(name);
-
       products.push({
-        partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
+        partNumber: name.slice(0, 80),
         name, url,
         price: price && price > 0 && price < 100000 ? price : undefined,
         formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
@@ -167,7 +172,7 @@ export async function scrapeFiberMall(): Promise<void> {
     "FiberMall",
     "compatible",
     "https://www.fibermall.com",
-    "https://www.fibermall.com/c/optical-transceivers/",
+    "https://www.fibermall.com/store-16528-100g-qsfp28.htm",
   );
 
   let totalProducts = 0;
@@ -181,21 +186,6 @@ export async function scrapeFiberMall(): Promise<void> {
       const html1 = await fetchPage(BASE + cat.path);
       const catProducts = parseProductList(html1, cat);
 
-      // Skip generic fallback if specific categories already scraped
-      if (cat.path.includes("/optical-transceivers/") && seenCategories.size > 3) {
-        console.log(`  Skipping generic fallback (${seenCategories.size} categories scraped)`);
-        continue;
-      }
-
-      if (catProducts.length === 0) {
-        console.log("  No products on page 1 — trying alternate pagination");
-        // Try ?page=1 format
-        try {
-          const html1alt = await fetchPage(BASE + cat.path + "?page=1");
-          catProducts.push(...parseProductList(html1alt, cat));
-        } catch { /* ignore */ }
-      }
-
       if (catProducts.length === 0) {
         console.log("  No products found — skipping");
         continue;
@@ -209,7 +199,7 @@ export async function scrapeFiberMall(): Promise<void> {
       for (let page = 2; page <= MAX_PAGES; page++) {
         await sleep(2000);
         try {
-          // Try both pagination formats
+          // FiberMall pagination: ?page=N
           const pageUrl = `${BASE}${cat.path}?page=${page}`;
           const html = await fetchPage(pageUrl);
           const pageProds = parseProductList(html, cat);

packages/scraper/src/scrapers/flexoptix-catalog.ts

@@ -72,6 +72,10 @@ const SEARCH_QUERIES = [
   { query: "OSFP LR4", formFactor: "OSFP", speed: "400G", speedGbps: 400 },
   { query: "OSFP ZR", formFactor: "OSFP", speed: "400G", speedGbps: 400 },
   { query: "OSFP 800G", formFactor: "OSFP", speed: "800G", speedGbps: 800 },
+  { query: "OSFP224 1.6T", formFactor: "OSFP224", speed: "1.6T", speedGbps: 1600 },
+  { query: "OSFP224", formFactor: "OSFP224", speed: "1.6T", speedGbps: 1600 },
+  { query: "1.6T DR4", formFactor: "OSFP224", speed: "1.6T", speedGbps: 1600 },
+  { query: "1.6T transceiver", formFactor: "OSFP224", speed: "1.6T", speedGbps: 1600 },
   // Additional granular queries for maximum coverage
   { query: "SFP+ copper", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
   { query: "SFP+ 10GBASE-T", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
@@ -266,11 +270,13 @@ async function searchProducts(query: string): Promise<SearchResult[]> {
       // Try data-price-amount attribute first (Magento Hyva theme)
       const attrMatch = s.match(/data-price-amount="([\d.]+)"/);
       if (attrMatch) return attrMatch[1];
-      // Try plain price text like "39.64 EUR"
+      // Try plain price text like "2,921.60 EUR" or "39.64 EUR"
-      const textMatch = s.match(/([\d.]+)\s*EUR/i);
+      // IMPORTANT: must include comma in char class to handle thousand separators
-      if (textMatch) return textMatch[1];
+      const textMatch = s.match(/([\d,]+\.?\d*)\s*EUR/i);
-      // Try bare number
+      if (textMatch) return textMatch[1].replace(/,/g, "");
-      const num = parseFloat(s);
+      // Try bare number (strip thousand-separator commas first)
+      const cleaned = s.replace(/,/g, "");
+      const num = parseFloat(cleaned);
       if (!isNaN(num) && num > 0) return String(num);
       return undefined;
     }