feat: rewrite ATGBICS scraper to use Shopify products.json API

Static HTML collection pages return wrong results (all redirect to same 9 products).
Switch to /collections/{handle}/products.json?limit=250&page=N API which is:
- Reliable JSON (no HTML parsing)
- Correct per-collection product lists
- Clean pagination (stop at < limit results)
- Covers 11 key transceiver collections (1G, 10G, 25G, 40G, 100G, 400G)

packages/scraper/src/scrapers/atgbics.ts

@@ -2,48 +2,59 @@
  * ATGBICS Scraper — Prices, Stock, Product Catalog
  *
  * ATGBICS is a UK-based independent compatible optics vendor.
- * Site uses Shopify. Prices ARE present in static HTML on collection pages.
+ * Site uses Shopify. Uses the /products.json API for reliable, JS-free data access.
  *
  * Strategy:
- *   1. Fetch each collection page (correct handles discovered 2026-04-18)
+ *   For each collection: GET /collections/{handle}/products.json?limit=250&page=N
- *   2. Parse product cards: name (aria-label), handle, price (£X.XX), image
+ *   Parse JSON: title, handle, variants[0].price (GBP string), images
- *   3. Paginate via ?page=N until empty
+ *   Paginate until response returns < limit products.
- *   4. Upsert to DB
  *
- * No Playwright required — static HTML contains all needed data.
+ * Rate limited: 1 req/1 sec. Runs from Mac or Erik.
- * Rate limited: 1 req/1 sec. Runs from Mac or Erik (no IP issues with static pages).
+ * Rewritten 2026-05-06: switched from HTML parsing to products.json API after
+ * Shopify's static HTML stopped rendering per-collection results correctly.
  */
 import { ensureVendor, upsertPriceObservation, findOrCreateScrapedTransceiver, markImageVerified, pool } from "../utils/db";
 import { contentHash } from "../utils/hash";
 
-const BASE_URL = "https://www.atgbics.com";
+const BASE_URL = "https://atgbics.com";
 const HEADERS = {
   "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+  Accept: "application/json",
-  "Accept-Language": "en-GB,en;q=0.9",
-  // Force GBP pricing regardless of visitor IP geolocation
-  Cookie: "cart_currency=GBP",
 };
 
-const MAX_PAGES_PER_CAT = 50;
+const LIMIT = 250; // Shopify products.json max per page
+const MAX_PAGES_PER_CAT = 40; // 40 × 250 = 10,000 products per collection
 
-// Correct collection handles discovered 2026-04-18 by fetching /collections/
+/** Collections for transceiver products — discovered 2026-05-06 via /collections.json */
-// Each collection has static-HTML-rendered prices (£X.XX in price__current span)
 const CATEGORIES = [
-  // === Core speeds by form factor ===
+  // Core speeds by form factor
   { handle: "compatible-transceivers-sfp-1-25g",     formFactor: "SFP",    speed: "1G",   speedGbps: 1   },
-  { handle: "compatible-transceivers-sfp-100m",            formFactor: "SFP",     speed: "1G",   speedGbps: 1   },
   { handle: "compatible-transceiver-sfp-bidi-1-25g", formFactor: "SFP",    speed: "1G",   speedGbps: 1   },
+  { handle: "compatible-transceivers-sfp-100m",      formFactor: "SFP",    speed: "1G",   speedGbps: 1   },
   { handle: "compatible-transceivers-sfpp-10g",      formFactor: "SFP+",   speed: "10G",  speedGbps: 10  },
   { handle: "compatible-transceivers-sfpp-bidi-10g", formFactor: "SFP+",   speed: "10G",  speedGbps: 10  },
   { handle: "compatible-transceivers-sfpp-cwdm-10g", formFactor: "SFP+",   speed: "10G",  speedGbps: 10  },
   { handle: "compatible-transceivers-sfp-dwdm-10g",  formFactor: "SFP+",   speed: "10G",  speedGbps: 10  },
   { handle: "compatible-transceiver-sfp-25g",        formFactor: "SFP28",  speed: "25G",  speedGbps: 25  },
-  { handle: "high-speed-sfp-transceivers",                 formFactor: "QSFP28",  speed: "100G", speedGbps: 100 },
+  { handle: "compatible-transceivers-qsfpp-40gbps",  formFactor: "QSFP+",  speed: "40G",  speedGbps: 40  },
-  { handle: "high-speed-sfp-transceivers-1",               formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
+  { handle: "compatible-transceivers-qsfp28-100gbps",formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
-  { handle: "compatible-tansceivers-qsfp-bidi-100gbps",    formFactor: "QSFP28",  speed: "100G", speedGbps: 100 },
+  { handle: "400gbase-products",                     formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 },
 ];
 
+interface ShopifyVariant {
+  price: string;
+  compare_at_price?: string | null;
+  available?: boolean;
+}
+
+interface ShopifyProduct {
+  title: string;
+  handle: string;
+  variants: ShopifyVariant[];
+  images?: Array<{ src: string }>;
+  tags?: string[];
+}
+
 interface AtgbicsProduct {
   partNumber: string;
   name: string;
@@ -127,11 +138,10 @@ function detectWavelength(text: string): string {
 
 /**
  * Extract OEM part number from the ATGBICS product name.
- * Name format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}"
+ * Format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}"
  * e.g. "SFP-10G-SR Cisco® Compatible Transceiver SFP+ 10GBase-SR ..."
  */
 function extractPartNumber(name: string): string {
-  // First token before first space-separated vendor name or ® symbol
   const pnMatch = name.match(/^([A-Z0-9][A-Z0-9._\-/+]+)/i);
   if (pnMatch && pnMatch[1].length >= 3 && pnMatch[1].length <= 60) {
     return pnMatch[1].toUpperCase();
@@ -139,45 +149,35 @@ function extractPartNumber(name: string): string {
   return name.split(/\s+/)[0]?.toUpperCase()?.slice(0, 60) || name.slice(0, 60);
 }
 
-/** Parse a collection page HTML — returns array of products */
+/** Parse a single Shopify product JSON into our AtgbicsProduct format */
-function parseCategoryPage(html: string, cat: typeof CATEGORIES[number]): AtgbicsProduct[] {
+function parseShopifyProduct(
-  const products: AtgbicsProduct[] = [];
+  sp: ShopifyProduct,
-  const seen = new Set<string>();
+  cat: typeof CATEGORIES[number]
+): AtgbicsProduct | null {
+  const priceStr = sp.variants[0]?.price;
+  const price = priceStr ? parseFloat(priceStr) : 0;
+  if (!price || price <= 0 || price > 100000) return null;
 
-  // Split by product cards — class="card__info" (theme updated 2025, was "card card--product")
+  const name = sp.title.replace(/®/g, "").replace(/\s+/g, " ").trim();
-  const cardParts = html.split(/class="card__info"/);
+  if (name.length < 5) return null;
 
-  for (const card of cardParts.slice(1)) {
+  const fullText = `${name} ${sp.handle}`;
-    // Product handle + name from <a href="/products/..." class="card-link text-current">NAME</a>
-    const hrefM = card.match(/href="\/products\/([^"?#]+)"[^>]*>\s*([^<]{8,}?)\s*<\/a>/s);
-    if (!hrefM) continue;
-    const handle = hrefM[1];
-    const name = hrefM[2].replace(/®/g, "").replace(/\s+/g, " ").trim();
-    if (seen.has(handle)) continue;
-    seen.add(handle);
-
-    // Price — £X.XX in price__current (may have newline before £)
-    const priceM = card.match(/price__current"[^>]*>\s*£([\d,]+(?:\.\d{0,2})?)/s);
-    const price = priceM ? parseFloat(priceM[1].replace(",", "")) : 0;
-    if (!price || price <= 0 || price > 100000) continue;
-
-    // Image from data-srcset (first src)
-    const imgM = card.match(/data-srcset="\/\/(atgbics\.com\/cdn\/shop\/files\/[^"\s]+)/);
-    const imageUrl = imgM ? `https://${imgM[1].split(" ")[0]}` : undefined;
-
-    const fullText = `${name} ${handle}`;
   const speedInfo = detectSpeed(fullText, cat.speedGbps);
   const ff = detectFormFactor(fullText, cat.formFactor);
   const reach = detectReach(fullText);
   const partNumber = extractPartNumber(name);
 
-    products.push({
+  // Image URL — first non-placeholder image
+  const rawImg = sp.images?.[0]?.src;
+  const imageUrl = rawImg && !rawImg.includes("no-image") ? rawImg : undefined;
+
+  return {
     partNumber,
     name,
     price,
     currency: "GBP",
-      stockLevel: "in_stock",    // ATGBICS only lists available items
+    stockLevel: "in_stock",
-      url: `${BASE_URL}/products/${handle}`,
+    url: `${BASE_URL}/products/${sp.handle}`,
     formFactor: ff,
     speed: speedInfo.speed,
     speedGbps: speedInfo.speedGbps,
@@ -185,40 +185,25 @@ function parseCategoryPage(html: string, cat: typeof CATEGORIES[number]): Atgbic
     reachMeters: reach?.meters,
     fiberType: detectFiber(fullText),
     wavelength: detectWavelength(fullText),
-      imageUrl: imageUrl?.includes("no-image") ? undefined : imageUrl,
+    imageUrl,
-    });
+  };
 }
 
-  return products;
+async function fetchProductsJson(url: string): Promise<ShopifyProduct[]> {
-}
-
-async function fetchPage(url: string): Promise<string> {
   const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20000) });
   if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
-  return resp.text();
+  const data = (await resp.json()) as { products: ShopifyProduct[] };
-}
+  return data.products ?? [];
-
-/** Check if a page has pagination links pointing to the next page.
- * Shopify theme embeds all page numbers in the pagination nav; we check for
- * a link whose href explicitly contains &page=N (not just page=N anywhere). */
-function hasNextPage(html: string, currentPage: number): boolean {
-  const nextPage = currentPage + 1;
-  // Look for an actual <a> href with page parameter — avoids matching JavaScript vars
-  return (
-    html.includes(`&page=${nextPage}`) ||
-    html.includes(`?page=${nextPage}`) ||
-    html.includes(`page%3D${nextPage}`)
-  );
 }
 
 export async function scrapeAtgbics(): Promise<void> {
-  console.log("=== ATGBICS Scraper Starting (static HTML, correct collection handles) ===\n");
+  console.log("=== ATGBICS Scraper Starting (products.json API) ===\n");
 
   const vendorId = await ensureVendor(
     "ATGBICS",
     "compatible",
-    "https://www.atgbics.com",
+    "https://atgbics.com",
-    "https://www.atgbics.com/collections/compatible-transceivers-sfpp-10g",
+    "https://atgbics.com/collections/compatible-transceivers-sfpp-10g",
   );
 
   let totalProducts = 0;
@@ -230,39 +215,31 @@ export async function scrapeAtgbics(): Promise<void> {
     console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.handle}] ---`);
     let catTotal = 0;
 
-    // Track page-level seen URLs to detect Shopify wrap-around
-    const catPageSeen = new Set<string>();
-
     for (let page = 1; page <= MAX_PAGES_PER_CAT; page++) {
-      const pageUrl = page === 1
+      const pageUrl = `${BASE_URL}/collections/${cat.handle}/products.json?limit=${LIMIT}&page=${page}`;
-        ? `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending&currency=GBP`
-        : `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending&currency=GBP&page=${page}`;
 
+      let shopifyProducts: ShopifyProduct[];
       try {
-        const html = await fetchPage(pageUrl);
+        shopifyProducts = await fetchProductsJson(pageUrl);
-        const pageProducts = parseCategoryPage(html, cat);
+      } catch (err) {
-
+        console.warn(`  Page ${page} error: ${(err as Error).message.slice(0, 80)}`);
-        if (pageProducts.length === 0) {
-          console.log(`  Page ${page}: 0 products — stopping`);
         break;
       }
 
-        // Detect Shopify catalog wrap-around: if ALL products on this page are already seen
+      if (shopifyProducts.length === 0) {
-        // from a previous page (from this category), Shopify is repeating from page 1.
+        if (page === 1) console.log(`  Empty collection — skipping`);
-        const newInPage = pageProducts.filter(p => !catPageSeen.has(p.url));
+        else console.log(`  Page ${page}: 0 products — done`);
-        if (page > 1 && newInPage.length === 0) {
-          console.log(`  Page ${page}: all ${pageProducts.length} already seen — catalog end`);
         break;
       }
-        pageProducts.forEach(p => catPageSeen.add(p.url));
+      console.log(`  Page ${page}: ${shopifyProducts.length} products`);
 
-        console.log(`  Page ${page}: ${pageProducts.length} products (${newInPage.length} new)`);
+      for (const sp of shopifyProducts) {
+        // Skip cross-category duplicates
+        if (seenHandles.has(sp.handle)) continue;
+        seenHandles.add(sp.handle);
 
-        for (const product of pageProducts) {
+        const product = parseShopifyProduct(sp, cat);
-          // Skip cross-category duplicates (same product may appear in multiple collections)
+        if (!product) continue;
-          const dedupKey = `${product.url}`;
-          if (seenHandles.has(dedupKey)) continue;
-          seenHandles.add(dedupKey);
 
         try {
           const txId = await findOrCreateScrapedTransceiver({
@@ -303,20 +280,13 @@ export async function scrapeAtgbics(): Promise<void> {
         }
       }
 
-        // Check pagination
+      // If we got fewer products than the limit, we're on the last page
-        if (!hasNextPage(html, page)) {
+      if (shopifyProducts.length < LIMIT) break;
-          console.log(`  No page ${page + 1} — collection done`);
-          break;
-        }
-      } catch (err) {
-        console.warn(`  Page ${page} error: ${(err as Error).message.slice(0, 80)}`);
-        break;
-      }
 
       await sleep(1000);
     }
 
-    console.log(`  Category total: ${catTotal} products`);
+    console.log(`  Category total: ${catTotal} new products saved`);
     await sleep(1000);
   }