diff --git a/packages/scraper/src/scrapers/atgbics.ts b/packages/scraper/src/scrapers/atgbics.ts index e000dd2..434e9f4 100644 --- a/packages/scraper/src/scrapers/atgbics.ts +++ b/packages/scraper/src/scrapers/atgbics.ts @@ -2,48 +2,59 @@ * ATGBICS Scraper — Prices, Stock, Product Catalog * * ATGBICS is a UK-based independent compatible optics vendor. - * Site uses Shopify. Prices ARE present in static HTML on collection pages. + * Site uses Shopify. Uses the /products.json API for reliable, JS-free data access. * * Strategy: - * 1. Fetch each collection page (correct handles discovered 2026-04-18) - * 2. Parse product cards: name (aria-label), handle, price (£X.XX), image - * 3. Paginate via ?page=N until empty - * 4. Upsert to DB + * For each collection: GET /collections/{handle}/products.json?limit=250&page=N + * Parse JSON: title, handle, variants[0].price (GBP string), images + * Paginate until response returns < limit products. * - * No Playwright required — static HTML contains all needed data. - * Rate limited: 1 req/1 sec. Runs from Mac or Erik (no IP issues with static pages). + * Rate limited: 1 req/1 sec. Runs from Mac or Erik. + * Rewritten 2026-05-06: switched from HTML parsing to products.json API after + * Shopify's static HTML stopped rendering per-collection results correctly. */ import { ensureVendor, upsertPriceObservation, findOrCreateScrapedTransceiver, markImageVerified, pool } from "../utils/db"; import { contentHash } from "../utils/hash"; -const BASE_URL = "https://www.atgbics.com"; +const BASE_URL = "https://atgbics.com"; const HEADERS = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-GB,en;q=0.9", - // Force GBP pricing regardless of visitor IP geolocation - Cookie: "cart_currency=GBP", + Accept: "application/json", }; -const MAX_PAGES_PER_CAT = 50; +const LIMIT = 250; // Shopify products.json max per page +const MAX_PAGES_PER_CAT = 40; // 40 × 250 = 10,000 products per collection -// Correct collection handles discovered 2026-04-18 by fetching /collections/ -// Each collection has static-HTML-rendered prices (£X.XX in price__current span) +/** Collections for transceiver products — discovered 2026-05-06 via /collections.json */ const CATEGORIES = [ - // === Core speeds by form factor === - { handle: "compatible-transceivers-sfp-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 }, - { handle: "compatible-transceivers-sfp-100m", formFactor: "SFP", speed: "1G", speedGbps: 1 }, - { handle: "compatible-transceiver-sfp-bidi-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 }, - { handle: "compatible-transceivers-sfpp-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, - { handle: "compatible-transceivers-sfpp-bidi-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, - { handle: "compatible-transceivers-sfpp-cwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, - { handle: "compatible-transceivers-sfp-dwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, - { handle: "compatible-transceiver-sfp-25g", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, - { handle: "high-speed-sfp-transceivers", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, - { handle: "high-speed-sfp-transceivers-1", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, - { handle: "compatible-tansceivers-qsfp-bidi-100gbps", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, + // Core speeds by form factor + { handle: "compatible-transceivers-sfp-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { handle: "compatible-transceiver-sfp-bidi-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { handle: "compatible-transceivers-sfp-100m", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { handle: "compatible-transceivers-sfpp-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { handle: "compatible-transceivers-sfpp-bidi-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { handle: "compatible-transceivers-sfpp-cwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { handle: "compatible-transceivers-sfp-dwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { handle: "compatible-transceiver-sfp-25g", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, + { handle: "compatible-transceivers-qsfpp-40gbps", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, + { handle: "compatible-transceivers-qsfp28-100gbps",formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, + { handle: "400gbase-products", formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 }, ]; +interface ShopifyVariant { + price: string; + compare_at_price?: string | null; + available?: boolean; +} + +interface ShopifyProduct { + title: string; + handle: string; + variants: ShopifyVariant[]; + images?: Array<{ src: string }>; + tags?: string[]; +} + interface AtgbicsProduct { partNumber: string; name: string; @@ -127,11 +138,10 @@ function detectWavelength(text: string): string { /** * Extract OEM part number from the ATGBICS product name. - * Name format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}" + * Format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}" * e.g. "SFP-10G-SR Cisco® Compatible Transceiver SFP+ 10GBase-SR ..." */ function extractPartNumber(name: string): string { - // First token before first space-separated vendor name or ® symbol const pnMatch = name.match(/^([A-Z0-9][A-Z0-9._\-/+]+)/i); if (pnMatch && pnMatch[1].length >= 3 && pnMatch[1].length <= 60) { return pnMatch[1].toUpperCase(); @@ -139,86 +149,61 @@ function extractPartNumber(name: string): string { return name.split(/\s+/)[0]?.toUpperCase()?.slice(0, 60) || name.slice(0, 60); } -/** Parse a collection page HTML — returns array of products */ -function parseCategoryPage(html: string, cat: typeof CATEGORIES[number]): AtgbicsProduct[] { - const products: AtgbicsProduct[] = []; - const seen = new Set(); +/** Parse a single Shopify product JSON into our AtgbicsProduct format */ +function parseShopifyProduct( + sp: ShopifyProduct, + cat: typeof CATEGORIES[number] +): AtgbicsProduct | null { + const priceStr = sp.variants[0]?.price; + const price = priceStr ? parseFloat(priceStr) : 0; + if (!price || price <= 0 || price > 100000) return null; - // Split by product cards — class="card__info" (theme updated 2025, was "card card--product") - const cardParts = html.split(/class="card__info"/); + const name = sp.title.replace(/®/g, "").replace(/\s+/g, " ").trim(); + if (name.length < 5) return null; - for (const card of cardParts.slice(1)) { - // Product handle + name from NAME - const hrefM = card.match(/href="\/products\/([^"?#]+)"[^>]*>\s*([^<]{8,}?)\s*<\/a>/s); - if (!hrefM) continue; - const handle = hrefM[1]; - const name = hrefM[2].replace(/®/g, "").replace(/\s+/g, " ").trim(); - if (seen.has(handle)) continue; - seen.add(handle); + const fullText = `${name} ${sp.handle}`; + const speedInfo = detectSpeed(fullText, cat.speedGbps); + const ff = detectFormFactor(fullText, cat.formFactor); + const reach = detectReach(fullText); + const partNumber = extractPartNumber(name); - // Price — £X.XX in price__current (may have newline before £) - const priceM = card.match(/price__current"[^>]*>\s*£([\d,]+(?:\.\d{0,2})?)/s); - const price = priceM ? parseFloat(priceM[1].replace(",", "")) : 0; - if (!price || price <= 0 || price > 100000) continue; + // Image URL — first non-placeholder image + const rawImg = sp.images?.[0]?.src; + const imageUrl = rawImg && !rawImg.includes("no-image") ? rawImg : undefined; - // Image from data-srcset (first src) - const imgM = card.match(/data-srcset="\/\/(atgbics\.com\/cdn\/shop\/files\/[^"\s]+)/); - const imageUrl = imgM ? `https://${imgM[1].split(" ")[0]}` : undefined; - - const fullText = `${name} ${handle}`; - const speedInfo = detectSpeed(fullText, cat.speedGbps); - const ff = detectFormFactor(fullText, cat.formFactor); - const reach = detectReach(fullText); - const partNumber = extractPartNumber(name); - - products.push({ - partNumber, - name, - price, - currency: "GBP", - stockLevel: "in_stock", // ATGBICS only lists available items - url: `${BASE_URL}/products/${handle}`, - formFactor: ff, - speed: speedInfo.speed, - speedGbps: speedInfo.speedGbps, - reachLabel: reach?.label, - reachMeters: reach?.meters, - fiberType: detectFiber(fullText), - wavelength: detectWavelength(fullText), - imageUrl: imageUrl?.includes("no-image") ? undefined : imageUrl, - }); - } - - return products; + return { + partNumber, + name, + price, + currency: "GBP", + stockLevel: "in_stock", + url: `${BASE_URL}/products/${sp.handle}`, + formFactor: ff, + speed: speedInfo.speed, + speedGbps: speedInfo.speedGbps, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(fullText), + wavelength: detectWavelength(fullText), + imageUrl, + }; } -async function fetchPage(url: string): Promise { +async function fetchProductsJson(url: string): Promise { const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20000) }); if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); - return resp.text(); -} - -/** Check if a page has pagination links pointing to the next page. - * Shopify theme embeds all page numbers in the pagination nav; we check for - * a link whose href explicitly contains &page=N (not just page=N anywhere). */ -function hasNextPage(html: string, currentPage: number): boolean { - const nextPage = currentPage + 1; - // Look for an actual href with page parameter — avoids matching JavaScript vars - return ( - html.includes(`&page=${nextPage}`) || - html.includes(`?page=${nextPage}`) || - html.includes(`page%3D${nextPage}`) - ); + const data = (await resp.json()) as { products: ShopifyProduct[] }; + return data.products ?? []; } export async function scrapeAtgbics(): Promise { - console.log("=== ATGBICS Scraper Starting (static HTML, correct collection handles) ===\n"); + console.log("=== ATGBICS Scraper Starting (products.json API) ===\n"); const vendorId = await ensureVendor( "ATGBICS", "compatible", - "https://www.atgbics.com", - "https://www.atgbics.com/collections/compatible-transceivers-sfpp-10g", + "https://atgbics.com", + "https://atgbics.com/collections/compatible-transceivers-sfpp-10g", ); let totalProducts = 0; @@ -230,93 +215,78 @@ export async function scrapeAtgbics(): Promise { console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.handle}] ---`); let catTotal = 0; - // Track page-level seen URLs to detect Shopify wrap-around - const catPageSeen = new Set(); - for (let page = 1; page <= MAX_PAGES_PER_CAT; page++) { - const pageUrl = page === 1 - ? `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending¤cy=GBP` - : `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending¤cy=GBP&page=${page}`; + const pageUrl = `${BASE_URL}/collections/${cat.handle}/products.json?limit=${LIMIT}&page=${page}`; + let shopifyProducts: ShopifyProduct[]; try { - const html = await fetchPage(pageUrl); - const pageProducts = parseCategoryPage(html, cat); - - if (pageProducts.length === 0) { - console.log(` Page ${page}: 0 products — stopping`); - break; - } - - // Detect Shopify catalog wrap-around: if ALL products on this page are already seen - // from a previous page (from this category), Shopify is repeating from page 1. - const newInPage = pageProducts.filter(p => !catPageSeen.has(p.url)); - if (page > 1 && newInPage.length === 0) { - console.log(` Page ${page}: all ${pageProducts.length} already seen — catalog end`); - break; - } - pageProducts.forEach(p => catPageSeen.add(p.url)); - - console.log(` Page ${page}: ${pageProducts.length} products (${newInPage.length} new)`); - - for (const product of pageProducts) { - // Skip cross-category duplicates (same product may appear in multiple collections) - const dedupKey = `${product.url}`; - if (seenHandles.has(dedupKey)) continue; - seenHandles.add(dedupKey); - - try { - const txId = await findOrCreateScrapedTransceiver({ - partNumber: product.partNumber, - vendorId, - productUrl: product.url, - formFactor: product.formFactor, - speedGbps: product.speedGbps, - speed: product.speed, - reachMeters: product.reachMeters, - reachLabel: product.reachLabel, - fiberType: product.fiberType, - wavelengths: product.wavelength, - category: "Compatible", - }); - - const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); - const updated = await upsertPriceObservation({ - transceiverId: txId, - sourceVendorId: vendorId, - price: product.price, - currency: product.currency, - stockLevel: product.stockLevel, - url: product.url, - contentHash: hash, - }); - if (updated) priceUpdates++; - - if (product.imageUrl) { - const updatedImage = await markImageVerified(txId, product.imageUrl); - if (updatedImage) imageUpdates++; - } - - totalProducts++; - catTotal++; - } catch (err) { - console.warn(` DB error ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); - } - } - - // Check pagination - if (!hasNextPage(html, page)) { - console.log(` No page ${page + 1} — collection done`); - break; - } + shopifyProducts = await fetchProductsJson(pageUrl); } catch (err) { console.warn(` Page ${page} error: ${(err as Error).message.slice(0, 80)}`); break; } + if (shopifyProducts.length === 0) { + if (page === 1) console.log(` Empty collection — skipping`); + else console.log(` Page ${page}: 0 products — done`); + break; + } + console.log(` Page ${page}: ${shopifyProducts.length} products`); + + for (const sp of shopifyProducts) { + // Skip cross-category duplicates + if (seenHandles.has(sp.handle)) continue; + seenHandles.add(sp.handle); + + const product = parseShopifyProduct(sp, cat); + if (!product) continue; + + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + productUrl: product.url, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "Compatible", + }); + + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: product.currency, + stockLevel: product.stockLevel, + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + + if (product.imageUrl) { + const updatedImage = await markImageVerified(txId, product.imageUrl); + if (updatedImage) imageUpdates++; + } + + totalProducts++; + catTotal++; + } catch (err) { + console.warn(` DB error ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + // If we got fewer products than the limit, we're on the last page + if (shopifyProducts.length < LIMIT) break; + await sleep(1000); } - console.log(` Category total: ${catTotal} products`); + console.log(` Category total: ${catTotal} new products saved`); await sleep(1000); }