feat: rewrite ATGBICS scraper to use Shopify products.json API
Static HTML collection pages return wrong results (all redirect to same 9 products).
Switch to /collections/{handle}/products.json?limit=250&page=N API which is:
- Reliable JSON (no HTML parsing)
- Correct per-collection product lists
- Clean pagination (stop at < limit results)
- Covers 11 key transceiver collections (1G, 10G, 25G, 40G, 100G, 400G)
This commit is contained in:
Rene Fichtmueller
parent
5c882c3a46
commit
efb0c24a19
@ -2,48 +2,59 @@
|
||||
* ATGBICS Scraper — Prices, Stock, Product Catalog
|
||||
*
|
||||
* ATGBICS is a UK-based independent compatible optics vendor.
|
||||
* Site uses Shopify. Prices ARE present in static HTML on collection pages.
|
||||
* Site uses Shopify. Uses the /products.json API for reliable, JS-free data access.
|
||||
*
|
||||
* Strategy:
|
||||
* 1. Fetch each collection page (correct handles discovered 2026-04-18)
|
||||
* 2. Parse product cards: name (aria-label), handle, price (£X.XX), image
|
||||
* 3. Paginate via ?page=N until empty
|
||||
* 4. Upsert to DB
|
||||
* For each collection: GET /collections/{handle}/products.json?limit=250&page=N
|
||||
* Parse JSON: title, handle, variants[0].price (GBP string), images
|
||||
* Paginate until response returns < limit products.
|
||||
*
|
||||
* No Playwright required — static HTML contains all needed data.
|
||||
* Rate limited: 1 req/1 sec. Runs from Mac or Erik (no IP issues with static pages).
|
||||
* Rate limited: 1 req/1 sec. Runs from Mac or Erik.
|
||||
* Rewritten 2026-05-06: switched from HTML parsing to products.json API after
|
||||
* Shopify's static HTML stopped rendering per-collection results correctly.
|
||||
*/
|
||||
import { ensureVendor, upsertPriceObservation, findOrCreateScrapedTransceiver, markImageVerified, pool } from "../utils/db";
|
||||
import { contentHash } from "../utils/hash";
|
||||
|
||||
const BASE_URL = "https://www.atgbics.com";
|
||||
const BASE_URL = "https://atgbics.com";
|
||||
const HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-GB,en;q=0.9",
|
||||
// Force GBP pricing regardless of visitor IP geolocation
|
||||
Cookie: "cart_currency=GBP",
|
||||
Accept: "application/json",
|
||||
};
|
||||
|
||||
const MAX_PAGES_PER_CAT = 50;
|
||||
const LIMIT = 250; // Shopify products.json max per page
|
||||
const MAX_PAGES_PER_CAT = 40; // 40 × 250 = 10,000 products per collection
|
||||
|
||||
// Correct collection handles discovered 2026-04-18 by fetching /collections/
|
||||
// Each collection has static-HTML-rendered prices (£X.XX in price__current span)
|
||||
/** Collections for transceiver products — discovered 2026-05-06 via /collections.json */
|
||||
const CATEGORIES = [
|
||||
// === Core speeds by form factor ===
|
||||
{ handle: "compatible-transceivers-sfp-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceivers-sfp-100m", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceiver-sfp-bidi-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceivers-sfpp-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfpp-bidi-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfpp-cwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfp-dwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceiver-sfp-25g", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
|
||||
{ handle: "high-speed-sfp-transceivers", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
|
||||
{ handle: "high-speed-sfp-transceivers-1", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
|
||||
{ handle: "compatible-tansceivers-qsfp-bidi-100gbps", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
|
||||
// Core speeds by form factor
|
||||
{ handle: "compatible-transceivers-sfp-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceiver-sfp-bidi-1-25g", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceivers-sfp-100m", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
||||
{ handle: "compatible-transceivers-sfpp-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfpp-bidi-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfpp-cwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceivers-sfp-dwdm-10g", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
||||
{ handle: "compatible-transceiver-sfp-25g", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
|
||||
{ handle: "compatible-transceivers-qsfpp-40gbps", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
|
||||
{ handle: "compatible-transceivers-qsfp28-100gbps",formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
|
||||
{ handle: "400gbase-products", formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 },
|
||||
];
|
||||
|
||||
interface ShopifyVariant {
|
||||
price: string;
|
||||
compare_at_price?: string | null;
|
||||
available?: boolean;
|
||||
}
|
||||
|
||||
interface ShopifyProduct {
|
||||
title: string;
|
||||
handle: string;
|
||||
variants: ShopifyVariant[];
|
||||
images?: Array<{ src: string }>;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
interface AtgbicsProduct {
|
||||
partNumber: string;
|
||||
name: string;
|
||||
@ -127,11 +138,10 @@ function detectWavelength(text: string): string {
|
||||
|
||||
/**
|
||||
* Extract OEM part number from the ATGBICS product name.
|
||||
* Name format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}"
|
||||
* Format: "{OEM_PN} {Vendor}® Compatible Transceiver {Specs}"
|
||||
* e.g. "SFP-10G-SR Cisco® Compatible Transceiver SFP+ 10GBase-SR ..."
|
||||
*/
|
||||
function extractPartNumber(name: string): string {
|
||||
// First token before first space-separated vendor name or ® symbol
|
||||
const pnMatch = name.match(/^([A-Z0-9][A-Z0-9._\-/+]+)/i);
|
||||
if (pnMatch && pnMatch[1].length >= 3 && pnMatch[1].length <= 60) {
|
||||
return pnMatch[1].toUpperCase();
|
||||
@ -139,86 +149,61 @@ function extractPartNumber(name: string): string {
|
||||
return name.split(/\s+/)[0]?.toUpperCase()?.slice(0, 60) || name.slice(0, 60);
|
||||
}
|
||||
|
||||
/** Parse a collection page HTML — returns array of products */
|
||||
function parseCategoryPage(html: string, cat: typeof CATEGORIES[number]): AtgbicsProduct[] {
|
||||
const products: AtgbicsProduct[] = [];
|
||||
const seen = new Set<string>();
|
||||
/** Parse a single Shopify product JSON into our AtgbicsProduct format */
|
||||
function parseShopifyProduct(
|
||||
sp: ShopifyProduct,
|
||||
cat: typeof CATEGORIES[number]
|
||||
): AtgbicsProduct | null {
|
||||
const priceStr = sp.variants[0]?.price;
|
||||
const price = priceStr ? parseFloat(priceStr) : 0;
|
||||
if (!price || price <= 0 || price > 100000) return null;
|
||||
|
||||
// Split by product cards — class="card__info" (theme updated 2025, was "card card--product")
|
||||
const cardParts = html.split(/class="card__info"/);
|
||||
const name = sp.title.replace(/®/g, "").replace(/\s+/g, " ").trim();
|
||||
if (name.length < 5) return null;
|
||||
|
||||
for (const card of cardParts.slice(1)) {
|
||||
// Product handle + name from <a href="/products/..." class="card-link text-current">NAME</a>
|
||||
const hrefM = card.match(/href="\/products\/([^"?#]+)"[^>]*>\s*([^<]{8,}?)\s*<\/a>/s);
|
||||
if (!hrefM) continue;
|
||||
const handle = hrefM[1];
|
||||
const name = hrefM[2].replace(/®/g, "").replace(/\s+/g, " ").trim();
|
||||
if (seen.has(handle)) continue;
|
||||
seen.add(handle);
|
||||
const fullText = `${name} ${sp.handle}`;
|
||||
const speedInfo = detectSpeed(fullText, cat.speedGbps);
|
||||
const ff = detectFormFactor(fullText, cat.formFactor);
|
||||
const reach = detectReach(fullText);
|
||||
const partNumber = extractPartNumber(name);
|
||||
|
||||
// Price — £X.XX in price__current (may have newline before £)
|
||||
const priceM = card.match(/price__current"[^>]*>\s*£([\d,]+(?:\.\d{0,2})?)/s);
|
||||
const price = priceM ? parseFloat(priceM[1].replace(",", "")) : 0;
|
||||
if (!price || price <= 0 || price > 100000) continue;
|
||||
// Image URL — first non-placeholder image
|
||||
const rawImg = sp.images?.[0]?.src;
|
||||
const imageUrl = rawImg && !rawImg.includes("no-image") ? rawImg : undefined;
|
||||
|
||||
// Image from data-srcset (first src)
|
||||
const imgM = card.match(/data-srcset="\/\/(atgbics\.com\/cdn\/shop\/files\/[^"\s]+)/);
|
||||
const imageUrl = imgM ? `https://${imgM[1].split(" ")[0]}` : undefined;
|
||||
|
||||
const fullText = `${name} ${handle}`;
|
||||
const speedInfo = detectSpeed(fullText, cat.speedGbps);
|
||||
const ff = detectFormFactor(fullText, cat.formFactor);
|
||||
const reach = detectReach(fullText);
|
||||
const partNumber = extractPartNumber(name);
|
||||
|
||||
products.push({
|
||||
partNumber,
|
||||
name,
|
||||
price,
|
||||
currency: "GBP",
|
||||
stockLevel: "in_stock", // ATGBICS only lists available items
|
||||
url: `${BASE_URL}/products/${handle}`,
|
||||
formFactor: ff,
|
||||
speed: speedInfo.speed,
|
||||
speedGbps: speedInfo.speedGbps,
|
||||
reachLabel: reach?.label,
|
||||
reachMeters: reach?.meters,
|
||||
fiberType: detectFiber(fullText),
|
||||
wavelength: detectWavelength(fullText),
|
||||
imageUrl: imageUrl?.includes("no-image") ? undefined : imageUrl,
|
||||
});
|
||||
}
|
||||
|
||||
return products;
|
||||
return {
|
||||
partNumber,
|
||||
name,
|
||||
price,
|
||||
currency: "GBP",
|
||||
stockLevel: "in_stock",
|
||||
url: `${BASE_URL}/products/${sp.handle}`,
|
||||
formFactor: ff,
|
||||
speed: speedInfo.speed,
|
||||
speedGbps: speedInfo.speedGbps,
|
||||
reachLabel: reach?.label,
|
||||
reachMeters: reach?.meters,
|
||||
fiberType: detectFiber(fullText),
|
||||
wavelength: detectWavelength(fullText),
|
||||
imageUrl,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchPage(url: string): Promise<string> {
|
||||
async function fetchProductsJson(url: string): Promise<ShopifyProduct[]> {
|
||||
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20000) });
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
|
||||
return resp.text();
|
||||
}
|
||||
|
||||
/** Check if a page has pagination links pointing to the next page.
|
||||
* Shopify theme embeds all page numbers in the pagination nav; we check for
|
||||
* a link whose href explicitly contains &page=N (not just page=N anywhere). */
|
||||
function hasNextPage(html: string, currentPage: number): boolean {
|
||||
const nextPage = currentPage + 1;
|
||||
// Look for an actual <a> href with page parameter — avoids matching JavaScript vars
|
||||
return (
|
||||
html.includes(`&page=${nextPage}`) ||
|
||||
html.includes(`?page=${nextPage}`) ||
|
||||
html.includes(`page%3D${nextPage}`)
|
||||
);
|
||||
const data = (await resp.json()) as { products: ShopifyProduct[] };
|
||||
return data.products ?? [];
|
||||
}
|
||||
|
||||
export async function scrapeAtgbics(): Promise<void> {
|
||||
console.log("=== ATGBICS Scraper Starting (static HTML, correct collection handles) ===\n");
|
||||
console.log("=== ATGBICS Scraper Starting (products.json API) ===\n");
|
||||
|
||||
const vendorId = await ensureVendor(
|
||||
"ATGBICS",
|
||||
"compatible",
|
||||
"https://www.atgbics.com",
|
||||
"https://www.atgbics.com/collections/compatible-transceivers-sfpp-10g",
|
||||
"https://atgbics.com",
|
||||
"https://atgbics.com/collections/compatible-transceivers-sfpp-10g",
|
||||
);
|
||||
|
||||
let totalProducts = 0;
|
||||
@ -230,93 +215,78 @@ export async function scrapeAtgbics(): Promise<void> {
|
||||
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.handle}] ---`);
|
||||
let catTotal = 0;
|
||||
|
||||
// Track page-level seen URLs to detect Shopify wrap-around
|
||||
const catPageSeen = new Set<string>();
|
||||
|
||||
for (let page = 1; page <= MAX_PAGES_PER_CAT; page++) {
|
||||
const pageUrl = page === 1
|
||||
? `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending¤cy=GBP`
|
||||
: `${BASE_URL}/collections/${cat.handle}?sort_by=price-ascending¤cy=GBP&page=${page}`;
|
||||
const pageUrl = `${BASE_URL}/collections/${cat.handle}/products.json?limit=${LIMIT}&page=${page}`;
|
||||
|
||||
let shopifyProducts: ShopifyProduct[];
|
||||
try {
|
||||
const html = await fetchPage(pageUrl);
|
||||
const pageProducts = parseCategoryPage(html, cat);
|
||||
|
||||
if (pageProducts.length === 0) {
|
||||
console.log(` Page ${page}: 0 products — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
// Detect Shopify catalog wrap-around: if ALL products on this page are already seen
|
||||
// from a previous page (from this category), Shopify is repeating from page 1.
|
||||
const newInPage = pageProducts.filter(p => !catPageSeen.has(p.url));
|
||||
if (page > 1 && newInPage.length === 0) {
|
||||
console.log(` Page ${page}: all ${pageProducts.length} already seen — catalog end`);
|
||||
break;
|
||||
}
|
||||
pageProducts.forEach(p => catPageSeen.add(p.url));
|
||||
|
||||
console.log(` Page ${page}: ${pageProducts.length} products (${newInPage.length} new)`);
|
||||
|
||||
for (const product of pageProducts) {
|
||||
// Skip cross-category duplicates (same product may appear in multiple collections)
|
||||
const dedupKey = `${product.url}`;
|
||||
if (seenHandles.has(dedupKey)) continue;
|
||||
seenHandles.add(dedupKey);
|
||||
|
||||
try {
|
||||
const txId = await findOrCreateScrapedTransceiver({
|
||||
partNumber: product.partNumber,
|
||||
vendorId,
|
||||
productUrl: product.url,
|
||||
formFactor: product.formFactor,
|
||||
speedGbps: product.speedGbps,
|
||||
speed: product.speed,
|
||||
reachMeters: product.reachMeters,
|
||||
reachLabel: product.reachLabel,
|
||||
fiberType: product.fiberType,
|
||||
wavelengths: product.wavelength,
|
||||
category: "Compatible",
|
||||
});
|
||||
|
||||
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
|
||||
const updated = await upsertPriceObservation({
|
||||
transceiverId: txId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price,
|
||||
currency: product.currency,
|
||||
stockLevel: product.stockLevel,
|
||||
url: product.url,
|
||||
contentHash: hash,
|
||||
});
|
||||
if (updated) priceUpdates++;
|
||||
|
||||
if (product.imageUrl) {
|
||||
const updatedImage = await markImageVerified(txId, product.imageUrl);
|
||||
if (updatedImage) imageUpdates++;
|
||||
}
|
||||
|
||||
totalProducts++;
|
||||
catTotal++;
|
||||
} catch (err) {
|
||||
console.warn(` DB error ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check pagination
|
||||
if (!hasNextPage(html, page)) {
|
||||
console.log(` No page ${page + 1} — collection done`);
|
||||
break;
|
||||
}
|
||||
shopifyProducts = await fetchProductsJson(pageUrl);
|
||||
} catch (err) {
|
||||
console.warn(` Page ${page} error: ${(err as Error).message.slice(0, 80)}`);
|
||||
break;
|
||||
}
|
||||
|
||||
if (shopifyProducts.length === 0) {
|
||||
if (page === 1) console.log(` Empty collection — skipping`);
|
||||
else console.log(` Page ${page}: 0 products — done`);
|
||||
break;
|
||||
}
|
||||
console.log(` Page ${page}: ${shopifyProducts.length} products`);
|
||||
|
||||
for (const sp of shopifyProducts) {
|
||||
// Skip cross-category duplicates
|
||||
if (seenHandles.has(sp.handle)) continue;
|
||||
seenHandles.add(sp.handle);
|
||||
|
||||
const product = parseShopifyProduct(sp, cat);
|
||||
if (!product) continue;
|
||||
|
||||
try {
|
||||
const txId = await findOrCreateScrapedTransceiver({
|
||||
partNumber: product.partNumber,
|
||||
vendorId,
|
||||
productUrl: product.url,
|
||||
formFactor: product.formFactor,
|
||||
speedGbps: product.speedGbps,
|
||||
speed: product.speed,
|
||||
reachMeters: product.reachMeters,
|
||||
reachLabel: product.reachLabel,
|
||||
fiberType: product.fiberType,
|
||||
wavelengths: product.wavelength,
|
||||
category: "Compatible",
|
||||
});
|
||||
|
||||
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
|
||||
const updated = await upsertPriceObservation({
|
||||
transceiverId: txId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price,
|
||||
currency: product.currency,
|
||||
stockLevel: product.stockLevel,
|
||||
url: product.url,
|
||||
contentHash: hash,
|
||||
});
|
||||
if (updated) priceUpdates++;
|
||||
|
||||
if (product.imageUrl) {
|
||||
const updatedImage = await markImageVerified(txId, product.imageUrl);
|
||||
if (updatedImage) imageUpdates++;
|
||||
}
|
||||
|
||||
totalProducts++;
|
||||
catTotal++;
|
||||
} catch (err) {
|
||||
console.warn(` DB error ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// If we got fewer products than the limit, we're on the last page
|
||||
if (shopifyProducts.length < LIMIT) break;
|
||||
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
console.log(` Category total: ${catTotal} products`);
|
||||
console.log(` Category total: ${catTotal} new products saved`);
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user