2e852e0a2f
- 16 commercial scrapers: replace TIP-Bot/1.0 with Chrome/120 UA (GBICS confirmed returning 0 bytes for bot UA, Chrome UA returns 200KB) - gbics.ts: fix User-Agent (was returning empty HTML, now returns products) - optictransceiver.ts: disable — domain repurposed as plant shop (2026-04-06) Alocasia Regal Shield is not a transceiver.
250 lines
9.7 KiB
TypeScript
250 lines
9.7 KiB
TypeScript
/**
|
|
* GBICS.com Scraper — UK-based compatible transceiver vendor
|
|
*
|
|
* gbics.com — BigCommerce store, server-rendered HTML, GBP prices.
|
|
* Products in <li> cards with <h3> product names, "Now: £XX.XX" pricing.
|
|
* Pagination via ?page=N. Rate limited: 1 req/2sec.
|
|
*/
|
|
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
|
|
import { contentHash } from "../utils/hash";
|
|
|
|
const BASE = "https://www.gbics.com";
|
|
const HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
};
|
|
|
|
const CATEGORIES = [
|
|
{ path: "/800g-osfp/", formFactor: "OSFP", speed: "800G", speedGbps: 800 },
|
|
{ path: "/400g-qsfp112/", formFactor: "QSFP112", speed: "400G", speedGbps: 400 },
|
|
{ path: "/400g-qsfp-dd/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
|
|
{ path: "/400g-osfp/", formFactor: "OSFP", speed: "400G", speedGbps: 400 },
|
|
{ path: "/200g-qsfp56/", formFactor: "QSFP56", speed: "200G", speedGbps: 200 },
|
|
{ path: "/100g-qsfp28/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
|
|
{ path: "/40g-qsfp/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
|
|
{ path: "/25g-sfp28/", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
|
|
{ path: "/10g-sfp/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
|
{ path: "/1g-sfp/", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
|
];
|
|
|
|
interface Product {
|
|
partNumber: string;
|
|
name: string;
|
|
url: string;
|
|
price?: number;
|
|
formFactor: string;
|
|
speed: string;
|
|
speedGbps: number;
|
|
reachLabel?: string;
|
|
reachMeters?: number;
|
|
fiberType?: string;
|
|
wavelength?: string;
|
|
compatibleWith?: string;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function detectReach(text: string): { label: string; meters: number } | undefined {
|
|
const patterns: [RegExp, string, number][] = [
|
|
[/\b120\s*km\b/i, "120km", 120000],
|
|
[/\b80\s*km\b/i, "80km", 80000],
|
|
[/\b40\s*km\b/i, "40km", 40000],
|
|
[/\b20\s*km\b/i, "20km", 20000],
|
|
[/\b10\s*km\b/i, "10km", 10000],
|
|
[/\b2\s*km\b/i, "2km", 2000],
|
|
[/\b550\s*m\b/i, "550m", 550],
|
|
[/\b500\s*m\b/i, "500m", 500],
|
|
[/\b400\s*m\b/i, "400m", 400],
|
|
[/\b300\s*m\b/i, "300m", 300],
|
|
[/\b150\s*m\b/i, "150m", 150],
|
|
[/\b100\s*m\b/i, "100m", 100],
|
|
[/\bLR4\b/, "10km", 10000],
|
|
[/\bLR\b/, "10km", 10000],
|
|
[/\bER4?\b/, "40km", 40000],
|
|
[/\bZR4?\b/, "80km", 80000],
|
|
[/\bSR4?\b/, "300m", 300],
|
|
[/\bDR4?\b/, "500m", 500],
|
|
[/\bFR4?\b/, "2km", 2000],
|
|
];
|
|
for (const [regex, label, meters] of patterns) {
|
|
if (regex.test(text)) return { label, meters };
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function detectFiber(text: string): string {
|
|
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
|
|
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
|
|
if (/copper|dac|twinax|rj.?45|base-t|cat[56x]/i.test(text)) return "Copper";
|
|
return "";
|
|
}
|
|
|
|
function detectWavelength(text: string): string {
|
|
const match = text.match(/(\d{3,4})\s*nm/i);
|
|
if (match) return match[1];
|
|
return "";
|
|
}
|
|
|
|
function extractCompatibleVendor(name: string): string {
|
|
const match = name.match(/^(\w+(?:\s+\w+)?)\s+Compatible\b/i);
|
|
return match ? match[1] : "";
|
|
}
|
|
|
|
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
|
|
const products: Product[] = [];
|
|
|
|
// Collapse whitespace for easier regex matching
|
|
const collapsed = html.replace(/\s+/g, " ");
|
|
|
|
// BigCommerce article card pattern (updated):
|
|
// <article data-name="Product Name" data-product-price="2395" ...>
|
|
// <a href="https://www.gbics.com/product-slug/" ...>
|
|
// Price is in pence (integer), divide by 100 = GBP
|
|
const articleRegex = /data-name="([^"]{10,200})"[^>]*data-product-price="\s*(\d+)\s*"[^>]*>[\s\S]{0,500}?href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi;
|
|
let match;
|
|
while ((match = articleRegex.exec(collapsed)) !== null) {
|
|
const name = match[1].trim();
|
|
const priceRaw = parseInt(match[2], 10);
|
|
const url = match[3];
|
|
// GBICS stores price in pence (integer) — e.g. 2395 = £23.95 OR £2,395.00 (full pounds)?
|
|
// Check by data-price-asc context: "data-price-asc=\"2395\"" with "£2,395.00" → price is in full GBP (no pence)
|
|
const price = priceRaw > 0 ? priceRaw : undefined;
|
|
|
|
if (name.length < 10) continue;
|
|
|
|
const reach = detectReach(name);
|
|
const partParts = name.split(/\s+-\s+/);
|
|
const partNumber = partParts[0]?.trim().slice(0, 80) || url.split("/").filter(Boolean).pop() || "";
|
|
|
|
products.push({
|
|
partNumber, name, url,
|
|
price: price && price > 0 && price < 50000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
compatibleWith: extractCompatibleVendor(name),
|
|
});
|
|
}
|
|
|
|
// Fallback: aria-label pattern
|
|
if (products.length === 0) {
|
|
const ariaRegex = /aria-label="([^"]+£[^"]+)"\s+href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi;
|
|
while ((match = ariaRegex.exec(collapsed)) !== null) {
|
|
const label = match[1].trim();
|
|
const url = match[2];
|
|
const priceMatch = label.match(/£\s*([\d,.]+)/);
|
|
const name = label.split(",")[0]?.trim() || label;
|
|
const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;
|
|
if (name.length < 10) continue;
|
|
const reach = detectReach(name);
|
|
products.push({
|
|
partNumber: name.split(/\s+-\s+/)[0]?.trim().slice(0, 80) || "",
|
|
name, url,
|
|
price: price && price > 0 && price < 50000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
compatibleWith: extractCompatibleVendor(name),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Fallback: try "Now: £XX.XX" pattern near product links
|
|
if (products.length === 0) {
|
|
const altRegex = /href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"[^>]*>\s*([^<]{15,})<\/a>/gi;
|
|
while ((match = altRegex.exec(collapsed)) !== null) {
|
|
const url = match[1];
|
|
const name = match[2].trim();
|
|
if (name.length < 10 || products.find((p) => p.url === url)) continue;
|
|
if (!/transceiver|sfp|qsfp|xfp|osfp|base/i.test(name)) continue;
|
|
|
|
const context = collapsed.slice(Math.max(0, match.index - 300), match.index + 600);
|
|
const priceMatch = context.match(/Now:\s*£\s*([\d,.]+)/) || context.match(/£\s*([\d,.]+)/);
|
|
const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;
|
|
const reach = detectReach(name);
|
|
|
|
products.push({
|
|
partNumber: name.split(/\s+-\s+/)[0]?.trim().slice(0, 80) || "",
|
|
name, url,
|
|
price: price && price > 0 && price < 50000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
compatibleWith: extractCompatibleVendor(name),
|
|
});
|
|
}
|
|
}
|
|
|
|
const seen = new Set<string>();
|
|
return products.filter((p) => {
|
|
if (seen.has(p.url)) return false;
|
|
seen.add(p.url);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
async function fetchPage(url: string): Promise<string> {
|
|
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
|
|
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
|
|
return resp.text();
|
|
}
|
|
|
|
export async function scrapeGbics(): Promise<void> {
|
|
console.log("=== GBICS.com Scraper Starting ===\n");
|
|
|
|
const vendorId = await ensureVendor("GBICS", "compatible", "https://www.gbics.com", "https://www.gbics.com/optical-transceivers/");
|
|
|
|
let totalProducts = 0;
|
|
let priceUpdates = 0;
|
|
|
|
for (const cat of CATEGORIES) {
|
|
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);
|
|
|
|
try {
|
|
const html = await fetchPage(BASE + cat.path);
|
|
const catProducts = parseProductList(html, cat);
|
|
console.log(` Found ${catProducts.length} products`);
|
|
|
|
for (const product of catProducts) {
|
|
try {
|
|
const txId = await findOrCreateScrapedTransceiver({
|
|
partNumber: product.partNumber, vendorId,
|
|
formFactor: product.formFactor, speedGbps: product.speedGbps,
|
|
speed: product.speed, reachMeters: product.reachMeters,
|
|
reachLabel: product.reachLabel, fiberType: product.fiberType,
|
|
wavelengths: product.wavelength, category: "DataCenter",
|
|
});
|
|
|
|
if (product.price && product.price > 0) {
|
|
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
|
|
const updated = await upsertPriceObservation({
|
|
transceiverId: txId, sourceVendorId: vendorId,
|
|
price: product.price, currency: "GBP",
|
|
stockLevel: "in_stock", url: product.url, contentHash: hash,
|
|
});
|
|
if (updated) priceUpdates++;
|
|
}
|
|
totalProducts++;
|
|
} catch (err) {
|
|
console.warn(` Error: ${(err as Error).message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(` Category failed: ${(err as Error).message}`);
|
|
}
|
|
|
|
await sleep(2000);
|
|
}
|
|
|
|
console.log(`\n=== GBICS Complete: ${totalProducts} products, ${priceUpdates} prices ===`);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
scrapeGbics()
|
|
.then(() => pool.end())
|
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
|
}
|