/** * Champion ONE Scraper — US-based compatible transceiver vendor * * championone.com — Server-rendered HTML, no JS required. * Large catalog: SFP, SFP+, SFP28, QSFP+, QSFP28, QSFP-DD, OSFP, XFP, X2, GBIC * * Rate limited: 1 req/2sec. */ import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; import { contentHash } from "../utils/hash"; const BASE = "https://www.championone.com"; const HEADERS = { "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", Accept: "text/html,application/xhtml+xml", }; const CATEGORIES = [ { path: "/sfp-transceivers", formFactor: "SFP", speed: "1G", speedGbps: 1 }, { path: "/sfp-plus-transceivers", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, { path: "/25g-sfp28-transceivers", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, { path: "/qsfp-plus-transceivers", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, { path: "/qsfp28-transceivers", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, { path: "/qsfp-dd-transceivers", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, { path: "/xfp-transceivers", formFactor: "XFP", speed: "10G", speedGbps: 10 }, ]; interface Product { partNumber: string; name: string; url: string; price?: number; currency?: string; formFactor: string; speed: string; speedGbps: number; reachLabel?: string; reachMeters?: number; fiberType?: string; wavelength?: string; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function detectReach(text: string): { label: string; meters: number } | undefined { const patterns: [RegExp, string, number][] = [ [/\b160\s*km\b/i, "160km", 160000], [/\b80\s*km\b/i, "80km", 80000], [/\b40\s*km\b/i, "40km", 40000], [/\b20\s*km\b/i, "20km", 20000], [/\b10\s*km\b/i, "10km", 10000], [/\b2\s*km\b/i, "2km", 2000], [/\b550\s*m\b/i, "550m", 550], [/\b500\s*m\b/i, "500m", 500], [/\b300\s*m\b/i, "300m", 300], [/\b100\s*m\b/i, "100m", 100], [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], [/\bCWDM4\b/i, "2km", 2000], [/\bPSM4\b/i, "500m", 500], ]; for (const [regex, label, meters] of patterns) { if (regex.test(text)) return { label, meters }; } return undefined; } function detectFiber(text: string): string { if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; return ""; } function detectWavelength(text: string): string { const match = text.match(/(\d{3,4})\s*nm/i); if (match) return match[1]; return ""; } function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] { const products: Product[] = []; // Champion ONE uses standard ecommerce HTML with product cards const productRegex = /href="(\/[^"]*?(?:transceiver|sfp|qsfp|osfp|xfp|optic)[^"]*)"[^>]*>([^<]{5,})<\/a>/gi; let match; while ((match = productRegex.exec(html)) !== null) { const url = match[1]; const name = match[2].trim(); if (name.length < 8 || name.length > 200) continue; const context = html.slice(Math.max(0, match.index - 200), match.index + 500); const priceMatch = context.match(/\$\s*([\d,]+\.?\d{0,2})/) || context.match(/USD\s*([\d,]+\.?\d{0,2})/i); const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined; const partNum = name.replace(/\s+/g, "-").replace(/[^a-zA-Z0-9\-]/g, "").slice(0, 80); const reach = detectReach(name); products.push({ partNumber: partNum, name, url: url.startsWith("http") ? url : BASE + url, price: price && price > 0 && price < 50000 ? price : undefined, currency: price ? "USD" : undefined, formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, reachLabel: reach?.label, reachMeters: reach?.meters, fiberType: detectFiber(name), wavelength: detectWavelength(name), }); } // Pattern 2: Generic product card pattern const cardRegex = /class="[^"]*product[^"]*"[\s\S]*?href="([^"]+)"[^>]*>[\s\S]*?(?:name|title)[^>]*>([^<]+)/gi; while ((match = cardRegex.exec(html)) !== null) { const url = match[1]; const name = match[2].trim(); if (products.find((p) => p.url === (url.startsWith("http") ? url : BASE + url))) continue; if (name.length < 8) continue; const context = html.slice(match.index, match.index + 500); const priceMatch = context.match(/\$\s*([\d,]+\.?\d{0,2})/); const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined; const reach = detectReach(name); products.push({ partNumber: name.replace(/\s+/g, "-").replace(/[^a-zA-Z0-9\-]/g, "").slice(0, 80), name, url: url.startsWith("http") ? url : BASE + url, price: price && price > 0 && price < 50000 ? price : undefined, currency: price ? "USD" : undefined, formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, reachLabel: reach?.label, reachMeters: reach?.meters, fiberType: detectFiber(name), wavelength: detectWavelength(name), }); } const seen = new Set(); return products.filter((p) => { if (seen.has(p.url)) return false; seen.add(p.url); return true; }); } function getMaxPage(html: string): number { const pageMatches = html.match(/[?&]page=(\d+)/g) || html.match(/\/page\/(\d+)/g); if (!pageMatches) return 1; let max = 1; for (const m of pageMatches) { const n = parseInt(m.replace(/[^0-9]/g, "")); if (n > max) max = n; } return Math.min(max, 30); } async function fetchPage(url: string): Promise { const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); return resp.text(); } export async function scrapeChampionOne(): Promise { console.log("=== Champion ONE Scraper Starting ===\n"); const vendorId = await ensureVendor("Champion ONE", "compatible", "https://www.championone.com", "https://www.championone.com"); let totalProducts = 0; let priceUpdates = 0; for (const cat of CATEGORIES) { console.log(`\n--- ${cat.formFactor} (${cat.speed}) ---`); try { const firstPage = await fetchPage(BASE + cat.path); const maxPage = getMaxPage(firstPage); console.log(` Pages: ${maxPage}`); let catProducts: Product[] = parseProductList(firstPage, cat); for (let page = 2; page <= maxPage; page++) { await sleep(2000); try { const html = await fetchPage(`${BASE}${cat.path}?page=${page}`); catProducts.push(...parseProductList(html, cat)); } catch (err) { console.warn(` Page ${page} failed: ${(err as Error).message}`); } } const seen = new Set(); catProducts = catProducts.filter((p) => { if (seen.has(p.url)) return false; seen.add(p.url); return true; }); console.log(` Found ${catProducts.length} products`); for (const product of catProducts) { try { const txId = await findOrCreateScrapedTransceiver({ partNumber: product.partNumber, vendorId, formFactor: product.formFactor, speedGbps: product.speedGbps, speed: product.speed, reachMeters: product.reachMeters, reachLabel: product.reachLabel, fiberType: product.fiberType, wavelengths: product.wavelength, category: "DataCenter", }); if (product.price && product.price > 0) { const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); const updated = await upsertPriceObservation({ transceiverId: txId, sourceVendorId: vendorId, price: product.price, currency: product.currency || "USD", stockLevel: "in_stock", url: product.url, contentHash: hash, }); if (updated) priceUpdates++; } totalProducts++; } catch (err) { console.warn(` Error: ${(err as Error).message.slice(0, 80)}`); } } } catch (err) { console.error(` Category failed: ${(err as Error).message}`); } await sleep(2000); } console.log(`\n=== Champion ONE Complete: ${totalProducts} products, ${priceUpdates} prices ===`); } if (require.main === module) { scrapeChampionOne() .then(() => pool.end()) .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); }