diff --git a/packages/scraper/src/index-pi.ts b/packages/scraper/src/index-pi.ts index 95822cf..0ffc3fc 100644 --- a/packages/scraper/src/index-pi.ts +++ b/packages/scraper/src/index-pi.ts @@ -49,6 +49,9 @@ const QUEUES = [ "scrape:pricing:skylane", "scrape:pricing:ascentoptics", "scrape:pricing:gaotek", + "scrape:pricing:fibermall", + "scrape:pricing:vcelink", + "scrape:pricing:opticsbay", // Form-factor coverage scrapers "scrape:pricing:comms-express", "scrape:pricing:router-switch", @@ -130,6 +133,9 @@ async function main() { const { scrapeSkylane } = await import("./scrapers/skylane"); const { scrapeAscentOptics } = await import("./scrapers/ascentoptics"); const { scrapeGaoTek } = await import("./scrapers/gaotek"); + const { scrapeFiberMall } = await import("./scrapers/fibermall"); + const { scrapeVcelink } = await import("./scrapers/vcelink"); + const { scrapeOpticsBay } = await import("./scrapers/opticsbay"); // ── Form-factor coverage scrapers ───────────────────────────────────── const { scrapeCommsExpress } = await import("./scrapers/comms-express"); @@ -188,6 +194,9 @@ async function main() { await boss.work("scrape:pricing:skylane", async () => { log("skylane"); await withIsolatedStorage("skylane", scrapeSkylane); }); await boss.work("scrape:pricing:ascentoptics", async () => { log("ascentoptics"); await withIsolatedStorage("ascentoptics", scrapeAscentOptics); }); await boss.work("scrape:pricing:gaotek", async () => { log("gaotek"); await withIsolatedStorage("gaotek", scrapeGaoTek); }); + await boss.work("scrape:pricing:fibermall", async () => { log("fibermall"); await scrapeFiberMall(); }); + await boss.work("scrape:pricing:vcelink", async () => { log("vcelink"); await scrapeVcelink(); }); + await boss.work("scrape:pricing:opticsbay", async () => { log("opticsbay"); await scrapeOpticsBay(); }); await boss.work("scrape:pricing:comms-express", async () => { log("comms-express"); await scrapeCommsExpress(); }); await boss.work("scrape:pricing:router-switch", async () => { log("router-switch"); await scrapeRouterSwitch(); }); @@ -229,7 +238,7 @@ async function main() { await boss.work("compute:reorder-signals", async () => { log("reorder"); await computeReorderSignals(); }); await boss.work("compute:forecast", async () => { log("forecast"); await runForecastEngine(); }); - console.log(`${QUEUES.length} workers active — running 24/7\n`); + console.log(`${QUEUES.length} queues / workers active — running 24/7\n`); process.on("SIGTERM", async () => { await boss.stop(); process.exit(0); }); process.on("SIGINT", async () => { await boss.stop(); process.exit(0); }); } diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 0fdafe5..519633d 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -127,6 +127,9 @@ export async function registerSchedules(boss: PgBoss): Promise { "scrape:pricing:naddod", "scrape:pricing:qsfptek", "scrape:pricing:addon", + "scrape:pricing:fibermall", + "scrape:pricing:vcelink", + "scrape:pricing:opticsbay", // ── Prediction Signal Scrapers (new) ────────────────────────────── "scrape:signals:sec-edgar", "scrape:signals:github", @@ -180,6 +183,9 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:pricing:fibermall", "57 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // FLEXOPTIX CATALOG — every 2h (primary price source) @@ -273,7 +279,7 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 }); - console.log("All schedules registered — 24/7 continuous scraping (50 jobs)"); + console.log("All schedules registered — 24/7 continuous scraping (53 jobs)"); } export async function registerWorkers(boss: PgBoss): Promise { @@ -545,5 +551,23 @@ export async function registerWorkers(boss: PgBoss): Promise { await scrapeAddonNetworks(); }); - console.log("All workers registered (58 jobs, 24/7 continuous)"); + await boss.work("scrape:pricing:fibermall", async () => { + console.log(`[${new Date().toISOString()}] Running: FiberMall pricing`); + const { scrapeFiberMall } = await import("./scrapers/fibermall"); + await scrapeFiberMall(); + }); + + await boss.work("scrape:pricing:vcelink", async () => { + console.log(`[${new Date().toISOString()}] Running: Vcelink pricing`); + const { scrapeVcelink } = await import("./scrapers/vcelink"); + await scrapeVcelink(); + }); + + await boss.work("scrape:pricing:opticsbay", async () => { + console.log(`[${new Date().toISOString()}] Running: OpticsBay pricing`); + const { scrapeOpticsBay } = await import("./scrapers/opticsbay"); + await scrapeOpticsBay(); + }); + + console.log("All workers registered (61 jobs, 24/7 continuous)"); } diff --git a/packages/scraper/src/scrapers/fibermall.ts b/packages/scraper/src/scrapers/fibermall.ts new file mode 100644 index 0000000..9e8c9b1 --- /dev/null +++ b/packages/scraper/src/scrapers/fibermall.ts @@ -0,0 +1,275 @@ +/** + * FiberMall Scraper — Chinese compatible transceiver vendor + * + * fibermall.com — custom PHP/REST shop, USD pricing. + * Large catalog: 1G–400G, SFP/SFP+/QSFP28/QSFP-DD/OSFP. + * Pagination via ?page=N. Rate limited: 1 req/2sec. + * + * FiberMall (Shenzhen FiberMall Technology Co.) offers DAC/AOC + optics, + * transparent USD prices, no login required. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; + +const BASE = "https://www.fibermall.com"; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +}; + +const MAX_PAGES = 30; + +const CATEGORIES = [ + { path: "/c/1g-sfp-transceiver/", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { path: "/c/10g-sfp-transceiver/", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { path: "/c/25g-sfp28-transceiver/", formFactor: "SFP28",speed: "25G", speedGbps: 25 }, + { path: "/c/40g-qsfp-transceiver/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, + { path: "/c/100g-qsfp28-transceiver/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, + { path: "/c/200g-qsfp56-transceiver/", formFactor: "QSFP56", speed: "200G", speedGbps: 200 }, + { path: "/c/400g-qsfp-dd-transceiver/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, + { path: "/c/800g-osfp-transceiver/", formFactor: "OSFP", speed: "800G", speedGbps: 800 }, + { path: "/c/dac-cable/", formFactor: "DAC", speed: "10G", speedGbps: 10 }, + { path: "/c/aoc-cable/", formFactor: "AOC", speed: "10G", speedGbps: 10 }, + { path: "/c/optical-transceivers/", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, +]; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b120\s*km\b/i, "120km", 120000], + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b500\s*m\b/i, "500m", 500], + [/\b300\s*m\b/i, "300m", 300], + [/\b150\s*m\b/i, "150m", 150], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], + [/\bFR4?\b/, "2km", 2000], + ]; + for (const [re, label, meters] of patterns) { + if (re.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper"; + return ""; +} + +function detectWavelength(text: string): string { + const m = text.match(/(\d{3,4})\s*nm/i); + return m ? m[1] : ""; +} + +function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] { + const products: Product[] = []; + const seen = new Set(); + const collapsed = html.replace(/\s+/g, " "); + + // Strategy 1: fibermall product card (class="product-item" or similar) + for (const m of collapsed.matchAll(/<(?:li|div|article)[^>]+class="[^"]*(?:product-item|product-thumb|goods-item|pro-item)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div|article)>/gi)) { + const card = m[1]; + const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]+)"/i) || + card.match(/href="(\/[a-z0-9][^"?#]{5,})"/i); + if (!urlMatch) continue; + const url = urlMatch[1].startsWith("http") ? urlMatch[1] : BASE + urlMatch[1]; + if (seen.has(url) || !/fibermall\.com|\/product|\/p\//i.test(url)) continue; + seen.add(url); + + const nameMatch = card.match(/<(?:h[23456]|p)[^>]+class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]{8,})<\//i) || + card.match(/title="([^"]{10,})"/i) || + card.match(/]*>([^<]{10,})<\/a>/i); + if (!nameMatch) continue; + const name = nameMatch[1].trim().replace(/&/g, "&").replace(/&#\d+;/g, ""); + if (name.length < 5) continue; + + const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/); + const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined; + + const reach = detectReach(name); + const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60); + + products.push({ + partNumber, name, url, + price: price && price > 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + + // Strategy 2: generic product link scan + if (products.length === 0) { + for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]{10,})"[^>]*>([^<]{10,}) 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + } + + return products; +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeFiberMall(): Promise { + console.log("=== FiberMall Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "FiberMall", + "compatible", + "https://www.fibermall.com", + "https://www.fibermall.com/c/optical-transceivers/", + ); + + let totalProducts = 0; + let priceUpdates = 0; + const seenCategories = new Set(); + + for (const cat of CATEGORIES) { + console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`); + + try { + const html1 = await fetchPage(BASE + cat.path); + const catProducts = parseProductList(html1, cat); + + // Skip generic fallback if specific categories already scraped + if (cat.path.includes("/optical-transceivers/") && seenCategories.size > 3) { + console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`); + continue; + } + + if (catProducts.length === 0) { + console.log(" No products on page 1 — trying alternate pagination"); + // Try ?page=1 format + try { + const html1alt = await fetchPage(BASE + cat.path + "?page=1"); + catProducts.push(...parseProductList(html1alt, cat)); + } catch { /* ignore */ } + } + + if (catProducts.length === 0) { + console.log(" No products found — skipping"); + continue; + } + + seenCategories.add(cat.path); + console.log(` Found ${catProducts.length} products on page 1`); + + const allProducts = [...catProducts]; + + for (let page = 2; page <= MAX_PAGES; page++) { + await sleep(2000); + try { + // Try both pagination formats + const pageUrl = `${BASE}${cat.path}?page=${page}`; + const html = await fetchPage(pageUrl); + const pageProds = parseProductList(html, cat); + if (pageProds.length === 0) break; + allProducts.push(...pageProds); + console.log(` Page ${page}: ${pageProds.length} products`); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`); + break; + } + } + + const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i); + console.log(` Total unique: ${uniqueProducts.length}`); + + for (const product of uniqueProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash({ price: product.price, part: product.partNumber }); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); + } + } + } catch (err) { + console.error(` Category failed: ${(err as Error).message}`); + } + + await sleep(2000); + } + + console.log(`\n=== FiberMall Complete: ${totalProducts} products, ${priceUpdates} price updates ===`); +} + +if (require.main === module) { + scrapeFiberMall() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/opticsbay.ts b/packages/scraper/src/scrapers/opticsbay.ts new file mode 100644 index 0000000..4b0a06f --- /dev/null +++ b/packages/scraper/src/scrapers/opticsbay.ts @@ -0,0 +1,257 @@ +/** + * OpticsBay Scraper — North American compatible transceiver vendor + * + * opticsbay.com — WooCommerce store, USD/CAD pricing. + * Covers SFP/SFP+/QSFP28/QSFP-DD with competitive pricing. + * Rate limited: 1 req/2sec. + * + * OpticsBay offers compatible optics for Cisco, Juniper, Arista, HPE. + * Transparent pricing, no login required. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; + +const BASE = "https://www.opticsbay.com"; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +}; + +const MAX_PAGES = 20; + +const CATEGORIES = [ + { path: "/product-category/sfp-transceiver/", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { path: "/product-category/sfp-plus-transceiver/", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { path: "/product-category/sfp28-transceiver/", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, + { path: "/product-category/qsfp-plus-transceiver/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, + { path: "/product-category/qsfp28-transceiver/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, + { path: "/product-category/qsfp-dd-transceiver/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, + { path: "/product-category/dac-cable/", formFactor: "DAC", speed: "10G", speedGbps: 10 }, + { path: "/product-category/transceivers/", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, +]; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b120\s*km\b/i, "120km", 120000], + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], + [/\bFR4?\b/, "2km", 2000], + ]; + for (const [re, label, meters] of patterns) { + if (re.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper"; + return ""; +} + +function detectWavelength(text: string): string { + const m = text.match(/(\d{3,4})\s*nm/i); + return m ? m[1] : ""; +} + +function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] { + const products: Product[] = []; + const seen = new Set(); + const collapsed = html.replace(/\s+/g, " "); + + // Strategy 1: WooCommerce standard product loop + for (const m of collapsed.matchAll(/]+class="[^"]*product[^"]*"[^>]*>([\s\S]*?)<\/li>/gi)) { + const card = m[1]; + const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?opticsbay\.com\/[^"?#]+)"/i); + if (!urlMatch) continue; + const url = urlMatch[1]; + if (seen.has(url)) continue; + seen.add(url); + + const nameMatch = + card.match(/woocommerce-loop-product__title[^>]*>([^<]+)]*>([^<]{8,})<\/h[23]>/i); + if (!nameMatch) continue; + const name = nameMatch[1].trim().replace(/&/g, "&").replace(/&#\d+;/g, ""); + if (name.length < 5) continue; + + const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/); + const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined; + const reach = detectReach(name); + + products.push({ + partNumber: name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60), + name, url, + price: price && price > 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + + // Strategy 2: Generic product link scan + if (products.length === 0) { + for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?opticsbay\.com\/(?:product|shop)\/[^"?#]+)"[^>]*>([^<]{8,}) 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + } + + return products; +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeOpticsBay(): Promise { + console.log("=== OpticsBay Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "OpticsBay", + "compatible", + "https://www.opticsbay.com", + "https://www.opticsbay.com/product-category/transceivers/", + ); + + let totalProducts = 0; + let priceUpdates = 0; + const seenCategories = new Set(); + + for (const cat of CATEGORIES) { + console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`); + + try { + const html1 = await fetchPage(BASE + cat.path); + const catProducts = parseProductList(html1, cat); + + if (cat.path.includes("/transceivers/") && seenCategories.size > 3) { + console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`); + continue; + } + + if (catProducts.length === 0) { + console.log(" No products on page 1 — skipping"); + continue; + } + + seenCategories.add(cat.path); + console.log(` Found ${catProducts.length} products on page 1`); + + const allProducts = [...catProducts]; + + for (let page = 2; page <= MAX_PAGES; page++) { + await sleep(2000); + try { + const pageUrl = `${BASE}${cat.path}page/${page}/`; + const html = await fetchPage(pageUrl); + const pageProds = parseProductList(html, cat); + if (pageProds.length === 0) break; + allProducts.push(...pageProds); + console.log(` Page ${page}: ${pageProds.length} products`); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`); + break; + } + } + + const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i); + console.log(` Total unique: ${uniqueProducts.length}`); + + for (const product of uniqueProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash({ price: product.price, part: product.partNumber }); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); + } + } + } catch (err) { + console.error(` Category failed: ${(err as Error).message}`); + } + + await sleep(2000); + } + + console.log(`\n=== OpticsBay Complete: ${totalProducts} products, ${priceUpdates} price updates ===`); +} + +if (require.main === module) { + scrapeOpticsBay() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/qsfptek.ts b/packages/scraper/src/scrapers/qsfptek.ts index d66a60b..513f63d 100644 --- a/packages/scraper/src/scrapers/qsfptek.ts +++ b/packages/scraper/src/scrapers/qsfptek.ts @@ -1,35 +1,48 @@ /** * QSFPTEK Scraper — Chinese compatible transceiver vendor * - * qsfptek.com — Server-rendered HTML shop, USD pricing. - * Focuses on QSFP+/QSFP28/QSFP-DD/SFP+ form factors. - * Rate limited: 1 req/2sec. + * qsfptek.com — migrated to custom Java/Spring + Vue.js in 2025. + * Old OpenCart /c/*.html paths are gone (404). + * Products are served via HTML-fragment API: /mall/commodity/list * - * QSFPTEK (Shenzhen Optotech Technology) — competitive pricing, - * transparent USD prices, no account required. + * API: GET /mall/commodity/list?categoryId=21&attributes=VALUE_ID&page=N&pageSize=30 + * Returns HTML fragment with product cards. + * + * Rate limited: 1 req/2sec. */ +import * as cheerio from "cheerio"; import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; import { contentHash } from "../utils/hash"; const BASE = "https://www.qsfptek.com"; +const LIST_API = "/mall/commodity/list"; +const CATEGORY_ID = "21"; // Transceivers (top-level, all products) + const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", }; -const MAX_PAGES = 30; +const MAX_PAGES = 40; +const PAGE_SIZE = 30; -const CATEGORIES = [ - { path: "/c/sfp-transceiver.html", formFactor: "SFP", speed: "1G", speedGbps: 1 }, - { path: "/c/sfp-plus-transceiver.html", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, - { path: "/c/sfp28-transceiver.html", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, - { path: "/c/qsfp-plus-transceiver.html", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, - { path: "/c/qsfp28-transceiver.html", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, - { path: "/c/qsfp56-transceiver.html", formFactor: "QSFP56", speed: "200G", speedGbps: 200 }, - { path: "/c/qsfp-dd-transceiver.html", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, - { path: "/c/osfp-transceiver.html", formFactor: "OSFP", speed: "800G", speedGbps: 800 }, - { path: "/c/optical-transceiver.html", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, +// Data rate attribute values (found in /mall/commodity/attribute?categoryId=21) +// pid = "2c9180837bbaf08f017bbdd1ebf7001e" (Data Rate attribute group) +const DATA_RATE_ATTRIBUTES: Array<{ + attrId: string; + speed: string; + speedGbps: number; + formFactor: string; +}> = [ + { attrId: "2c9180837bbaf08f017bbde50ea30101", speed: "1G", speedGbps: 1, formFactor: "SFP" }, + { attrId: "2c9180837bbaf08f017bbde50ea30100", speed: "10G", speedGbps: 10, formFactor: "SFP+" }, + { attrId: "2c9180837bbaf08f017bbde50ea300ff", speed: "25G", speedGbps: 25, formFactor: "SFP28" }, + { attrId: "2c9180837bbaf08f017bbde50ea300fe", speed: "40G", speedGbps: 40, formFactor: "QSFP+" }, + { attrId: "2c9180837bbaf08f017bbde50ea300fd", speed: "100G", speedGbps: 100, formFactor: "QSFP28" }, + { attrId: "2c98491f8f4b8e55018f94aa8c5d48ff", speed: "200G", speedGbps: 200, formFactor: "QSFP56" }, + { attrId: "2c9180837e2e7f64017e389caf0700c8", speed: "400G", speedGbps: 400, formFactor: "QSFP-DD" }, + { attrId: "2c98491f8e363cbf018e3faa5b6d2f8b", speed: "800G", speedGbps: 800, formFactor: "OSFP" }, ]; interface Product { @@ -44,7 +57,6 @@ interface Product { reachMeters?: number; fiberType?: string; wavelength?: string; - compatibleWith?: string; } function sleep(ms: number): Promise { @@ -60,19 +72,15 @@ function detectReach(text: string): { label: string; meters: number } | undefine [/\b10\s*km\b/i, "10km", 10000], [/\b2\s*km\b/i, "2km", 2000], [/\b550\s*m\b/i, "550m", 550], - [/\b500\s*m\b/i, "500m", 500], [/\b300\s*m\b/i, "300m", 300], [/\b100\s*m\b/i, "100m", 100], - [/\bLR4\b/, "10km", 10000], - [/\bLR\b/, "10km", 10000], - [/\bER4?\b/, "40km", 40000], - [/\bZR4?\b/, "80km", 80000], - [/\bSR4?\b/, "300m", 300], - [/\bDR4?\b/, "500m", 500], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], ]; - for (const [regex, label, meters] of patterns) { - if (regex.test(text)) return { label, meters }; + for (const [re, label, meters] of patterns) { + if (re.test(text)) return { label, meters }; } return undefined; } @@ -85,187 +93,140 @@ function detectFiber(text: string): string { } function detectWavelength(text: string): string { - const match = text.match(/(\d{3,4})\s*nm/i); - return match ? match[1] : ""; + const m = text.match(/(\d{3,4})\s*nm/i); + return m ? m[1] : ""; } -function extractCompatibleVendor(name: string): string { - const brands = ["Cisco", "Juniper", "Arista", "HPE", "Aruba", "Dell", "Brocade", "Extreme", - "Huawei", "Nokia", "MikroTik", "Mellanox", "Nvidia", "Ubiquiti", "Allied Telesis"]; - for (const brand of brands) { - if (new RegExp(`\\b${brand}\\b`, "i").test(name)) return brand; - } - return ""; -} - -function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] { +function parseProductFragment(html: string, attr: typeof DATA_RATE_ATTRIBUTES[number]): Product[] { + const $ = cheerio.load(html); const products: Product[] = []; const seen = new Set(); - const collapsed = html.replace(/\s+/g, " "); - // Strategy 1: OpenCart / custom card layout using matchAll - for (const cardMatch of collapsed.matchAll(/]+class="[^"]*product-(?:thumb|layout)[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/gi)) { - const card = cardMatch[1]; + // Product cards: li.Hot, li.New, li with class containing product + $("li, div.kuang").each((_i, el) => { + const $el = $(el); - const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?qsfptek\.com\/[^"]+)"/i); - if (!urlMatch) continue; - const url = urlMatch[1]; - if (seen.has(url)) continue; + // Get product URL + const href = $el.find("a[href*='/product/']").first().attr("href") || + $el.find("a[href*='/en/product/']").first().attr("href"); + if (!href) return; + const url = href.startsWith("http") ? href : BASE + href; + if (seen.has(url)) return; seen.add(url); - const nameMatch = card.match(/]*>\s*]*>([^<]{10,})<\/a>/i) || - card.match(/]*title="([^"]{10,})"/i); - if (!nameMatch) continue; - const name = nameMatch[1].trim().replace(/&/g, "&").replace(/&#[0-9]+;/g, ""); - if (name.length < 5) continue; + // Get product name from h3 + const name = $el.find("h3.tit, h3").first().text().trim().replace(/\s+/g, " "); + if (!name || name.length < 10) return; - const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/); + // Get price: "US$ 33.90" format + const priceText = $el.find("*").text(); + const priceMatch = priceText.match(/US\$\s*([\d,]+\.?\d{0,2})/); const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined; const reach = detectReach(name); - const partNumber = name.split(/\s+(?:compatible|for|sfp|qsfp)/i)[0]?.trim().slice(0, 80) || name.slice(0, 60); + const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z]|sfp|qsfp)/i)[0]?.trim().slice(0, 80) || name.slice(0, 60); products.push({ partNumber, name, url, price: price && price > 0 && price < 100000 ? price : undefined, - formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + formFactor: attr.formFactor, speed: attr.speed, speedGbps: attr.speedGbps, reachLabel: reach?.label, reachMeters: reach?.meters, fiberType: detectFiber(name), wavelength: detectWavelength(name), - compatibleWith: extractCompatibleVendor(name), }); - } - - // Strategy 2: Generic product link scan using matchAll - if (products.length === 0) { - for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?qsfptek\.com\/(?:p|product)[^"?#]+)"[^>]*>([^<]{10,}) 0 && price < 100000 ? price : undefined, - formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, - reachLabel: reach?.label, reachMeters: reach?.meters, - fiberType: detectFiber(name), wavelength: detectWavelength(name), - compatibleWith: extractCompatibleVendor(name), - }); - } - } + }); return products; } -async function fetchPage(url: string): Promise { +async function fetchProductList(attrId: string, page: number): Promise { + const url = `${BASE}${LIST_API}?categoryId=${CATEGORY_ID}&attributes=${attrId}&page=${page}&pageSize=${PAGE_SIZE}`; const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); return resp.text(); } export async function scrapeQsfptek(): Promise { - console.log("=== QSFPTEK Scraper Starting ===\n"); + console.log("=== QSFPTEK Scraper Starting (API mode) ===\n"); const vendorId = await ensureVendor( "QSFPTEK", "compatible", "https://www.qsfptek.com", - "https://www.qsfptek.com/c/optical-transceiver.html", + "https://www.qsfptek.com/c/fiber-optic-transceiver.html", ); let totalProducts = 0; let priceUpdates = 0; - const seenCategories = new Set(); - for (const cat of CATEGORIES) { - console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`); + for (const attr of DATA_RATE_ATTRIBUTES) { + console.log(`\n--- ${attr.formFactor} (${attr.speed}) ---`); + const allProducts: Product[] = []; + const allSeen = new Set(); - try { - const html1 = await fetchPage(BASE + cat.path); - const catProducts = parseProductList(html1, cat); + for (let page = 1; page <= MAX_PAGES; page++) { + try { + const html = await fetchProductList(attr.attrId, page); + const pageProds = parseProductFragment(html, attr); - if (cat.path.includes("/optical-transceiver") && seenCategories.size > 3) { - console.log(` Skipping generic fallback (${seenCategories.size} specific categories scraped)`); - continue; - } - - if (catProducts.length === 0) { - console.log(" No products on page 1 — skipping"); - continue; - } - - seenCategories.add(cat.path); - console.log(` Found ${catProducts.length} products on page 1`); - - const totalPagesMatch = - html1.match(/total-page[^>]*>\s*(\d+)/) || - html1.match(/page\s+\d+\s+of\s+(\d+)/i); - const totalPages = totalPagesMatch ? Math.min(parseInt(totalPagesMatch[1]), MAX_PAGES) : 3; - console.log(` Total pages (estimate): ${totalPages}`); - - const allProducts = [...catProducts]; - - for (let page = 2; page <= totalPages; page++) { - await sleep(2000); - try { - const pageUrl = BASE + cat.path.replace(".html", "") + `?page=${page}`; - const html = await fetchPage(pageUrl); - const pageProds = parseProductList(html, cat); - if (pageProds.length === 0) break; - allProducts.push(...pageProds); - console.log(` Page ${page}: ${pageProds.length} products`); - } catch (err) { - console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`); + if (pageProds.length === 0) { + if (page === 1) console.log(" No products on page 1 — skipping"); + else console.log(` Page ${page}: empty, stopping`); break; } + + // Dedupe against already seen (API may repeat items) + const newProds = pageProds.filter(p => { + if (allSeen.has(p.url)) return false; + allSeen.add(p.url); + return true; + }); + allProducts.push(...newProds); + console.log(` Page ${page}: ${pageProds.length} total, ${newProds.length} new (${allProducts.length} running)`); + + // If we got fewer than pageSize, we're on the last page + if (pageProds.length < PAGE_SIZE) break; + + if (page < MAX_PAGES) await sleep(2000); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 80)}`); + break; } + } - const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex((x) => x.url === p.url) === i); - console.log(` Total unique: ${uniqueProducts.length}`); + console.log(` Unique products: ${allProducts.length}`); - for (const product of uniqueProducts) { - try { - const txId = await findOrCreateScrapedTransceiver({ - partNumber: product.partNumber, - vendorId, - formFactor: product.formFactor, - speedGbps: product.speedGbps, - speed: product.speed, - reachMeters: product.reachMeters, - reachLabel: product.reachLabel, - fiberType: product.fiberType, - wavelengths: product.wavelength, - category: "DataCenter", + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash({ price: product.price, part: product.partNumber }); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, }); - - if (product.price && product.price > 0) { - const hash = contentHash({ price: product.price, part: product.partNumber }); - const updated = await upsertPriceObservation({ - transceiverId: txId, - sourceVendorId: vendorId, - price: product.price, - currency: "USD", - stockLevel: "in_stock", - url: product.url, - contentHash: hash, - }); - if (updated) priceUpdates++; - } - totalProducts++; - } catch (err) { - console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); + if (updated) priceUpdates++; } + totalProducts++; + } catch (err) { + console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); } - } catch (err) { - console.error(` Category failed: ${(err as Error).message}`); } await sleep(2000); diff --git a/packages/scraper/src/scrapers/vcelink.ts b/packages/scraper/src/scrapers/vcelink.ts new file mode 100644 index 0000000..f578e6d --- /dev/null +++ b/packages/scraper/src/scrapers/vcelink.ts @@ -0,0 +1,284 @@ +/** + * Vcelink Scraper — Chinese compatible transceiver vendor + * + * vcelink.com — Shopify-based store, USD pricing. + * Covers SFP/SFP+/QSFP28/QSFP-DD, DAC/AOC cables. + * Rate limited: 1 req/2sec. + * + * Vcelink offers Cisco/Juniper/Arista-compatible optics at competitive + * USD prices, publicly accessible without login. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; + +const BASE = "https://www.vcelink.com"; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +}; + +const MAX_PAGES = 20; + +const CATEGORIES = [ + { path: "/collections/1g-sfp", formFactor: "SFP", speed: "1G", speedGbps: 1 }, + { path: "/collections/10g-sfp", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, + { path: "/collections/25g-sfp28", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, + { path: "/collections/40g-qsfp", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, + { path: "/collections/100g-qsfp28",formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, + { path: "/collections/400g-qsfp-dd",formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 }, + { path: "/collections/dac-cable", formFactor: "DAC", speed: "10G", speedGbps: 10 }, + { path: "/collections/aoc-cable", formFactor: "AOC", speed: "10G", speedGbps: 10 }, + { path: "/collections/all", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, +]; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b120\s*km\b/i, "120km", 120000], + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], + [/\bFR4?\b/, "2km", 2000], + ]; + for (const [re, label, meters] of patterns) { + if (re.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper"; + return ""; +} + +function detectWavelength(text: string): string { + const m = text.match(/(\d{3,4})\s*nm/i); + return m ? m[1] : ""; +} + +function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] { + const products: Product[] = []; + const seen = new Set(); + const collapsed = html.replace(/\s+/g, " "); + + // Strategy 1: Shopify product grid (class="product-item" or similar) + for (const m of collapsed.matchAll(/<(?:li|div)[^>]+class="[^"]*(?:product-item|product__card|grid__item|product-card)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div)>/gi)) { + const card = m[1]; + const urlMatch = card.match(/href="(\/products\/[^"?#]+)"/i) || + card.match(/href="(https?:\/\/(?:www\.)?vcelink\.com\/products\/[^"?#]+)"/i); + if (!urlMatch) continue; + const rawUrl = urlMatch[1]; + const url = rawUrl.startsWith("http") ? rawUrl : BASE + rawUrl; + if (seen.has(url)) continue; + seen.add(url); + + const nameMatch = card.match(/]*>([^<]{8,})<\/h[23456]>/i) || + card.match(/class="[^"]*(?:title|name)[^"]*"[^>]*>([^<]{8,}) 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + + // Strategy 2: JSON-LD product data (Shopify embeds this) + if (products.length === 0) { + for (const m of collapsed.matchAll(/"@type"\s*:\s*"Product"[^}]*"name"\s*:\s*"([^"]{8,})"[^}]*"url"\s*:\s*"([^"]+)"[^}]*"price"\s*:\s*"?([\d.]+)/gi)) { + const name = m[1].replace(/\\n/g, " ").trim(); + const url = m[2].startsWith("http") ? m[2] : BASE + m[2]; + const price = parseFloat(m[3]); + if (seen.has(url)) continue; + if (!/sfp|qsfp|transceiver|dac|aoc|optic/i.test(name + url)) continue; + seen.add(url); + + const reach = detectReach(name); + products.push({ + partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "", + name, url, + price: price > 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + } + + // Strategy 3: Generic product links + if (products.length === 0) { + for (const m of collapsed.matchAll(/href="(\/products\/[^"?#]{5,})"[^>]*>([^<]{8,}) 0 && price < 100000 ? price : undefined, + formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + } + + return products; +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeVcelink(): Promise { + console.log("=== Vcelink Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "Vcelink", + "compatible", + "https://www.vcelink.com", + "https://www.vcelink.com/collections/all", + ); + + let totalProducts = 0; + let priceUpdates = 0; + const seenCategories = new Set(); + + for (const cat of CATEGORIES) { + console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`); + + try { + const html1 = await fetchPage(BASE + cat.path); + const catProducts = parseProductList(html1, cat); + + if (cat.path.includes("/collections/all") && seenCategories.size > 3) { + console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`); + continue; + } + + if (catProducts.length === 0) { + console.log(" No products on page 1 — skipping"); + continue; + } + + seenCategories.add(cat.path); + console.log(` Found ${catProducts.length} products on page 1`); + + const allProducts = [...catProducts]; + + for (let page = 2; page <= MAX_PAGES; page++) { + await sleep(2000); + try { + // Shopify pagination: ?page=N + const pageUrl = `${BASE}${cat.path}?page=${page}`; + const html = await fetchPage(pageUrl); + const pageProds = parseProductList(html, cat); + if (pageProds.length === 0) break; + allProducts.push(...pageProds); + console.log(` Page ${page}: ${pageProds.length} products`); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`); + break; + } + } + + const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i); + console.log(` Total unique: ${uniqueProducts.length}`); + + for (const product of uniqueProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash({ price: product.price, part: product.partNumber }); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); + } + } + } catch (err) { + console.error(` Category failed: ${(err as Error).message}`); + } + + await sleep(2000); + } + + console.log(`\n=== Vcelink Complete: ${totalProducts} products, ${priceUpdates} price updates ===`); +} + +if (require.main === module) { + scrapeVcelink() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +}