rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scraper): add FiberMall/Vcelink/OpticsBay scrapers, fix QSFPTEK API migration

Browse Source 
- New scrapers: fibermall.ts (WooCommerce), vcelink.ts (Shopify), opticsbay.ts (WooCommerce)
- QSFPTEK rewritten to use /mall/commodity/list API (old OpenCart /c/*.html paths gone 404)
  - New: attribute-based filtering by data rate (1G/10G/25G/40G/100G/200G/400G/800G)
  - Scrapes HTML fragments, extracts US$ prices and product URLs
- scheduler.ts: +3 queues/schedules/workers (fibermall, vcelink, opticsbay) → 61 total workers
- index-pi.ts: Pi fleet picks up all 3 new scrapers
This commit is contained in:
Rene Fichtmueller 2026-04-11 19:13:36 +02:00
parent 8905382a49
commit cdb8ef6e61
6 changed files with 973 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/scraper/src/index-pi.ts
View File

@ -49,6 +49,9 @@ const QUEUES = [
"scrape:pricing:skylane", "scrape:pricing:skylane",
"scrape:pricing:ascentoptics", "scrape:pricing:ascentoptics",
"scrape:pricing:gaotek", "scrape:pricing:gaotek",
"scrape:pricing:fibermall",
"scrape:pricing:vcelink",
"scrape:pricing:opticsbay",
// Form-factor coverage scrapers // Form-factor coverage scrapers
"scrape:pricing:comms-express", "scrape:pricing:comms-express",
"scrape:pricing:router-switch", "scrape:pricing:router-switch",
@ -130,6 +133,9 @@ async function main() {
const { scrapeSkylane } = await import("./scrapers/skylane"); const { scrapeSkylane } = await import("./scrapers/skylane");
const { scrapeAscentOptics } = await import("./scrapers/ascentoptics"); const { scrapeAscentOptics } = await import("./scrapers/ascentoptics");
const { scrapeGaoTek } = await import("./scrapers/gaotek"); const { scrapeGaoTek } = await import("./scrapers/gaotek");
const { scrapeFiberMall } = await import("./scrapers/fibermall");
const { scrapeVcelink } = await import("./scrapers/vcelink");
const { scrapeOpticsBay } = await import("./scrapers/opticsbay");
// ── Form-factor coverage scrapers ───────────────────────────────────── // ── Form-factor coverage scrapers ─────────────────────────────────────
const { scrapeCommsExpress } = await import("./scrapers/comms-express"); const { scrapeCommsExpress } = await import("./scrapers/comms-express");
@ -188,6 +194,9 @@ async function main() {
await boss.work("scrape:pricing:skylane", async () => { log("skylane"); await withIsolatedStorage("skylane", scrapeSkylane); }); await boss.work("scrape:pricing:skylane", async () => { log("skylane"); await withIsolatedStorage("skylane", scrapeSkylane); });
await boss.work("scrape:pricing:ascentoptics", async () => { log("ascentoptics"); await withIsolatedStorage("ascentoptics", scrapeAscentOptics); }); await boss.work("scrape:pricing:ascentoptics", async () => { log("ascentoptics"); await withIsolatedStorage("ascentoptics", scrapeAscentOptics); });
await boss.work("scrape:pricing:gaotek", async () => { log("gaotek"); await withIsolatedStorage("gaotek", scrapeGaoTek); }); await boss.work("scrape:pricing:gaotek", async () => { log("gaotek"); await withIsolatedStorage("gaotek", scrapeGaoTek); });
await boss.work("scrape:pricing:fibermall", async () => { log("fibermall"); await scrapeFiberMall(); });
await boss.work("scrape:pricing:vcelink", async () => { log("vcelink"); await scrapeVcelink(); });
await boss.work("scrape:pricing:opticsbay", async () => { log("opticsbay"); await scrapeOpticsBay(); });
await boss.work("scrape:pricing:comms-express", async () => { log("comms-express"); await scrapeCommsExpress(); }); await boss.work("scrape:pricing:comms-express", async () => { log("comms-express"); await scrapeCommsExpress(); });
await boss.work("scrape:pricing:router-switch", async () => { log("router-switch"); await scrapeRouterSwitch(); }); await boss.work("scrape:pricing:router-switch", async () => { log("router-switch"); await scrapeRouterSwitch(); });
@ -229,7 +238,7 @@ async function main() {
await boss.work("compute:reorder-signals", async () => { log("reorder"); await computeReorderSignals(); }); await boss.work("compute:reorder-signals", async () => { log("reorder"); await computeReorderSignals(); });
await boss.work("compute:forecast", async () => { log("forecast"); await runForecastEngine(); }); await boss.work("compute:forecast", async () => { log("forecast"); await runForecastEngine(); });
console.log(`${QUEUES.length} workers active — running 24/7\n`); console.log(`${QUEUES.length} queues / workers active — running 24/7\n`);
process.on("SIGTERM", async () => { await boss.stop(); process.exit(0); }); process.on("SIGTERM", async () => { await boss.stop(); process.exit(0); });
process.on("SIGINT", async () => { await boss.stop(); process.exit(0); }); process.on("SIGINT", async () => { await boss.stop(); process.exit(0); });
} }

28
packages/scraper/src/scheduler.ts
View File

@ -127,6 +127,9 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
"scrape:pricing:naddod", "scrape:pricing:naddod",
"scrape:pricing:qsfptek", "scrape:pricing:qsfptek",
"scrape:pricing:addon", "scrape:pricing:addon",
"scrape:pricing:fibermall",
"scrape:pricing:vcelink",
"scrape:pricing:opticsbay",
// ── Prediction Signal Scrapers (new) ────────────────────────────── // ── Prediction Signal Scrapers (new) ──────────────────────────────
"scrape:signals:sec-edgar", "scrape:signals:sec-edgar",
"scrape:signals:github", "scrape:signals:github",
@ -180,6 +183,9 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:fibermall", "57 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════
// FLEXOPTIX CATALOG — every 2h (primary price source) // FLEXOPTIX CATALOG — every 2h (primary price source)
@ -273,7 +279,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 }); await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
console.log("All schedules registered — 24/7 continuous scraping (50 jobs)"); console.log("All schedules registered — 24/7 continuous scraping (53 jobs)");
} }
export async function registerWorkers(boss: PgBoss): Promise<void> { export async function registerWorkers(boss: PgBoss): Promise<void> {
@ -545,5 +551,23 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
await scrapeAddonNetworks(); await scrapeAddonNetworks();
}); });
console.log("All workers registered (58 jobs, 24/7 continuous)"); await boss.work("scrape:pricing:fibermall", async () => {
console.log(`[${new Date().toISOString()}] Running: FiberMall pricing`);
const { scrapeFiberMall } = await import("./scrapers/fibermall");
await scrapeFiberMall();
});
await boss.work("scrape:pricing:vcelink", async () => {
console.log(`[${new Date().toISOString()}] Running: Vcelink pricing`);
const { scrapeVcelink } = await import("./scrapers/vcelink");
await scrapeVcelink();
});
await boss.work("scrape:pricing:opticsbay", async () => {
console.log(`[${new Date().toISOString()}] Running: OpticsBay pricing`);
const { scrapeOpticsBay } = await import("./scrapers/opticsbay");
await scrapeOpticsBay();
});
console.log("All workers registered (61 jobs, 24/7 continuous)");
} }

275
packages/scraper/src/scrapers/fibermall.ts Normal file
View File

@ -0,0 +1,275 @@
/**
* FiberMall Scraper — Chinese compatible transceiver vendor
*
* fibermall.com — custom PHP/REST shop, USD pricing.
* Large catalog: 1G–400G, SFP/SFP+/QSFP28/QSFP-DD/OSFP.
* Pagination via ?page=N. Rate limited: 1 req/2sec.
*
* FiberMall (Shenzhen FiberMall Technology Co.) offers DAC/AOC + optics,
* transparent USD prices, no login required.
*/
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash";
const BASE = "https://www.fibermall.com";
const HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
};
const MAX_PAGES = 30;
const CATEGORIES = [
{ path: "/c/1g-sfp-transceiver/", formFactor: "SFP", speed: "1G", speedGbps: 1 },
{ path: "/c/10g-sfp-transceiver/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
{ path: "/c/25g-sfp28-transceiver/", formFactor: "SFP28",speed: "25G", speedGbps: 25 },
{ path: "/c/40g-qsfp-transceiver/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
{ path: "/c/100g-qsfp28-transceiver/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
{ path: "/c/200g-qsfp56-transceiver/", formFactor: "QSFP56", speed: "200G", speedGbps: 200 },
{ path: "/c/400g-qsfp-dd-transceiver/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
{ path: "/c/800g-osfp-transceiver/", formFactor: "OSFP", speed: "800G", speedGbps: 800 },
{ path: "/c/dac-cable/", formFactor: "DAC", speed: "10G", speedGbps: 10 },
{ path: "/c/aoc-cable/", formFactor: "AOC", speed: "10G", speedGbps: 10 },
{ path: "/c/optical-transceivers/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
];
interface Product {
partNumber: string;
name: string;
url: string;
price?: number;
formFactor: string;
speed: string;
speedGbps: number;
reachLabel?: string;
reachMeters?: number;
fiberType?: string;
wavelength?: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function detectReach(text: string): { label: string; meters: number } | undefined {
const patterns: [RegExp, string, number][] = [
[/\b120\s*km\b/i, "120km", 120000],
[/\b80\s*km\b/i, "80km", 80000],
[/\b40\s*km\b/i, "40km", 40000],
[/\b20\s*km\b/i, "20km", 20000],
[/\b10\s*km\b/i, "10km", 10000],
[/\b2\s*km\b/i, "2km", 2000],
[/\b550\s*m\b/i, "550m", 550],
[/\b500\s*m\b/i, "500m", 500],
[/\b300\s*m\b/i, "300m", 300],
[/\b150\s*m\b/i, "150m", 150],
[/\b100\s*m\b/i, "100m", 100],
[/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
[/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
[/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500],
[/\bFR4?\b/, "2km", 2000],
];
for (const [re, label, meters] of patterns) {
if (re.test(text)) return { label, meters };
}
return undefined;
}
function detectFiber(text: string): string {
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper";
return "";
}
function detectWavelength(text: string): string {
const m = text.match(/(\d{3,4})\s*nm/i);
return m ? m[1] : "";
}
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
const products: Product[] = [];
const seen = new Set<string>();
const collapsed = html.replace(/\s+/g, " ");
// Strategy 1: fibermall product card (class="product-item" or similar)
for (const m of collapsed.matchAll(/<(?:li|div|article)[^>]+class="[^"]*(?:product-item|product-thumb|goods-item|pro-item)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div|article)>/gi)) {
const card = m[1];
const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]+)"/i) ||
card.match(/href="(\/[a-z0-9][^"?#]{5,})"/i);
if (!urlMatch) continue;
const url = urlMatch[1].startsWith("http") ? urlMatch[1] : BASE + urlMatch[1];
if (seen.has(url) || !/fibermall\.com|\/product|\/p\//i.test(url)) continue;
seen.add(url);
const nameMatch = card.match(/<(?:h[23456]|p)[^>]+class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]{8,})<\//i) ||
card.match(/title="([^"]{10,})"/i) ||
card.match(/<a[^>]*>([^<]{10,})<\/a>/i);
if (!nameMatch) continue;
const name = nameMatch[1].trim().replace(/&amp;/g, "&").replace(/&#\d+;/g, "");
if (name.length < 5) continue;
const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);
products.push({
partNumber, name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
// Strategy 2: generic product link scan
if (products.length === 0) {
for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?fibermall\.com\/[^"?#]{10,})"[^>]*>([^<]{10,})</gi)) {
const url = m[1];
const name = m[2].trim().replace(/&amp;/g, "&");
if (seen.has(url) || name.length < 8) continue;
if (!/transceiver|sfp|qsfp|osfp|dac|aoc|optic/i.test(name)) continue;
seen.add(url);
const idx = collapsed.indexOf(url);
const ctx = collapsed.slice(Math.max(0, idx - 300), idx + 600);
const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
}
return products;
}
async function fetchPage(url: string): Promise<string> {
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
return resp.text();
}
export async function scrapeFiberMall(): Promise<void> {
console.log("=== FiberMall Scraper Starting ===\n");
const vendorId = await ensureVendor(
"FiberMall",
"compatible",
"https://www.fibermall.com",
"https://www.fibermall.com/c/optical-transceivers/",
);
let totalProducts = 0;
let priceUpdates = 0;
const seenCategories = new Set<string>();
for (const cat of CATEGORIES) {
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);
try {
const html1 = await fetchPage(BASE + cat.path);
const catProducts = parseProductList(html1, cat);
// Skip generic fallback if specific categories already scraped
if (cat.path.includes("/optical-transceivers/") && seenCategories.size > 3) {
console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`);
continue;
}
if (catProducts.length === 0) {
console.log(" No products on page 1 — trying alternate pagination");
// Try ?page=1 format
try {
const html1alt = await fetchPage(BASE + cat.path + "?page=1");
catProducts.push(...parseProductList(html1alt, cat));
} catch { /* ignore */ }
}
if (catProducts.length === 0) {
console.log(" No products found — skipping");
continue;
}
seenCategories.add(cat.path);
console.log(` Found ${catProducts.length} products on page 1`);
const allProducts = [...catProducts];
for (let page = 2; page <= MAX_PAGES; page++) {
await sleep(2000);
try {
// Try both pagination formats
const pageUrl = `${BASE}${cat.path}?page=${page}`;
const html = await fetchPage(pageUrl);
const pageProds = parseProductList(html, cat);
if (pageProds.length === 0) break;
allProducts.push(...pageProds);
console.log(` Page ${page}: ${pageProds.length} products`);
} catch (err) {
console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`);
break;
}
}
const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i);
console.log(` Total unique: ${uniqueProducts.length}`);
for (const product of uniqueProducts) {
try {
const txId = await findOrCreateScrapedTransceiver({
partNumber: product.partNumber,
vendorId,
formFactor: product.formFactor,
speedGbps: product.speedGbps,
speed: product.speed,
reachMeters: product.reachMeters,
reachLabel: product.reachLabel,
fiberType: product.fiberType,
wavelengths: product.wavelength,
category: "DataCenter",
});
if (product.price && product.price > 0) {
const hash = contentHash({ price: product.price, part: product.partNumber });
const updated = await upsertPriceObservation({
transceiverId: txId,
sourceVendorId: vendorId,
price: product.price,
currency: "USD",
stockLevel: "in_stock",
url: product.url,
contentHash: hash,
});
if (updated) priceUpdates++;
}
totalProducts++;
} catch (err) {
console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`);
}
}
} catch (err) {
console.error(` Category failed: ${(err as Error).message}`);
}
await sleep(2000);
}
console.log(`\n=== FiberMall Complete: ${totalProducts} products, ${priceUpdates} price updates ===`);
}
if (require.main === module) {
scrapeFiberMall()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}

257
packages/scraper/src/scrapers/opticsbay.ts Normal file
View File

@ -0,0 +1,257 @@
/**
* OpticsBay Scraper — North American compatible transceiver vendor
*
* opticsbay.com — WooCommerce store, USD/CAD pricing.
* Covers SFP/SFP+/QSFP28/QSFP-DD with competitive pricing.
* Rate limited: 1 req/2sec.
*
* OpticsBay offers compatible optics for Cisco, Juniper, Arista, HPE.
* Transparent pricing, no login required.
*/
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash";
const BASE = "https://www.opticsbay.com";
const HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
};
const MAX_PAGES = 20;
const CATEGORIES = [
{ path: "/product-category/sfp-transceiver/", formFactor: "SFP", speed: "1G", speedGbps: 1 },
{ path: "/product-category/sfp-plus-transceiver/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
{ path: "/product-category/sfp28-transceiver/", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
{ path: "/product-category/qsfp-plus-transceiver/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
{ path: "/product-category/qsfp28-transceiver/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
{ path: "/product-category/qsfp-dd-transceiver/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
{ path: "/product-category/dac-cable/", formFactor: "DAC", speed: "10G", speedGbps: 10 },
{ path: "/product-category/transceivers/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
];
interface Product {
partNumber: string;
name: string;
url: string;
price?: number;
formFactor: string;
speed: string;
speedGbps: number;
reachLabel?: string;
reachMeters?: number;
fiberType?: string;
wavelength?: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function detectReach(text: string): { label: string; meters: number } | undefined {
const patterns: [RegExp, string, number][] = [
[/\b120\s*km\b/i, "120km", 120000],
[/\b80\s*km\b/i, "80km", 80000],
[/\b40\s*km\b/i, "40km", 40000],
[/\b20\s*km\b/i, "20km", 20000],
[/\b10\s*km\b/i, "10km", 10000],
[/\b2\s*km\b/i, "2km", 2000],
[/\b550\s*m\b/i, "550m", 550],
[/\b300\s*m\b/i, "300m", 300],
[/\b100\s*m\b/i, "100m", 100],
[/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
[/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
[/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500],
[/\bFR4?\b/, "2km", 2000],
];
for (const [re, label, meters] of patterns) {
if (re.test(text)) return { label, meters };
}
return undefined;
}
function detectFiber(text: string): string {
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper";
return "";
}
function detectWavelength(text: string): string {
const m = text.match(/(\d{3,4})\s*nm/i);
return m ? m[1] : "";
}
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
const products: Product[] = [];
const seen = new Set<string>();
const collapsed = html.replace(/\s+/g, " ");
// Strategy 1: WooCommerce standard product loop
for (const m of collapsed.matchAll(/<li[^>]+class="[^"]*product[^"]*"[^>]*>([\s\S]*?)<\/li>/gi)) {
const card = m[1];
const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?opticsbay\.com\/[^"?#]+)"/i);
if (!urlMatch) continue;
const url = urlMatch[1];
if (seen.has(url)) continue;
seen.add(url);
const nameMatch =
card.match(/woocommerce-loop-product__title[^>]*>([^<]+)</i) ||
card.match(/<h[23][^>]*>([^<]{8,})<\/h[23]>/i);
if (!nameMatch) continue;
const name = nameMatch[1].trim().replace(/&amp;/g, "&").replace(/&#\d+;/g, "");
if (name.length < 5) continue;
const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60),
name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
// Strategy 2: Generic product link scan
if (products.length === 0) {
for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?opticsbay\.com\/(?:product|shop)\/[^"?#]+)"[^>]*>([^<]{8,})</gi)) {
const url = m[1];
const name = m[2].trim().replace(/&amp;/g, "&");
if (seen.has(url) || name.length < 8) continue;
if (!/transceiver|sfp|qsfp|dac|optic/i.test(name + url)) continue;
seen.add(url);
const idx = collapsed.indexOf(url);
const ctx = collapsed.slice(Math.max(0, idx - 300), idx + 600);
const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
}
return products;
}
async function fetchPage(url: string): Promise<string> {
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
return resp.text();
}
export async function scrapeOpticsBay(): Promise<void> {
console.log("=== OpticsBay Scraper Starting ===\n");
const vendorId = await ensureVendor(
"OpticsBay",
"compatible",
"https://www.opticsbay.com",
"https://www.opticsbay.com/product-category/transceivers/",
);
let totalProducts = 0;
let priceUpdates = 0;
const seenCategories = new Set<string>();
for (const cat of CATEGORIES) {
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);
try {
const html1 = await fetchPage(BASE + cat.path);
const catProducts = parseProductList(html1, cat);
if (cat.path.includes("/transceivers/") && seenCategories.size > 3) {
console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`);
continue;
}
if (catProducts.length === 0) {
console.log(" No products on page 1 — skipping");
continue;
}
seenCategories.add(cat.path);
console.log(` Found ${catProducts.length} products on page 1`);
const allProducts = [...catProducts];
for (let page = 2; page <= MAX_PAGES; page++) {
await sleep(2000);
try {
const pageUrl = `${BASE}${cat.path}page/${page}/`;
const html = await fetchPage(pageUrl);
const pageProds = parseProductList(html, cat);
if (pageProds.length === 0) break;
allProducts.push(...pageProds);
console.log(` Page ${page}: ${pageProds.length} products`);
} catch (err) {
console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`);
break;
}
}
const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i);
console.log(` Total unique: ${uniqueProducts.length}`);
for (const product of uniqueProducts) {
try {
const txId = await findOrCreateScrapedTransceiver({
partNumber: product.partNumber,
vendorId,
formFactor: product.formFactor,
speedGbps: product.speedGbps,
speed: product.speed,
reachMeters: product.reachMeters,
reachLabel: product.reachLabel,
fiberType: product.fiberType,
wavelengths: product.wavelength,
category: "DataCenter",
});
if (product.price && product.price > 0) {
const hash = contentHash({ price: product.price, part: product.partNumber });
const updated = await upsertPriceObservation({
transceiverId: txId,
sourceVendorId: vendorId,
price: product.price,
currency: "USD",
stockLevel: "in_stock",
url: product.url,
contentHash: hash,
});
if (updated) priceUpdates++;
}
totalProducts++;
} catch (err) {
console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`);
}
}
} catch (err) {
console.error(` Category failed: ${(err as Error).message}`);
}
await sleep(2000);
}
console.log(`\n=== OpticsBay Complete: ${totalProducts} products, ${priceUpdates} price updates ===`);
}
if (require.main === module) {
scrapeOpticsBay()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}

211
packages/scraper/src/scrapers/qsfptek.ts
View File

@ -1,35 +1,48 @@
/** /**
* QSFPTEK Scraper — Chinese compatible transceiver vendor * QSFPTEK Scraper — Chinese compatible transceiver vendor
* *
* qsfptek.com — Server-rendered HTML shop, USD pricing. * qsfptek.com — migrated to custom Java/Spring + Vue.js in 2025.
* Focuses on QSFP+/QSFP28/QSFP-DD/SFP+ form factors. * Old OpenCart /c/*.html paths are gone (404).
* Rate limited: 1 req/2sec. * Products are served via HTML-fragment API: /mall/commodity/list
* *
* QSFPTEK (Shenzhen Optotech Technology) — competitive pricing, * API: GET /mall/commodity/list?categoryId=21&attributes=VALUE_ID&page=N&pageSize=30
* transparent USD prices, no account required. * Returns HTML fragment with product cards.
*
* Rate limited: 1 req/2sec.
*/ */
import * as cheerio from "cheerio";
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash"; import { contentHash } from "../utils/hash";
const BASE = "https://www.qsfptek.com"; const BASE = "https://www.qsfptek.com";
const LIST_API = "/mall/commodity/list";
const CATEGORY_ID = "21"; // Transceivers (top-level, all products)
const HEADERS = { const HEADERS = {
"User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", Accept: "text/html,application/xhtml+xml",
"Accept-Language": "en-US,en;q=0.9", "Accept-Language": "en-US,en;q=0.9",
}; };
const MAX_PAGES = 30; const MAX_PAGES = 40;
const PAGE_SIZE = 30;
const CATEGORIES = [ // Data rate attribute values (found in /mall/commodity/attribute?categoryId=21)
{ path: "/c/sfp-transceiver.html", formFactor: "SFP", speed: "1G", speedGbps: 1 }, // pid = "2c9180837bbaf08f017bbdd1ebf7001e" (Data Rate attribute group)
{ path: "/c/sfp-plus-transceiver.html", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, const DATA_RATE_ATTRIBUTES: Array<{
{ path: "/c/sfp28-transceiver.html", formFactor: "SFP28", speed: "25G", speedGbps: 25 }, attrId: string;
{ path: "/c/qsfp-plus-transceiver.html", formFactor: "QSFP+", speed: "40G", speedGbps: 40 }, speed: string;
{ path: "/c/qsfp28-transceiver.html", formFactor: "QSFP28", speed: "100G", speedGbps: 100 }, speedGbps: number;
{ path: "/c/qsfp56-transceiver.html", formFactor: "QSFP56", speed: "200G", speedGbps: 200 }, formFactor: string;
{ path: "/c/qsfp-dd-transceiver.html", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }, }> = [
{ path: "/c/osfp-transceiver.html", formFactor: "OSFP", speed: "800G", speedGbps: 800 }, { attrId: "2c9180837bbaf08f017bbde50ea30101", speed: "1G", speedGbps: 1, formFactor: "SFP" },
{ path: "/c/optical-transceiver.html", formFactor: "SFP+", speed: "10G", speedGbps: 10 }, { attrId: "2c9180837bbaf08f017bbde50ea30100", speed: "10G", speedGbps: 10, formFactor: "SFP+" },
{ attrId: "2c9180837bbaf08f017bbde50ea300ff", speed: "25G", speedGbps: 25, formFactor: "SFP28" },
{ attrId: "2c9180837bbaf08f017bbde50ea300fe", speed: "40G", speedGbps: 40, formFactor: "QSFP+" },
{ attrId: "2c9180837bbaf08f017bbde50ea300fd", speed: "100G", speedGbps: 100, formFactor: "QSFP28" },
{ attrId: "2c98491f8f4b8e55018f94aa8c5d48ff", speed: "200G", speedGbps: 200, formFactor: "QSFP56" },
{ attrId: "2c9180837e2e7f64017e389caf0700c8", speed: "400G", speedGbps: 400, formFactor: "QSFP-DD" },
{ attrId: "2c98491f8e363cbf018e3faa5b6d2f8b", speed: "800G", speedGbps: 800, formFactor: "OSFP" },
]; ];
interface Product { interface Product {
@ -44,7 +57,6 @@ interface Product {
reachMeters?: number; reachMeters?: number;
fiberType?: string; fiberType?: string;
wavelength?: string; wavelength?: string;
compatibleWith?: string;
} }
function sleep(ms: number): Promise<void> { function sleep(ms: number): Promise<void> {
@ -60,19 +72,15 @@ function detectReach(text: string): { label: string; meters: number } | undefine
[/\b10\s*km\b/i, "10km", 10000], [/\b10\s*km\b/i, "10km", 10000],
[/\b2\s*km\b/i, "2km", 2000], [/\b2\s*km\b/i, "2km", 2000],
[/\b550\s*m\b/i, "550m", 550], [/\b550\s*m\b/i, "550m", 550],
[/\b500\s*m\b/i, "500m", 500],
[/\b300\s*m\b/i, "300m", 300], [/\b300\s*m\b/i, "300m", 300],
[/\b100\s*m\b/i, "100m", 100], [/\b100\s*m\b/i, "100m", 100],
[/\bLR4\b/, "10km", 10000], [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
[/\bLR\b/, "10km", 10000], [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
[/\bER4?\b/, "40km", 40000], [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500],
[/\bZR4?\b/, "80km", 80000],
[/\bSR4?\b/, "300m", 300],
[/\bDR4?\b/, "500m", 500],
[/\bFR4?\b/, "2km", 2000], [/\bFR4?\b/, "2km", 2000],
]; ];
for (const [regex, label, meters] of patterns) { for (const [re, label, meters] of patterns) {
if (regex.test(text)) return { label, meters }; if (re.test(text)) return { label, meters };
} }
return undefined; return undefined;
} }
@ -85,153 +93,109 @@ function detectFiber(text: string): string {
} }
function detectWavelength(text: string): string { function detectWavelength(text: string): string {
const match = text.match(/(\d{3,4})\s*nm/i); const m = text.match(/(\d{3,4})\s*nm/i);
return match ? match[1] : ""; return m ? m[1] : "";
} }
function extractCompatibleVendor(name: string): string { function parseProductFragment(html: string, attr: typeof DATA_RATE_ATTRIBUTES[number]): Product[] {
const brands = ["Cisco", "Juniper", "Arista", "HPE", "Aruba", "Dell", "Brocade", "Extreme", const $ = cheerio.load(html);
"Huawei", "Nokia", "MikroTik", "Mellanox", "Nvidia", "Ubiquiti", "Allied Telesis"];
for (const brand of brands) {
if (new RegExp(`\\b${brand}\\b`, "i").test(name)) return brand;
}
return "";
}
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
const products: Product[] = []; const products: Product[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
const collapsed = html.replace(/\s+/g, " ");
// Strategy 1: OpenCart / custom card layout using matchAll // Product cards: li.Hot, li.New, li with class containing product
for (const cardMatch of collapsed.matchAll(/<div[^>]+class="[^"]*product-(?:thumb|layout)[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/gi)) { $("li, div.kuang").each((_i, el) => {
const card = cardMatch[1]; const $el = $(el);
const urlMatch = card.match(/href="(https?:\/\/(?:www\.)?qsfptek\.com\/[^"]+)"/i); // Get product URL
if (!urlMatch) continue; const href = $el.find("a[href*='/product/']").first().attr("href") ||
const url = urlMatch[1]; $el.find("a[href*='/en/product/']").first().attr("href");
if (seen.has(url)) continue; if (!href) return;
const url = href.startsWith("http") ? href : BASE + href;
if (seen.has(url)) return;
seen.add(url); seen.add(url);
const nameMatch = card.match(/<h[34][^>]*>\s*<a[^>]*>([^<]{10,})<\/a>/i) || // Get product name from h3
card.match(/<a[^>]*title="([^"]{10,})"/i); const name = $el.find("h3.tit, h3").first().text().trim().replace(/\s+/g, " ");
if (!nameMatch) continue; if (!name || name.length < 10) return;
const name = nameMatch[1].trim().replace(/&amp;/g, "&").replace(/&#[0-9]+;/g, "");
if (name.length < 5) continue;
const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/); // Get price: "US$ 33.90" format
const priceText = $el.find("*").text();
const priceMatch = priceText.match(/US\$\s*([\d,]+\.?\d{0,2})/);
const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined; const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name); const reach = detectReach(name);
const partNumber = name.split(/\s+(?:compatible|for|sfp|qsfp)/i)[0]?.trim().slice(0, 80) || name.slice(0, 60); const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z]|sfp|qsfp)/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);
products.push({ products.push({
partNumber, name, url, partNumber, name, url,
price: price && price > 0 && price < 100000 ? price : undefined, price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps, formFactor: attr.formFactor, speed: attr.speed, speedGbps: attr.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters, reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name), fiberType: detectFiber(name), wavelength: detectWavelength(name),
compatibleWith: extractCompatibleVendor(name),
}); });
}
// Strategy 2: Generic product link scan using matchAll
if (products.length === 0) {
for (const m of collapsed.matchAll(/href="(https?:\/\/(?:www\.)?qsfptek\.com\/(?:p|product)[^"?#]+)"[^>]*>([^<]{10,})</gi)) {
const url = m[1];
const name = m[2].trim().replace(/&amp;/g, "&");
if (seen.has(url) || name.length < 10) continue;
if (!/transceiver|sfp|qsfp|osfp|dac|aoc/i.test(name)) continue;
seen.add(url);
const idx = collapsed.indexOf(url);
const ctx = collapsed.slice(Math.max(0, idx - 300), idx + 600);
const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
compatibleWith: extractCompatibleVendor(name),
}); });
}
}
return products; return products;
} }
async function fetchPage(url: string): Promise<string> { async function fetchProductList(attrId: string, page: number): Promise<string> {
const url = `${BASE}${LIST_API}?categoryId=${CATEGORY_ID}&attributes=${attrId}&page=${page}&pageSize=${PAGE_SIZE}`;
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
return resp.text(); return resp.text();
} }
export async function scrapeQsfptek(): Promise<void> { export async function scrapeQsfptek(): Promise<void> {
console.log("=== QSFPTEK Scraper Starting ===\n"); console.log("=== QSFPTEK Scraper Starting (API mode) ===\n");
const vendorId = await ensureVendor( const vendorId = await ensureVendor(
"QSFPTEK", "QSFPTEK",
"compatible", "compatible",
"https://www.qsfptek.com", "https://www.qsfptek.com",
"https://www.qsfptek.com/c/optical-transceiver.html", "https://www.qsfptek.com/c/fiber-optic-transceiver.html",
); );
let totalProducts = 0; let totalProducts = 0;
let priceUpdates = 0; let priceUpdates = 0;
const seenCategories = new Set<string>();
for (const cat of CATEGORIES) { for (const attr of DATA_RATE_ATTRIBUTES) {
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`); console.log(`\n--- ${attr.formFactor} (${attr.speed}) ---`);
const allProducts: Product[] = [];
const allSeen = new Set<string>();
for (let page = 1; page <= MAX_PAGES; page++) {
try { try {
const html1 = await fetchPage(BASE + cat.path); const html = await fetchProductList(attr.attrId, page);
const catProducts = parseProductList(html1, cat); const pageProds = parseProductFragment(html, attr);
if (cat.path.includes("/optical-transceiver") && seenCategories.size > 3) { if (pageProds.length === 0) {
console.log(` Skipping generic fallback (${seenCategories.size} specific categories scraped)`); if (page === 1) console.log(" No products on page 1 — skipping");
continue; else console.log(` Page ${page}: empty, stopping`);
break;
} }
if (catProducts.length === 0) { // Dedupe against already seen (API may repeat items)
console.log(" No products on page 1 — skipping"); const newProds = pageProds.filter(p => {
continue; if (allSeen.has(p.url)) return false;
} allSeen.add(p.url);
return true;
});
allProducts.push(...newProds);
console.log(` Page ${page}: ${pageProds.length} total, ${newProds.length} new (${allProducts.length} running)`);
seenCategories.add(cat.path); // If we got fewer than pageSize, we're on the last page
console.log(` Found ${catProducts.length} products on page 1`); if (pageProds.length < PAGE_SIZE) break;
const totalPagesMatch = if (page < MAX_PAGES) await sleep(2000);
html1.match(/total-page[^>]*>\s*(\d+)/) ||
html1.match(/page\s+\d+\s+of\s+(\d+)/i);
const totalPages = totalPagesMatch ? Math.min(parseInt(totalPagesMatch[1]), MAX_PAGES) : 3;
console.log(` Total pages (estimate): ${totalPages}`);
const allProducts = [...catProducts];
for (let page = 2; page <= totalPages; page++) {
await sleep(2000);
try {
const pageUrl = BASE + cat.path.replace(".html", "") + `?page=${page}`;
const html = await fetchPage(pageUrl);
const pageProds = parseProductList(html, cat);
if (pageProds.length === 0) break;
allProducts.push(...pageProds);
console.log(` Page ${page}: ${pageProds.length} products`);
} catch (err) { } catch (err) {
console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`); console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 80)}`);
break; break;
} }
} }
const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex((x) => x.url === p.url) === i); console.log(` Unique products: ${allProducts.length}`);
console.log(` Total unique: ${uniqueProducts.length}`);
for (const product of uniqueProducts) { for (const product of allProducts) {
try { try {
const txId = await findOrCreateScrapedTransceiver({ const txId = await findOrCreateScrapedTransceiver({
partNumber: product.partNumber, partNumber: product.partNumber,
@ -264,9 +228,6 @@ export async function scrapeQsfptek(): Promise<void> {
console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`); console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`);
} }
} }
} catch (err) {
console.error(` Category failed: ${(err as Error).message}`);
}
await sleep(2000); await sleep(2000);
} }

284
packages/scraper/src/scrapers/vcelink.ts Normal file
View File

@ -0,0 +1,284 @@
/**
* Vcelink Scraper — Chinese compatible transceiver vendor
*
* vcelink.com — Shopify-based store, USD pricing.
* Covers SFP/SFP+/QSFP28/QSFP-DD, DAC/AOC cables.
* Rate limited: 1 req/2sec.
*
* Vcelink offers Cisco/Juniper/Arista-compatible optics at competitive
* USD prices, publicly accessible without login.
*/
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash";
const BASE = "https://www.vcelink.com";
const HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
};
const MAX_PAGES = 20;
const CATEGORIES = [
{ path: "/collections/1g-sfp", formFactor: "SFP", speed: "1G", speedGbps: 1 },
{ path: "/collections/10g-sfp", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
{ path: "/collections/25g-sfp28", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
{ path: "/collections/40g-qsfp", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
{ path: "/collections/100g-qsfp28",formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
{ path: "/collections/400g-qsfp-dd",formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 },
{ path: "/collections/dac-cable", formFactor: "DAC", speed: "10G", speedGbps: 10 },
{ path: "/collections/aoc-cable", formFactor: "AOC", speed: "10G", speedGbps: 10 },
{ path: "/collections/all", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
];
interface Product {
partNumber: string;
name: string;
url: string;
price?: number;
formFactor: string;
speed: string;
speedGbps: number;
reachLabel?: string;
reachMeters?: number;
fiberType?: string;
wavelength?: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function detectReach(text: string): { label: string; meters: number } | undefined {
const patterns: [RegExp, string, number][] = [
[/\b120\s*km\b/i, "120km", 120000],
[/\b80\s*km\b/i, "80km", 80000],
[/\b40\s*km\b/i, "40km", 40000],
[/\b20\s*km\b/i, "20km", 20000],
[/\b10\s*km\b/i, "10km", 10000],
[/\b2\s*km\b/i, "2km", 2000],
[/\b550\s*m\b/i, "550m", 550],
[/\b300\s*m\b/i, "300m", 300],
[/\b100\s*m\b/i, "100m", 100],
[/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
[/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
[/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500],
[/\bFR4?\b/, "2km", 2000],
];
for (const [re, label, meters] of patterns) {
if (re.test(text)) return { label, meters };
}
return undefined;
}
function detectFiber(text: string): string {
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper";
return "";
}
function detectWavelength(text: string): string {
const m = text.match(/(\d{3,4})\s*nm/i);
return m ? m[1] : "";
}
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
const products: Product[] = [];
const seen = new Set<string>();
const collapsed = html.replace(/\s+/g, " ");
// Strategy 1: Shopify product grid (class="product-item" or similar)
for (const m of collapsed.matchAll(/<(?:li|div)[^>]+class="[^"]*(?:product-item|product__card|grid__item|product-card)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div)>/gi)) {
const card = m[1];
const urlMatch = card.match(/href="(\/products\/[^"?#]+)"/i) ||
card.match(/href="(https?:\/\/(?:www\.)?vcelink\.com\/products\/[^"?#]+)"/i);
if (!urlMatch) continue;
const rawUrl = urlMatch[1];
const url = rawUrl.startsWith("http") ? rawUrl : BASE + rawUrl;
if (seen.has(url)) continue;
seen.add(url);
const nameMatch = card.match(/<h[23456][^>]*>([^<]{8,})<\/h[23456]>/i) ||
card.match(/class="[^"]*(?:title|name)[^"]*"[^>]*>([^<]{8,})</i) ||
card.match(/aria-label="([^"]{8,})"/i);
if (!nameMatch) continue;
const name = nameMatch[1].trim().replace(/&amp;/g, "&").replace(/&#\d+;/g, "");
if (name.length < 5 || !/sfp|qsfp|transceiver|dac|aoc|optic/i.test(name + url)) continue;
const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);
products.push({
partNumber, name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
// Strategy 2: JSON-LD product data (Shopify embeds this)
if (products.length === 0) {
for (const m of collapsed.matchAll(/"@type"\s*:\s*"Product"[^}]*"name"\s*:\s*"([^"]{8,})"[^}]*"url"\s*:\s*"([^"]+)"[^}]*"price"\s*:\s*"?([\d.]+)/gi)) {
const name = m[1].replace(/\\n/g, " ").trim();
const url = m[2].startsWith("http") ? m[2] : BASE + m[2];
const price = parseFloat(m[3]);
if (seen.has(url)) continue;
if (!/sfp|qsfp|transceiver|dac|aoc|optic/i.test(name + url)) continue;
seen.add(url);
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
name, url,
price: price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
}
// Strategy 3: Generic product links
if (products.length === 0) {
for (const m of collapsed.matchAll(/href="(\/products\/[^"?#]{5,})"[^>]*>([^<]{8,})</gi)) {
const url = BASE + m[1];
const name = m[2].trim().replace(/&amp;/g, "&");
if (seen.has(url) || name.length < 8) continue;
if (!/transceiver|sfp|qsfp|dac|aoc|optic/i.test(name + url)) continue;
seen.add(url);
const idx = collapsed.indexOf(m[1]);
const ctx = collapsed.slice(Math.max(0, idx - 200), idx + 400);
const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
const reach = detectReach(name);
products.push({
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
name, url,
price: price && price > 0 && price < 100000 ? price : undefined,
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
reachLabel: reach?.label, reachMeters: reach?.meters,
fiberType: detectFiber(name), wavelength: detectWavelength(name),
});
}
}
return products;
}
async function fetchPage(url: string): Promise<string> {
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
return resp.text();
}
export async function scrapeVcelink(): Promise<void> {
console.log("=== Vcelink Scraper Starting ===\n");
const vendorId = await ensureVendor(
"Vcelink",
"compatible",
"https://www.vcelink.com",
"https://www.vcelink.com/collections/all",
);
let totalProducts = 0;
let priceUpdates = 0;
const seenCategories = new Set<string>();
for (const cat of CATEGORIES) {
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);
try {
const html1 = await fetchPage(BASE + cat.path);
const catProducts = parseProductList(html1, cat);
if (cat.path.includes("/collections/all") && seenCategories.size > 3) {
console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`);
continue;
}
if (catProducts.length === 0) {
console.log(" No products on page 1 — skipping");
continue;
}
seenCategories.add(cat.path);
console.log(` Found ${catProducts.length} products on page 1`);
const allProducts = [...catProducts];
for (let page = 2; page <= MAX_PAGES; page++) {
await sleep(2000);
try {
// Shopify pagination: ?page=N
const pageUrl = `${BASE}${cat.path}?page=${page}`;
const html = await fetchPage(pageUrl);
const pageProds = parseProductList(html, cat);
if (pageProds.length === 0) break;
allProducts.push(...pageProds);
console.log(` Page ${page}: ${pageProds.length} products`);
} catch (err) {
console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`);
break;
}
}
const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i);
console.log(` Total unique: ${uniqueProducts.length}`);
for (const product of uniqueProducts) {
try {
const txId = await findOrCreateScrapedTransceiver({
partNumber: product.partNumber,
vendorId,
formFactor: product.formFactor,
speedGbps: product.speedGbps,
speed: product.speed,
reachMeters: product.reachMeters,
reachLabel: product.reachLabel,
fiberType: product.fiberType,
wavelengths: product.wavelength,
category: "DataCenter",
});
if (product.price && product.price > 0) {
const hash = contentHash({ price: product.price, part: product.partNumber });
const updated = await upsertPriceObservation({
transceiverId: txId,
sourceVendorId: vendorId,
price: product.price,
currency: "USD",
stockLevel: "in_stock",
url: product.url,
contentHash: hash,
});
if (updated) priceUpdates++;
}
totalProducts++;
} catch (err) {
console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`);
}
}
} catch (err) {
console.error(` Category failed: ${(err as Error).message}`);
}
await sleep(2000);
}
console.log(`\n=== Vcelink Complete: ${totalProducts} products, ${priceUpdates} price updates ===`);
}
if (require.main === module) {
scrapeVcelink()
.then(() => pool.end())
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}