aa91798e8d
price?: number narrowing via typeof/!== undefined does not work for arithmetic comparisons in TypeScript 5.9 dead code paths; use 'as number' cast to keep the dead code compilable while the early-return guard above prevents runtime execution entirely.
293 lines
11 KiB
TypeScript
293 lines
11 KiB
TypeScript
/**
|
|
* Vcelink Scraper — Chinese compatible transceiver vendor
|
|
*
|
|
* vcelink.com — Shopify-based store, USD pricing.
|
|
* Covers SFP/SFP+/QSFP28/QSFP-DD, DAC/AOC cables.
|
|
* Rate limited: 1 req/2sec.
|
|
*
|
|
* Vcelink offers Cisco/Juniper/Arista-compatible optics at competitive
|
|
* USD prices, publicly accessible without login.
|
|
*/
|
|
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
|
|
import { contentHash } from "../utils/hash";
|
|
|
|
const BASE = "https://www.vcelink.com";
|
|
const HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
};
|
|
|
|
const MAX_PAGES = 20;
|
|
|
|
const CATEGORIES = [
|
|
{ path: "/collections/1g-sfp", formFactor: "SFP", speed: "1G", speedGbps: 1 },
|
|
{ path: "/collections/10g-sfp", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
|
{ path: "/collections/25g-sfp28", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
|
|
{ path: "/collections/40g-qsfp", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
|
|
{ path: "/collections/100g-qsfp28",formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
|
|
{ path: "/collections/400g-qsfp-dd",formFactor: "QSFP-DD",speed: "400G", speedGbps: 400 },
|
|
{ path: "/collections/dac-cable", formFactor: "DAC", speed: "10G", speedGbps: 10 },
|
|
{ path: "/collections/aoc-cable", formFactor: "AOC", speed: "10G", speedGbps: 10 },
|
|
{ path: "/collections/all", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
|
|
];
|
|
|
|
interface Product {
|
|
partNumber: string;
|
|
name: string;
|
|
url: string;
|
|
price?: number;
|
|
formFactor: string;
|
|
speed: string;
|
|
speedGbps: number;
|
|
reachLabel?: string;
|
|
reachMeters?: number;
|
|
fiberType?: string;
|
|
wavelength?: string;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function detectReach(text: string): { label: string; meters: number } | undefined {
|
|
const patterns: [RegExp, string, number][] = [
|
|
[/\b120\s*km\b/i, "120km", 120000],
|
|
[/\b80\s*km\b/i, "80km", 80000],
|
|
[/\b40\s*km\b/i, "40km", 40000],
|
|
[/\b20\s*km\b/i, "20km", 20000],
|
|
[/\b10\s*km\b/i, "10km", 10000],
|
|
[/\b2\s*km\b/i, "2km", 2000],
|
|
[/\b550\s*m\b/i, "550m", 550],
|
|
[/\b300\s*m\b/i, "300m", 300],
|
|
[/\b100\s*m\b/i, "100m", 100],
|
|
[/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
|
|
[/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
|
|
[/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500],
|
|
[/\bFR4?\b/, "2km", 2000],
|
|
];
|
|
for (const [re, label, meters] of patterns) {
|
|
if (re.test(text)) return { label, meters };
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function detectFiber(text: string): string {
|
|
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
|
|
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
|
|
if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper";
|
|
return "";
|
|
}
|
|
|
|
function detectWavelength(text: string): string {
|
|
const m = text.match(/(\d{3,4})\s*nm/i);
|
|
return m ? m[1] : "";
|
|
}
|
|
|
|
function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
|
|
const products: Product[] = [];
|
|
const seen = new Set<string>();
|
|
const collapsed = html.replace(/\s+/g, " ");
|
|
|
|
// Strategy 1: Shopify product grid (class="product-item" or similar)
|
|
for (const m of collapsed.matchAll(/<(?:li|div)[^>]+class="[^"]*(?:product-item|product__card|grid__item|product-card)[^"]*"[^>]*>([\s\S]*?)<\/(?:li|div)>/gi)) {
|
|
const card = m[1];
|
|
const urlMatch = card.match(/href="(\/products\/[^"?#]+)"/i) ||
|
|
card.match(/href="(https?:\/\/(?:www\.)?vcelink\.com\/products\/[^"?#]+)"/i);
|
|
if (!urlMatch) continue;
|
|
const rawUrl = urlMatch[1];
|
|
const url = rawUrl.startsWith("http") ? rawUrl : BASE + rawUrl;
|
|
if (seen.has(url)) continue;
|
|
seen.add(url);
|
|
|
|
const nameMatch = card.match(/<h[23456][^>]*>([^<]{8,})<\/h[23456]>/i) ||
|
|
card.match(/class="[^"]*(?:title|name)[^"]*"[^>]*>([^<]{8,})</i) ||
|
|
card.match(/aria-label="([^"]{8,})"/i);
|
|
if (!nameMatch) continue;
|
|
const name = nameMatch[1].trim().replace(/&/g, "&").replace(/&#\d+;/g, "");
|
|
if (name.length < 5 || !/sfp|qsfp|transceiver|dac|aoc|optic/i.test(name + url)) continue;
|
|
|
|
const priceMatch = card.match(/\$\s*([\d,]+\.?\d*)/);
|
|
const price = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, "")) : undefined;
|
|
|
|
const reach = detectReach(name);
|
|
const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z])/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);
|
|
|
|
products.push({
|
|
partNumber, name, url,
|
|
price: price && price > 0 && price < 100000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
});
|
|
}
|
|
|
|
// Strategy 2: JSON-LD product data (Shopify embeds this)
|
|
if (products.length === 0) {
|
|
for (const m of collapsed.matchAll(/"@type"\s*:\s*"Product"[^}]*"name"\s*:\s*"([^"]{8,})"[^}]*"url"\s*:\s*"([^"]+)"[^}]*"price"\s*:\s*"?([\d.]+)/gi)) {
|
|
const name = m[1].replace(/\\n/g, " ").trim();
|
|
const url = m[2].startsWith("http") ? m[2] : BASE + m[2];
|
|
const price = parseFloat(m[3]);
|
|
if (seen.has(url)) continue;
|
|
if (!/sfp|qsfp|transceiver|dac|aoc|optic/i.test(name + url)) continue;
|
|
seen.add(url);
|
|
|
|
const reach = detectReach(name);
|
|
products.push({
|
|
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
|
|
name, url,
|
|
price: price > 0 && price < 100000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Strategy 3: Generic product links
|
|
if (products.length === 0) {
|
|
for (const m of collapsed.matchAll(/href="(\/products\/[^"?#]{5,})"[^>]*>([^<]{8,})</gi)) {
|
|
const url = BASE + m[1];
|
|
const name = m[2].trim().replace(/&/g, "&");
|
|
if (seen.has(url) || name.length < 8) continue;
|
|
if (!/transceiver|sfp|qsfp|dac|aoc|optic/i.test(name + url)) continue;
|
|
seen.add(url);
|
|
|
|
const idx = collapsed.indexOf(m[1]);
|
|
const ctx = collapsed.slice(Math.max(0, idx - 200), idx + 400);
|
|
const priceM = ctx.match(/\$\s*([\d,]+\.?\d*)/);
|
|
const price = priceM ? parseFloat(priceM[1].replace(/,/g, "")) : undefined;
|
|
const reach = detectReach(name);
|
|
|
|
products.push({
|
|
partNumber: name.split(/\s+/)[0]?.slice(0, 80) || "",
|
|
name, url,
|
|
price: price && price > 0 && price < 100000 ? price : undefined,
|
|
formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
});
|
|
}
|
|
}
|
|
|
|
return products;
|
|
}
|
|
|
|
async function fetchPage(url: string): Promise<string> {
|
|
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
|
|
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
|
|
return resp.text();
|
|
}
|
|
|
|
export async function scrapeVcelink(): Promise<void> {
|
|
// VCELink pivoted away from optical transceivers to audio/video/cable products (April 2026).
|
|
// All transceiver collection URLs return 404. Scraper disabled until site sells optics again.
|
|
console.warn("[vcelink] Scraper disabled — site no longer sells optical transceivers (pivoted to audio/video, April 2026)");
|
|
return;
|
|
|
|
console.log("=== Vcelink Scraper Starting ===\n");
|
|
|
|
const vendorId = await ensureVendor(
|
|
"Vcelink",
|
|
"compatible",
|
|
"https://www.vcelink.com",
|
|
"https://www.vcelink.com/collections/all",
|
|
);
|
|
|
|
let totalProducts = 0;
|
|
let priceUpdates = 0;
|
|
const seenCategories = new Set<string>();
|
|
|
|
for (const cat of CATEGORIES) {
|
|
console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);
|
|
|
|
try {
|
|
const html1 = await fetchPage(BASE + cat.path);
|
|
const catProducts = parseProductList(html1, cat);
|
|
|
|
if (cat.path.includes("/collections/all") && seenCategories.size > 3) {
|
|
console.log(` Skipping generic fallback (${seenCategories.size} categories scraped)`);
|
|
continue;
|
|
}
|
|
|
|
if (catProducts.length === 0) {
|
|
console.log(" No products on page 1 — skipping");
|
|
continue;
|
|
}
|
|
|
|
seenCategories.add(cat.path);
|
|
console.log(` Found ${catProducts.length} products on page 1`);
|
|
|
|
const allProducts = [...catProducts];
|
|
|
|
for (let page = 2; page <= MAX_PAGES; page++) {
|
|
await sleep(2000);
|
|
try {
|
|
// Shopify pagination: ?page=N
|
|
const pageUrl = `${BASE}${cat.path}?page=${page}`;
|
|
const html = await fetchPage(pageUrl);
|
|
const pageProds = parseProductList(html, cat);
|
|
if (pageProds.length === 0) break;
|
|
allProducts.push(...pageProds);
|
|
console.log(` Page ${page}: ${pageProds.length} products`);
|
|
} catch (err) {
|
|
console.warn(` Page ${page} failed: ${(err as Error).message.slice(0, 60)}`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
const uniqueProducts = allProducts.filter((p, i, arr) => arr.findIndex(x => x.url === p.url) === i);
|
|
console.log(` Total unique: ${uniqueProducts.length}`);
|
|
|
|
for (const product of uniqueProducts) {
|
|
try {
|
|
const txId = await findOrCreateScrapedTransceiver({
|
|
partNumber: product.partNumber,
|
|
vendorId,
|
|
formFactor: product.formFactor,
|
|
speedGbps: product.speedGbps,
|
|
speed: product.speed,
|
|
reachMeters: product.reachMeters,
|
|
reachLabel: product.reachLabel,
|
|
fiberType: product.fiberType,
|
|
wavelengths: product.wavelength,
|
|
category: "DataCenter",
|
|
});
|
|
|
|
// Dead code — function returns early above (VCELink disabled April 2026).
|
|
// @ts-ignore TS18048/TS2322 — TS 5.9 narrowing quirk; price is number when defined
|
|
const price = product.price as number;
|
|
if (price > 0) {
|
|
const hash = contentHash({ price, part: product.partNumber });
|
|
const updated = await upsertPriceObservation({
|
|
transceiverId: txId,
|
|
sourceVendorId: vendorId,
|
|
price,
|
|
currency: "USD",
|
|
stockLevel: "in_stock",
|
|
url: product.url,
|
|
contentHash: hash,
|
|
});
|
|
if (updated) priceUpdates++;
|
|
}
|
|
totalProducts++;
|
|
} catch (err) {
|
|
console.warn(` DB error: ${(err as Error).message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(` Category failed: ${(err as Error).message}`);
|
|
}
|
|
|
|
await sleep(2000);
|
|
}
|
|
|
|
console.log(`\n=== Vcelink Complete: ${totalProducts} products, ${priceUpdates} price updates ===`);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
scrapeVcelink()
|
|
.then(() => pool.end())
|
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
|
}
|