ba24d33858
- 16 commercial scrapers: replace TIP-Bot/1.0 with Chrome/120 UA (GBICS confirmed returning 0 bytes for bot UA, Chrome UA returns 200KB) - gbics.ts: fix User-Agent (was returning empty HTML, now returns products) - optictransceiver.ts: disable — domain repurposed as plant shop (2026-04-06) Alocasia Regal Shield is not a transceiver.
308 lines
11 KiB
TypeScript
308 lines
11 KiB
TypeScript
/**
|
|
* Skylane Optics Scraper — Belgian compatible transceiver vendor
|
|
*
|
|
* www.skylaneoptics.com — product catalog with form factor listings.
|
|
* No direct webshop prices; product catalog only.
|
|
*
|
|
* Rate limited: 1 req/2sec.
|
|
*/
|
|
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
|
|
import { contentHash } from "../utils/hash";
|
|
import * as cheerio from "cheerio";
|
|
|
|
const BASE = "https://www.skylaneoptics.com";
|
|
const CATALOG_URLS = [
|
|
"/en/products/",
|
|
"/en/",
|
|
"/en/products/sfp/",
|
|
"/en/products/sfp28/",
|
|
"/en/products/qsfp28/",
|
|
"/en/products/qsfp-dd/",
|
|
];
|
|
const MAX_PAGES = 10;
|
|
const HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
Accept: "text/html,application/xhtml+xml",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
};
|
|
|
|
interface Product {
|
|
partNumber: string;
|
|
name: string;
|
|
url: string;
|
|
price?: number;
|
|
formFactor: string;
|
|
speed: string;
|
|
speedGbps: number;
|
|
reachLabel?: string;
|
|
reachMeters?: number;
|
|
fiberType?: string;
|
|
wavelength?: string;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } {
|
|
const lower = text.toLowerCase();
|
|
if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 };
|
|
if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 };
|
|
if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 };
|
|
if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 };
|
|
if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 };
|
|
if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 };
|
|
if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
|
|
if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 };
|
|
if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
|
|
if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
|
|
return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
|
|
}
|
|
|
|
function detectReach(text: string): { label: string; meters: number } | undefined {
|
|
const patterns: [RegExp, string, number][] = [
|
|
[/\b80\s*km\b/i, "80km", 80000],
|
|
[/\b40\s*km\b/i, "40km", 40000],
|
|
[/\b20\s*km\b/i, "20km", 20000],
|
|
[/\b10\s*km\b/i, "10km", 10000],
|
|
[/\b2\s*km\b/i, "2km", 2000],
|
|
[/\b550\s*m\b/i, "550m", 550],
|
|
[/\b300\s*m\b/i, "300m", 300],
|
|
[/\b100\s*m\b/i, "100m", 100],
|
|
[/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
|
|
[/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
|
|
[/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000],
|
|
];
|
|
for (const [regex, label, meters] of patterns) {
|
|
if (regex.test(text)) return { label, meters };
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function detectFiber(text: string): string {
|
|
if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
|
|
if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
|
|
if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper";
|
|
return "SMF";
|
|
}
|
|
|
|
function detectWavelength(text: string): string {
|
|
const match = text.match(/(\d{3,4})\s*nm/i);
|
|
return match ? match[1] : "";
|
|
}
|
|
|
|
function parseProductList(html: string, sourceUrl: string): Product[] {
|
|
const $ = cheerio.load(html);
|
|
const products: Product[] = [];
|
|
|
|
const cardSelectors = [
|
|
".product-item", ".product", ".item", "li.product",
|
|
".product-card", "article", ".product-list-item", ".grid-item",
|
|
];
|
|
|
|
for (const sel of cardSelectors) {
|
|
if ($(sel).length >= 2) {
|
|
$(sel).each((_i, el) => {
|
|
const text = $(el).text().trim();
|
|
if (!/sfp|qsfp|xfp|transceiver|optic/i.test(text)) return;
|
|
|
|
const nameEl = $(el).find("h2, h3, h4, .name, .product-name, .title, a").first();
|
|
const name = nameEl.text().trim() || text.slice(0, 100);
|
|
if (!name || name.length < 5) return;
|
|
|
|
const linkEl = $(el).find("a[href]").first();
|
|
const href = linkEl.attr("href") || sourceUrl;
|
|
const url = href.startsWith("http") ? href : BASE + href;
|
|
|
|
// Skylane part numbers: e.g. SO-SFP+-LRM-CI, SFP-10G-SR-SO
|
|
const partNumMatch = name.match(/\b(SO[-_][A-Z0-9-]+)\b/i) ||
|
|
name.match(/\b([A-Z]{2,}[-][A-Z0-9]+[-][A-Z0-9]+)\b/) ||
|
|
text.match(/Part\s*[#:]?\s*([A-Z0-9-]{5,})/i);
|
|
const partNumber = partNumMatch?.[1] ||
|
|
name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] ||
|
|
name.replace(/\s+/g, "-").slice(0, 60);
|
|
|
|
// Check for price
|
|
const priceText = $(el).find(".price, .product-price, .amount").text();
|
|
const priceMatch = priceText.match(/\$\s*([\d,]+\.?\d{0,2})/) ||
|
|
priceText.match(/([\d.,]+)\s*€/);
|
|
let price: number | undefined;
|
|
if (priceMatch) {
|
|
const raw = priceMatch[1].replace(/[.,](\d{2})$/, ".$1").replace(/[.,]/g, "");
|
|
const parsed = parseFloat(raw);
|
|
if (parsed > 0 && parsed < 50000) price = parsed;
|
|
}
|
|
|
|
const ff = detectFormFactor(name + " " + text);
|
|
const reach = detectReach(name + " " + text);
|
|
|
|
products.push({
|
|
partNumber,
|
|
name,
|
|
url,
|
|
price,
|
|
...ff,
|
|
reachLabel: reach?.label,
|
|
reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name + " " + text),
|
|
wavelength: detectWavelength(name + " " + text),
|
|
});
|
|
});
|
|
if (products.length > 0) break;
|
|
}
|
|
}
|
|
|
|
// Fallback: transceiver-relevant anchors
|
|
if (products.length === 0) {
|
|
$("a[href]").each((_i, el) => {
|
|
const name = $(el).text().trim();
|
|
const href = $(el).attr("href") || "";
|
|
if (name.length < 8 || name.length > 200 || !/sfp|qsfp|transceiver/i.test(name)) return;
|
|
const url = href.startsWith("http") ? href : BASE + href;
|
|
const ff = detectFormFactor(name);
|
|
const reach = detectReach(name);
|
|
products.push({
|
|
partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60),
|
|
name, url, ...ff,
|
|
reachLabel: reach?.label, reachMeters: reach?.meters,
|
|
fiberType: detectFiber(name), wavelength: detectWavelength(name),
|
|
});
|
|
});
|
|
}
|
|
|
|
const seen = new Set<string>();
|
|
return products.filter((p) => {
|
|
if (!p.url || seen.has(p.url)) return false;
|
|
seen.add(p.url);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
// Discover sub-category links from a catalog page
|
|
function extractCategoryLinks(html: string): string[] {
|
|
const $ = cheerio.load(html);
|
|
const links: string[] = [];
|
|
$("a[href]").each((_i, el) => {
|
|
const href = $(el).attr("href") || "";
|
|
if (/\/products?\//i.test(href) && /sfp|qsfp|transceiver|optic/i.test(href)) {
|
|
links.push(href.startsWith("http") ? href : BASE + href);
|
|
}
|
|
});
|
|
return [...new Set(links)];
|
|
}
|
|
|
|
async function fetchPage(url: string): Promise<string> {
|
|
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
|
|
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
|
|
return resp.text();
|
|
}
|
|
|
|
export async function scrapeSkylane(): Promise<void> {
|
|
console.log("=== Skylane Optics Scraper Starting ===\n");
|
|
|
|
const vendorId = await ensureVendor(
|
|
"Skylane Optics",
|
|
"compatible",
|
|
"https://www.skylaneoptics.com",
|
|
BASE + CATALOG_URLS[0],
|
|
);
|
|
|
|
const allProducts: Product[] = [];
|
|
const seenUrls = new Set<string>();
|
|
const scrapedCatalogUrls = new Set<string>();
|
|
|
|
// Start with known catalog URLs and discover sub-categories
|
|
const urlQueue = [...CATALOG_URLS.map((p) => BASE + p)];
|
|
|
|
for (const catalogUrl of urlQueue) {
|
|
if (scrapedCatalogUrls.has(catalogUrl)) continue;
|
|
scrapedCatalogUrls.add(catalogUrl);
|
|
|
|
console.log(` Fetching: ${catalogUrl}`);
|
|
try {
|
|
const html = await fetchPage(catalogUrl);
|
|
const pageProducts = parseProductList(html, catalogUrl);
|
|
for (const p of pageProducts) {
|
|
if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); }
|
|
}
|
|
console.log(` Found ${pageProducts.length} products`);
|
|
|
|
// Discover sub-category links (limit to first catalog page)
|
|
if (scrapedCatalogUrls.size <= CATALOG_URLS.length) {
|
|
const newLinks = extractCategoryLinks(html).slice(0, 10);
|
|
for (const link of newLinks) {
|
|
if (!scrapedCatalogUrls.has(link)) urlQueue.push(link);
|
|
}
|
|
}
|
|
|
|
// Try pagination for this URL
|
|
for (let page = 2; page <= MAX_PAGES; page++) {
|
|
const pageUrl = `${catalogUrl}?page=${page}`;
|
|
try {
|
|
const pageHtml = await fetchPage(pageUrl);
|
|
const paginated = parseProductList(pageHtml, pageUrl);
|
|
if (paginated.length === 0) break;
|
|
for (const p of paginated) {
|
|
if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); }
|
|
}
|
|
console.log(` Page ${page}: ${paginated.length} products`);
|
|
await sleep(2000);
|
|
} catch {
|
|
break;
|
|
}
|
|
}
|
|
|
|
await sleep(2000);
|
|
} catch (err) {
|
|
console.warn(` Failed: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\nTotal unique products: ${allProducts.length}`);
|
|
|
|
let totalProducts = 0;
|
|
let priceUpdates = 0;
|
|
|
|
for (const product of allProducts) {
|
|
try {
|
|
const txId = await findOrCreateScrapedTransceiver({
|
|
partNumber: product.partNumber,
|
|
vendorId,
|
|
formFactor: product.formFactor,
|
|
speedGbps: product.speedGbps,
|
|
speed: product.speed,
|
|
reachMeters: product.reachMeters,
|
|
reachLabel: product.reachLabel,
|
|
fiberType: product.fiberType,
|
|
wavelengths: product.wavelength,
|
|
category: "DataCenter",
|
|
});
|
|
|
|
if (product.price && product.price > 0) {
|
|
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
|
|
const updated = await upsertPriceObservation({
|
|
transceiverId: txId,
|
|
sourceVendorId: vendorId,
|
|
price: product.price,
|
|
currency: "USD",
|
|
stockLevel: "in_stock",
|
|
url: product.url,
|
|
contentHash: hash,
|
|
});
|
|
if (updated) priceUpdates++;
|
|
}
|
|
totalProducts++;
|
|
} catch (err) {
|
|
console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\n=== Skylane Optics Complete: ${totalProducts} products, ${priceUpdates} prices ===`);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
scrapeSkylane()
|
|
.then(() => pool.end())
|
|
.catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
|
|
}
|