Model:\s*([^<]+)<\/div>/i);
+ let partNumber = modelMatch ? modelMatch[1].trim() : "";
+
+ // XFP fallback: parse title after "|"
+ if (!partNumber && rawTitle.includes("|")) {
+ const afterPipe = rawTitle.split("|").pop()?.trim() ?? "";
+ if (afterPipe.length >= 3 && /^[A-Z0-9]/i.test(afterPipe)) {
+ partNumber = afterPipe.trim();
+ }
}
- // Need at least 6 columns, first cell must look like a part number (starts with A or contains letters+digits)
- if (cells.length < 6) continue;
- const partNumber = cells[0];
+ // Skip products without a usable part number
if (!partNumber || partNumber.length < 3) continue;
- // Skip header rows
- if (/^Part\s*No/i.test(partNumber) || /^Spec/i.test(partNumber)) continue;
- // Part numbers typically start with A (ASF, AXS, AXQ, AQS, etc.) or contain alphanumeric
- if (!/^[A-Z][A-Z0-9]/i.test(partNumber)) continue;
- const spec = cells[1] || "";
- const dataRate = cells[2] || "";
- const wavelength = cells.length >= 4 ? cells[3] : "";
- const fiberType = cells.length >= 5 ? cells[4] : "";
- const distance = cells.length >= 6 ? cells[5] : "";
- const txPower = cells.length >= 8 ? cells[7] : "";
+ // Extract price — first occurrence of US$X.XX in the block
+ const priceMatch = block.match(/
US\$([0-9]+(?:\.[0-9]{1,2})?)<\/span>/);
+ const price = priceMatch ? parseFloat(priceMatch[1]) : undefined;
- // Build descriptive name
- const name = `${partNumber} ${spec} ${dataRate}`.trim();
- const reach = parseDistance(distance) || detectReach(spec + " " + distance);
-
- // Determine fiber type from table cell or spec
- let fiber = "";
- if (/SMF|single/i.test(fiberType)) fiber = "SMF";
- else if (/MMF|multi/i.test(fiberType)) fiber = "MMF";
- else if (/CAT|RJ|copper/i.test(fiberType)) fiber = "Copper";
- else fiber = detectFiber(spec);
-
- // Extract wavelength
- const wl = wavelength.replace(/[^0-9]/g, "");
+ // Detect reach and fiber type from product title
+ const reach = detectReach(rawTitle);
+ const fiber = detectFiber(rawTitle);
products.push({
partNumber,
- name,
- url: `${BASE}${cat.path}#${partNumber}`,
+ name: rawTitle,
+ url: productUrl,
+ price,
+ currency: price !== undefined ? "USD" : undefined,
formFactor: cat.formFactor,
speed: cat.speed,
speedGbps: cat.speedGbps,
reachLabel: reach?.label,
reachMeters: reach?.meters,
- fiberType: fiber,
+ fiberType: fiber || undefined,
});
}
- // Dedupe by part number
- const seen = new Set();
- return products.filter((p) => {
- if (seen.has(p.partNumber)) return false;
- seen.add(p.partNumber);
- return true;
- });
+ return products;
}
async function fetchPage(url: string): Promise {
@@ -165,21 +167,72 @@ async function fetchPage(url: string): Promise {
return resp.text();
}
-export async function scrape10Gtek(): Promise {
- console.log("=== 10Gtek Scraper Starting ===\n");
+/**
+ * Fetch all pages of a category, stopping when no new part numbers appear.
+ * Magento loops back to page 1 for out-of-range page numbers, so we detect
+ * this by checking for duplicate part numbers from previous pages.
+ */
+async function fetchAllPages(cat: typeof CATEGORIES[number]): Promise {
+ const MAX_PAGES = 30;
+ const allProducts: Product[] = [];
+ const seenPartNumbers = new Set();
- const vendorId = await ensureVendor("10Gtek", "compatible", "https://www.10gtek.com", "https://www.10gtek.com");
+ for (let page = 1; page <= MAX_PAGES; page++) {
+ const url = page === 1 ? `${BASE}/${cat.slug}` : `${BASE}/${cat.slug}?p=${page}`;
+
+ let html: string;
+ try {
+ html = await fetchPage(url);
+ } catch (err) {
+ console.error(` Fetch error on p${page}: ${(err as Error).message}`);
+ break;
+ }
+
+ const pageProducts = parseProductsFromPage(html, cat);
+
+ // Stop if no products found (truly empty page)
+ if (pageProducts.length === 0) break;
+
+ // Count new (unseen) products — detects Magento catalog wrap-around
+ let newCount = 0;
+ for (const p of pageProducts) {
+ if (!seenPartNumbers.has(p.partNumber)) {
+ seenPartNumbers.add(p.partNumber);
+ allProducts.push(p);
+ newCount++;
+ }
+ }
+
+ console.log(` p${page}: ${pageProducts.length} parsed, ${newCount} new`);
+
+ // All products on this page already seen → we've looped back to start
+ if (newCount === 0) break;
+
+ await sleep(2000);
+ }
+
+ return allProducts;
+}
+
+export async function scrape10Gtek(): Promise {
+ console.log("=== 10Gtek Scraper Starting (via sfpcables.com) ===\n");
+
+ const vendorId = await ensureVendor(
+ "10Gtek",
+ "compatible",
+ "https://www.10gtek.com",
+ "https://www.sfpcables.com"
+ );
let totalProducts = 0;
let priceUpdates = 0;
for (const cat of CATEGORIES) {
- console.log(`\n--- ${cat.formFactor} (${cat.speed}) ---`);
+ console.log(`\n--- ${cat.formFactor} (${cat.speed}) [/${cat.slug}] ---`);
try {
- const html = await fetchPage(BASE + cat.path);
- const catProducts = parseProductList(html, cat);
- console.log(` Found ${catProducts.length} products`);
+ const catProducts = await fetchAllPages(cat);
+ console.log(` Fetched: ${catProducts.length} unique products`);
for (const product of catProducts) {
try {
@@ -201,7 +254,7 @@ export async function scrape10Gtek(): Promise {
transceiverId: txId,
sourceVendorId: vendorId,
price: product.price,
- currency: product.currency || "USD",
+ currency: product.currency ?? "USD",
stockLevel: "in_stock",
url: product.url,
contentHash: hash,
@@ -211,14 +264,12 @@ export async function scrape10Gtek(): Promise {
totalProducts++;
} catch (err) {
- console.warn(` Error: ${(err as Error).message.slice(0, 80)}`);
+ console.warn(` Error [${product.partNumber}]: ${(err as Error).message.slice(0, 80)}`);
}
}
} catch (err) {
console.error(` Category failed: ${(err as Error).message}`);
}
-
- await sleep(2000);
}
console.log(`\n=== 10Gtek Complete: ${totalProducts} products, ${priceUpdates} prices ===`);
@@ -227,5 +278,9 @@ export async function scrape10Gtek(): Promise {
if (require.main === module) {
scrape10Gtek()
.then(() => pool.end())
- .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
+ .catch((err) => {
+ console.error("Fatal:", err);
+ pool.end();
+ process.exit(1);
+ });
}