f146ac873e
Add Comms-Express, Router-Switch.com, Multimode Inc, OpticTransceiver.com, and Wiitek scrapers covering CFP2-DCO, CFP4, OSFP224, QSFP112, CXP, GBIC, XENPAK, CSFP, SFP-DD, SFP56, QSFP56 and other previously-uncovered form factors. Each scheduled every 8h. Worker registrations added to scheduler. Also export db alias in utils/db.ts to fix eBay enricher + community scrapers crashing with 'Cannot read properties of undefined (reading query)'.
91 lines
4.0 KiB
TypeScript
91 lines
4.0 KiB
TypeScript
/**
|
|
* Wiitek Scraper — wiitek.com
|
|
*
|
|
* Chinese manufacturer/reseller, direct factory prices.
|
|
* Covers: CSFP, SFP-DD, CFP, CFP2, OSFP224, QSFP112 and all standard forms.
|
|
* Good for price benchmarking.
|
|
*
|
|
* Schedule: every 8h
|
|
*/
|
|
import * as cheerio from "cheerio";
|
|
import { ensureVendor, upsertPriceObservation, findOrCreateScrapedTransceiver } from "../utils/db";
|
|
import { contentHash, parsePrice } from "../utils/hash";
|
|
import { logger } from "../utils/logger";
|
|
|
|
const BASE = "https://www.wiitek.com";
|
|
|
|
const CATEGORIES: Array<{ path: string; form_factor: string }> = [
|
|
{ path: "/SFP-Transceiver/", form_factor: "SFP" },
|
|
{ path: "/SFP-Plus-Transceiver/", form_factor: "SFP+" },
|
|
{ path: "/SFP28-Transceiver/", form_factor: "SFP28" },
|
|
{ path: "/SFP56-Transceiver/", form_factor: "SFP56" },
|
|
{ path: "/SFP-DD-Transceiver/", form_factor: "SFP-DD" },
|
|
{ path: "/CSFP-Transceiver/", form_factor: "CSFP" },
|
|
{ path: "/QSFP-Transceiver/", form_factor: "QSFP+" },
|
|
{ path: "/QSFP28-Transceiver/", form_factor: "QSFP28" },
|
|
{ path: "/QSFP56-Transceiver/", form_factor: "QSFP56" },
|
|
{ path: "/QSFP-DD-Transceiver/", form_factor: "QSFP-DD" },
|
|
{ path: "/QSFP-DD800-Transceiver/", form_factor: "QSFP-DD800" },
|
|
{ path: "/QSFP112-Transceiver/", form_factor: "QSFP112" },
|
|
{ path: "/OSFP-Transceiver/", form_factor: "OSFP" },
|
|
{ path: "/OSFP112-Transceiver/", form_factor: "OSFP112" },
|
|
{ path: "/OSFP224-Transceiver/", form_factor: "OSFP224" },
|
|
{ path: "/CFP-Transceiver/", form_factor: "CFP" },
|
|
{ path: "/CFP2-Transceiver/", form_factor: "CFP2" },
|
|
{ path: "/XFP-Transceiver/", form_factor: "XFP" },
|
|
{ path: "/GBIC-Transceiver/", form_factor: "GBIC" },
|
|
{ path: "/XENPAK-Transceiver/", form_factor: "XENPAK" },
|
|
{ path: "/CXP-Transceiver/", form_factor: "CXP" },
|
|
];
|
|
|
|
export async function scrapeWiitek(): Promise<void> {
|
|
logger.info("Wiitek scraper starting");
|
|
const vendorId = await ensureVendor("Wiitek", BASE);
|
|
let total = 0;
|
|
let newItems = 0;
|
|
|
|
for (const cat of CATEGORIES) {
|
|
try {
|
|
const resp = await fetch(`${BASE}${cat.path}`, {
|
|
headers: { "User-Agent": "Mozilla/5.0 (compatible; TIPBot/1.0)" },
|
|
signal: AbortSignal.timeout(20_000),
|
|
});
|
|
if (!resp.ok) continue;
|
|
const $ = cheerio.load(await resp.text());
|
|
|
|
const items = $(".product-item, .goods-item, .pro-item, [class*=product]");
|
|
for (let i = 0; i < items.length; i++) {
|
|
const $el = $(items[i]);
|
|
const name = $el.find("h2,h3,h4,.title,.name").first().text().trim();
|
|
const priceText = $el.find(".price,.cost,[class*=price]").first().text().trim();
|
|
const href = $el.find("a[href]").first().attr("href") || "";
|
|
if (!name || !priceText) continue;
|
|
|
|
const partMatch = name.match(/([A-Z0-9]{2,8}[-\/][A-Z0-9][A-Z0-9\-\.\/]{3,35})/);
|
|
const partNumber = (partMatch ? partMatch[1] : name.substring(0, 50)).toUpperCase();
|
|
const { price, currency } = parsePrice(priceText);
|
|
if (price <= 0) continue;
|
|
|
|
try {
|
|
const t = await findOrCreateScrapedTransceiver({
|
|
partNumber, vendorId, formFactor: cat.form_factor, name,
|
|
url: href.startsWith("http") ? href : `${BASE}${href}`,
|
|
});
|
|
const isNew = await upsertPriceObservation({
|
|
transceiverId: t.id, sourceVendorId: vendorId,
|
|
price, currency: currency || "USD",
|
|
stockLevel: "unknown",
|
|
url: href.startsWith("http") ? href : `${BASE}${href}`,
|
|
contentHash: contentHash(`${partNumber}:${price}:${currency}`),
|
|
});
|
|
if (isNew) newItems++;
|
|
total++;
|
|
} catch { /* skip */ }
|
|
}
|
|
} catch (e) {
|
|
logger.warn(`Wiitek ${cat.form_factor} failed`, { err: e });
|
|
}
|
|
}
|
|
logger.info(`Wiitek done — ${total} total, ${newItems} new`);
|
|
}
|