45c48755e4
- Register scrape:pricing:naddod (48 */2), qsfptek (52 */2), addon (55 */2) in pg-boss - Add boss.work() handlers for all three (fetch-based, run on Erik) - Fix findOrCreateScrapedTransceiver callers: remove invalid `name`/`url` params, fix `t.id` → `t` (function already returns string ID) - Fix ebay-enricher: remove invalid `extractType` option, use extraction.standard_name instead of non-existent `.description`, fix cheerio type incompatibility - Fix community-issues: description → summary, publishedDate → published_at - Startup zombie cleanup already deployed (index.ts) — no changes needed - ProLabs rewritten to fetch-based catalog scraper (no Playwright, bypasses WAF)
82 lines
3.5 KiB
TypeScript
82 lines
3.5 KiB
TypeScript
/**
|
|
* MultiMode Inc Scraper — multimode-inc.com
|
|
*
|
|
* Specialist for high-speed coherent transceivers:
|
|
* CFP, CFP2, CFP2-DCO, CFP4, QSFP112, OSFP112, OSFP224
|
|
* Plus broad 400G/800G coverage.
|
|
*
|
|
* Schedule: every 8h
|
|
*/
|
|
import * as cheerio from "cheerio";
|
|
import { ensureVendor, upsertPriceObservation, findOrCreateScrapedTransceiver } from "../utils/db";
|
|
import { contentHash, parsePrice, parseStockLevel } from "../utils/hash";
|
|
import { logger } from "../utils/logger";
|
|
|
|
const BASE = "https://www.multimode.com";
|
|
|
|
const CATEGORIES: Array<{ path: string; form_factor: string }> = [
|
|
{ path: "/cfp-transceivers/", form_factor: "CFP" },
|
|
{ path: "/cfp2-transceivers/", form_factor: "CFP2" },
|
|
{ path: "/cfp2-dco/", form_factor: "CFP2-DCO" },
|
|
{ path: "/cfp4-transceivers/", form_factor: "CFP4" },
|
|
{ path: "/osfp-transceivers/", form_factor: "OSFP" },
|
|
{ path: "/osfp112/", form_factor: "OSFP112" },
|
|
{ path: "/osfp224/", form_factor: "OSFP224" },
|
|
{ path: "/qsfp112/", form_factor: "QSFP112" },
|
|
{ path: "/qsfp-dd-800/", form_factor: "QSFP-DD800" },
|
|
{ path: "/qsfp-dd/", form_factor: "QSFP-DD" },
|
|
{ path: "/sfp-dd/", form_factor: "SFP-DD" },
|
|
{ path: "/qsfp28-transceivers/", form_factor: "QSFP28" },
|
|
];
|
|
|
|
export async function scrapeMultimodeInc(): Promise<void> {
|
|
logger.info("Multimode Inc scraper starting");
|
|
const vendorId = await ensureVendor("Multimode Inc", BASE);
|
|
let total = 0;
|
|
let newItems = 0;
|
|
|
|
for (const cat of CATEGORIES) {
|
|
try {
|
|
const resp = await fetch(`${BASE}${cat.path}`, {
|
|
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" },
|
|
signal: AbortSignal.timeout(25_000),
|
|
});
|
|
if (!resp.ok) continue;
|
|
const $ = cheerio.load(await resp.text());
|
|
|
|
const rows = $(".product, .woocommerce-loop-product__link, article.product");
|
|
for (let i = 0; i < rows.length; i++) {
|
|
const $el = $(rows[i]);
|
|
const name = $el.find(".woocommerce-loop-product__title, h2, h3").first().text().trim();
|
|
const priceText = $el.find(".price, .woocommerce-Price-amount").first().text().trim();
|
|
const href = $el.find("a").first().attr("href") || $el.closest("a").attr("href") || "";
|
|
if (!name) continue;
|
|
|
|
const partMatch = name.match(/([A-Z0-9]{2,8}-[A-Z0-9][A-Z0-9\-\/\.]{3,30})/);
|
|
const partNumber = partMatch ? partMatch[1].toUpperCase() : name.substring(0, 50);
|
|
const { price, currency } = parsePrice(priceText);
|
|
if (price <= 0) continue;
|
|
|
|
try {
|
|
const transceiverId = await findOrCreateScrapedTransceiver({
|
|
partNumber, vendorId, formFactor: cat.form_factor,
|
|
});
|
|
const isNew = await upsertPriceObservation({
|
|
transceiverId, sourceVendorId: vendorId,
|
|
price, currency: currency || "USD",
|
|
stockLevel: "unknown",
|
|
url: href.startsWith("http") ? href : `${BASE}${href}`,
|
|
contentHash: contentHash(`${partNumber}:${price}:${currency}`),
|
|
});
|
|
if (isNew) newItems++;
|
|
total++;
|
|
} catch { /* skip */ }
|
|
}
|
|
if (rows.length > 0) logger.info(`Multimode Inc ${cat.form_factor}: ${rows.length} products`);
|
|
} catch (e) {
|
|
logger.warn(`Multimode Inc ${cat.form_factor} failed`, { err: e });
|
|
}
|
|
}
|
|
logger.info(`Multimode Inc done — ${total} total, ${newItems} new`);
|
|
}
|