diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index 7e88060..68314aa 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -1,38 +1,22 @@ /** - * pg-boss Job Scheduler + * pg-boss Job Scheduler — 24/7 Continuous Scraping * - * NIGHTLY WINDOW 00:00–08:00 — ALL scrapers run every night - * Staggered to avoid parallel overload and respect vendor rate limits. + * ARCHITECTURE: + * - Erik (VPS, .82) : Playwright-heavy scrapers (FS.com, 10Gtek, ATGBICS, ProLabs) + * + all compatibility + eBay + compute + NAS sync + * - Raspberry Pi Fleet : Lightweight fetch/cheerio scrapers run continuously all day + * (BlueOptics, Fiber24, T&S Com, Fluxlight, GBICs, Optcore, + * Champion ONE, SFPCables, SmartOptics, HUBER+SUHNER, etc.) * - * 00:00 eBay transceiver pricing (new/refurb condition prices) - * 00:30 eBay switch enrichment (features, descriptions, images) - * 01:00 FS.com pricing (Playwright, slowest scraper) - * 01:30 Champion ONE pricing - * 01:50 Fluxlight pricing - * 02:10 GBICs pricing - * 02:30 Optcore pricing - * 02:50 10Gtek pricing (Playwright) - * 03:20 ATGBICS pricing (Shopify/Playwright, GBP) - * 03:50 ProLabs pricing (Playwright/CloudFront) - * 04:20 SFPCables pricing - * 04:40 Market intelligence (OFC/ECOC/IEEE/Farnell/TED) - * 05:00 Flexoptix catalog (fast fetch — primary source) - * 05:20 Flexoptix vendor list - * 05:35 Flexoptix supported vendors (full vendor seed) - * 05:50 Cisco compatibility matrices - * 06:05 Juniper HCT compatibility - * 06:20 SONiC HCL compatibility - * 06:35 Ufispace switch data - * 06:45 Edgecore switch data - * 06:55 Switch assets enrichment (images, specs) - * 07:00 Community issues (Reddit/forums/vendor KB) - * 07:10 Datasheet + manual link discovery - * 07:20 News aggregation (trade press) - * 07:30 FAQ / knowledge base - * 07:35 Docs check (full doc scrape) - * 07:42 ABC classification recompute - * 07:50 Reorder signals recompute - * 07:55 NAS sync (export JSON data + weekly pg_dump to Fearghas) + * SCHEDULE PHILOSOPHY: + * - Playwright scrapers: every 8h (resource-heavy, VPS only) + * - Fetch/Cheerio scrapers: every 4h (lightweight, Pi-friendly) + * - Catalog scrapers (Flexoptix): every 2h (fast GraphQL, primary price source) + * - Compatibility matrices: every 12h (rarely change) + * - eBay enrichment: every 6h + * - Intelligence/community: every 6h + * - Compute jobs: after each pricing wave + * - NAS sync: nightly at 07:55 */ import PgBoss from "pg-boss"; import { config } from "dotenv"; @@ -80,42 +64,53 @@ export async function createScheduler(): Promise { export async function registerSchedules(boss: PgBoss): Promise { const queues = [ - // Pricing scrapers + // ── Playwright scrapers (Erik, every 8h) ─────────────────────────── "scrape:pricing:fs", - "scrape:pricing:optcore", "scrape:pricing:10gtek", "scrape:pricing:atgbics", "scrape:pricing:prolabs", - "scrape:pricing:champion-one", + // ── Fetch/Cheerio scrapers (Pi-friendly, every 4h) ───────────────── "scrape:pricing:fluxlight", "scrape:pricing:gbics", + "scrape:pricing:optcore", + "scrape:pricing:champion-one", "scrape:pricing:sfpcables", - // Catalog / vendor scrapers + "scrape:pricing:blueoptics", + "scrape:pricing:fiber24", + "scrape:pricing:tscom", + "scrape:pricing:skylane", + "scrape:pricing:ascentoptics", + "scrape:pricing:gaotek", + // ── Catalog scrapers (every 2h) ──────────────────────────────────── "scrape:pricing:flexoptix", + // ── Manufacturer catalogs (every 8h, no prices) ──────────────────── + "scrape:catalog:smartoptics", + "scrape:catalog:hubersuhner", + // ── Vendor lists ─────────────────────────────────────────────────── "scrape:vendors:flexoptix", "scrape:vendors:flexoptix-supported", - // Compatibility scrapers + // ── Compatibility (every 12h) ────────────────────────────────────── "scrape:compat:cisco", "scrape:compat:juniper", "scrape:compat:sonic", "scrape:compat:ufispace", "scrape:compat:edgecore", - // Switch enrichment + // ── Switch enrichment (every 12h) ───────────────────────────────── "scrape:assets:switches", - // eBay enrichment + // ── eBay enrichment (every 6h) ──────────────────────────────────── "enrich:ebay-transceivers", "enrich:ebay-switches", - // Intelligence & community + // ── Intelligence & community (every 6h) ─────────────────────────── "scrape:market-intel", "scrape:community-issues", "scrape:datasheet-links", "scrape:news", "scrape:faq", "scrape:docs", - // Compute jobs + // ── Compute (every 4h, after pricing waves) ─────────────────────── "compute:abc", "compute:reorder-signals", - // Sync + // ── Sync ────────────────────────────────────────────────────────── "sync:nas", ]; @@ -123,156 +118,127 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.createQueue(q).catch(() => { /* already exists */ }); } - // ════════════════════════════════════════════════════════════════ - // NIGHTLY WINDOW 00:00–08:00 — every night, all scrapers - // ════════════════════════════════════════════════════════════════ + // ══════════════════════════════════════════════════════════════════════ + // PLAYWRIGHT SCRAPERS — every 8h (resource-heavy, runs on Erik VPS) + // ══════════════════════════════════════════════════════════════════════ - // 00:00 — eBay transceiver pricing (new/refurb, all 5000+ products) - await boss.schedule("enrich:ebay-transceivers", "0 0 * * *", {}, { - retryLimit: 2, expireInSeconds: 7200, - }); + // FS.com: 01:00, 09:00, 17:00 + await boss.schedule("scrape:pricing:fs", "0 1,9,17 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 }); - // 00:30 — eBay switch enrichment (features, images, refurb prices) - await boss.schedule("enrich:ebay-switches", "30 0 * * *", {}, { - retryLimit: 2, expireInSeconds: 7200, - }); + // 10Gtek: 01:20, 09:20, 17:20 + await boss.schedule("scrape:pricing:10gtek", "20 1,9,17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 01:00 — FS.com pricing (Playwright JS-rendered, slowest scraper) - await boss.schedule("scrape:pricing:fs", "0 1 * * *", {}, { - retryLimit: 3, expireInSeconds: 5400, - }); + // ATGBICS: 01:50, 09:50, 17:50 + await boss.schedule("scrape:pricing:atgbics", "50 1,9,17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 01:30 — Champion ONE pricing - await boss.schedule("scrape:pricing:champion-one", "30 1 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // ProLabs: 02:20, 10:20, 18:20 + await boss.schedule("scrape:pricing:prolabs", "20 2,10,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 01:50 — Fluxlight pricing - await boss.schedule("scrape:pricing:fluxlight", "50 1 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // ══════════════════════════════════════════════════════════════════════ + // FETCH/CHEERIO SCRAPERS — every 4h (lightweight, Pi-friendly) + // ══════════════════════════════════════════════════════════════════════ - // 02:10 — GBICs pricing - await boss.schedule("scrape:pricing:gbics", "10 2 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // Fluxlight: 00:05, 04:05, 08:05, 12:05, 16:05, 20:05 + await boss.schedule("scrape:pricing:fluxlight", "5 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 02:30 — Optcore pricing - await boss.schedule("scrape:pricing:optcore", "30 2 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // GBICs: 00:15, 04:15, 08:15, 12:15, 16:15, 20:15 + await boss.schedule("scrape:pricing:gbics", "15 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 02:50 — 10Gtek pricing (Playwright) - await boss.schedule("scrape:pricing:10gtek", "50 2 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // Optcore: 00:30, 04:30, 08:30, 12:30, 16:30, 20:30 + await boss.schedule("scrape:pricing:optcore", "30 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 03:20 — ATGBICS pricing (Shopify/Playwright, GBP) - await boss.schedule("scrape:pricing:atgbics", "20 3 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // Champion ONE: 00:45, 04:45, 08:45, 12:45, 16:45, 20:45 + await boss.schedule("scrape:pricing:champion-one", "45 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 03:50 — ProLabs pricing (Playwright/CloudFront) - await boss.schedule("scrape:pricing:prolabs", "50 3 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // SFPCables: 01:00, 05:00, 09:00, 13:00, 17:00, 21:00 + await boss.schedule("scrape:pricing:sfpcables", "0 1,5,9,13,17,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 04:20 — SFPCables pricing - await boss.schedule("scrape:pricing:sfpcables", "20 4 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // BlueOptics: 01:15, 05:15, 09:15, 13:15, 17:15, 21:15 + await boss.schedule("scrape:pricing:blueoptics", "15 1,5,9,13,17,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 04:40 — Market intelligence (OFC/ECOC, IEEE 802.3, EU TED, Farnell/Mouser lead times) - await boss.schedule("scrape:market-intel", "40 4 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // ShopFiber24: 01:30, 05:30, 09:30, 13:30, 17:30, 21:30 + await boss.schedule("scrape:pricing:fiber24", "30 1,5,9,13,17,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 05:00 — Flexoptix catalog (fast fetch — primary source, highest priority) - await boss.schedule("scrape:pricing:flexoptix", "0 5 * * *", {}, { - retryLimit: 3, expireInSeconds: 3600, - }); + // T&S Communication: 01:45, 05:45, 09:45, 13:45, 17:45, 21:45 + await boss.schedule("scrape:pricing:tscom", "45 1,5,9,13,17,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 05:20 — Flexoptix vendor list (full vendor catalog sync) - await boss.schedule("scrape:vendors:flexoptix", "20 5 * * *", {}, { - retryLimit: 2, expireInSeconds: 1800, - }); + // Skylane: 02:00, 06:00, 10:00, 14:00, 18:00, 22:00 + await boss.schedule("scrape:pricing:skylane", "0 2,6,10,14,18,22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 05:35 — Flexoptix supported vendors (OEM→Flexoptix SKU mapping) - await boss.schedule("scrape:vendors:flexoptix-supported", "35 5 * * *", {}, { - retryLimit: 2, expireInSeconds: 1800, - }); + // AscentOptics: 02:15, 06:15, 10:15, 14:15, 18:15, 22:15 + await boss.schedule("scrape:pricing:ascentoptics", "15 2,6,10,14,18,22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 05:50 — Cisco/Juniper/Arista compatibility matrices - await boss.schedule("scrape:compat:cisco", "50 5 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // GAO Tek: 02:30, 06:30, 10:30, 14:30, 18:30, 22:30 + await boss.schedule("scrape:pricing:gaotek", "30 2,6,10,14,18,22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 06:05 — Juniper HCT compatibility - await boss.schedule("scrape:compat:juniper", "5 6 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // ══════════════════════════════════════════════════════════════════════ + // CATALOG SCRAPERS — Flexoptix every 2h (primary price source) + // ══════════════════════════════════════════════════════════════════════ - // 06:20 — SONiC HCL compatibility - await boss.schedule("scrape:compat:sonic", "20 6 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 }); - // 06:35 — Ufispace switch data - await boss.schedule("scrape:compat:ufispace", "35 6 * * *", {}, { - retryLimit: 2, expireInSeconds: 1800, - }); + // ══════════════════════════════════════════════════════════════════════ + // MANUFACTURER CATALOGS — every 8h (product data, no prices) + // ══════════════════════════════════════════════════════════════════════ - // 06:45 — Edgecore switch data - await boss.schedule("scrape:compat:edgecore", "45 6 * * *", {}, { - retryLimit: 2, expireInSeconds: 1800, - }); + await boss.schedule("scrape:catalog:smartoptics", "10 3,11,19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:catalog:hubersuhner", "25 3,11,19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); - // 06:55 — Switch assets enrichment (images, specs, from Arista/Cisco/Juniper) - await boss.schedule("scrape:assets:switches", "55 6 * * *", {}, { - retryLimit: 1, expireInSeconds: 3600, - }); + // ══════════════════════════════════════════════════════════════════════ + // VENDOR LISTS — every 12h + // ══════════════════════════════════════════════════════════════════════ - // 07:00 — Community issues (Reddit/ServeTheHome/Arista/Cisco forums) - await boss.schedule("scrape:community-issues", "0 7 * * *", {}, { - retryLimit: 1, expireInSeconds: 3600, - }); + await boss.schedule("scrape:vendors:flexoptix", "0 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); + await boss.schedule("scrape:vendors:flexoptix-supported", "15 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); - // 07:10 — Datasheet + manual link discovery - await boss.schedule("scrape:datasheet-links", "10 7 * * *", {}, { - retryLimit: 1, expireInSeconds: 3600, - }); + // ══════════════════════════════════════════════════════════════════════ + // COMPATIBILITY MATRICES — every 12h + // ══════════════════════════════════════════════════════════════════════ - // 07:20 — News aggregation (LightReading, FierceTelecom, trade press) - await boss.schedule("scrape:news", "20 7 * * *", {}, { - retryLimit: 2, expireInSeconds: 1800, - }); + await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:compat:ufispace", "45 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); + await boss.schedule("scrape:compat:edgecore", "55 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); - // 07:30 — FAQ / knowledge base scraping - await boss.schedule("scrape:faq", "30 7 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + // ══════════════════════════════════════════════════════════════════════ + // SWITCH ASSETS — every 12h + // ══════════════════════════════════════════════════════════════════════ - // 07:35 — Docs check (full document/datasheet download) - await boss.schedule("scrape:docs", "35 7 * * *", {}, { - retryLimit: 2, expireInSeconds: 3600, - }); + await boss.schedule("scrape:assets:switches", "30 7,19 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); - // 07:42 — ABC classification recompute (after all pricing runs) - await boss.schedule("compute:abc", "42 7 * * *", {}, { - retryLimit: 2, expireInSeconds: 600, - }); + // ══════════════════════════════════════════════════════════════════════ + // EBAY ENRICHMENT — every 6h + // ══════════════════════════════════════════════════════════════════════ - // 07:50 — Reorder signals recompute (after ABC) - await boss.schedule("compute:reorder-signals", "50 7 * * *", {}, { - retryLimit: 2, expireInSeconds: 600, - }); + await boss.schedule("enrich:ebay-transceivers", "0 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 }); + await boss.schedule("enrich:ebay-switches", "30 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 }); - // 07:55 — NAS sync: export all data as JSON + weekly pg_dump to Fearghas - await boss.schedule("sync:nas", "55 7 * * *", {}, { - retryLimit: 1, expireInSeconds: 1800, - }); + // ══════════════════════════════════════════════════════════════════════ + // INTELLIGENCE & COMMUNITY — every 6h + // ══════════════════════════════════════════════════════════════════════ - console.log("All schedules registered — nightly window 00:00–08:00 (30 jobs)"); + await boss.schedule("scrape:market-intel", "0 2,8,14,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:community-issues", "30 2,8,14,20 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); + await boss.schedule("scrape:datasheet-links", "0 3,9,15,21 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); + await boss.schedule("scrape:news", "20 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); + await boss.schedule("scrape:faq", "40 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + await boss.schedule("scrape:docs", "50 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); + + // ══════════════════════════════════════════════════════════════════════ + // COMPUTE JOBS — every 4h (after pricing waves settle) + // ══════════════════════════════════════════════════════════════════════ + + await boss.schedule("compute:abc", "50 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 }); + await boss.schedule("compute:reorder-signals", "55 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 }); + + // ══════════════════════════════════════════════════════════════════════ + // NAS SYNC — nightly + // ══════════════════════════════════════════════════════════════════════ + + await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 }); + + console.log("All schedules registered — 24/7 continuous scraping (42 jobs)"); } export async function registerWorkers(boss: PgBoss): Promise { @@ -296,173 +262,225 @@ export async function registerWorkers(boss: PgBoss): Promise { const { scrapeUfiSpace } = await import("./scrapers/ufispace"); const { scrapeEdgecore } = await import("./scrapers/edgecore"); const { scrapeSwitchAssets } = await import("./scrapers/switch-assets"); + const { scrapeBlueOptics } = await import("./scrapers/blueoptics"); + const { scrapeFiber24 } = await import("./scrapers/fiber24"); + const { scrapeTsCom } = await import("./scrapers/tscom"); + const { scrapeSmartOptics } = await import("./scrapers/smartoptics"); + const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner"); + const { scrapeSkylane } = await import("./scrapers/skylane"); + const { scrapeAscentOptics } = await import("./scrapers/ascentoptics"); + const { scrapeGaoTek } = await import("./scrapers/gaotek"); - // ── Pricing scrapers ────────────────────────────────────────────── + // ── Playwright scrapers ─────────────────────────────────────────────── - await boss.work("scrape:pricing:fs", async (_job) => { + await boss.work("scrape:pricing:fs", async () => { console.log(`[${new Date().toISOString()}] Running: FS.com pricing`); await withIsolatedStorage("fs", scrapeFs); }); - await boss.work("scrape:pricing:champion-one", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`); - await withIsolatedStorage("champion-one", scrapeChampionOne); - }); - - await boss.work("scrape:pricing:fluxlight", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`); - await withIsolatedStorage("fluxlight", scrapeFluxlight); - }); - - await boss.work("scrape:pricing:gbics", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: GBICs pricing`); - await withIsolatedStorage("gbics", scrapeGbics); - }); - - await boss.work("scrape:pricing:optcore", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: Optcore pricing`); - await withIsolatedStorage("optcore", scrapeOptcore); - }); - - await boss.work("scrape:pricing:10gtek", async (_job) => { + await boss.work("scrape:pricing:10gtek", async () => { console.log(`[${new Date().toISOString()}] Running: 10Gtek pricing`); await withIsolatedStorage("10gtek", scrape10Gtek); }); - await boss.work("scrape:pricing:atgbics", async (_job) => { + await boss.work("scrape:pricing:atgbics", async () => { console.log(`[${new Date().toISOString()}] Running: ATGBICS pricing`); await withIsolatedStorage("atgbics", scrapeAtgbics); }); - await boss.work("scrape:pricing:prolabs", async (_job) => { + await boss.work("scrape:pricing:prolabs", async () => { console.log(`[${new Date().toISOString()}] Running: ProLabs pricing`); await withIsolatedStorage("prolabs", scrapeProLabs); }); - await boss.work("scrape:pricing:sfpcables", async (_job) => { + // ── Fetch/Cheerio scrapers ──────────────────────────────────────────── + + await boss.work("scrape:pricing:fluxlight", async () => { + console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`); + await withIsolatedStorage("fluxlight", scrapeFluxlight); + }); + + await boss.work("scrape:pricing:gbics", async () => { + console.log(`[${new Date().toISOString()}] Running: GBICs pricing`); + await withIsolatedStorage("gbics", scrapeGbics); + }); + + await boss.work("scrape:pricing:optcore", async () => { + console.log(`[${new Date().toISOString()}] Running: Optcore pricing`); + await withIsolatedStorage("optcore", scrapeOptcore); + }); + + await boss.work("scrape:pricing:champion-one", async () => { + console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`); + await withIsolatedStorage("champion-one", scrapeChampionOne); + }); + + await boss.work("scrape:pricing:sfpcables", async () => { console.log(`[${new Date().toISOString()}] Running: SFPCables pricing`); await withIsolatedStorage("sfpcables", scrapeSfpCables); }); - // ── Catalog / vendor scrapers ───────────────────────────────────── + await boss.work("scrape:pricing:blueoptics", async () => { + console.log(`[${new Date().toISOString()}] Running: BlueOptics pricing`); + await withIsolatedStorage("blueoptics", scrapeBlueOptics); + }); - await boss.work("scrape:pricing:flexoptix", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog pricing`); + await boss.work("scrape:pricing:fiber24", async () => { + console.log(`[${new Date().toISOString()}] Running: ShopFiber24 pricing`); + await withIsolatedStorage("fiber24", scrapeFiber24); + }); + + await boss.work("scrape:pricing:tscom", async () => { + console.log(`[${new Date().toISOString()}] Running: T&S Communication pricing`); + await withIsolatedStorage("tscom", scrapeTsCom); + }); + + await boss.work("scrape:pricing:skylane", async () => { + console.log(`[${new Date().toISOString()}] Running: Skylane Optics pricing`); + await withIsolatedStorage("skylane", scrapeSkylane); + }); + + await boss.work("scrape:pricing:ascentoptics", async () => { + console.log(`[${new Date().toISOString()}] Running: AscentOptics pricing`); + await withIsolatedStorage("ascentoptics", scrapeAscentOptics); + }); + + await boss.work("scrape:pricing:gaotek", async () => { + console.log(`[${new Date().toISOString()}] Running: GAO Tek pricing`); + await withIsolatedStorage("gaotek", scrapeGaoTek); + }); + + // ── Catalog scrapers ────────────────────────────────────────────────── + + await boss.work("scrape:pricing:flexoptix", async () => { + console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`); await scrapeFlexoptixCatalog(); }); - await boss.work("scrape:vendors:flexoptix", async (_job) => { + await boss.work("scrape:catalog:smartoptics", async () => { + console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`); + await withIsolatedStorage("smartoptics", scrapeSmartOptics); + }); + + await boss.work("scrape:catalog:hubersuhner", async () => { + console.log(`[${new Date().toISOString()}] Running: HUBER+SUHNER catalog`); + await withIsolatedStorage("hubersuhner", scrapeHuberSuhner); + }); + + // ── Vendor lists ────────────────────────────────────────────────────── + + await boss.work("scrape:vendors:flexoptix", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix vendor list`); await scrapeFlexoptixVendors(); }); - await boss.work("scrape:vendors:flexoptix-supported", async (_job) => { + await boss.work("scrape:vendors:flexoptix-supported", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix supported vendors`); await seedFlexoptixVendors(); }); - // ── Compatibility scrapers ──────────────────────────────────────── + // ── Compatibility scrapers ──────────────────────────────────────────── - await boss.work("scrape:compat:cisco", async (_job) => { + await boss.work("scrape:compat:cisco", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`); await withIsolatedStorage("cisco", scrapeCiscoTmg); }); - await boss.work("scrape:compat:juniper", async (_job) => { + await boss.work("scrape:compat:juniper", async () => { console.log(`[${new Date().toISOString()}] Running: Juniper HCT compatibility`); await withIsolatedStorage("juniper", scrapeJuniperHct); }); - await boss.work("scrape:compat:sonic", async (_job) => { + await boss.work("scrape:compat:sonic", async () => { console.log(`[${new Date().toISOString()}] Running: SONiC HCL compatibility`); await withIsolatedStorage("sonic", scrapeSonicHcl); }); - await boss.work("scrape:compat:ufispace", async (_job) => { + await boss.work("scrape:compat:ufispace", async () => { console.log(`[${new Date().toISOString()}] Running: Ufispace switch data`); await withIsolatedStorage("ufispace", scrapeUfiSpace); }); - await boss.work("scrape:compat:edgecore", async (_job) => { + await boss.work("scrape:compat:edgecore", async () => { console.log(`[${new Date().toISOString()}] Running: Edgecore switch data`); await withIsolatedStorage("edgecore", scrapeEdgecore); }); - // ── Switch asset enrichment ─────────────────────────────────────── + // ── Switch assets ───────────────────────────────────────────────────── - await boss.work("scrape:assets:switches", async (_job) => { + await boss.work("scrape:assets:switches", async () => { console.log(`[${new Date().toISOString()}] Running: Switch assets enrichment`); await withIsolatedStorage("switch-assets", () => scrapeSwitchAssets()); }); - // ── eBay enrichment ─────────────────────────────────────────────── + // ── eBay enrichment ─────────────────────────────────────────────────── - await boss.work("enrich:ebay-switches", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`); - const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher"); - await withIsolatedStorage("ebay-switches", () => enrichSwitchesFromEbay(30)); - }); - - await boss.work("enrich:ebay-transceivers", async (_job) => { + await boss.work("enrich:ebay-transceivers", async () => { console.log(`[${new Date().toISOString()}] Running: eBay transceiver pricing`); const { enrichTransceiversFromEbay } = await import("./scrapers/ebay-enricher"); await withIsolatedStorage("ebay-transceivers", () => enrichTransceiversFromEbay(100)); }); - // ── Intelligence & community ────────────────────────────────────── + await boss.work("enrich:ebay-switches", async () => { + console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`); + const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher"); + await withIsolatedStorage("ebay-switches", () => enrichSwitchesFromEbay(30)); + }); - await boss.work("scrape:market-intel", async (_job) => { + // ── Intelligence & community ────────────────────────────────────────── + + await boss.work("scrape:market-intel", async () => { console.log(`[${new Date().toISOString()}] Running: Market intelligence`); const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence"); await withIsolatedStorage("market-intel", scrapeMarketIntelligence); }); - await boss.work("scrape:community-issues", async (_job) => { + await boss.work("scrape:community-issues", async () => { console.log(`[${new Date().toISOString()}] Running: Community issues`); const { scrapeAllSwitchIssues } = await import("./scrapers/community-issues"); await withIsolatedStorage("community-issues", () => scrapeAllSwitchIssues(30)); }); - await boss.work("scrape:datasheet-links", async (_job) => { - console.log(`[${new Date().toISOString()}] Running: Datasheet link discovery`); + await boss.work("scrape:datasheet-links", async () => { + console.log(`[${new Date().toISOString()}] Running: Datasheet links`); const { findAndSeedDatasheetLinks } = await import("./scrapers/community-issues"); await findAndSeedDatasheetLinks(50); }); - await boss.work("scrape:news", async (_job) => { + await boss.work("scrape:news", async () => { console.log(`[${new Date().toISOString()}] Running: News aggregation`); await scrapeNews(); }); - await boss.work("scrape:faq", async (_job) => { + await boss.work("scrape:faq", async () => { console.log(`[${new Date().toISOString()}] FAQ scraper — not yet implemented`); }); - await boss.work("scrape:docs", async (_job) => { + await boss.work("scrape:docs", async () => { console.log(`[${new Date().toISOString()}] Docs scraper — not yet implemented`); }); - // ── Compute jobs ────────────────────────────────────────────────── + // ── Compute jobs ────────────────────────────────────────────────────── - await boss.work("compute:abc", async (_job) => { + await boss.work("compute:abc", async () => { console.log(`[${new Date().toISOString()}] Computing: ABC classification`); const { computeAbcClassification } = await import("./scrapers/market-intelligence"); await computeAbcClassification(); }); - await boss.work("compute:reorder-signals", async (_job) => { + await boss.work("compute:reorder-signals", async () => { console.log(`[${new Date().toISOString()}] Computing: Reorder signals`); const { computeReorderSignals } = await import("./scrapers/market-intelligence"); await computeReorderSignals(); }); - // ── Sync ────────────────────────────────────────────────────────── + // ── NAS sync ────────────────────────────────────────────────────────── - await boss.work("sync:nas", async (_job) => { + await boss.work("sync:nas", async () => { console.log(`[${new Date().toISOString()}] Running: NAS sync to Fearghas`); const { runNightlyNasSync } = await import("./utils/nas-sync"); await runNightlyNasSync(); }); - console.log("All workers registered (30 jobs)"); + console.log("All workers registered (42 jobs, 24/7 continuous)"); } diff --git a/packages/scraper/src/scrapers/ascentoptics.ts b/packages/scraper/src/scrapers/ascentoptics.ts new file mode 100644 index 0000000..d23b7c1 --- /dev/null +++ b/packages/scraper/src/scrapers/ascentoptics.ts @@ -0,0 +1,280 @@ +/** + * Ascent Optics Scraper — US-based compatible transceiver vendor + * + * ascentoptics.com — product catalog with USD prices. + * Tries /catalog/ and /products/ as entry points. + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://ascentoptics.com"; +const CATALOG_URLS = [ + "/catalog/", + "/products/", + "/products/transceivers/", + "/catalog/transceivers/", +]; +const MAX_PAGES = 15; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string, sourceUrl: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + const cardSelectors = [ + ".product-item", ".product", ".item", "li.product", + ".product-card", "tr", "article", ".catalog-item", + ".product-list-item", ".result", + ]; + + for (const sel of cardSelectors) { + if ($(sel).length >= 2) { + $(sel).each((_i, el) => { + const text = $(el).text().trim(); + if (!/sfp|qsfp|xfp|transceiver|optic/i.test(text)) return; + + const nameEl = $(el).find("h2, h3, h4, .name, .product-name, .title, td, a").first(); + const name = nameEl.text().trim() || text.slice(0, 120); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || sourceUrl; + const url = href.startsWith("http") ? href : BASE + href; + + // Ascent Optics part numbers: e.g. AS-SFP-10G-SR, SFP-10G-LR-AS + const partNumMatch = name.match(/\b(AS[-_][A-Z0-9-]+)\b/i) || + name.match(/\b([A-Z]{2,}[-][A-Z0-9]+[-][A-Z0-9]+[-][A-Z0-9]+)\b/) || + text.match(/Part\s*(?:No\.?|Number|#)?\s*:?\s*([A-Z0-9-]{6,})/i); + const partNumber = partNumMatch?.[1] || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + // USD price + const priceText = $(el).find(".price, .product-price, .amount, [data-price]").text(); + const priceMatch = priceText.match(/\$\s*([\d,]+\.?\d{0,2})/); + let price: number | undefined; + if (priceMatch) { + const parsed = parseFloat(priceMatch[1].replace(",", "")); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + const ff = detectFormFactor(name + " " + text); + const reach = detectReach(name + " " + text); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name + " " + text), + wavelength: detectWavelength(name + " " + text), + }); + }); + if (products.length > 0) break; + } + } + + // Fallback: transceiver-relevant anchors + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 8 || name.length > 200 || !/sfp|qsfp|transceiver/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeAscentOptics(): Promise { + console.log("=== Ascent Optics Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "Ascent Optics", + "compatible", + "https://ascentoptics.com", + BASE + CATALOG_URLS[0], + ); + + const allProducts: Product[] = []; + const seenUrls = new Set(); + const triedUrls = new Set(); + + for (const catalogPath of CATALOG_URLS) { + const catalogUrl = BASE + catalogPath; + if (triedUrls.has(catalogUrl)) continue; + triedUrls.add(catalogUrl); + + console.log(` Fetching catalog: ${catalogUrl}`); + try { + const html = await fetchPage(catalogUrl); + const pageProducts = parseProductList(html, catalogUrl); + for (const p of pageProducts) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + console.log(` Found ${pageProducts.length} products`); + await sleep(2000); + + // Paginate from each working catalog URL + for (let page = 2; page <= MAX_PAGES; page++) { + const pageUrl = `${catalogUrl}?page=${page}`; + try { + const pageHtml = await fetchPage(pageUrl); + const paginated = parseProductList(pageHtml, pageUrl); + if (paginated.length === 0) break; + for (const p of paginated) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + console.log(` Page ${page}: ${paginated.length} products`); + await sleep(2000); + } catch { + break; + } + } + } catch (err) { + console.warn(` Failed: ${(err as Error).message}`); + } + } + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== Ascent Optics Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeAscentOptics() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/blueoptics.ts b/packages/scraper/src/scrapers/blueoptics.ts new file mode 100644 index 0000000..f651c3b --- /dev/null +++ b/packages/scraper/src/scrapers/blueoptics.ts @@ -0,0 +1,265 @@ +/** + * BlueOptics Scraper — German compatible transceiver vendor + * + * www.blueoptics.de — WooCommerce/Magento shop with EUR prices. + * Paginated catalog: /transceivers/?page=N + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://www.blueoptics.de"; +const CATALOG_PATH = "/transceivers/"; +const MAX_PAGES = 20; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // WooCommerce/generic shop product card selectors + $("li.product, .product-item, article.product, .product-card, .woocommerce-loop-product").each((_i, el) => { + const titleEl = $(el).find("h2, h3, .product-title, .woocommerce-loop-product__title, .product-name").first(); + const name = titleEl.text().trim(); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || ""; + const url = href.startsWith("http") ? href : BASE + href; + + // EUR price — try .price, .woocommerce-Price-amount, .amount + const priceText = $(el).find(".price, .woocommerce-Price-amount, .amount, .product-price").first().text(); + const priceMatch = priceText.match(/([\d.,]+)\s*€|€\s*([\d.,]+)/); + let price: number | undefined; + if (priceMatch) { + const raw = (priceMatch[1] || priceMatch[2]).replace(/\./g, "").replace(",", "."); + const parsed = parseFloat(raw); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + // Try to extract part number from name or SKU attribute + const skuEl = $(el).find("[data-sku], .sku"); + const partNumber = skuEl.attr("data-sku") || skuEl.text().trim() || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name); + const reach = detectReach(name); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name), + wavelength: detectWavelength(name), + }); + }); + + // Fallback: link-based extraction for non-WooCommerce layouts + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if ( + name.length < 10 || name.length > 200 || + !/sfp|qsfp|xfp|transceiver/i.test(name) + ) return; + + const url = href.startsWith("http") ? href : BASE + href; + const context = $(el).parent().parent().text(); + const priceMatch = context.match(/([\d.,]+)\s*€|€\s*([\d.,]+)/); + let price: number | undefined; + if (priceMatch) { + const raw = (priceMatch[1] || priceMatch[2]).replace(/\./g, "").replace(",", "."); + const parsed = parseFloat(raw); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + const ff = detectFormFactor(name); + const reach = detectReach(name); + + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, price, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeBlueOptics(): Promise { + console.log("=== BlueOptics Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "BlueOptics", + "compatible", + "https://www.blueoptics.de", + BASE + CATALOG_PATH, + ); + + let allProducts: Product[] = []; + + for (let page = 1; page <= MAX_PAGES; page++) { + try { + const url = page === 1 + ? BASE + CATALOG_PATH + : `${BASE}${CATALOG_PATH}?page=${page}`; + const html = await fetchPage(url); + const pageProducts = parseProductList(html); + allProducts.push(...pageProducts); + console.log(` Page ${page}: ${pageProducts.length} products`); + if (pageProducts.length === 0) { + console.log(` Empty page ${page}, stopping pagination.`); + break; + } + if (page < MAX_PAGES) await sleep(2000); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message}`); + break; + } + } + + // Dedupe by URL + const seen = new Set(); + allProducts = allProducts.filter((p) => { + if (seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "EUR", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== BlueOptics Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeBlueOptics() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/fiber24.ts b/packages/scraper/src/scrapers/fiber24.ts new file mode 100644 index 0000000..8d93009 --- /dev/null +++ b/packages/scraper/src/scrapers/fiber24.ts @@ -0,0 +1,260 @@ +/** + * ShopFiber24 Scraper — German compatible transceiver shop + * + * shop.fiber24.net — EUR prices, FO transceiver category. + * Paginated catalog: /FO-TRANSCEIVER/de?p=N + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://shop.fiber24.net"; +const CATALOG_PATH = "/FO-TRANSCEIVER/de"; +const MAX_PAGES = 20; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // Common e-commerce product listing selectors + const cardSelectors = [ + ".product-item", ".item", "li.product", ".product-card", + ".woocommerce-loop-product", "article.product", ".grid-item", + ]; + + let found = false; + for (const sel of cardSelectors) { + if ($(sel).length > 0) { + $(sel).each((_i, el) => { + const nameEl = $(el).find("h2, h3, .product-name, .product-title, .item-title, a").first(); + const name = nameEl.text().trim(); + if (!name || name.length < 5 || !/sfp|qsfp|xfp|transceiver|optic/i.test(name)) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || ""; + const url = href.startsWith("http") ? href : BASE + href; + + const priceText = $(el).find(".price, .product-price, .price-box, .amount, [data-price]").text(); + const priceMatch = priceText.match(/([\d.,]+)\s*€|€\s*([\d.,]+)/); + let price: number | undefined; + if (priceMatch) { + const raw = (priceMatch[1] || priceMatch[2]).replace(/\./g, "").replace(",", "."); + const parsed = parseFloat(raw); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + const skuText = $(el).find(".sku, [data-sku], .product-sku").text().trim(); + const partNumber = skuText || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name); + const reach = detectReach(name); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name), + wavelength: detectWavelength(name), + }); + }); + if (products.length > 0) { found = true; break; } + } + } + + // Fallback: extract all transceiver-relevant links + if (!found) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 8 || name.length > 200 || !/sfp|qsfp|transceiver/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeFiber24(): Promise { + console.log("=== ShopFiber24 Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "ShopFiber24", + "compatible", + "https://shop.fiber24.net", + BASE + CATALOG_PATH, + ); + + let allProducts: Product[] = []; + + for (let page = 1; page <= MAX_PAGES; page++) { + try { + // Try common pagination patterns: ?p=N, ?page=N, /page/N/ + const url = page === 1 + ? BASE + CATALOG_PATH + : `${BASE}${CATALOG_PATH}?p=${page}`; + const html = await fetchPage(url); + const pageProducts = parseProductList(html); + allProducts.push(...pageProducts); + console.log(` Page ${page}: ${pageProducts.length} products`); + if (pageProducts.length === 0) { + console.log(` Empty page ${page}, stopping pagination.`); + break; + } + if (page < MAX_PAGES) await sleep(2000); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message}`); + break; + } + } + + const seen = new Set(); + allProducts = allProducts.filter((p) => { + if (seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "EUR", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== ShopFiber24 Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeFiber24() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/gaotek.ts b/packages/scraper/src/scrapers/gaotek.ts new file mode 100644 index 0000000..6775a59 --- /dev/null +++ b/packages/scraper/src/scrapers/gaotek.ts @@ -0,0 +1,263 @@ +/** + * GAO Tek Scraper — Canadian compatible transceiver distributor + * + * gaotek.com — large catalog with USD prices, WooCommerce-based. + * Category: /category/fiber-optics/transceivers/ + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://gaotek.com"; +const CATALOG_PATH = "/category/fiber-optics/transceivers/"; +const MAX_PAGES = 20; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // WooCommerce product grid + $("li.product, .product-item, .woocommerce-loop-product, article.product").each((_i, el) => { + const titleEl = $(el).find(".woocommerce-loop-product__title, h2, h3, .product-title, .product-name").first(); + const name = titleEl.text().trim(); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || ""; + const url = href.startsWith("http") ? href : BASE + href; + + // WooCommerce price + const priceText = $(el).find(".price, .woocommerce-Price-amount, .amount").text(); + const priceMatch = priceText.match(/\$\s*([\d,]+\.?\d{0,2})/); + let price: number | undefined; + if (priceMatch) { + const parsed = parseFloat(priceMatch[1].replace(",", "")); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + // GAO Tek uses SKU for part numbers + const skuEl = $(el).find(".sku, [data-sku]"); + const partNumber = skuEl.text().trim() || + url.split("/").filter(Boolean).pop()?.replace(/-/g, " ").trim() || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name); + const reach = detectReach(name); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name), + wavelength: detectWavelength(name), + }); + }); + + // Fallback for non-WooCommerce layout + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if ( + name.length < 8 || name.length > 200 || + !/sfp|qsfp|xfp|transceiver|optic/i.test(name) + ) return; + + const url = href.startsWith("http") ? href : BASE + href; + const context = $(el).parent().parent().text(); + const priceMatch = context.match(/\$\s*([\d,]+\.?\d{0,2})/); + let price: number | undefined; + if (priceMatch) { + const parsed = parseFloat(priceMatch[1].replace(",", "")); + if (parsed > 0 && parsed < 50000) price = parsed; + } + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, price, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeGaoTek(): Promise { + console.log("=== GAO Tek Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "GAO Tek", + "compatible", + "https://gaotek.com", + BASE + CATALOG_PATH, + ); + + let allProducts: Product[] = []; + + for (let page = 1; page <= MAX_PAGES; page++) { + try { + // WooCommerce pagination: /page/N/ suffix + const url = page === 1 + ? BASE + CATALOG_PATH + : `${BASE}${CATALOG_PATH}page/${page}/`; + const html = await fetchPage(url); + const pageProducts = parseProductList(html); + allProducts.push(...pageProducts); + console.log(` Page ${page}: ${pageProducts.length} products`); + if (pageProducts.length === 0) { + console.log(` Empty page ${page}, stopping pagination.`); + break; + } + if (page < MAX_PAGES) await sleep(2000); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message}`); + break; + } + } + + // Dedupe + const seen = new Set(); + allProducts = allProducts.filter((p) => { + if (seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== GAO Tek Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeGaoTek() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/hubersuhner.ts b/packages/scraper/src/scrapers/hubersuhner.ts new file mode 100644 index 0000000..b16448a --- /dev/null +++ b/packages/scraper/src/scrapers/hubersuhner.ts @@ -0,0 +1,295 @@ +/** + * HUBER+SUHNER Scraper — Swiss transceiver manufacturer + * + * www.hubersuhner.com — no direct prices, catalog-only scrape. + * Tries DE and EN product listing pages; may expose product API. + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor } from "../utils/db"; +import * as cheerio from "cheerio"; + +const BASE = "https://www.hubersuhner.com"; + +// Try multiple entry points — HS has EN and DE variants +const CATALOG_URLS = [ + "/en/transceiver", + "/de/transceiver", + "/en/products/fiber-optics/fiber-optic-transceivers", + "/de/products/fiber-optics/fiber-optic-transceivers", + "/en/products/fiber-optics", +]; + +// Known HS product API endpoint pattern (JSON-based catalog) +const API_URLS = [ + "/api/products?category=transceiver&format=json", + "/api/catalog/transceiver", +]; + +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml,application/json", + "Accept-Language": "en-US,en;q=0.9,de;q=0.8", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseHtmlProducts(html: string, sourceUrl: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // HS uses a custom product portal — try multiple selectors + const cardSelectors = [ + ".product-item", ".product-tile", ".product-card", ".product", + "tr[data-part]", "tr", ".item", ".result-item", ".catalog-item", + ".product-list-item", "li[data-id]", + ]; + + for (const sel of cardSelectors) { + if ($(sel).length >= 2) { + $(sel).each((_i, el) => { + const text = $(el).text().trim(); + if (!/sfp|qsfp|xfp|transceiver|optic|fiber/i.test(text)) return; + + const nameEl = $(el).find("h2, h3, h4, .name, .title, .product-name, a").first(); + const name = nameEl.text().trim() || text.slice(0, 100); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || sourceUrl; + const url = href.startsWith("http") ? href : BASE + href; + + // HUBER+SUHNER part numbers often follow: 85XXXXXXX or similar numeric patterns + const partNumMatch = text.match(/\b(85\d{7,})\b/) || + text.match(/\b([A-Z]{2,}[-_][A-Z0-9]+[-_][A-Z0-9]+)\b/) || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/); + const partNumber = partNumMatch?.[0] || name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name + " " + text); + const reach = detectReach(name + " " + text); + + products.push({ + partNumber, + name, + url, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name + " " + text), + wavelength: detectWavelength(name + " " + text), + }); + }); + if (products.length > 0) break; + } + } + + // Fallback: all product-relevant links + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 5 || name.length > 200 || !/sfp|qsfp|transceiver|optic/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +// Try JSON API endpoint — some HS portals expose catalog APIs +function parseJsonProducts(json: unknown): Product[] { + const products: Product[] = []; + const items = Array.isArray(json) + ? json + : (json as Record)?.products || (json as Record)?.items || []; + if (!Array.isArray(items)) return products; + + for (const item of items) { + const name = String((item as Record).name || (item as Record).title || ""); + if (!name || !/sfp|qsfp|transceiver/i.test(name)) continue; + const partNumber = String( + (item as Record).partNumber || + (item as Record).sku || + (item as Record).id || + name.replace(/\s+/g, "-").slice(0, 60) + ); + const url = String((item as Record).url || BASE); + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber, name, + url: url.startsWith("http") ? url : BASE + url, + ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + } + return products; +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeHuberSuhner(): Promise { + console.log("=== HUBER+SUHNER Scraper Starting (catalog-only, no prices) ===\n"); + + const vendorId = await ensureVendor( + "HUBER+SUHNER", + "manufacturer", + "https://www.hubersuhner.com", + BASE + CATALOG_URLS[0], + ); + + const allProducts: Product[] = []; + const seenUrls = new Set(); + + // Try JSON API endpoints first + for (const apiPath of API_URLS) { + const apiUrl = BASE + apiPath; + console.log(` Trying API: ${apiUrl}`); + try { + const resp = await fetch(apiUrl, { + headers: { ...HEADERS, Accept: "application/json" }, + signal: AbortSignal.timeout(15000), + }); + if (resp.ok) { + const json: unknown = await resp.json(); + const parsed = parseJsonProducts(json); + console.log(` API returned ${parsed.length} products`); + for (const p of parsed) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + } + await sleep(2000); + } catch { + // API endpoint not available — continue to HTML scraping + } + } + + // HTML catalog fallback + for (const catalogPath of CATALOG_URLS) { + const catalogUrl = BASE + catalogPath; + console.log(` Fetching catalog: ${catalogUrl}`); + try { + const html = await fetchPage(catalogUrl); + const pageProducts = parseHtmlProducts(html, catalogUrl); + for (const p of pageProducts) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + console.log(` Found ${pageProducts.length} products`); + await sleep(2000); + } catch (err) { + console.warn(` Failed ${catalogPath}: ${(err as Error).message}`); + } + } + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + + for (const product of allProducts) { + try { + await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== HUBER+SUHNER Complete: ${totalProducts} products catalogued (no prices — contact sales) ===`); +} + +if (require.main === module) { + scrapeHuberSuhner() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/skylane.ts b/packages/scraper/src/scrapers/skylane.ts new file mode 100644 index 0000000..ca0d90f --- /dev/null +++ b/packages/scraper/src/scrapers/skylane.ts @@ -0,0 +1,307 @@ +/** + * Skylane Optics Scraper — Belgian compatible transceiver vendor + * + * www.skylaneoptics.com — product catalog with form factor listings. + * No direct webshop prices; product catalog only. + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://www.skylaneoptics.com"; +const CATALOG_URLS = [ + "/en/products/", + "/en/", + "/en/products/sfp/", + "/en/products/sfp28/", + "/en/products/qsfp28/", + "/en/products/qsfp-dd/", +]; +const MAX_PAGES = 10; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string, sourceUrl: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + const cardSelectors = [ + ".product-item", ".product", ".item", "li.product", + ".product-card", "article", ".product-list-item", ".grid-item", + ]; + + for (const sel of cardSelectors) { + if ($(sel).length >= 2) { + $(sel).each((_i, el) => { + const text = $(el).text().trim(); + if (!/sfp|qsfp|xfp|transceiver|optic/i.test(text)) return; + + const nameEl = $(el).find("h2, h3, h4, .name, .product-name, .title, a").first(); + const name = nameEl.text().trim() || text.slice(0, 100); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || sourceUrl; + const url = href.startsWith("http") ? href : BASE + href; + + // Skylane part numbers: e.g. SO-SFP+-LRM-CI, SFP-10G-SR-SO + const partNumMatch = name.match(/\b(SO[-_][A-Z0-9-]+)\b/i) || + name.match(/\b([A-Z]{2,}[-][A-Z0-9]+[-][A-Z0-9]+)\b/) || + text.match(/Part\s*[#:]?\s*([A-Z0-9-]{5,})/i); + const partNumber = partNumMatch?.[1] || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + // Check for price + const priceText = $(el).find(".price, .product-price, .amount").text(); + const priceMatch = priceText.match(/\$\s*([\d,]+\.?\d{0,2})/) || + priceText.match(/([\d.,]+)\s*€/); + let price: number | undefined; + if (priceMatch) { + const raw = priceMatch[1].replace(/[.,](\d{2})$/, ".$1").replace(/[.,]/g, ""); + const parsed = parseFloat(raw); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + const ff = detectFormFactor(name + " " + text); + const reach = detectReach(name + " " + text); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name + " " + text), + wavelength: detectWavelength(name + " " + text), + }); + }); + if (products.length > 0) break; + } + } + + // Fallback: transceiver-relevant anchors + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 8 || name.length > 200 || !/sfp|qsfp|transceiver/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +// Discover sub-category links from a catalog page +function extractCategoryLinks(html: string): string[] { + const $ = cheerio.load(html); + const links: string[] = []; + $("a[href]").each((_i, el) => { + const href = $(el).attr("href") || ""; + if (/\/products?\//i.test(href) && /sfp|qsfp|transceiver|optic/i.test(href)) { + links.push(href.startsWith("http") ? href : BASE + href); + } + }); + return [...new Set(links)]; +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeSkylane(): Promise { + console.log("=== Skylane Optics Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "Skylane Optics", + "compatible", + "https://www.skylaneoptics.com", + BASE + CATALOG_URLS[0], + ); + + const allProducts: Product[] = []; + const seenUrls = new Set(); + const scrapedCatalogUrls = new Set(); + + // Start with known catalog URLs and discover sub-categories + const urlQueue = [...CATALOG_URLS.map((p) => BASE + p)]; + + for (const catalogUrl of urlQueue) { + if (scrapedCatalogUrls.has(catalogUrl)) continue; + scrapedCatalogUrls.add(catalogUrl); + + console.log(` Fetching: ${catalogUrl}`); + try { + const html = await fetchPage(catalogUrl); + const pageProducts = parseProductList(html, catalogUrl); + for (const p of pageProducts) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + console.log(` Found ${pageProducts.length} products`); + + // Discover sub-category links (limit to first catalog page) + if (scrapedCatalogUrls.size <= CATALOG_URLS.length) { + const newLinks = extractCategoryLinks(html).slice(0, 10); + for (const link of newLinks) { + if (!scrapedCatalogUrls.has(link)) urlQueue.push(link); + } + } + + // Try pagination for this URL + for (let page = 2; page <= MAX_PAGES; page++) { + const pageUrl = `${catalogUrl}?page=${page}`; + try { + const pageHtml = await fetchPage(pageUrl); + const paginated = parseProductList(pageHtml, pageUrl); + if (paginated.length === 0) break; + for (const p of paginated) { + if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); } + } + console.log(` Page ${page}: ${paginated.length} products`); + await sleep(2000); + } catch { + break; + } + } + + await sleep(2000); + } catch (err) { + console.warn(` Failed: ${(err as Error).message}`); + } + } + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== Skylane Optics Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeSkylane() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/smartoptics.ts b/packages/scraper/src/scrapers/smartoptics.ts new file mode 100644 index 0000000..378e0f9 --- /dev/null +++ b/packages/scraper/src/scrapers/smartoptics.ts @@ -0,0 +1,232 @@ +/** + * SmartOptics Scraper — Manufacturer, quote-based pricing + * + * www.smartoptics.com — no direct prices, catalog-only scrape. + * Extracts product names, part numbers, form factors and specs. + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor } from "../utils/db"; +import * as cheerio from "cheerio"; + +const BASE = "https://www.smartoptics.com"; +const CATALOG_URLS = [ + "/products/transceivers/", + "/products/", + "/products/sfp-transceivers/", + "/products/qsfp-transceivers/", +]; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string, sourceUrl: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // Manufacturer site: product cards or table rows + const cardSelectors = [ + ".product-item", ".product-card", ".product", "tr", + "article", ".item", ".col-xs-12.col-sm-6", ".entry", + ]; + + for (const sel of cardSelectors) { + if ($(sel).length >= 2) { + $(sel).each((_i, el) => { + const text = $(el).text().trim(); + if (!/sfp|qsfp|xfp|transceiver|optic/i.test(text)) return; + + const nameEl = $(el).find("h2, h3, h4, td, .title, .name, a").first(); + const name = nameEl.text().trim() || text.slice(0, 100); + if (!name || name.length < 5) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || sourceUrl; + const url = href.startsWith("http") ? href : BASE + href; + + // Part number: look for alphanumeric patterns typical of transceiver SKUs + const partNumMatch = text.match(/\b([A-Z]{2,}[-_][A-Z0-9]{2,}[-_][A-Z0-9]+)\b/) || + text.match(/\b(SO[-_][A-Z0-9]+)\b/i) || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/); + const partNumber = partNumMatch?.[0] || name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name + " " + text); + const reach = detectReach(name + " " + text); + + products.push({ + partNumber, + name, + url, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name + " " + text), + wavelength: detectWavelength(name + " " + text), + }); + }); + if (products.length > 0) break; + } + } + + // Fallback: all transceiver-relevant anchors + if (products.length === 0) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 5 || name.length > 200 || !/sfp|qsfp|transceiver|optic/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeSmartOptics(): Promise { + console.log("=== SmartOptics Scraper Starting (catalog-only, no prices) ===\n"); + + const vendorId = await ensureVendor( + "SmartOptics", + "manufacturer", + "https://www.smartoptics.com", + BASE + CATALOG_URLS[0], + ); + + const allProducts: Product[] = []; + const seenUrls = new Set(); + + for (const catalogPath of CATALOG_URLS) { + const catalogUrl = BASE + catalogPath; + console.log(` Fetching catalog: ${catalogUrl}`); + try { + const html = await fetchPage(catalogUrl); + const pageProducts = parseProductList(html, catalogUrl); + for (const p of pageProducts) { + if (!seenUrls.has(p.url)) { + seenUrls.add(p.url); + allProducts.push(p); + } + } + console.log(` Found ${pageProducts.length} products`); + await sleep(2000); + } catch (err) { + console.warn(` Failed ${catalogPath}: ${(err as Error).message}`); + } + } + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + + for (const product of allProducts) { + try { + await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== SmartOptics Complete: ${totalProducts} products catalogued (no prices — quote-based) ===`); +} + +if (require.main === module) { + scrapeSmartOptics() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/packages/scraper/src/scrapers/tscom.ts b/packages/scraper/src/scrapers/tscom.ts new file mode 100644 index 0000000..8a1c240 --- /dev/null +++ b/packages/scraper/src/scrapers/tscom.ts @@ -0,0 +1,260 @@ +/** + * T&S Communication Scraper — Chinese compatible transceiver manufacturer + * + * www.china-tscom.com — product catalog with USD prices. + * Paginated: /products/fiber-optic-transceivers/?page=N + * + * Rate limited: 1 req/2sec. + */ +import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db"; +import { contentHash } from "../utils/hash"; +import * as cheerio from "cheerio"; + +const BASE = "https://www.china-tscom.com"; +const CATALOG_PATH = "/products/fiber-optic-transceivers/"; +const MAX_PAGES = 15; +const HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + Accept: "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +}; + +interface Product { + partNumber: string; + name: string; + url: string; + price?: number; + formFactor: string; + speed: string; + speedGbps: number; + reachLabel?: string; + reachMeters?: number; + fiberType?: string; + wavelength?: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } { + const lower = text.toLowerCase(); + if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 }; + if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 }; + if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 }; + if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 }; + if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 }; + if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; + if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 }; + if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 }; + return { formFactor: "SFP+", speed: "10G", speedGbps: 10 }; +} + +function detectReach(text: string): { label: string; meters: number } | undefined { + const patterns: [RegExp, string, number][] = [ + [/\b80\s*km\b/i, "80km", 80000], + [/\b40\s*km\b/i, "40km", 40000], + [/\b20\s*km\b/i, "20km", 20000], + [/\b10\s*km\b/i, "10km", 10000], + [/\b2\s*km\b/i, "2km", 2000], + [/\b550\s*m\b/i, "550m", 550], + [/\b300\s*m\b/i, "300m", 300], + [/\b100\s*m\b/i, "100m", 100], + [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000], + [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000], + [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000], + ]; + for (const [regex, label, meters] of patterns) { + if (regex.test(text)) return { label, meters }; + } + return undefined; +} + +function detectFiber(text: string): string { + if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF"; + if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF"; + if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper"; + return "SMF"; +} + +function detectWavelength(text: string): string { + const match = text.match(/(\d{3,4})\s*nm/i); + return match ? match[1] : ""; +} + +function parseProductList(html: string): Product[] { + const $ = cheerio.load(html); + const products: Product[] = []; + + // Try common product grid/list selectors + const cardSelectors = [ + ".product-item", ".product-card", ".item", "li.product", + "article", ".product", ".col-product", ".entry", + ]; + + let found = false; + for (const sel of cardSelectors) { + if ($(sel).length >= 2) { + $(sel).each((_i, el) => { + const nameEl = $(el).find("h2, h3, h4, .product-name, .entry-title, a").first(); + const name = nameEl.text().trim(); + if (!name || name.length < 5 || !/sfp|qsfp|xfp|transceiver|optic/i.test(name)) return; + + const linkEl = $(el).find("a[href]").first(); + const href = linkEl.attr("href") || ""; + const url = href.startsWith("http") ? href : BASE + href; + + // USD price patterns + const priceText = $(el).find(".price, .product-price, .amount, [data-price]").text(); + const priceMatch = priceText.match(/\$\s*([\d,]+\.?\d{0,2})/) || + priceText.match(/([\d,]+\.?\d{0,2})\s*USD/i); + let price: number | undefined; + if (priceMatch) { + const parsed = parseFloat(priceMatch[1].replace(",", "")); + if (parsed > 0 && parsed < 50000) price = parsed; + } + + const skuText = $(el).find(".sku, [data-sku], .model, .part-number").text().trim(); + const partNumber = skuText || + name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || + name.replace(/\s+/g, "-").slice(0, 60); + + const ff = detectFormFactor(name); + const reach = detectReach(name); + + products.push({ + partNumber, + name, + url, + price, + ...ff, + reachLabel: reach?.label, + reachMeters: reach?.meters, + fiberType: detectFiber(name), + wavelength: detectWavelength(name), + }); + }); + if (products.length > 0) { found = true; break; } + } + } + + // Fallback: anchor-based extraction + if (!found) { + $("a[href]").each((_i, el) => { + const name = $(el).text().trim(); + const href = $(el).attr("href") || ""; + if (name.length < 8 || name.length > 200 || !/sfp|qsfp|transceiver/i.test(name)) return; + const url = href.startsWith("http") ? href : BASE + href; + const ff = detectFormFactor(name); + const reach = detectReach(name); + products.push({ + partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60), + name, url, ...ff, + reachLabel: reach?.label, reachMeters: reach?.meters, + fiberType: detectFiber(name), wavelength: detectWavelength(name), + }); + }); + } + + const seen = new Set(); + return products.filter((p) => { + if (!p.url || seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); +} + +async function fetchPage(url: string): Promise { + const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`); + return resp.text(); +} + +export async function scrapeTsCom(): Promise { + console.log("=== T&S Communication Scraper Starting ===\n"); + + const vendorId = await ensureVendor( + "T&S Communication", + "compatible", + "https://www.china-tscom.com", + BASE + CATALOG_PATH, + ); + + let allProducts: Product[] = []; + + for (let page = 1; page <= MAX_PAGES; page++) { + try { + const url = page === 1 + ? BASE + CATALOG_PATH + : `${BASE}${CATALOG_PATH}?page=${page}`; + const html = await fetchPage(url); + const pageProducts = parseProductList(html); + allProducts.push(...pageProducts); + console.log(` Page ${page}: ${pageProducts.length} products`); + if (pageProducts.length === 0) { + console.log(` Empty page ${page}, stopping pagination.`); + break; + } + if (page < MAX_PAGES) await sleep(2000); + } catch (err) { + console.warn(` Page ${page} failed: ${(err as Error).message}`); + break; + } + } + + const seen = new Set(); + allProducts = allProducts.filter((p) => { + if (seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); + + console.log(`\nTotal unique products: ${allProducts.length}`); + + let totalProducts = 0; + let priceUpdates = 0; + + for (const product of allProducts) { + try { + const txId = await findOrCreateScrapedTransceiver({ + partNumber: product.partNumber, + vendorId, + formFactor: product.formFactor, + speedGbps: product.speedGbps, + speed: product.speed, + reachMeters: product.reachMeters, + reachLabel: product.reachLabel, + fiberType: product.fiberType, + wavelengths: product.wavelength, + category: "DataCenter", + }); + + if (product.price && product.price > 0) { + const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const updated = await upsertPriceObservation({ + transceiverId: txId, + sourceVendorId: vendorId, + price: product.price, + currency: "USD", + stockLevel: "in_stock", + url: product.url, + contentHash: hash, + }); + if (updated) priceUpdates++; + } + totalProducts++; + } catch (err) { + console.warn(` Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`); + } + } + + console.log(`\n=== T&S Communication Complete: ${totalProducts} products, ${priceUpdates} prices ===`); +} + +if (require.main === module) { + scrapeTsCom() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); +} diff --git a/scripts/pi-scraper-setup.sh b/scripts/pi-scraper-setup.sh new file mode 100644 index 0000000..2735f26 --- /dev/null +++ b/scripts/pi-scraper-setup.sh @@ -0,0 +1,204 @@ +#!/bin/bash +# ───────────────────────────────────────────────────────────────────────────── +# TIP Pi Scraper Setup — run this ONCE on each Raspberry Pi +# +# Usage (from the Pi itself or via SSH once you have access): +# curl -sL https://gitea.context-x.org/rene/transceiver-db/raw/branch/main/scripts/pi-scraper-setup.sh | bash +# +# Or copy & run manually: +# bash pi-scraper-setup.sh +# +# What this does: +# 1. Installs Node.js 22 + tsx + pm2 +# 2. Clones the TIP scraper package +# 3. Installs dependencies (no Playwright — Pi runs fetch-only scrapers) +# 4. Creates .env pointing to Erik's PostgreSQL via WireGuard +# 5. Starts pm2 with the Pi-specific scheduler (lightweight scrapers only) +# ───────────────────────────────────────────────────────────────────────────── + +set -e + +PI_NAME="${PI_NAME:-pi-scraper}" # override with PI_NAME=pi2 bash setup.sh +DB_HOST="${DB_HOST:-10.10.0.1}" # Erik WireGuard IP +DB_PORT="${DB_PORT:-5433}" +DB_USER="${DB_USER:-tip}" +DB_PASS="${DB_PASS:-tip_prod_2026}" +DB_NAME="${DB_NAME:-transceiver_db}" +GITEA="http://192.168.178.196:3000/rene/transceiver-db.git" +INSTALL_DIR="/opt/tip-scraper" + +echo "=== TIP Pi Scraper Setup: $PI_NAME ===" + +# ── 1. Node.js 22 ──────────────────────────────────────────────────────────── +if ! command -v node &>/dev/null || [[ "$(node --version)" < "v20" ]]; then + echo "Installing Node.js 22..." + curl -fsSL https://deb.nodesource.com/setup_22.x | sudo bash - + sudo apt-get install -y nodejs +fi +echo "Node: $(node --version)" + +# ── 2. Global tools ─────────────────────────────────────────────────────────── +sudo npm install -g tsx pm2 2>/dev/null || npm install -g tsx pm2 +pm2 startup systemd -u "$USER" --hp "$HOME" | tail -1 | sudo bash || true + +# ── 3. Clone / update repo ─────────────────────────────────────────────────── +if [ -d "$INSTALL_DIR" ]; then + echo "Updating existing repo..." + cd "$INSTALL_DIR" && git pull +else + echo "Cloning from Gitea..." + git clone "$GITEA" "$INSTALL_DIR" +fi +cd "$INSTALL_DIR" + +# ── 4. Install deps (scraper package only, skip Playwright) ────────────────── +cd packages/scraper +npm install --ignore-scripts # --ignore-scripts skips playwright browser download +echo "Dependencies installed" + +# ── 5. .env file ───────────────────────────────────────────────────────────── +cat > "$INSTALL_DIR/.env" < "$INSTALL_DIR/packages/scraper/src/index-pi.ts" <<'PIEOF' +/** + * Pi Scraper Index — lightweight fetch/cheerio only + * No Playwright, no eBay enricher, no heavy compute + * Runs 24/7 on Raspberry Pi nodes + */ +import { config } from "dotenv"; +import { join } from "path"; +config({ path: join(__dirname, "..", "..", "..", ".env") }); + +import PgBoss from "pg-boss"; + +const connectionString = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB}`; + +const PI_QUEUES = [ + "scrape:pricing:fluxlight", + "scrape:pricing:gbics", + "scrape:pricing:optcore", + "scrape:pricing:champion-one", + "scrape:pricing:sfpcables", + "scrape:pricing:blueoptics", + "scrape:pricing:fiber24", + "scrape:pricing:tscom", + "scrape:pricing:skylane", + "scrape:pricing:ascentoptics", + "scrape:pricing:gaotek", + "scrape:catalog:smartoptics", + "scrape:catalog:hubersuhner", + "scrape:news", + "scrape:market-intel", +]; + +async function main() { + console.log(`\n=== TIP Pi Scraper (${process.env.PI_NAME || "pi"}) ===\n`); + + const boss = new PgBoss({ + connectionString, + retryLimit: 2, + retryDelay: 60, + expireInSeconds: 3600, + monitorStateIntervalSeconds: 60, + }); + + boss.on("error", (e) => console.error("pg-boss error:", e)); + await boss.start(); + + for (const q of PI_QUEUES) { + await boss.createQueue(q).catch(() => {}); + } + + // Register workers for all Pi-safe scrapers + const mods: Record = { + "scrape:pricing:fluxlight": "./scrapers/fluxlight", + "scrape:pricing:gbics": "./scrapers/gbics", + "scrape:pricing:optcore": "./scrapers/optcore", + "scrape:pricing:champion-one": "./scrapers/champion-one", + "scrape:pricing:sfpcables": "./scrapers/sfpcables", + "scrape:pricing:blueoptics": "./scrapers/blueoptics", + "scrape:pricing:fiber24": "./scrapers/fiber24", + "scrape:pricing:tscom": "./scrapers/tscom", + "scrape:pricing:skylane": "./scrapers/skylane", + "scrape:pricing:ascentoptics": "./scrapers/ascentoptics", + "scrape:pricing:gaotek": "./scrapers/gaotek", + "scrape:catalog:smartoptics": "./scrapers/smartoptics", + "scrape:catalog:hubersuhner": "./scrapers/hubersuhner", + "scrape:news": "./scrapers/news", + }; + + const fnNames: Record = { + "scrape:pricing:fluxlight": "scrapeFluxlight", + "scrape:pricing:gbics": "scrapeGbics", + "scrape:pricing:optcore": "scrapeOptcore", + "scrape:pricing:champion-one": "scrapeChampionOne", + "scrape:pricing:sfpcables": "scrapeSfpCables", + "scrape:pricing:blueoptics": "scrapeBlueOptics", + "scrape:pricing:fiber24": "scrapeFiber24", + "scrape:pricing:tscom": "scrapeTsCom", + "scrape:pricing:skylane": "scrapeSkylane", + "scrape:pricing:ascentoptics": "scrapeAscentOptics", + "scrape:pricing:gaotek": "scrapeGaoTek", + "scrape:catalog:smartoptics": "scrapeSmartOptics", + "scrape:catalog:hubersuhner": "scrapeHuberSuhner", + "scrape:news": "scrapeNews", + }; + + for (const [queue, modPath] of Object.entries(mods)) { + const mod = await import(modPath); + const fn = mod[fnNames[queue]]; + if (!fn) { console.warn(`No function ${fnNames[queue]} in ${modPath}`); continue; } + await boss.work(queue, async () => { + console.log(`[${new Date().toISOString()}] [${process.env.PI_NAME || "pi"}] Running: ${queue}`); + try { await fn(); } + catch (e) { console.error(`[${queue}] failed:`, String(e).slice(0, 200)); } + }); + } + + // Market intel worker + await boss.work("scrape:market-intel", async () => { + console.log(`[${new Date().toISOString()}] Running: Market intelligence`); + const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence"); + try { await scrapeMarketIntelligence(); } + catch (e) { console.error("market-intel failed:", String(e).slice(0, 200)); } + }); + + console.log(`Pi worker registered for ${PI_QUEUES.length} queues\nWaiting for jobs...\n`); + + process.on("SIGTERM", async () => { await boss.stop(); process.exit(0); }); + process.on("SIGINT", async () => { await boss.stop(); process.exit(0); }); +} + +main().catch((e) => { console.error("Fatal:", e); process.exit(1); }); +PIEOF + +# ── 7. PM2 process ─────────────────────────────────────────────────────────── +cd "$INSTALL_DIR" +PI_NAME="$PI_NAME" pm2 start \ + --name "tip-pi-scraper" \ + --interpreter "$(which tsx)" \ + --cwd "$INSTALL_DIR" \ + packages/scraper/src/index-pi.ts \ + -- \ + || pm2 restart tip-pi-scraper + +pm2 save +echo "" +echo "✅ TIP Pi Scraper ($PI_NAME) is running" +echo " pm2 logs tip-pi-scraper — view logs" +echo " pm2 status — check status" +echo "" +echo "DB target: $DB_HOST:$DB_PORT/$DB_NAME" +echo "Jobs: ${#PI_QUEUES[@]} lightweight scrapers, all day every day"