/** * pg-boss Job Scheduler — 24/7 Continuous Scraping * * ARCHITECTURE: * - Erik (VPS, .82) : Playwright-heavy scrapers (FS.com, 10Gtek, ATGBICS, ProLabs) * + all compatibility + eBay + compute + NAS sync * - Raspberry Pi Fleet : Lightweight fetch/cheerio scrapers run continuously all day * (BlueOptics, Fiber24, T&S Com, Fluxlight, GBICs, Optcore, * Champion ONE, SFPCables, SmartOptics, HUBER+SUHNER, etc.) * * SCHEDULE PHILOSOPHY: * - Playwright scrapers: every 8h (resource-heavy, VPS only) * - Fetch/Cheerio scrapers: every 4h (lightweight, Pi-friendly) * - Catalog scrapers (Flexoptix): every 2h (fast GraphQL, primary price source) * - Compatibility matrices: every 12h (rarely change) * - eBay enrichment: every 6h * - Intelligence/community: every 6h * - Compute jobs: after each pricing wave * - NAS sync: nightly at 07:55 */ import PgBoss from "pg-boss"; import { config } from "dotenv"; import { join } from "path"; import { loadavg } from "os"; // withIsolatedStorage removed — all Crawlee scrapers now use makeCrawleeConfig() // for instance-level storage isolation. See packages/scraper/src/utils/crawlee-config.ts /** * Load-aware guard — skip heavy scrapers when the server is already busy. * Uses the 1-minute load average; maxLoad defaults to 2.5 (50% of 5 vCPUs). * Logs a warning and returns false when load is too high. */ function isLoadAcceptable(maxLoad = 2.5): boolean { const [avg1] = loadavg(); if (avg1 > maxLoad) { console.warn(`[load-guard] 1m load avg ${avg1.toFixed(2)} > ${maxLoad} — deferring heavy scraper`); return false; } return true; } config({ path: join(__dirname, "..", "..", "..", ".env") }); const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`; type EquivalenceProduct = { part_number?: string | null; standard_name?: string | null; form_factor?: string | null; speed_gbps?: number | string | null; fiber_type?: string | null; reach_meters?: number | string | null; wavelengths?: string | null; connector?: string | null; }; type EquivalenceResearchResult = { decision: "approve" | "reject"; confidence: number; basis: string[]; reasons: string[]; rejectReason?: string; }; function normalizeEquivalenceText(value: unknown): string | null { if (value === null || value === undefined) return null; const text = String(value).trim().toUpperCase(); return text.length > 0 ? text : null; } function numericEquivalenceValue(value: unknown): number | null { if (value === null || value === undefined || value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function extractPrimaryNm(wavelengths: unknown): number | null { if (!wavelengths) return null; const match = String(wavelengths).match(/(\d{3,4})/); return match ? parseInt(match[1], 10) : null; } function evaluateEquivalenceResearch( fx: EquivalenceProduct, cp: EquivalenceProduct, hasRecentPrice: boolean, ): EquivalenceResearchResult { const basis: string[] = []; const reasons: string[] = []; let score = 0; const fxForm = normalizeEquivalenceText(fx.form_factor); const cpForm = normalizeEquivalenceText(cp.form_factor); const fxSpeed = numericEquivalenceValue(fx.speed_gbps); const cpSpeed = numericEquivalenceValue(cp.speed_gbps); const fxStandard = normalizeEquivalenceText(fx.standard_name); const cpStandard = normalizeEquivalenceText(cp.standard_name); const fxFiber = normalizeEquivalenceText(fx.fiber_type); const cpFiber = normalizeEquivalenceText(cp.fiber_type); const fxReach = numericEquivalenceValue(fx.reach_meters); const cpReach = numericEquivalenceValue(cp.reach_meters); const fxNm = extractPrimaryNm(fx.wavelengths); const cpNm = extractPrimaryNm(cp.wavelengths); if (!hasRecentPrice) { reasons.push("no recent competitor price observation"); return { decision: "reject", confidence: 0, basis, reasons, rejectReason: "automated research: competitor has no recent price observation", }; } if (fxForm && cpForm && fxForm === cpForm) { score += 25; basis.push("form_factor"); } else { reasons.push("form factor mismatch or missing"); } if (fxSpeed !== null && cpSpeed !== null && fxSpeed === cpSpeed) { score += 20; basis.push("speed_gbps"); } else { reasons.push("speed mismatch or missing"); } if (fxStandard && cpStandard && fxStandard === cpStandard) { score += 30; basis.push("standard_name"); } else { reasons.push("standard name not identical"); } if (fxNm !== null && cpNm !== null) { if (Math.abs(fxNm - cpNm) <= 15) { score += 20; basis.push(`wavelength_${fxNm}nm`); } else { reasons.push(`wavelength mismatch ${fxNm}nm vs ${cpNm}nm`); score -= 20; } } else { reasons.push("wavelength missing"); } if (fxFiber && cpFiber) { if (fxFiber === cpFiber) { score += 10; basis.push("fiber_type"); } else { reasons.push(`fiber mismatch ${fxFiber} vs ${cpFiber}`); score -= 15; } } else { reasons.push("fiber type missing"); } if (fxReach !== null && cpReach !== null && fxReach > 0 && cpReach > 0) { const ratio = Math.min(fxReach, cpReach) / Math.max(fxReach, cpReach); if (ratio >= 0.85) { score += 10; basis.push("reach"); } else { reasons.push(`reach mismatch ${fxReach}m vs ${cpReach}m`); score -= 15; } } else { reasons.push("reach missing"); } const confidence = Math.max(0, Math.min(1, score / 115)); const criticalMismatch = reasons.some((reason) => reason.startsWith("wavelength mismatch") || reason.startsWith("fiber mismatch") || reason.startsWith("reach mismatch") || reason.startsWith("form factor mismatch") || reason.startsWith("speed mismatch") ); const missingCriticalEvidence = reasons.some((reason) => reason === "wavelength missing" || reason === "fiber type missing" || reason === "reach missing" ); if (criticalMismatch) { return { decision: "reject", confidence, basis, reasons, rejectReason: `automated research: technical mismatch (${reasons.join("; ")})`, }; } if (missingCriticalEvidence) { return { decision: "reject", confidence, basis, reasons, rejectReason: `automated research: insufficient technical evidence (${reasons.join("; ")})`, }; } if (confidence >= 0.73) { return { decision: "approve", confidence, basis, reasons }; } return { decision: "reject", confidence, basis, reasons, rejectReason: `automated research: confidence ${confidence.toFixed(3)} below approval threshold`, }; } export async function createScheduler(): Promise { const boss = new PgBoss({ connectionString, retryLimit: 3, retryDelay: 30, retryBackoff: true, expireInSeconds: 300, monitorStateIntervalSeconds: 60, }); boss.on("error", (error) => console.error("pg-boss error:", error)); await boss.start(); console.log("pg-boss scheduler started"); return boss; } export async function registerSchedules(boss: PgBoss): Promise { // pg-boss v10: boss.schedule() requires the queue to already exist in pgboss.queue. // After a DB reset (e.g. server outage), all queue rows are wiped. // Patch boss.schedule to auto-create queues idempotently before each schedule call, // so the 277 individual schedule() calls below don't need to be touched. const _origSchedule = boss.schedule.bind(boss) as typeof boss.schedule; (boss as unknown as Record).schedule = async ( name: string, cron: string, data?: unknown, opts?: unknown, ) => { await boss.createQueue(name).catch(() => { /* already exists */ }); return _origSchedule(name, cron, data as object, opts as object); }; const queues = [ // ── Playwright scrapers (Erik, every 8h) ─────────────────────────── "scrape:pricing:fs", "scrape:pricing:10gtek", "scrape:pricing:atgbics", "scrape:pricing:prolabs", // ── Fetch/Cheerio scrapers (Pi-friendly, every 4h) ───────────────── "scrape:pricing:fluxlight", "scrape:pricing:gbics", "scrape:pricing:optcore", "scrape:pricing:champion-one", "scrape:pricing:sfpcables", "scrape:pricing:blueoptics", "scrape:pricing:fiber24", "scrape:pricing:tscom", "scrape:pricing:skylane", "scrape:pricing:ascentoptics", "scrape:pricing:gaotek", // ── Catalog scrapers (every 2h) ──────────────────────────────────── "scrape:pricing:flexoptix", // ── Manufacturer catalogs (every 8h, no prices) ──────────────────── "scrape:catalog:smartoptics", "scrape:catalog:hubersuhner", "scrape:catalog:eoptolink", // ── Vendor lists ─────────────────────────────────────────────────── "scrape:vendors:flexoptix", "scrape:vendors:flexoptix-supported", // ── Compatibility (every 12h) ────────────────────────────────────── "scrape:compat:flexoptix", "scrape:compat:cisco", "scrape:compat:juniper", "scrape:compat:sonic", "scrape:compat:ufispace", "scrape:compat:edgecore", // ── Switch enrichment (every 12h) ───────────────────────────────── "scrape:assets:switches", // ── Switch og:image fetcher (daily, after switch-assets) ────────── "scrape:images:switches", // ── Playwright image fetcher for bot-blocked vendors (every 3d) ─── "scrape:images:switches:playwright", // ── eBay enrichment (every 6h) ──────────────────────────────────── "enrich:ebay-transceivers", "enrich:ebay-switches", // ── Intelligence & community (every 6h) ─────────────────────────── "scrape:market-intel", "scrape:nog-talks", "scrape:community-issues", "scrape:datasheet-links", "scrape:news", "scrape:faq", "scrape:docs", // ── Compute (every 4h, after pricing waves) ─────────────────────── "compute:abc", "compute:reorder-signals", // ── New form-factor coverage scrapers (every 8h) ────────────────── "scrape:pricing:comms-express", "scrape:pricing:router-switch", "scrape:pricing:multimode-inc", "scrape:pricing:optictransceiver", "scrape:pricing:wiitek", // ── Fetch-based catalog+pricing scrapers (every 2h) ────────────── "scrape:pricing:naddod", "scrape:pricing:qsfptek", "scrape:pricing:addon", "scrape:pricing:fibermall", "scrape:pricing:vcelink", "scrape:pricing:opticsbay", // ── OEM Reference Prices (Mouser API, once daily) ───────────────── "scrape:pricing:mouser-oem", // ── Prediction Signal Scrapers (new) ────────────────────────────── "scrape:signals:sec-edgar", "scrape:signals:github", "scrape:signals:ebay-velocity", "scrape:signals:ai-clusters", "scrape:signals:distributor-leads", "scrape:signals:standards", // ── Forecast Engine ─────────────────────────────────────────────── "compute:forecast", // ── Hype Cycle Engine (Norton-Bass, daily) ──────────────────────── "compute:hype-cycle", // ── Sync ────────────────────────────────────────────────────────── "sync:nas", // ── Health Monitoring ───────────────────────────────────────────── "monitor:scraper-health", // ── Price denormalization refresh ───────────────────────────────── "compute:price-denorm", // ── Verification Reconciliation ─────────────────────────────────── "maintenance:reconcile-verification", // ── Competitor Equivalence Matching ─────────────────────────────── "maintenance:find-equivalences", // ── Re-Research approved equivalences ───────────────────────────── "maintenance:re-research-equivalences", // ── Vendor Discovery Crawlers (TIPLLM training data + DB seeding) ───── "discover:vendor:cisco-tmg", "discover:vendor:juniper", "discover:vendor:arista", "discover:vendor:fs-com", "discover:vendor:flexoptix", "discover:vendor:nokia", "discover:vendor:huawei", "discover:vendor:ii-vi", ]; for (const q of queues) { await boss.createQueue(q).catch(() => { /* already exists */ }); } // ══════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════ // ALL PRICING SCRAPERS — 24/7, every 2h, staggered by 10min // Goal: complete competitor coverage, no gaps, database always fresh // ══════════════════════════════════════════════════════════════════════ // Playwright scrapers (resource-heavy) — every 2h, 10min apart await boss.schedule("scrape:pricing:fs", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 }); await boss.schedule("scrape:pricing:10gtek", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:atgbics", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:prolabs", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // Fetch/Cheerio scrapers (lightweight) — every 2h, 5min apart await boss.schedule("scrape:pricing:fluxlight", "0 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:gbics", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:optcore", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:champion-one", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:sfpcables", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:blueoptics", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:fiber24", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:tscom", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:skylane", "40 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:ascentoptics", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:gaotek", "50 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // Form-factor coverage scrapers — every 2h await boss.schedule("scrape:pricing:comms-express", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 }); await boss.schedule("scrape:pricing:router-switch", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 }); await boss.schedule("scrape:pricing:multimode-inc", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:optictransceiver", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:wiitek", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // Fetch-based scrapers running on Erik — every 2h, end-of-cycle slots await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:fibermall", "57 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // OEM reference prices via Mouser API — once daily at 03:00 (slow: 2s/PID × 475 PIDs ≈ 16min) // Requires MOUSER_API_KEY env var (free at mouser.com/api-hub) await boss.schedule("scrape:pricing:mouser-oem", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // FLEXOPTIX CATALOG — every 2h (primary price source) // ══════════════════════════════════════════════════════════════════════ await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // MANUFACTURER CATALOGS — every 4h (product data, no prices) // ══════════════════════════════════════════════════════════════════════ await boss.schedule("scrape:catalog:smartoptics", "10 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hubersuhner", "25 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:eoptolink", "40 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // OEM vendor seed catalogs — daily at 04:00+ (stable data, rarely changes) await boss.schedule("scrape:catalog:arista-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:juniper-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hpe-aruba-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nokia-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:huawei-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:dell-emc-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:extreme-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nvidia-mellanox-oem","45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:brocade-ruckus-oem","0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:zte-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ciena-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ericsson-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:fortinet-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:palo-alto-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:f5-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ubiquiti-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:mikrotik-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:netgear-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:adtran-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:calix-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:checkpoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:allied-telesis-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:zyxel-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:dlink-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ale-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ribbon-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:infinera-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:tplink-oem", "45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cambium-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:moxa-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:lumentum-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:coherent-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:brocade-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ruggedcom-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hirschmann-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:viavi-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:westermo-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:lancom-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:dzs-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:watchguard-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:sonicwall-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:advantech-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:redlion-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:exfo-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:mellanox-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:intel-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:antaira-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:transition-networks-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:perle-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:barracuda-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:etherwan-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:blackbox-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:packetlight-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:spirent-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:comnet-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:korenix-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:sophos-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cradlepoint-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:fujitsu-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nec-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:arris-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hillstone-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:vecima-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:stordis-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:keysight-oem", "0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:sycamore-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ekinops-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:adva-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:coriant-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:casa-systems-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:harmonic-oem", "30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:solarflare-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:marvell-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:broadcom-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:calix-access-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ribbon-comms-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:infinera-groove-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ciena-waveserver-oem","15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:commscope-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:teleste-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:tejas-networks-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ericsson-transport-oem","15 0 * * *",{}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:adtran-ta-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:isolan-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:telco-systems-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:rad-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:comtrend-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:packetfront-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:edgewater-networks-oem","0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:corning-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ofs-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:kontron-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ipinfusion-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:telrad-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:siklu-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ceragon-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:datang-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:viptela-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:versa-networks-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:vmware-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cimc-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:qlogic-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:emulex-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:netapp-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:pure-storage-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hpe-storage-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ibm-storage-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:dell-storage-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hitachi-vantara-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:aws-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:azure-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:google-cloud-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:meta-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nokia-access-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:huawei-access-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:zte-access-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:calix-gigapoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:samsung-networks-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nokia-airscale-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ericsson-ran-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:mavenir-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ixia-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:exfo-network-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cumulus-networks-oem","45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:sonic-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:h3c-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ruijie-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:centec-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:supermicro-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cisco-meraki-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cisco-catalyst-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cisco-nexus-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cisco-asr-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:juniper-mx-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:juniper-qfx-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:aruba-cx-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:extreme-campus-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:arista-7000-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:pica8-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:pluribus-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:drivenets-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:phoenix-contact-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:beckhoff-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:omron-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:abb-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:siemens-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:schneider-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:rockwell-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:belden-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:ge-grid-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:schweitzer-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:moxa-industrial-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cisco-ie-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Media / Broadcast / Pro-AV ──────────────────────────────────────── await boss.schedule("scrape:catalog:evertz-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:grass-valley-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:haivision-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:viasat-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Asian Optical Vendors ───────────────────────────────────────────── await boss.schedule("scrape:catalog:fiberhome-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:oplink-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:accelink-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:hisense-broadband-oem","0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Optical Transceiver Manufacturers (batch 24–25) ─────────────────── await boss.schedule("scrape:catalog:ii-vi-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:source-photonics-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:o-net-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:innolight-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:aoi-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:sumitomo-electric-oem","30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:neophotonics-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Test & Measurement / Defense / Telecom Access (batch 29–30) ──────── await boss.schedule("scrape:catalog:audiocodes-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:anritsu-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:netscout-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:curtiss-wright-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:eci-telecom-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:rohde-schwarz-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:l3harris-oem", "15 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:zhone-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:cambium-networks-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:tektronix-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:clearfield-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:lanner-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:black-box-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:radiflow-oem", "0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:dragonwave-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:teledyne-lecroy-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:finisar-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:acacia-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:inphi-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Batch 35: Sierra Wireless, Senao, EMCORE, Reflex Photonics ──────── await boss.schedule("scrape:catalog:sierra-wireless-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:senao-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:emcore-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:reflex-photonics-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Batch 36: EnGenius, Palo Alto Networks, Brocade (Legacy), Foundry ─ await boss.schedule("scrape:catalog:engenius-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:paloalto-networks-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:brocade-legacy-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:foundry-networks-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ── Batch 37: Extreme (Legacy), Nortel, 3Com, Avaya ────────────────── await boss.schedule("scrape:catalog:extreme-legacy-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:nortel-legacy-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:3com-legacy-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:catalog:avaya-legacy-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // VENDOR DISCOVERY CRAWLERS — daily, permanent 24/7 rotation // Each run: crawls catalog → LLM extract → spec validate → DB + Gitea SFT // 8 vendors × 3h stagger = full rotation every 24h, no overlap // ══════════════════════════════════════════════════════════════════════ await boss.schedule("discover:vendor:cisco-tmg", "0 20 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:juniper", "0 22 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:arista", "0 0 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:fs-com", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:flexoptix", "0 4 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:nokia", "0 6 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:huawei", "0 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("discover:vendor:ii-vi", "0 10 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); // ══════════════════════════════════════════════════════════════════════ // VENDOR LISTS — every 12h // ══════════════════════════════════════════════════════════════════════ await boss.schedule("scrape:vendors:flexoptix", "0 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); await boss.schedule("scrape:vendors:flexoptix-supported", "15 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); // ══════════════════════════════════════════════════════════════════════ // COMPATIBILITY MATRICES — every 12h // ══════════════════════════════════════════════════════════════════════ // Flexoptix compatibility — every 24h at 09:00 (after both switch-assets + image-fetcher) await boss.schedule("scrape:compat:flexoptix", "0 9 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:compat:ufispace", "45 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); await boss.schedule("scrape:compat:edgecore", "55 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); // ══════════════════════════════════════════════════════════════════════ // SWITCH ASSETS — every 12h // ══════════════════════════════════════════════════════════════════════ await boss.schedule("scrape:assets:switches", "30 7,19 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); // og:image fetcher: daily at 08:30, after switch-assets completes at 07:30 await boss.schedule("scrape:images:switches", "30 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 }); // Playwright image scraper for bot-blocked vendors (Arista/Dell/Edgecore/Fortinet/Extreme) // Every 3 days at 09:00 — Playwright is slower and heavier than plain HTTP await boss.schedule("scrape:images:switches:playwright", "0 9 */3 * *", {}, { retryLimit: 1, expireInSeconds: 10800 }); // ══════════════════════════════════════════════════════════════════════ // EBAY ENRICHMENT — every 6h // ══════════════════════════════════════════════════════════════════════ await boss.schedule("enrich:ebay-transceivers", "0 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 }); await boss.schedule("enrich:ebay-switches", "30 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 }); // ══════════════════════════════════════════════════════════════════════ // INTELLIGENCE & COMMUNITY — every 6h // ══════════════════════════════════════════════════════════════════════ await boss.schedule("scrape:market-intel", "0 2,8,14,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // NOG conference talks — weekly on Mondays 06:00 UTC await boss.schedule("scrape:nog-talks", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 7200 }); await boss.schedule("scrape:community-issues", "30 2,8,14,20 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); await boss.schedule("scrape:datasheet-links", "0 3,9,15,21 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); await boss.schedule("scrape:news", "20 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); await boss.schedule("scrape:faq", "40 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); await boss.schedule("scrape:docs", "50 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // COMPUTE JOBS — every 4h (after pricing waves settle) // ══════════════════════════════════════════════════════════════════════ await boss.schedule("compute:abc", "50 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 }); await boss.schedule("compute:reorder-signals", "55 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 }); // Price denorm refresh: daily 05:30 UTC after overnight scraping waves settle await boss.schedule("compute:price-denorm", "30 5 * * *", {}, { retryLimit: 1, expireInSeconds: 600 }); // ══════════════════════════════════════════════════════════════════════ // PREDICTION SIGNAL SCRAPERS // ══════════════════════════════════════════════════════════════════════ // SEC EDGAR CapEx — weekly Monday 06:00 (filings don't change that fast) await boss.schedule("scrape:signals:sec-edgar", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 3600 }); // GitHub signals — weekly Sunday 05:00 await boss.schedule("scrape:signals:github", "0 5 * * 0", {}, { retryLimit: 2, expireInSeconds: 7200 }); // eBay sold velocity — every 12h (fast-moving market signal) await boss.schedule("scrape:signals:ebay-velocity", "0 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // AI cluster RSS feeds — every 4h (news moves fast) await boss.schedule("scrape:signals:ai-clusters", "10 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 }); // Distributor lead times — daily 03:30 (stock changes overnight) await boss.schedule("scrape:signals:distributor-leads","30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 }); // Standards tracker — weekly Wednesday 04:00 (standards move slowly) await boss.schedule("scrape:signals:standards", "0 4 * * 3", {}, { retryLimit: 1, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // FORECAST ENGINE — daily at 08:00 (after all nightly scrapers done) // ══════════════════════════════════════════════════════════════════════ await boss.schedule("compute:forecast", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 600 }); // Hype Cycle Engine runs daily at 04:30 (after Mouser OEM scraper at 03:00) await boss.schedule("compute:hype-cycle", "30 4 * * *", {}, { retryLimit: 1, expireInSeconds: 600 }); // ══════════════════════════════════════════════════════════════════════ // NAS SYNC — nightly // ══════════════════════════════════════════════════════════════════════ await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 }); // Health check: every 3h — warns if any vendor has no new prices recently await boss.schedule("monitor:scraper-health", "17 */3 * * *", {}, { retryLimit: 1, expireInSeconds: 600 }); // Verification reconciliation: nightly at 01:00 UTC // Resets competitor_verified/fully_verified for any transceiver that no longer // has a real non-Flexoptix price in the last 30 days — prevents stale ★ 100% badges await boss.schedule("maintenance:reconcile-verification", "0 1 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 }); // Equivalence matching: nightly at 02:00 UTC (after reconcile) await boss.schedule("maintenance:find-equivalences", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); // Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); console.log("All schedules registered — 24/7 continuous scraping (60 jobs)"); } export async function registerWorkers(boss: PgBoss): Promise { // Lazy-load all scrapers const { scrapeFs } = await import("./scrapers/fs-com"); const { scrapeCiscoTmg } = await import("./scrapers/cisco-tmg"); const { scrapeSmartOptics } = await import("./scrapers/smartoptics"); const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner"); const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence"); const { scrapeNews } = await import("./scrapers/news"); const { scrape10Gtek } = await import("./scrapers/tenGtek"); const { scrapeFlexoptixCatalog } = await import("./scrapers/flexoptix-catalog"); const { scrapeFlexoptixVendors } = await import("./scrapers/flexoptix-vendors"); const { seedFlexoptixVendors } = await import("./scrapers/flexoptix-supported-vendors"); const { scrapeAtgbics } = await import("./scrapers/atgbics"); const { scrapeProLabs } = await import("./scrapers/prolabs"); const { scrapeJuniperHct } = await import("./scrapers/juniper-hct"); const { scrapeSonicHcl } = await import("./scrapers/sonic-hcl"); const { scrapeUfiSpace } = await import("./scrapers/ufispace"); const { scrapeEdgecore } = await import("./scrapers/edgecore"); const { scrapeSwitchAssets } = await import("./scrapers/switch-assets"); const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher"); const { fetchSwitchImagesPlaywright } = await import("./scrapers/switch-image-playwright"); const { scrapeFlexoptixCompatibility } = await import("./scrapers/flexoptix-compat"); // ── Prediction signal scrapers ──────────────────────────────────────── const { scrapeSecEdgar } = await import("./scrapers/sec-edgar"); const { scrapeGithubSignals } = await import("./scrapers/github-signals"); const { scrapeEbayVelocity } = await import("./scrapers/ebay-velocity"); const { scrapeAiClusters } = await import("./scrapers/ai-clusters"); const { scrapeDistributorLeads }= await import("./scrapers/distributor-leads"); const { scrapeStandardsTracker }= await import("./scrapers/standards-tracker"); const { runForecastEngine } = await import("./utils/forecast-engine"); // ── Playwright scrapers ─────────────────────────────────────────────── await boss.work("scrape:pricing:fs", async () => { // FS.com uses Playwright + Cloudflare bypass. On datacenter servers the // datacenter IP is blocked by Cloudflare WAF. Set SKIP_FS_SCRAPER=true to // skip on Erik; the Mac launchd cron handles FS.com from a residential IP. if (process.env["SKIP_FS_SCRAPER"] === "true") { console.log(`[${new Date().toISOString()}] FS.com pricing: SKIPPED (SKIP_FS_SCRAPER=true)`); return; } console.log(`[${new Date().toISOString()}] Running: FS.com pricing`); await scrapeFs(); }); await boss.work("scrape:pricing:10gtek", async () => { console.log(`[${new Date().toISOString()}] Running: 10Gtek pricing`); await scrape10Gtek(); }); await boss.work("scrape:pricing:atgbics", async () => { console.log(`[${new Date().toISOString()}] Running: ATGBICS pricing`); await scrapeAtgbics(); }); await boss.work("scrape:pricing:prolabs", async () => { console.log(`[${new Date().toISOString()}] Running: ProLabs pricing`); await scrapeProLabs(); }); // ── Lightweight fetch/cheerio scrapers ─────────────────────────────── await boss.work("scrape:pricing:fluxlight", async () => { console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`); const { scrapeFluxlight } = await import("./scrapers/fluxlight"); await scrapeFluxlight(); }); await boss.work("scrape:pricing:gbics", async () => { console.log(`[${new Date().toISOString()}] Running: GBICS pricing`); const { scrapeGbics } = await import("./scrapers/gbics"); await scrapeGbics(); }); await boss.work("scrape:pricing:optcore", async () => { // Optcore.net WP REST API is blocked by Cloudflare WAF for Erik's datacenter // IP (82.165.222.127). Set SKIP_OPTCORE_SCRAPER=true to suppress the wasted // run. A residential IP (Mac launchd) would be needed to scrape Optcore. if (process.env["SKIP_OPTCORE_SCRAPER"] === "true") { console.log(`[${new Date().toISOString()}] Optcore pricing: SKIPPED (SKIP_OPTCORE_SCRAPER=true)`); return; } console.log(`[${new Date().toISOString()}] Running: Optcore pricing`); const { scrapeOptcore } = await import("./scrapers/optcore"); await scrapeOptcore(); }); await boss.work("scrape:pricing:champion-one", async () => { console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`); const { scrapeChampionOne } = await import("./scrapers/champion-one"); await scrapeChampionOne(); }); await boss.work("scrape:pricing:sfpcables", async () => { console.log(`[${new Date().toISOString()}] Running: SFPcables pricing`); const { scrapeSfpCables } = await import("./scrapers/sfpcables"); await scrapeSfpCables(); }); await boss.work("scrape:pricing:blueoptics", async () => { console.log(`[${new Date().toISOString()}] Running: BlueOptics pricing`); const { scrapeBlueOptics } = await import("./scrapers/blueoptics"); await scrapeBlueOptics(); }); await boss.work("scrape:pricing:fiber24", async () => { console.log(`[${new Date().toISOString()}] Running: Fiber24 pricing`); const { scrapeFiber24 } = await import("./scrapers/fiber24"); await scrapeFiber24(); }); await boss.work("scrape:pricing:tscom", async () => { console.log(`[${new Date().toISOString()}] Running: T&S Communication pricing`); const { scrapeTsCom } = await import("./scrapers/tscom"); await scrapeTsCom(); }); await boss.work("scrape:pricing:skylane", async () => { console.log(`[${new Date().toISOString()}] Running: Skylane pricing`); const { scrapeSkylane } = await import("./scrapers/skylane"); await scrapeSkylane(); }); await boss.work("scrape:pricing:ascentoptics", async () => { console.log(`[${new Date().toISOString()}] Running: Ascent Optics pricing`); const { scrapeAscentOptics } = await import("./scrapers/ascentoptics"); await scrapeAscentOptics(); }); await boss.work("scrape:pricing:gaotek", async () => { console.log(`[${new Date().toISOString()}] Running: GAO Tek pricing`); const { scrapeGaoTek } = await import("./scrapers/gaotek"); await scrapeGaoTek(); }); // ── Catalog scrapers ────────────────────────────────────────────────── await boss.work("scrape:pricing:flexoptix", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`); await scrapeFlexoptixCatalog(); }); await boss.work("scrape:catalog:smartoptics", async () => { console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`); await scrapeSmartOptics(); }); await boss.work("scrape:catalog:hubersuhner", async () => { console.log(`[${new Date().toISOString()}] Running: HUBER+SUHNER catalog`); await scrapeHuberSuhner(); }); await boss.work("scrape:catalog:eoptolink", async () => { console.log(`[${new Date().toISOString()}] Running: Eoptolink OEM catalog`); const { scrapeEoptolink } = await import("./scrapers/eoptolink"); await scrapeEoptolink(); }); await boss.work("scrape:catalog:arista-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Arista OEM catalog seed`); const { scrapeAristaOem } = await import("./scrapers/arista-oem"); await scrapeAristaOem(); }); await boss.work("scrape:catalog:juniper-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Juniper OEM catalog seed`); const { scrapeJuniperOem } = await import("./scrapers/juniper-oem"); await scrapeJuniperOem(); }); await boss.work("scrape:catalog:hpe-aruba-oem", async () => { console.log(`[${new Date().toISOString()}] Running: HPE/Aruba OEM catalog seed`); const { scrapeHpeArubaOem } = await import("./scrapers/hpe-aruba-oem"); await scrapeHpeArubaOem(); }); await boss.work("scrape:catalog:nokia-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Nokia OEM catalog seed`); const { scrapeNokiaOem } = await import("./scrapers/nokia-oem"); await scrapeNokiaOem(); }); await boss.work("scrape:catalog:huawei-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Huawei OEM catalog seed`); const { scrapeHuaweiOem } = await import("./scrapers/huawei-oem"); await scrapeHuaweiOem(); }); await boss.work("scrape:catalog:dell-emc-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Dell EMC OEM catalog seed`); const { scrapeDellEmcOem } = await import("./scrapers/dell-emc-oem"); await scrapeDellEmcOem(); }); await boss.work("scrape:catalog:extreme-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Extreme Networks OEM catalog seed`); const { scrapeExtremeOem } = await import("./scrapers/extreme-oem"); await scrapeExtremeOem(); }); await boss.work("scrape:catalog:nvidia-mellanox-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NVIDIA/Mellanox OEM catalog seed`); const { scrapeNvidiaMellanoxOem } = await import("./scrapers/nvidia-mellanox-oem"); await scrapeNvidiaMellanoxOem(); }); await boss.work("scrape:catalog:brocade-ruckus-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Brocade/RUCKUS OEM catalog seed`); const { scrapeBrocadeRuckusOem } = await import("./scrapers/brocade-ruckus-oem"); await scrapeBrocadeRuckusOem(); }); await boss.work("scrape:catalog:zte-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ZTE OEM catalog seed`); const { scrapeZteOem } = await import("./scrapers/zte-oem"); await scrapeZteOem(); }); await boss.work("scrape:catalog:ciena-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ciena OEM catalog seed`); const { scrapeCienaOem } = await import("./scrapers/ciena-oem"); await scrapeCienaOem(); }); await boss.work("scrape:catalog:ericsson-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ericsson OEM catalog seed`); const { scrapeEricssonOem } = await import("./scrapers/ericsson-oem"); await scrapeEricssonOem(); }); await boss.work("scrape:catalog:fortinet-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Fortinet OEM catalog seed`); const { scrapeFortinetOem } = await import("./scrapers/fortinet-oem"); await scrapeFortinetOem(); }); await boss.work("scrape:catalog:palo-alto-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`); const { scrapePaloAltoOem } = await import("./scrapers/palo-alto-oem"); await scrapePaloAltoOem(); }); await boss.work("scrape:catalog:f5-oem", async () => { console.log(`[${new Date().toISOString()}] Running: F5 Networks OEM catalog seed`); const { scrapeF5Oem } = await import("./scrapers/f5-oem"); await scrapeF5Oem(); }); await boss.work("scrape:catalog:ubiquiti-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ubiquiti OEM catalog seed`); const { scrapeUbiquitiOem } = await import("./scrapers/ubiquiti-oem"); await scrapeUbiquitiOem(); }); await boss.work("scrape:catalog:mikrotik-oem", async () => { console.log(`[${new Date().toISOString()}] Running: MikroTik OEM catalog seed`); const { scrapeMikrotikOem } = await import("./scrapers/mikrotik-oem"); await scrapeMikrotikOem(); }); await boss.work("scrape:catalog:netgear-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Netgear OEM catalog seed`); const { scrapeNetgearOem } = await import("./scrapers/netgear-oem"); await scrapeNetgearOem(); }); await boss.work("scrape:catalog:adtran-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ADTRAN OEM catalog seed`); const { scrapeAdtranOem } = await import("./scrapers/adtran-oem"); await scrapeAdtranOem(); }); await boss.work("scrape:catalog:calix-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Calix OEM catalog seed`); const { scrapeCalixOem } = await import("./scrapers/calix-oem"); await scrapeCalixOem(); }); await boss.work("scrape:catalog:checkpoint-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Check Point OEM catalog seed`); const { scrapeCheckpointOem } = await import("./scrapers/checkpoint-oem"); await scrapeCheckpointOem(); }); await boss.work("scrape:catalog:allied-telesis-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Allied Telesis OEM catalog seed`); const { scrapeAlliedTelesisOem } = await import("./scrapers/allied-telesis-oem"); await scrapeAlliedTelesisOem(); }); await boss.work("scrape:catalog:zyxel-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Zyxel OEM catalog seed`); const { scrapeZyxelOem } = await import("./scrapers/zyxel-oem"); await scrapeZyxelOem(); }); await boss.work("scrape:catalog:dlink-oem", async () => { console.log(`[${new Date().toISOString()}] Running: D-Link OEM catalog seed`); const { scrapeDlinkOem } = await import("./scrapers/dlink-oem"); await scrapeDlinkOem(); }); await boss.work("scrape:catalog:ale-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Alcatel-Lucent Enterprise OEM catalog seed`); const { scrapeAleOem } = await import("./scrapers/ale-oem"); await scrapeAleOem(); }); await boss.work("scrape:catalog:ribbon-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`); const { scrapeRibbonOem } = await import("./scrapers/ribbon-oem"); await scrapeRibbonOem(); }); await boss.work("scrape:catalog:infinera-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Infinera OEM catalog seed`); const { scrapeInfineraOem } = await import("./scrapers/infinera-oem"); await scrapeInfineraOem(); }); await boss.work("scrape:catalog:tplink-oem", async () => { console.log(`[${new Date().toISOString()}] Running: TP-Link OEM catalog seed`); const { scrapeTplinkOem } = await import("./scrapers/tplink-oem"); await scrapeTplinkOem(); }); await boss.work("scrape:catalog:cambium-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`); const { scrapeCambiumOem } = await import("./scrapers/cambium-oem"); await scrapeCambiumOem(); }); await boss.work("scrape:catalog:moxa-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Moxa OEM catalog seed`); const { scrapeMoxaOem } = await import("./scrapers/moxa-oem"); await scrapeMoxaOem(); }); await boss.work("scrape:catalog:lumentum-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Lumentum OEM catalog seed`); const { scrapeLumentumOem } = await import("./scrapers/lumentum-oem"); await scrapeLumentumOem(); }); await boss.work("scrape:catalog:coherent-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Coherent Corp OEM catalog seed`); const { scrapeCoherentOem } = await import("./scrapers/coherent-oem"); await scrapeCoherentOem(); }); await boss.work("scrape:catalog:brocade-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Brocade OEM catalog seed`); const { scrapeBrocadeOem } = await import("./scrapers/brocade-oem"); await scrapeBrocadeOem(); }); await boss.work("scrape:catalog:ruggedcom-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Siemens RUGGEDCOM OEM catalog seed`); const { scrapeRuggedcomOem } = await import("./scrapers/ruggedcom-oem"); await scrapeRuggedcomOem(); }); await boss.work("scrape:catalog:hirschmann-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Hirschmann OEM catalog seed`); const { scrapeHirschmannOem } = await import("./scrapers/hirschmann-oem"); await scrapeHirschmannOem(); }); await boss.work("scrape:catalog:viavi-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Viavi Solutions OEM catalog seed`); const { scrapeViaviOem } = await import("./scrapers/viavi-oem"); await scrapeViaviOem(); }); await boss.work("scrape:catalog:westermo-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Westermo OEM catalog seed`); const { scrapeWestermoOem } = await import("./scrapers/westermo-oem"); await scrapeWestermoOem(); }); await boss.work("scrape:catalog:lancom-oem", async () => { console.log(`[${new Date().toISOString()}] Running: LANCOM Systems OEM catalog seed`); const { scrapeLancomOem } = await import("./scrapers/lancom-oem"); await scrapeLancomOem(); }); await boss.work("scrape:catalog:dzs-oem", async () => { console.log(`[${new Date().toISOString()}] Running: DZS OEM catalog seed`); const { scrapeDzsOem } = await import("./scrapers/dzs-oem"); await scrapeDzsOem(); }); await boss.work("scrape:catalog:watchguard-oem", async () => { console.log(`[${new Date().toISOString()}] Running: WatchGuard OEM catalog seed`); const { scrapeWatchguardOem } = await import("./scrapers/watchguard-oem"); await scrapeWatchguardOem(); }); await boss.work("scrape:catalog:sonicwall-oem", async () => { console.log(`[${new Date().toISOString()}] Running: SonicWall OEM catalog seed`); const { scrapeSonicwallOem } = await import("./scrapers/sonicwall-oem"); await scrapeSonicwallOem(); }); await boss.work("scrape:catalog:advantech-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Advantech OEM catalog seed`); const { scrapeAdvantechOem } = await import("./scrapers/advantech-oem"); await scrapeAdvantechOem(); }); await boss.work("scrape:catalog:redlion-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Red Lion Controls OEM catalog seed`); const { scrapeRedlionOem } = await import("./scrapers/redlion-oem"); await scrapeRedlionOem(); }); await boss.work("scrape:catalog:exfo-oem", async () => { console.log(`[${new Date().toISOString()}] Running: EXFO OEM catalog seed`); const { scrapeExfoOem } = await import("./scrapers/exfo-oem"); await scrapeExfoOem(); }); await boss.work("scrape:catalog:mellanox-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NVIDIA Mellanox OEM catalog seed`); const { scrapeMellanoxOem } = await import("./scrapers/mellanox-oem"); await scrapeMellanoxOem(); }); await boss.work("scrape:catalog:intel-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Intel OEM catalog seed`); const { scrapeIntelOem } = await import("./scrapers/intel-oem"); await scrapeIntelOem(); }); await boss.work("scrape:catalog:antaira-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Antaira Technologies OEM catalog seed`); const { scrapeAntairaOem } = await import("./scrapers/antaira-oem"); await scrapeAntairaOem(); }); await boss.work("scrape:catalog:transition-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Transition Networks OEM catalog seed`); const { scrapeTransitionNetworksOem } = await import("./scrapers/transition-networks-oem"); await scrapeTransitionNetworksOem(); }); await boss.work("scrape:catalog:perle-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Perle Systems OEM catalog seed`); const { scrapePerleOem } = await import("./scrapers/perle-oem"); await scrapePerleOem(); }); await boss.work("scrape:catalog:barracuda-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Barracuda Networks OEM catalog seed`); const { scrapeBarracudaOem } = await import("./scrapers/barracuda-oem"); await scrapeBarracudaOem(); }); await boss.work("scrape:catalog:etherwan-oem", async () => { console.log(`[${new Date().toISOString()}] Running: EtherWAN Systems OEM catalog seed`); const { scrapeEtherwanOem } = await import("./scrapers/etherwan-oem"); await scrapeEtherwanOem(); }); await boss.work("scrape:catalog:blackbox-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`); const { scrapeBlackboxOem } = await import("./scrapers/blackbox-oem"); await scrapeBlackboxOem(); }); await boss.work("scrape:catalog:packetlight-oem", async () => { console.log(`[${new Date().toISOString()}] Running: PacketLight Networks OEM catalog seed`); const { scrapePacketLightOem } = await import("./scrapers/packetlight-oem"); await scrapePacketLightOem(); }); await boss.work("scrape:catalog:spirent-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Spirent Communications OEM catalog seed`); const { scrapeSpirentOem } = await import("./scrapers/spirent-oem"); await scrapeSpirentOem(); }); await boss.work("scrape:catalog:comnet-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Comnet OEM catalog seed`); const { scrapeComnetOem } = await import("./scrapers/comnet-oem"); await scrapeComnetOem(); }); await boss.work("scrape:catalog:korenix-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Korenix Technology OEM catalog seed`); const { scrapeKorenixOem } = await import("./scrapers/korenix-oem"); await scrapeKorenixOem(); }); await boss.work("scrape:catalog:sophos-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Sophos OEM catalog seed`); const { scrapeSophosOem } = await import("./scrapers/sophos-oem"); await scrapeSophosOem(); }); await boss.work("scrape:catalog:cradlepoint-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cradlepoint OEM catalog seed`); const { scrapeCradlepointOem } = await import("./scrapers/cradlepoint-oem"); await scrapeCradlepointOem(); }); await boss.work("scrape:catalog:fujitsu-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Fujitsu Network Communications OEM catalog seed`); const { scrapeFujitsuOem } = await import("./scrapers/fujitsu-oem"); await scrapeFujitsuOem(); }); await boss.work("scrape:catalog:nec-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NEC Corporation OEM catalog seed`); const { scrapeNecOem } = await import("./scrapers/nec-oem"); await scrapeNecOem(); }); await boss.work("scrape:catalog:arris-oem", async () => { console.log(`[${new Date().toISOString()}] Running: CommScope ARRIS OEM catalog seed`); const { scrapeArrisOem } = await import("./scrapers/arris-oem"); await scrapeArrisOem(); }); await boss.work("scrape:catalog:hillstone-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Hillstone Networks OEM catalog seed`); const { scrapeHillstoneOem } = await import("./scrapers/hillstone-oem"); await scrapeHillstoneOem(); }); await boss.work("scrape:catalog:vecima-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Vecima Networks OEM catalog seed`); const { scrapeVecimaOem } = await import("./scrapers/vecima-oem"); await scrapeVecimaOem(); }); await boss.work("scrape:catalog:stordis-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Stordis OEM catalog seed`); const { scrapeStordisOem } = await import("./scrapers/stordis-oem"); await scrapeStordisOem(); }); await boss.work("scrape:catalog:keysight-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Keysight OEM catalog seed`); const { scrapeKeysightOem } = await import("./scrapers/keysight-oem"); await scrapeKeysightOem(); }); await boss.work("scrape:catalog:sycamore-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Sycamore OEM catalog seed`); const { scrapeSycamoreOem } = await import("./scrapers/sycamore-oem"); await scrapeSycamoreOem(); }); await boss.work("scrape:catalog:ekinops-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ekinops OEM catalog seed`); const { scrapeEkinopsOem } = await import("./scrapers/ekinops-oem"); await scrapeEkinopsOem(); }); await boss.work("scrape:catalog:adva-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ADVA Optical OEM catalog seed`); const { scrapeAdvaOem } = await import("./scrapers/adva-oem"); await scrapeAdvaOem(); }); await boss.work("scrape:catalog:coriant-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Coriant OEM catalog seed`); const { scrapeCoriantOem } = await import("./scrapers/coriant-oem"); await scrapeCoriantOem(); }); await boss.work("scrape:catalog:casa-systems-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Casa Systems OEM catalog seed`); const { scrapeCasaSystemsOem } = await import("./scrapers/casa-systems-oem"); await scrapeCasaSystemsOem(); }); await boss.work("scrape:catalog:harmonic-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Harmonic OEM catalog seed`); const { scrapeHarmonicOem } = await import("./scrapers/harmonic-oem"); await scrapeHarmonicOem(); }); await boss.work("scrape:catalog:solarflare-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Solarflare OEM catalog seed`); const { scrapeSolarflareOem } = await import("./scrapers/solarflare-oem"); await scrapeSolarflareOem(); }); await boss.work("scrape:catalog:marvell-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Marvell OEM catalog seed`); const { scrapeMarvellOem } = await import("./scrapers/marvell-oem"); await scrapeMarvellOem(); }); await boss.work("scrape:catalog:broadcom-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Broadcom OEM catalog seed`); const { scrapeBroadcomOem } = await import("./scrapers/broadcom-oem"); await scrapeBroadcomOem(); }); await boss.work("scrape:catalog:calix-access-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Calix Access OEM catalog seed`); const { scrapeCalixAccessOem } = await import("./scrapers/calix-access-oem"); await scrapeCalixAccessOem(); }); await boss.work("scrape:catalog:ribbon-comms-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`); const { scrapeRibbonCommsOem } = await import("./scrapers/ribbon-comms-oem"); await scrapeRibbonCommsOem(); }); await boss.work("scrape:catalog:infinera-groove-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Infinera Groove OEM catalog seed`); const { scrapeInfineraGrooveOem } = await import("./scrapers/infinera-groove-oem"); await scrapeInfineraGrooveOem(); }); await boss.work("scrape:catalog:ciena-waveserver-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ciena WaveServer OEM catalog seed`); const { scrapeCienaWaveserverOem } = await import("./scrapers/ciena-waveserver-oem"); await scrapeCienaWaveserverOem(); }); await boss.work("scrape:catalog:commscope-oem", async () => { console.log(`[${new Date().toISOString()}] Running: CommScope OEM catalog seed`); const { scrapeCommScopeOem } = await import("./scrapers/commscope-oem"); await scrapeCommScopeOem(); }); await boss.work("scrape:catalog:teleste-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Teleste OEM catalog seed`); const { scrapeTelesteOem } = await import("./scrapers/teleste-oem"); await scrapeTelesteOem(); }); await boss.work("scrape:catalog:tejas-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Tejas Networks OEM catalog seed`); const { scrapeTejasNetworksOem } = await import("./scrapers/tejas-networks-oem"); await scrapeTejasNetworksOem(); }); await boss.work("scrape:catalog:ericsson-transport-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ericsson Transport OEM catalog seed`); const { scrapeEricssonTransportOem } = await import("./scrapers/ericsson-transport-oem"); await scrapeEricssonTransportOem(); }); await boss.work("scrape:catalog:adtran-ta-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ADTRAN TA OEM catalog seed`); const { scrapeAdtranTaOem } = await import("./scrapers/adtran-ta-oem"); await scrapeAdtranTaOem(); }); await boss.work("scrape:catalog:isolan-oem", async () => { console.log(`[${new Date().toISOString()}] Running: iSolan OEM catalog seed`); const { scrapeIsolanOem } = await import("./scrapers/isolan-oem"); await scrapeIsolanOem(); }); await boss.work("scrape:catalog:telco-systems-oem", async () => { console.log(`[${new Date().toISOString()}] Running: TELCO Systems OEM catalog seed`); const { scrapeTelcoSystemsOem } = await import("./scrapers/telco-systems-oem"); await scrapeTelcoSystemsOem(); }); await boss.work("scrape:catalog:rad-oem", async () => { console.log(`[${new Date().toISOString()}] Running: RAD Data Communications OEM catalog seed`); const { scrapeRadOem } = await import("./scrapers/rad-oem"); await scrapeRadOem(); }); await boss.work("scrape:catalog:comtrend-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Comtrend OEM catalog seed`); const { scrapeComtrendOem } = await import("./scrapers/comtrend-oem"); await scrapeComtrendOem(); }); await boss.work("scrape:catalog:packetfront-oem", async () => { console.log(`[${new Date().toISOString()}] Running: PacketFront OEM catalog seed`); const { scrapePacketfrontOem } = await import("./scrapers/packetfront-oem"); await scrapePacketfrontOem(); }); await boss.work("scrape:catalog:edgewater-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Edgewater Networks OEM catalog seed`); const { scrapeEdgewaterNetworksOem } = await import("./scrapers/edgewater-networks-oem"); await scrapeEdgewaterNetworksOem(); }); await boss.work("scrape:catalog:corning-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Corning OEM catalog seed`); const { scrapeCorningOem } = await import("./scrapers/corning-oem"); await scrapeCorningOem(); }); await boss.work("scrape:catalog:ofs-oem", async () => { console.log(`[${new Date().toISOString()}] Running: OFS OEM catalog seed`); const { scrapeOfsOem } = await import("./scrapers/ofs-oem"); await scrapeOfsOem(); }); await boss.work("scrape:catalog:kontron-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Kontron OEM catalog seed`); const { scrapeKontronOem } = await import("./scrapers/kontron-oem"); await scrapeKontronOem(); }); await boss.work("scrape:catalog:ipinfusion-oem", async () => { console.log(`[${new Date().toISOString()}] Running: IP Infusion OEM catalog seed`); const { scrapeIpInfusionOem } = await import("./scrapers/ipinfusion-oem"); await scrapeIpInfusionOem(); }); await boss.work("scrape:catalog:telrad-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Telrad Networks OEM catalog seed`); const { scrapeTelradOem } = await import("./scrapers/telrad-oem"); await scrapeTelradOem(); }); await boss.work("scrape:catalog:siklu-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Siklu OEM catalog seed`); const { scrapeSikluOem } = await import("./scrapers/siklu-oem"); await scrapeSikluOem(); }); await boss.work("scrape:catalog:ceragon-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ceragon OEM catalog seed`); const { scrapeCeragonOem } = await import("./scrapers/ceragon-oem"); await scrapeCeragonOem(); }); await boss.work("scrape:catalog:datang-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Datang Telecom OEM catalog seed`); const { scrapeDatangOem } = await import("./scrapers/datang-oem"); await scrapeDatangOem(); }); await boss.work("scrape:catalog:viptela-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Viptela OEM catalog seed`); const { scrapeViptelaOem } = await import("./scrapers/viptela-oem"); await scrapeViptelaOem(); }); await boss.work("scrape:catalog:versa-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Versa Networks OEM catalog seed`); const { scrapeVersaNetworksOem } = await import("./scrapers/versa-networks-oem"); await scrapeVersaNetworksOem(); }); await boss.work("scrape:catalog:vmware-oem", async () => { console.log(`[${new Date().toISOString()}] Running: VMware OEM catalog seed`); const { scrapeVmwareOem } = await import("./scrapers/vmware-oem"); await scrapeVmwareOem(); }); await boss.work("scrape:catalog:cimc-oem", async () => { console.log(`[${new Date().toISOString()}] Running: CIMC Semiconductors OEM catalog seed`); const { scrapeCimcOem } = await import("./scrapers/cimc-oem"); await scrapeCimcOem(); }); await boss.work("scrape:catalog:qlogic-oem", async () => { console.log(`[${new Date().toISOString()}] Running: QLogic OEM catalog seed`); const { scrapeQlogicOem } = await import("./scrapers/qlogic-oem"); await scrapeQlogicOem(); }); await boss.work("scrape:catalog:emulex-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Emulex OEM catalog seed`); const { scrapeEmulexOem } = await import("./scrapers/emulex-oem"); await scrapeEmulexOem(); }); await boss.work("scrape:catalog:netapp-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NetApp OEM catalog seed`); const { scrapeNetappOem } = await import("./scrapers/netapp-oem"); await scrapeNetappOem(); }); await boss.work("scrape:catalog:pure-storage-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Pure Storage OEM catalog seed`); const { scrapePureStorageOem } = await import("./scrapers/pure-storage-oem"); await scrapePureStorageOem(); }); await boss.work("scrape:catalog:hpe-storage-oem", async () => { console.log(`[${new Date().toISOString()}] Running: HPE Storage OEM catalog seed`); const { scrapeHpeStorageOem } = await import("./scrapers/hpe-storage-oem"); await scrapeHpeStorageOem(); }); await boss.work("scrape:catalog:ibm-storage-oem", async () => { console.log(`[${new Date().toISOString()}] Running: IBM Storage OEM catalog seed`); const { scrapeIbmStorageOem } = await import("./scrapers/ibm-storage-oem"); await scrapeIbmStorageOem(); }); await boss.work("scrape:catalog:dell-storage-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Dell Storage OEM catalog seed`); const { scrapeDellStorageOem } = await import("./scrapers/dell-storage-oem"); await scrapeDellStorageOem(); }); await boss.work("scrape:catalog:hitachi-vantara-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Hitachi Vantara OEM catalog seed`); const { scrapeHitachiVantaraOem } = await import("./scrapers/hitachi-vantara-oem"); await scrapeHitachiVantaraOem(); }); await boss.work("scrape:catalog:aws-oem", async () => { console.log(`[${new Date().toISOString()}] Running: AWS OEM catalog seed`); const { scrapeAwsOem } = await import("./scrapers/aws-oem"); await scrapeAwsOem(); }); await boss.work("scrape:catalog:azure-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Microsoft Azure OEM catalog seed`); const { scrapeAzureOem } = await import("./scrapers/azure-oem"); await scrapeAzureOem(); }); await boss.work("scrape:catalog:google-cloud-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Google Cloud OEM catalog seed`); const { scrapeGoogleCloudOem } = await import("./scrapers/google-cloud-oem"); await scrapeGoogleCloudOem(); }); await boss.work("scrape:catalog:meta-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Meta OEM catalog seed`); const { scrapeMetaOem } = await import("./scrapers/meta-oem"); await scrapeMetaOem(); }); await boss.work("scrape:catalog:nokia-access-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Nokia Access OEM catalog seed`); const { scrapeNokiaAccessOem } = await import("./scrapers/nokia-access-oem"); await scrapeNokiaAccessOem(); }); await boss.work("scrape:catalog:huawei-access-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Huawei Access OEM catalog seed`); const { scrapeHuaweiAccessOem } = await import("./scrapers/huawei-access-oem"); await scrapeHuaweiAccessOem(); }); await boss.work("scrape:catalog:zte-access-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ZTE Access OEM catalog seed`); const { scrapeZteAccessOem } = await import("./scrapers/zte-access-oem"); await scrapeZteAccessOem(); }); await boss.work("scrape:catalog:calix-gigapoint-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Calix GigaPoint OEM catalog seed`); const { scrapeCalixGigapointOem } = await import("./scrapers/calix-gigapoint-oem"); await scrapeCalixGigapointOem(); }); await boss.work("scrape:catalog:samsung-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Samsung Networks OEM catalog seed`); const { scrapeSamsungNetworksOem } = await import("./scrapers/samsung-networks-oem"); await scrapeSamsungNetworksOem(); }); await boss.work("scrape:catalog:nokia-airscale-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Nokia AirScale OEM catalog seed`); const { scrapeNokiaAirscaleOem } = await import("./scrapers/nokia-airscale-oem"); await scrapeNokiaAirscaleOem(); }); await boss.work("scrape:catalog:ericsson-ran-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ericsson RAN OEM catalog seed`); const { scrapeEricssonRanOem } = await import("./scrapers/ericsson-ran-oem"); await scrapeEricssonRanOem(); }); await boss.work("scrape:catalog:mavenir-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Mavenir OEM catalog seed`); const { scrapeMavenirOem } = await import("./scrapers/mavenir-oem"); await scrapeMavenirOem(); }); await boss.work("scrape:catalog:ixia-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ixia OEM catalog seed`); const { scrapeIxiaOem } = await import("./scrapers/ixia-oem"); await scrapeIxiaOem(); }); await boss.work("scrape:catalog:exfo-network-oem", async () => { console.log(`[${new Date().toISOString()}] Running: EXFO Network OEM catalog seed`); const { scrapeExfoNetworkOem } = await import("./scrapers/exfo-network-oem"); await scrapeExfoNetworkOem(); }); await boss.work("scrape:catalog:cumulus-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cumulus Networks OEM catalog seed`); const { scrapeCumulusNetworksOem } = await import("./scrapers/cumulus-networks-oem"); await scrapeCumulusNetworksOem(); }); await boss.work("scrape:catalog:sonic-oem", async () => { console.log(`[${new Date().toISOString()}] Running: SONiC OEM catalog seed`); const { scrapeSonicOem } = await import("./scrapers/sonic-oem"); await scrapeSonicOem(); }); await boss.work("scrape:catalog:h3c-oem", async () => { console.log(`[${new Date().toISOString()}] Running: H3C OEM catalog seed`); const { scrapeH3cOem } = await import("./scrapers/h3c-oem"); await scrapeH3cOem(); }); await boss.work("scrape:catalog:ruijie-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Ruijie Networks OEM catalog seed`); const { scrapeRuijieOem } = await import("./scrapers/ruijie-oem"); await scrapeRuijieOem(); }); await boss.work("scrape:catalog:centec-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Centec Networks OEM catalog seed`); const { scrapeCentecOem } = await import("./scrapers/centec-oem"); await scrapeCentecOem(); }); await boss.work("scrape:catalog:supermicro-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Supermicro OEM catalog seed`); const { scrapeSupermicroOem } = await import("./scrapers/supermicro-oem"); await scrapeSupermicroOem(); }); await boss.work("scrape:catalog:cisco-meraki-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco Meraki OEM catalog seed`); const { scrapeCiscoMerakiOem } = await import("./scrapers/cisco-meraki-oem"); await scrapeCiscoMerakiOem(); }); await boss.work("scrape:catalog:cisco-catalyst-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco Catalyst OEM catalog seed`); const { scrapeCiscoCatalystOem } = await import("./scrapers/cisco-catalyst-oem"); await scrapeCiscoCatalystOem(); }); await boss.work("scrape:catalog:cisco-nexus-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco Nexus OEM catalog seed`); const { scrapeCiscoNexusOem } = await import("./scrapers/cisco-nexus-oem"); await scrapeCiscoNexusOem(); }); await boss.work("scrape:catalog:cisco-asr-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco ASR OEM catalog seed`); const { scrapeCiscoAsrOem } = await import("./scrapers/cisco-asr-oem"); await scrapeCiscoAsrOem(); }); await boss.work("scrape:catalog:juniper-mx-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Juniper MX OEM catalog seed`); const { scrapeJuniperMxOem } = await import("./scrapers/juniper-mx-oem"); await scrapeJuniperMxOem(); }); await boss.work("scrape:catalog:juniper-qfx-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Juniper QFX OEM catalog seed`); const { scrapeJuniperQfxOem } = await import("./scrapers/juniper-qfx-oem"); await scrapeJuniperQfxOem(); }); await boss.work("scrape:catalog:aruba-cx-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Aruba CX OEM catalog seed`); const { scrapeArubaCxOem } = await import("./scrapers/aruba-cx-oem"); await scrapeArubaCxOem(); }); await boss.work("scrape:catalog:extreme-campus-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Extreme Campus OEM catalog seed`); const { scrapeExtremeCampusOem } = await import("./scrapers/extreme-campus-oem"); await scrapeExtremeCampusOem(); }); await boss.work("scrape:catalog:arista-7000-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Arista 7000 OEM catalog seed`); const { scrapeArista7000Oem } = await import("./scrapers/arista-7000-oem"); await scrapeArista7000Oem(); }); await boss.work("scrape:catalog:pica8-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Pica8 OEM catalog seed`); const { scrapePica8Oem } = await import("./scrapers/pica8-oem"); await scrapePica8Oem(); }); await boss.work("scrape:catalog:pluribus-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Pluribus Networks OEM catalog seed`); const { scrapePluribusOem } = await import("./scrapers/pluribus-oem"); await scrapePluribusOem(); }); await boss.work("scrape:catalog:drivenets-oem", async () => { console.log(`[${new Date().toISOString()}] Running: DriveNets OEM catalog seed`); const { scrapeDrivenetsOem } = await import("./scrapers/drivenets-oem"); await scrapeDrivenetsOem(); }); await boss.work("scrape:catalog:phoenix-contact-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Phoenix Contact OEM catalog seed`); const { scrapePhoenixContactOem } = await import("./scrapers/phoenix-contact-oem"); await scrapePhoenixContactOem(); }); await boss.work("scrape:catalog:beckhoff-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Beckhoff OEM catalog seed`); const { scrapeBeckhoffOem } = await import("./scrapers/beckhoff-oem"); await scrapeBeckhoffOem(); }); await boss.work("scrape:catalog:omron-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Omron OEM catalog seed`); const { scrapeOmronOem } = await import("./scrapers/omron-oem"); await scrapeOmronOem(); }); await boss.work("scrape:catalog:abb-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ABB OEM catalog seed`); const { scrapeAbbOem } = await import("./scrapers/abb-oem"); await scrapeAbbOem(); }); await boss.work("scrape:catalog:siemens-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Siemens SCALANCE OEM catalog seed`); const { scrapeSiemensOem } = await import("./scrapers/siemens-oem"); await scrapeSiemensOem(); }); await boss.work("scrape:catalog:schneider-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Schneider Electric OEM catalog seed`); const { scrapeSchneiderOem } = await import("./scrapers/schneider-oem"); await scrapeSchneiderOem(); }); await boss.work("scrape:catalog:rockwell-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Rockwell Automation OEM catalog seed`); const { scrapeRockwellOem } = await import("./scrapers/rockwell-oem"); await scrapeRockwellOem(); }); await boss.work("scrape:catalog:belden-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Belden OEM catalog seed`); const { scrapeBeldenOem } = await import("./scrapers/belden-oem"); await scrapeBeldenOem(); }); await boss.work("scrape:catalog:ge-grid-oem", async () => { console.log(`[${new Date().toISOString()}] Running: GE Grid Solutions OEM catalog seed`); const { scrapeGeGridOem } = await import("./scrapers/ge-grid-oem"); await scrapeGeGridOem(); }); await boss.work("scrape:catalog:schweitzer-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Schweitzer Engineering OEM catalog seed`); const { scrapeSchweiterEngOem } = await import("./scrapers/schweitzer-oem"); await scrapeSchweiterEngOem(); }); await boss.work("scrape:catalog:moxa-industrial-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Moxa Industrial OEM catalog seed`); const { scrapeIndustrialMoxaOem } = await import("./scrapers/moxa-industrial-oem"); await scrapeIndustrialMoxaOem(); }); await boss.work("scrape:catalog:cisco-ie-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco Industrial Ethernet OEM catalog seed`); const { scrapeCiscoIeOem } = await import("./scrapers/cisco-ie-oem"); await scrapeCiscoIeOem(); }); // ── Media / Broadcast / Pro-AV ──────────────────────────────────────── await boss.work("scrape:catalog:evertz-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Evertz Microsystems OEM catalog seed`); const { scrapeEvertzOem } = await import("./scrapers/evertz-oem"); await scrapeEvertzOem(); }); await boss.work("scrape:catalog:grass-valley-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Grass Valley OEM catalog seed`); const { scrapeGrassValleyOem } = await import("./scrapers/grass-valley-oem"); await scrapeGrassValleyOem(); }); await boss.work("scrape:catalog:haivision-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Haivision OEM catalog seed`); const { scrapeHaivisionOem } = await import("./scrapers/haivision-oem"); await scrapeHaivisionOem(); }); await boss.work("scrape:catalog:viasat-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Viasat OEM catalog seed`); const { scrapeViasatOem } = await import("./scrapers/viasat-oem"); await scrapeViasatOem(); }); // ── Asian Optical Vendors ───────────────────────────────────────────── await boss.work("scrape:catalog:fiberhome-oem", async () => { console.log(`[${new Date().toISOString()}] Running: FiberHome Technologies OEM catalog seed`); const { scrapeFiberhomeOem } = await import("./scrapers/fiberhome-oem"); await scrapeFiberhomeOem(); }); await boss.work("scrape:catalog:oplink-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Oplink Communications OEM catalog seed`); const { scrapeOplinkOem } = await import("./scrapers/oplink-oem"); await scrapeOplinkOem(); }); await boss.work("scrape:catalog:accelink-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Accelink Technologies OEM catalog seed`); const { scrapeAccelinkOem } = await import("./scrapers/accelink-oem"); await scrapeAccelinkOem(); }); await boss.work("scrape:catalog:hisense-broadband-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Hisense Broadband OEM catalog seed`); const { scrapeHisenseBroadbandOem } = await import("./scrapers/hisense-broadband-oem"); await scrapeHisenseBroadbandOem(); }); // ── Optical Transceiver Manufacturers (batch 24–25) ─────────────────── await boss.work("scrape:catalog:ii-vi-oem", async () => { console.log(`[${new Date().toISOString()}] Running: II-VI / Coherent OEM catalog seed`); const { scrapeIiViOem } = await import("./scrapers/ii-vi-oem"); await scrapeIiViOem(); }); await boss.work("scrape:catalog:source-photonics-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Source Photonics OEM catalog seed`); const { scrapeSourcePhotonicsOem } = await import("./scrapers/source-photonics-oem"); await scrapeSourcePhotonicsOem(); }); await boss.work("scrape:catalog:o-net-oem", async () => { console.log(`[${new Date().toISOString()}] Running: O-Net Technologies OEM catalog seed`); const { scrapeONetOem } = await import("./scrapers/o-net-oem"); await scrapeONetOem(); }); await boss.work("scrape:catalog:innolight-oem", async () => { console.log(`[${new Date().toISOString()}] Running: InnoLight Technology OEM catalog seed`); const { scrapeInnoLightOem } = await import("./scrapers/innolight-oem"); await scrapeInnoLightOem(); }); await boss.work("scrape:catalog:aoi-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Applied Optoelectronics (AOI) OEM catalog seed`); const { scrapeAoiOem } = await import("./scrapers/aoi-oem"); await scrapeAoiOem(); }); await boss.work("scrape:catalog:sumitomo-electric-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Sumitomo Electric Networks OEM catalog seed`); const { scrapeSumitomoElectricOem } = await import("./scrapers/sumitomo-electric-oem"); await scrapeSumitomoElectricOem(); }); await boss.work("scrape:catalog:neophotonics-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NeoPhotonics OEM catalog seed`); const { scrapeNeoPhotonicsOem } = await import("./scrapers/neophotonics-oem"); await scrapeNeoPhotonicsOem(); }); await boss.work("scrape:catalog:audiocodes-oem", async () => { console.log(`[${new Date().toISOString()}] Running: AudioCodes OEM catalog seed`); const { scrapeAudioCodesOem } = await import("./scrapers/audiocodes-oem"); await scrapeAudioCodesOem(); }); await boss.work("scrape:catalog:anritsu-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Anritsu OEM catalog seed`); const { scrapeAnritsuOem } = await import("./scrapers/anritsu-oem"); await scrapeAnritsuOem(); }); await boss.work("scrape:catalog:netscout-oem", async () => { console.log(`[${new Date().toISOString()}] Running: NETSCOUT OEM catalog seed`); const { scrapeNetScoutOem } = await import("./scrapers/netscout-oem"); await scrapeNetScoutOem(); }); await boss.work("scrape:catalog:curtiss-wright-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Curtiss-Wright OEM catalog seed`); const { scrapeCurtissWrightOem } = await import("./scrapers/curtiss-wright-oem"); await scrapeCurtissWrightOem(); }); await boss.work("scrape:catalog:eci-telecom-oem", async () => { console.log(`[${new Date().toISOString()}] Running: ECI Telecom OEM catalog seed`); const { scrapeEciTelecomOem } = await import("./scrapers/eci-telecom-oem"); await scrapeEciTelecomOem(); }); await boss.work("scrape:catalog:utstarcom-oem", async () => { console.log(`[${new Date().toISOString()}] Running: UTStarcom OEM catalog seed`); const { scrapeUTStarcomOem } = await import("./scrapers/utstarcom-oem"); await scrapeUTStarcomOem(); }); await boss.work("scrape:catalog:turbolink-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Turbolink OEM catalog seed`); const { scrapeTurbolinkOem } = await import("./scrapers/turbolink-oem"); await scrapeTurbolinkOem(); }); await boss.work("scrape:catalog:chelsio-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Chelsio OEM catalog seed`); const { scrapeChelsioOem } = await import("./scrapers/chelsio-oem"); await scrapeChelsioOem(); }); await boss.work("scrape:catalog:rohde-schwarz-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Rohde & Schwarz OEM catalog seed`); const { scrapeRohdeScharzOem } = await import("./scrapers/rohde-schwarz-oem"); await scrapeRohdeScharzOem(); }); await boss.work("scrape:catalog:l3harris-oem", async () => { console.log(`[${new Date().toISOString()}] Running: L3Harris OEM catalog seed`); const { scrapeL3HarrisOem } = await import("./scrapers/l3harris-oem"); await scrapeL3HarrisOem(); }); await boss.work("scrape:catalog:zhone-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Zhone/DZS OEM catalog seed`); const { scrapeZhoneOem } = await import("./scrapers/zhone-oem"); await scrapeZhoneOem(); }); await boss.work("scrape:catalog:cambium-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`); const { scrapeCambiumNetworksOem } = await import("./scrapers/cambium-networks-oem"); await scrapeCambiumNetworksOem(); }); await boss.work("scrape:catalog:tektronix-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Tektronix OEM catalog seed`); const { scrapeTektronixOem } = await import("./scrapers/tektronix-oem"); await scrapeTektronixOem(); }); await boss.work("scrape:catalog:clearfield-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Clearfield OEM catalog seed`); const { scrapeClearfieldOem } = await import("./scrapers/clearfield-oem"); await scrapeClearfieldOem(); }); await boss.work("scrape:catalog:lanner-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Lanner Electronics OEM catalog seed`); const { scrapeLannerOem } = await import("./scrapers/lanner-oem"); await scrapeLannerOem(); }); await boss.work("scrape:catalog:black-box-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`); const { scrapeBlackBoxOem } = await import("./scrapers/black-box-oem"); await scrapeBlackBoxOem(); }); await boss.work("scrape:catalog:radiflow-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Radiflow OEM catalog seed`); const { scrapeRadiflowOem } = await import("./scrapers/radiflow-oem"); await scrapeRadiflowOem(); }); await boss.work("scrape:catalog:dragonwave-oem", async () => { console.log(`[${new Date().toISOString()}] Running: DragonWave OEM catalog seed`); const { scrapeDragonwaveOem } = await import("./scrapers/dragonwave-oem"); await scrapeDragonwaveOem(); }); await boss.work("scrape:catalog:teledyne-lecroy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Teledyne LeCroy OEM catalog seed`); const { scrapeTeledyneLeCroyOem } = await import("./scrapers/teledyne-lecroy-oem"); await scrapeTeledyneLeCroyOem(); }); await boss.work("scrape:catalog:finisar-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Finisar OEM catalog seed`); const { scrapeFinisarOem } = await import("./scrapers/finisar-oem"); await scrapeFinisarOem(); }); await boss.work("scrape:catalog:acacia-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Acacia Communications OEM catalog seed`); const { scrapeAcaciaOem } = await import("./scrapers/acacia-oem"); await scrapeAcaciaOem(); }); await boss.work("scrape:catalog:inphi-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Inphi (Marvell) OEM catalog seed`); const { scrapeInphiOem } = await import("./scrapers/inphi-oem"); await scrapeInphiOem(); }); // ── Batch 35 workers ───────────────────────────────────────────────── await boss.work("scrape:catalog:sierra-wireless-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Sierra Wireless (Semtech) OEM catalog seed`); const { scrapeSierraWirelessOem } = await import("./scrapers/sierra-wireless-oem"); await scrapeSierraWirelessOem(); }); await boss.work("scrape:catalog:senao-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Senao (EnGenius) OEM catalog seed`); const { scrapeSenaoOem } = await import("./scrapers/senao-oem"); await scrapeSenaoOem(); }); await boss.work("scrape:catalog:emcore-oem", async () => { console.log(`[${new Date().toISOString()}] Running: EMCORE OEM catalog seed`); const { scrapeEmcoreOem } = await import("./scrapers/emcore-oem"); await scrapeEmcoreOem(); }); await boss.work("scrape:catalog:reflex-photonics-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Reflex Photonics OEM catalog seed`); const { scrapeReflexPhotonicsOem } = await import("./scrapers/reflex-photonics-oem"); await scrapeReflexPhotonicsOem(); }); // ── Batch 36 workers ───────────────────────────────────────────────── await boss.work("scrape:catalog:engenius-oem", async () => { console.log(`[${new Date().toISOString()}] Running: EnGenius OEM catalog seed`); const { scrapeEngeniusOem } = await import("./scrapers/engenius-oem"); await scrapeEngeniusOem(); }); await boss.work("scrape:catalog:paloalto-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`); const { scrapePaloaltoNetworksOem } = await import("./scrapers/paloalto-networks-oem"); await scrapePaloaltoNetworksOem(); }); await boss.work("scrape:catalog:brocade-legacy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Brocade (Legacy) OEM catalog seed`); const { scrapeBrocadeLegacyOem } = await import("./scrapers/brocade-legacy-oem"); await scrapeBrocadeLegacyOem(); }); await boss.work("scrape:catalog:foundry-networks-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Foundry Networks (Legacy) OEM catalog seed`); const { scrapeFoundryNetworksOem } = await import("./scrapers/foundry-networks-oem"); await scrapeFoundryNetworksOem(); }); // ── Batch 37 workers ───────────────────────────────────────────────── await boss.work("scrape:catalog:extreme-legacy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Extreme Networks (Legacy) OEM catalog seed`); const { scrapeExtremeLegacyOem } = await import("./scrapers/extreme-legacy-oem"); await scrapeExtremeLegacyOem(); }); await boss.work("scrape:catalog:nortel-legacy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Nortel Networks (Legacy) OEM catalog seed`); const { scrapeNortelLegacyOem } = await import("./scrapers/nortel-legacy-oem"); await scrapeNortelLegacyOem(); }); await boss.work("scrape:catalog:3com-legacy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: 3Com (Legacy) OEM catalog seed`); const { scrape3comLegacyOem } = await import("./scrapers/3com-legacy-oem"); await scrape3comLegacyOem(); }); await boss.work("scrape:catalog:avaya-legacy-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Avaya (Legacy) OEM catalog seed`); const { scrapeAvayaLegacyOem } = await import("./scrapers/avaya-legacy-oem"); await scrapeAvayaLegacyOem(); }); // ── Vendor lists ────────────────────────────────────────────────────── await boss.work("scrape:vendors:flexoptix", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix vendor list`); await scrapeFlexoptixVendors(); }); await boss.work("scrape:vendors:flexoptix-supported", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix supported vendors`); await seedFlexoptixVendors(); }); // ── Compatibility scrapers ──────────────────────────────────────────── await boss.work("scrape:compat:flexoptix", async () => { console.log(`[${new Date().toISOString()}] Running: Flexoptix compatibility mapping`); if (!isLoadAcceptable(2.5)) { console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Flexoptix compat scrape`); return; } await scrapeFlexoptixCompatibility(); }); await boss.work("scrape:compat:cisco", async () => { console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`); await scrapeCiscoTmg(); }); await boss.work("scrape:compat:juniper", async () => { console.log(`[${new Date().toISOString()}] Running: Juniper HCT compatibility`); await scrapeJuniperHct(); }); await boss.work("scrape:compat:sonic", async () => { console.log(`[${new Date().toISOString()}] Running: SONiC HCL compatibility`); await scrapeSonicHcl(); }); await boss.work("scrape:compat:ufispace", async () => { console.log(`[${new Date().toISOString()}] Running: Ufispace switch data`); await scrapeUfiSpace(); }); await boss.work("scrape:compat:edgecore", async () => { console.log(`[${new Date().toISOString()}] Running: Edgecore switch data`); await scrapeEdgecore(); }); // ── Switch assets ───────────────────────────────────────────────────── await boss.work("scrape:assets:switches", async () => { console.log(`[${new Date().toISOString()}] Running: Switch assets enrichment`); await scrapeSwitchAssets(); }); await boss.work("scrape:images:switches", async () => { console.log(`[${new Date().toISOString()}] Running: Switch og:image fetcher`); if (!isLoadAcceptable(2.5)) { console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping switch image fetch`); return; } await fetchSwitchImages(); }); await boss.work("scrape:images:switches:playwright", async () => { console.log(`[${new Date().toISOString()}] Running: Switch image fetcher (Playwright — bot-blocked vendors)`); if (!isLoadAcceptable(2.0)) { console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Playwright image fetch`); return; } await fetchSwitchImagesPlaywright(); }); // ── eBay enrichment ─────────────────────────────────────────────────── await boss.work("enrich:ebay-transceivers", async () => { console.log(`[${new Date().toISOString()}] Running: eBay transceiver pricing`); const { enrichTransceiversFromEbay } = await import("./scrapers/ebay-enricher"); await enrichTransceiversFromEbay(100); }); await boss.work("enrich:ebay-switches", async () => { console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`); const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher"); await enrichSwitchesFromEbay(30); }); // ── Intelligence & community ────────────────────────────────────────── await boss.work("scrape:market-intel", async () => { console.log(`[${new Date().toISOString()}] Running: Market intelligence`); await scrapeMarketIntelligence(); }); await boss.work("scrape:nog-talks", async () => { console.log(`[${new Date().toISOString()}] Running: NOG conference talks`); const { scrapeNogTalks } = await import("./scrapers/nog-talks"); await scrapeNogTalks(); }); await boss.work("scrape:community-issues", async () => { console.log(`[${new Date().toISOString()}] Running: Community issues`); const { scrapeAllSwitchIssues, scrapeTransceiverCompatIssues } = await import("./scrapers/community-issues"); await scrapeAllSwitchIssues(30); await scrapeTransceiverCompatIssues(15); }); await boss.work("scrape:datasheet-links", async () => { console.log(`[${new Date().toISOString()}] Running: Datasheet links`); const { findAndSeedDatasheetLinks } = await import("./scrapers/community-issues"); await findAndSeedDatasheetLinks(50); }); await boss.work("scrape:news", async () => { console.log(`[${new Date().toISOString()}] Running: News scraper`); await scrapeNews(); }); await boss.work("scrape:faq", async () => { console.log(`[${new Date().toISOString()}] FAQ scraper — not yet implemented`); }); await boss.work("scrape:docs", async () => { console.log(`[${new Date().toISOString()}] Docs scraper — not yet implemented`); }); // ── Compute jobs ────────────────────────────────────────────────────── await boss.work("compute:abc", async () => { console.log(`[${new Date().toISOString()}] Computing: ABC classification`); const { computeAbcClassification } = await import("./scrapers/market-intelligence"); await computeAbcClassification(); }); await boss.work("compute:reorder-signals", async () => { console.log(`[${new Date().toISOString()}] Computing: Reorder signals`); const { computeReorderSignals } = await import("./scrapers/market-intelligence"); await computeReorderSignals(); }); // ── NAS sync ────────────────────────────────────────────────────────── await boss.work("sync:nas", async () => { console.log(`[${new Date().toISOString()}] Running: NAS sync to Fearghas`); const { runNightlyNasSync } = await import("./utils/nas-sync"); await runNightlyNasSync(); }); // ── Prediction signal scrapers ──────────────────────────────────────── await boss.work("scrape:signals:sec-edgar", async () => { console.log(`[${new Date().toISOString()}] Running: SEC EDGAR CapEx`); await scrapeSecEdgar(); }); await boss.work("scrape:signals:github", async () => { console.log(`[${new Date().toISOString()}] Running: GitHub tech signals`); await scrapeGithubSignals(); }); await boss.work("scrape:signals:ebay-velocity", async () => { console.log(`[${new Date().toISOString()}] Running: eBay sold velocity`); await scrapeEbayVelocity(); }); await boss.work("scrape:signals:ai-clusters", async () => { console.log(`[${new Date().toISOString()}] Running: AI cluster announcements`); await scrapeAiClusters(); }); await boss.work("scrape:signals:distributor-leads", async () => { console.log(`[${new Date().toISOString()}] Running: Distributor lead times`); await scrapeDistributorLeads(); }); await boss.work("scrape:signals:standards", async () => { console.log(`[${new Date().toISOString()}] Running: Standards tracker`); await scrapeStandardsTracker(); }); // ── Forecast engine ─────────────────────────────────────────────────── await boss.work("compute:forecast", async () => { console.log(`[${new Date().toISOString()}] Running: Forecast engine`); await runForecastEngine(); }); // ── Hype Cycle Engine (Norton-Bass diffusion model) ─────────────────── await boss.work("compute:hype-cycle", async () => { console.log(`[${new Date().toISOString()}] Running: Hype Cycle Engine (Norton-Bass)`); const { computeHypeCycle } = await import("./utils/hype-cycle-engine"); await computeHypeCycle(); }); // ── Form-factor coverage scrapers ───────────────────────────────────── await boss.work("scrape:pricing:comms-express", async () => { console.log(`[${new Date().toISOString()}] Running: Comms-Express pricing`); const { scrapeCommsExpress } = await import("./scrapers/comms-express"); await scrapeCommsExpress(); }); await boss.work("scrape:pricing:router-switch", async () => { console.log(`[${new Date().toISOString()}] Running: Router-Switch.com pricing`); const { scrapeRouterSwitch } = await import("./scrapers/router-switch"); await scrapeRouterSwitch(); }); await boss.work("scrape:pricing:multimode-inc", async () => { console.log(`[${new Date().toISOString()}] Running: Multimode Inc pricing`); const { scrapeMultimodeInc } = await import("./scrapers/multimode-inc"); await scrapeMultimodeInc(); }); await boss.work("scrape:pricing:optictransceiver", async () => { console.log(`[${new Date().toISOString()}] Running: OpticTransceiver.com pricing`); const { scrapeOpticTransceiver } = await import("./scrapers/optictransceiver"); await scrapeOpticTransceiver(); }); await boss.work("scrape:pricing:wiitek", async () => { console.log(`[${new Date().toISOString()}] Running: Wiitek pricing`); const { scrapeWiitek } = await import("./scrapers/wiitek"); await scrapeWiitek(); }); await boss.work("scrape:pricing:naddod", async () => { console.log(`[${new Date().toISOString()}] Running: NADDOD pricing`); const { scrapeNaddod } = await import("./scrapers/naddod"); await scrapeNaddod(); }); await boss.work("scrape:pricing:qsfptek", async () => { console.log(`[${new Date().toISOString()}] Running: QSFPTEK pricing`); const { scrapeQsfptek } = await import("./scrapers/qsfptek"); await scrapeQsfptek(); }); await boss.work("scrape:pricing:addon", async () => { console.log(`[${new Date().toISOString()}] Running: AddOn Networks pricing`); const { scrapeAddonNetworks } = await import("./scrapers/addon-networks"); await scrapeAddonNetworks(); }); await boss.work("scrape:pricing:fibermall", async () => { console.log(`[${new Date().toISOString()}] Running: FiberMall pricing`); const { scrapeFiberMall } = await import("./scrapers/fibermall"); await scrapeFiberMall(); }); await boss.work("scrape:pricing:vcelink", async () => { console.log(`[${new Date().toISOString()}] Running: Vcelink pricing`); const { scrapeVcelink } = await import("./scrapers/vcelink"); await scrapeVcelink(); }); await boss.work("scrape:pricing:opticsbay", async () => { console.log(`[${new Date().toISOString()}] Running: OpticsBay pricing`); const { scrapeOpticsBay } = await import("./scrapers/opticsbay"); await scrapeOpticsBay(); }); // ── Mouser OEM reference prices ──────────────────────────────────────── await boss.work("scrape:pricing:mouser-oem", async () => { console.log(`[${new Date().toISOString()}] Running: Mouser OEM reference prices`); if (!process.env["MOUSER_API_KEY"]) { console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set"); return; } const { scrapeMouser } = await import("./scrapers/mouser"); await scrapeMouser(); }); // ── Price denormalization refresh ───────────────────────────────────────── // Refreshes street_price_usd / price_verified_eur on the transceivers table from // the price_observations hypertable. Without this, denormalized prices go stale // even when scrapers are collecting new observations. await boss.work("compute:price-denorm", async () => { const { pool } = await import("./utils/db"); const ts = new Date().toISOString(); console.log(`[${ts}] Running: Price denormalization refresh`); const result = await pool.query(` UPDATE transceivers t SET price_verified_eur = sub.price_eur, street_price_usd = sub.price_usd, updated_at = NOW() FROM ( SELECT po.transceiver_id, MAX(po.price) FILTER (WHERE po.currency = 'EUR') AS price_eur, MAX(po.price) FILTER (WHERE po.currency = 'USD') AS price_usd FROM price_observations po WHERE po.time > NOW() - INTERVAL '180 days' AND po.price > 0 GROUP BY po.transceiver_id ) sub WHERE t.id = sub.transceiver_id AND (sub.price_eur IS NOT NULL OR sub.price_usd IS NOT NULL) `); console.log(`[price-denorm] refreshed ${result.rowCount} transceivers`); }); // ── Health monitor ────────────────────────────────────────────────────── await boss.work("monitor:scraper-health", async () => { const { pool } = await import("./utils/db"); // Vendors we expect to see prices from regularly. // Mapped: display name → pg-boss job name prefix (for last-run lookup). const EXPECTED_VENDORS: Array<{ name: string; jobName: string }> = [ { name: "FiberMall", jobName: "scrape:pricing:fibermall" }, { name: "QSFPTEK", jobName: "scrape:pricing:qsfptek" }, { name: "Flexoptix", jobName: "scrape:pricing:flexoptix" }, // FS.COM is skipped on Erik (SKIP_FS_SCRAPER=true) — Mac launchd handles it. // Exclude from monitor so stale last_seen doesn't trigger false positives. { name: "10Gtek", jobName: "scrape:pricing:10gtek" }, { name: "ATGBICS", jobName: "scrape:pricing:atgbics" }, { name: "GBICS", jobName: "scrape:pricing:gbics" }, { name: "SFPcables", jobName: "scrape:pricing:sfpcables" }, { name: "NADDOD", jobName: "scrape:pricing:naddod" }, ]; const vendorNames = EXPECTED_VENDORS.map((v) => v.name); const jobNames = EXPECTED_VENDORS.map((v) => v.jobName); // Price observation recency per vendor const priceResult = await pool.query(` SELECT v.name, SUM(CASE WHEN po.time > NOW() - INTERVAL '6 hours' THEN 1 ELSE 0 END) AS prices_6h, MAX(po.time) AS last_seen, EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 3600.0 AS hours_since FROM vendors v LEFT JOIN price_observations po ON po.source_vendor_id = v.id WHERE v.name = ANY($1) GROUP BY v.name ORDER BY last_seen ASC NULLS FIRST `, [vendorNames]); // Last successful pg-boss job per vendor scraper (within last 26h — covers any // 2h-scheduled job at least once plus 2h slack for daemon restart delays). // Also pull the most recent failed job so we can distinguish "running but // prices stable" from "job consistently failing". const jobResult = await pool.query(` SELECT DISTINCT ON (name) name, state, completed_on FROM pgboss.job WHERE name = ANY($1) AND created_on > NOW() - INTERVAL '26 hours' ORDER BY name, created_on DESC `, [jobNames]); const jobMap = new Map(); for (const row of jobResult.rows) { jobMap.set(row.name as string, { state: row.state as string, completed_on: row.completed_on as Date | null }); } // Last COMPLETED job per vendor (to know when the job last ran successfully) const completedResult = await pool.query(` SELECT DISTINCT ON (name) name, completed_on AS last_completed FROM pgboss.job WHERE name = ANY($1) AND state = 'completed' AND created_on > NOW() - INTERVAL '26 hours' ORDER BY name, completed_on DESC `, [jobNames]); const completedMap = new Map(); for (const row of completedResult.rows) { if (row.last_completed) completedMap.set(row.name as string, new Date(row.last_completed as string)); } // Thresholds for alerting: // CRITICAL (🔴): no job completed in 26h AND last price > 168h (7 days) // — scraper genuinely not running or consistently failing // WARNING (🟡): no job completed in 26h AND last price > 48h // — scraper may be broken but recently seen // STABLE (✅): job completed in last 26h AND vendor has historical prices // — prices unchanged (hash dedup), scraper is healthy const CRITICAL_HOURS = 168; const WARN_HOURS = 48; const critical: string[] = []; const warnings: string[] = []; const stable: string[] = []; for (const row of priceResult.rows) { const h = parseFloat(row.hours_since ?? "9999"); const n = parseInt(row.prices_6h ?? "0", 10); if (n > 0) continue; // new prices written → definitely healthy const lastStr = row.last_seen ? `last price ${h.toFixed(1)}h ago (${new Date(row.last_seen as string).toISOString().slice(0, 16)})` : "NEVER scraped"; const vendor = EXPECTED_VENDORS.find((v) => v.name === row.name); const jobInfo = vendor ? jobMap.get(vendor.jobName) : undefined; const lastCompleted = vendor ? completedMap.get(vendor.jobName) : undefined; const jobStr = jobInfo ? ` | job=${jobInfo.state} at ${jobInfo.completed_on ? new Date(jobInfo.completed_on).toISOString().slice(11, 16) : "?"}` : " | job=not run in 26h"; // If the job completed successfully in the last 26h AND the vendor has // historical prices, prices are just stable (hash dedup) — not an outage. const jobRunningOk = !!lastCompleted && row.last_seen; if (jobRunningOk) { stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged, job OK)${jobStr}`); continue; } if (!row.last_seen || h > CRITICAL_HOURS) { critical.push(`🔴 ${row.name}: ${lastStr}${jobStr}`); } else if (h > WARN_HOURS) { warnings.push(`🟡 ${row.name}: ${lastStr}${jobStr}`); } else { stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged)${jobStr}`); } } if (critical.length > 0 || warnings.length > 0) { if (critical.length > 0) { console.error("=== 🔴 SCRAPER CRITICAL — vendors with no prices for 7+ days ==="); for (const p of critical) console.error(p); } if (warnings.length > 0) { console.warn("=== 🟡 SCRAPER WARNING — vendors with stale prices (48h+) ==="); for (const p of warnings) console.warn(p); } console.error("=== Check: pm2 logs tip-scraper-daemon ==="); } else { const activeCount = EXPECTED_VENDORS.length - stable.length; if (stable.length > 0) { console.log(`[monitor] Scraper health OK — ${activeCount} vendors active, ${stable.length} stable (no price changes)`); for (const s of stable) console.log(` ${s}`); } else { console.log(`[monitor] Scraper health OK — all ${EXPECTED_VENDORS.length} vendors active in last 6h`); } } }); // ── Verification reconciliation ───────────────────────────────────────── await boss.work("maintenance:reconcile-verification", async () => { const { pool } = await import("./utils/db"); // 1. Backfill product_page_url from the latest real price URL. Many // fetch scrapers write URL only to price_observations; details verification // needs the canonical product source on transceivers as well. const backfillProductUrls = await pool.query(` UPDATE transceivers t SET product_page_url = latest.url, updated_at = NOW() FROM ( SELECT DISTINCT ON (po.transceiver_id) po.transceiver_id, po.url FROM price_observations po WHERE po.url IS NOT NULL AND po.url != '' AND po.time > NOW() - INTERVAL '180 days' ORDER BY po.transceiver_id, po.time DESC ) latest WHERE t.id = latest.transceiver_id AND (t.product_page_url IS NULL OR t.product_page_url = '') `); // 2. Promote stored product images to image_verified. Scrapers already // filter placeholders before writing image_url; this keeps older crawls from // being invisible to the dashboard badge math. const verifyImages = await pool.query(` UPDATE transceivers SET has_image = true, image_verified = true, image_verified_at = COALESCE(image_verified_at, NOW()), image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url), updated_at = NOW() WHERE image_url IS NOT NULL AND image_url != '' AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)' AND (image_verified = false OR image_verified IS NULL) `); // 3. Promote crawled products with source URL + core spec fields to // details_verified. This is intentionally stricter than "has any row": // product identity, reach and media type all need to be present. const verifyDetails = await pool.query(` UPDATE transceivers SET details_verified = true, details_verified_at = COALESCE(details_verified_at, NOW()), details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url), data_confidence = CASE WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated') THEN 'scraped_unverified' ELSE data_confidence END, updated_at = NOW() WHERE product_page_url IS NOT NULL AND product_page_url != '' AND product_page_url NOT LIKE '%/category/%' AND COALESCE(category, '') NOT IN ( 'NonTransceiver', 'Accessory', 'Adapter / Converter', 'Switch / Media Converter', 'Switch / Network Infrastructure', 'NIC / Adapter', 'Mux / Passive Optical', 'Product Family', 'Loopback / Test Module' ) AND form_factor IS NOT NULL AND speed_gbps IS NOT NULL AND part_number IS NOT NULL AND part_number != '' AND reach_label IS NOT NULL AND reach_label != '' AND fiber_type IS NOT NULL AND fiber_type != '' AND COALESCE(data_confidence, 'unknown') != 'garbage' AND (details_verified = false OR details_verified IS NULL) `); // 4. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days const resetComp = await pool.query(` UPDATE transceivers t SET competitor_verified = false, competitor_verified_at = NULL, competitor_status = 'needs_research', competitor_status_updated_at = NOW() WHERE competitor_verified = true AND COALESCE(competitor_status, 'matched') != 'no_valid_match' AND NOT EXISTS ( SELECT 1 FROM price_observations po JOIN vendors v ON po.source_vendor_id = v.id WHERE po.transceiver_id = t.id AND po.time > NOW() - INTERVAL '30 days' AND UPPER(v.name) NOT LIKE '%FLEXOPTIX%' ) `); // 5. Reset fully_verified=false for products that lost a required signal const resetFull = await pool.query(` UPDATE transceivers SET fully_verified = false, fully_verified_at = NULL WHERE fully_verified = true AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false) `); // 6. Set fully_verified=true for products that now meet all 4 criteria const setFull = await pool.query(` UPDATE transceivers SET fully_verified = true, fully_verified_at = COALESCE(fully_verified_at, NOW()) WHERE competitor_verified = true AND price_verified = true AND image_verified = true AND details_verified = true AND fully_verified = false `); console.log( `[reconcile] product URLs backfilled: ${backfillProductUrls.rowCount}, ` + `images verified: ${verifyImages.rowCount}, ` + `details verified: ${verifyDetails.rowCount}, ` + `competitor_verified reset: ${resetComp.rowCount}, ` + `fully_verified cleared: ${resetFull.rowCount}, ` + `fully_verified earned: ${setFull.rowCount}` ); }); // ── Equivalence matching ──────────────────────────────────────────────── // Matches Flexoptix SKUs to technically equivalent competitor products by specs. // Confidence scoring: standard_name(35) + form_factor(25) + speed_gbps(20) + // fiber_type(10) + reach±25%(10) = 100 pts max // ≥0.85 → auto_approved + competitor_verified=true // 0.50–0.84 → pending (Manual Review queue in dashboard) // <0.50 → skipped await boss.work("maintenance:find-equivalences", async () => { const { pool } = await import("./utils/db"); const ts = new Date().toISOString(); console.log(`[${ts}] Running: Equivalence matching`); // Find all Flexoptix transceivers that are NOT yet competitor_verified const flexResult = await pool.query(` SELECT t.id, t.part_number, t.standard_name, t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, t.connector, t.wdm_type, t.coherent FROM transceivers t JOIN vendors v ON v.id = t.vendor_id WHERE UPPER(v.name) LIKE '%FLEXOPTIX%' AND t.competitor_verified = false `); let autoApproved = 0; let queued = 0; let skipped = 0; for (const fx of flexResult.rows) { let fxMatched = false; let fxQueued = false; // Find competitor transceivers with recent price observations and matching specs const candidates = await pool.query(` SELECT t.id AS competitor_id, t.part_number, t.standard_name, t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, t.connector, v.name AS vendor_name, MAX(po.time) AS last_price, COUNT(*) AS price_count FROM transceivers t JOIN vendors v ON v.id = t.vendor_id JOIN price_observations po ON po.transceiver_id = t.id WHERE UPPER(v.name) NOT LIKE '%FLEXOPTIX%' AND po.time > NOW() - INTERVAL '30 days' AND t.form_factor = $1 AND t.speed_gbps = $2 AND t.id != $3 GROUP BY t.id, t.part_number, t.standard_name, t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, t.connector, v.name `, [fx.form_factor, fx.speed_gbps, fx.id]); for (const cand of candidates.rows) { // Confidence scoring // Max points: form_factor(25) + speed_gbps(20) + standard_name(30) + // wavelength_nm(20) + fiber_type(10) + reach(10) = 115 let score = 0; const basis: string[] = []; // form_factor already matched (pre-filter), award points score += 25; basis.push("form_factor"); // speed_gbps already matched (pre-filter) score += 20; basis.push("speed_gbps"); // standard_name match (strong signal — e.g. "10GBASE-LR") if (fx.standard_name && cand.standard_name && fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) { score += 30; basis.push("standard_name"); } // wavelength match — extract first numeric nm value and compare within ±15nm // "wavelengths" is text: "1310 nm", "850nm", "1270/1290/1310/1330 nm" etc. const extractNm = (w: string | null): number | null => { if (!w) return null; const m = w.match(/(\d{3,4})/); return m ? parseInt(m[1], 10) : null; }; const fxNm = extractNm(fx.wavelengths); const candNm = extractNm(cand.wavelengths); if (fxNm !== null && candNm !== null) { if (Math.abs(fxNm - candNm) <= 15) { score += 20; basis.push(`wavelength_${fxNm}nm`); } else { score -= 20; // hard penalize wrong wavelength (1310 vs 1550 = completely different product) } } // fiber_type match (SMF vs MMF — critical) if (fx.fiber_type && cand.fiber_type) { if (fx.fiber_type.trim().toUpperCase() === cand.fiber_type.trim().toUpperCase()) { score += 10; basis.push("fiber_type"); } else { score -= 15; // SMF vs MMF = wrong product } } // reach within ±25% if (fx.reach_meters && cand.reach_meters && fx.reach_meters > 0 && cand.reach_meters > 0) { const diff = Math.abs(fx.reach_meters - cand.reach_meters); const tolerance = Math.max(fx.reach_meters, 1) * 0.25; if (diff <= tolerance) { score += 10; basis.push("reach"); } else { score -= 15; // penalize mismatched reach } } else if (!fx.reach_meters && !cand.reach_meters) { score += 5; basis.push("reach_null"); } const confidence = Math.max(0, Math.min(1, score / 115)); if (confidence < 0.50) { skipped++; continue; } const notes = `${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` + `basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` + `wavelength: ${fx.wavelengths||"?"} vs ${cand.wavelengths||"?"}`; // Upsert equivalence candidate const status = confidence >= 0.73 ? "auto_approved" : "pending"; await pool.query(` INSERT INTO transceiver_equivalences (flexoptix_id, competitor_id, confidence, match_basis, match_notes, status) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET confidence = EXCLUDED.confidence, match_basis = EXCLUDED.match_basis, match_notes = EXCLUDED.match_notes, updated_at = NOW() WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected') `, [fx.id, cand.competitor_id, confidence, basis, notes, status]); if (confidence >= 0.73) { // Auto-approve: set competitor_verified on the Flexoptix transceiver await pool.query(` UPDATE transceivers SET competitor_verified = true, competitor_verified_at = NOW(), competitor_status = 'matched', competitor_status_updated_at = NOW() WHERE id = $1 AND competitor_verified = false `, [fx.id]); await pool.query(` INSERT INTO transceiver_verification_evidence ( transceiver_id, verification_type, source_url, source_vendor_id, evidence_value, evidence_hash, robot_name, confidence ) VALUES ( $1, 'competitor_match', NULL, NULL, $2::jsonb, md5($2::text), 'maintenance:find-equivalences', $3 ) ON CONFLICT DO NOTHING `, [ fx.id, JSON.stringify({ competitor_id: cand.competitor_id, competitor_part_number: cand.part_number, competitor_vendor: cand.vendor_name, match_basis: basis, notes, }), confidence, ]); autoApproved++; fxMatched = true; } else { queued++; fxQueued = true; } } if (!fxMatched && fxQueued) { await pool.query(` UPDATE transceivers SET competitor_status = 'ambiguous', competitor_status_updated_at = NOW() WHERE id = $1 AND competitor_verified = false AND COALESCE(competitor_status, 'unknown') NOT IN ('no_valid_match') `, [fx.id]); } else if (!fxMatched && !fxQueued) { await pool.query(` UPDATE transceivers SET competitor_status = 'needs_research', competitor_status_updated_at = NOW() WHERE id = $1 AND competitor_verified = false AND COALESCE(competitor_status, 'unknown') NOT IN ('no_valid_match', 'ambiguous') `, [fx.id]); } } console.log( `[find-equivalences] auto_approved: ${autoApproved}, ` + `queued for review: ${queued}, skipped (low confidence): ${skipped}` ); // After auto-approvals, rerun fully_verified check if (autoApproved > 0) { await pool.query(` UPDATE transceivers SET fully_verified = true, fully_verified_at = COALESCE(fully_verified_at, NOW()) WHERE competitor_verified = true AND price_verified = true AND image_verified = true AND details_verified = true AND fully_verified = false `); } }); // ── Re-research equivalences ──────────────────────────────────────────────── // Confirms only well-evidenced matches. Weak, stale, incomplete, or technically // contradictory matches are rejected automatically instead of going back to a // manual queue. await boss.work("maintenance:re-research-equivalences", async () => { const { pool } = await import("./utils/db"); const ts = new Date().toISOString(); const batchLimit = Math.max(1, Math.min(10000, parseInt(process.env["EQUIVALENCE_RESEARCH_BATCH_LIMIT"] || "2000", 10))); console.log(`[${ts}] Running: Equivalence automated research`); const batch = await pool.query(` SELECT eq.id, eq.flexoptix_id, eq.competitor_id, eq.status, eq.confidence, fx.part_number AS fx_part_number, fx.form_factor AS fx_form_factor, fx.speed_gbps AS fx_speed_gbps, fx.standard_name AS fx_standard_name, fx.fiber_type AS fx_fiber_type, fx.reach_meters AS fx_reach_meters, fx.wavelengths AS fx_wavelengths, fx.connector AS fx_connector, cp.part_number AS cp_part_number, cp.form_factor AS cp_form_factor, cp.speed_gbps AS cp_speed_gbps, cp.standard_name AS cp_standard_name, cp.fiber_type AS cp_fiber_type, cp.reach_meters AS cp_reach_meters, cp.wavelengths AS cp_wavelengths, cp.connector AS cp_connector, cpv.name AS competitor_vendor, ( SELECT COUNT(*) FROM price_observations po WHERE po.transceiver_id = eq.competitor_id AND po.time > NOW() - INTERVAL '45 days' ) AS recent_price_count FROM transceiver_equivalences eq JOIN transceivers fx ON eq.flexoptix_id = fx.id JOIN transceivers cp ON eq.competitor_id = cp.id JOIN vendors cpv ON cpv.id = cp.vendor_id WHERE eq.status IN ('pending', 'approved', 'auto_approved') AND eq.re_research_due_at IS NOT NULL AND eq.re_research_due_at <= NOW() ORDER BY eq.re_research_due_at ASC LIMIT $1 `, [batchLimit]); let confirmed = 0; let rejected = 0; for (const eq of batch.rows) { const research = evaluateEquivalenceResearch( { part_number: eq.fx_part_number, form_factor: eq.fx_form_factor, speed_gbps: eq.fx_speed_gbps, standard_name: eq.fx_standard_name, fiber_type: eq.fx_fiber_type, reach_meters: eq.fx_reach_meters, wavelengths: eq.fx_wavelengths, connector: eq.fx_connector, }, { part_number: eq.cp_part_number, form_factor: eq.cp_form_factor, speed_gbps: eq.cp_speed_gbps, standard_name: eq.cp_standard_name, fiber_type: eq.cp_fiber_type, reach_meters: eq.cp_reach_meters, wavelengths: eq.cp_wavelengths, connector: eq.cp_connector, }, parseInt(eq.recent_price_count, 10) > 0, ); if (research.decision === "reject") { await pool.query(` UPDATE transceiver_equivalences SET status = 'rejected', confidence = $2, match_basis = $3, reject_reason = $4, reviewed_by = 'automated-research', reviewed_at = NOW(), re_research_due_at = NULL, re_researched_at = NOW(), match_notes = CONCAT( COALESCE(match_notes, ''), E'\n[Automated research ' || NOW()::date || ': rejected; ' || $5 || ']' ), updated_at = NOW() WHERE id = $1 `, [ eq.id, research.confidence, research.basis, research.rejectReason || "automated research: rejected", research.reasons.join("; "), ]); await pool.query(` UPDATE transceivers SET competitor_verified = false, competitor_verified_at = NULL, fully_verified = false, fully_verified_at = NULL WHERE id = $1 AND NOT EXISTS ( SELECT 1 FROM transceiver_equivalences WHERE flexoptix_id = $1 AND status IN ('approved', 'auto_approved') AND id != $2 ) `, [eq.flexoptix_id, eq.id]); rejected++; } else { await pool.query(` UPDATE transceiver_equivalences SET status = CASE WHEN status = 'pending' THEN 'auto_approved' ELSE status END, confidence = $2, match_basis = $3, reviewed_by = COALESCE(reviewed_by, 'automated-research'), reviewed_at = COALESCE(reviewed_at, NOW()), reject_reason = NULL, re_researched_at = NOW(), re_research_due_at = NOW() + INTERVAL '30 days', match_notes = CONCAT( COALESCE(match_notes, ''), E'\n[Automated research ' || NOW()::date || ': confirmed; basis: ' || $4 || ']' ), updated_at = NOW() WHERE id = $1 `, [eq.id, research.confidence, research.basis, research.basis.join(", ")]); await pool.query(` UPDATE transceivers SET competitor_verified = true, competitor_verified_at = COALESCE(competitor_verified_at, NOW()) WHERE id = $1 AND competitor_verified = false `, [eq.flexoptix_id]); confirmed++; } } if (confirmed > 0) { await pool.query(` UPDATE transceivers SET fully_verified = true, fully_verified_at = COALESCE(fully_verified_at, NOW()) WHERE competitor_verified = true AND price_verified = true AND image_verified = true AND details_verified = true AND fully_verified = false `); } console.log(`[equivalence-research] confirmed: ${confirmed}, rejected: ${rejected}, batch size: ${batch.rows.length}`); }); // ══════════════════════════════════════════════════════════════════════ // VENDOR DISCOVERY CRAWLER WORKERS // Each worker calls discoverVendorCatalog() for the matching slug. // Results go to: TIP DB (findOrCreateScrapedTransceiver) + // Gitea tip-training-data repo (SFT JSONL pairs) // ══════════════════════════════════════════════════════════════════════ const { discoverVendorCatalog, VENDOR_CATALOG_REGISTRY } = await import("./crawler-llm/vendor-discovery-crawler"); for (const vendorConfig of VENDOR_CATALOG_REGISTRY) { const jobName = `discover:vendor:${vendorConfig.slug}`; boss.work(jobName, async () => { if (!isLoadAcceptable(3.0)) { console.warn(`[${jobName}] Load too high — skipping deep crawl`); return; } console.log(`[${jobName}] Starting vendor discovery crawl…`); try { await discoverVendorCatalog(vendorConfig, { verbose: false }); } catch (err) { console.error(`[${jobName}] Fatal:`, (err as Error).message); throw err; // let pg-boss retry } }); } console.log("All workers registered (102 jobs, 24/7 continuous + 8 weekly discovery crawlers)"); }