rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
transceiver-db/packages/scraper/src/scheduler.ts
Rene Fichtmueller 199f36be48 fix(scraper): auto-create pg-boss queues before scheduling + worker/schedule order 
- scheduler: patch boss.schedule() to call createQueue() first (idempotent),
  fixing FK constraint errors after DB reset — no need to touch 277 call sites
- index: registerWorkers() before registerSchedules() since boss.work() must
  register handlers before schedules fire
- dashboard: fix switchBlogLlm() to use api() helper (adds Bearer auth token)
  instead of raw fetch() which was returning 401 Unauthorized
2026-04-29 16:14:25 +02:00

2773 lines
147 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* pg-boss Job Scheduler — 24/7 Continuous Scraping
*
* ARCHITECTURE:
* - Erik (VPS, .82) : Playwright-heavy scrapers (FS.com, 10Gtek, ATGBICS, ProLabs)
* + all compatibility + eBay + compute + NAS sync
* - Raspberry Pi Fleet : Lightweight fetch/cheerio scrapers run continuously all day
* (BlueOptics, Fiber24, T&S Com, Fluxlight, GBICs, Optcore,
* Champion ONE, SFPCables, SmartOptics, HUBER+SUHNER, etc.)
*
* SCHEDULE PHILOSOPHY:
* - Playwright scrapers: every 8h (resource-heavy, VPS only)
* - Fetch/Cheerio scrapers: every 4h (lightweight, Pi-friendly)
* - Catalog scrapers (Flexoptix): every 2h (fast GraphQL, primary price source)
* - Compatibility matrices: every 12h (rarely change)
* - eBay enrichment: every 6h
* - Intelligence/community: every 6h
* - Compute jobs: after each pricing wave
* - NAS sync: nightly at 07:55
*/
import PgBoss from "pg-boss";
import { config } from "dotenv";
import { join } from "path";
import { loadavg } from "os";
// withIsolatedStorage removed — all Crawlee scrapers now use makeCrawleeConfig()
// for instance-level storage isolation. See packages/scraper/src/utils/crawlee-config.ts
/**
* Load-aware guard — skip heavy scrapers when the server is already busy.
* Uses the 1-minute load average; maxLoad defaults to 2.5 (50% of 5 vCPUs).
* Logs a warning and returns false when load is too high.
*/
function isLoadAcceptable(maxLoad = 2.5): boolean {
const [avg1] = loadavg();
if (avg1 > maxLoad) {
console.warn(`[load-guard] 1m load avg ${avg1.toFixed(2)} > ${maxLoad} — deferring heavy scraper`);
return false;
}
return true;
}
config({ path: join(__dirname, "..", "..", "..", ".env") });
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
export async function createScheduler(): Promise<PgBoss> {
const boss = new PgBoss({
connectionString,
retryLimit: 3,
retryDelay: 30,
retryBackoff: true,
expireInSeconds: 300,
monitorStateIntervalSeconds: 60,
});
boss.on("error", (error) => console.error("pg-boss error:", error));
await boss.start();
console.log("pg-boss scheduler started");
return boss;
}
export async function registerSchedules(boss: PgBoss): Promise<void> {
// pg-boss v10: boss.schedule() requires the queue to already exist in pgboss.queue.
// After a DB reset (e.g. server outage), all queue rows are wiped.
// Patch boss.schedule to auto-create queues idempotently before each schedule call,
// so the 277 individual schedule() calls below don't need to be touched.
const _origSchedule = boss.schedule.bind(boss) as typeof boss.schedule;
(boss as unknown as Record<string, unknown>).schedule = async (
name: string, cron: string, data?: unknown, opts?: unknown,
) => {
await boss.createQueue(name).catch(() => { /* already exists */ });
return _origSchedule(name, cron, data as object, opts as object);
};
const queues = [
// ── Playwright scrapers (Erik, every 8h) ───────────────────────────
"scrape:pricing:fs",
"scrape:pricing:10gtek",
"scrape:pricing:atgbics",
"scrape:pricing:prolabs",
// ── Fetch/Cheerio scrapers (Pi-friendly, every 4h) ─────────────────
"scrape:pricing:fluxlight",
"scrape:pricing:gbics",
"scrape:pricing:optcore",
"scrape:pricing:champion-one",
"scrape:pricing:sfpcables",
"scrape:pricing:blueoptics",
"scrape:pricing:fiber24",
"scrape:pricing:tscom",
"scrape:pricing:skylane",
"scrape:pricing:ascentoptics",
"scrape:pricing:gaotek",
// ── Catalog scrapers (every 2h) ────────────────────────────────────
"scrape:pricing:flexoptix",
// ── Manufacturer catalogs (every 8h, no prices) ────────────────────
"scrape:catalog:smartoptics",
"scrape:catalog:hubersuhner",
"scrape:catalog:eoptolink",
// ── Vendor lists ───────────────────────────────────────────────────
"scrape:vendors:flexoptix",
"scrape:vendors:flexoptix-supported",
// ── Compatibility (every 12h) ──────────────────────────────────────
"scrape:compat:flexoptix",
"scrape:compat:cisco",
"scrape:compat:juniper",
"scrape:compat:sonic",
"scrape:compat:ufispace",
"scrape:compat:edgecore",
// ── Switch enrichment (every 12h) ─────────────────────────────────
"scrape:assets:switches",
// ── Switch og:image fetcher (daily, after switch-assets) ──────────
"scrape:images:switches",
// ── Playwright image fetcher for bot-blocked vendors (every 3d) ───
"scrape:images:switches:playwright",
// ── eBay enrichment (every 6h) ────────────────────────────────────
"enrich:ebay-transceivers",
"enrich:ebay-switches",
// ── Intelligence & community (every 6h) ───────────────────────────
"scrape:market-intel",
"scrape:nog-talks",
"scrape:community-issues",
"scrape:datasheet-links",
"scrape:news",
"scrape:faq",
"scrape:docs",
// ── Compute (every 4h, after pricing waves) ───────────────────────
"compute:abc",
"compute:reorder-signals",
// ── New form-factor coverage scrapers (every 8h) ──────────────────
"scrape:pricing:comms-express",
"scrape:pricing:router-switch",
"scrape:pricing:multimode-inc",
"scrape:pricing:optictransceiver",
"scrape:pricing:wiitek",
// ── Fetch-based catalog+pricing scrapers (every 2h) ──────────────
"scrape:pricing:naddod",
"scrape:pricing:qsfptek",
"scrape:pricing:addon",
"scrape:pricing:fibermall",
"scrape:pricing:vcelink",
"scrape:pricing:opticsbay",
// ── OEM Reference Prices (Mouser API, once daily) ─────────────────
"scrape:pricing:mouser-oem",
// ── Prediction Signal Scrapers (new) ──────────────────────────────
"scrape:signals:sec-edgar",
"scrape:signals:github",
"scrape:signals:ebay-velocity",
"scrape:signals:ai-clusters",
"scrape:signals:distributor-leads",
"scrape:signals:standards",
// ── Forecast Engine ───────────────────────────────────────────────
"compute:forecast",
// ── Hype Cycle Engine (Norton-Bass, daily) ────────────────────────
"compute:hype-cycle",
// ── Sync ──────────────────────────────────────────────────────────
"sync:nas",
// ── Health Monitoring ─────────────────────────────────────────────
"monitor:scraper-health",
// ── Price denormalization refresh ─────────────────────────────────
"compute:price-denorm",
// ── Verification Reconciliation ───────────────────────────────────
"maintenance:reconcile-verification",
// ── Competitor Equivalence Matching ───────────────────────────────
"maintenance:find-equivalences",
// ── Re-Research approved equivalences ─────────────────────────────
"maintenance:re-research-equivalences",
// ── Vendor Discovery Crawlers (TIPLLM training data + DB seeding) ─────
"discover:vendor:cisco-tmg",
"discover:vendor:juniper",
"discover:vendor:arista",
"discover:vendor:fs-com",
"discover:vendor:flexoptix",
"discover:vendor:nokia",
"discover:vendor:huawei",
"discover:vendor:ii-vi",
];
for (const q of queues) {
await boss.createQueue(q).catch(() => { /* already exists */ });
}
// ══════════════════════════════════════════════════════════════════════
// ══════════════════════════════════════════════════════════════════════
// ALL PRICING SCRAPERS — 24/7, every 2h, staggered by 10min
// Goal: complete competitor coverage, no gaps, database always fresh
// ══════════════════════════════════════════════════════════════════════
// Playwright scrapers (resource-heavy) — every 2h, 10min apart
await boss.schedule("scrape:pricing:fs", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 });
await boss.schedule("scrape:pricing:10gtek", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:atgbics", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:prolabs", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// Fetch/Cheerio scrapers (lightweight) — every 2h, 5min apart
await boss.schedule("scrape:pricing:fluxlight", "0 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:gbics", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:optcore", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:champion-one", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:sfpcables", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:blueoptics", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:fiber24", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:tscom", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:skylane", "40 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:ascentoptics", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:gaotek", "50 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// Form-factor coverage scrapers — every 2h
await boss.schedule("scrape:pricing:comms-express", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 });
await boss.schedule("scrape:pricing:router-switch", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 });
await boss.schedule("scrape:pricing:multimode-inc", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:optictransceiver", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:wiitek", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// Fetch-based scrapers running on Erik — every 2h, end-of-cycle slots
await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:fibermall", "57 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// OEM reference prices via Mouser API — once daily at 03:00 (slow: 2s/PID × 475 PIDs ≈ 16min)
// Requires MOUSER_API_KEY env var (free at mouser.com/api-hub)
await boss.schedule("scrape:pricing:mouser-oem", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════
// FLEXOPTIX CATALOG — every 2h (primary price source)
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════
// MANUFACTURER CATALOGS — every 4h (product data, no prices)
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("scrape:catalog:smartoptics", "10 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hubersuhner", "25 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:eoptolink", "40 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// OEM vendor seed catalogs — daily at 04:00+ (stable data, rarely changes)
await boss.schedule("scrape:catalog:arista-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:juniper-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hpe-aruba-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nokia-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:huawei-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:dell-emc-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:extreme-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nvidia-mellanox-oem","45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:brocade-ruckus-oem","0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:zte-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ciena-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ericsson-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:fortinet-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:palo-alto-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:f5-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ubiquiti-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:mikrotik-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:netgear-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:adtran-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:calix-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:checkpoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:allied-telesis-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:zyxel-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:dlink-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ale-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ribbon-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:infinera-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:tplink-oem", "45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cambium-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:moxa-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:lumentum-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:coherent-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:brocade-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ruggedcom-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hirschmann-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:viavi-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:westermo-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:lancom-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:dzs-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:watchguard-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:sonicwall-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:advantech-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:redlion-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:exfo-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:mellanox-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:intel-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:antaira-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:transition-networks-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:perle-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:barracuda-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:etherwan-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:blackbox-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:packetlight-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:spirent-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:comnet-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:korenix-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:sophos-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cradlepoint-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:fujitsu-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nec-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:arris-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hillstone-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:vecima-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:stordis-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:keysight-oem", "0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:sycamore-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ekinops-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:adva-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:coriant-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:casa-systems-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:harmonic-oem", "30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:solarflare-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:marvell-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:broadcom-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:calix-access-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ribbon-comms-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:infinera-groove-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ciena-waveserver-oem","15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:commscope-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:teleste-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:tejas-networks-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ericsson-transport-oem","15 0 * * *",{}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:adtran-ta-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:isolan-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:telco-systems-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:rad-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:comtrend-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:packetfront-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:edgewater-networks-oem","0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:corning-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ofs-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:kontron-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ipinfusion-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:telrad-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:siklu-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ceragon-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:datang-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:viptela-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:versa-networks-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:vmware-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cimc-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:qlogic-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:emulex-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:netapp-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:pure-storage-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hpe-storage-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ibm-storage-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:dell-storage-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hitachi-vantara-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:aws-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:azure-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:google-cloud-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:meta-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nokia-access-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:huawei-access-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:zte-access-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:calix-gigapoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:samsung-networks-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nokia-airscale-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ericsson-ran-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:mavenir-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ixia-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:exfo-network-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cumulus-networks-oem","45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:sonic-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:h3c-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ruijie-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:centec-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:supermicro-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cisco-meraki-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cisco-catalyst-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cisco-nexus-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cisco-asr-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:juniper-mx-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:juniper-qfx-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:aruba-cx-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:extreme-campus-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:arista-7000-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:pica8-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:pluribus-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:drivenets-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:phoenix-contact-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:beckhoff-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:omron-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:abb-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:siemens-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:schneider-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:rockwell-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:belden-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:ge-grid-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:schweitzer-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:moxa-industrial-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cisco-ie-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Media / Broadcast / Pro-AV ────────────────────────────────────────
await boss.schedule("scrape:catalog:evertz-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:grass-valley-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:haivision-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:viasat-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Asian Optical Vendors ─────────────────────────────────────────────
await boss.schedule("scrape:catalog:fiberhome-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:oplink-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:accelink-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:hisense-broadband-oem","0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Optical Transceiver Manufacturers (batch 24–25) ───────────────────
await boss.schedule("scrape:catalog:ii-vi-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:source-photonics-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:o-net-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:innolight-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:aoi-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:sumitomo-electric-oem","30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:neophotonics-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Test & Measurement / Defense / Telecom Access (batch 29–30) ────────
await boss.schedule("scrape:catalog:audiocodes-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:anritsu-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:netscout-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:curtiss-wright-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:eci-telecom-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:rohde-schwarz-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:l3harris-oem", "15 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:zhone-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:cambium-networks-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:tektronix-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:clearfield-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:lanner-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:black-box-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:radiflow-oem", "0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:dragonwave-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:teledyne-lecroy-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:finisar-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:acacia-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:inphi-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Batch 35: Sierra Wireless, Senao, EMCORE, Reflex Photonics ────────
await boss.schedule("scrape:catalog:sierra-wireless-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:senao-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:emcore-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:reflex-photonics-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Batch 36: EnGenius, Palo Alto Networks, Brocade (Legacy), Foundry ─
await boss.schedule("scrape:catalog:engenius-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:paloalto-networks-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:brocade-legacy-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:foundry-networks-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ── Batch 37: Extreme (Legacy), Nortel, 3Com, Avaya ──────────────────
await boss.schedule("scrape:catalog:extreme-legacy-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:nortel-legacy-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:3com-legacy-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:catalog:avaya-legacy-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════
// VENDOR DISCOVERY CRAWLERS — daily, permanent 24/7 rotation
// Each run: crawls catalog → LLM extract → spec validate → DB + Gitea SFT
// 8 vendors × 3h stagger = full rotation every 24h, no overlap
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("discover:vendor:cisco-tmg", "0 20 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:juniper", "0 22 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:arista", "0 0 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:fs-com", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:flexoptix", "0 4 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:nokia", "0 6 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:huawei", "0 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("discover:vendor:ii-vi", "0 10 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
// ══════════════════════════════════════════════════════════════════════
// VENDOR LISTS — every 12h
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("scrape:vendors:flexoptix", "0 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
await boss.schedule("scrape:vendors:flexoptix-supported", "15 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
// ══════════════════════════════════════════════════════════════════════
// COMPATIBILITY MATRICES — every 12h
// ══════════════════════════════════════════════════════════════════════
// Flexoptix compatibility — every 24h at 09:00 (after both switch-assets + image-fetcher)
await boss.schedule("scrape:compat:flexoptix", "0 9 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:compat:ufispace", "45 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
await boss.schedule("scrape:compat:edgecore", "55 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
// ══════════════════════════════════════════════════════════════════════
// SWITCH ASSETS — every 12h
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("scrape:assets:switches", "30 7,19 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
// og:image fetcher: daily at 08:30, after switch-assets completes at 07:30
await boss.schedule("scrape:images:switches", "30 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
// Playwright image scraper for bot-blocked vendors (Arista/Dell/Edgecore/Fortinet/Extreme)
// Every 3 days at 09:00 — Playwright is slower and heavier than plain HTTP
await boss.schedule("scrape:images:switches:playwright", "0 9 */3 * *", {}, { retryLimit: 1, expireInSeconds: 10800 });
// ══════════════════════════════════════════════════════════════════════
// EBAY ENRICHMENT — every 6h
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("enrich:ebay-transceivers", "0 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 });
await boss.schedule("enrich:ebay-switches", "30 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 });
// ══════════════════════════════════════════════════════════════════════
// INTELLIGENCE & COMMUNITY — every 6h
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("scrape:market-intel", "0 2,8,14,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// NOG conference talks — weekly on Mondays 06:00 UTC
await boss.schedule("scrape:nog-talks", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 7200 });
await boss.schedule("scrape:community-issues", "30 2,8,14,20 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
await boss.schedule("scrape:datasheet-links", "0 3,9,15,21 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
await boss.schedule("scrape:news", "20 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
await boss.schedule("scrape:faq", "40 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
await boss.schedule("scrape:docs", "50 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════
// COMPUTE JOBS — every 4h (after pricing waves settle)
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("compute:abc", "50 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
await boss.schedule("compute:reorder-signals", "55 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
// Price denorm refresh: daily 05:30 UTC after overnight scraping waves settle
await boss.schedule("compute:price-denorm", "30 5 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
// ══════════════════════════════════════════════════════════════════════
// PREDICTION SIGNAL SCRAPERS
// ══════════════════════════════════════════════════════════════════════
// SEC EDGAR CapEx — weekly Monday 06:00 (filings don't change that fast)
await boss.schedule("scrape:signals:sec-edgar", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 3600 });
// GitHub signals — weekly Sunday 05:00
await boss.schedule("scrape:signals:github", "0 5 * * 0", {}, { retryLimit: 2, expireInSeconds: 7200 });
// eBay sold velocity — every 12h (fast-moving market signal)
await boss.schedule("scrape:signals:ebay-velocity", "0 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// AI cluster RSS feeds — every 4h (news moves fast)
await boss.schedule("scrape:signals:ai-clusters", "10 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
// Distributor lead times — daily 03:30 (stock changes overnight)
await boss.schedule("scrape:signals:distributor-leads","30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
// Standards tracker — weekly Wednesday 04:00 (standards move slowly)
await boss.schedule("scrape:signals:standards", "0 4 * * 3", {}, { retryLimit: 1, expireInSeconds: 3600 });
// ══════════════════════════════════════════════════════════════════════
// FORECAST ENGINE — daily at 08:00 (after all nightly scrapers done)
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("compute:forecast", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
// Hype Cycle Engine runs daily at 04:30 (after Mouser OEM scraper at 03:00)
await boss.schedule("compute:hype-cycle", "30 4 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
// ══════════════════════════════════════════════════════════════════════
// NAS SYNC — nightly
// ══════════════════════════════════════════════════════════════════════
await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
// Health check: every 3h — warns if any vendor has no new prices recently
await boss.schedule("monitor:scraper-health", "17 */3 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
// Verification reconciliation: nightly at 01:00 UTC
// Resets competitor_verified/fully_verified for any transceiver that no longer
// has a real non-Flexoptix price in the last 30 days — prevents stale ★ 100% badges
await boss.schedule("maintenance:reconcile-verification", "0 1 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
// Equivalence matching: nightly at 02:00 UTC (after reconcile)
await boss.schedule("maintenance:find-equivalences", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
// Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run
await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
console.log("All schedules registered — 24/7 continuous scraping (60 jobs)");
}
export async function registerWorkers(boss: PgBoss): Promise<void> {
// Lazy-load all scrapers
const { scrapeFs } = await import("./scrapers/fs-com");
const { scrapeCiscoTmg } = await import("./scrapers/cisco-tmg");
const { scrapeSmartOptics } = await import("./scrapers/smartoptics");
const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner");
const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence");
const { scrapeNews } = await import("./scrapers/news");
const { scrape10Gtek } = await import("./scrapers/tenGtek");
const { scrapeFlexoptixCatalog } = await import("./scrapers/flexoptix-catalog");
const { scrapeFlexoptixVendors } = await import("./scrapers/flexoptix-vendors");
const { seedFlexoptixVendors } = await import("./scrapers/flexoptix-supported-vendors");
const { scrapeAtgbics } = await import("./scrapers/atgbics");
const { scrapeProLabs } = await import("./scrapers/prolabs");
const { scrapeJuniperHct } = await import("./scrapers/juniper-hct");
const { scrapeSonicHcl } = await import("./scrapers/sonic-hcl");
const { scrapeUfiSpace } = await import("./scrapers/ufispace");
const { scrapeEdgecore } = await import("./scrapers/edgecore");
const { scrapeSwitchAssets } = await import("./scrapers/switch-assets");
const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher");
const { fetchSwitchImagesPlaywright } = await import("./scrapers/switch-image-playwright");
const { scrapeFlexoptixCompatibility } = await import("./scrapers/flexoptix-compat");
// ── Prediction signal scrapers ────────────────────────────────────────
const { scrapeSecEdgar } = await import("./scrapers/sec-edgar");
const { scrapeGithubSignals } = await import("./scrapers/github-signals");
const { scrapeEbayVelocity } = await import("./scrapers/ebay-velocity");
const { scrapeAiClusters } = await import("./scrapers/ai-clusters");
const { scrapeDistributorLeads }= await import("./scrapers/distributor-leads");
const { scrapeStandardsTracker }= await import("./scrapers/standards-tracker");
const { runForecastEngine } = await import("./utils/forecast-engine");
// ── Playwright scrapers ───────────────────────────────────────────────
await boss.work("scrape:pricing:fs", async () => {
// FS.com uses Playwright + Cloudflare bypass. On datacenter servers the
// datacenter IP is blocked by Cloudflare WAF. Set SKIP_FS_SCRAPER=true to
// skip on Erik; the Mac launchd cron handles FS.com from a residential IP.
if (process.env["SKIP_FS_SCRAPER"] === "true") {
console.log(`[${new Date().toISOString()}] FS.com pricing: SKIPPED (SKIP_FS_SCRAPER=true)`);
return;
}
console.log(`[${new Date().toISOString()}] Running: FS.com pricing`);
await scrapeFs();
});
await boss.work("scrape:pricing:10gtek", async () => {
console.log(`[${new Date().toISOString()}] Running: 10Gtek pricing`);
await scrape10Gtek();
});
await boss.work("scrape:pricing:atgbics", async () => {
console.log(`[${new Date().toISOString()}] Running: ATGBICS pricing`);
await scrapeAtgbics();
});
await boss.work("scrape:pricing:prolabs", async () => {
console.log(`[${new Date().toISOString()}] Running: ProLabs pricing`);
await scrapeProLabs();
});
// ── Lightweight fetch/cheerio scrapers ───────────────────────────────
await boss.work("scrape:pricing:fluxlight", async () => {
console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`);
const { scrapeFluxlight } = await import("./scrapers/fluxlight");
await scrapeFluxlight();
});
await boss.work("scrape:pricing:gbics", async () => {
console.log(`[${new Date().toISOString()}] Running: GBICS pricing`);
const { scrapeGbics } = await import("./scrapers/gbics");
await scrapeGbics();
});
await boss.work("scrape:pricing:optcore", async () => {
// Optcore.net WP REST API is blocked by Cloudflare WAF for Erik's datacenter
// IP (82.165.222.127). Set SKIP_OPTCORE_SCRAPER=true to suppress the wasted
// run. A residential IP (Mac launchd) would be needed to scrape Optcore.
if (process.env["SKIP_OPTCORE_SCRAPER"] === "true") {
console.log(`[${new Date().toISOString()}] Optcore pricing: SKIPPED (SKIP_OPTCORE_SCRAPER=true)`);
return;
}
console.log(`[${new Date().toISOString()}] Running: Optcore pricing`);
const { scrapeOptcore } = await import("./scrapers/optcore");
await scrapeOptcore();
});
await boss.work("scrape:pricing:champion-one", async () => {
console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`);
const { scrapeChampionOne } = await import("./scrapers/champion-one");
await scrapeChampionOne();
});
await boss.work("scrape:pricing:sfpcables", async () => {
console.log(`[${new Date().toISOString()}] Running: SFPcables pricing`);
const { scrapeSfpCables } = await import("./scrapers/sfpcables");
await scrapeSfpCables();
});
await boss.work("scrape:pricing:blueoptics", async () => {
console.log(`[${new Date().toISOString()}] Running: BlueOptics pricing`);
const { scrapeBlueOptics } = await import("./scrapers/blueoptics");
await scrapeBlueOptics();
});
await boss.work("scrape:pricing:fiber24", async () => {
console.log(`[${new Date().toISOString()}] Running: Fiber24 pricing`);
const { scrapeFiber24 } = await import("./scrapers/fiber24");
await scrapeFiber24();
});
await boss.work("scrape:pricing:tscom", async () => {
console.log(`[${new Date().toISOString()}] Running: T&S Communication pricing`);
const { scrapeTsCom } = await import("./scrapers/tscom");
await scrapeTsCom();
});
await boss.work("scrape:pricing:skylane", async () => {
console.log(`[${new Date().toISOString()}] Running: Skylane pricing`);
const { scrapeSkylane } = await import("./scrapers/skylane");
await scrapeSkylane();
});
await boss.work("scrape:pricing:ascentoptics", async () => {
console.log(`[${new Date().toISOString()}] Running: Ascent Optics pricing`);
const { scrapeAscentOptics } = await import("./scrapers/ascentoptics");
await scrapeAscentOptics();
});
await boss.work("scrape:pricing:gaotek", async () => {
console.log(`[${new Date().toISOString()}] Running: GAO Tek pricing`);
const { scrapeGaoTek } = await import("./scrapers/gaotek");
await scrapeGaoTek();
});
// ── Catalog scrapers ──────────────────────────────────────────────────
await boss.work("scrape:pricing:flexoptix", async () => {
console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`);
await scrapeFlexoptixCatalog();
});
await boss.work("scrape:catalog:smartoptics", async () => {
console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
await scrapeSmartOptics();
});
await boss.work("scrape:catalog:hubersuhner", async () => {
console.log(`[${new Date().toISOString()}] Running: HUBER+SUHNER catalog`);
await scrapeHuberSuhner();
});
await boss.work("scrape:catalog:eoptolink", async () => {
console.log(`[${new Date().toISOString()}] Running: Eoptolink OEM catalog`);
const { scrapeEoptolink } = await import("./scrapers/eoptolink");
await scrapeEoptolink();
});
await boss.work("scrape:catalog:arista-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Arista OEM catalog seed`);
const { scrapeAristaOem } = await import("./scrapers/arista-oem");
await scrapeAristaOem();
});
await boss.work("scrape:catalog:juniper-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Juniper OEM catalog seed`);
const { scrapeJuniperOem } = await import("./scrapers/juniper-oem");
await scrapeJuniperOem();
});
await boss.work("scrape:catalog:hpe-aruba-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: HPE/Aruba OEM catalog seed`);
const { scrapeHpeArubaOem } = await import("./scrapers/hpe-aruba-oem");
await scrapeHpeArubaOem();
});
await boss.work("scrape:catalog:nokia-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Nokia OEM catalog seed`);
const { scrapeNokiaOem } = await import("./scrapers/nokia-oem");
await scrapeNokiaOem();
});
await boss.work("scrape:catalog:huawei-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Huawei OEM catalog seed`);
const { scrapeHuaweiOem } = await import("./scrapers/huawei-oem");
await scrapeHuaweiOem();
});
await boss.work("scrape:catalog:dell-emc-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Dell EMC OEM catalog seed`);
const { scrapeDellEmcOem } = await import("./scrapers/dell-emc-oem");
await scrapeDellEmcOem();
});
await boss.work("scrape:catalog:extreme-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Extreme Networks OEM catalog seed`);
const { scrapeExtremeOem } = await import("./scrapers/extreme-oem");
await scrapeExtremeOem();
});
await boss.work("scrape:catalog:nvidia-mellanox-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NVIDIA/Mellanox OEM catalog seed`);
const { scrapeNvidiaMellanoxOem } = await import("./scrapers/nvidia-mellanox-oem");
await scrapeNvidiaMellanoxOem();
});
await boss.work("scrape:catalog:brocade-ruckus-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Brocade/RUCKUS OEM catalog seed`);
const { scrapeBrocadeRuckusOem } = await import("./scrapers/brocade-ruckus-oem");
await scrapeBrocadeRuckusOem();
});
await boss.work("scrape:catalog:zte-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ZTE OEM catalog seed`);
const { scrapeZteOem } = await import("./scrapers/zte-oem");
await scrapeZteOem();
});
await boss.work("scrape:catalog:ciena-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ciena OEM catalog seed`);
const { scrapeCienaOem } = await import("./scrapers/ciena-oem");
await scrapeCienaOem();
});
await boss.work("scrape:catalog:ericsson-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ericsson OEM catalog seed`);
const { scrapeEricssonOem } = await import("./scrapers/ericsson-oem");
await scrapeEricssonOem();
});
await boss.work("scrape:catalog:fortinet-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Fortinet OEM catalog seed`);
const { scrapeFortinetOem } = await import("./scrapers/fortinet-oem");
await scrapeFortinetOem();
});
await boss.work("scrape:catalog:palo-alto-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`);
const { scrapePaloAltoOem } = await import("./scrapers/palo-alto-oem");
await scrapePaloAltoOem();
});
await boss.work("scrape:catalog:f5-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: F5 Networks OEM catalog seed`);
const { scrapeF5Oem } = await import("./scrapers/f5-oem");
await scrapeF5Oem();
});
await boss.work("scrape:catalog:ubiquiti-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ubiquiti OEM catalog seed`);
const { scrapeUbiquitiOem } = await import("./scrapers/ubiquiti-oem");
await scrapeUbiquitiOem();
});
await boss.work("scrape:catalog:mikrotik-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: MikroTik OEM catalog seed`);
const { scrapeMikrotikOem } = await import("./scrapers/mikrotik-oem");
await scrapeMikrotikOem();
});
await boss.work("scrape:catalog:netgear-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Netgear OEM catalog seed`);
const { scrapeNetgearOem } = await import("./scrapers/netgear-oem");
await scrapeNetgearOem();
});
await boss.work("scrape:catalog:adtran-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ADTRAN OEM catalog seed`);
const { scrapeAdtranOem } = await import("./scrapers/adtran-oem");
await scrapeAdtranOem();
});
await boss.work("scrape:catalog:calix-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Calix OEM catalog seed`);
const { scrapeCalixOem } = await import("./scrapers/calix-oem");
await scrapeCalixOem();
});
await boss.work("scrape:catalog:checkpoint-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Check Point OEM catalog seed`);
const { scrapeCheckpointOem } = await import("./scrapers/checkpoint-oem");
await scrapeCheckpointOem();
});
await boss.work("scrape:catalog:allied-telesis-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Allied Telesis OEM catalog seed`);
const { scrapeAlliedTelesisOem } = await import("./scrapers/allied-telesis-oem");
await scrapeAlliedTelesisOem();
});
await boss.work("scrape:catalog:zyxel-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Zyxel OEM catalog seed`);
const { scrapeZyxelOem } = await import("./scrapers/zyxel-oem");
await scrapeZyxelOem();
});
await boss.work("scrape:catalog:dlink-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: D-Link OEM catalog seed`);
const { scrapeDlinkOem } = await import("./scrapers/dlink-oem");
await scrapeDlinkOem();
});
await boss.work("scrape:catalog:ale-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Alcatel-Lucent Enterprise OEM catalog seed`);
const { scrapeAleOem } = await import("./scrapers/ale-oem");
await scrapeAleOem();
});
await boss.work("scrape:catalog:ribbon-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`);
const { scrapeRibbonOem } = await import("./scrapers/ribbon-oem");
await scrapeRibbonOem();
});
await boss.work("scrape:catalog:infinera-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Infinera OEM catalog seed`);
const { scrapeInfineraOem } = await import("./scrapers/infinera-oem");
await scrapeInfineraOem();
});
await boss.work("scrape:catalog:tplink-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: TP-Link OEM catalog seed`);
const { scrapeTplinkOem } = await import("./scrapers/tplink-oem");
await scrapeTplinkOem();
});
await boss.work("scrape:catalog:cambium-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`);
const { scrapeCambiumOem } = await import("./scrapers/cambium-oem");
await scrapeCambiumOem();
});
await boss.work("scrape:catalog:moxa-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Moxa OEM catalog seed`);
const { scrapeMoxaOem } = await import("./scrapers/moxa-oem");
await scrapeMoxaOem();
});
await boss.work("scrape:catalog:lumentum-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Lumentum OEM catalog seed`);
const { scrapeLumentumOem } = await import("./scrapers/lumentum-oem");
await scrapeLumentumOem();
});
await boss.work("scrape:catalog:coherent-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Coherent Corp OEM catalog seed`);
const { scrapeCoherentOem } = await import("./scrapers/coherent-oem");
await scrapeCoherentOem();
});
await boss.work("scrape:catalog:brocade-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Brocade OEM catalog seed`);
const { scrapeBrocadeOem } = await import("./scrapers/brocade-oem");
await scrapeBrocadeOem();
});
await boss.work("scrape:catalog:ruggedcom-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Siemens RUGGEDCOM OEM catalog seed`);
const { scrapeRuggedcomOem } = await import("./scrapers/ruggedcom-oem");
await scrapeRuggedcomOem();
});
await boss.work("scrape:catalog:hirschmann-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Hirschmann OEM catalog seed`);
const { scrapeHirschmannOem } = await import("./scrapers/hirschmann-oem");
await scrapeHirschmannOem();
});
await boss.work("scrape:catalog:viavi-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Viavi Solutions OEM catalog seed`);
const { scrapeViaviOem } = await import("./scrapers/viavi-oem");
await scrapeViaviOem();
});
await boss.work("scrape:catalog:westermo-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Westermo OEM catalog seed`);
const { scrapeWestermoOem } = await import("./scrapers/westermo-oem");
await scrapeWestermoOem();
});
await boss.work("scrape:catalog:lancom-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: LANCOM Systems OEM catalog seed`);
const { scrapeLancomOem } = await import("./scrapers/lancom-oem");
await scrapeLancomOem();
});
await boss.work("scrape:catalog:dzs-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: DZS OEM catalog seed`);
const { scrapeDzsOem } = await import("./scrapers/dzs-oem");
await scrapeDzsOem();
});
await boss.work("scrape:catalog:watchguard-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: WatchGuard OEM catalog seed`);
const { scrapeWatchguardOem } = await import("./scrapers/watchguard-oem");
await scrapeWatchguardOem();
});
await boss.work("scrape:catalog:sonicwall-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: SonicWall OEM catalog seed`);
const { scrapeSonicwallOem } = await import("./scrapers/sonicwall-oem");
await scrapeSonicwallOem();
});
await boss.work("scrape:catalog:advantech-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Advantech OEM catalog seed`);
const { scrapeAdvantechOem } = await import("./scrapers/advantech-oem");
await scrapeAdvantechOem();
});
await boss.work("scrape:catalog:redlion-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Red Lion Controls OEM catalog seed`);
const { scrapeRedlionOem } = await import("./scrapers/redlion-oem");
await scrapeRedlionOem();
});
await boss.work("scrape:catalog:exfo-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: EXFO OEM catalog seed`);
const { scrapeExfoOem } = await import("./scrapers/exfo-oem");
await scrapeExfoOem();
});
await boss.work("scrape:catalog:mellanox-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NVIDIA Mellanox OEM catalog seed`);
const { scrapeMellanoxOem } = await import("./scrapers/mellanox-oem");
await scrapeMellanoxOem();
});
await boss.work("scrape:catalog:intel-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Intel OEM catalog seed`);
const { scrapeIntelOem } = await import("./scrapers/intel-oem");
await scrapeIntelOem();
});
await boss.work("scrape:catalog:antaira-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Antaira Technologies OEM catalog seed`);
const { scrapeAntairaOem } = await import("./scrapers/antaira-oem");
await scrapeAntairaOem();
});
await boss.work("scrape:catalog:transition-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Transition Networks OEM catalog seed`);
const { scrapeTransitionNetworksOem } = await import("./scrapers/transition-networks-oem");
await scrapeTransitionNetworksOem();
});
await boss.work("scrape:catalog:perle-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Perle Systems OEM catalog seed`);
const { scrapePerleOem } = await import("./scrapers/perle-oem");
await scrapePerleOem();
});
await boss.work("scrape:catalog:barracuda-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Barracuda Networks OEM catalog seed`);
const { scrapeBarracudaOem } = await import("./scrapers/barracuda-oem");
await scrapeBarracudaOem();
});
await boss.work("scrape:catalog:etherwan-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: EtherWAN Systems OEM catalog seed`);
const { scrapeEtherwanOem } = await import("./scrapers/etherwan-oem");
await scrapeEtherwanOem();
});
await boss.work("scrape:catalog:blackbox-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`);
const { scrapeBlackboxOem } = await import("./scrapers/blackbox-oem");
await scrapeBlackboxOem();
});
await boss.work("scrape:catalog:packetlight-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: PacketLight Networks OEM catalog seed`);
const { scrapePacketLightOem } = await import("./scrapers/packetlight-oem");
await scrapePacketLightOem();
});
await boss.work("scrape:catalog:spirent-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Spirent Communications OEM catalog seed`);
const { scrapeSpirentOem } = await import("./scrapers/spirent-oem");
await scrapeSpirentOem();
});
await boss.work("scrape:catalog:comnet-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Comnet OEM catalog seed`);
const { scrapeComnetOem } = await import("./scrapers/comnet-oem");
await scrapeComnetOem();
});
await boss.work("scrape:catalog:korenix-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Korenix Technology OEM catalog seed`);
const { scrapeKorenixOem } = await import("./scrapers/korenix-oem");
await scrapeKorenixOem();
});
await boss.work("scrape:catalog:sophos-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Sophos OEM catalog seed`);
const { scrapeSophosOem } = await import("./scrapers/sophos-oem");
await scrapeSophosOem();
});
await boss.work("scrape:catalog:cradlepoint-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cradlepoint OEM catalog seed`);
const { scrapeCradlepointOem } = await import("./scrapers/cradlepoint-oem");
await scrapeCradlepointOem();
});
await boss.work("scrape:catalog:fujitsu-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Fujitsu Network Communications OEM catalog seed`);
const { scrapeFujitsuOem } = await import("./scrapers/fujitsu-oem");
await scrapeFujitsuOem();
});
await boss.work("scrape:catalog:nec-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NEC Corporation OEM catalog seed`);
const { scrapeNecOem } = await import("./scrapers/nec-oem");
await scrapeNecOem();
});
await boss.work("scrape:catalog:arris-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: CommScope ARRIS OEM catalog seed`);
const { scrapeArrisOem } = await import("./scrapers/arris-oem");
await scrapeArrisOem();
});
await boss.work("scrape:catalog:hillstone-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Hillstone Networks OEM catalog seed`);
const { scrapeHillstoneOem } = await import("./scrapers/hillstone-oem");
await scrapeHillstoneOem();
});
await boss.work("scrape:catalog:vecima-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Vecima Networks OEM catalog seed`);
const { scrapeVecimaOem } = await import("./scrapers/vecima-oem");
await scrapeVecimaOem();
});
await boss.work("scrape:catalog:stordis-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Stordis OEM catalog seed`);
const { scrapeStordisOem } = await import("./scrapers/stordis-oem");
await scrapeStordisOem();
});
await boss.work("scrape:catalog:keysight-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Keysight OEM catalog seed`);
const { scrapeKeysightOem } = await import("./scrapers/keysight-oem");
await scrapeKeysightOem();
});
await boss.work("scrape:catalog:sycamore-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Sycamore OEM catalog seed`);
const { scrapeSycamoreOem } = await import("./scrapers/sycamore-oem");
await scrapeSycamoreOem();
});
await boss.work("scrape:catalog:ekinops-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ekinops OEM catalog seed`);
const { scrapeEkinopsOem } = await import("./scrapers/ekinops-oem");
await scrapeEkinopsOem();
});
await boss.work("scrape:catalog:adva-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ADVA Optical OEM catalog seed`);
const { scrapeAdvaOem } = await import("./scrapers/adva-oem");
await scrapeAdvaOem();
});
await boss.work("scrape:catalog:coriant-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Coriant OEM catalog seed`);
const { scrapeCoriantOem } = await import("./scrapers/coriant-oem");
await scrapeCoriantOem();
});
await boss.work("scrape:catalog:casa-systems-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Casa Systems OEM catalog seed`);
const { scrapeCasaSystemsOem } = await import("./scrapers/casa-systems-oem");
await scrapeCasaSystemsOem();
});
await boss.work("scrape:catalog:harmonic-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Harmonic OEM catalog seed`);
const { scrapeHarmonicOem } = await import("./scrapers/harmonic-oem");
await scrapeHarmonicOem();
});
await boss.work("scrape:catalog:solarflare-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Solarflare OEM catalog seed`);
const { scrapeSolarflareOem } = await import("./scrapers/solarflare-oem");
await scrapeSolarflareOem();
});
await boss.work("scrape:catalog:marvell-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Marvell OEM catalog seed`);
const { scrapeMarvellOem } = await import("./scrapers/marvell-oem");
await scrapeMarvellOem();
});
await boss.work("scrape:catalog:broadcom-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Broadcom OEM catalog seed`);
const { scrapeBroadcomOem } = await import("./scrapers/broadcom-oem");
await scrapeBroadcomOem();
});
await boss.work("scrape:catalog:calix-access-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Calix Access OEM catalog seed`);
const { scrapeCalixAccessOem } = await import("./scrapers/calix-access-oem");
await scrapeCalixAccessOem();
});
await boss.work("scrape:catalog:ribbon-comms-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`);
const { scrapeRibbonCommsOem } = await import("./scrapers/ribbon-comms-oem");
await scrapeRibbonCommsOem();
});
await boss.work("scrape:catalog:infinera-groove-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Infinera Groove OEM catalog seed`);
const { scrapeInfineraGrooveOem } = await import("./scrapers/infinera-groove-oem");
await scrapeInfineraGrooveOem();
});
await boss.work("scrape:catalog:ciena-waveserver-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ciena WaveServer OEM catalog seed`);
const { scrapeCienaWaveserverOem } = await import("./scrapers/ciena-waveserver-oem");
await scrapeCienaWaveserverOem();
});
await boss.work("scrape:catalog:commscope-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: CommScope OEM catalog seed`);
const { scrapeCommScopeOem } = await import("./scrapers/commscope-oem");
await scrapeCommScopeOem();
});
await boss.work("scrape:catalog:teleste-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Teleste OEM catalog seed`);
const { scrapeTelesteOem } = await import("./scrapers/teleste-oem");
await scrapeTelesteOem();
});
await boss.work("scrape:catalog:tejas-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Tejas Networks OEM catalog seed`);
const { scrapeTejasNetworksOem } = await import("./scrapers/tejas-networks-oem");
await scrapeTejasNetworksOem();
});
await boss.work("scrape:catalog:ericsson-transport-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ericsson Transport OEM catalog seed`);
const { scrapeEricssonTransportOem } = await import("./scrapers/ericsson-transport-oem");
await scrapeEricssonTransportOem();
});
await boss.work("scrape:catalog:adtran-ta-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ADTRAN TA OEM catalog seed`);
const { scrapeAdtranTaOem } = await import("./scrapers/adtran-ta-oem");
await scrapeAdtranTaOem();
});
await boss.work("scrape:catalog:isolan-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: iSolan OEM catalog seed`);
const { scrapeIsolanOem } = await import("./scrapers/isolan-oem");
await scrapeIsolanOem();
});
await boss.work("scrape:catalog:telco-systems-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: TELCO Systems OEM catalog seed`);
const { scrapeTelcoSystemsOem } = await import("./scrapers/telco-systems-oem");
await scrapeTelcoSystemsOem();
});
await boss.work("scrape:catalog:rad-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: RAD Data Communications OEM catalog seed`);
const { scrapeRadOem } = await import("./scrapers/rad-oem");
await scrapeRadOem();
});
await boss.work("scrape:catalog:comtrend-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Comtrend OEM catalog seed`);
const { scrapeComtrendOem } = await import("./scrapers/comtrend-oem");
await scrapeComtrendOem();
});
await boss.work("scrape:catalog:packetfront-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: PacketFront OEM catalog seed`);
const { scrapePacketfrontOem } = await import("./scrapers/packetfront-oem");
await scrapePacketfrontOem();
});
await boss.work("scrape:catalog:edgewater-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Edgewater Networks OEM catalog seed`);
const { scrapeEdgewaterNetworksOem } = await import("./scrapers/edgewater-networks-oem");
await scrapeEdgewaterNetworksOem();
});
await boss.work("scrape:catalog:corning-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Corning OEM catalog seed`);
const { scrapeCorningOem } = await import("./scrapers/corning-oem");
await scrapeCorningOem();
});
await boss.work("scrape:catalog:ofs-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: OFS OEM catalog seed`);
const { scrapeOfsOem } = await import("./scrapers/ofs-oem");
await scrapeOfsOem();
});
await boss.work("scrape:catalog:kontron-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Kontron OEM catalog seed`);
const { scrapeKontronOem } = await import("./scrapers/kontron-oem");
await scrapeKontronOem();
});
await boss.work("scrape:catalog:ipinfusion-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: IP Infusion OEM catalog seed`);
const { scrapeIpInfusionOem } = await import("./scrapers/ipinfusion-oem");
await scrapeIpInfusionOem();
});
await boss.work("scrape:catalog:telrad-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Telrad Networks OEM catalog seed`);
const { scrapeTelradOem } = await import("./scrapers/telrad-oem");
await scrapeTelradOem();
});
await boss.work("scrape:catalog:siklu-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Siklu OEM catalog seed`);
const { scrapeSikluOem } = await import("./scrapers/siklu-oem");
await scrapeSikluOem();
});
await boss.work("scrape:catalog:ceragon-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ceragon OEM catalog seed`);
const { scrapeCeragonOem } = await import("./scrapers/ceragon-oem");
await scrapeCeragonOem();
});
await boss.work("scrape:catalog:datang-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Datang Telecom OEM catalog seed`);
const { scrapeDatangOem } = await import("./scrapers/datang-oem");
await scrapeDatangOem();
});
await boss.work("scrape:catalog:viptela-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Viptela OEM catalog seed`);
const { scrapeViptelaOem } = await import("./scrapers/viptela-oem");
await scrapeViptelaOem();
});
await boss.work("scrape:catalog:versa-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Versa Networks OEM catalog seed`);
const { scrapeVersaNetworksOem } = await import("./scrapers/versa-networks-oem");
await scrapeVersaNetworksOem();
});
await boss.work("scrape:catalog:vmware-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: VMware OEM catalog seed`);
const { scrapeVmwareOem } = await import("./scrapers/vmware-oem");
await scrapeVmwareOem();
});
await boss.work("scrape:catalog:cimc-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: CIMC Semiconductors OEM catalog seed`);
const { scrapeCimcOem } = await import("./scrapers/cimc-oem");
await scrapeCimcOem();
});
await boss.work("scrape:catalog:qlogic-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: QLogic OEM catalog seed`);
const { scrapeQlogicOem } = await import("./scrapers/qlogic-oem");
await scrapeQlogicOem();
});
await boss.work("scrape:catalog:emulex-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Emulex OEM catalog seed`);
const { scrapeEmulexOem } = await import("./scrapers/emulex-oem");
await scrapeEmulexOem();
});
await boss.work("scrape:catalog:netapp-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NetApp OEM catalog seed`);
const { scrapeNetappOem } = await import("./scrapers/netapp-oem");
await scrapeNetappOem();
});
await boss.work("scrape:catalog:pure-storage-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Pure Storage OEM catalog seed`);
const { scrapePureStorageOem } = await import("./scrapers/pure-storage-oem");
await scrapePureStorageOem();
});
await boss.work("scrape:catalog:hpe-storage-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: HPE Storage OEM catalog seed`);
const { scrapeHpeStorageOem } = await import("./scrapers/hpe-storage-oem");
await scrapeHpeStorageOem();
});
await boss.work("scrape:catalog:ibm-storage-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: IBM Storage OEM catalog seed`);
const { scrapeIbmStorageOem } = await import("./scrapers/ibm-storage-oem");
await scrapeIbmStorageOem();
});
await boss.work("scrape:catalog:dell-storage-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Dell Storage OEM catalog seed`);
const { scrapeDellStorageOem } = await import("./scrapers/dell-storage-oem");
await scrapeDellStorageOem();
});
await boss.work("scrape:catalog:hitachi-vantara-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Hitachi Vantara OEM catalog seed`);
const { scrapeHitachiVantaraOem } = await import("./scrapers/hitachi-vantara-oem");
await scrapeHitachiVantaraOem();
});
await boss.work("scrape:catalog:aws-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: AWS OEM catalog seed`);
const { scrapeAwsOem } = await import("./scrapers/aws-oem");
await scrapeAwsOem();
});
await boss.work("scrape:catalog:azure-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Microsoft Azure OEM catalog seed`);
const { scrapeAzureOem } = await import("./scrapers/azure-oem");
await scrapeAzureOem();
});
await boss.work("scrape:catalog:google-cloud-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Google Cloud OEM catalog seed`);
const { scrapeGoogleCloudOem } = await import("./scrapers/google-cloud-oem");
await scrapeGoogleCloudOem();
});
await boss.work("scrape:catalog:meta-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Meta OEM catalog seed`);
const { scrapeMetaOem } = await import("./scrapers/meta-oem");
await scrapeMetaOem();
});
await boss.work("scrape:catalog:nokia-access-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Nokia Access OEM catalog seed`);
const { scrapeNokiaAccessOem } = await import("./scrapers/nokia-access-oem");
await scrapeNokiaAccessOem();
});
await boss.work("scrape:catalog:huawei-access-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Huawei Access OEM catalog seed`);
const { scrapeHuaweiAccessOem } = await import("./scrapers/huawei-access-oem");
await scrapeHuaweiAccessOem();
});
await boss.work("scrape:catalog:zte-access-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ZTE Access OEM catalog seed`);
const { scrapeZteAccessOem } = await import("./scrapers/zte-access-oem");
await scrapeZteAccessOem();
});
await boss.work("scrape:catalog:calix-gigapoint-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Calix GigaPoint OEM catalog seed`);
const { scrapeCalixGigapointOem } = await import("./scrapers/calix-gigapoint-oem");
await scrapeCalixGigapointOem();
});
await boss.work("scrape:catalog:samsung-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Samsung Networks OEM catalog seed`);
const { scrapeSamsungNetworksOem } = await import("./scrapers/samsung-networks-oem");
await scrapeSamsungNetworksOem();
});
await boss.work("scrape:catalog:nokia-airscale-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Nokia AirScale OEM catalog seed`);
const { scrapeNokiaAirscaleOem } = await import("./scrapers/nokia-airscale-oem");
await scrapeNokiaAirscaleOem();
});
await boss.work("scrape:catalog:ericsson-ran-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ericsson RAN OEM catalog seed`);
const { scrapeEricssonRanOem } = await import("./scrapers/ericsson-ran-oem");
await scrapeEricssonRanOem();
});
await boss.work("scrape:catalog:mavenir-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Mavenir OEM catalog seed`);
const { scrapeMavenirOem } = await import("./scrapers/mavenir-oem");
await scrapeMavenirOem();
});
await boss.work("scrape:catalog:ixia-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ixia OEM catalog seed`);
const { scrapeIxiaOem } = await import("./scrapers/ixia-oem");
await scrapeIxiaOem();
});
await boss.work("scrape:catalog:exfo-network-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: EXFO Network OEM catalog seed`);
const { scrapeExfoNetworkOem } = await import("./scrapers/exfo-network-oem");
await scrapeExfoNetworkOem();
});
await boss.work("scrape:catalog:cumulus-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cumulus Networks OEM catalog seed`);
const { scrapeCumulusNetworksOem } = await import("./scrapers/cumulus-networks-oem");
await scrapeCumulusNetworksOem();
});
await boss.work("scrape:catalog:sonic-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: SONiC OEM catalog seed`);
const { scrapeSonicOem } = await import("./scrapers/sonic-oem");
await scrapeSonicOem();
});
await boss.work("scrape:catalog:h3c-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: H3C OEM catalog seed`);
const { scrapeH3cOem } = await import("./scrapers/h3c-oem");
await scrapeH3cOem();
});
await boss.work("scrape:catalog:ruijie-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Ruijie Networks OEM catalog seed`);
const { scrapeRuijieOem } = await import("./scrapers/ruijie-oem");
await scrapeRuijieOem();
});
await boss.work("scrape:catalog:centec-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Centec Networks OEM catalog seed`);
const { scrapeCentecOem } = await import("./scrapers/centec-oem");
await scrapeCentecOem();
});
await boss.work("scrape:catalog:supermicro-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Supermicro OEM catalog seed`);
const { scrapeSupermicroOem } = await import("./scrapers/supermicro-oem");
await scrapeSupermicroOem();
});
await boss.work("scrape:catalog:cisco-meraki-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco Meraki OEM catalog seed`);
const { scrapeCiscoMerakiOem } = await import("./scrapers/cisco-meraki-oem");
await scrapeCiscoMerakiOem();
});
await boss.work("scrape:catalog:cisco-catalyst-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco Catalyst OEM catalog seed`);
const { scrapeCiscoCatalystOem } = await import("./scrapers/cisco-catalyst-oem");
await scrapeCiscoCatalystOem();
});
await boss.work("scrape:catalog:cisco-nexus-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco Nexus OEM catalog seed`);
const { scrapeCiscoNexusOem } = await import("./scrapers/cisco-nexus-oem");
await scrapeCiscoNexusOem();
});
await boss.work("scrape:catalog:cisco-asr-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco ASR OEM catalog seed`);
const { scrapeCiscoAsrOem } = await import("./scrapers/cisco-asr-oem");
await scrapeCiscoAsrOem();
});
await boss.work("scrape:catalog:juniper-mx-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Juniper MX OEM catalog seed`);
const { scrapeJuniperMxOem } = await import("./scrapers/juniper-mx-oem");
await scrapeJuniperMxOem();
});
await boss.work("scrape:catalog:juniper-qfx-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Juniper QFX OEM catalog seed`);
const { scrapeJuniperQfxOem } = await import("./scrapers/juniper-qfx-oem");
await scrapeJuniperQfxOem();
});
await boss.work("scrape:catalog:aruba-cx-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Aruba CX OEM catalog seed`);
const { scrapeArubaCxOem } = await import("./scrapers/aruba-cx-oem");
await scrapeArubaCxOem();
});
await boss.work("scrape:catalog:extreme-campus-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Extreme Campus OEM catalog seed`);
const { scrapeExtremeCampusOem } = await import("./scrapers/extreme-campus-oem");
await scrapeExtremeCampusOem();
});
await boss.work("scrape:catalog:arista-7000-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Arista 7000 OEM catalog seed`);
const { scrapeArista7000Oem } = await import("./scrapers/arista-7000-oem");
await scrapeArista7000Oem();
});
await boss.work("scrape:catalog:pica8-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Pica8 OEM catalog seed`);
const { scrapePica8Oem } = await import("./scrapers/pica8-oem");
await scrapePica8Oem();
});
await boss.work("scrape:catalog:pluribus-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Pluribus Networks OEM catalog seed`);
const { scrapePluribusOem } = await import("./scrapers/pluribus-oem");
await scrapePluribusOem();
});
await boss.work("scrape:catalog:drivenets-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: DriveNets OEM catalog seed`);
const { scrapeDrivenetsOem } = await import("./scrapers/drivenets-oem");
await scrapeDrivenetsOem();
});
await boss.work("scrape:catalog:phoenix-contact-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Phoenix Contact OEM catalog seed`);
const { scrapePhoenixContactOem } = await import("./scrapers/phoenix-contact-oem");
await scrapePhoenixContactOem();
});
await boss.work("scrape:catalog:beckhoff-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Beckhoff OEM catalog seed`);
const { scrapeBeckhoffOem } = await import("./scrapers/beckhoff-oem");
await scrapeBeckhoffOem();
});
await boss.work("scrape:catalog:omron-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Omron OEM catalog seed`);
const { scrapeOmronOem } = await import("./scrapers/omron-oem");
await scrapeOmronOem();
});
await boss.work("scrape:catalog:abb-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ABB OEM catalog seed`);
const { scrapeAbbOem } = await import("./scrapers/abb-oem");
await scrapeAbbOem();
});
await boss.work("scrape:catalog:siemens-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Siemens SCALANCE OEM catalog seed`);
const { scrapeSiemensOem } = await import("./scrapers/siemens-oem");
await scrapeSiemensOem();
});
await boss.work("scrape:catalog:schneider-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Schneider Electric OEM catalog seed`);
const { scrapeSchneiderOem } = await import("./scrapers/schneider-oem");
await scrapeSchneiderOem();
});
await boss.work("scrape:catalog:rockwell-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Rockwell Automation OEM catalog seed`);
const { scrapeRockwellOem } = await import("./scrapers/rockwell-oem");
await scrapeRockwellOem();
});
await boss.work("scrape:catalog:belden-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Belden OEM catalog seed`);
const { scrapeBeldenOem } = await import("./scrapers/belden-oem");
await scrapeBeldenOem();
});
await boss.work("scrape:catalog:ge-grid-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: GE Grid Solutions OEM catalog seed`);
const { scrapeGeGridOem } = await import("./scrapers/ge-grid-oem");
await scrapeGeGridOem();
});
await boss.work("scrape:catalog:schweitzer-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Schweitzer Engineering OEM catalog seed`);
const { scrapeSchweiterEngOem } = await import("./scrapers/schweitzer-oem");
await scrapeSchweiterEngOem();
});
await boss.work("scrape:catalog:moxa-industrial-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Moxa Industrial OEM catalog seed`);
const { scrapeIndustrialMoxaOem } = await import("./scrapers/moxa-industrial-oem");
await scrapeIndustrialMoxaOem();
});
await boss.work("scrape:catalog:cisco-ie-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco Industrial Ethernet OEM catalog seed`);
const { scrapeCiscoIeOem } = await import("./scrapers/cisco-ie-oem");
await scrapeCiscoIeOem();
});
// ── Media / Broadcast / Pro-AV ────────────────────────────────────────
await boss.work("scrape:catalog:evertz-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Evertz Microsystems OEM catalog seed`);
const { scrapeEvertzOem } = await import("./scrapers/evertz-oem");
await scrapeEvertzOem();
});
await boss.work("scrape:catalog:grass-valley-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Grass Valley OEM catalog seed`);
const { scrapeGrassValleyOem } = await import("./scrapers/grass-valley-oem");
await scrapeGrassValleyOem();
});
await boss.work("scrape:catalog:haivision-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Haivision OEM catalog seed`);
const { scrapeHaivisionOem } = await import("./scrapers/haivision-oem");
await scrapeHaivisionOem();
});
await boss.work("scrape:catalog:viasat-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Viasat OEM catalog seed`);
const { scrapeViasatOem } = await import("./scrapers/viasat-oem");
await scrapeViasatOem();
});
// ── Asian Optical Vendors ─────────────────────────────────────────────
await boss.work("scrape:catalog:fiberhome-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: FiberHome Technologies OEM catalog seed`);
const { scrapeFiberhomeOem } = await import("./scrapers/fiberhome-oem");
await scrapeFiberhomeOem();
});
await boss.work("scrape:catalog:oplink-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Oplink Communications OEM catalog seed`);
const { scrapeOplinkOem } = await import("./scrapers/oplink-oem");
await scrapeOplinkOem();
});
await boss.work("scrape:catalog:accelink-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Accelink Technologies OEM catalog seed`);
const { scrapeAccelinkOem } = await import("./scrapers/accelink-oem");
await scrapeAccelinkOem();
});
await boss.work("scrape:catalog:hisense-broadband-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Hisense Broadband OEM catalog seed`);
const { scrapeHisenseBroadbandOem } = await import("./scrapers/hisense-broadband-oem");
await scrapeHisenseBroadbandOem();
});
// ── Optical Transceiver Manufacturers (batch 24–25) ───────────────────
await boss.work("scrape:catalog:ii-vi-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: II-VI / Coherent OEM catalog seed`);
const { scrapeIiViOem } = await import("./scrapers/ii-vi-oem");
await scrapeIiViOem();
});
await boss.work("scrape:catalog:source-photonics-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Source Photonics OEM catalog seed`);
const { scrapeSourcePhotonicsOem } = await import("./scrapers/source-photonics-oem");
await scrapeSourcePhotonicsOem();
});
await boss.work("scrape:catalog:o-net-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: O-Net Technologies OEM catalog seed`);
const { scrapeONetOem } = await import("./scrapers/o-net-oem");
await scrapeONetOem();
});
await boss.work("scrape:catalog:innolight-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: InnoLight Technology OEM catalog seed`);
const { scrapeInnoLightOem } = await import("./scrapers/innolight-oem");
await scrapeInnoLightOem();
});
await boss.work("scrape:catalog:aoi-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Applied Optoelectronics (AOI) OEM catalog seed`);
const { scrapeAoiOem } = await import("./scrapers/aoi-oem");
await scrapeAoiOem();
});
await boss.work("scrape:catalog:sumitomo-electric-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Sumitomo Electric Networks OEM catalog seed`);
const { scrapeSumitomoElectricOem } = await import("./scrapers/sumitomo-electric-oem");
await scrapeSumitomoElectricOem();
});
await boss.work("scrape:catalog:neophotonics-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NeoPhotonics OEM catalog seed`);
const { scrapeNeoPhotonicsOem } = await import("./scrapers/neophotonics-oem");
await scrapeNeoPhotonicsOem();
});
await boss.work("scrape:catalog:audiocodes-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: AudioCodes OEM catalog seed`);
const { scrapeAudioCodesOem } = await import("./scrapers/audiocodes-oem");
await scrapeAudioCodesOem();
});
await boss.work("scrape:catalog:anritsu-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Anritsu OEM catalog seed`);
const { scrapeAnritsuOem } = await import("./scrapers/anritsu-oem");
await scrapeAnritsuOem();
});
await boss.work("scrape:catalog:netscout-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: NETSCOUT OEM catalog seed`);
const { scrapeNetScoutOem } = await import("./scrapers/netscout-oem");
await scrapeNetScoutOem();
});
await boss.work("scrape:catalog:curtiss-wright-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Curtiss-Wright OEM catalog seed`);
const { scrapeCurtissWrightOem } = await import("./scrapers/curtiss-wright-oem");
await scrapeCurtissWrightOem();
});
await boss.work("scrape:catalog:eci-telecom-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: ECI Telecom OEM catalog seed`);
const { scrapeEciTelecomOem } = await import("./scrapers/eci-telecom-oem");
await scrapeEciTelecomOem();
});
await boss.work("scrape:catalog:utstarcom-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: UTStarcom OEM catalog seed`);
const { scrapeUTStarcomOem } = await import("./scrapers/utstarcom-oem");
await scrapeUTStarcomOem();
});
await boss.work("scrape:catalog:turbolink-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Turbolink OEM catalog seed`);
const { scrapeTurbolinkOem } = await import("./scrapers/turbolink-oem");
await scrapeTurbolinkOem();
});
await boss.work("scrape:catalog:chelsio-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Chelsio OEM catalog seed`);
const { scrapeChelsioOem } = await import("./scrapers/chelsio-oem");
await scrapeChelsioOem();
});
await boss.work("scrape:catalog:rohde-schwarz-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Rohde & Schwarz OEM catalog seed`);
const { scrapeRohdeScharzOem } = await import("./scrapers/rohde-schwarz-oem");
await scrapeRohdeScharzOem();
});
await boss.work("scrape:catalog:l3harris-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: L3Harris OEM catalog seed`);
const { scrapeL3HarrisOem } = await import("./scrapers/l3harris-oem");
await scrapeL3HarrisOem();
});
await boss.work("scrape:catalog:zhone-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Zhone/DZS OEM catalog seed`);
const { scrapeZhoneOem } = await import("./scrapers/zhone-oem");
await scrapeZhoneOem();
});
await boss.work("scrape:catalog:cambium-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`);
const { scrapeCambiumNetworksOem } = await import("./scrapers/cambium-networks-oem");
await scrapeCambiumNetworksOem();
});
await boss.work("scrape:catalog:tektronix-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Tektronix OEM catalog seed`);
const { scrapeTektronixOem } = await import("./scrapers/tektronix-oem");
await scrapeTektronixOem();
});
await boss.work("scrape:catalog:clearfield-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Clearfield OEM catalog seed`);
const { scrapeClearfieldOem } = await import("./scrapers/clearfield-oem");
await scrapeClearfieldOem();
});
await boss.work("scrape:catalog:lanner-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Lanner Electronics OEM catalog seed`);
const { scrapeLannerOem } = await import("./scrapers/lanner-oem");
await scrapeLannerOem();
});
await boss.work("scrape:catalog:black-box-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`);
const { scrapeBlackBoxOem } = await import("./scrapers/black-box-oem");
await scrapeBlackBoxOem();
});
await boss.work("scrape:catalog:radiflow-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Radiflow OEM catalog seed`);
const { scrapeRadiflowOem } = await import("./scrapers/radiflow-oem");
await scrapeRadiflowOem();
});
await boss.work("scrape:catalog:dragonwave-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: DragonWave OEM catalog seed`);
const { scrapeDragonwaveOem } = await import("./scrapers/dragonwave-oem");
await scrapeDragonwaveOem();
});
await boss.work("scrape:catalog:teledyne-lecroy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Teledyne LeCroy OEM catalog seed`);
const { scrapeTeledyneLeCroyOem } = await import("./scrapers/teledyne-lecroy-oem");
await scrapeTeledyneLeCroyOem();
});
await boss.work("scrape:catalog:finisar-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Finisar OEM catalog seed`);
const { scrapeFinisarOem } = await import("./scrapers/finisar-oem");
await scrapeFinisarOem();
});
await boss.work("scrape:catalog:acacia-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Acacia Communications OEM catalog seed`);
const { scrapeAcaciaOem } = await import("./scrapers/acacia-oem");
await scrapeAcaciaOem();
});
await boss.work("scrape:catalog:inphi-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Inphi (Marvell) OEM catalog seed`);
const { scrapeInphiOem } = await import("./scrapers/inphi-oem");
await scrapeInphiOem();
});
// ── Batch 35 workers ─────────────────────────────────────────────────
await boss.work("scrape:catalog:sierra-wireless-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Sierra Wireless (Semtech) OEM catalog seed`);
const { scrapeSierraWirelessOem } = await import("./scrapers/sierra-wireless-oem");
await scrapeSierraWirelessOem();
});
await boss.work("scrape:catalog:senao-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Senao (EnGenius) OEM catalog seed`);
const { scrapeSenaoOem } = await import("./scrapers/senao-oem");
await scrapeSenaoOem();
});
await boss.work("scrape:catalog:emcore-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: EMCORE OEM catalog seed`);
const { scrapeEmcoreOem } = await import("./scrapers/emcore-oem");
await scrapeEmcoreOem();
});
await boss.work("scrape:catalog:reflex-photonics-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Reflex Photonics OEM catalog seed`);
const { scrapeReflexPhotonicsOem } = await import("./scrapers/reflex-photonics-oem");
await scrapeReflexPhotonicsOem();
});
// ── Batch 36 workers ─────────────────────────────────────────────────
await boss.work("scrape:catalog:engenius-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: EnGenius OEM catalog seed`);
const { scrapeEngeniusOem } = await import("./scrapers/engenius-oem");
await scrapeEngeniusOem();
});
await boss.work("scrape:catalog:paloalto-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`);
const { scrapePaloaltoNetworksOem } = await import("./scrapers/paloalto-networks-oem");
await scrapePaloaltoNetworksOem();
});
await boss.work("scrape:catalog:brocade-legacy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Brocade (Legacy) OEM catalog seed`);
const { scrapeBrocadeLegacyOem } = await import("./scrapers/brocade-legacy-oem");
await scrapeBrocadeLegacyOem();
});
await boss.work("scrape:catalog:foundry-networks-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Foundry Networks (Legacy) OEM catalog seed`);
const { scrapeFoundryNetworksOem } = await import("./scrapers/foundry-networks-oem");
await scrapeFoundryNetworksOem();
});
// ── Batch 37 workers ─────────────────────────────────────────────────
await boss.work("scrape:catalog:extreme-legacy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Extreme Networks (Legacy) OEM catalog seed`);
const { scrapeExtremeLegacyOem } = await import("./scrapers/extreme-legacy-oem");
await scrapeExtremeLegacyOem();
});
await boss.work("scrape:catalog:nortel-legacy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Nortel Networks (Legacy) OEM catalog seed`);
const { scrapeNortelLegacyOem } = await import("./scrapers/nortel-legacy-oem");
await scrapeNortelLegacyOem();
});
await boss.work("scrape:catalog:3com-legacy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: 3Com (Legacy) OEM catalog seed`);
const { scrape3comLegacyOem } = await import("./scrapers/3com-legacy-oem");
await scrape3comLegacyOem();
});
await boss.work("scrape:catalog:avaya-legacy-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Avaya (Legacy) OEM catalog seed`);
const { scrapeAvayaLegacyOem } = await import("./scrapers/avaya-legacy-oem");
await scrapeAvayaLegacyOem();
});
// ── Vendor lists ──────────────────────────────────────────────────────
await boss.work("scrape:vendors:flexoptix", async () => {
console.log(`[${new Date().toISOString()}] Running: Flexoptix vendor list`);
await scrapeFlexoptixVendors();
});
await boss.work("scrape:vendors:flexoptix-supported", async () => {
console.log(`[${new Date().toISOString()}] Running: Flexoptix supported vendors`);
await seedFlexoptixVendors();
});
// ── Compatibility scrapers ────────────────────────────────────────────
await boss.work("scrape:compat:flexoptix", async () => {
console.log(`[${new Date().toISOString()}] Running: Flexoptix compatibility mapping`);
if (!isLoadAcceptable(2.5)) {
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Flexoptix compat scrape`);
return;
}
await scrapeFlexoptixCompatibility();
});
await boss.work("scrape:compat:cisco", async () => {
console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`);
await scrapeCiscoTmg();
});
await boss.work("scrape:compat:juniper", async () => {
console.log(`[${new Date().toISOString()}] Running: Juniper HCT compatibility`);
await scrapeJuniperHct();
});
await boss.work("scrape:compat:sonic", async () => {
console.log(`[${new Date().toISOString()}] Running: SONiC HCL compatibility`);
await scrapeSonicHcl();
});
await boss.work("scrape:compat:ufispace", async () => {
console.log(`[${new Date().toISOString()}] Running: Ufispace switch data`);
await scrapeUfiSpace();
});
await boss.work("scrape:compat:edgecore", async () => {
console.log(`[${new Date().toISOString()}] Running: Edgecore switch data`);
await scrapeEdgecore();
});
// ── Switch assets ─────────────────────────────────────────────────────
await boss.work("scrape:assets:switches", async () => {
console.log(`[${new Date().toISOString()}] Running: Switch assets enrichment`);
await scrapeSwitchAssets();
});
await boss.work("scrape:images:switches", async () => {
console.log(`[${new Date().toISOString()}] Running: Switch og:image fetcher`);
if (!isLoadAcceptable(2.5)) {
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping switch image fetch`);
return;
}
await fetchSwitchImages();
});
await boss.work("scrape:images:switches:playwright", async () => {
console.log(`[${new Date().toISOString()}] Running: Switch image fetcher (Playwright — bot-blocked vendors)`);
if (!isLoadAcceptable(2.0)) {
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Playwright image fetch`);
return;
}
await fetchSwitchImagesPlaywright();
});
// ── eBay enrichment ───────────────────────────────────────────────────
await boss.work("enrich:ebay-transceivers", async () => {
console.log(`[${new Date().toISOString()}] Running: eBay transceiver pricing`);
const { enrichTransceiversFromEbay } = await import("./scrapers/ebay-enricher");
await enrichTransceiversFromEbay(100);
});
await boss.work("enrich:ebay-switches", async () => {
console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`);
const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher");
await enrichSwitchesFromEbay(30);
});
// ── Intelligence & community ──────────────────────────────────────────
await boss.work("scrape:market-intel", async () => {
console.log(`[${new Date().toISOString()}] Running: Market intelligence`);
await scrapeMarketIntelligence();
});
await boss.work("scrape:nog-talks", async () => {
console.log(`[${new Date().toISOString()}] Running: NOG conference talks`);
const { scrapeNogTalks } = await import("./scrapers/nog-talks");
await scrapeNogTalks();
});
await boss.work("scrape:community-issues", async () => {
console.log(`[${new Date().toISOString()}] Running: Community issues`);
const { scrapeAllSwitchIssues, scrapeTransceiverCompatIssues } = await import("./scrapers/community-issues");
await scrapeAllSwitchIssues(30);
await scrapeTransceiverCompatIssues(15);
});
await boss.work("scrape:datasheet-links", async () => {
console.log(`[${new Date().toISOString()}] Running: Datasheet links`);
const { findAndSeedDatasheetLinks } = await import("./scrapers/community-issues");
await findAndSeedDatasheetLinks(50);
});
await boss.work("scrape:news", async () => {
console.log(`[${new Date().toISOString()}] Running: News scraper`);
await scrapeNews();
});
await boss.work("scrape:faq", async () => {
console.log(`[${new Date().toISOString()}] FAQ scraper — not yet implemented`);
});
await boss.work("scrape:docs", async () => {
console.log(`[${new Date().toISOString()}] Docs scraper — not yet implemented`);
});
// ── Compute jobs ──────────────────────────────────────────────────────
await boss.work("compute:abc", async () => {
console.log(`[${new Date().toISOString()}] Computing: ABC classification`);
const { computeAbcClassification } = await import("./scrapers/market-intelligence");
await computeAbcClassification();
});
await boss.work("compute:reorder-signals", async () => {
console.log(`[${new Date().toISOString()}] Computing: Reorder signals`);
const { computeReorderSignals } = await import("./scrapers/market-intelligence");
await computeReorderSignals();
});
// ── NAS sync ──────────────────────────────────────────────────────────
await boss.work("sync:nas", async () => {
console.log(`[${new Date().toISOString()}] Running: NAS sync to Fearghas`);
const { runNightlyNasSync } = await import("./utils/nas-sync");
await runNightlyNasSync();
});
// ── Prediction signal scrapers ────────────────────────────────────────
await boss.work("scrape:signals:sec-edgar", async () => {
console.log(`[${new Date().toISOString()}] Running: SEC EDGAR CapEx`);
await scrapeSecEdgar();
});
await boss.work("scrape:signals:github", async () => {
console.log(`[${new Date().toISOString()}] Running: GitHub tech signals`);
await scrapeGithubSignals();
});
await boss.work("scrape:signals:ebay-velocity", async () => {
console.log(`[${new Date().toISOString()}] Running: eBay sold velocity`);
await scrapeEbayVelocity();
});
await boss.work("scrape:signals:ai-clusters", async () => {
console.log(`[${new Date().toISOString()}] Running: AI cluster announcements`);
await scrapeAiClusters();
});
await boss.work("scrape:signals:distributor-leads", async () => {
console.log(`[${new Date().toISOString()}] Running: Distributor lead times`);
await scrapeDistributorLeads();
});
await boss.work("scrape:signals:standards", async () => {
console.log(`[${new Date().toISOString()}] Running: Standards tracker`);
await scrapeStandardsTracker();
});
// ── Forecast engine ───────────────────────────────────────────────────
await boss.work("compute:forecast", async () => {
console.log(`[${new Date().toISOString()}] Running: Forecast engine`);
await runForecastEngine();
});
// ── Hype Cycle Engine (Norton-Bass diffusion model) ───────────────────
await boss.work("compute:hype-cycle", async () => {
console.log(`[${new Date().toISOString()}] Running: Hype Cycle Engine (Norton-Bass)`);
const { computeHypeCycle } = await import("./utils/hype-cycle-engine");
await computeHypeCycle();
});
// ── Form-factor coverage scrapers ─────────────────────────────────────
await boss.work("scrape:pricing:comms-express", async () => {
console.log(`[${new Date().toISOString()}] Running: Comms-Express pricing`);
const { scrapeCommsExpress } = await import("./scrapers/comms-express");
await scrapeCommsExpress();
});
await boss.work("scrape:pricing:router-switch", async () => {
console.log(`[${new Date().toISOString()}] Running: Router-Switch.com pricing`);
const { scrapeRouterSwitch } = await import("./scrapers/router-switch");
await scrapeRouterSwitch();
});
await boss.work("scrape:pricing:multimode-inc", async () => {
console.log(`[${new Date().toISOString()}] Running: Multimode Inc pricing`);
const { scrapeMultimodeInc } = await import("./scrapers/multimode-inc");
await scrapeMultimodeInc();
});
await boss.work("scrape:pricing:optictransceiver", async () => {
console.log(`[${new Date().toISOString()}] Running: OpticTransceiver.com pricing`);
const { scrapeOpticTransceiver } = await import("./scrapers/optictransceiver");
await scrapeOpticTransceiver();
});
await boss.work("scrape:pricing:wiitek", async () => {
console.log(`[${new Date().toISOString()}] Running: Wiitek pricing`);
const { scrapeWiitek } = await import("./scrapers/wiitek");
await scrapeWiitek();
});
await boss.work("scrape:pricing:naddod", async () => {
console.log(`[${new Date().toISOString()}] Running: NADDOD pricing`);
const { scrapeNaddod } = await import("./scrapers/naddod");
await scrapeNaddod();
});
await boss.work("scrape:pricing:qsfptek", async () => {
console.log(`[${new Date().toISOString()}] Running: QSFPTEK pricing`);
const { scrapeQsfptek } = await import("./scrapers/qsfptek");
await scrapeQsfptek();
});
await boss.work("scrape:pricing:addon", async () => {
console.log(`[${new Date().toISOString()}] Running: AddOn Networks pricing`);
const { scrapeAddonNetworks } = await import("./scrapers/addon-networks");
await scrapeAddonNetworks();
});
await boss.work("scrape:pricing:fibermall", async () => {
console.log(`[${new Date().toISOString()}] Running: FiberMall pricing`);
const { scrapeFiberMall } = await import("./scrapers/fibermall");
await scrapeFiberMall();
});
await boss.work("scrape:pricing:vcelink", async () => {
console.log(`[${new Date().toISOString()}] Running: Vcelink pricing`);
const { scrapeVcelink } = await import("./scrapers/vcelink");
await scrapeVcelink();
});
await boss.work("scrape:pricing:opticsbay", async () => {
console.log(`[${new Date().toISOString()}] Running: OpticsBay pricing`);
const { scrapeOpticsBay } = await import("./scrapers/opticsbay");
await scrapeOpticsBay();
});
// ── Mouser OEM reference prices ────────────────────────────────────────
await boss.work("scrape:pricing:mouser-oem", async () => {
console.log(`[${new Date().toISOString()}] Running: Mouser OEM reference prices`);
if (!process.env["MOUSER_API_KEY"]) {
console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set");
return;
}
const { scrapeMouser } = await import("./scrapers/mouser");
await scrapeMouser();
});
// ── Price denormalization refresh ─────────────────────────────────────────
// Refreshes street_price_usd / price_verified_eur on the transceivers table from
// the price_observations hypertable. Without this, denormalized prices go stale
// even when scrapers are collecting new observations.
await boss.work("compute:price-denorm", async () => {
const { pool } = await import("./utils/db");
const ts = new Date().toISOString();
console.log(`[${ts}] Running: Price denormalization refresh`);
const result = await pool.query(`
UPDATE transceivers t
SET price_verified_eur = sub.price_eur,
street_price_usd = sub.price_usd,
updated_at = NOW()
FROM (
SELECT po.transceiver_id,
MAX(po.price) FILTER (WHERE po.currency = 'EUR') AS price_eur,
MAX(po.price) FILTER (WHERE po.currency = 'USD') AS price_usd
FROM price_observations po
WHERE po.time > NOW() - INTERVAL '180 days'
AND po.price > 0
GROUP BY po.transceiver_id
) sub
WHERE t.id = sub.transceiver_id
AND (sub.price_eur IS NOT NULL OR sub.price_usd IS NOT NULL)
`);
console.log(`[price-denorm] refreshed ${result.rowCount} transceivers`);
});
// ── Health monitor ──────────────────────────────────────────────────────
await boss.work("monitor:scraper-health", async () => {
const { pool } = await import("./utils/db");
// Vendors we expect to see prices from regularly.
// Mapped: display name → pg-boss job name prefix (for last-run lookup).
const EXPECTED_VENDORS: Array<{ name: string; jobName: string }> = [
{ name: "FiberMall", jobName: "scrape:pricing:fibermall" },
{ name: "QSFPTEK", jobName: "scrape:pricing:qsfptek" },
{ name: "Flexoptix", jobName: "scrape:pricing:flexoptix" },
// FS.COM is skipped on Erik (SKIP_FS_SCRAPER=true) — Mac launchd handles it.
// Exclude from monitor so stale last_seen doesn't trigger false positives.
{ name: "10Gtek", jobName: "scrape:pricing:10gtek" },
{ name: "ATGBICS", jobName: "scrape:pricing:atgbics" },
{ name: "GBICS", jobName: "scrape:pricing:gbics" },
{ name: "SFPcables", jobName: "scrape:pricing:sfpcables" },
{ name: "NADDOD", jobName: "scrape:pricing:naddod" },
];
const vendorNames = EXPECTED_VENDORS.map((v) => v.name);
const jobNames = EXPECTED_VENDORS.map((v) => v.jobName);
// Price observation recency per vendor
const priceResult = await pool.query(`
SELECT v.name,
SUM(CASE WHEN po.time > NOW() - INTERVAL '6 hours' THEN 1 ELSE 0 END) AS prices_6h,
MAX(po.time) AS last_seen,
EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 3600.0 AS hours_since
FROM vendors v
LEFT JOIN price_observations po ON po.source_vendor_id = v.id
WHERE v.name = ANY($1)
GROUP BY v.name
ORDER BY last_seen ASC NULLS FIRST
`, [vendorNames]);
// Last successful pg-boss job per vendor scraper (within last 26h — covers any
// 2h-scheduled job at least once plus 2h slack for daemon restart delays).
// Also pull the most recent failed job so we can distinguish "running but
// prices stable" from "job consistently failing".
const jobResult = await pool.query(`
SELECT DISTINCT ON (name) name, state, completed_on
FROM pgboss.job
WHERE name = ANY($1)
AND created_on > NOW() - INTERVAL '26 hours'
ORDER BY name, created_on DESC
`, [jobNames]);
const jobMap = new Map<string, { state: string; completed_on: Date | null }>();
for (const row of jobResult.rows) {
jobMap.set(row.name as string, { state: row.state as string, completed_on: row.completed_on as Date | null });
}
// Last COMPLETED job per vendor (to know when the job last ran successfully)
const completedResult = await pool.query(`
SELECT DISTINCT ON (name) name, completed_on AS last_completed
FROM pgboss.job
WHERE name = ANY($1)
AND state = 'completed'
AND created_on > NOW() - INTERVAL '26 hours'
ORDER BY name, completed_on DESC
`, [jobNames]);
const completedMap = new Map<string, Date>();
for (const row of completedResult.rows) {
if (row.last_completed) completedMap.set(row.name as string, new Date(row.last_completed as string));
}
// Thresholds for alerting:
// CRITICAL (🔴): no job completed in 26h AND last price > 168h (7 days)
// — scraper genuinely not running or consistently failing
// WARNING (🟡): no job completed in 26h AND last price > 48h
// — scraper may be broken but recently seen
// STABLE (✅): job completed in last 26h AND vendor has historical prices
// — prices unchanged (hash dedup), scraper is healthy
const CRITICAL_HOURS = 168;
const WARN_HOURS = 48;
const critical: string[] = [];
const warnings: string[] = [];
const stable: string[] = [];
for (const row of priceResult.rows) {
const h = parseFloat(row.hours_since ?? "9999");
const n = parseInt(row.prices_6h ?? "0", 10);
if (n > 0) continue; // new prices written → definitely healthy
const lastStr = row.last_seen
? `last price ${h.toFixed(1)}h ago (${new Date(row.last_seen as string).toISOString().slice(0, 16)})`
: "NEVER scraped";
const vendor = EXPECTED_VENDORS.find((v) => v.name === row.name);
const jobInfo = vendor ? jobMap.get(vendor.jobName) : undefined;
const lastCompleted = vendor ? completedMap.get(vendor.jobName) : undefined;
const jobStr = jobInfo
? ` | job=${jobInfo.state} at ${jobInfo.completed_on ? new Date(jobInfo.completed_on).toISOString().slice(11, 16) : "?"}`
: " | job=not run in 26h";
// If the job completed successfully in the last 26h AND the vendor has
// historical prices, prices are just stable (hash dedup) — not an outage.
const jobRunningOk = !!lastCompleted && row.last_seen;
if (jobRunningOk) {
stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged, job OK)${jobStr}`);
continue;
}
if (!row.last_seen || h > CRITICAL_HOURS) {
critical.push(`🔴 ${row.name}: ${lastStr}${jobStr}`);
} else if (h > WARN_HOURS) {
warnings.push(`🟡 ${row.name}: ${lastStr}${jobStr}`);
} else {
stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged)${jobStr}`);
}
}
if (critical.length > 0 || warnings.length > 0) {
if (critical.length > 0) {
console.error("=== 🔴 SCRAPER CRITICAL — vendors with no prices for 7+ days ===");
for (const p of critical) console.error(p);
}
if (warnings.length > 0) {
console.warn("=== 🟡 SCRAPER WARNING — vendors with stale prices (48h+) ===");
for (const p of warnings) console.warn(p);
}
console.error("=== Check: pm2 logs tip-scraper-daemon ===");
} else {
const activeCount = EXPECTED_VENDORS.length - stable.length;
if (stable.length > 0) {
console.log(`[monitor] Scraper health OK — ${activeCount} vendors active, ${stable.length} stable (no price changes)`);
for (const s of stable) console.log(` ${s}`);
} else {
console.log(`[monitor] Scraper health OK — all ${EXPECTED_VENDORS.length} vendors active in last 6h`);
}
}
});
// ── Verification reconciliation ─────────────────────────────────────────
await boss.work("maintenance:reconcile-verification", async () => {
const { pool } = await import("./utils/db");
// 1. Backfill product_page_url from the latest real price URL. Many
// fetch scrapers write URL only to price_observations; details verification
// needs the canonical product source on transceivers as well.
const backfillProductUrls = await pool.query(`
UPDATE transceivers t
SET product_page_url = latest.url,
updated_at = NOW()
FROM (
SELECT DISTINCT ON (po.transceiver_id)
po.transceiver_id, po.url
FROM price_observations po
WHERE po.url IS NOT NULL
AND po.url != ''
AND po.time > NOW() - INTERVAL '180 days'
ORDER BY po.transceiver_id, po.time DESC
) latest
WHERE t.id = latest.transceiver_id
AND (t.product_page_url IS NULL OR t.product_page_url = '')
`);
// 2. Promote stored product images to image_verified. Scrapers already
// filter placeholders before writing image_url; this keeps older crawls from
// being invisible to the dashboard badge math.
const verifyImages = await pool.query(`
UPDATE transceivers
SET has_image = true,
image_verified = true,
image_verified_at = COALESCE(image_verified_at, NOW()),
image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url),
updated_at = NOW()
WHERE image_url IS NOT NULL
AND image_url != ''
AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)'
AND (image_verified = false OR image_verified IS NULL)
`);
// 3. Promote crawled products with source URL + core spec fields to
// details_verified. This is intentionally stricter than "has any row":
// product identity, reach and media type all need to be present.
const verifyDetails = await pool.query(`
UPDATE transceivers
SET details_verified = true,
details_verified_at = COALESCE(details_verified_at, NOW()),
details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url),
data_confidence = CASE
WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated')
THEN 'scraped_unverified'
ELSE data_confidence
END,
updated_at = NOW()
WHERE product_page_url IS NOT NULL
AND product_page_url != ''
AND form_factor IS NOT NULL
AND speed_gbps IS NOT NULL
AND part_number IS NOT NULL
AND part_number != ''
AND reach_label IS NOT NULL
AND reach_label != ''
AND fiber_type IS NOT NULL
AND fiber_type != ''
AND COALESCE(data_confidence, 'unknown') != 'garbage'
AND (details_verified = false OR details_verified IS NULL)
`);
// 4. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days
const resetComp = await pool.query(`
UPDATE transceivers t
SET competitor_verified = false,
competitor_verified_at = NULL
WHERE competitor_verified = true
AND NOT EXISTS (
SELECT 1 FROM price_observations po
JOIN vendors v ON po.source_vendor_id = v.id
WHERE po.transceiver_id = t.id
AND po.time > NOW() - INTERVAL '30 days'
AND UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
)
`);
// 5. Reset fully_verified=false for products that lost a required signal
const resetFull = await pool.query(`
UPDATE transceivers
SET fully_verified = false,
fully_verified_at = NULL
WHERE fully_verified = true
AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false)
`);
// 6. Set fully_verified=true for products that now meet all 4 criteria
const setFull = await pool.query(`
UPDATE transceivers
SET fully_verified = true,
fully_verified_at = COALESCE(fully_verified_at, NOW())
WHERE competitor_verified = true
AND price_verified = true
AND image_verified = true
AND details_verified = true
AND fully_verified = false
`);
console.log(
`[reconcile] product URLs backfilled: ${backfillProductUrls.rowCount}, ` +
`images verified: ${verifyImages.rowCount}, ` +
`details verified: ${verifyDetails.rowCount}, ` +
`competitor_verified reset: ${resetComp.rowCount}, ` +
`fully_verified cleared: ${resetFull.rowCount}, ` +
`fully_verified earned: ${setFull.rowCount}`
);
});
// ── Equivalence matching ────────────────────────────────────────────────
// Matches Flexoptix SKUs to technically equivalent competitor products by specs.
// Confidence scoring: standard_name(35) + form_factor(25) + speed_gbps(20) +
// fiber_type(10) + reach±25%(10) = 100 pts max
// ≥0.85 → auto_approved + competitor_verified=true
// 0.50–0.84 → pending (Manual Review queue in dashboard)
// <0.50 → skipped
await boss.work("maintenance:find-equivalences", async () => {
const { pool } = await import("./utils/db");
const ts = new Date().toISOString();
console.log(`[${ts}] Running: Equivalence matching`);
// Find all Flexoptix transceivers that are NOT yet competitor_verified
const flexResult = await pool.query(`
SELECT t.id, t.part_number, t.standard_name, t.form_factor,
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
t.connector, t.wdm_type, t.coherent
FROM transceivers t
JOIN vendors v ON v.id = t.vendor_id
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
AND t.competitor_verified = false
`);
let autoApproved = 0;
let queued = 0;
let skipped = 0;
for (const fx of flexResult.rows) {
// Find competitor transceivers with recent price observations and matching specs
const candidates = await pool.query(`
SELECT t.id AS competitor_id, t.part_number, t.standard_name,
t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters,
t.wavelengths, t.connector, v.name AS vendor_name,
MAX(po.time) AS last_price, COUNT(*) AS price_count
FROM transceivers t
JOIN vendors v ON v.id = t.vendor_id
JOIN price_observations po ON po.transceiver_id = t.id
WHERE UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
AND po.time > NOW() - INTERVAL '30 days'
AND t.form_factor = $1
AND t.speed_gbps = $2
AND t.id != $3
GROUP BY t.id, t.part_number, t.standard_name, t.form_factor,
t.speed_gbps, t.fiber_type, t.reach_meters,
t.wavelengths, t.connector, v.name
`, [fx.form_factor, fx.speed_gbps, fx.id]);
for (const cand of candidates.rows) {
// Confidence scoring
// Max points: form_factor(25) + speed_gbps(20) + standard_name(30) +
// wavelength_nm(20) + fiber_type(10) + reach(10) = 115
let score = 0;
const basis: string[] = [];
// form_factor already matched (pre-filter), award points
score += 25; basis.push("form_factor");
// speed_gbps already matched (pre-filter)
score += 20; basis.push("speed_gbps");
// standard_name match (strong signal — e.g. "10GBASE-LR")
if (fx.standard_name && cand.standard_name &&
fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) {
score += 30; basis.push("standard_name");
}
// wavelength match — extract first numeric nm value and compare within ±15nm
// "wavelengths" is text: "1310 nm", "850nm", "1270/1290/1310/1330 nm" etc.
const extractNm = (w: string | null): number | null => {
if (!w) return null;
const m = w.match(/(\d{3,4})/);
return m ? parseInt(m[1], 10) : null;
};
const fxNm = extractNm(fx.wavelengths);
const candNm = extractNm(cand.wavelengths);
if (fxNm !== null && candNm !== null) {
if (Math.abs(fxNm - candNm) <= 15) {
score += 20; basis.push(`wavelength_${fxNm}nm`);
} else {
score -= 20; // hard penalize wrong wavelength (1310 vs 1550 = completely different product)
}
}
// fiber_type match (SMF vs MMF — critical)
if (fx.fiber_type && cand.fiber_type) {
if (fx.fiber_type.trim().toUpperCase() === cand.fiber_type.trim().toUpperCase()) {
score += 10; basis.push("fiber_type");
} else {
score -= 15; // SMF vs MMF = wrong product
}
}
// reach within ±25%
if (fx.reach_meters && cand.reach_meters && fx.reach_meters > 0 && cand.reach_meters > 0) {
const diff = Math.abs(fx.reach_meters - cand.reach_meters);
const tolerance = Math.max(fx.reach_meters, 1) * 0.25;
if (diff <= tolerance) {
score += 10; basis.push("reach");
} else {
score -= 15; // penalize mismatched reach
}
} else if (!fx.reach_meters && !cand.reach_meters) {
score += 5; basis.push("reach_null");
}
const confidence = Math.max(0, Math.min(1, score / 115));
if (confidence < 0.50) { skipped++; continue; }
const notes = `${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` +
`basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` +
`wavelength: ${fx.wavelengths||"?"} vs ${cand.wavelengths||"?"}`;
// Upsert equivalence candidate
const status = confidence >= 0.73 ? "auto_approved" : "pending";
await pool.query(`
INSERT INTO transceiver_equivalences
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
confidence = EXCLUDED.confidence,
match_basis = EXCLUDED.match_basis,
match_notes = EXCLUDED.match_notes,
updated_at = NOW()
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected')
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]);
if (confidence >= 0.73) {
// Auto-approve: set competitor_verified on the Flexoptix transceiver
await pool.query(`
UPDATE transceivers
SET competitor_verified = true,
competitor_verified_at = NOW()
WHERE id = $1 AND competitor_verified = false
`, [fx.id]);
autoApproved++;
} else {
queued++;
}
}
}
console.log(
`[find-equivalences] auto_approved: ${autoApproved}, ` +
`queued for review: ${queued}, skipped (low confidence): ${skipped}`
);
// After auto-approvals, rerun fully_verified check
if (autoApproved > 0) {
await pool.query(`
UPDATE transceivers
SET fully_verified = true,
fully_verified_at = COALESCE(fully_verified_at, NOW())
WHERE competitor_verified = true
AND price_verified = true
AND image_verified = true
AND details_verified = true
AND fully_verified = false
`);
}
});
// ── Re-research approved equivalences ────────────────────────────────────────
// Processes up to 200 approved equivalences per day that have re_research_due_at <= NOW().
// Re-runs the confidence check: if competitor still has recent prices and specs still match,
// the approval is confirmed (re_researched_at = NOW(), next check in 30 days).
// If confidence drops or competitor has no recent price: reverts to pending.
await boss.work("maintenance:re-research-equivalences", async () => {
const { pool } = await import("./utils/db");
const ts = new Date().toISOString();
console.log(`[${ts}] Running: Re-research approved equivalences`);
const batch = await pool.query(`
SELECT eq.id, eq.flexoptix_id, eq.competitor_id, eq.confidence,
fx.form_factor, fx.speed_gbps, fx.standard_name, fx.fiber_type,
fx.reach_meters, fx.wavelengths
FROM transceiver_equivalences eq
JOIN transceivers fx ON eq.flexoptix_id = fx.id
WHERE eq.status IN ('approved', 'auto_approved')
AND eq.re_research_due_at IS NOT NULL
AND eq.re_research_due_at <= NOW()
ORDER BY eq.re_research_due_at ASC
LIMIT 200
`);
let confirmed = 0;
let reverted = 0;
for (const eq of batch.rows) {
// Check if competitor still has a recent price observation
const priceCheck = await pool.query(`
SELECT COUNT(*) AS cnt
FROM price_observations
WHERE transceiver_id = $1 AND time > NOW() - INTERVAL '45 days'
`, [eq.competitor_id]);
const hasRecentPrice = parseInt(priceCheck.rows[0].cnt, 10) > 0;
if (!hasRecentPrice) {
// Competitor no longer carries this — revert to pending for manual review
await pool.query(`
UPDATE transceiver_equivalences
SET status = 'pending', re_research_due_at = NULL, re_researched_at = NULL,
match_notes = CONCAT(match_notes, E'\n[Re-research ' || NOW()::date || ': no recent price — reverted to pending]')
WHERE id = $1
`, [eq.id]);
// Reset competitor_verified if no other approved equivalence covers this transceiver
await pool.query(`
UPDATE transceivers
SET competitor_verified = false, competitor_verified_at = NULL,
fully_verified = false, fully_verified_at = NULL
WHERE id = $1
AND NOT EXISTS (
SELECT 1 FROM transceiver_equivalences
WHERE flexoptix_id = $1
AND status IN ('approved', 'auto_approved')
AND id != $2
)
`, [eq.flexoptix_id, eq.id]);
reverted++;
} else {
// Still valid — confirm and schedule next re-research in 30 days
await pool.query(`
UPDATE transceiver_equivalences
SET re_researched_at = NOW(),
re_research_due_at = NOW() + INTERVAL '30 days'
WHERE id = $1
`, [eq.id]);
confirmed++;
}
}
console.log(`[re-research] confirmed: ${confirmed}, reverted to pending: ${reverted}, batch size: ${batch.rows.length}`);
});
// ══════════════════════════════════════════════════════════════════════
// VENDOR DISCOVERY CRAWLER WORKERS
// Each worker calls discoverVendorCatalog() for the matching slug.
// Results go to: TIP DB (findOrCreateScrapedTransceiver) +
// Gitea tip-training-data repo (SFT JSONL pairs)
// ══════════════════════════════════════════════════════════════════════
const { discoverVendorCatalog, VENDOR_CATALOG_REGISTRY } = await import("./crawler-llm/vendor-discovery-crawler");
for (const vendorConfig of VENDOR_CATALOG_REGISTRY) {
const jobName = `discover:vendor:${vendorConfig.slug}`;
boss.work(jobName, async () => {
if (!isLoadAcceptable(3.0)) {
console.warn(`[${jobName}] Load too high — skipping deep crawl`);
return;
}
console.log(`[${jobName}] Starting vendor discovery crawl…`);
try {
await discoverVendorCatalog(vendorConfig, { verbose: false });
} catch (err) {
console.error(`[${jobName}] Fatal:`, (err as Error).message);
throw err; // let pg-boss retry
}
});
}
console.log("All workers registered (102 jobs, 24/7 continuous + 8 weekly discovery crawlers)");
}
Reference in New Issue View Git Blame Copy Permalink