3032 lines
155 KiB
TypeScript
3032 lines
155 KiB
TypeScript
/**
|
||
* pg-boss Job Scheduler — 24/7 Continuous Scraping
|
||
*
|
||
* ARCHITECTURE:
|
||
* - Erik (VPS, .82) : Playwright-heavy scrapers (FS.com, 10Gtek, ATGBICS, ProLabs)
|
||
* + all compatibility + eBay + compute + NAS sync
|
||
* - Raspberry Pi Fleet : Lightweight fetch/cheerio scrapers run continuously all day
|
||
* (BlueOptics, Fiber24, T&S Com, Fluxlight, GBICs, Optcore,
|
||
* Champion ONE, SFPCables, SmartOptics, HUBER+SUHNER, etc.)
|
||
*
|
||
* SCHEDULE PHILOSOPHY:
|
||
* - Playwright scrapers: every 8h (resource-heavy, VPS only)
|
||
* - Fetch/Cheerio scrapers: every 4h (lightweight, Pi-friendly)
|
||
* - Catalog scrapers (Flexoptix): every 2h (fast GraphQL, primary price source)
|
||
* - Compatibility matrices: every 12h (rarely change)
|
||
* - eBay enrichment: every 6h
|
||
* - Intelligence/community: every 6h
|
||
* - Compute jobs: after each pricing wave
|
||
* - NAS sync: nightly at 07:55
|
||
*/
|
||
import PgBoss from "pg-boss";
|
||
import { config } from "dotenv";
|
||
import { join } from "path";
|
||
import { loadavg } from "os";
|
||
|
||
// withIsolatedStorage removed — all Crawlee scrapers now use makeCrawleeConfig()
|
||
// for instance-level storage isolation. See packages/scraper/src/utils/crawlee-config.ts
|
||
|
||
/**
|
||
* Load-aware guard — skip heavy scrapers when the server is already busy.
|
||
* Uses the 1-minute load average; maxLoad defaults to 2.5 (50% of 5 vCPUs).
|
||
* Logs a warning and returns false when load is too high.
|
||
*/
|
||
function isLoadAcceptable(maxLoad = 2.5): boolean {
|
||
const [avg1] = loadavg();
|
||
if (avg1 > maxLoad) {
|
||
console.warn(`[load-guard] 1m load avg ${avg1.toFixed(2)} > ${maxLoad} — deferring heavy scraper`);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
config({ path: join(__dirname, "..", "..", "..", ".env") });
|
||
|
||
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
|
||
|
||
type EquivalenceProduct = {
|
||
part_number?: string | null;
|
||
standard_name?: string | null;
|
||
form_factor?: string | null;
|
||
speed_gbps?: number | string | null;
|
||
fiber_type?: string | null;
|
||
reach_meters?: number | string | null;
|
||
wavelengths?: string | null;
|
||
connector?: string | null;
|
||
};
|
||
|
||
type EquivalenceResearchResult = {
|
||
decision: "approve" | "reject";
|
||
confidence: number;
|
||
basis: string[];
|
||
reasons: string[];
|
||
rejectReason?: string;
|
||
};
|
||
|
||
function normalizeEquivalenceText(value: unknown): string | null {
|
||
if (value === null || value === undefined) return null;
|
||
const text = String(value).trim().toUpperCase();
|
||
return text.length > 0 ? text : null;
|
||
}
|
||
|
||
function numericEquivalenceValue(value: unknown): number | null {
|
||
if (value === null || value === undefined || value === "") return null;
|
||
const parsed = Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function extractPrimaryNm(wavelengths: unknown): number | null {
|
||
if (!wavelengths) return null;
|
||
const match = String(wavelengths).match(/(\d{3,4})/);
|
||
return match ? parseInt(match[1], 10) : null;
|
||
}
|
||
|
||
function evaluateEquivalenceResearch(
|
||
fx: EquivalenceProduct,
|
||
cp: EquivalenceProduct,
|
||
hasRecentPrice: boolean,
|
||
): EquivalenceResearchResult {
|
||
const basis: string[] = [];
|
||
const reasons: string[] = [];
|
||
let score = 0;
|
||
|
||
const fxForm = normalizeEquivalenceText(fx.form_factor);
|
||
const cpForm = normalizeEquivalenceText(cp.form_factor);
|
||
const fxSpeed = numericEquivalenceValue(fx.speed_gbps);
|
||
const cpSpeed = numericEquivalenceValue(cp.speed_gbps);
|
||
const fxStandard = normalizeEquivalenceText(fx.standard_name);
|
||
const cpStandard = normalizeEquivalenceText(cp.standard_name);
|
||
const fxFiber = normalizeEquivalenceText(fx.fiber_type);
|
||
const cpFiber = normalizeEquivalenceText(cp.fiber_type);
|
||
const fxReach = numericEquivalenceValue(fx.reach_meters);
|
||
const cpReach = numericEquivalenceValue(cp.reach_meters);
|
||
const fxNm = extractPrimaryNm(fx.wavelengths);
|
||
const cpNm = extractPrimaryNm(cp.wavelengths);
|
||
|
||
if (!hasRecentPrice) {
|
||
reasons.push("no recent competitor price observation");
|
||
return {
|
||
decision: "reject",
|
||
confidence: 0,
|
||
basis,
|
||
reasons,
|
||
rejectReason: "automated research: competitor has no recent price observation",
|
||
};
|
||
}
|
||
|
||
if (fxForm && cpForm && fxForm === cpForm) {
|
||
score += 25;
|
||
basis.push("form_factor");
|
||
} else {
|
||
reasons.push("form factor mismatch or missing");
|
||
}
|
||
|
||
if (fxSpeed !== null && cpSpeed !== null && fxSpeed === cpSpeed) {
|
||
score += 20;
|
||
basis.push("speed_gbps");
|
||
} else {
|
||
reasons.push("speed mismatch or missing");
|
||
}
|
||
|
||
if (fxStandard && cpStandard && fxStandard === cpStandard) {
|
||
score += 30;
|
||
basis.push("standard_name");
|
||
} else {
|
||
reasons.push("standard name not identical");
|
||
}
|
||
|
||
if (fxNm !== null && cpNm !== null) {
|
||
if (Math.abs(fxNm - cpNm) <= 15) {
|
||
score += 20;
|
||
basis.push(`wavelength_${fxNm}nm`);
|
||
} else {
|
||
reasons.push(`wavelength mismatch ${fxNm}nm vs ${cpNm}nm`);
|
||
score -= 20;
|
||
}
|
||
} else {
|
||
reasons.push("wavelength missing");
|
||
}
|
||
|
||
if (fxFiber && cpFiber) {
|
||
if (fxFiber === cpFiber) {
|
||
score += 10;
|
||
basis.push("fiber_type");
|
||
} else {
|
||
reasons.push(`fiber mismatch ${fxFiber} vs ${cpFiber}`);
|
||
score -= 15;
|
||
}
|
||
} else {
|
||
reasons.push("fiber type missing");
|
||
}
|
||
|
||
if (fxReach !== null && cpReach !== null && fxReach > 0 && cpReach > 0) {
|
||
const ratio = Math.min(fxReach, cpReach) / Math.max(fxReach, cpReach);
|
||
if (ratio >= 0.85) {
|
||
score += 10;
|
||
basis.push("reach");
|
||
} else {
|
||
reasons.push(`reach mismatch ${fxReach}m vs ${cpReach}m`);
|
||
score -= 15;
|
||
}
|
||
} else {
|
||
reasons.push("reach missing");
|
||
}
|
||
|
||
const confidence = Math.max(0, Math.min(1, score / 115));
|
||
const criticalMismatch = reasons.some((reason) =>
|
||
reason.startsWith("wavelength mismatch") ||
|
||
reason.startsWith("fiber mismatch") ||
|
||
reason.startsWith("reach mismatch") ||
|
||
reason.startsWith("form factor mismatch") ||
|
||
reason.startsWith("speed mismatch")
|
||
);
|
||
const missingCriticalEvidence = reasons.some((reason) =>
|
||
reason === "wavelength missing" ||
|
||
reason === "fiber type missing" ||
|
||
reason === "reach missing"
|
||
);
|
||
|
||
if (criticalMismatch) {
|
||
return {
|
||
decision: "reject",
|
||
confidence,
|
||
basis,
|
||
reasons,
|
||
rejectReason: `automated research: technical mismatch (${reasons.join("; ")})`,
|
||
};
|
||
}
|
||
|
||
if (missingCriticalEvidence) {
|
||
return {
|
||
decision: "reject",
|
||
confidence,
|
||
basis,
|
||
reasons,
|
||
rejectReason: `automated research: insufficient technical evidence (${reasons.join("; ")})`,
|
||
};
|
||
}
|
||
|
||
if (confidence >= 0.73) {
|
||
return { decision: "approve", confidence, basis, reasons };
|
||
}
|
||
|
||
return {
|
||
decision: "reject",
|
||
confidence,
|
||
basis,
|
||
reasons,
|
||
rejectReason: `automated research: confidence ${confidence.toFixed(3)} below approval threshold`,
|
||
};
|
||
}
|
||
|
||
export async function createScheduler(): Promise<PgBoss> {
|
||
const boss = new PgBoss({
|
||
connectionString,
|
||
retryLimit: 3,
|
||
retryDelay: 30,
|
||
retryBackoff: true,
|
||
expireInSeconds: 300,
|
||
monitorStateIntervalSeconds: 60,
|
||
});
|
||
|
||
boss.on("error", (error) => console.error("pg-boss error:", error));
|
||
|
||
await boss.start();
|
||
console.log("pg-boss scheduler started");
|
||
|
||
return boss;
|
||
}
|
||
|
||
export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||
// pg-boss v10: boss.schedule() requires the queue to already exist in pgboss.queue.
|
||
// After a DB reset (e.g. server outage), all queue rows are wiped.
|
||
// Patch boss.schedule to auto-create queues idempotently before each schedule call,
|
||
// so the 277 individual schedule() calls below don't need to be touched.
|
||
const _origSchedule = boss.schedule.bind(boss) as typeof boss.schedule;
|
||
(boss as unknown as Record<string, unknown>).schedule = async (
|
||
name: string, cron: string, data?: unknown, opts?: unknown,
|
||
) => {
|
||
await boss.createQueue(name).catch(() => { /* already exists */ });
|
||
return _origSchedule(name, cron, data as object, opts as object);
|
||
};
|
||
|
||
const queues = [
|
||
// ── Playwright scrapers (Erik, every 8h) ───────────────────────────
|
||
"scrape:pricing:fs",
|
||
"scrape:pricing:10gtek",
|
||
"scrape:pricing:atgbics",
|
||
"scrape:pricing:prolabs",
|
||
// ── Fetch/Cheerio scrapers (Pi-friendly, every 4h) ─────────────────
|
||
"scrape:pricing:fluxlight",
|
||
"scrape:pricing:gbics",
|
||
"scrape:pricing:optcore",
|
||
"scrape:pricing:champion-one",
|
||
"scrape:pricing:sfpcables",
|
||
"scrape:pricing:blueoptics",
|
||
"scrape:pricing:fiber24",
|
||
"scrape:pricing:tscom",
|
||
"scrape:pricing:skylane",
|
||
"scrape:pricing:ascentoptics",
|
||
"scrape:pricing:gaotek",
|
||
// ── Catalog scrapers (every 2h) ────────────────────────────────────
|
||
"scrape:pricing:flexoptix",
|
||
// ── Manufacturer catalogs (every 8h, no prices) ────────────────────
|
||
"scrape:catalog:smartoptics",
|
||
"scrape:catalog:hubersuhner",
|
||
"scrape:catalog:eoptolink",
|
||
// ── Vendor lists ───────────────────────────────────────────────────
|
||
"scrape:vendors:flexoptix",
|
||
"scrape:vendors:flexoptix-supported",
|
||
// ── Compatibility (every 12h) ──────────────────────────────────────
|
||
"scrape:compat:flexoptix",
|
||
"scrape:compat:cisco",
|
||
"scrape:compat:juniper",
|
||
"scrape:compat:sonic",
|
||
"scrape:compat:ufispace",
|
||
"scrape:compat:edgecore",
|
||
// ── Switch enrichment (every 12h) ─────────────────────────────────
|
||
"scrape:assets:switches",
|
||
// ── Switch og:image fetcher (daily, after switch-assets) ──────────
|
||
"scrape:images:switches",
|
||
// ── Playwright image fetcher for bot-blocked vendors (every 3d) ───
|
||
"scrape:images:switches:playwright",
|
||
// ── eBay enrichment (every 6h) ────────────────────────────────────
|
||
"enrich:ebay-transceivers",
|
||
"enrich:ebay-switches",
|
||
// ── Intelligence & community (every 6h) ───────────────────────────
|
||
"scrape:market-intel",
|
||
"scrape:nog-talks",
|
||
"scrape:community-issues",
|
||
"scrape:datasheet-links",
|
||
"scrape:news",
|
||
"scrape:faq",
|
||
"scrape:docs",
|
||
// ── Compute (every 4h, after pricing waves) ───────────────────────
|
||
"compute:abc",
|
||
"compute:reorder-signals",
|
||
// ── New form-factor coverage scrapers (every 8h) ──────────────────
|
||
"scrape:pricing:comms-express",
|
||
"scrape:pricing:router-switch",
|
||
"scrape:pricing:multimode-inc",
|
||
"scrape:pricing:optictransceiver",
|
||
"scrape:pricing:wiitek",
|
||
// ── Fetch-based catalog+pricing scrapers (every 2h) ──────────────
|
||
"scrape:pricing:naddod",
|
||
"scrape:pricing:qsfptek",
|
||
"scrape:pricing:addon",
|
||
"scrape:pricing:fibermall",
|
||
"scrape:pricing:vcelink",
|
||
"scrape:pricing:opticsbay",
|
||
// ── OEM Reference Prices (Mouser API, once daily) ─────────────────
|
||
"scrape:pricing:mouser-oem",
|
||
// ── Prediction Signal Scrapers (new) ──────────────────────────────
|
||
"scrape:signals:sec-edgar",
|
||
"scrape:signals:github",
|
||
"scrape:signals:ebay-velocity",
|
||
"scrape:signals:ai-clusters",
|
||
"scrape:signals:distributor-leads",
|
||
"scrape:signals:standards",
|
||
// ── Forecast Engine ───────────────────────────────────────────────
|
||
"compute:forecast",
|
||
// ── Hype Cycle Engine (Norton-Bass, daily) ────────────────────────
|
||
"compute:hype-cycle",
|
||
// ── Sync ──────────────────────────────────────────────────────────
|
||
"sync:nas",
|
||
// ── Health Monitoring ─────────────────────────────────────────────
|
||
"monitor:scraper-health",
|
||
// ── Price denormalization refresh ─────────────────────────────────
|
||
"compute:price-denorm",
|
||
// ── Verification Reconciliation ───────────────────────────────────
|
||
"maintenance:reconcile-verification",
|
||
// ── Competitor Equivalence Matching ───────────────────────────────
|
||
"maintenance:find-equivalences",
|
||
// ── Re-Research approved equivalences ─────────────────────────────
|
||
"maintenance:re-research-equivalences",
|
||
// ── Vendor Discovery Crawlers (TIPLLM training data + DB seeding) ─────
|
||
"discover:vendor:cisco-tmg",
|
||
"discover:vendor:juniper",
|
||
"discover:vendor:arista",
|
||
"discover:vendor:fs-com",
|
||
"discover:vendor:flexoptix",
|
||
"discover:vendor:nokia",
|
||
"discover:vendor:huawei",
|
||
"discover:vendor:ii-vi",
|
||
];
|
||
|
||
for (const q of queues) {
|
||
await boss.createQueue(q).catch(() => { /* already exists */ });
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// ALL PRICING SCRAPERS — 24/7, every 2h, staggered by 10min
|
||
// Goal: complete competitor coverage, no gaps, database always fresh
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
// Playwright scrapers (resource-heavy) — every 2h, 10min apart
|
||
await boss.schedule("scrape:pricing:fs", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 5400 });
|
||
await boss.schedule("scrape:pricing:10gtek", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:atgbics", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:prolabs", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// Fetch/Cheerio scrapers (lightweight) — every 2h, 5min apart
|
||
await boss.schedule("scrape:pricing:fluxlight", "0 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:gbics", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:optcore", "10 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:champion-one", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:sfpcables", "20 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:blueoptics", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:fiber24", "30 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:tscom", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:skylane", "40 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:ascentoptics", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:gaotek", "50 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// Form-factor coverage scrapers — every 2h
|
||
await boss.schedule("scrape:pricing:comms-express", "5 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 });
|
||
await boss.schedule("scrape:pricing:router-switch", "15 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 5400 });
|
||
await boss.schedule("scrape:pricing:multimode-inc", "25 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:optictransceiver", "35 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:wiitek", "45 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// Fetch-based scrapers running on Erik — every 2h, end-of-cycle slots
|
||
await boss.schedule("scrape:pricing:naddod", "48 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:qsfptek", "52 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:addon", "55 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:fibermall", "57 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// OEM reference prices via Mouser API — once daily at 03:00 (slow: 2s/PID × 475 PIDs ≈ 16min)
|
||
// Requires MOUSER_API_KEY env var (free at mouser.com/api-hub)
|
||
await boss.schedule("scrape:pricing:mouser-oem", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// FLEXOPTIX CATALOG — every 2h (primary price source)
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// MANUFACTURER CATALOGS — every 4h (product data, no prices)
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("scrape:catalog:smartoptics", "10 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hubersuhner", "25 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:eoptolink", "40 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// OEM vendor seed catalogs — daily at 04:00+ (stable data, rarely changes)
|
||
await boss.schedule("scrape:catalog:arista-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:juniper-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hpe-aruba-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nokia-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:huawei-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:dell-emc-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:extreme-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nvidia-mellanox-oem","45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:brocade-ruckus-oem","0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:zte-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ciena-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ericsson-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:fortinet-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:palo-alto-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:f5-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ubiquiti-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:mikrotik-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:netgear-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:adtran-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:calix-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:checkpoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:allied-telesis-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:zyxel-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:dlink-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ale-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ribbon-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:infinera-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:tplink-oem", "45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cambium-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:moxa-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:lumentum-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:coherent-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:brocade-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ruggedcom-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hirschmann-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:viavi-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:westermo-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:lancom-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:dzs-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:watchguard-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:sonicwall-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:advantech-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:redlion-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:exfo-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:mellanox-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:intel-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:antaira-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:transition-networks-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:perle-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:barracuda-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:etherwan-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:blackbox-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:packetlight-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:spirent-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:comnet-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:korenix-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:sophos-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cradlepoint-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:fujitsu-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nec-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:arris-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hillstone-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:vecima-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:stordis-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:keysight-oem", "0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:sycamore-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ekinops-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:adva-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:coriant-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:casa-systems-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:harmonic-oem", "30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:solarflare-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:marvell-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:broadcom-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:calix-access-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ribbon-comms-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:infinera-groove-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ciena-waveserver-oem","15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:commscope-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:teleste-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:tejas-networks-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ericsson-transport-oem","15 0 * * *",{}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:adtran-ta-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:isolan-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:telco-systems-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:rad-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:comtrend-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:packetfront-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:edgewater-networks-oem","0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:corning-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ofs-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:kontron-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ipinfusion-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:telrad-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:siklu-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ceragon-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:datang-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:viptela-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:versa-networks-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:vmware-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cimc-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:qlogic-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:emulex-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:netapp-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:pure-storage-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hpe-storage-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ibm-storage-oem", "30 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:dell-storage-oem", "45 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hitachi-vantara-oem", "0 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:aws-oem", "15 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:azure-oem", "30 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:google-cloud-oem", "45 7 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:meta-oem", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nokia-access-oem", "15 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:huawei-access-oem", "30 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:zte-access-oem", "45 8 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:calix-gigapoint-oem", "0 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:samsung-networks-oem","15 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nokia-airscale-oem", "30 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ericsson-ran-oem", "45 9 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:mavenir-oem", "0 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ixia-oem", "15 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:exfo-network-oem", "30 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cumulus-networks-oem","45 10 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:sonic-oem", "0 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:h3c-oem", "15 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ruijie-oem", "30 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:centec-oem", "45 11 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:supermicro-oem", "0 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cisco-meraki-oem", "15 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cisco-catalyst-oem", "30 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cisco-nexus-oem", "45 12 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cisco-asr-oem", "0 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:juniper-mx-oem", "15 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:juniper-qfx-oem", "30 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:aruba-cx-oem", "45 13 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:extreme-campus-oem", "0 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:arista-7000-oem", "15 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:pica8-oem", "30 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:pluribus-oem", "45 14 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:drivenets-oem", "0 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:phoenix-contact-oem", "15 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:beckhoff-oem", "30 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:omron-oem", "45 15 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:abb-oem", "0 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:siemens-oem", "15 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:schneider-oem", "30 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:rockwell-oem", "45 16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:belden-oem", "0 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:ge-grid-oem", "15 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:schweitzer-oem", "30 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:moxa-industrial-oem", "45 17 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cisco-ie-oem", "0 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// ── Media / Broadcast / Pro-AV ────────────────────────────────────────
|
||
await boss.schedule("scrape:catalog:evertz-oem", "15 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:grass-valley-oem", "30 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:haivision-oem", "45 18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:viasat-oem", "0 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// ── Asian Optical Vendors ─────────────────────────────────────────────
|
||
await boss.schedule("scrape:catalog:fiberhome-oem", "15 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:oplink-oem", "30 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:accelink-oem", "45 19 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:hisense-broadband-oem","0 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// ── Optical Transceiver Manufacturers (batch 24–25) ───────────────────
|
||
await boss.schedule("scrape:catalog:ii-vi-oem", "15 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:source-photonics-oem", "30 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:o-net-oem", "45 20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:innolight-oem", "0 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:aoi-oem", "15 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:sumitomo-electric-oem","30 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:neophotonics-oem", "45 21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// ── Test & Measurement / Defense / Telecom Access (batch 29–30) ────────
|
||
await boss.schedule("scrape:catalog:audiocodes-oem", "0 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:anritsu-oem", "15 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:netscout-oem", "30 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:curtiss-wright-oem", "45 22 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:eci-telecom-oem", "0 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:utstarcom-oem", "15 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:turbolink-oem", "30 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:chelsio-oem", "45 23 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:rohde-schwarz-oem", "0 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:l3harris-oem", "15 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:zhone-oem", "30 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:cambium-networks-oem", "45 0 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:tektronix-oem", "0 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:clearfield-oem", "15 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:lanner-oem", "30 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:black-box-oem", "45 1 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:radiflow-oem", "0 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:dragonwave-oem", "15 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:teledyne-lecroy-oem", "30 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:finisar-oem", "45 2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:acacia-oem", "0 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:inphi-oem", "15 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// ── Batch 35: Sierra Wireless, Senao, EMCORE, Reflex Photonics ────────
|
||
await boss.schedule("scrape:catalog:sierra-wireless-oem", "30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:senao-oem", "45 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:emcore-oem", "0 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:reflex-photonics-oem", "15 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// ── Batch 36: EnGenius, Palo Alto Networks, Brocade (Legacy), Foundry ─
|
||
await boss.schedule("scrape:catalog:engenius-oem", "30 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:paloalto-networks-oem", "45 4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:brocade-legacy-oem", "0 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:foundry-networks-oem", "15 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// ── Batch 37: Extreme (Legacy), Nortel, 3Com, Avaya ──────────────────
|
||
await boss.schedule("scrape:catalog:extreme-legacy-oem", "30 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:nortel-legacy-oem", "45 5 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:3com-legacy-oem", "0 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:catalog:avaya-legacy-oem", "15 6 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// VENDOR DISCOVERY CRAWLERS — daily, permanent 24/7 rotation
|
||
// Each run: crawls catalog → LLM extract → spec validate → DB + Gitea SFT
|
||
// 8 vendors × 3h stagger = full rotation every 24h, no overlap
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("discover:vendor:cisco-tmg", "0 20 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:juniper", "0 22 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:arista", "0 0 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:fs-com", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:flexoptix", "0 4 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:nokia", "0 6 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:huawei", "0 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("discover:vendor:ii-vi", "0 10 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// VENDOR LISTS — every 12h
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("scrape:vendors:flexoptix", "0 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
await boss.schedule("scrape:vendors:flexoptix-supported", "15 5,17 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// COMPATIBILITY MATRICES — every 12h
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
// Flexoptix compatibility — every 24h at 09:00 (after both switch-assets + image-fetcher)
|
||
await boss.schedule("scrape:compat:flexoptix", "0 9 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
await boss.schedule("scrape:compat:cisco", "0 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:compat:juniper", "15 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:compat:sonic", "30 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:compat:ufispace", "45 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
await boss.schedule("scrape:compat:edgecore", "55 6,18 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// SWITCH ASSETS — every 12h
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("scrape:assets:switches", "30 7,19 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
// og:image fetcher: daily at 08:30, after switch-assets completes at 07:30
|
||
await boss.schedule("scrape:images:switches", "30 8 * * *", {}, { retryLimit: 1, expireInSeconds: 7200 });
|
||
// Playwright image scraper for bot-blocked vendors (Arista/Dell/Edgecore/Fortinet/Extreme)
|
||
// Every 3 days at 09:00 — Playwright is slower and heavier than plain HTTP
|
||
await boss.schedule("scrape:images:switches:playwright", "0 9 */3 * *", {}, { retryLimit: 1, expireInSeconds: 10800 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// EBAY ENRICHMENT — every 6h
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("enrich:ebay-transceivers", "0 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 });
|
||
await boss.schedule("enrich:ebay-switches", "30 0,6,12,18 * * *", {}, { retryLimit: 2, expireInSeconds: 7200 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// INTELLIGENCE & COMMUNITY — every 6h
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("scrape:market-intel", "0 2,8,14,20 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// NOG conference talks — weekly on Mondays 06:00 UTC
|
||
await boss.schedule("scrape:nog-talks", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 7200 });
|
||
await boss.schedule("scrape:community-issues", "30 2,8,14,20 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:datasheet-links", "0 3,9,15,21 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:news", "20 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
await boss.schedule("scrape:faq", "40 3,9,15,21 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
await boss.schedule("scrape:docs", "50 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// COMPUTE JOBS — every 4h (after pricing waves settle)
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("compute:abc", "50 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
|
||
await boss.schedule("compute:reorder-signals", "55 3,7,11,15,19,23 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
|
||
// Price denorm refresh: daily 05:30 UTC after overnight scraping waves settle
|
||
await boss.schedule("compute:price-denorm", "30 5 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// PREDICTION SIGNAL SCRAPERS
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
// SEC EDGAR CapEx — weekly Monday 06:00 (filings don't change that fast)
|
||
await boss.schedule("scrape:signals:sec-edgar", "0 6 * * 1", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// GitHub signals — weekly Sunday 05:00
|
||
await boss.schedule("scrape:signals:github", "0 5 * * 0", {}, { retryLimit: 2, expireInSeconds: 7200 });
|
||
// eBay sold velocity — every 12h (fast-moving market signal)
|
||
await boss.schedule("scrape:signals:ebay-velocity", "0 4,16 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// AI cluster RSS feeds — every 4h (news moves fast)
|
||
await boss.schedule("scrape:signals:ai-clusters", "10 0,4,8,12,16,20 * * *", {}, { retryLimit: 2, expireInSeconds: 1800 });
|
||
// Distributor lead times — daily 03:30 (stock changes overnight)
|
||
await boss.schedule("scrape:signals:distributor-leads","30 3 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||
// Standards tracker — weekly Wednesday 04:00 (standards move slowly)
|
||
await boss.schedule("scrape:signals:standards", "0 4 * * 3", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// FORECAST ENGINE — daily at 08:00 (after all nightly scrapers done)
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("compute:forecast", "0 8 * * *", {}, { retryLimit: 2, expireInSeconds: 600 });
|
||
// Hype Cycle Engine runs daily at 04:30 (after Mouser OEM scraper at 03:00)
|
||
await boss.schedule("compute:hype-cycle", "30 4 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// NAS SYNC — nightly
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
|
||
|
||
// Health check: every 3h — warns if any vendor has no new prices recently
|
||
await boss.schedule("monitor:scraper-health", "17 */3 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
|
||
|
||
// Verification reconciliation: nightly at 01:00 UTC
|
||
// Resets competitor_verified/fully_verified for any transceiver that no longer
|
||
// has a real non-Flexoptix price in the last 30 days — prevents stale ★ 100% badges
|
||
await boss.schedule("maintenance:reconcile-verification", "0 1 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
|
||
|
||
// Equivalence matching: nightly at 02:00 UTC (after reconcile)
|
||
await boss.schedule("maintenance:find-equivalences", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
|
||
// Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run
|
||
await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||
|
||
console.log("All schedules registered — 24/7 continuous scraping (60 jobs)");
|
||
}
|
||
|
||
export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||
// Lazy-load all scrapers
|
||
const { scrapeFs } = await import("./scrapers/fs-com");
|
||
const { scrapeCiscoTmg } = await import("./scrapers/cisco-tmg");
|
||
const { scrapeSmartOptics } = await import("./scrapers/smartoptics");
|
||
const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner");
|
||
const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence");
|
||
const { scrapeNews } = await import("./scrapers/news");
|
||
const { scrape10Gtek } = await import("./scrapers/tenGtek");
|
||
const { scrapeFlexoptixCatalog } = await import("./scrapers/flexoptix-catalog");
|
||
const { scrapeFlexoptixVendors } = await import("./scrapers/flexoptix-vendors");
|
||
const { seedFlexoptixVendors } = await import("./scrapers/flexoptix-supported-vendors");
|
||
const { scrapeAtgbics } = await import("./scrapers/atgbics");
|
||
const { scrapeProLabs } = await import("./scrapers/prolabs");
|
||
const { scrapeJuniperHct } = await import("./scrapers/juniper-hct");
|
||
const { scrapeSonicHcl } = await import("./scrapers/sonic-hcl");
|
||
const { scrapeUfiSpace } = await import("./scrapers/ufispace");
|
||
const { scrapeEdgecore } = await import("./scrapers/edgecore");
|
||
const { scrapeSwitchAssets } = await import("./scrapers/switch-assets");
|
||
const { fetchSwitchImages } = await import("./scrapers/switch-image-fetcher");
|
||
const { fetchSwitchImagesPlaywright } = await import("./scrapers/switch-image-playwright");
|
||
const { scrapeFlexoptixCompatibility } = await import("./scrapers/flexoptix-compat");
|
||
// ── Prediction signal scrapers ────────────────────────────────────────
|
||
const { scrapeSecEdgar } = await import("./scrapers/sec-edgar");
|
||
const { scrapeGithubSignals } = await import("./scrapers/github-signals");
|
||
const { scrapeEbayVelocity } = await import("./scrapers/ebay-velocity");
|
||
const { scrapeAiClusters } = await import("./scrapers/ai-clusters");
|
||
const { scrapeDistributorLeads }= await import("./scrapers/distributor-leads");
|
||
const { scrapeStandardsTracker }= await import("./scrapers/standards-tracker");
|
||
const { runForecastEngine } = await import("./utils/forecast-engine");
|
||
|
||
// ── Playwright scrapers ───────────────────────────────────────────────
|
||
|
||
await boss.work("scrape:pricing:fs", async () => {
|
||
// FS.com uses Playwright + Cloudflare bypass. On datacenter servers the
|
||
// datacenter IP is blocked by Cloudflare WAF. Set SKIP_FS_SCRAPER=true to
|
||
// skip on Erik; the Mac launchd cron handles FS.com from a residential IP.
|
||
if (process.env["SKIP_FS_SCRAPER"] === "true") {
|
||
console.log(`[${new Date().toISOString()}] FS.com pricing: SKIPPED (SKIP_FS_SCRAPER=true)`);
|
||
return;
|
||
}
|
||
console.log(`[${new Date().toISOString()}] Running: FS.com pricing`);
|
||
await scrapeFs();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:10gtek", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: 10Gtek pricing`);
|
||
await scrape10Gtek();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:atgbics", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ATGBICS pricing`);
|
||
await scrapeAtgbics();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:prolabs", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ProLabs pricing`);
|
||
await scrapeProLabs();
|
||
});
|
||
|
||
// ── Lightweight fetch/cheerio scrapers ───────────────────────────────
|
||
await boss.work("scrape:pricing:fluxlight", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`);
|
||
const { scrapeFluxlight } = await import("./scrapers/fluxlight");
|
||
await scrapeFluxlight();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:gbics", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: GBICS pricing`);
|
||
const { scrapeGbics } = await import("./scrapers/gbics");
|
||
await scrapeGbics();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:optcore", async () => {
|
||
// Optcore.net WP REST API is blocked by Cloudflare WAF for Erik's datacenter
|
||
// IP (82.165.222.127). Set SKIP_OPTCORE_SCRAPER=true to suppress the wasted
|
||
// run. A residential IP (Mac launchd) would be needed to scrape Optcore.
|
||
if (process.env["SKIP_OPTCORE_SCRAPER"] === "true") {
|
||
console.log(`[${new Date().toISOString()}] Optcore pricing: SKIPPED (SKIP_OPTCORE_SCRAPER=true)`);
|
||
return;
|
||
}
|
||
console.log(`[${new Date().toISOString()}] Running: Optcore pricing`);
|
||
const { scrapeOptcore } = await import("./scrapers/optcore");
|
||
await scrapeOptcore();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:champion-one", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`);
|
||
const { scrapeChampionOne } = await import("./scrapers/champion-one");
|
||
await scrapeChampionOne();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:sfpcables", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SFPcables pricing`);
|
||
const { scrapeSfpCables } = await import("./scrapers/sfpcables");
|
||
await scrapeSfpCables();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:blueoptics", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: BlueOptics pricing`);
|
||
const { scrapeBlueOptics } = await import("./scrapers/blueoptics");
|
||
await scrapeBlueOptics();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:fiber24", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Fiber24 pricing`);
|
||
const { scrapeFiber24 } = await import("./scrapers/fiber24");
|
||
await scrapeFiber24();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:tscom", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: T&S Communication pricing`);
|
||
const { scrapeTsCom } = await import("./scrapers/tscom");
|
||
await scrapeTsCom();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:skylane", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Skylane pricing`);
|
||
const { scrapeSkylane } = await import("./scrapers/skylane");
|
||
await scrapeSkylane();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:ascentoptics", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ascent Optics pricing`);
|
||
const { scrapeAscentOptics } = await import("./scrapers/ascentoptics");
|
||
await scrapeAscentOptics();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:gaotek", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: GAO Tek pricing`);
|
||
const { scrapeGaoTek } = await import("./scrapers/gaotek");
|
||
await scrapeGaoTek();
|
||
});
|
||
|
||
// ── Catalog scrapers ──────────────────────────────────────────────────
|
||
|
||
await boss.work("scrape:pricing:flexoptix", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`);
|
||
await scrapeFlexoptixCatalog();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:smartoptics", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
|
||
await scrapeSmartOptics();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hubersuhner", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: HUBER+SUHNER catalog`);
|
||
await scrapeHuberSuhner();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:eoptolink", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Eoptolink OEM catalog`);
|
||
const { scrapeEoptolink } = await import("./scrapers/eoptolink");
|
||
await scrapeEoptolink();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:arista-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Arista OEM catalog seed`);
|
||
const { scrapeAristaOem } = await import("./scrapers/arista-oem");
|
||
await scrapeAristaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:juniper-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Juniper OEM catalog seed`);
|
||
const { scrapeJuniperOem } = await import("./scrapers/juniper-oem");
|
||
await scrapeJuniperOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hpe-aruba-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: HPE/Aruba OEM catalog seed`);
|
||
const { scrapeHpeArubaOem } = await import("./scrapers/hpe-aruba-oem");
|
||
await scrapeHpeArubaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nokia-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Nokia OEM catalog seed`);
|
||
const { scrapeNokiaOem } = await import("./scrapers/nokia-oem");
|
||
await scrapeNokiaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:huawei-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Huawei OEM catalog seed`);
|
||
const { scrapeHuaweiOem } = await import("./scrapers/huawei-oem");
|
||
await scrapeHuaweiOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:dell-emc-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Dell EMC OEM catalog seed`);
|
||
const { scrapeDellEmcOem } = await import("./scrapers/dell-emc-oem");
|
||
await scrapeDellEmcOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:extreme-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Extreme Networks OEM catalog seed`);
|
||
const { scrapeExtremeOem } = await import("./scrapers/extreme-oem");
|
||
await scrapeExtremeOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nvidia-mellanox-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NVIDIA/Mellanox OEM catalog seed`);
|
||
const { scrapeNvidiaMellanoxOem } = await import("./scrapers/nvidia-mellanox-oem");
|
||
await scrapeNvidiaMellanoxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:brocade-ruckus-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Brocade/RUCKUS OEM catalog seed`);
|
||
const { scrapeBrocadeRuckusOem } = await import("./scrapers/brocade-ruckus-oem");
|
||
await scrapeBrocadeRuckusOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:zte-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ZTE OEM catalog seed`);
|
||
const { scrapeZteOem } = await import("./scrapers/zte-oem");
|
||
await scrapeZteOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ciena-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ciena OEM catalog seed`);
|
||
const { scrapeCienaOem } = await import("./scrapers/ciena-oem");
|
||
await scrapeCienaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ericsson-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ericsson OEM catalog seed`);
|
||
const { scrapeEricssonOem } = await import("./scrapers/ericsson-oem");
|
||
await scrapeEricssonOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:fortinet-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Fortinet OEM catalog seed`);
|
||
const { scrapeFortinetOem } = await import("./scrapers/fortinet-oem");
|
||
await scrapeFortinetOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:palo-alto-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`);
|
||
const { scrapePaloAltoOem } = await import("./scrapers/palo-alto-oem");
|
||
await scrapePaloAltoOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:f5-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: F5 Networks OEM catalog seed`);
|
||
const { scrapeF5Oem } = await import("./scrapers/f5-oem");
|
||
await scrapeF5Oem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ubiquiti-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ubiquiti OEM catalog seed`);
|
||
const { scrapeUbiquitiOem } = await import("./scrapers/ubiquiti-oem");
|
||
await scrapeUbiquitiOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:mikrotik-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: MikroTik OEM catalog seed`);
|
||
const { scrapeMikrotikOem } = await import("./scrapers/mikrotik-oem");
|
||
await scrapeMikrotikOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:netgear-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Netgear OEM catalog seed`);
|
||
const { scrapeNetgearOem } = await import("./scrapers/netgear-oem");
|
||
await scrapeNetgearOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:adtran-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ADTRAN OEM catalog seed`);
|
||
const { scrapeAdtranOem } = await import("./scrapers/adtran-oem");
|
||
await scrapeAdtranOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:calix-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Calix OEM catalog seed`);
|
||
const { scrapeCalixOem } = await import("./scrapers/calix-oem");
|
||
await scrapeCalixOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:checkpoint-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Check Point OEM catalog seed`);
|
||
const { scrapeCheckpointOem } = await import("./scrapers/checkpoint-oem");
|
||
await scrapeCheckpointOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:allied-telesis-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Allied Telesis OEM catalog seed`);
|
||
const { scrapeAlliedTelesisOem } = await import("./scrapers/allied-telesis-oem");
|
||
await scrapeAlliedTelesisOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:zyxel-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Zyxel OEM catalog seed`);
|
||
const { scrapeZyxelOem } = await import("./scrapers/zyxel-oem");
|
||
await scrapeZyxelOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:dlink-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: D-Link OEM catalog seed`);
|
||
const { scrapeDlinkOem } = await import("./scrapers/dlink-oem");
|
||
await scrapeDlinkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ale-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Alcatel-Lucent Enterprise OEM catalog seed`);
|
||
const { scrapeAleOem } = await import("./scrapers/ale-oem");
|
||
await scrapeAleOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ribbon-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`);
|
||
const { scrapeRibbonOem } = await import("./scrapers/ribbon-oem");
|
||
await scrapeRibbonOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:infinera-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Infinera OEM catalog seed`);
|
||
const { scrapeInfineraOem } = await import("./scrapers/infinera-oem");
|
||
await scrapeInfineraOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:tplink-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: TP-Link OEM catalog seed`);
|
||
const { scrapeTplinkOem } = await import("./scrapers/tplink-oem");
|
||
await scrapeTplinkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cambium-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`);
|
||
const { scrapeCambiumOem } = await import("./scrapers/cambium-oem");
|
||
await scrapeCambiumOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:moxa-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Moxa OEM catalog seed`);
|
||
const { scrapeMoxaOem } = await import("./scrapers/moxa-oem");
|
||
await scrapeMoxaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:lumentum-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Lumentum OEM catalog seed`);
|
||
const { scrapeLumentumOem } = await import("./scrapers/lumentum-oem");
|
||
await scrapeLumentumOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:coherent-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Coherent Corp OEM catalog seed`);
|
||
const { scrapeCoherentOem } = await import("./scrapers/coherent-oem");
|
||
await scrapeCoherentOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:brocade-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Brocade OEM catalog seed`);
|
||
const { scrapeBrocadeOem } = await import("./scrapers/brocade-oem");
|
||
await scrapeBrocadeOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ruggedcom-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Siemens RUGGEDCOM OEM catalog seed`);
|
||
const { scrapeRuggedcomOem } = await import("./scrapers/ruggedcom-oem");
|
||
await scrapeRuggedcomOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hirschmann-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Hirschmann OEM catalog seed`);
|
||
const { scrapeHirschmannOem } = await import("./scrapers/hirschmann-oem");
|
||
await scrapeHirschmannOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:viavi-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Viavi Solutions OEM catalog seed`);
|
||
const { scrapeViaviOem } = await import("./scrapers/viavi-oem");
|
||
await scrapeViaviOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:westermo-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Westermo OEM catalog seed`);
|
||
const { scrapeWestermoOem } = await import("./scrapers/westermo-oem");
|
||
await scrapeWestermoOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:lancom-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: LANCOM Systems OEM catalog seed`);
|
||
const { scrapeLancomOem } = await import("./scrapers/lancom-oem");
|
||
await scrapeLancomOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:dzs-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: DZS OEM catalog seed`);
|
||
const { scrapeDzsOem } = await import("./scrapers/dzs-oem");
|
||
await scrapeDzsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:watchguard-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: WatchGuard OEM catalog seed`);
|
||
const { scrapeWatchguardOem } = await import("./scrapers/watchguard-oem");
|
||
await scrapeWatchguardOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:sonicwall-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SonicWall OEM catalog seed`);
|
||
const { scrapeSonicwallOem } = await import("./scrapers/sonicwall-oem");
|
||
await scrapeSonicwallOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:advantech-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Advantech OEM catalog seed`);
|
||
const { scrapeAdvantechOem } = await import("./scrapers/advantech-oem");
|
||
await scrapeAdvantechOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:redlion-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Red Lion Controls OEM catalog seed`);
|
||
const { scrapeRedlionOem } = await import("./scrapers/redlion-oem");
|
||
await scrapeRedlionOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:exfo-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: EXFO OEM catalog seed`);
|
||
const { scrapeExfoOem } = await import("./scrapers/exfo-oem");
|
||
await scrapeExfoOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:mellanox-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NVIDIA Mellanox OEM catalog seed`);
|
||
const { scrapeMellanoxOem } = await import("./scrapers/mellanox-oem");
|
||
await scrapeMellanoxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:intel-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Intel OEM catalog seed`);
|
||
const { scrapeIntelOem } = await import("./scrapers/intel-oem");
|
||
await scrapeIntelOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:antaira-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Antaira Technologies OEM catalog seed`);
|
||
const { scrapeAntairaOem } = await import("./scrapers/antaira-oem");
|
||
await scrapeAntairaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:transition-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Transition Networks OEM catalog seed`);
|
||
const { scrapeTransitionNetworksOem } = await import("./scrapers/transition-networks-oem");
|
||
await scrapeTransitionNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:perle-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Perle Systems OEM catalog seed`);
|
||
const { scrapePerleOem } = await import("./scrapers/perle-oem");
|
||
await scrapePerleOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:barracuda-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Barracuda Networks OEM catalog seed`);
|
||
const { scrapeBarracudaOem } = await import("./scrapers/barracuda-oem");
|
||
await scrapeBarracudaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:etherwan-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: EtherWAN Systems OEM catalog seed`);
|
||
const { scrapeEtherwanOem } = await import("./scrapers/etherwan-oem");
|
||
await scrapeEtherwanOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:blackbox-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`);
|
||
const { scrapeBlackboxOem } = await import("./scrapers/blackbox-oem");
|
||
await scrapeBlackboxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:packetlight-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: PacketLight Networks OEM catalog seed`);
|
||
const { scrapePacketLightOem } = await import("./scrapers/packetlight-oem");
|
||
await scrapePacketLightOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:spirent-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Spirent Communications OEM catalog seed`);
|
||
const { scrapeSpirentOem } = await import("./scrapers/spirent-oem");
|
||
await scrapeSpirentOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:comnet-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Comnet OEM catalog seed`);
|
||
const { scrapeComnetOem } = await import("./scrapers/comnet-oem");
|
||
await scrapeComnetOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:korenix-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Korenix Technology OEM catalog seed`);
|
||
const { scrapeKorenixOem } = await import("./scrapers/korenix-oem");
|
||
await scrapeKorenixOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:sophos-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Sophos OEM catalog seed`);
|
||
const { scrapeSophosOem } = await import("./scrapers/sophos-oem");
|
||
await scrapeSophosOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cradlepoint-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cradlepoint OEM catalog seed`);
|
||
const { scrapeCradlepointOem } = await import("./scrapers/cradlepoint-oem");
|
||
await scrapeCradlepointOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:fujitsu-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Fujitsu Network Communications OEM catalog seed`);
|
||
const { scrapeFujitsuOem } = await import("./scrapers/fujitsu-oem");
|
||
await scrapeFujitsuOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nec-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NEC Corporation OEM catalog seed`);
|
||
const { scrapeNecOem } = await import("./scrapers/nec-oem");
|
||
await scrapeNecOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:arris-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: CommScope ARRIS OEM catalog seed`);
|
||
const { scrapeArrisOem } = await import("./scrapers/arris-oem");
|
||
await scrapeArrisOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hillstone-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Hillstone Networks OEM catalog seed`);
|
||
const { scrapeHillstoneOem } = await import("./scrapers/hillstone-oem");
|
||
await scrapeHillstoneOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:vecima-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Vecima Networks OEM catalog seed`);
|
||
const { scrapeVecimaOem } = await import("./scrapers/vecima-oem");
|
||
await scrapeVecimaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:stordis-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Stordis OEM catalog seed`);
|
||
const { scrapeStordisOem } = await import("./scrapers/stordis-oem");
|
||
await scrapeStordisOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:keysight-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Keysight OEM catalog seed`);
|
||
const { scrapeKeysightOem } = await import("./scrapers/keysight-oem");
|
||
await scrapeKeysightOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:sycamore-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Sycamore OEM catalog seed`);
|
||
const { scrapeSycamoreOem } = await import("./scrapers/sycamore-oem");
|
||
await scrapeSycamoreOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ekinops-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ekinops OEM catalog seed`);
|
||
const { scrapeEkinopsOem } = await import("./scrapers/ekinops-oem");
|
||
await scrapeEkinopsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:adva-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ADVA Optical OEM catalog seed`);
|
||
const { scrapeAdvaOem } = await import("./scrapers/adva-oem");
|
||
await scrapeAdvaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:coriant-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Coriant OEM catalog seed`);
|
||
const { scrapeCoriantOem } = await import("./scrapers/coriant-oem");
|
||
await scrapeCoriantOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:casa-systems-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Casa Systems OEM catalog seed`);
|
||
const { scrapeCasaSystemsOem } = await import("./scrapers/casa-systems-oem");
|
||
await scrapeCasaSystemsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:harmonic-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Harmonic OEM catalog seed`);
|
||
const { scrapeHarmonicOem } = await import("./scrapers/harmonic-oem");
|
||
await scrapeHarmonicOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:solarflare-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Solarflare OEM catalog seed`);
|
||
const { scrapeSolarflareOem } = await import("./scrapers/solarflare-oem");
|
||
await scrapeSolarflareOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:marvell-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Marvell OEM catalog seed`);
|
||
const { scrapeMarvellOem } = await import("./scrapers/marvell-oem");
|
||
await scrapeMarvellOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:broadcom-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Broadcom OEM catalog seed`);
|
||
const { scrapeBroadcomOem } = await import("./scrapers/broadcom-oem");
|
||
await scrapeBroadcomOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:calix-access-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Calix Access OEM catalog seed`);
|
||
const { scrapeCalixAccessOem } = await import("./scrapers/calix-access-oem");
|
||
await scrapeCalixAccessOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ribbon-comms-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ribbon Communications OEM catalog seed`);
|
||
const { scrapeRibbonCommsOem } = await import("./scrapers/ribbon-comms-oem");
|
||
await scrapeRibbonCommsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:infinera-groove-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Infinera Groove OEM catalog seed`);
|
||
const { scrapeInfineraGrooveOem } = await import("./scrapers/infinera-groove-oem");
|
||
await scrapeInfineraGrooveOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ciena-waveserver-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ciena WaveServer OEM catalog seed`);
|
||
const { scrapeCienaWaveserverOem } = await import("./scrapers/ciena-waveserver-oem");
|
||
await scrapeCienaWaveserverOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:commscope-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: CommScope OEM catalog seed`);
|
||
const { scrapeCommScopeOem } = await import("./scrapers/commscope-oem");
|
||
await scrapeCommScopeOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:teleste-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Teleste OEM catalog seed`);
|
||
const { scrapeTelesteOem } = await import("./scrapers/teleste-oem");
|
||
await scrapeTelesteOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:tejas-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Tejas Networks OEM catalog seed`);
|
||
const { scrapeTejasNetworksOem } = await import("./scrapers/tejas-networks-oem");
|
||
await scrapeTejasNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ericsson-transport-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ericsson Transport OEM catalog seed`);
|
||
const { scrapeEricssonTransportOem } = await import("./scrapers/ericsson-transport-oem");
|
||
await scrapeEricssonTransportOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:adtran-ta-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ADTRAN TA OEM catalog seed`);
|
||
const { scrapeAdtranTaOem } = await import("./scrapers/adtran-ta-oem");
|
||
await scrapeAdtranTaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:isolan-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: iSolan OEM catalog seed`);
|
||
const { scrapeIsolanOem } = await import("./scrapers/isolan-oem");
|
||
await scrapeIsolanOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:telco-systems-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: TELCO Systems OEM catalog seed`);
|
||
const { scrapeTelcoSystemsOem } = await import("./scrapers/telco-systems-oem");
|
||
await scrapeTelcoSystemsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:rad-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: RAD Data Communications OEM catalog seed`);
|
||
const { scrapeRadOem } = await import("./scrapers/rad-oem");
|
||
await scrapeRadOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:comtrend-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Comtrend OEM catalog seed`);
|
||
const { scrapeComtrendOem } = await import("./scrapers/comtrend-oem");
|
||
await scrapeComtrendOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:packetfront-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: PacketFront OEM catalog seed`);
|
||
const { scrapePacketfrontOem } = await import("./scrapers/packetfront-oem");
|
||
await scrapePacketfrontOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:edgewater-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Edgewater Networks OEM catalog seed`);
|
||
const { scrapeEdgewaterNetworksOem } = await import("./scrapers/edgewater-networks-oem");
|
||
await scrapeEdgewaterNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:corning-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Corning OEM catalog seed`);
|
||
const { scrapeCorningOem } = await import("./scrapers/corning-oem");
|
||
await scrapeCorningOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ofs-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: OFS OEM catalog seed`);
|
||
const { scrapeOfsOem } = await import("./scrapers/ofs-oem");
|
||
await scrapeOfsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:kontron-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Kontron OEM catalog seed`);
|
||
const { scrapeKontronOem } = await import("./scrapers/kontron-oem");
|
||
await scrapeKontronOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ipinfusion-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: IP Infusion OEM catalog seed`);
|
||
const { scrapeIpInfusionOem } = await import("./scrapers/ipinfusion-oem");
|
||
await scrapeIpInfusionOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:telrad-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Telrad Networks OEM catalog seed`);
|
||
const { scrapeTelradOem } = await import("./scrapers/telrad-oem");
|
||
await scrapeTelradOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:siklu-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Siklu OEM catalog seed`);
|
||
const { scrapeSikluOem } = await import("./scrapers/siklu-oem");
|
||
await scrapeSikluOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ceragon-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ceragon OEM catalog seed`);
|
||
const { scrapeCeragonOem } = await import("./scrapers/ceragon-oem");
|
||
await scrapeCeragonOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:datang-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Datang Telecom OEM catalog seed`);
|
||
const { scrapeDatangOem } = await import("./scrapers/datang-oem");
|
||
await scrapeDatangOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:viptela-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Viptela OEM catalog seed`);
|
||
const { scrapeViptelaOem } = await import("./scrapers/viptela-oem");
|
||
await scrapeViptelaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:versa-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Versa Networks OEM catalog seed`);
|
||
const { scrapeVersaNetworksOem } = await import("./scrapers/versa-networks-oem");
|
||
await scrapeVersaNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:vmware-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: VMware OEM catalog seed`);
|
||
const { scrapeVmwareOem } = await import("./scrapers/vmware-oem");
|
||
await scrapeVmwareOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cimc-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: CIMC Semiconductors OEM catalog seed`);
|
||
const { scrapeCimcOem } = await import("./scrapers/cimc-oem");
|
||
await scrapeCimcOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:qlogic-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: QLogic OEM catalog seed`);
|
||
const { scrapeQlogicOem } = await import("./scrapers/qlogic-oem");
|
||
await scrapeQlogicOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:emulex-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Emulex OEM catalog seed`);
|
||
const { scrapeEmulexOem } = await import("./scrapers/emulex-oem");
|
||
await scrapeEmulexOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:netapp-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NetApp OEM catalog seed`);
|
||
const { scrapeNetappOem } = await import("./scrapers/netapp-oem");
|
||
await scrapeNetappOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:pure-storage-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Pure Storage OEM catalog seed`);
|
||
const { scrapePureStorageOem } = await import("./scrapers/pure-storage-oem");
|
||
await scrapePureStorageOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hpe-storage-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: HPE Storage OEM catalog seed`);
|
||
const { scrapeHpeStorageOem } = await import("./scrapers/hpe-storage-oem");
|
||
await scrapeHpeStorageOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ibm-storage-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: IBM Storage OEM catalog seed`);
|
||
const { scrapeIbmStorageOem } = await import("./scrapers/ibm-storage-oem");
|
||
await scrapeIbmStorageOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:dell-storage-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Dell Storage OEM catalog seed`);
|
||
const { scrapeDellStorageOem } = await import("./scrapers/dell-storage-oem");
|
||
await scrapeDellStorageOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hitachi-vantara-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Hitachi Vantara OEM catalog seed`);
|
||
const { scrapeHitachiVantaraOem } = await import("./scrapers/hitachi-vantara-oem");
|
||
await scrapeHitachiVantaraOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:aws-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: AWS OEM catalog seed`);
|
||
const { scrapeAwsOem } = await import("./scrapers/aws-oem");
|
||
await scrapeAwsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:azure-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Microsoft Azure OEM catalog seed`);
|
||
const { scrapeAzureOem } = await import("./scrapers/azure-oem");
|
||
await scrapeAzureOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:google-cloud-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Google Cloud OEM catalog seed`);
|
||
const { scrapeGoogleCloudOem } = await import("./scrapers/google-cloud-oem");
|
||
await scrapeGoogleCloudOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:meta-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Meta OEM catalog seed`);
|
||
const { scrapeMetaOem } = await import("./scrapers/meta-oem");
|
||
await scrapeMetaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nokia-access-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Nokia Access OEM catalog seed`);
|
||
const { scrapeNokiaAccessOem } = await import("./scrapers/nokia-access-oem");
|
||
await scrapeNokiaAccessOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:huawei-access-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Huawei Access OEM catalog seed`);
|
||
const { scrapeHuaweiAccessOem } = await import("./scrapers/huawei-access-oem");
|
||
await scrapeHuaweiAccessOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:zte-access-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ZTE Access OEM catalog seed`);
|
||
const { scrapeZteAccessOem } = await import("./scrapers/zte-access-oem");
|
||
await scrapeZteAccessOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:calix-gigapoint-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Calix GigaPoint OEM catalog seed`);
|
||
const { scrapeCalixGigapointOem } = await import("./scrapers/calix-gigapoint-oem");
|
||
await scrapeCalixGigapointOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:samsung-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Samsung Networks OEM catalog seed`);
|
||
const { scrapeSamsungNetworksOem } = await import("./scrapers/samsung-networks-oem");
|
||
await scrapeSamsungNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nokia-airscale-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Nokia AirScale OEM catalog seed`);
|
||
const { scrapeNokiaAirscaleOem } = await import("./scrapers/nokia-airscale-oem");
|
||
await scrapeNokiaAirscaleOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ericsson-ran-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ericsson RAN OEM catalog seed`);
|
||
const { scrapeEricssonRanOem } = await import("./scrapers/ericsson-ran-oem");
|
||
await scrapeEricssonRanOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:mavenir-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Mavenir OEM catalog seed`);
|
||
const { scrapeMavenirOem } = await import("./scrapers/mavenir-oem");
|
||
await scrapeMavenirOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ixia-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ixia OEM catalog seed`);
|
||
const { scrapeIxiaOem } = await import("./scrapers/ixia-oem");
|
||
await scrapeIxiaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:exfo-network-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: EXFO Network OEM catalog seed`);
|
||
const { scrapeExfoNetworkOem } = await import("./scrapers/exfo-network-oem");
|
||
await scrapeExfoNetworkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cumulus-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cumulus Networks OEM catalog seed`);
|
||
const { scrapeCumulusNetworksOem } = await import("./scrapers/cumulus-networks-oem");
|
||
await scrapeCumulusNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:sonic-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SONiC OEM catalog seed`);
|
||
const { scrapeSonicOem } = await import("./scrapers/sonic-oem");
|
||
await scrapeSonicOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:h3c-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: H3C OEM catalog seed`);
|
||
const { scrapeH3cOem } = await import("./scrapers/h3c-oem");
|
||
await scrapeH3cOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ruijie-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ruijie Networks OEM catalog seed`);
|
||
const { scrapeRuijieOem } = await import("./scrapers/ruijie-oem");
|
||
await scrapeRuijieOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:centec-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Centec Networks OEM catalog seed`);
|
||
const { scrapeCentecOem } = await import("./scrapers/centec-oem");
|
||
await scrapeCentecOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:supermicro-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Supermicro OEM catalog seed`);
|
||
const { scrapeSupermicroOem } = await import("./scrapers/supermicro-oem");
|
||
await scrapeSupermicroOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cisco-meraki-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco Meraki OEM catalog seed`);
|
||
const { scrapeCiscoMerakiOem } = await import("./scrapers/cisco-meraki-oem");
|
||
await scrapeCiscoMerakiOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cisco-catalyst-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco Catalyst OEM catalog seed`);
|
||
const { scrapeCiscoCatalystOem } = await import("./scrapers/cisco-catalyst-oem");
|
||
await scrapeCiscoCatalystOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cisco-nexus-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco Nexus OEM catalog seed`);
|
||
const { scrapeCiscoNexusOem } = await import("./scrapers/cisco-nexus-oem");
|
||
await scrapeCiscoNexusOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cisco-asr-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco ASR OEM catalog seed`);
|
||
const { scrapeCiscoAsrOem } = await import("./scrapers/cisco-asr-oem");
|
||
await scrapeCiscoAsrOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:juniper-mx-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Juniper MX OEM catalog seed`);
|
||
const { scrapeJuniperMxOem } = await import("./scrapers/juniper-mx-oem");
|
||
await scrapeJuniperMxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:juniper-qfx-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Juniper QFX OEM catalog seed`);
|
||
const { scrapeJuniperQfxOem } = await import("./scrapers/juniper-qfx-oem");
|
||
await scrapeJuniperQfxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:aruba-cx-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Aruba CX OEM catalog seed`);
|
||
const { scrapeArubaCxOem } = await import("./scrapers/aruba-cx-oem");
|
||
await scrapeArubaCxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:extreme-campus-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Extreme Campus OEM catalog seed`);
|
||
const { scrapeExtremeCampusOem } = await import("./scrapers/extreme-campus-oem");
|
||
await scrapeExtremeCampusOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:arista-7000-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Arista 7000 OEM catalog seed`);
|
||
const { scrapeArista7000Oem } = await import("./scrapers/arista-7000-oem");
|
||
await scrapeArista7000Oem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:pica8-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Pica8 OEM catalog seed`);
|
||
const { scrapePica8Oem } = await import("./scrapers/pica8-oem");
|
||
await scrapePica8Oem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:pluribus-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Pluribus Networks OEM catalog seed`);
|
||
const { scrapePluribusOem } = await import("./scrapers/pluribus-oem");
|
||
await scrapePluribusOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:drivenets-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: DriveNets OEM catalog seed`);
|
||
const { scrapeDrivenetsOem } = await import("./scrapers/drivenets-oem");
|
||
await scrapeDrivenetsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:phoenix-contact-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Phoenix Contact OEM catalog seed`);
|
||
const { scrapePhoenixContactOem } = await import("./scrapers/phoenix-contact-oem");
|
||
await scrapePhoenixContactOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:beckhoff-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Beckhoff OEM catalog seed`);
|
||
const { scrapeBeckhoffOem } = await import("./scrapers/beckhoff-oem");
|
||
await scrapeBeckhoffOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:omron-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Omron OEM catalog seed`);
|
||
const { scrapeOmronOem } = await import("./scrapers/omron-oem");
|
||
await scrapeOmronOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:abb-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ABB OEM catalog seed`);
|
||
const { scrapeAbbOem } = await import("./scrapers/abb-oem");
|
||
await scrapeAbbOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:siemens-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Siemens SCALANCE OEM catalog seed`);
|
||
const { scrapeSiemensOem } = await import("./scrapers/siemens-oem");
|
||
await scrapeSiemensOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:schneider-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Schneider Electric OEM catalog seed`);
|
||
const { scrapeSchneiderOem } = await import("./scrapers/schneider-oem");
|
||
await scrapeSchneiderOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:rockwell-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Rockwell Automation OEM catalog seed`);
|
||
const { scrapeRockwellOem } = await import("./scrapers/rockwell-oem");
|
||
await scrapeRockwellOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:belden-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Belden OEM catalog seed`);
|
||
const { scrapeBeldenOem } = await import("./scrapers/belden-oem");
|
||
await scrapeBeldenOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:ge-grid-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: GE Grid Solutions OEM catalog seed`);
|
||
const { scrapeGeGridOem } = await import("./scrapers/ge-grid-oem");
|
||
await scrapeGeGridOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:schweitzer-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Schweitzer Engineering OEM catalog seed`);
|
||
const { scrapeSchweiterEngOem } = await import("./scrapers/schweitzer-oem");
|
||
await scrapeSchweiterEngOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:moxa-industrial-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Moxa Industrial OEM catalog seed`);
|
||
const { scrapeIndustrialMoxaOem } = await import("./scrapers/moxa-industrial-oem");
|
||
await scrapeIndustrialMoxaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cisco-ie-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco Industrial Ethernet OEM catalog seed`);
|
||
const { scrapeCiscoIeOem } = await import("./scrapers/cisco-ie-oem");
|
||
await scrapeCiscoIeOem();
|
||
});
|
||
|
||
// ── Media / Broadcast / Pro-AV ────────────────────────────────────────
|
||
await boss.work("scrape:catalog:evertz-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Evertz Microsystems OEM catalog seed`);
|
||
const { scrapeEvertzOem } = await import("./scrapers/evertz-oem");
|
||
await scrapeEvertzOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:grass-valley-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Grass Valley OEM catalog seed`);
|
||
const { scrapeGrassValleyOem } = await import("./scrapers/grass-valley-oem");
|
||
await scrapeGrassValleyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:haivision-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Haivision OEM catalog seed`);
|
||
const { scrapeHaivisionOem } = await import("./scrapers/haivision-oem");
|
||
await scrapeHaivisionOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:viasat-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Viasat OEM catalog seed`);
|
||
const { scrapeViasatOem } = await import("./scrapers/viasat-oem");
|
||
await scrapeViasatOem();
|
||
});
|
||
|
||
// ── Asian Optical Vendors ─────────────────────────────────────────────
|
||
await boss.work("scrape:catalog:fiberhome-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: FiberHome Technologies OEM catalog seed`);
|
||
const { scrapeFiberhomeOem } = await import("./scrapers/fiberhome-oem");
|
||
await scrapeFiberhomeOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:oplink-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Oplink Communications OEM catalog seed`);
|
||
const { scrapeOplinkOem } = await import("./scrapers/oplink-oem");
|
||
await scrapeOplinkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:accelink-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Accelink Technologies OEM catalog seed`);
|
||
const { scrapeAccelinkOem } = await import("./scrapers/accelink-oem");
|
||
await scrapeAccelinkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:hisense-broadband-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Hisense Broadband OEM catalog seed`);
|
||
const { scrapeHisenseBroadbandOem } = await import("./scrapers/hisense-broadband-oem");
|
||
await scrapeHisenseBroadbandOem();
|
||
});
|
||
|
||
// ── Optical Transceiver Manufacturers (batch 24–25) ───────────────────
|
||
await boss.work("scrape:catalog:ii-vi-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: II-VI / Coherent OEM catalog seed`);
|
||
const { scrapeIiViOem } = await import("./scrapers/ii-vi-oem");
|
||
await scrapeIiViOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:source-photonics-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Source Photonics OEM catalog seed`);
|
||
const { scrapeSourcePhotonicsOem } = await import("./scrapers/source-photonics-oem");
|
||
await scrapeSourcePhotonicsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:o-net-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: O-Net Technologies OEM catalog seed`);
|
||
const { scrapeONetOem } = await import("./scrapers/o-net-oem");
|
||
await scrapeONetOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:innolight-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: InnoLight Technology OEM catalog seed`);
|
||
const { scrapeInnoLightOem } = await import("./scrapers/innolight-oem");
|
||
await scrapeInnoLightOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:aoi-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Applied Optoelectronics (AOI) OEM catalog seed`);
|
||
const { scrapeAoiOem } = await import("./scrapers/aoi-oem");
|
||
await scrapeAoiOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:sumitomo-electric-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Sumitomo Electric Networks OEM catalog seed`);
|
||
const { scrapeSumitomoElectricOem } = await import("./scrapers/sumitomo-electric-oem");
|
||
await scrapeSumitomoElectricOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:neophotonics-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NeoPhotonics OEM catalog seed`);
|
||
const { scrapeNeoPhotonicsOem } = await import("./scrapers/neophotonics-oem");
|
||
await scrapeNeoPhotonicsOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:audiocodes-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: AudioCodes OEM catalog seed`);
|
||
const { scrapeAudioCodesOem } = await import("./scrapers/audiocodes-oem");
|
||
await scrapeAudioCodesOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:anritsu-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Anritsu OEM catalog seed`);
|
||
const { scrapeAnritsuOem } = await import("./scrapers/anritsu-oem");
|
||
await scrapeAnritsuOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:netscout-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NETSCOUT OEM catalog seed`);
|
||
const { scrapeNetScoutOem } = await import("./scrapers/netscout-oem");
|
||
await scrapeNetScoutOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:curtiss-wright-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Curtiss-Wright OEM catalog seed`);
|
||
const { scrapeCurtissWrightOem } = await import("./scrapers/curtiss-wright-oem");
|
||
await scrapeCurtissWrightOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:eci-telecom-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: ECI Telecom OEM catalog seed`);
|
||
const { scrapeEciTelecomOem } = await import("./scrapers/eci-telecom-oem");
|
||
await scrapeEciTelecomOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:utstarcom-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: UTStarcom OEM catalog seed`);
|
||
const { scrapeUTStarcomOem } = await import("./scrapers/utstarcom-oem");
|
||
await scrapeUTStarcomOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:turbolink-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Turbolink OEM catalog seed`);
|
||
const { scrapeTurbolinkOem } = await import("./scrapers/turbolink-oem");
|
||
await scrapeTurbolinkOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:chelsio-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Chelsio OEM catalog seed`);
|
||
const { scrapeChelsioOem } = await import("./scrapers/chelsio-oem");
|
||
await scrapeChelsioOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:rohde-schwarz-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Rohde & Schwarz OEM catalog seed`);
|
||
const { scrapeRohdeScharzOem } = await import("./scrapers/rohde-schwarz-oem");
|
||
await scrapeRohdeScharzOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:l3harris-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: L3Harris OEM catalog seed`);
|
||
const { scrapeL3HarrisOem } = await import("./scrapers/l3harris-oem");
|
||
await scrapeL3HarrisOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:zhone-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Zhone/DZS OEM catalog seed`);
|
||
const { scrapeZhoneOem } = await import("./scrapers/zhone-oem");
|
||
await scrapeZhoneOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:cambium-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cambium Networks OEM catalog seed`);
|
||
const { scrapeCambiumNetworksOem } = await import("./scrapers/cambium-networks-oem");
|
||
await scrapeCambiumNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:tektronix-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Tektronix OEM catalog seed`);
|
||
const { scrapeTektronixOem } = await import("./scrapers/tektronix-oem");
|
||
await scrapeTektronixOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:clearfield-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Clearfield OEM catalog seed`);
|
||
const { scrapeClearfieldOem } = await import("./scrapers/clearfield-oem");
|
||
await scrapeClearfieldOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:lanner-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Lanner Electronics OEM catalog seed`);
|
||
const { scrapeLannerOem } = await import("./scrapers/lanner-oem");
|
||
await scrapeLannerOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:black-box-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Black Box OEM catalog seed`);
|
||
const { scrapeBlackBoxOem } = await import("./scrapers/black-box-oem");
|
||
await scrapeBlackBoxOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:radiflow-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Radiflow OEM catalog seed`);
|
||
const { scrapeRadiflowOem } = await import("./scrapers/radiflow-oem");
|
||
await scrapeRadiflowOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:dragonwave-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: DragonWave OEM catalog seed`);
|
||
const { scrapeDragonwaveOem } = await import("./scrapers/dragonwave-oem");
|
||
await scrapeDragonwaveOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:teledyne-lecroy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Teledyne LeCroy OEM catalog seed`);
|
||
const { scrapeTeledyneLeCroyOem } = await import("./scrapers/teledyne-lecroy-oem");
|
||
await scrapeTeledyneLeCroyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:finisar-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Finisar OEM catalog seed`);
|
||
const { scrapeFinisarOem } = await import("./scrapers/finisar-oem");
|
||
await scrapeFinisarOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:acacia-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Acacia Communications OEM catalog seed`);
|
||
const { scrapeAcaciaOem } = await import("./scrapers/acacia-oem");
|
||
await scrapeAcaciaOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:inphi-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Inphi (Marvell) OEM catalog seed`);
|
||
const { scrapeInphiOem } = await import("./scrapers/inphi-oem");
|
||
await scrapeInphiOem();
|
||
});
|
||
|
||
// ── Batch 35 workers ─────────────────────────────────────────────────
|
||
await boss.work("scrape:catalog:sierra-wireless-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Sierra Wireless (Semtech) OEM catalog seed`);
|
||
const { scrapeSierraWirelessOem } = await import("./scrapers/sierra-wireless-oem");
|
||
await scrapeSierraWirelessOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:senao-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Senao (EnGenius) OEM catalog seed`);
|
||
const { scrapeSenaoOem } = await import("./scrapers/senao-oem");
|
||
await scrapeSenaoOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:emcore-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: EMCORE OEM catalog seed`);
|
||
const { scrapeEmcoreOem } = await import("./scrapers/emcore-oem");
|
||
await scrapeEmcoreOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:reflex-photonics-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Reflex Photonics OEM catalog seed`);
|
||
const { scrapeReflexPhotonicsOem } = await import("./scrapers/reflex-photonics-oem");
|
||
await scrapeReflexPhotonicsOem();
|
||
});
|
||
|
||
// ── Batch 36 workers ─────────────────────────────────────────────────
|
||
await boss.work("scrape:catalog:engenius-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: EnGenius OEM catalog seed`);
|
||
const { scrapeEngeniusOem } = await import("./scrapers/engenius-oem");
|
||
await scrapeEngeniusOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:paloalto-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Palo Alto Networks OEM catalog seed`);
|
||
const { scrapePaloaltoNetworksOem } = await import("./scrapers/paloalto-networks-oem");
|
||
await scrapePaloaltoNetworksOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:brocade-legacy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Brocade (Legacy) OEM catalog seed`);
|
||
const { scrapeBrocadeLegacyOem } = await import("./scrapers/brocade-legacy-oem");
|
||
await scrapeBrocadeLegacyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:foundry-networks-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Foundry Networks (Legacy) OEM catalog seed`);
|
||
const { scrapeFoundryNetworksOem } = await import("./scrapers/foundry-networks-oem");
|
||
await scrapeFoundryNetworksOem();
|
||
});
|
||
|
||
// ── Batch 37 workers ─────────────────────────────────────────────────
|
||
await boss.work("scrape:catalog:extreme-legacy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Extreme Networks (Legacy) OEM catalog seed`);
|
||
const { scrapeExtremeLegacyOem } = await import("./scrapers/extreme-legacy-oem");
|
||
await scrapeExtremeLegacyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:nortel-legacy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Nortel Networks (Legacy) OEM catalog seed`);
|
||
const { scrapeNortelLegacyOem } = await import("./scrapers/nortel-legacy-oem");
|
||
await scrapeNortelLegacyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:3com-legacy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: 3Com (Legacy) OEM catalog seed`);
|
||
const { scrape3comLegacyOem } = await import("./scrapers/3com-legacy-oem");
|
||
await scrape3comLegacyOem();
|
||
});
|
||
|
||
await boss.work("scrape:catalog:avaya-legacy-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Avaya (Legacy) OEM catalog seed`);
|
||
const { scrapeAvayaLegacyOem } = await import("./scrapers/avaya-legacy-oem");
|
||
await scrapeAvayaLegacyOem();
|
||
});
|
||
|
||
// ── Vendor lists ──────────────────────────────────────────────────────
|
||
|
||
await boss.work("scrape:vendors:flexoptix", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Flexoptix vendor list`);
|
||
await scrapeFlexoptixVendors();
|
||
});
|
||
|
||
await boss.work("scrape:vendors:flexoptix-supported", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Flexoptix supported vendors`);
|
||
await seedFlexoptixVendors();
|
||
});
|
||
|
||
// ── Compatibility scrapers ────────────────────────────────────────────
|
||
|
||
await boss.work("scrape:compat:flexoptix", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Flexoptix compatibility mapping`);
|
||
if (!isLoadAcceptable(2.5)) {
|
||
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Flexoptix compat scrape`);
|
||
return;
|
||
}
|
||
await scrapeFlexoptixCompatibility();
|
||
});
|
||
|
||
await boss.work("scrape:compat:cisco", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Cisco TMG compatibility`);
|
||
await scrapeCiscoTmg();
|
||
});
|
||
|
||
await boss.work("scrape:compat:juniper", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Juniper HCT compatibility`);
|
||
await scrapeJuniperHct();
|
||
});
|
||
|
||
await boss.work("scrape:compat:sonic", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SONiC HCL compatibility`);
|
||
await scrapeSonicHcl();
|
||
});
|
||
|
||
await boss.work("scrape:compat:ufispace", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Ufispace switch data`);
|
||
await scrapeUfiSpace();
|
||
});
|
||
|
||
await boss.work("scrape:compat:edgecore", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Edgecore switch data`);
|
||
await scrapeEdgecore();
|
||
});
|
||
|
||
// ── Switch assets ─────────────────────────────────────────────────────
|
||
|
||
await boss.work("scrape:assets:switches", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Switch assets enrichment`);
|
||
await scrapeSwitchAssets();
|
||
});
|
||
|
||
await boss.work("scrape:images:switches", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Switch og:image fetcher`);
|
||
if (!isLoadAcceptable(2.5)) {
|
||
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping switch image fetch`);
|
||
return;
|
||
}
|
||
await fetchSwitchImages();
|
||
});
|
||
|
||
await boss.work("scrape:images:switches:playwright", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Switch image fetcher (Playwright — bot-blocked vendors)`);
|
||
if (!isLoadAcceptable(2.0)) {
|
||
console.warn(`[${new Date().toISOString()}] ⚠ Load too high — skipping Playwright image fetch`);
|
||
return;
|
||
}
|
||
await fetchSwitchImagesPlaywright();
|
||
});
|
||
|
||
// ── eBay enrichment ───────────────────────────────────────────────────
|
||
|
||
await boss.work("enrich:ebay-transceivers", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: eBay transceiver pricing`);
|
||
const { enrichTransceiversFromEbay } = await import("./scrapers/ebay-enricher");
|
||
await enrichTransceiversFromEbay(100);
|
||
});
|
||
|
||
await boss.work("enrich:ebay-switches", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: eBay switch enrichment`);
|
||
const { enrichSwitchesFromEbay } = await import("./scrapers/ebay-enricher");
|
||
await enrichSwitchesFromEbay(30);
|
||
});
|
||
|
||
// ── Intelligence & community ──────────────────────────────────────────
|
||
|
||
await boss.work("scrape:market-intel", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Market intelligence`);
|
||
await scrapeMarketIntelligence();
|
||
});
|
||
|
||
await boss.work("scrape:nog-talks", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NOG conference talks`);
|
||
const { scrapeNogTalks } = await import("./scrapers/nog-talks");
|
||
await scrapeNogTalks();
|
||
});
|
||
|
||
await boss.work("scrape:community-issues", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Community issues`);
|
||
const { scrapeAllSwitchIssues, scrapeTransceiverCompatIssues } = await import("./scrapers/community-issues");
|
||
await scrapeAllSwitchIssues(30);
|
||
await scrapeTransceiverCompatIssues(15);
|
||
});
|
||
|
||
await boss.work("scrape:datasheet-links", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Datasheet links`);
|
||
const { findAndSeedDatasheetLinks } = await import("./scrapers/community-issues");
|
||
await findAndSeedDatasheetLinks(50);
|
||
});
|
||
|
||
await boss.work("scrape:news", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: News scraper`);
|
||
await scrapeNews();
|
||
});
|
||
|
||
await boss.work("scrape:faq", async () => {
|
||
console.log(`[${new Date().toISOString()}] FAQ scraper — not yet implemented`);
|
||
});
|
||
|
||
await boss.work("scrape:docs", async () => {
|
||
console.log(`[${new Date().toISOString()}] Docs scraper — not yet implemented`);
|
||
});
|
||
|
||
// ── Compute jobs ──────────────────────────────────────────────────────
|
||
|
||
await boss.work("compute:abc", async () => {
|
||
console.log(`[${new Date().toISOString()}] Computing: ABC classification`);
|
||
const { computeAbcClassification } = await import("./scrapers/market-intelligence");
|
||
await computeAbcClassification();
|
||
});
|
||
|
||
await boss.work("compute:reorder-signals", async () => {
|
||
console.log(`[${new Date().toISOString()}] Computing: Reorder signals`);
|
||
const { computeReorderSignals } = await import("./scrapers/market-intelligence");
|
||
await computeReorderSignals();
|
||
});
|
||
|
||
// ── NAS sync ──────────────────────────────────────────────────────────
|
||
|
||
await boss.work("sync:nas", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NAS sync to Fearghas`);
|
||
const { runNightlyNasSync } = await import("./utils/nas-sync");
|
||
await runNightlyNasSync();
|
||
});
|
||
|
||
// ── Prediction signal scrapers ────────────────────────────────────────
|
||
|
||
await boss.work("scrape:signals:sec-edgar", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: SEC EDGAR CapEx`);
|
||
await scrapeSecEdgar();
|
||
});
|
||
|
||
await boss.work("scrape:signals:github", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: GitHub tech signals`);
|
||
await scrapeGithubSignals();
|
||
});
|
||
|
||
await boss.work("scrape:signals:ebay-velocity", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: eBay sold velocity`);
|
||
await scrapeEbayVelocity();
|
||
});
|
||
|
||
await boss.work("scrape:signals:ai-clusters", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: AI cluster announcements`);
|
||
await scrapeAiClusters();
|
||
});
|
||
|
||
await boss.work("scrape:signals:distributor-leads", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Distributor lead times`);
|
||
await scrapeDistributorLeads();
|
||
});
|
||
|
||
await boss.work("scrape:signals:standards", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Standards tracker`);
|
||
await scrapeStandardsTracker();
|
||
});
|
||
|
||
// ── Forecast engine ───────────────────────────────────────────────────
|
||
|
||
await boss.work("compute:forecast", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Forecast engine`);
|
||
await runForecastEngine();
|
||
});
|
||
|
||
// ── Hype Cycle Engine (Norton-Bass diffusion model) ───────────────────
|
||
|
||
await boss.work("compute:hype-cycle", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Hype Cycle Engine (Norton-Bass)`);
|
||
const { computeHypeCycle } = await import("./utils/hype-cycle-engine");
|
||
await computeHypeCycle();
|
||
});
|
||
|
||
// ── Form-factor coverage scrapers ─────────────────────────────────────
|
||
|
||
await boss.work("scrape:pricing:comms-express", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Comms-Express pricing`);
|
||
const { scrapeCommsExpress } = await import("./scrapers/comms-express");
|
||
await scrapeCommsExpress();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:router-switch", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Router-Switch.com pricing`);
|
||
const { scrapeRouterSwitch } = await import("./scrapers/router-switch");
|
||
await scrapeRouterSwitch();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:multimode-inc", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Multimode Inc pricing`);
|
||
const { scrapeMultimodeInc } = await import("./scrapers/multimode-inc");
|
||
await scrapeMultimodeInc();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:optictransceiver", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: OpticTransceiver.com pricing`);
|
||
const { scrapeOpticTransceiver } = await import("./scrapers/optictransceiver");
|
||
await scrapeOpticTransceiver();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:wiitek", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Wiitek pricing`);
|
||
const { scrapeWiitek } = await import("./scrapers/wiitek");
|
||
await scrapeWiitek();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:naddod", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: NADDOD pricing`);
|
||
const { scrapeNaddod } = await import("./scrapers/naddod");
|
||
await scrapeNaddod();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:qsfptek", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: QSFPTEK pricing`);
|
||
const { scrapeQsfptek } = await import("./scrapers/qsfptek");
|
||
await scrapeQsfptek();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:addon", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: AddOn Networks pricing`);
|
||
const { scrapeAddonNetworks } = await import("./scrapers/addon-networks");
|
||
await scrapeAddonNetworks();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:fibermall", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: FiberMall pricing`);
|
||
const { scrapeFiberMall } = await import("./scrapers/fibermall");
|
||
await scrapeFiberMall();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:vcelink", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Vcelink pricing`);
|
||
const { scrapeVcelink } = await import("./scrapers/vcelink");
|
||
await scrapeVcelink();
|
||
});
|
||
|
||
await boss.work("scrape:pricing:opticsbay", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: OpticsBay pricing`);
|
||
const { scrapeOpticsBay } = await import("./scrapers/opticsbay");
|
||
await scrapeOpticsBay();
|
||
});
|
||
|
||
// ── Mouser OEM reference prices ────────────────────────────────────────
|
||
await boss.work("scrape:pricing:mouser-oem", async () => {
|
||
console.log(`[${new Date().toISOString()}] Running: Mouser OEM reference prices`);
|
||
if (!process.env["MOUSER_API_KEY"]) {
|
||
console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set");
|
||
return;
|
||
}
|
||
const { scrapeMouser } = await import("./scrapers/mouser");
|
||
await scrapeMouser();
|
||
});
|
||
|
||
// ── Price denormalization refresh ─────────────────────────────────────────
|
||
// Refreshes street_price_usd / price_verified_eur on the transceivers table from
|
||
// the price_observations hypertable. Without this, denormalized prices go stale
|
||
// even when scrapers are collecting new observations.
|
||
await boss.work("compute:price-denorm", async () => {
|
||
const { pool } = await import("./utils/db");
|
||
const ts = new Date().toISOString();
|
||
console.log(`[${ts}] Running: Price denormalization refresh`);
|
||
|
||
const result = await pool.query(`
|
||
UPDATE transceivers t
|
||
SET price_verified_eur = sub.price_eur,
|
||
street_price_usd = sub.price_usd,
|
||
updated_at = NOW()
|
||
FROM (
|
||
SELECT po.transceiver_id,
|
||
MAX(po.price) FILTER (WHERE po.currency = 'EUR') AS price_eur,
|
||
MAX(po.price) FILTER (WHERE po.currency = 'USD') AS price_usd
|
||
FROM price_observations po
|
||
WHERE po.time > NOW() - INTERVAL '180 days'
|
||
AND po.price > 0
|
||
GROUP BY po.transceiver_id
|
||
) sub
|
||
WHERE t.id = sub.transceiver_id
|
||
AND (sub.price_eur IS NOT NULL OR sub.price_usd IS NOT NULL)
|
||
`);
|
||
|
||
console.log(`[price-denorm] refreshed ${result.rowCount} transceivers`);
|
||
});
|
||
|
||
// ── Health monitor ──────────────────────────────────────────────────────
|
||
await boss.work("monitor:scraper-health", async () => {
|
||
const { pool } = await import("./utils/db");
|
||
|
||
// Vendors we expect to see prices from regularly.
|
||
// Mapped: display name → pg-boss job name prefix (for last-run lookup).
|
||
const EXPECTED_VENDORS: Array<{ name: string; jobName: string }> = [
|
||
{ name: "FiberMall", jobName: "scrape:pricing:fibermall" },
|
||
{ name: "QSFPTEK", jobName: "scrape:pricing:qsfptek" },
|
||
{ name: "Flexoptix", jobName: "scrape:pricing:flexoptix" },
|
||
// FS.COM is skipped on Erik (SKIP_FS_SCRAPER=true) — Mac launchd handles it.
|
||
// Exclude from monitor so stale last_seen doesn't trigger false positives.
|
||
{ name: "10Gtek", jobName: "scrape:pricing:10gtek" },
|
||
{ name: "ATGBICS", jobName: "scrape:pricing:atgbics" },
|
||
{ name: "GBICS", jobName: "scrape:pricing:gbics" },
|
||
{ name: "SFPcables", jobName: "scrape:pricing:sfpcables" },
|
||
{ name: "NADDOD", jobName: "scrape:pricing:naddod" },
|
||
];
|
||
|
||
const vendorNames = EXPECTED_VENDORS.map((v) => v.name);
|
||
const jobNames = EXPECTED_VENDORS.map((v) => v.jobName);
|
||
|
||
// Price observation recency per vendor
|
||
const priceResult = await pool.query(`
|
||
SELECT v.name,
|
||
SUM(CASE WHEN po.time > NOW() - INTERVAL '6 hours' THEN 1 ELSE 0 END) AS prices_6h,
|
||
MAX(po.time) AS last_seen,
|
||
EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 3600.0 AS hours_since
|
||
FROM vendors v
|
||
LEFT JOIN price_observations po ON po.source_vendor_id = v.id
|
||
WHERE v.name = ANY($1)
|
||
GROUP BY v.name
|
||
ORDER BY last_seen ASC NULLS FIRST
|
||
`, [vendorNames]);
|
||
|
||
// Last successful pg-boss job per vendor scraper (within last 26h — covers any
|
||
// 2h-scheduled job at least once plus 2h slack for daemon restart delays).
|
||
// Also pull the most recent failed job so we can distinguish "running but
|
||
// prices stable" from "job consistently failing".
|
||
const jobResult = await pool.query(`
|
||
SELECT DISTINCT ON (name) name, state, completed_on
|
||
FROM pgboss.job
|
||
WHERE name = ANY($1)
|
||
AND created_on > NOW() - INTERVAL '26 hours'
|
||
ORDER BY name, created_on DESC
|
||
`, [jobNames]);
|
||
|
||
const jobMap = new Map<string, { state: string; completed_on: Date | null }>();
|
||
for (const row of jobResult.rows) {
|
||
jobMap.set(row.name as string, { state: row.state as string, completed_on: row.completed_on as Date | null });
|
||
}
|
||
|
||
// Last COMPLETED job per vendor (to know when the job last ran successfully)
|
||
const completedResult = await pool.query(`
|
||
SELECT DISTINCT ON (name) name, completed_on AS last_completed
|
||
FROM pgboss.job
|
||
WHERE name = ANY($1)
|
||
AND state = 'completed'
|
||
AND created_on > NOW() - INTERVAL '26 hours'
|
||
ORDER BY name, completed_on DESC
|
||
`, [jobNames]);
|
||
|
||
const completedMap = new Map<string, Date>();
|
||
for (const row of completedResult.rows) {
|
||
if (row.last_completed) completedMap.set(row.name as string, new Date(row.last_completed as string));
|
||
}
|
||
|
||
// Thresholds for alerting:
|
||
// CRITICAL (🔴): no job completed in 26h AND last price > 168h (7 days)
|
||
// — scraper genuinely not running or consistently failing
|
||
// WARNING (🟡): no job completed in 26h AND last price > 48h
|
||
// — scraper may be broken but recently seen
|
||
// STABLE (✅): job completed in last 26h AND vendor has historical prices
|
||
// — prices unchanged (hash dedup), scraper is healthy
|
||
const CRITICAL_HOURS = 168;
|
||
const WARN_HOURS = 48;
|
||
|
||
const critical: string[] = [];
|
||
const warnings: string[] = [];
|
||
const stable: string[] = [];
|
||
|
||
for (const row of priceResult.rows) {
|
||
const h = parseFloat(row.hours_since ?? "9999");
|
||
const n = parseInt(row.prices_6h ?? "0", 10);
|
||
if (n > 0) continue; // new prices written → definitely healthy
|
||
|
||
const lastStr = row.last_seen
|
||
? `last price ${h.toFixed(1)}h ago (${new Date(row.last_seen as string).toISOString().slice(0, 16)})`
|
||
: "NEVER scraped";
|
||
|
||
const vendor = EXPECTED_VENDORS.find((v) => v.name === row.name);
|
||
const jobInfo = vendor ? jobMap.get(vendor.jobName) : undefined;
|
||
const lastCompleted = vendor ? completedMap.get(vendor.jobName) : undefined;
|
||
|
||
const jobStr = jobInfo
|
||
? ` | job=${jobInfo.state} at ${jobInfo.completed_on ? new Date(jobInfo.completed_on).toISOString().slice(11, 16) : "?"}`
|
||
: " | job=not run in 26h";
|
||
|
||
// If the job completed successfully in the last 26h AND the vendor has
|
||
// historical prices, prices are just stable (hash dedup) — not an outage.
|
||
const jobRunningOk = !!lastCompleted && row.last_seen;
|
||
if (jobRunningOk) {
|
||
stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged, job OK)${jobStr}`);
|
||
continue;
|
||
}
|
||
|
||
if (!row.last_seen || h > CRITICAL_HOURS) {
|
||
critical.push(`🔴 ${row.name}: ${lastStr}${jobStr}`);
|
||
} else if (h > WARN_HOURS) {
|
||
warnings.push(`🟡 ${row.name}: ${lastStr}${jobStr}`);
|
||
} else {
|
||
stable.push(`✅ ${row.name}: prices stable (${h.toFixed(1)}h unchanged)${jobStr}`);
|
||
}
|
||
}
|
||
|
||
if (critical.length > 0 || warnings.length > 0) {
|
||
if (critical.length > 0) {
|
||
console.error("=== 🔴 SCRAPER CRITICAL — vendors with no prices for 7+ days ===");
|
||
for (const p of critical) console.error(p);
|
||
}
|
||
if (warnings.length > 0) {
|
||
console.warn("=== 🟡 SCRAPER WARNING — vendors with stale prices (48h+) ===");
|
||
for (const p of warnings) console.warn(p);
|
||
}
|
||
console.error("=== Check: pm2 logs tip-scraper-daemon ===");
|
||
} else {
|
||
const activeCount = EXPECTED_VENDORS.length - stable.length;
|
||
if (stable.length > 0) {
|
||
console.log(`[monitor] Scraper health OK — ${activeCount} vendors active, ${stable.length} stable (no price changes)`);
|
||
for (const s of stable) console.log(` ${s}`);
|
||
} else {
|
||
console.log(`[monitor] Scraper health OK — all ${EXPECTED_VENDORS.length} vendors active in last 6h`);
|
||
}
|
||
}
|
||
});
|
||
|
||
// ── Verification reconciliation ─────────────────────────────────────────
|
||
await boss.work("maintenance:reconcile-verification", async () => {
|
||
const { pool } = await import("./utils/db");
|
||
|
||
// 1. Backfill product_page_url from the latest real price URL. Many
|
||
// fetch scrapers write URL only to price_observations; details verification
|
||
// needs the canonical product source on transceivers as well.
|
||
const backfillProductUrls = await pool.query(`
|
||
UPDATE transceivers t
|
||
SET product_page_url = latest.url,
|
||
updated_at = NOW()
|
||
FROM (
|
||
SELECT DISTINCT ON (po.transceiver_id)
|
||
po.transceiver_id, po.url
|
||
FROM price_observations po
|
||
WHERE po.url IS NOT NULL
|
||
AND po.url != ''
|
||
AND po.time > NOW() - INTERVAL '180 days'
|
||
ORDER BY po.transceiver_id, po.time DESC
|
||
) latest
|
||
WHERE t.id = latest.transceiver_id
|
||
AND (t.product_page_url IS NULL OR t.product_page_url = '')
|
||
`);
|
||
|
||
// 2. Promote stored product images to image_verified. Scrapers already
|
||
// filter placeholders before writing image_url; this keeps older crawls from
|
||
// being invisible to the dashboard badge math.
|
||
const verifyImages = await pool.query(`
|
||
UPDATE transceivers
|
||
SET has_image = true,
|
||
image_verified = true,
|
||
image_verified_at = COALESCE(image_verified_at, NOW()),
|
||
image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url),
|
||
updated_at = NOW()
|
||
WHERE image_url IS NOT NULL
|
||
AND image_url != ''
|
||
AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)'
|
||
AND (image_verified = false OR image_verified IS NULL)
|
||
`);
|
||
|
||
// 3. Promote crawled products with source URL + core spec fields to
|
||
// details_verified. This is intentionally stricter than "has any row":
|
||
// product identity, reach and media type all need to be present.
|
||
const verifyDetails = await pool.query(`
|
||
UPDATE transceivers
|
||
SET details_verified = true,
|
||
details_verified_at = COALESCE(details_verified_at, NOW()),
|
||
details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url),
|
||
data_confidence = CASE
|
||
WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated')
|
||
THEN 'scraped_unverified'
|
||
ELSE data_confidence
|
||
END,
|
||
updated_at = NOW()
|
||
WHERE product_page_url IS NOT NULL
|
||
AND product_page_url != ''
|
||
AND form_factor IS NOT NULL
|
||
AND speed_gbps IS NOT NULL
|
||
AND part_number IS NOT NULL
|
||
AND part_number != ''
|
||
AND reach_label IS NOT NULL
|
||
AND reach_label != ''
|
||
AND fiber_type IS NOT NULL
|
||
AND fiber_type != ''
|
||
AND COALESCE(data_confidence, 'unknown') != 'garbage'
|
||
AND (details_verified = false OR details_verified IS NULL)
|
||
`);
|
||
|
||
// 4. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days
|
||
const resetComp = await pool.query(`
|
||
UPDATE transceivers t
|
||
SET competitor_verified = false,
|
||
competitor_verified_at = NULL
|
||
WHERE competitor_verified = true
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM price_observations po
|
||
JOIN vendors v ON po.source_vendor_id = v.id
|
||
WHERE po.transceiver_id = t.id
|
||
AND po.time > NOW() - INTERVAL '30 days'
|
||
AND UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
|
||
)
|
||
`);
|
||
|
||
// 5. Reset fully_verified=false for products that lost a required signal
|
||
const resetFull = await pool.query(`
|
||
UPDATE transceivers
|
||
SET fully_verified = false,
|
||
fully_verified_at = NULL
|
||
WHERE fully_verified = true
|
||
AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false)
|
||
`);
|
||
|
||
// 6. Set fully_verified=true for products that now meet all 4 criteria
|
||
const setFull = await pool.query(`
|
||
UPDATE transceivers
|
||
SET fully_verified = true,
|
||
fully_verified_at = COALESCE(fully_verified_at, NOW())
|
||
WHERE competitor_verified = true
|
||
AND price_verified = true
|
||
AND image_verified = true
|
||
AND details_verified = true
|
||
AND fully_verified = false
|
||
`);
|
||
|
||
console.log(
|
||
`[reconcile] product URLs backfilled: ${backfillProductUrls.rowCount}, ` +
|
||
`images verified: ${verifyImages.rowCount}, ` +
|
||
`details verified: ${verifyDetails.rowCount}, ` +
|
||
`competitor_verified reset: ${resetComp.rowCount}, ` +
|
||
`fully_verified cleared: ${resetFull.rowCount}, ` +
|
||
`fully_verified earned: ${setFull.rowCount}`
|
||
);
|
||
});
|
||
|
||
// ── Equivalence matching ────────────────────────────────────────────────
|
||
// Matches Flexoptix SKUs to technically equivalent competitor products by specs.
|
||
// Confidence scoring: standard_name(35) + form_factor(25) + speed_gbps(20) +
|
||
// fiber_type(10) + reach±25%(10) = 100 pts max
|
||
// ≥0.85 → auto_approved + competitor_verified=true
|
||
// 0.50–0.84 → pending (Manual Review queue in dashboard)
|
||
// <0.50 → skipped
|
||
await boss.work("maintenance:find-equivalences", async () => {
|
||
const { pool } = await import("./utils/db");
|
||
const ts = new Date().toISOString();
|
||
console.log(`[${ts}] Running: Equivalence matching`);
|
||
|
||
// Find all Flexoptix transceivers that are NOT yet competitor_verified
|
||
const flexResult = await pool.query(`
|
||
SELECT t.id, t.part_number, t.standard_name, t.form_factor,
|
||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
|
||
t.connector, t.wdm_type, t.coherent
|
||
FROM transceivers t
|
||
JOIN vendors v ON v.id = t.vendor_id
|
||
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
|
||
AND t.competitor_verified = false
|
||
`);
|
||
|
||
let autoApproved = 0;
|
||
let queued = 0;
|
||
let skipped = 0;
|
||
|
||
for (const fx of flexResult.rows) {
|
||
// Find competitor transceivers with recent price observations and matching specs
|
||
const candidates = await pool.query(`
|
||
SELECT t.id AS competitor_id, t.part_number, t.standard_name,
|
||
t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters,
|
||
t.wavelengths, t.connector, v.name AS vendor_name,
|
||
MAX(po.time) AS last_price, COUNT(*) AS price_count
|
||
FROM transceivers t
|
||
JOIN vendors v ON v.id = t.vendor_id
|
||
JOIN price_observations po ON po.transceiver_id = t.id
|
||
WHERE UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
|
||
AND po.time > NOW() - INTERVAL '30 days'
|
||
AND t.form_factor = $1
|
||
AND t.speed_gbps = $2
|
||
AND t.id != $3
|
||
GROUP BY t.id, t.part_number, t.standard_name, t.form_factor,
|
||
t.speed_gbps, t.fiber_type, t.reach_meters,
|
||
t.wavelengths, t.connector, v.name
|
||
`, [fx.form_factor, fx.speed_gbps, fx.id]);
|
||
|
||
for (const cand of candidates.rows) {
|
||
// Confidence scoring
|
||
// Max points: form_factor(25) + speed_gbps(20) + standard_name(30) +
|
||
// wavelength_nm(20) + fiber_type(10) + reach(10) = 115
|
||
let score = 0;
|
||
const basis: string[] = [];
|
||
|
||
// form_factor already matched (pre-filter), award points
|
||
score += 25; basis.push("form_factor");
|
||
|
||
// speed_gbps already matched (pre-filter)
|
||
score += 20; basis.push("speed_gbps");
|
||
|
||
// standard_name match (strong signal — e.g. "10GBASE-LR")
|
||
if (fx.standard_name && cand.standard_name &&
|
||
fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) {
|
||
score += 30; basis.push("standard_name");
|
||
}
|
||
|
||
// wavelength match — extract first numeric nm value and compare within ±15nm
|
||
// "wavelengths" is text: "1310 nm", "850nm", "1270/1290/1310/1330 nm" etc.
|
||
const extractNm = (w: string | null): number | null => {
|
||
if (!w) return null;
|
||
const m = w.match(/(\d{3,4})/);
|
||
return m ? parseInt(m[1], 10) : null;
|
||
};
|
||
const fxNm = extractNm(fx.wavelengths);
|
||
const candNm = extractNm(cand.wavelengths);
|
||
if (fxNm !== null && candNm !== null) {
|
||
if (Math.abs(fxNm - candNm) <= 15) {
|
||
score += 20; basis.push(`wavelength_${fxNm}nm`);
|
||
} else {
|
||
score -= 20; // hard penalize wrong wavelength (1310 vs 1550 = completely different product)
|
||
}
|
||
}
|
||
|
||
// fiber_type match (SMF vs MMF — critical)
|
||
if (fx.fiber_type && cand.fiber_type) {
|
||
if (fx.fiber_type.trim().toUpperCase() === cand.fiber_type.trim().toUpperCase()) {
|
||
score += 10; basis.push("fiber_type");
|
||
} else {
|
||
score -= 15; // SMF vs MMF = wrong product
|
||
}
|
||
}
|
||
|
||
// reach within ±25%
|
||
if (fx.reach_meters && cand.reach_meters && fx.reach_meters > 0 && cand.reach_meters > 0) {
|
||
const diff = Math.abs(fx.reach_meters - cand.reach_meters);
|
||
const tolerance = Math.max(fx.reach_meters, 1) * 0.25;
|
||
if (diff <= tolerance) {
|
||
score += 10; basis.push("reach");
|
||
} else {
|
||
score -= 15; // penalize mismatched reach
|
||
}
|
||
} else if (!fx.reach_meters && !cand.reach_meters) {
|
||
score += 5; basis.push("reach_null");
|
||
}
|
||
|
||
const confidence = Math.max(0, Math.min(1, score / 115));
|
||
|
||
if (confidence < 0.50) { skipped++; continue; }
|
||
|
||
const notes = `${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` +
|
||
`basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` +
|
||
`wavelength: ${fx.wavelengths||"?"} vs ${cand.wavelengths||"?"}`;
|
||
|
||
// Upsert equivalence candidate
|
||
const status = confidence >= 0.73 ? "auto_approved" : "pending";
|
||
|
||
await pool.query(`
|
||
INSERT INTO transceiver_equivalences
|
||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status)
|
||
VALUES ($1, $2, $3, $4, $5, $6)
|
||
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
|
||
confidence = EXCLUDED.confidence,
|
||
match_basis = EXCLUDED.match_basis,
|
||
match_notes = EXCLUDED.match_notes,
|
||
updated_at = NOW()
|
||
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected')
|
||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]);
|
||
|
||
if (confidence >= 0.73) {
|
||
// Auto-approve: set competitor_verified on the Flexoptix transceiver
|
||
await pool.query(`
|
||
UPDATE transceivers
|
||
SET competitor_verified = true,
|
||
competitor_verified_at = NOW()
|
||
WHERE id = $1 AND competitor_verified = false
|
||
`, [fx.id]);
|
||
autoApproved++;
|
||
} else {
|
||
queued++;
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log(
|
||
`[find-equivalences] auto_approved: ${autoApproved}, ` +
|
||
`queued for review: ${queued}, skipped (low confidence): ${skipped}`
|
||
);
|
||
|
||
// After auto-approvals, rerun fully_verified check
|
||
if (autoApproved > 0) {
|
||
await pool.query(`
|
||
UPDATE transceivers
|
||
SET fully_verified = true,
|
||
fully_verified_at = COALESCE(fully_verified_at, NOW())
|
||
WHERE competitor_verified = true
|
||
AND price_verified = true
|
||
AND image_verified = true
|
||
AND details_verified = true
|
||
AND fully_verified = false
|
||
`);
|
||
}
|
||
});
|
||
|
||
// ── Re-research equivalences ────────────────────────────────────────────────
|
||
// Confirms only well-evidenced matches. Weak, stale, incomplete, or technically
|
||
// contradictory matches are rejected automatically instead of going back to a
|
||
// manual queue.
|
||
await boss.work("maintenance:re-research-equivalences", async () => {
|
||
const { pool } = await import("./utils/db");
|
||
const ts = new Date().toISOString();
|
||
const batchLimit = Math.max(1, Math.min(10000, parseInt(process.env["EQUIVALENCE_RESEARCH_BATCH_LIMIT"] || "2000", 10)));
|
||
console.log(`[${ts}] Running: Equivalence automated research`);
|
||
|
||
const batch = await pool.query(`
|
||
SELECT eq.id, eq.flexoptix_id, eq.competitor_id, eq.status, eq.confidence,
|
||
fx.part_number AS fx_part_number,
|
||
fx.form_factor AS fx_form_factor,
|
||
fx.speed_gbps AS fx_speed_gbps,
|
||
fx.standard_name AS fx_standard_name,
|
||
fx.fiber_type AS fx_fiber_type,
|
||
fx.reach_meters AS fx_reach_meters,
|
||
fx.wavelengths AS fx_wavelengths,
|
||
fx.connector AS fx_connector,
|
||
cp.part_number AS cp_part_number,
|
||
cp.form_factor AS cp_form_factor,
|
||
cp.speed_gbps AS cp_speed_gbps,
|
||
cp.standard_name AS cp_standard_name,
|
||
cp.fiber_type AS cp_fiber_type,
|
||
cp.reach_meters AS cp_reach_meters,
|
||
cp.wavelengths AS cp_wavelengths,
|
||
cp.connector AS cp_connector,
|
||
cpv.name AS competitor_vendor,
|
||
(
|
||
SELECT COUNT(*)
|
||
FROM price_observations po
|
||
WHERE po.transceiver_id = eq.competitor_id
|
||
AND po.time > NOW() - INTERVAL '45 days'
|
||
) AS recent_price_count
|
||
FROM transceiver_equivalences eq
|
||
JOIN transceivers fx ON eq.flexoptix_id = fx.id
|
||
JOIN transceivers cp ON eq.competitor_id = cp.id
|
||
JOIN vendors cpv ON cpv.id = cp.vendor_id
|
||
WHERE eq.status IN ('pending', 'approved', 'auto_approved')
|
||
AND eq.re_research_due_at IS NOT NULL
|
||
AND eq.re_research_due_at <= NOW()
|
||
ORDER BY eq.re_research_due_at ASC
|
||
LIMIT $1
|
||
`, [batchLimit]);
|
||
|
||
let confirmed = 0;
|
||
let rejected = 0;
|
||
|
||
for (const eq of batch.rows) {
|
||
const research = evaluateEquivalenceResearch(
|
||
{
|
||
part_number: eq.fx_part_number,
|
||
form_factor: eq.fx_form_factor,
|
||
speed_gbps: eq.fx_speed_gbps,
|
||
standard_name: eq.fx_standard_name,
|
||
fiber_type: eq.fx_fiber_type,
|
||
reach_meters: eq.fx_reach_meters,
|
||
wavelengths: eq.fx_wavelengths,
|
||
connector: eq.fx_connector,
|
||
},
|
||
{
|
||
part_number: eq.cp_part_number,
|
||
form_factor: eq.cp_form_factor,
|
||
speed_gbps: eq.cp_speed_gbps,
|
||
standard_name: eq.cp_standard_name,
|
||
fiber_type: eq.cp_fiber_type,
|
||
reach_meters: eq.cp_reach_meters,
|
||
wavelengths: eq.cp_wavelengths,
|
||
connector: eq.cp_connector,
|
||
},
|
||
parseInt(eq.recent_price_count, 10) > 0,
|
||
);
|
||
|
||
if (research.decision === "reject") {
|
||
await pool.query(`
|
||
UPDATE transceiver_equivalences
|
||
SET status = 'rejected',
|
||
confidence = $2,
|
||
match_basis = $3,
|
||
reject_reason = $4,
|
||
reviewed_by = 'automated-research',
|
||
reviewed_at = NOW(),
|
||
re_research_due_at = NULL,
|
||
re_researched_at = NOW(),
|
||
match_notes = CONCAT(
|
||
COALESCE(match_notes, ''),
|
||
E'\n[Automated research ' || NOW()::date || ': rejected; ' || $5 || ']'
|
||
),
|
||
updated_at = NOW()
|
||
WHERE id = $1
|
||
`, [
|
||
eq.id,
|
||
research.confidence,
|
||
research.basis,
|
||
research.rejectReason || "automated research: rejected",
|
||
research.reasons.join("; "),
|
||
]);
|
||
|
||
await pool.query(`
|
||
UPDATE transceivers
|
||
SET competitor_verified = false, competitor_verified_at = NULL,
|
||
fully_verified = false, fully_verified_at = NULL
|
||
WHERE id = $1
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM transceiver_equivalences
|
||
WHERE flexoptix_id = $1
|
||
AND status IN ('approved', 'auto_approved')
|
||
AND id != $2
|
||
)
|
||
`, [eq.flexoptix_id, eq.id]);
|
||
|
||
rejected++;
|
||
} else {
|
||
await pool.query(`
|
||
UPDATE transceiver_equivalences
|
||
SET status = CASE WHEN status = 'pending' THEN 'auto_approved' ELSE status END,
|
||
confidence = $2,
|
||
match_basis = $3,
|
||
reviewed_by = COALESCE(reviewed_by, 'automated-research'),
|
||
reviewed_at = COALESCE(reviewed_at, NOW()),
|
||
reject_reason = NULL,
|
||
re_researched_at = NOW(),
|
||
re_research_due_at = NOW() + INTERVAL '30 days',
|
||
match_notes = CONCAT(
|
||
COALESCE(match_notes, ''),
|
||
E'\n[Automated research ' || NOW()::date || ': confirmed; basis: ' || $4 || ']'
|
||
),
|
||
updated_at = NOW()
|
||
WHERE id = $1
|
||
`, [eq.id, research.confidence, research.basis, research.basis.join(", ")]);
|
||
|
||
await pool.query(`
|
||
UPDATE transceivers
|
||
SET competitor_verified = true,
|
||
competitor_verified_at = COALESCE(competitor_verified_at, NOW())
|
||
WHERE id = $1 AND competitor_verified = false
|
||
`, [eq.flexoptix_id]);
|
||
|
||
confirmed++;
|
||
}
|
||
}
|
||
|
||
if (confirmed > 0) {
|
||
await pool.query(`
|
||
UPDATE transceivers
|
||
SET fully_verified = true,
|
||
fully_verified_at = COALESCE(fully_verified_at, NOW())
|
||
WHERE competitor_verified = true
|
||
AND price_verified = true
|
||
AND image_verified = true
|
||
AND details_verified = true
|
||
AND fully_verified = false
|
||
`);
|
||
}
|
||
|
||
console.log(`[equivalence-research] confirmed: ${confirmed}, rejected: ${rejected}, batch size: ${batch.rows.length}`);
|
||
});
|
||
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
// VENDOR DISCOVERY CRAWLER WORKERS
|
||
// Each worker calls discoverVendorCatalog() for the matching slug.
|
||
// Results go to: TIP DB (findOrCreateScrapedTransceiver) +
|
||
// Gitea tip-training-data repo (SFT JSONL pairs)
|
||
// ══════════════════════════════════════════════════════════════════════
|
||
|
||
const { discoverVendorCatalog, VENDOR_CATALOG_REGISTRY } = await import("./crawler-llm/vendor-discovery-crawler");
|
||
|
||
for (const vendorConfig of VENDOR_CATALOG_REGISTRY) {
|
||
const jobName = `discover:vendor:${vendorConfig.slug}`;
|
||
boss.work(jobName, async () => {
|
||
if (!isLoadAcceptable(3.0)) {
|
||
console.warn(`[${jobName}] Load too high — skipping deep crawl`);
|
||
return;
|
||
}
|
||
console.log(`[${jobName}] Starting vendor discovery crawl…`);
|
||
try {
|
||
await discoverVendorCatalog(vendorConfig, { verbose: false });
|
||
} catch (err) {
|
||
console.error(`[${jobName}] Fatal:`, (err as Error).message);
|
||
throw err; // let pg-boss retry
|
||
}
|
||
});
|
||
}
|
||
|
||
console.log("All workers registered (102 jobs, 24/7 continuous + 8 weekly discovery crawlers)");
|
||
}
|