0edc6e3f3a
- index-pi.ts: removed Playwright scrapers (FS.COM, eBay enricher, switch assets) added NADDOD (fetch-based, benefits from residential IP) now 32 fetch-only queues safe for ARM/Pi without Chromium - index-fs-only.ts: new dedicated FS.COM + NADDOD worker for Erik routes through Pi SOCKS5 via PROXY_URLS=socks5://10.10.0.6:1080 Crawlee ProxyConfiguration automatically applies to Playwright crawler - pi-scraper-setup.sh: removed inline index-pi.ts override (repo version now authoritative) - CODEX-TASK-pi-scraper-deploy.md: full 9-step Codex spec for Pi fleet setup covers WireGuard keypair, Erik peer config, setup script, ecosystem.config.js - CODEX-TASK-zero-manual-review.md: deterministic equivalence matcher spec
223 lines
14 KiB
TypeScript
223 lines
14 KiB
TypeScript
/**
|
|
* TIP Pi Scraper Node — index-pi.ts
|
|
*
|
|
* FETCH-ONLY worker for Raspberry Pi fleet (Pi1/Pi2/Pi3).
|
|
* NO Playwright — pure fetch/cheerio scrapers that run on ARM without Chromium.
|
|
* Connects to production DB via WireGuard VPN (10.10.0.1 = Erik).
|
|
*
|
|
* Playwright-heavy scrapers (FS.COM, switch-assets, eBay enricher) stay on ERIK
|
|
* and route through the Pi's SOCKS5 proxy for IP diversity.
|
|
*
|
|
* Architecture:
|
|
* Pi node 24/7: fetch-only scrapers → pg-boss queues on Erik's PostgreSQL
|
|
* Pi SOCKS5: dante-server on WireGuard IP, Erik's FS.COM routes through it
|
|
* Erik only: Playwright scrapers, API, MCP, Qdrant, heavy compute
|
|
*/
|
|
import { config } from "dotenv";
|
|
import { join } from "path";
|
|
config({ path: join(__dirname, "..", "..", "..", ".env") });
|
|
|
|
import PgBoss from "pg-boss";
|
|
import { mkdirSync, rmSync } from "fs";
|
|
|
|
const connectionString = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT || "5432"}/${process.env.POSTGRES_DB}`;
|
|
const PI_NAME = process.env.PI_NAME || "pi";
|
|
|
|
async function withIsolatedStorage(name: string, fn: () => Promise<void>): Promise<void> {
|
|
const dir = `/tmp/tip-crawlee-${name}-${Date.now()}`;
|
|
mkdirSync(join(dir, "request_queues", "default"), { recursive: true });
|
|
mkdirSync(join(dir, "datasets", "default"), { recursive: true });
|
|
mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true });
|
|
const prev = process.env.CRAWLEE_STORAGE_DIR;
|
|
process.env.CRAWLEE_STORAGE_DIR = dir;
|
|
try { await fn(); }
|
|
finally {
|
|
process.env.CRAWLEE_STORAGE_DIR = prev ?? "";
|
|
try { rmSync(dir, { recursive: true, force: true }); } catch {}
|
|
}
|
|
}
|
|
|
|
// ── FETCH-ONLY scrapers — safe on ARM/Pi without Playwright ──────────────────
|
|
// FS.COM intentionally EXCLUDED: uses Playwright, runs on Erik via Pi SOCKS5 proxy
|
|
// ATGBICS intentionally EXCLUDED: uses Playwright detail pages
|
|
// eBay enricher intentionally EXCLUDED: Playwright
|
|
// switch-assets intentionally EXCLUDED: Playwright variants
|
|
const QUEUES = [
|
|
// Pricing scrapers — all fetch/cheerio based
|
|
"scrape:pricing:naddod", // fetch + LD+JSON — residential IP beneficial
|
|
"scrape:pricing:10gtek",
|
|
"scrape:pricing:prolabs",
|
|
"scrape:pricing:optcore",
|
|
"scrape:pricing:fluxlight",
|
|
"scrape:pricing:gbics",
|
|
"scrape:pricing:champion-one",
|
|
"scrape:pricing:sfpcables",
|
|
"scrape:pricing:blueoptics",
|
|
"scrape:pricing:fiber24",
|
|
"scrape:pricing:tscom",
|
|
"scrape:pricing:skylane",
|
|
"scrape:pricing:ascentoptics",
|
|
"scrape:pricing:gaotek",
|
|
"scrape:pricing:fibermall",
|
|
"scrape:pricing:vcelink",
|
|
"scrape:pricing:opticsbay",
|
|
// Form-factor coverage scrapers
|
|
"scrape:pricing:comms-express",
|
|
"scrape:pricing:router-switch",
|
|
"scrape:pricing:multimode-inc",
|
|
"scrape:pricing:optictransceiver",
|
|
"scrape:pricing:wiitek",
|
|
// Catalog scrapers
|
|
"scrape:pricing:flexoptix",
|
|
"scrape:catalog:smartoptics",
|
|
"scrape:catalog:hubersuhner",
|
|
// Vendor scrapers
|
|
"scrape:vendors:flexoptix",
|
|
"scrape:vendors:flexoptix-supported",
|
|
// Compatibility scrapers (static HCL pages, fetch-based)
|
|
"scrape:compat:sonic",
|
|
"scrape:compat:ufispace",
|
|
"scrape:compat:edgecore",
|
|
// Intelligence
|
|
"scrape:news",
|
|
"scrape:market-intel",
|
|
"scrape:nog-talks",
|
|
"scrape:community-issues",
|
|
"scrape:datasheet-links",
|
|
// Prediction signals (API-based, no browser)
|
|
"scrape:signals:sec-edgar",
|
|
"scrape:signals:github",
|
|
"scrape:signals:ai-clusters",
|
|
"scrape:signals:distributor-leads",
|
|
"scrape:signals:standards",
|
|
];
|
|
|
|
async function main() {
|
|
console.log(`\n=== TIP Pi Scraper Node [${PI_NAME}] ===`);
|
|
console.log(`DB: ${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT || 5432}`);
|
|
console.log(`Queues: ${QUEUES.length} workers\n`);
|
|
|
|
const boss = new PgBoss({
|
|
connectionString,
|
|
retryLimit: 2,
|
|
retryDelay: 60,
|
|
expireInSeconds: 3600,
|
|
monitorStateIntervalSeconds: 60,
|
|
});
|
|
|
|
boss.on("error", (e: Error) => console.error("pg-boss error:", e.message));
|
|
await boss.start();
|
|
|
|
for (const q of QUEUES) {
|
|
await boss.createQueue(q).catch(() => {});
|
|
}
|
|
|
|
const log = (q: string) => console.log(`[${new Date().toISOString()}] [${PI_NAME}] ${q}`);
|
|
|
|
// ── Pricing scrapers (fetch/cheerio — NO Playwright) ─────────────────
|
|
const { scrapeNaddod } = await import("./scrapers/naddod");
|
|
const { scrape10Gtek } = await import("./scrapers/tenGtek");
|
|
const { scrapeProLabs } = await import("./scrapers/prolabs");
|
|
const { scrapeOptcore } = await import("./scrapers/optcore");
|
|
const { scrapeFluxlight } = await import("./scrapers/fluxlight");
|
|
const { scrapeGbics } = await import("./scrapers/gbics");
|
|
const { scrapeChampionOne } = await import("./scrapers/champion-one");
|
|
const { scrapeSfpCables } = await import("./scrapers/sfpcables");
|
|
const { scrapeBlueOptics } = await import("./scrapers/blueoptics");
|
|
const { scrapeFiber24 } = await import("./scrapers/fiber24");
|
|
const { scrapeTsCom } = await import("./scrapers/tscom");
|
|
const { scrapeSkylane } = await import("./scrapers/skylane");
|
|
const { scrapeAscentOptics } = await import("./scrapers/ascentoptics");
|
|
const { scrapeGaoTek } = await import("./scrapers/gaotek");
|
|
const { scrapeFiberMall } = await import("./scrapers/fibermall");
|
|
const { scrapeVcelink } = await import("./scrapers/vcelink");
|
|
const { scrapeOpticsBay } = await import("./scrapers/opticsbay");
|
|
|
|
// ── Form-factor coverage scrapers ─────────────────────────────────────
|
|
const { scrapeCommsExpress } = await import("./scrapers/comms-express");
|
|
const { scrapeRouterSwitch } = await import("./scrapers/router-switch");
|
|
const { scrapeMultimodeInc } = await import("./scrapers/multimode-inc");
|
|
const { scrapeOpticTransceiver } = await import("./scrapers/optictransceiver");
|
|
const { scrapeWiitek } = await import("./scrapers/wiitek");
|
|
|
|
// ── Catalog scrapers ──────────────────────────────────────────────────
|
|
const { scrapeFlexoptixCatalog } = await import("./scrapers/flexoptix-catalog");
|
|
const { scrapeSmartOptics } = await import("./scrapers/smartoptics");
|
|
const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner");
|
|
|
|
// ── Vendor scrapers ───────────────────────────────────────────────────
|
|
const { scrapeFlexoptixVendors } = await import("./scrapers/flexoptix-vendors");
|
|
const { seedFlexoptixVendors } = await import("./scrapers/flexoptix-supported-vendors");
|
|
|
|
// ── Compatibility scrapers (static HTML, fetch-based) ────────────────
|
|
const { scrapeSonicHcl } = await import("./scrapers/sonic-hcl");
|
|
const { scrapeUfiSpace } = await import("./scrapers/ufispace");
|
|
const { scrapeEdgecore } = await import("./scrapers/edgecore");
|
|
|
|
// ── Intelligence scrapers ─────────────────────────────────────────────
|
|
const { scrapeNews } = await import("./scrapers/news");
|
|
const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence");
|
|
const { scrapeAllSwitchIssues, findAndSeedDatasheetLinks } = await import("./scrapers/community-issues");
|
|
|
|
// ── Prediction signals (API-based, no browser) ────────────────────────
|
|
const { scrapeSecEdgar } = await import("./scrapers/sec-edgar");
|
|
const { scrapeGithubSignals } = await import("./scrapers/github-signals");
|
|
const { scrapeAiClusters } = await import("./scrapers/ai-clusters");
|
|
const { scrapeDistributorLeads } = await import("./scrapers/distributor-leads");
|
|
const { scrapeStandardsTracker } = await import("./scrapers/standards-tracker");
|
|
|
|
// ── Register workers ──────────────────────────────────────────────────
|
|
await boss.work("scrape:pricing:naddod", async () => { log("naddod"); await scrapeNaddod(); });
|
|
await boss.work("scrape:pricing:10gtek", async () => { log("10gtek"); await withIsolatedStorage("10gtek", scrape10Gtek); });
|
|
await boss.work("scrape:pricing:prolabs", async () => { log("prolabs"); await withIsolatedStorage("prolabs", scrapeProLabs); });
|
|
await boss.work("scrape:pricing:optcore", async () => { log("optcore"); await withIsolatedStorage("optcore", scrapeOptcore); });
|
|
await boss.work("scrape:pricing:fluxlight", async () => { log("fluxlight"); await withIsolatedStorage("fluxlight", scrapeFluxlight); });
|
|
await boss.work("scrape:pricing:gbics", async () => { log("gbics"); await withIsolatedStorage("gbics", scrapeGbics); });
|
|
await boss.work("scrape:pricing:champion-one", async () => { log("champion-one"); await withIsolatedStorage("champion-one", scrapeChampionOne); });
|
|
await boss.work("scrape:pricing:sfpcables", async () => { log("sfpcables"); await withIsolatedStorage("sfpcables", scrapeSfpCables); });
|
|
await boss.work("scrape:pricing:blueoptics", async () => { log("blueoptics"); await withIsolatedStorage("blueoptics", scrapeBlueOptics); });
|
|
await boss.work("scrape:pricing:fiber24", async () => { log("fiber24"); await withIsolatedStorage("fiber24", scrapeFiber24); });
|
|
await boss.work("scrape:pricing:tscom", async () => { log("tscom"); await withIsolatedStorage("tscom", scrapeTsCom); });
|
|
await boss.work("scrape:pricing:skylane", async () => { log("skylane"); await withIsolatedStorage("skylane", scrapeSkylane); });
|
|
await boss.work("scrape:pricing:ascentoptics", async () => { log("ascentoptics"); await withIsolatedStorage("ascentoptics", scrapeAscentOptics); });
|
|
await boss.work("scrape:pricing:gaotek", async () => { log("gaotek"); await withIsolatedStorage("gaotek", scrapeGaoTek); });
|
|
await boss.work("scrape:pricing:fibermall", async () => { log("fibermall"); await scrapeFiberMall(); });
|
|
await boss.work("scrape:pricing:vcelink", async () => { log("vcelink"); await scrapeVcelink(); });
|
|
await boss.work("scrape:pricing:opticsbay", async () => { log("opticsbay"); await scrapeOpticsBay(); });
|
|
|
|
await boss.work("scrape:pricing:comms-express", async () => { log("comms-express"); await scrapeCommsExpress(); });
|
|
await boss.work("scrape:pricing:router-switch", async () => { log("router-switch"); await scrapeRouterSwitch(); });
|
|
await boss.work("scrape:pricing:multimode-inc", async () => { log("multimode-inc"); await scrapeMultimodeInc(); });
|
|
await boss.work("scrape:pricing:optictransceiver", async () => { log("optictransceiver"); await scrapeOpticTransceiver(); });
|
|
await boss.work("scrape:pricing:wiitek", async () => { log("wiitek"); await scrapeWiitek(); });
|
|
|
|
await boss.work("scrape:pricing:flexoptix", async () => { log("flexoptix-catalog"); await scrapeFlexoptixCatalog(); });
|
|
await boss.work("scrape:catalog:smartoptics", async () => { log("smartoptics"); await withIsolatedStorage("smartoptics", scrapeSmartOptics); });
|
|
await boss.work("scrape:catalog:hubersuhner", async () => { log("hubersuhner"); await withIsolatedStorage("hubersuhner", scrapeHuberSuhner); });
|
|
|
|
await boss.work("scrape:vendors:flexoptix", async () => { log("flexoptix-vendors"); await scrapeFlexoptixVendors(); });
|
|
await boss.work("scrape:vendors:flexoptix-supported", async () => { log("flexoptix-supported"); await seedFlexoptixVendors(); });
|
|
|
|
await boss.work("scrape:compat:sonic", async () => { log("sonic-compat"); await withIsolatedStorage("sonic", scrapeSonicHcl); });
|
|
await boss.work("scrape:compat:ufispace", async () => { log("ufispace"); await withIsolatedStorage("ufispace", scrapeUfiSpace); });
|
|
await boss.work("scrape:compat:edgecore", async () => { log("edgecore"); await withIsolatedStorage("edgecore", scrapeEdgecore); });
|
|
|
|
await boss.work("scrape:news", async () => { log("news"); await scrapeNews(); });
|
|
await boss.work("scrape:market-intel", async () => { log("market-intel"); await withIsolatedStorage("market-intel", scrapeMarketIntelligence); });
|
|
await boss.work("scrape:nog-talks", async () => { log("nog-talks"); const { scrapeNogTalks } = await import("./scrapers/nog-talks"); await scrapeNogTalks(); });
|
|
await boss.work("scrape:community-issues", async () => { log("community"); await withIsolatedStorage("community", () => scrapeAllSwitchIssues(30)); });
|
|
await boss.work("scrape:datasheet-links", async () => { log("datasheets"); await findAndSeedDatasheetLinks(50); });
|
|
|
|
await boss.work("scrape:signals:sec-edgar", async () => { log("sec-edgar"); await scrapeSecEdgar(); });
|
|
await boss.work("scrape:signals:github", async () => { log("github-signals"); await scrapeGithubSignals(); });
|
|
await boss.work("scrape:signals:ai-clusters", async () => { log("ai-clusters"); await scrapeAiClusters(); });
|
|
await boss.work("scrape:signals:distributor-leads", async () => { log("distributor-leads"); await scrapeDistributorLeads(); });
|
|
await boss.work("scrape:signals:standards", async () => { log("standards"); await scrapeStandardsTracker(); });
|
|
|
|
console.log(`${QUEUES.length} queues / workers active — running 24/7\n`);
|
|
process.on("SIGTERM", async () => { await boss.stop(); process.exit(0); });
|
|
process.on("SIGINT", async () => { await boss.stop(); process.exit(0); });
|
|
}
|
|
|
|
main().catch((e) => { console.error("Fatal:", e); process.exit(1); });
|