rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: register fiber24 + fibermall in index, move atgbics to fetch-only section

Browse Source
This commit is contained in:
Rene Fichtmueller 2026-04-18 22:50:52 +02:00
parent 017ed78d2b
commit 5405685d24
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
packages/scraper/src/index.ts
View File

@ -30,6 +30,8 @@
* tsx src/index.ts --naddod — Run NADDOD scraper once * tsx src/index.ts --naddod — Run NADDOD scraper once
* tsx src/index.ts --qsfptek — Run QSFPTEK scraper once * tsx src/index.ts --qsfptek — Run QSFPTEK scraper once
* tsx src/index.ts --addon — Run AddOn Networks scraper once * tsx src/index.ts --addon — Run AddOn Networks scraper once
* tsx src/index.ts --fiber24 — Run ShopFiber24 scraper once (sitemap-based)
* tsx src/index.ts --fibermall — Run FiberMall scraper once
*/ */
import { createScheduler, registerSchedules, registerWorkers } from "./scheduler"; import { createScheduler, registerSchedules, registerWorkers } from "./scheduler";
import { scrapeFs } from "./scrapers/fs-com"; import { scrapeFs } from "./scrapers/fs-com";
@ -60,6 +62,8 @@ import { scrapeProLabs } from "./scrapers/prolabs";
import { scrapeNaddod } from "./scrapers/naddod"; import { scrapeNaddod } from "./scrapers/naddod";
import { scrapeQsfptek } from "./scrapers/qsfptek"; import { scrapeQsfptek } from "./scrapers/qsfptek";
import { scrapeAddonNetworks } from "./scrapers/addon-networks"; import { scrapeAddonNetworks } from "./scrapers/addon-networks";
import { scrapeFiber24 } from "./scrapers/fiber24";
import { scrapeFiberMall } from "./scrapers/fibermall";
import { pool } from "./utils/db"; import { pool } from "./utils/db";
const args = process.argv.slice(2); const args = process.argv.slice(2);
@ -142,6 +146,17 @@ async function runOnce(): Promise<void> {
await scrapeUfiSpace(); await scrapeUfiSpace();
} }
// Fetch-based price scrapers (added 2026-04-18)
if (args.includes("--atgbics") || isAll || isFetchOnly) {
await scrapeAtgbics(); // static HTML, no Playwright (rewritten 2026-04-18)
}
if (args.includes("--fiber24") || isAll || isFetchOnly) {
await scrapeFiber24(); // sitemap-based, microdata parsing
}
if (args.includes("--fibermall") || isAll || isFetchOnly) {
await scrapeFiberMall();
}
// Playwright-based scrapers (need Chromium installed) // Playwright-based scrapers (need Chromium installed)
if (!isFetchOnly) { if (!isFetchOnly) {
if (args.includes("--fs") || isAll) { if (args.includes("--fs") || isAll) {
@ -157,9 +172,6 @@ async function runOnce(): Promise<void> {
const vendor = args.find((a) => a.startsWith("--vendor="))?.split("=")[1]; const vendor = args.find((a) => a.startsWith("--vendor="))?.split("=")[1];
await crawlSwitchAssetsPlaywright(vendor); await crawlSwitchAssetsPlaywright(vendor);
} }
if (args.includes("--atgbics") || isAll) {
await scrapeAtgbics();
}
} }
await pool.end(); await pool.end();
@ -223,7 +235,7 @@ async function runScheduler(): Promise<void> {
process.on("SIGTERM", shutdown); process.on("SIGTERM", shutdown);
} }
const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics"]; const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics", "--fiber24", "--fibermall"];
if (args.some((a) => ALL_FLAGS.includes(a))) { if (args.some((a) => ALL_FLAGS.includes(a))) {
runOnce().catch((err) => { runOnce().catch((err) => {