rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): auto-create pg-boss queues before scheduling + worker/schedule order

Browse Source 
- scheduler: patch boss.schedule() to call createQueue() first (idempotent),
  fixing FK constraint errors after DB reset — no need to touch 277 call sites
- index: registerWorkers() before registerSchedules() since boss.work() must
  register handlers before schedules fire
- dashboard: fix switchBlogLlm() to use api() helper (adds Bearer auth token)
  instead of raw fetch() which was returning 401 Unauthorized
This commit is contained in:
Rene Fichtmueller 2026-04-29 16:14:25 +02:00
parent 270bd12382
commit 199f36be48
3 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/dashboard/index.html
View File

@ -5644,12 +5644,11 @@ async function switchBlogLlm(providerKey, model) {
try {
var body = { provider: providerKey };
if (model) body.model = model;
var res = await fetch(API + '/api/blog/llm/switch', {
var data = await api('/api/blog/llm/switch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
var data = await res.json();
if (data.success) {
if (msgEl) {
msgEl.style.background = '#d1fae5';

11
packages/scraper/src/index.ts
View File

@ -32,6 +32,7 @@
* tsx src/index.ts --addon — Run AddOn Networks scraper once
* tsx src/index.ts --fiber24 — Run ShopFiber24 scraper once (sitemap-based)
* tsx src/index.ts --fibermall — Run FiberMall scraper once
* tsx src/index.ts --backfill-images — Fill missing transceiver product photos
*/
import { createScheduler, registerSchedules, registerWorkers } from "./scheduler";
import { scrapeFs } from "./scrapers/fs-com";
@ -156,6 +157,10 @@ async function runOnce(): Promise<void> {
if (args.includes("--fibermall") || isAll || isFetchOnly) {
await scrapeFiberMall();
}
if (args.includes("--backfill-images")) {
const { backfillImages } = await import("./utils/backfill-images");
await backfillImages();
}
// Playwright-based scrapers (need Chromium installed)
if (!isFetchOnly) {
@ -218,8 +223,10 @@ async function runScheduler(): Promise<void> {
console.warn("Startup zombie cleanup failed (non-fatal):", (err as Error).message);
}
await registerSchedules(boss);
// Workers must register before schedules — boss.work() auto-creates queues,
// boss.schedule() requires the queue to already exist (pg-boss v10 FK constraint)
await registerWorkers(boss);
await registerSchedules(boss);
console.log("\nScheduler running. Press Ctrl+C to stop.\n");
@ -235,7 +242,7 @@ async function runScheduler(): Promise<void> {
process.on("SIGTERM", shutdown);
}
const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics", "--fiber24", "--fibermall"];
const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics", "--fiber24", "--fibermall", "--backfill-images"];
if (args.some((a) => ALL_FLAGS.includes(a))) {
runOnce().catch((err) => {

12
packages/scraper/src/scheduler.ts
View File

@ -63,6 +63,18 @@ export async function createScheduler(): Promise<PgBoss> {
}
export async function registerSchedules(boss: PgBoss): Promise<void> {
// pg-boss v10: boss.schedule() requires the queue to already exist in pgboss.queue.
// After a DB reset (e.g. server outage), all queue rows are wiped.
// Patch boss.schedule to auto-create queues idempotently before each schedule call,
// so the 277 individual schedule() calls below don't need to be touched.
const _origSchedule = boss.schedule.bind(boss) as typeof boss.schedule;
(boss as unknown as Record<string, unknown>).schedule = async (
name: string, cron: string, data?: unknown, opts?: unknown,
) => {
await boss.createQueue(name).catch(() => { /* already exists */ });
return _origSchedule(name, cron, data as object, opts as object);
};
const queues = [
// ── Playwright scrapers (Erik, every 8h) ───────────────────────────
"scrape:pricing:fs",