fix(scraper): auto-create pg-boss queues before scheduling + worker/schedule order
- scheduler: patch boss.schedule() to call createQueue() first (idempotent), fixing FK constraint errors after DB reset — no need to touch 277 call sites - index: registerWorkers() before registerSchedules() since boss.work() must register handlers before schedules fire - dashboard: fix switchBlogLlm() to use api() helper (adds Bearer auth token) instead of raw fetch() which was returning 401 Unauthorized
This commit is contained in:
parent
270bd12382
commit
199f36be48
@ -5644,12 +5644,11 @@ async function switchBlogLlm(providerKey, model) {
|
|||||||
try {
|
try {
|
||||||
var body = { provider: providerKey };
|
var body = { provider: providerKey };
|
||||||
if (model) body.model = model;
|
if (model) body.model = model;
|
||||||
var res = await fetch(API + '/api/blog/llm/switch', {
|
var data = await api('/api/blog/llm/switch', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify(body)
|
body: JSON.stringify(body)
|
||||||
});
|
});
|
||||||
var data = await res.json();
|
|
||||||
if (data.success) {
|
if (data.success) {
|
||||||
if (msgEl) {
|
if (msgEl) {
|
||||||
msgEl.style.background = '#d1fae5';
|
msgEl.style.background = '#d1fae5';
|
||||||
|
|||||||
@ -32,6 +32,7 @@
|
|||||||
* tsx src/index.ts --addon — Run AddOn Networks scraper once
|
* tsx src/index.ts --addon — Run AddOn Networks scraper once
|
||||||
* tsx src/index.ts --fiber24 — Run ShopFiber24 scraper once (sitemap-based)
|
* tsx src/index.ts --fiber24 — Run ShopFiber24 scraper once (sitemap-based)
|
||||||
* tsx src/index.ts --fibermall — Run FiberMall scraper once
|
* tsx src/index.ts --fibermall — Run FiberMall scraper once
|
||||||
|
* tsx src/index.ts --backfill-images — Fill missing transceiver product photos
|
||||||
*/
|
*/
|
||||||
import { createScheduler, registerSchedules, registerWorkers } from "./scheduler";
|
import { createScheduler, registerSchedules, registerWorkers } from "./scheduler";
|
||||||
import { scrapeFs } from "./scrapers/fs-com";
|
import { scrapeFs } from "./scrapers/fs-com";
|
||||||
@ -156,6 +157,10 @@ async function runOnce(): Promise<void> {
|
|||||||
if (args.includes("--fibermall") || isAll || isFetchOnly) {
|
if (args.includes("--fibermall") || isAll || isFetchOnly) {
|
||||||
await scrapeFiberMall();
|
await scrapeFiberMall();
|
||||||
}
|
}
|
||||||
|
if (args.includes("--backfill-images")) {
|
||||||
|
const { backfillImages } = await import("./utils/backfill-images");
|
||||||
|
await backfillImages();
|
||||||
|
}
|
||||||
|
|
||||||
// Playwright-based scrapers (need Chromium installed)
|
// Playwright-based scrapers (need Chromium installed)
|
||||||
if (!isFetchOnly) {
|
if (!isFetchOnly) {
|
||||||
@ -218,8 +223,10 @@ async function runScheduler(): Promise<void> {
|
|||||||
console.warn("Startup zombie cleanup failed (non-fatal):", (err as Error).message);
|
console.warn("Startup zombie cleanup failed (non-fatal):", (err as Error).message);
|
||||||
}
|
}
|
||||||
|
|
||||||
await registerSchedules(boss);
|
// Workers must register before schedules — boss.work() auto-creates queues,
|
||||||
|
// boss.schedule() requires the queue to already exist (pg-boss v10 FK constraint)
|
||||||
await registerWorkers(boss);
|
await registerWorkers(boss);
|
||||||
|
await registerSchedules(boss);
|
||||||
|
|
||||||
console.log("\nScheduler running. Press Ctrl+C to stop.\n");
|
console.log("\nScheduler running. Press Ctrl+C to stop.\n");
|
||||||
|
|
||||||
@ -235,7 +242,7 @@ async function runScheduler(): Promise<void> {
|
|||||||
process.on("SIGTERM", shutdown);
|
process.on("SIGTERM", shutdown);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics", "--fiber24", "--fibermall"];
|
const ALL_FLAGS = ["--all", "--fs", "--cisco", "--optcore", "--news", "--flexoptix", "--vendors", "--10gtek", "--champion", "--fluxlight", "--sfpcables", "--gbics", "--prolabs", "--naddod", "--qsfptek", "--addon", "--juniper", "--switches", "--whitebox", "--switches-ext", "--flexoptix-vendors", "--sonic-hcl", "--edgecore", "--ufispace", "--switch-assets", "--switch-crawl", "--switch-crawl-pw", "--fetch-only", "--atgbics", "--fiber24", "--fibermall", "--backfill-images"];
|
||||||
|
|
||||||
if (args.some((a) => ALL_FLAGS.includes(a))) {
|
if (args.some((a) => ALL_FLAGS.includes(a))) {
|
||||||
runOnce().catch((err) => {
|
runOnce().catch((err) => {
|
||||||
|
|||||||
@ -63,6 +63,18 @@ export async function createScheduler(): Promise<PgBoss> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function registerSchedules(boss: PgBoss): Promise<void> {
|
export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||||
|
// pg-boss v10: boss.schedule() requires the queue to already exist in pgboss.queue.
|
||||||
|
// After a DB reset (e.g. server outage), all queue rows are wiped.
|
||||||
|
// Patch boss.schedule to auto-create queues idempotently before each schedule call,
|
||||||
|
// so the 277 individual schedule() calls below don't need to be touched.
|
||||||
|
const _origSchedule = boss.schedule.bind(boss) as typeof boss.schedule;
|
||||||
|
(boss as unknown as Record<string, unknown>).schedule = async (
|
||||||
|
name: string, cron: string, data?: unknown, opts?: unknown,
|
||||||
|
) => {
|
||||||
|
await boss.createQueue(name).catch(() => { /* already exists */ });
|
||||||
|
return _origSchedule(name, cron, data as object, opts as object);
|
||||||
|
};
|
||||||
|
|
||||||
const queues = [
|
const queues = [
|
||||||
// ── Playwright scrapers (Erik, every 8h) ───────────────────────────
|
// ── Playwright scrapers (Erik, every 8h) ───────────────────────────
|
||||||
"scrape:pricing:fs",
|
"scrape:pricing:fs",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user