diff --git a/packages/scraper/src/robots/catalog-reconcile.ts b/packages/scraper/src/robots/catalog-reconcile.ts index 5627b81..bf8a23c 100644 --- a/packages/scraper/src/robots/catalog-reconcile.ts +++ b/packages/scraper/src/robots/catalog-reconcile.ts @@ -249,18 +249,25 @@ export async function runCatalogReconcile(): Promise { `last_price: ${cand.last_price?.toISOString() ?? "never"} | ` + `source: catalog-reconcile`; - // Upsert — bereits approved/rejected Einträge nicht überschreiben + // Upsert — bereits approved/rejected Einträge nicht überschreiben. Pending + // rows get re_research_due_at = NOW() so the daily + // maintenance:re-research-equivalences job (scheduler.ts) actually drains + // them — that job only evaluates rows where re_research_due_at IS NOT NULL, + // and without this the backlog only grows (found live 2026-07-13: 289,681 + // pending rows had accumulated with re_research_due_at always NULL). + const reResearchDueAt = status === "pending" ? new Date() : null; const { rowCount } = await pool.query(` INSERT INTO transceiver_equivalences - (flexoptix_id, competitor_id, confidence, match_basis, match_notes, status) - VALUES ($1, $2, $3, $4, $5, $6) + (flexoptix_id, competitor_id, confidence, match_basis, match_notes, status, re_research_due_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET confidence = EXCLUDED.confidence, match_basis = EXCLUDED.match_basis, match_notes = EXCLUDED.match_notes, + re_research_due_at = COALESCE(transceiver_equivalences.re_research_due_at, EXCLUDED.re_research_due_at), updated_at = NOW() WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected', 'auto_approved') - `, [fx.id, cand.competitor_id, confidence, basis, notes, status]); + `, [fx.id, cand.competitor_id, confidence, basis, notes, status, reResearchDueAt]); const wasInsertOrUpdate = (rowCount ?? 0) > 0; if (!wasInsertOrUpdate) { diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index f8171f5..9266d98 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -3182,6 +3182,13 @@ export async function registerWorkers(boss: PgBoss): Promise { ); if (research.decision === "reject") { + const rejectReason = research.rejectReason || "automated research: rejected"; + // Data-gap rejects (missing wavelength/fiber/reach evidence, not a + // confirmed mismatch) get a future re-check — once a scraper fills the + // gap, re-research picks it back up. Confirmed mismatches / no-price / + // low-confidence rejects get re_research_due_at=NULL (permanent; nothing + // will change on a re-check with the same stored specs). + const isDataGap = rejectReason.includes("insufficient technical evidence"); await pool.query(` UPDATE transceiver_equivalences SET status = 'rejected', @@ -3190,7 +3197,7 @@ export async function registerWorkers(boss: PgBoss): Promise { reject_reason = $4, reviewed_by = 'automated-research', reviewed_at = NOW(), - re_research_due_at = NULL, + re_research_due_at = CASE WHEN $6 THEN NOW() + INTERVAL '21 days' ELSE NULL END, re_researched_at = NOW(), match_notes = CONCAT( COALESCE(match_notes, ''), @@ -3202,8 +3209,9 @@ export async function registerWorkers(boss: PgBoss): Promise { eq.id, research.confidence, research.basis, - research.rejectReason || "automated research: rejected", + rejectReason, research.reasons.join("; "), + isDataGap, ]); await pool.query(`