rene.fichtmueller/transceiver-db
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(equivalences): drain pending-review backlog automatically

Browse Source 
catalog-reconcile.ts inserted new pending equivalences with
re_research_due_at always NULL. The daily maintenance:re-research-equivalences
job only evaluates rows where re_research_due_at IS NOT NULL AND <= NOW() -- so
the job that was built to auto-approve/reject the queue never saw fresh
pending rows. Backlog reached 289,681 rows before anyone noticed (found live
2026-07-13, one-time triage in sql/139: 41,963 approved, 247,718 rejected,
57,683 of those flagged for re-research once missing specs get enriched).

Fix: pending inserts now get re_research_due_at=NOW() so the job drains them
going forward. Also: reject branch now distinguishes data-gap rejects
(missing wavelength/fiber/reach evidence -> re-checked in 21 days once a
scraper fills the gap) from confirmed-mismatch/no-price/low-confidence
rejects (permanent, re_research_due_at stays NULL -- nothing changes on a
re-check against the same stored specs).
This commit is contained in:
root 2026-07-17 19:58:01 +00:00
parent a4fffd05fd
commit 9320a236c8
2 changed files with 21 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
packages/scraper/src/robots/catalog-reconcile.ts
View File

@ -249,18 +249,25 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
`last_price: ${cand.last_price?.toISOString() ?? "never"} | ` + `last_price: ${cand.last_price?.toISOString() ?? "never"} | ` +
`source: catalog-reconcile`; `source: catalog-reconcile`;
// Upsert — bereits approved/rejected Einträge nicht überschreiben // Upsert — bereits approved/rejected Einträge nicht überschreiben. Pending
// rows get re_research_due_at = NOW() so the daily
// maintenance:re-research-equivalences job (scheduler.ts) actually drains
// them — that job only evaluates rows where re_research_due_at IS NOT NULL,
// and without this the backlog only grows (found live 2026-07-13: 289,681
// pending rows had accumulated with re_research_due_at always NULL).
const reResearchDueAt = status === "pending" ? new Date() : null;
const { rowCount } = await pool.query(` const { rowCount } = await pool.query(`
INSERT INTO transceiver_equivalences INSERT INTO transceiver_equivalences
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status) (flexoptix_id, competitor_id, confidence, match_basis, match_notes, status, re_research_due_at)
VALUES ($1, $2, $3, $4, $5, $6) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
confidence = EXCLUDED.confidence, confidence = EXCLUDED.confidence,
match_basis = EXCLUDED.match_basis, match_basis = EXCLUDED.match_basis,
match_notes = EXCLUDED.match_notes, match_notes = EXCLUDED.match_notes,
re_research_due_at = COALESCE(transceiver_equivalences.re_research_due_at, EXCLUDED.re_research_due_at),
updated_at = NOW() updated_at = NOW()
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected', 'auto_approved') WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected', 'auto_approved')
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]); `, [fx.id, cand.competitor_id, confidence, basis, notes, status, reResearchDueAt]);
const wasInsertOrUpdate = (rowCount ?? 0) > 0; const wasInsertOrUpdate = (rowCount ?? 0) > 0;
if (!wasInsertOrUpdate) { if (!wasInsertOrUpdate) {

12
packages/scraper/src/scheduler.ts
View File

@ -3182,6 +3182,13 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
); );
if (research.decision === "reject") { if (research.decision === "reject") {
const rejectReason = research.rejectReason || "automated research: rejected";
// Data-gap rejects (missing wavelength/fiber/reach evidence, not a
// confirmed mismatch) get a future re-check — once a scraper fills the
// gap, re-research picks it back up. Confirmed mismatches / no-price /
// low-confidence rejects get re_research_due_at=NULL (permanent; nothing
// will change on a re-check with the same stored specs).
const isDataGap = rejectReason.includes("insufficient technical evidence");
await pool.query(` await pool.query(`
UPDATE transceiver_equivalences UPDATE transceiver_equivalences
SET status = 'rejected', SET status = 'rejected',
@ -3190,7 +3197,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
reject_reason = $4, reject_reason = $4,
reviewed_by = 'automated-research', reviewed_by = 'automated-research',
reviewed_at = NOW(), reviewed_at = NOW(),
re_research_due_at = NULL, re_research_due_at = CASE WHEN $6 THEN NOW() + INTERVAL '21 days' ELSE NULL END,
re_researched_at = NOW(), re_researched_at = NOW(),
match_notes = CONCAT( match_notes = CONCAT(
COALESCE(match_notes, ''), COALESCE(match_notes, ''),
@ -3202,8 +3209,9 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
eq.id, eq.id,
research.confidence, research.confidence,
research.basis, research.basis,
research.rejectReason || "automated research: rejected", rejectReason,
research.reasons.join("; "), research.reasons.join("; "),
isDataGap,
]); ]);
await pool.query(` await pool.query(`