fix: close stale TIP manual review queue

packages/api/src/routes/review.ts

@@ -92,9 +92,17 @@ reviewRouter.get("/equivalences", async (req: Request, res: Response) => {
     params = [limit, offset];
     limitIdx = 1; offsetIdx = 2;
   } else if (status === "needs_research") {
-    where = `WHERE eq.status IN ('pending','approved','auto_approved') AND eq.re_research_due_at IS NOT NULL AND eq.re_research_due_at <= NOW()`;
+    where = `WHERE eq.status IN ('pending','approved','auto_approved')
+       AND eq.re_research_due_at IS NOT NULL
+       AND eq.re_research_due_at <= NOW()
+       AND COALESCE(fx.competitor_status, 'needs_research') IN ('unknown', 'needs_research')`;
     params = [limit, offset];
     limitIdx = 1; offsetIdx = 2;
+  } else if (status === "pending") {
+    where = `WHERE eq.status = $1
+       AND COALESCE(fx.competitor_status, 'needs_research') IN ('unknown', 'needs_research')`;
+    params = [status, limit, offset];
+    limitIdx = 2; offsetIdx = 3;
   } else {
     where = `WHERE eq.status = $1`;
     params = [status, limit, offset];
@@ -163,7 +171,11 @@ reviewRouter.get("/equivalences", async (req: Request, res: Response) => {
   `, params);
 
   const countResult = await pool.query(
-    `SELECT COUNT(*) FROM transceiver_equivalences eq ${where}`,
+    `SELECT COUNT(*)
+     FROM transceiver_equivalences eq
+     JOIN transceivers fx ON fx.id = eq.flexoptix_id
+     JOIN transceivers cp ON cp.id = eq.competitor_id
+     ${where}`,
     (status === "all" || status === "needs_research") ? [] : [status]
   );
 
@@ -180,15 +192,20 @@ reviewRouter.get("/equivalences", async (req: Request, res: Response) => {
 reviewRouter.get("/equivalences/stats", async (_req: Request, res: Response) => {
   const result = await pool.query(`
     SELECT
-      SUM(CASE WHEN status = 'pending'       THEN 1 ELSE 0 END) AS pending,
+      SUM(CASE WHEN eq.status = 'pending'
-      SUM(CASE WHEN status = 'approved'      THEN 1 ELSE 0 END) AS approved,
+                AND COALESCE(fx.competitor_status, 'needs_research') IN ('unknown', 'needs_research')
-      SUM(CASE WHEN status = 'auto_approved' THEN 1 ELSE 0 END) AS auto_approved,
+               THEN 1 ELSE 0 END) AS pending,
-      SUM(CASE WHEN status = 'rejected'      THEN 1 ELSE 0 END) AS rejected,
+      SUM(CASE WHEN eq.status = 'approved'      THEN 1 ELSE 0 END) AS approved,
-      SUM(CASE WHEN status IN ('pending','approved','auto_approved')
+      SUM(CASE WHEN eq.status = 'auto_approved' THEN 1 ELSE 0 END) AS auto_approved,
-               AND re_research_due_at IS NOT NULL
+      SUM(CASE WHEN eq.status = 'rejected'      THEN 1 ELSE 0 END) AS rejected,
-               AND re_research_due_at <= NOW()        THEN 1 ELSE 0 END) AS needs_research,
+      SUM(CASE WHEN eq.status IN ('pending','approved','auto_approved')
+               AND eq.re_research_due_at IS NOT NULL
+               AND eq.re_research_due_at <= NOW()
+               AND COALESCE(fx.competitor_status, 'needs_research') IN ('unknown', 'needs_research')
+               THEN 1 ELSE 0 END) AS needs_research,
       COUNT(*) AS total
-    FROM transceiver_equivalences
+    FROM transceiver_equivalences eq
+    JOIN transceivers fx ON fx.id = eq.flexoptix_id
   `);
   const productStatus = await pool.query(`
     SELECT

packages/scraper/src/scheduler.ts

@@ -2706,7 +2706,9 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
     const ts = new Date().toISOString();
     console.log(`[${ts}] Running: Equivalence matching`);
 
-    // Find all Flexoptix transceivers that are NOT yet competitor_verified
+    // Find Flexoptix transceivers whose competitor research is still open.
+    // Terminal product-level states are not manual-review work and must not
+    // recreate stale pending equivalence candidates.
     const flexResult = await pool.query(`
       SELECT t.id, t.part_number, t.standard_name, t.form_factor,
              t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
@@ -2715,6 +2717,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
       JOIN vendors v ON v.id = t.vendor_id
       WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
         AND t.competitor_verified = false
+        AND COALESCE(t.competitor_status, 'needs_research') IN ('unknown', 'needs_research')
     `);
 
     let autoApproved = 0;

sql/107-close-stale-manual-review-queue.sql

@@ -0,0 +1,30 @@
+-- Migration 107: Close stale manual equivalence review debt.
+--
+-- Pending equivalence candidates are only actionable while the Flexoptix
+-- product still has unresolved competitor research. Once the product-level
+-- state is terminal (`matched`, `no_valid_match`, or `ambiguous`), leftover
+-- pending candidates are stale review debt and must not ask for manual work.
+
+UPDATE transceiver_equivalences eq
+SET status = 'rejected',
+    reviewed_by = 'verify:close-stale-review-queue',
+    reviewed_at = NOW(),
+    reject_reason = CASE
+      WHEN fx.competitor_status = 'matched'
+        THEN 'automated closure: Flexoptix product already has a resolved competitor match; stale alternate candidate'
+      WHEN fx.competitor_status = 'ambiguous'
+        THEN 'automated closure: Flexoptix product competitor state is resolved ambiguous; no safe deterministic 1:1 match'
+      WHEN fx.competitor_status = 'no_valid_match'
+        THEN 'automated closure: Flexoptix product has resolved no-valid-match state'
+      ELSE 'automated closure: stale non-actionable equivalence candidate'
+    END,
+    re_research_due_at = NULL,
+    re_researched_at = NOW(),
+    updated_at = NOW()
+FROM transceivers fx
+WHERE eq.flexoptix_id = fx.id
+  AND eq.status = 'pending'
+  AND COALESCE(fx.competitor_status, 'needs_research') IN ('matched', 'no_valid_match', 'ambiguous');
+
+COMMENT ON TABLE transceiver_equivalences IS
+  'Candidate equivalences between Flexoptix and competitor transceivers. Pending rows are actionable only while product-level competitor research is unresolved; resolved product states close stale pending candidates automatically.';

sync/CURRENT.md

@@ -1,9 +1,42 @@
 # Current TIP Sync State
 
-Updated: 2026-05-10 08:10 UTC
+Updated: 2026-05-10 08:32 UTC
 
 ## Newest Work
 
+- TIP manual review queue closure on 2026-05-10 UTC:
+  - user correctly reported that the dashboard still showed `Review 13374`, so TIP was not done from the UI/workflow perspective
+  - live DB diagnosis:
+    - `transceiver_equivalences.status=pending`: `13374`
+    - these pending rows belonged to only `506` Flexoptix products
+    - product-level competitor state was already terminal:
+      - `13323` pending candidates had Flexoptix `competitor_status=matched`
+      - `51` pending candidates had Flexoptix `competitor_status=ambiguous`
+    - therefore the visible Manual Review badge represented stale equivalence candidates, not actionable manual work
+  - added `sql/107-close-stale-manual-review-queue.sql`
+    - rejects stale pending equivalences when the Flexoptix product is already `matched`, `no_valid_match`, or `ambiguous`
+    - writes explicit automated reject reasons
+  - hardened `maintenance:find-equivalences`
+    - matcher now only creates pending equivalences for Flexoptix products whose competitor research is still `unknown` or `needs_research`
+    - terminal product states cannot recreate stale manual-review debt
+  - hardened Review API counts/listing:
+    - Pending badge now counts only genuinely actionable rows where the Flexoptix product still has open competitor research
+  - live deployment:
+    - migration applied on Erik: `UPDATE 13374`
+    - API and scraper builds passed
+    - `tip-api` and `tip-scraper-daemon` restarted
+  - final review table status:
+    - `approved=1987`
+    - `auto_approved=33464`
+    - `rejected=161756`
+    - `pending=0`
+  - final active-base health remains:
+    - research resolved: `16236 / 16236 = 100%`
+    - all price/image/details/competitor `needs_research` buckets: `0`
+  - TIPLLM training pool updated with:
+    - stale manual review queue rule
+    - review badge zero-count rule
+
 - TIP research-resolution closure on 2026-05-10 UTC:
   - added explicit image/detail research status model:
     - `sql/106-research-resolution-status.sql`

sync/history/2026-05-10-tip-manual-review-queue-closure.md

@@ -0,0 +1,70 @@
+# TIP Manual Review Queue Closure
+
+Date: 2026-05-10 UTC
+Owner: Codex
+
+## Summary
+
+The dashboard still showed `Review 13374` after research resolution was complete.
+
+This was a real workflow issue: the product research queues were closed, but stale equivalence candidates still appeared as manual review work.
+
+## Diagnosis
+
+Live DB state before cleanup:
+
+- `pending=13374`
+- `approved=1987`
+- `auto_approved=33464`
+- `rejected=148382`
+
+The `13374` pending rows belonged to only `506` Flexoptix products.
+
+Their product-level competitor states were already terminal:
+
+- `13323` pending rows had Flexoptix `competitor_status=matched`
+- `51` pending rows had Flexoptix `competitor_status=ambiguous`
+
+Therefore these rows were stale review debt, not actionable manual work.
+
+## Implemented
+
+- Added `sql/107-close-stale-manual-review-queue.sql`
+  - closes stale pending equivalences when product-level competitor research is already terminal
+  - writes explicit automated reject reasons
+- Hardened `maintenance:find-equivalences`
+  - only creates pending candidates for Flexoptix products with `competitor_status IN ('unknown','needs_research')`
+  - terminal states (`matched`, `no_valid_match`, `ambiguous`) cannot recreate manual-review debt
+- Hardened Review API
+  - pending badge/listing only counts genuinely actionable pending rows
+  - stale candidates for terminal product states are not shown as manual work
+
+## Live Deployment
+
+- Migration applied on Erik:
+  - `UPDATE 13374`
+- API build passed.
+- Scraper build passed.
+- `tip-api` restarted.
+- `tip-scraper-daemon` restarted.
+
+## Final Review State
+
+- `approved=1987`
+- `auto_approved=33464`
+- `rejected=161756`
+- `pending=0`
+
+## Final Health Still Intact
+
+- `research_resolved=16236`
+- `total=16236`
+- `research_resolved_pct=100`
+- price/image/details/competitor `needs_research=0`
+
+## Training Pool
+
+Added two TIPLLM examples to `training-data/tip-llm-capabilities-v1.jsonl`:
+
+- stale manual review queue closure rule
+- review badge zero-count rule

training-data/tip-llm-capabilities-v1.jsonl

@@ -62,3 +62,5 @@
 {"id":"tip-20260510-research-resolution-status-model-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Close research loops with explicit states and evidence; never turn missing public data into fake verification."},{"role":"user","content":"How should TIP represent a product when public price, image, details, or competitor research has been exhausted but no source-backed value exists?"},{"role":"assistant","content":"Keep strict verification booleans source-backed only. Use status columns to represent resolved research states: `price_status=public_price|no_public_price|ambiguous|needs_research`, `image_status=public_image|no_public_image|ambiguous|needs_research`, `details_status=public_details|no_public_details|ambiguous|needs_research`, and `competitor_status=matched|no_valid_match|ambiguous|needs_research`. Write evidence for unavailable states (`price_unavailable`, `image_unavailable`, `details_unavailable`) or competitor outcomes. Count `research_resolved` when all four domains are in a terminal state, even if `fully_verified` remains false because public evidence was unavailable."}]}
 {"id":"tip-20260510-research-status-resolver-runbook-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Operate TIP robots on Erik conservatively and in auditable batches."},{"role":"user","content":"What is the safe runbook for closing remaining image/details/competitor needs_research states?"},{"role":"assistant","content":"Run `verify:research-status` dry-run first with a small `RESEARCH_STATUS_LIMIT`. Apply in serial batches with `RESEARCH_STATUS_APPLY=1`, checking `/api/health` between batches. Do not restart `tip-scraper-daemon` while pg-boss jobs are active. For missing images/details, record `no_public_image` or `no_public_details` with source-aware unavailable evidence instead of marking verified. For competitors, use strict form factor, speed, fiber and reach matching; set `no_valid_match` if no resolved candidate exists, or `ambiguous` if candidates exist but no deterministic 1:1 match is safe."}]}
 {"id":"tip-20260510-research-resolution-live-result-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Use live health metrics to distinguish strict verification from completed research."},{"role":"user","content":"What was the TIP research-resolution closure result on 2026-05-10?"},{"role":"assistant","content":"After migration `106-research-resolution-status.sql` and batched `verify:research-status` runs, active TIP research was closed for `16236 / 16236` products (`research_resolved_pct=100`). Final active-base buckets: price `public_price=10851`, `no_public_price=5385`, `needs_research=0`; image `public_image=11602`, `no_public_image=4634`, `needs_research=0`; details `public_details=16005`, `no_public_details=231`, `needs_research=0`; competitor `matched=10838`, `no_valid_match=561`, `ambiguous=4837`, `needs_research=0`. Strict `fully_verified` remained `10600` because no-public-image/details and ambiguous competitor states are not fake green checks."}]}
+{"id":"tip-20260510-stale-manual-review-queue-001","source":"codex-tip-manual-review-closure-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Manual review queues must reflect genuinely actionable work, not stale candidates already superseded by product-level research outcomes."},{"role":"user","content":"The dashboard shows thousands of pending manual equivalence reviews, but product-level competitor_status is already matched or ambiguous. What should TIP do?"},{"role":"assistant","content":"Treat those pending equivalence rows as stale review debt. If the Flexoptix product has terminal `competitor_status` such as `matched`, `no_valid_match`, or `ambiguous`, pending candidate rows are no longer actionable manual work. Reject them with an automated reason that explains whether they were superseded by an existing match, resolved no-valid-match, or resolved ambiguity. Also harden the matcher so it only generates pending equivalences for products whose competitor research is still `unknown` or `needs_research`."}]}
+{"id":"tip-20260510-review-badge-zero-rule-001","source":"codex-tip-manual-review-closure-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Dashboard counters should show current work, not historical tables."},{"role":"user","content":"How should the Manual Review badge be counted after research-resolution closure?"},{"role":"assistant","content":"Count `pending` manual review only for equivalence rows where `eq.status='pending'` and the Flexoptix product still has `competitor_status IN ('unknown','needs_research')`. Do not show stale pending candidates for products already resolved as `matched`, `no_valid_match`, or `ambiguous`. After the 2026-05-10 cleanup, `transceiver_equivalences` had `pending=0`; the dashboard Review badge should hide."}]}