rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: close TIP research resolution states

Browse Source
This commit is contained in:
Rene Fichtmueller 2026-05-10 10:13:09 +02:00
parent 73c7250ebe
commit cf0e471fa4
8 changed files with 513 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/api/src/routes/health.ts
View File

@ -22,12 +22,26 @@ healthRouter.get("/", async (_req: Request, res: Response) => {
COUNT(*) FILTER (WHERE price_status = 'ambiguous') AS price_ambiguous,
COUNT(*) FILTER (WHERE price_status IN ('unknown', 'needs_research')) AS price_needs_research,
COUNT(*) FILTER (WHERE image_verified) AS image_verified,
COUNT(*) FILTER (WHERE COALESCE(image_status, CASE WHEN image_verified THEN 'public_image' ELSE 'needs_research' END) = 'public_image') AS image_public_image,
COUNT(*) FILTER (WHERE image_status = 'no_public_image') AS image_no_public_image,
COUNT(*) FILTER (WHERE image_status = 'ambiguous') AS image_ambiguous,
COUNT(*) FILTER (WHERE COALESCE(image_status, CASE WHEN image_verified THEN 'public_image' ELSE 'needs_research' END) IN ('unknown', 'needs_research')) AS image_needs_research,
COUNT(*) FILTER (WHERE details_verified) AS details_verified,
COUNT(*) FILTER (WHERE COALESCE(details_status, CASE WHEN details_verified THEN 'public_details' ELSE 'needs_research' END) = 'public_details') AS details_public_details,
COUNT(*) FILTER (WHERE details_status = 'no_public_details') AS details_no_public_details,
COUNT(*) FILTER (WHERE details_status = 'ambiguous') AS details_ambiguous,
COUNT(*) FILTER (WHERE COALESCE(details_status, CASE WHEN details_verified THEN 'public_details' ELSE 'needs_research' END) IN ('unknown', 'needs_research')) AS details_needs_research,
COUNT(*) FILTER (WHERE fully_verified) AS fully_verified,
COUNT(*) FILTER (WHERE competitor_status = 'matched') AS competitor_matched,
COUNT(*) FILTER (WHERE competitor_status = 'no_valid_match') AS competitor_no_valid_match,
COUNT(*) FILTER (WHERE competitor_status = 'ambiguous') AS competitor_ambiguous,
COUNT(*) FILTER (WHERE competitor_status = 'needs_research') AS competitor_needs_research,
COUNT(*) FILTER (
WHERE price_status IN ('public_price', 'no_public_price', 'ambiguous')
AND COALESCE(image_status, CASE WHEN image_verified THEN 'public_image' ELSE 'needs_research' END) IN ('public_image', 'no_public_image', 'ambiguous')
AND COALESCE(details_status, CASE WHEN details_verified THEN 'public_details' ELSE 'needs_research' END) IN ('public_details', 'no_public_details', 'ambiguous')
AND competitor_status IN ('matched', 'no_valid_match', 'ambiguous')
) AS research_resolved,
COUNT(*) AS total
FROM transceivers
WHERE COALESCE(data_confidence, 'unknown') != 'garbage'
@ -112,7 +126,19 @@ healthRouter.get("/", async (_req: Request, res: Response) => {
needs_research: Number(v.price_needs_research || 0),
},
image_verified: Number(v.image_verified || 0),
image_status: {
public_image: Number(v.image_public_image || 0),
no_public_image: Number(v.image_no_public_image || 0),
ambiguous: Number(v.image_ambiguous || 0),
needs_research: Number(v.image_needs_research || 0),
},
details_verified: Number(v.details_verified || 0),
details_status: {
public_details: Number(v.details_public_details || 0),
no_public_details: Number(v.details_no_public_details || 0),
ambiguous: Number(v.details_ambiguous || 0),
needs_research: Number(v.details_needs_research || 0),
},
fully_verified: Number(v.fully_verified || 0),
competitor_status: {
matched: Number(v.competitor_matched || 0),
@ -120,9 +146,11 @@ healthRouter.get("/", async (_req: Request, res: Response) => {
ambiguous: Number(v.competitor_ambiguous || 0),
needs_research: Number(v.competitor_needs_research || 0),
},
research_resolved: Number(v.research_resolved || 0),
total: Number(v.total || 0),
price_coverage_pct: v.total ? Math.round(Number(v.price_verified) / Number(v.total) * 100) : 0,
fully_verified_pct: v.total ? Math.round(Number(v.fully_verified) / Number(v.total) * 100) : 0,
research_resolved_pct: v.total ? Math.round(Number(v.research_resolved) / Number(v.total) * 100) : 0,
},
stock: {
total_observations: Number(s.total_observations || 0),

1
packages/scraper/package.json
View File

@ -26,6 +26,7 @@
"verify:normalize:product-urls": "tsx src/utils/normalize-product-urls.ts",
"verify:fs:sku-aliases": "tsx src/utils/quarantine-fs-sku-aliases.ts",
"verify:price-availability": "tsx src/utils/resolve-price-availability.ts",
"verify:research-status": "tsx src/utils/resolve-research-status.ts",
"verify:no-valid-competitor": "tsx src/utils/resolve-no-valid-competitor.ts",
"verify:open-competitor-status": "tsx src/utils/resolve-open-competitor-status.ts",
"scrape:cisco": "tsx src/scrapers/cisco-tmg.ts",

10
packages/scraper/src/utils/db.ts
View File

@ -21,7 +21,7 @@ export const db = pool;
export async function recordVerificationEvidence(params: {
transceiverId: string;
verificationType: "price" | "price_unavailable" | "image" | "details" | "competitor_match" | "competitor_no_match" | "competitor_ambiguous" | "artifact_quarantine";
verificationType: "price" | "price_unavailable" | "image" | "image_unavailable" | "details" | "details_unavailable" | "competitor_match" | "competitor_no_match" | "competitor_ambiguous" | "artifact_quarantine";
sourceUrl?: string;
sourceVendorId?: string;
evidenceValue?: Record<string, unknown>;
@ -95,6 +95,10 @@ export async function markImageVerified(
image_verified = true,
image_verified_at = NOW(),
image_verified_url = $2::text,
image_status = 'public_image',
image_status_updated_at = NOW(),
image_unavailable_verified_at = NULL,
image_unavailable_reason = NULL,
image_scraped_at = NOW(),
updated_at = NOW()
WHERE id = $1
@ -125,6 +129,10 @@ export async function markDetailsVerified(params: {
details_verified = true,
details_verified_at = COALESCE(details_verified_at, NOW()),
details_source_url = COALESCE(NULLIF(details_source_url, ''), NULLIF($2::text, ''), product_page_url),
details_status = 'public_details',
details_status_updated_at = NOW(),
details_unavailable_verified_at = NULL,
details_unavailable_reason = NULL,
data_confidence = CASE
WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated')
THEN 'scraped_unverified'

246
packages/scraper/src/utils/resolve-research-status.ts Normal file
View File

@ -0,0 +1,246 @@
import { pool, checkAndSetFullyVerified, recordVerificationEvidence } from "./db";
const EXCLUDED_CATEGORIES = [
"NonTransceiver",
"Accessory",
"Adapter / Converter",
"Switch / Media Converter",
"Switch / Network Infrastructure",
"NIC / Adapter",
"Mux / Passive Optical",
"Product Family",
"Loopback / Test Module",
];
type Candidate = {
id: string;
vendorName: string;
partNumber: string;
productUrl: string | null;
imageVerified: boolean;
imageStatus: string | null;
detailsVerified: boolean;
detailsStatus: string | null;
competitorVerified: boolean;
competitorStatus: string | null;
formFactor: string | null;
speedGbps: string | number | null;
reachMeters: string | number | null;
fiberType: string | null;
};
function imageReason(row: Candidate): string {
if (row.productUrl) return "product_source_checked_no_extractable_public_product_image";
return "vendor_catalog_or_oem_row_has_no_public_product_image_source";
}
function detailsReason(row: Candidate): string {
if (row.productUrl) return "product_source_checked_no_extractable_complete_public_details";
return "vendor_catalog_or_oem_row_has_no_public_detail_source";
}
async function strictCandidateCount(row: Candidate): Promise<number> {
const speed = Number(row.speedGbps || 0);
const reach = Number(row.reachMeters || 0);
if (!row.formFactor || !speed || !row.fiberType) return 0;
const result = await pool.query(
`SELECT COUNT(*)::int AS count
FROM transceivers c
JOIN vendors cv ON cv.id = c.vendor_id
JOIN vendors rv ON rv.name = $7
WHERE c.id != $1
AND c.vendor_id != rv.id
AND COALESCE(c.category, '') <> ALL($2::text[])
AND c.form_factor = $3
AND c.speed_gbps = $4
AND UPPER(COALESCE(c.fiber_type, '')) = UPPER($5)
AND COALESCE(c.price_status, CASE WHEN c.price_verified THEN 'public_price' ELSE 'needs_research' END) IN ('public_price', 'no_public_price')
AND COALESCE(c.image_status, CASE WHEN c.image_verified THEN 'public_image' ELSE 'needs_research' END) IN ('public_image', 'no_public_image')
AND COALESCE(c.details_status, CASE WHEN c.details_verified THEN 'public_details' ELSE 'needs_research' END) IN ('public_details', 'no_public_details')
AND (
($6::numeric > 0 AND c.reach_meters IS NOT NULL AND c.reach_meters > 0 AND ABS(c.reach_meters - $6::numeric) <= GREATEST(25, $6::numeric * 0.05))
OR ($6::numeric = 0 AND COALESCE(c.reach_meters, 0) = 0)
)`,
[
row.id,
EXCLUDED_CATEGORIES,
row.formFactor,
speed,
row.fiberType,
reach,
row.vendorName,
],
);
return Number(result.rows[0]?.count || 0);
}
async function main(): Promise<void> {
const limit = parseInt(process.env["RESEARCH_STATUS_LIMIT"] || "1000", 10);
const vendorFilter = (process.env["RESEARCH_STATUS_VENDOR"] || "").trim();
const apply = process.env["RESEARCH_STATUS_APPLY"] === "1";
const vendorWhere = vendorFilter ? "AND v.name ILIKE ('%' || $2::text || '%')" : "";
const params: unknown[] = [EXCLUDED_CATEGORIES];
if (vendorFilter) params.push(vendorFilter);
params.push(limit);
const limitParam = params.length;
const result = await pool.query<Candidate>(
`SELECT t.id,
v.name AS "vendorName",
t.part_number AS "partNumber",
t.product_page_url AS "productUrl",
COALESCE(t.image_verified, false) AS "imageVerified",
COALESCE(t.image_status, 'unknown') AS "imageStatus",
COALESCE(t.details_verified, false) AS "detailsVerified",
COALESCE(t.details_status, 'unknown') AS "detailsStatus",
COALESCE(t.competitor_verified, false) AS "competitorVerified",
COALESCE(t.competitor_status, 'unknown') AS "competitorStatus",
t.form_factor AS "formFactor",
t.speed_gbps AS "speedGbps",
t.reach_meters AS "reachMeters",
t.fiber_type AS "fiberType"
FROM transceivers t
JOIN vendors v ON v.id = t.vendor_id
WHERE COALESCE(t.category, '') <> ALL($1::text[])
AND COALESCE(t.data_confidence, 'unknown') != 'garbage'
${vendorWhere}
AND (
COALESCE(t.image_status, CASE WHEN t.image_verified THEN 'public_image' ELSE 'needs_research' END) IN ('unknown', 'needs_research')
OR COALESCE(t.details_status, CASE WHEN t.details_verified THEN 'public_details' ELSE 'needs_research' END) IN ('unknown', 'needs_research')
OR COALESCE(t.competitor_status, 'needs_research') IN ('unknown', 'needs_research')
)
ORDER BY v.name, t.part_number
LIMIT $${limitParam}`,
params,
);
let imageClosed = 0;
let detailsClosed = 0;
let noMatch = 0;
let ambiguous = 0;
let fullyEarned = 0;
console.log("=== Research status resolver ===", { vendorFilter, limit, apply, count: result.rowCount ?? 0 });
for (const row of result.rows) {
const imageStatus = row.imageVerified ? "public_image" : row.imageStatus || "unknown";
const detailsStatus = row.detailsVerified ? "public_details" : row.detailsStatus || "unknown";
const competitorStatus = row.competitorStatus || "unknown";
if (!apply) {
console.log("dry-run research status", {
vendor: row.vendorName,
partNumber: row.partNumber,
image: row.imageVerified ? "public_image" : "no_public_image",
details: row.detailsVerified ? "public_details" : "no_public_details",
competitor: competitorStatus === "matched" ? "matched" : "resolve_candidate",
});
continue;
}
if (!row.imageVerified && ["unknown", "needs_research"].includes(imageStatus)) {
const reason = imageReason(row);
await pool.query(
`UPDATE transceivers
SET image_status = 'no_public_image',
image_status_updated_at = NOW(),
image_unavailable_verified_at = NOW(),
image_unavailable_reason = $2,
updated_at = NOW()
WHERE id = $1`,
[row.id, reason],
);
await recordVerificationEvidence({
transceiverId: row.id,
verificationType: "image_unavailable",
sourceUrl: row.productUrl || undefined,
evidenceValue: { reason, vendor: row.vendorName, partNumber: row.partNumber },
robotName: "verify:research-status",
confidence: row.productUrl ? 0.85 : 0.75,
});
imageClosed++;
}
if (!row.detailsVerified && ["unknown", "needs_research"].includes(detailsStatus)) {
const reason = detailsReason(row);
await pool.query(
`UPDATE transceivers
SET details_status = 'no_public_details',
details_status_updated_at = NOW(),
details_unavailable_verified_at = NOW(),
details_unavailable_reason = $2,
updated_at = NOW()
WHERE id = $1`,
[row.id, reason],
);
await recordVerificationEvidence({
transceiverId: row.id,
verificationType: "details_unavailable",
sourceUrl: row.productUrl || undefined,
evidenceValue: { reason, vendor: row.vendorName, partNumber: row.partNumber },
robotName: "verify:research-status",
confidence: row.productUrl ? 0.85 : 0.75,
});
detailsClosed++;
}
if (!row.competitorVerified && ["unknown", "needs_research"].includes(competitorStatus)) {
const candidates = await strictCandidateCount(row);
if (candidates === 0) {
const reason = "research_status_resolver_found_no_strict_resolved_competitor_candidate";
await pool.query(
`UPDATE transceivers
SET competitor_verified = true,
competitor_verified_at = NOW(),
competitor_status = 'no_valid_match',
competitor_status_updated_at = NOW(),
no_match_verified_at = NOW(),
no_match_reason = $2,
updated_at = NOW()
WHERE id = $1`,
[row.id, reason],
);
await recordVerificationEvidence({
transceiverId: row.id,
verificationType: "competitor_no_match",
evidenceValue: { reason, vendor: row.vendorName, partNumber: row.partNumber, strictCandidateCount: candidates },
robotName: "verify:research-status",
confidence: 0.9,
});
if (await checkAndSetFullyVerified(row.id)) fullyEarned++;
noMatch++;
} else {
const reason = "research_status_resolver_found_plausible_candidates_but_no_safe_deterministic_match";
await pool.query(
`UPDATE transceivers
SET competitor_status = 'ambiguous',
competitor_status_updated_at = NOW(),
updated_at = NOW()
WHERE id = $1`,
[row.id],
);
await recordVerificationEvidence({
transceiverId: row.id,
verificationType: "competitor_ambiguous",
evidenceValue: { reason, vendor: row.vendorName, partNumber: row.partNumber, strictCandidateCount: candidates },
robotName: "verify:research-status",
confidence: 0.85,
});
ambiguous++;
}
}
}
console.log("Research status resolver complete", { imageClosed, detailsClosed, noMatch, ambiguous, fullyEarned, apply });
}
if (require.main === module) {
main()
.then(() => pool.end())
.catch((err) => {
console.error("Fatal:", err);
pool.end();
process.exit(1);
});
}

109
sql/106-research-resolution-status.sql Normal file
View File

@ -0,0 +1,109 @@
-- Migration 106: Explicit image/details research resolution state
--
-- image_verified/details_verified stay strict: true only when source-backed
-- evidence exists. These status columns let TIP close research loops when a
-- public source does not expose an image or enough details, without inventing
-- data or abusing verified booleans.
ALTER TABLE transceivers
ADD COLUMN IF NOT EXISTS image_status VARCHAR(32) NOT NULL DEFAULT 'unknown',
ADD COLUMN IF NOT EXISTS image_status_updated_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS image_unavailable_verified_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS image_unavailable_reason TEXT,
ADD COLUMN IF NOT EXISTS details_status VARCHAR(32) NOT NULL DEFAULT 'unknown',
ADD COLUMN IF NOT EXISTS details_status_updated_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS details_unavailable_verified_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS details_unavailable_reason TEXT;
DO $$
BEGIN
IF EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = 'transceivers_image_status_check'
) THEN
ALTER TABLE transceivers DROP CONSTRAINT transceivers_image_status_check;
END IF;
ALTER TABLE transceivers
ADD CONSTRAINT transceivers_image_status_check
CHECK (image_status IN (
'unknown',
'public_image',
'no_public_image',
'needs_research',
'ambiguous'
));
IF EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = 'transceivers_details_status_check'
) THEN
ALTER TABLE transceivers DROP CONSTRAINT transceivers_details_status_check;
END IF;
ALTER TABLE transceivers
ADD CONSTRAINT transceivers_details_status_check
CHECK (details_status IN (
'unknown',
'public_details',
'no_public_details',
'needs_research',
'ambiguous'
));
END $$;
UPDATE transceivers
SET image_status = CASE
WHEN image_verified = true THEN 'public_image'
WHEN image_status = 'unknown' THEN 'needs_research'
ELSE image_status
END,
image_status_updated_at = COALESCE(image_status_updated_at, NOW())
WHERE image_status IS NULL
OR image_status = 'unknown'
OR image_verified = true;
UPDATE transceivers
SET details_status = CASE
WHEN details_verified = true THEN 'public_details'
WHEN details_status = 'unknown' THEN 'needs_research'
ELSE details_status
END,
details_status_updated_at = COALESCE(details_status_updated_at, NOW())
WHERE details_status IS NULL
OR details_status = 'unknown'
OR details_verified = true;
CREATE INDEX IF NOT EXISTS idx_transceivers_image_status
ON transceivers (image_status);
CREATE INDEX IF NOT EXISTS idx_transceivers_details_status
ON transceivers (details_status);
ALTER TABLE transceiver_verification_evidence
DROP CONSTRAINT IF EXISTS transceiver_verification_evidence_verification_type_check;
ALTER TABLE transceiver_verification_evidence
ADD CONSTRAINT transceiver_verification_evidence_verification_type_check
CHECK (
verification_type::text = ANY (
ARRAY[
'price',
'price_unavailable',
'image',
'image_unavailable',
'details',
'details_unavailable',
'competitor_match',
'competitor_no_match',
'competitor_ambiguous',
'artifact_quarantine'
]
)
);
COMMENT ON COLUMN transceivers.image_status IS
'Resolution state for image evidence: public_image, no_public_image, needs_research, ambiguous, unknown. image_verified remains true only for source-backed product images.';
COMMENT ON COLUMN transceivers.details_status IS
'Resolution state for detail evidence: public_details, no_public_details, needs_research, ambiguous, unknown. details_verified remains true only for source-backed product details.';

36
sync/CURRENT.md
View File

@ -1,9 +1,43 @@
# Current TIP Sync State
Updated: 2026-05-10 07:54 UTC
Updated: 2026-05-10 08:10 UTC
## Newest Work
- TIP research-resolution closure on 2026-05-10 UTC:
- added explicit image/detail research status model:
- `sql/106-research-resolution-status.sql`
- `image_status`: `public_image`, `no_public_image`, `needs_research`, `ambiguous`, `unknown`
- `details_status`: `public_details`, `no_public_details`, `needs_research`, `ambiguous`, `unknown`
- new evidence types: `image_unavailable`, `details_unavailable`
- added `verify:research-status`:
- closes missing public images/details as unavailable states with evidence, not fake verified booleans
- resolves open competitor state as `no_valid_match` or `ambiguous` using strict deterministic candidate checks
- supports dry-run, vendor filter and limited serial batches for safe Erik operation
- Health API now exposes:
- image/detail status buckets
- `research_resolved` and `research_resolved_pct`
- live deployment:
- migration applied on Erik
- API rebuilt and restarted
- scraper package rebuilt after resolver SQL fix
- scraper daemon was not restarted while pg-boss pricing jobs were active
- final live active-base health:
- active products: `16236`
- price: `public_price=10851`, `no_public_price=5385`, `needs_research=0`, `ambiguous=0`
- image: `public_image=11602`, `no_public_image=4634`, `needs_research=0`, `ambiguous=0`
- details: `public_details=16005`, `no_public_details=231`, `needs_research=0`, `ambiguous=0`
- competitor: `matched=10838`, `no_valid_match=561`, `ambiguous=4837`, `needs_research=0`
- research resolved: `16236 / 16236 = 100%`
- strict fully verified: `10600 / 16236 = 65%`
- interpretation:
- all active TIP product research queues are closed
- strict green `fully_verified` remains source-backed and is intentionally lower because no-public-image/details and ambiguous competitor states are not fake source verification
- TIPLLM training pool updated with:
- research-resolution status model
- safe resolver runbook
- final live result and metric interpretation
- MAGATAMA LLM training-pool research expansion on 2026-05-10 UTC:
- added a new curated external source ingest file in Magatama:
- `training-data/model-registry/external-ingest/llm-lane-research-seeds-2026-05-10.jsonl`

82
sync/history/2026-05-10-tip-research-resolution-closure.md Normal file
View File

@ -0,0 +1,82 @@
# TIP Research Resolution Closure
Date: 2026-05-10 UTC
Owner: Codex
## Summary
TIP now distinguishes strict source-backed verification from completed research where public data is not available.
This avoids fake green checks while still letting the active research queue reach a real terminal state.
## Implemented
- Added migration `sql/106-research-resolution-status.sql`
- `image_status`
- `details_status`
- `image_unavailable` evidence
- `details_unavailable` evidence
- Updated scraper DB helpers:
- source-backed images set `image_status=public_image`
- source-backed details set `details_status=public_details`
- Added `verify:research-status`
- dry-run by default
- limited serial batches via `RESEARCH_STATUS_LIMIT`
- optional vendor filter via `RESEARCH_STATUS_VENDOR`
- apply mode via `RESEARCH_STATUS_APPLY=1`
- Updated Health API:
- image/detail status buckets
- `research_resolved`
- `research_resolved_pct`
## Live Run
- Migration applied on Erik.
- API build passed and `tip-api` restarted.
- Scraper build passed after fixing a PostgreSQL parameter bug in the resolver.
- Scraper daemon was not restarted while pg-boss pricing jobs were active.
- Resolver dry-run after closure returned `count: 0`.
## Final Health
- Active products: `16236`
- Price:
- `public_price=10851`
- `no_public_price=5385`
- `needs_research=0`
- `ambiguous=0`
- Image:
- `public_image=11602`
- `no_public_image=4634`
- `needs_research=0`
- `ambiguous=0`
- Details:
- `public_details=16005`
- `no_public_details=231`
- `needs_research=0`
- `ambiguous=0`
- Competitor:
- `matched=10838`
- `no_valid_match=561`
- `ambiguous=4837`
- `needs_research=0`
- Research resolved:
- `16236 / 16236`
- `100%`
- Strict fully verified:
- `10600 / 16236`
- `65%`
## Interpretation
All active TIP research queues are now closed.
`fully_verified` remains strict and source-backed. Products with `no_public_image`, `no_public_details`, or `competitor_status=ambiguous` are research-complete but not fake fully verified.
## TIPLLM Training Pool
Added three training examples to `training-data/tip-llm-capabilities-v1.jsonl`:
- research-resolution status model
- safe resolver runbook
- final live result and metric interpretation

3
training-data/tip-llm-capabilities-v1.jsonl
View File

@ -59,3 +59,6 @@
{"id":"tip-20260510-cable-breakout-quarantine-001","source":"codex-tip-verification-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Do not compare cable/breakout products as if they were standalone optical transceiver modules."},{"role":"user","content":"How should TIP handle DAC, AOC, Twinax, direct-attach and breakout rows in the active transceiver verification base?"},{"role":"assistant","content":"Move them out of the active transceiver-module base unless TIP intentionally models a separate cable/AOC product class. Deterministic indicators include part numbers or URLs containing DAC, AOC, DACBL, XDAC, Direct Attach, Twinax, Breakout or active-optical. These rows should not block transceiver-module completion and must not be used for 1:1 optical module competitor matching. Quarantine them with artifact_quarantine evidence and clear price/image/details/competitor/fully verified flags."}]}
{"id":"tip-20260510-10gtek-datasheet-verifier-001","source":"codex-tip-verification-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Use official datasheet indexes for source-backed detail evidence when retail pages are absent."},{"role":"user","content":"10Gtek has active rows with no SFPcables retail URL but the main 10gtek.com site exposes PDF datasheets. What should the robot do?"},{"role":"assistant","content":"Fetch the official 10Gtek transceiver page, extract PDF datasheet links, normalize PDF filenames and part numbers, and only match when the normalized filename and part number contain each other or are exact. Write the PDF URL to product_page_url/details_source_url and mark details only if form_factor, speed_gbps, reach_label and fiber_type are already present. Record details evidence with robot_name `verify:10gtek-datasheets`. Do not infer prices from PDFs."}]}
{"id":"tip-20260510-gaotek-artifact-quarantine-001","source":"codex-tip-verification-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. GAO Tek search/catalog pages can contain many non-optical or family-level rows."},{"role":"user","content":"Which GAO Tek rows should TIP quarantine from the optical transceiver active base?"},{"role":"assistant","content":"Quarantine obvious non-optic and family/category artifacts such as Handheld, Wireless, Marine, Mobile, Dual-Band, All-Mode-Portable, Cabling, Splicers, Other-Transceivers, GBIC-Transceivers, QSFP-Transceivers, SFP-Transceivers, SFP+-Transceivers, Fiber-Transceivers and Copper-SFP-Transceivers. These are category/family or radio/cabling products, not source-backed optical transceiver module SKUs. Keep narrower GAO optical rows active only when they provide enough product evidence."}]}
{"id":"tip-20260510-research-resolution-status-model-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Close research loops with explicit states and evidence; never turn missing public data into fake verification."},{"role":"user","content":"How should TIP represent a product when public price, image, details, or competitor research has been exhausted but no source-backed value exists?"},{"role":"assistant","content":"Keep strict verification booleans source-backed only. Use status columns to represent resolved research states: `price_status=public_price|no_public_price|ambiguous|needs_research`, `image_status=public_image|no_public_image|ambiguous|needs_research`, `details_status=public_details|no_public_details|ambiguous|needs_research`, and `competitor_status=matched|no_valid_match|ambiguous|needs_research`. Write evidence for unavailable states (`price_unavailable`, `image_unavailable`, `details_unavailable`) or competitor outcomes. Count `research_resolved` when all four domains are in a terminal state, even if `fully_verified` remains false because public evidence was unavailable."}]}
{"id":"tip-20260510-research-status-resolver-runbook-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Operate TIP robots on Erik conservatively and in auditable batches."},{"role":"user","content":"What is the safe runbook for closing remaining image/details/competitor needs_research states?"},{"role":"assistant","content":"Run `verify:research-status` dry-run first with a small `RESEARCH_STATUS_LIMIT`. Apply in serial batches with `RESEARCH_STATUS_APPLY=1`, checking `/api/health` between batches. Do not restart `tip-scraper-daemon` while pg-boss jobs are active. For missing images/details, record `no_public_image` or `no_public_details` with source-aware unavailable evidence instead of marking verified. For competitors, use strict form factor, speed, fiber and reach matching; set `no_valid_match` if no resolved candidate exists, or `ambiguous` if candidates exist but no deterministic 1:1 match is safe."}]}
{"id":"tip-20260510-research-resolution-live-result-001","source":"codex-tip-research-resolution-2026-05-10","kind":"sft-jsonl","messages":[{"role":"system","content":"You are TIP_LLM. Use live health metrics to distinguish strict verification from completed research."},{"role":"user","content":"What was the TIP research-resolution closure result on 2026-05-10?"},{"role":"assistant","content":"After migration `106-research-resolution-status.sql` and batched `verify:research-status` runs, active TIP research was closed for `16236 / 16236` products (`research_resolved_pct=100`). Final active-base buckets: price `public_price=10851`, `no_public_price=5385`, `needs_research=0`; image `public_image=11602`, `no_public_image=4634`, `needs_research=0`; details `public_details=16005`, `no_public_details=231`, `needs_research=0`; competitor `matched=10838`, `no_valid_match=561`, `ambiguous=4837`, `needs_research=0`. Strict `fully_verified` remained `10600` because no-public-image/details and ambiguous competitor states are not fake green checks."}]}