From 71d6c20b5e1f96071f5a114936e8e93b26a3ea50 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Sat, 4 Apr 2026 07:14:26 +0200 Subject: [PATCH] fix: auto-set image_verified and price_verified in db utils - findOrCreateScrapedTransceiver now sets image_verified=true when writing image_url - upsertPriceObservation now sets price_verified=true on the transceiver after inserting price - Both INSERT and UPDATE paths covered for image_verified sync - Eliminates need for manual backfill after scraper runs --- packages/scraper/src/utils/db.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index d8c358a..8db97e7 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -56,6 +56,11 @@ export async function upsertPriceObservation(params: { params.contentHash, ] ); + // Mark the transceiver as price-verified whenever we successfully record a price + await pool.query( + `UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`, + [params.transceiverId] + ); return true; // New observation written } @@ -79,10 +84,10 @@ export async function findOrCreateScrapedTransceiver(params: { ); if (existing.rows.length > 0) { - // Update image_url if we have one and the record doesn't yet + // Update image_url and image_verified if we have a new image for a record without one if (params.imageUrl && !existing.rows[0].image_url) { await pool.query( - `UPDATE transceivers SET image_url = $1, updated_at = NOW() WHERE id = $2`, + `UPDATE transceivers SET image_url = $1, image_verified = true, updated_at = NOW() WHERE id = $2`, [params.imageUrl, existing.rows[0].id] ); } @@ -92,9 +97,9 @@ export async function findOrCreateScrapedTransceiver(params: { // Create new transceiver entry const slug = `scraped-${params.partNumber.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; const result = await pool.query( - `INSERT INTO transceivers (slug, part_number, vendor_id, form_factor, speed_gbps, speed, reach_meters, reach_label, fiber_type, wavelengths, category, market_status, image_url) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'Mainstream', $12) - ON CONFLICT (slug) DO UPDATE SET image_url = COALESCE(transceivers.image_url, EXCLUDED.image_url), updated_at = NOW() + `INSERT INTO transceivers (slug, part_number, vendor_id, form_factor, speed_gbps, speed, reach_meters, reach_label, fiber_type, wavelengths, category, market_status, image_url, image_verified) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'Mainstream', $12, $13) + ON CONFLICT (slug) DO UPDATE SET image_url = COALESCE(transceivers.image_url, EXCLUDED.image_url), image_verified = COALESCE(transceivers.image_verified, EXCLUDED.image_verified), updated_at = NOW() RETURNING id`, [ slug, @@ -109,6 +114,7 @@ export async function findOrCreateScrapedTransceiver(params: { params.wavelengths || "", params.category || "DataCenter", params.imageUrl || null, + params.imageUrl ? true : null, ] ); return result.rows[0].id;