rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: auto-set image_verified and price_verified in db utils

Browse Source 
- findOrCreateScrapedTransceiver now sets image_verified=true when writing image_url
- upsertPriceObservation now sets price_verified=true on the transceiver after inserting price
- Both INSERT and UPDATE paths covered for image_verified sync
- Eliminates need for manual backfill after scraper runs
This commit is contained in:
Rene Fichtmueller 2026-04-04 07:14:26 +02:00
parent 2913ad451b
commit c179b236d7
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
packages/scraper/src/utils/db.ts
View File

@ -56,6 +56,11 @@ export async function upsertPriceObservation(params: {
params.contentHash, params.contentHash,
] ]
); );
// Mark the transceiver as price-verified whenever we successfully record a price
await pool.query(
`UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`,
[params.transceiverId]
);
return true; // New observation written return true; // New observation written
} }
@ -79,10 +84,10 @@ export async function findOrCreateScrapedTransceiver(params: {
); );
if (existing.rows.length > 0) { if (existing.rows.length > 0) {
// Update image_url if we have one and the record doesn't yet // Update image_url and image_verified if we have a new image for a record without one
if (params.imageUrl && !existing.rows[0].image_url) { if (params.imageUrl && !existing.rows[0].image_url) {
await pool.query( await pool.query(
`UPDATE transceivers SET image_url = $1, updated_at = NOW() WHERE id = $2`, `UPDATE transceivers SET image_url = $1, image_verified = true, updated_at = NOW() WHERE id = $2`,
[params.imageUrl, existing.rows[0].id] [params.imageUrl, existing.rows[0].id]
); );
} }
@ -92,9 +97,9 @@ export async function findOrCreateScrapedTransceiver(params: {
// Create new transceiver entry // Create new transceiver entry
const slug = `scraped-${params.partNumber.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; const slug = `scraped-${params.partNumber.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
const result = await pool.query( const result = await pool.query(
`INSERT INTO transceivers (slug, part_number, vendor_id, form_factor, speed_gbps, speed, reach_meters, reach_label, fiber_type, wavelengths, category, market_status, image_url) `INSERT INTO transceivers (slug, part_number, vendor_id, form_factor, speed_gbps, speed, reach_meters, reach_label, fiber_type, wavelengths, category, market_status, image_url, image_verified)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'Mainstream', $12) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'Mainstream', $12, $13)
ON CONFLICT (slug) DO UPDATE SET image_url = COALESCE(transceivers.image_url, EXCLUDED.image_url), updated_at = NOW() ON CONFLICT (slug) DO UPDATE SET image_url = COALESCE(transceivers.image_url, EXCLUDED.image_url), image_verified = COALESCE(transceivers.image_verified, EXCLUDED.image_verified), updated_at = NOW()
RETURNING id`, RETURNING id`,
[ [
slug, slug,
@ -109,6 +114,7 @@ export async function findOrCreateScrapedTransceiver(params: {
params.wavelengths || "", params.wavelengths || "",
params.category || "DataCenter", params.category || "DataCenter",
params.imageUrl || null, params.imageUrl || null,
params.imageUrl ? true : null,
] ]
); );
return result.rows[0].id; return result.rows[0].id;