diff --git a/packages/api/src/routes/finder.ts b/packages/api/src/routes/finder.ts index ca5a0a7..9ec9c76 100644 --- a/packages/api/src/routes/finder.ts +++ b/packages/api/src/routes/finder.ts @@ -30,6 +30,10 @@ finderRouter.get("/", async (req, res) => { const switchResult = await pool.query( `SELECT sw.id, sw.model, sw.series, sw.ports_config, sw.max_speed_gbps, + sw.system_type, sw.is_linecard, sw.chassis_model, sw.slot_type, + sw.flexbox_compat_mode, sw.flexbox_notes, sw.description, + sw.switching_capacity_tbps, sw.total_ports, sw.category, + sw.lifecycle_status, sw.features, sw.use_cases, v.name AS vendor_name, sw.image_url, sw.datasheet_r2_key FROM switches sw JOIN vendors v ON sw.vendor_id = v.id @@ -144,15 +148,53 @@ finderRouter.get("/", async (req, res) => { // Step 4: Extract port types from switch for "what can this switch accept?" const portTypes = sw.ports_config || {}; + // Linecard system: fetch sibling linecards if this is a chassis or linecard + let linecards: any[] = []; + if (sw.is_linecard && sw.chassis_model) { + const siblingResult = await pool.query( + `SELECT sw2.model, sw2.total_ports, sw2.max_speed_gbps, sw2.switching_capacity_tbps, sw2.features, sw2.slot_type + FROM switches sw2 + WHERE sw2.chassis_model = $1 AND sw2.id != $2 AND sw2.is_linecard = true + ORDER BY sw2.model LIMIT 20`, + [sw.chassis_model, sw.id] + ); + linecards = siblingResult.rows; + } else if (!sw.is_linecard && sw.system_type === "modular") { + const lcResult = await pool.query( + `SELECT sw2.model, sw2.total_ports, sw2.max_speed_gbps, sw2.switching_capacity_tbps, sw2.features, sw2.slot_type + FROM switches sw2 + WHERE sw2.chassis_model = $1 AND sw2.is_linecard = true + ORDER BY sw2.model LIMIT 20`, + [sw.series] + ); + linecards = lcResult.rows; + } + res.json({ switch: { id: sw.id, model: sw.model, series: sw.series, vendor: sw.vendor_name, + category: sw.category, + description: sw.description, max_speed_gbps: sw.max_speed_gbps, + switching_capacity_tbps: sw.switching_capacity_tbps ? parseFloat(sw.switching_capacity_tbps) : null, + total_ports: sw.total_ports, + features: sw.features || [], + use_cases: sw.use_cases || [], + lifecycle_status: sw.lifecycle_status, ports: portTypes, image_url: sw.image_url, + // Linecard / Modular system info + system_type: sw.system_type || "fixed", + is_linecard: sw.is_linecard === true, + chassis_model: sw.chassis_model || null, + slot_type: sw.slot_type || null, + linecards: linecards, + // Flexbox programming info + flexbox_compat_mode: sw.flexbox_compat_mode || null, + flexbox_notes: sw.flexbox_notes || null, }, compatible_transceivers: compatResult.rows.map(r => ({ id: r.id, diff --git a/packages/api/src/routes/transceivers.ts b/packages/api/src/routes/transceivers.ts index cd4b3a2..fd40107 100644 --- a/packages/api/src/routes/transceivers.ts +++ b/packages/api/src/routes/transceivers.ts @@ -40,7 +40,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => { return; } - // Latest price per source vendor — last 30 days only + // Latest price per source vendor — last 30 days, exclude anomalous prices const pricesResult = await pool.query( `SELECT DISTINCT ON (po.source_vendor_id) po.price, po.currency, po.url, po.time, po.stock_level, po.is_verified, @@ -49,6 +49,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => { JOIN vendors v ON po.source_vendor_id = v.id WHERE po.transceiver_id = $1 AND po.time > NOW() - INTERVAL '30 days' + AND COALESCE(po.is_anomalous, false) = false ORDER BY po.source_vendor_id, po.time DESC`, [transceiver.id] ); @@ -93,6 +94,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => { AND po.time > NOW() - INTERVAL '30 days' AND po.price > 0 AND po.url IS NOT NULL + AND COALESCE(po.is_anomalous, false) = false -- Exclude vendors that already appear in direct prices AND sv.id NOT IN ( SELECT source_vendor_id FROM price_observations diff --git a/packages/scraper/src/scrapers/skylane.ts b/packages/scraper/src/scrapers/skylane.ts index 2576880..c8efdde 100644 --- a/packages/scraper/src/scrapers/skylane.ts +++ b/packages/scraper/src/scrapers/skylane.ts @@ -279,7 +279,7 @@ export async function scrapeSkylane(): Promise { }); if (product.price && product.price > 0) { - const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const hash = contentHash({ price: product.price, part: product.partNumber }); const updated = await upsertPriceObservation({ transceiverId: txId, sourceVendorId: vendorId, diff --git a/packages/scraper/src/scrapers/tscom.ts b/packages/scraper/src/scrapers/tscom.ts index 79e59d8..7301328 100644 --- a/packages/scraper/src/scrapers/tscom.ts +++ b/packages/scraper/src/scrapers/tscom.ts @@ -232,7 +232,7 @@ export async function scrapeTsCom(): Promise { }); if (product.price && product.price > 0) { - const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber })); + const hash = contentHash({ price: product.price, part: product.partNumber }); const updated = await upsertPriceObservation({ transceiverId: txId, sourceVendorId: vendorId, diff --git a/packages/scraper/src/scrapers/wiitek.ts b/packages/scraper/src/scrapers/wiitek.ts index 82edec3..1eabe61 100644 --- a/packages/scraper/src/scrapers/wiitek.ts +++ b/packages/scraper/src/scrapers/wiitek.ts @@ -67,16 +67,15 @@ export async function scrapeWiitek(): Promise { if (price <= 0) continue; try { - const t = await findOrCreateScrapedTransceiver({ - partNumber, vendorId, formFactor: cat.form_factor, name, - url: href.startsWith("http") ? href : `${BASE}${href}`, + const transceiverId = await findOrCreateScrapedTransceiver({ + partNumber, vendorId, formFactor: cat.form_factor, }); const isNew = await upsertPriceObservation({ - transceiverId: t.id, sourceVendorId: vendorId, + transceiverId, sourceVendorId: vendorId, price, currency: currency || "USD", stockLevel: "unknown", url: href.startsWith("http") ? href : `${BASE}${href}`, - contentHash: contentHash(`${partNumber}:${price}:${currency}`), + contentHash: contentHash({ partNumber, price, currency: currency || "USD" }), }); if (isNew) newItems++; total++; diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index 14ad53f..5da175e 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -18,6 +18,39 @@ export const pool = new Pool({ // Alias — some scrapers import { db } instead of { pool } export const db = pool; +// Per-form-factor price bounds [min, max] in USD equivalent +const PRICE_BOUNDS: Record = { + "SFP": [2, 3000], + "SFP+": [4, 5000], + "SFP28": [10, 8000], + "SFP56": [20, 10000], + "SFP-DD": [30, 12000], + "QSFP+": [15, 6000], + "QSFP28": [20, 10000], + "QSFP56": [50, 15000], + "QSFP-DD": [60, 20000], + "QSFP112": [80, 25000], + "OSFP": [100, 35000], + "OSFP112": [150, 40000], + "OSFP224": [200, 60000], + "CFP": [100, 30000], + "CFP2": [100, 30000], + "XFP": [10, 5000], + "GBIC": [2, 2000], +}; + +async function isPriceAnomalous(transceiverId: string, priceUsd: number): Promise { + const row = await pool.query( + `SELECT form_factor FROM transceivers WHERE id = $1`, + [transceiverId] + ); + const formFactor = row.rows[0]?.form_factor as string | undefined; + if (!formFactor) return false; + const bounds = PRICE_BOUNDS[formFactor]; + if (!bounds) return false; + return priceUsd < bounds[0] || priceUsd > bounds[1]; +} + export async function upsertPriceObservation(params: { transceiverId: string; sourceVendorId: string; @@ -29,6 +62,16 @@ export async function upsertPriceObservation(params: { url?: string; contentHash: string; }): Promise { + // Normalize price to USD for sanity check (rough conversion) + const priceUsd = params.currency === "EUR" ? params.price * 1.09 + : params.currency === "GBP" ? params.price * 1.27 + : params.price; + + const anomalous = await isPriceAnomalous(params.transceiverId, priceUsd); + if (anomalous) { + return false; // Reject price outside form-factor bounds + } + // Check if price changed via content hash const existing = await pool.query( `SELECT content_hash FROM price_observations @@ -37,10 +80,20 @@ export async function upsertPriceObservation(params: { [params.transceiverId, params.sourceVendorId] ); + // Check if vendor is a competitor (non-Flexoptix) for competitor_verified flag + const vendorRow = await pool.query( + `SELECT is_competitor FROM vendors WHERE id = $1`, + [params.sourceVendorId] + ); + const isCompetitor = vendorRow.rows[0]?.is_competitor === true; + if (existing.rows.length > 0 && existing.rows[0].content_hash === params.contentHash) { - // Price unchanged — but still ensure price_verified is set (in case it wasn't before) + // Price unchanged — still ensure verified flags are current await pool.query( - `UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`, + `UPDATE transceivers SET + price_verified = true + ${isCompetitor ? ", competitor_verified = true, competitor_verified_at = COALESCE(competitor_verified_at, NOW())" : ""} + WHERE id = $1 AND (price_verified IS NULL OR price_verified = false OR ${isCompetitor ? "competitor_verified IS NULL OR competitor_verified = false" : "false"})`, [params.transceiverId] ); return false; // No change @@ -61,11 +114,23 @@ export async function upsertPriceObservation(params: { params.contentHash, ] ); - // Mark the transceiver as price-verified whenever we successfully record a price - await pool.query( - `UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`, - [params.transceiverId] - ); + + // Mark price_verified always; competitor_verified only for non-Flexoptix vendors + if (isCompetitor) { + await pool.query( + `UPDATE transceivers SET + price_verified = true, + competitor_verified = true, + competitor_verified_at = COALESCE(competitor_verified_at, NOW()) + WHERE id = $1`, + [params.transceiverId] + ); + } else { + await pool.query( + `UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`, + [params.transceiverId] + ); + } return true; // New observation written } diff --git a/packages/scraper/src/utils/hash.ts b/packages/scraper/src/utils/hash.ts index a3d3f8a..17faa33 100644 --- a/packages/scraper/src/utils/hash.ts +++ b/packages/scraper/src/utils/hash.ts @@ -2,10 +2,12 @@ import { createHash } from "crypto"; /** * Generate SHA-256 content hash for change detection. - * Only hashes the fields that matter (price, stock, quantity). + * Accepts an object (preferred) or a plain string (legacy scrapers). */ -export function contentHash(data: Record): string { - const normalized = JSON.stringify(data, Object.keys(data).sort()); +export function contentHash(data: Record | string): string { + const normalized = typeof data === "string" + ? data + : JSON.stringify(data, Object.keys(data).sort()); return createHash("sha256").update(normalized).digest("hex").slice(0, 16); } diff --git a/scripts/migrate.ts b/scripts/migrate.ts index 6e54317..268ce90 100644 --- a/scripts/migrate.ts +++ b/scripts/migrate.ts @@ -19,16 +19,50 @@ async function migrate(): Promise { .filter((f) => f.endsWith(".sql")) .sort(); - console.log(`Found ${files.length} migration files`); - const client = await pool.connect(); try { - for (const file of files) { + // Create migration tracker if it doesn't exist + await client.query(` + CREATE TABLE IF NOT EXISTS _migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `); + + // Get already-applied migrations + const { rows } = await client.query( + "SELECT filename FROM _migrations ORDER BY filename" + ); + const applied = new Set(rows.map((r: { filename: string }) => r.filename)); + + const pending = files.filter((f) => !applied.has(f)); + if (pending.length === 0) { + console.log("All migrations already applied. Nothing to do."); + return; + } + + console.log( + `${applied.size} already applied, running ${pending.length} pending migrations` + ); + + for (const file of pending) { const sql = readFileSync(join(sqlDir, file), "utf-8"); console.log(`Running: ${file}...`); - await client.query(sql); - console.log(` Done: ${file}`); + await client.query("BEGIN"); + try { + await client.query(sql); + await client.query( + "INSERT INTO _migrations (filename) VALUES ($1)", + [file] + ); + await client.query("COMMIT"); + console.log(` Done: ${file}`); + } catch (err) { + await client.query("ROLLBACK"); + throw err; + } } + console.log("\nAll migrations completed successfully."); } catch (err) { console.error("Migration failed:", err);