rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: linecard system support, Cisco 8000 accuracy, price anomaly detection

Browse Source 
API/finder:
- Add modular chassis support: sibling linecards fetched when is_linecard=true
- Add chassis linecards when system_type=modular
- Extend switch response: system_type, is_linecard, chassis_model, slot_type,
  flexbox_compat_mode, flexbox_notes, description, switching_capacity_tbps,
  total_ports, category, lifecycle_status, features, use_cases, linecards[]

API/transceivers:
- Filter price_observations with COALESCE(is_anomalous, false) = false
  (direct prices + comparable market prices)

Scraper/db:
- Add PRICE_BOUNDS map (per form-factor min/max USD sanity bounds)
- Add isPriceAnomalous() — marks DB price_observations as is_anomalous=true
- Add competitor_verified flag: set true when valid competitor price stored
- upsertPriceObservation: skip prices outside sanity bounds, set competitor_verified

Scraper/hash:
- contentHash() now accepts Record<string,unknown> | string (union type)
  to support both structured objects and legacy string callers

Scrapers (skylane, tscom, wiitek):
- Fix contentHash() call signature: pass objects not JSON.stringify strings
- Fix wiitek: remove invalid 'name' param, fix t.id → transceiverId

Migrations:
- Add is_anomalous, competitor_verified, competitor_verified_at,
  image_primary columns
- Recreate sync_fully_verified trigger to include competitor_verified
- Add is_linecard, chassis_model, system_type, slot_type,
  flexbox_compat_mode, flexbox_notes to switches table
This commit is contained in:
Rene Fichtmueller 2026-04-09 09:06:22 +02:00
parent 240e7f46f2
commit cf75eee8ad
8 changed files with 167 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
packages/api/src/routes/finder.ts
View File

@ -30,6 +30,10 @@ finderRouter.get("/", async (req, res) => {
const switchResult = await pool.query(
`SELECT sw.id, sw.model, sw.series, sw.ports_config, sw.max_speed_gbps,
sw.system_type, sw.is_linecard, sw.chassis_model, sw.slot_type,
sw.flexbox_compat_mode, sw.flexbox_notes, sw.description,
sw.switching_capacity_tbps, sw.total_ports, sw.category,
sw.lifecycle_status, sw.features, sw.use_cases,
v.name AS vendor_name, sw.image_url, sw.datasheet_r2_key
FROM switches sw
JOIN vendors v ON sw.vendor_id = v.id
@ -144,15 +148,53 @@ finderRouter.get("/", async (req, res) => {
// Step 4: Extract port types from switch for "what can this switch accept?"
const portTypes = sw.ports_config || {};
// Linecard system: fetch sibling linecards if this is a chassis or linecard
let linecards: any[] = [];
if (sw.is_linecard && sw.chassis_model) {
const siblingResult = await pool.query(
`SELECT sw2.model, sw2.total_ports, sw2.max_speed_gbps, sw2.switching_capacity_tbps, sw2.features, sw2.slot_type
FROM switches sw2
WHERE sw2.chassis_model = $1 AND sw2.id != $2 AND sw2.is_linecard = true
ORDER BY sw2.model LIMIT 20`,
[sw.chassis_model, sw.id]
);
linecards = siblingResult.rows;
} else if (!sw.is_linecard && sw.system_type === "modular") {
const lcResult = await pool.query(
`SELECT sw2.model, sw2.total_ports, sw2.max_speed_gbps, sw2.switching_capacity_tbps, sw2.features, sw2.slot_type
FROM switches sw2
WHERE sw2.chassis_model = $1 AND sw2.is_linecard = true
ORDER BY sw2.model LIMIT 20`,
[sw.series]
);
linecards = lcResult.rows;
}
res.json({
switch: {
id: sw.id,
model: sw.model,
series: sw.series,
vendor: sw.vendor_name,
category: sw.category,
description: sw.description,
max_speed_gbps: sw.max_speed_gbps,
switching_capacity_tbps: sw.switching_capacity_tbps ? parseFloat(sw.switching_capacity_tbps) : null,
total_ports: sw.total_ports,
features: sw.features || [],
use_cases: sw.use_cases || [],
lifecycle_status: sw.lifecycle_status,
ports: portTypes,
image_url: sw.image_url,
// Linecard / Modular system info
system_type: sw.system_type || "fixed",
is_linecard: sw.is_linecard === true,
chassis_model: sw.chassis_model || null,
slot_type: sw.slot_type || null,
linecards: linecards,
// Flexbox programming info
flexbox_compat_mode: sw.flexbox_compat_mode || null,
flexbox_notes: sw.flexbox_notes || null,
},
compatible_transceivers: compatResult.rows.map(r => ({
id: r.id,

4
packages/api/src/routes/transceivers.ts
View File

@ -40,7 +40,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => {
return;
}
// Latest price per source vendor — last 30 days only
// Latest price per source vendor — last 30 days, exclude anomalous prices
const pricesResult = await pool.query(
`SELECT DISTINCT ON (po.source_vendor_id)
po.price, po.currency, po.url, po.time, po.stock_level, po.is_verified,
@ -49,6 +49,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => {
JOIN vendors v ON po.source_vendor_id = v.id
WHERE po.transceiver_id = $1
AND po.time > NOW() - INTERVAL '30 days'
AND COALESCE(po.is_anomalous, false) = false
ORDER BY po.source_vendor_id, po.time DESC`,
[transceiver.id]
);
@ -93,6 +94,7 @@ transceiverRouter.get("/:id", async (req: Request, res: Response) => {
AND po.time > NOW() - INTERVAL '30 days'
AND po.price > 0
AND po.url IS NOT NULL
AND COALESCE(po.is_anomalous, false) = false
-- Exclude vendors that already appear in direct prices
AND sv.id NOT IN (
SELECT source_vendor_id FROM price_observations

2
packages/scraper/src/scrapers/skylane.ts
View File

@ -279,7 +279,7 @@ export async function scrapeSkylane(): Promise<void> {
});
if (product.price && product.price > 0) {
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
const hash = contentHash({ price: product.price, part: product.partNumber });
const updated = await upsertPriceObservation({
transceiverId: txId,
sourceVendorId: vendorId,

2
packages/scraper/src/scrapers/tscom.ts
View File

@ -232,7 +232,7 @@ export async function scrapeTsCom(): Promise<void> {
});
if (product.price && product.price > 0) {
const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
const hash = contentHash({ price: product.price, part: product.partNumber });
const updated = await upsertPriceObservation({
transceiverId: txId,
sourceVendorId: vendorId,

9
packages/scraper/src/scrapers/wiitek.ts
View File

@ -67,16 +67,15 @@ export async function scrapeWiitek(): Promise<void> {
if (price <= 0) continue;
try {
const t = await findOrCreateScrapedTransceiver({
partNumber, vendorId, formFactor: cat.form_factor, name,
url: href.startsWith("http") ? href : `${BASE}${href}`,
const transceiverId = await findOrCreateScrapedTransceiver({
partNumber, vendorId, formFactor: cat.form_factor,
});
const isNew = await upsertPriceObservation({
transceiverId: t.id, sourceVendorId: vendorId,
transceiverId, sourceVendorId: vendorId,
price, currency: currency || "USD",
stockLevel: "unknown",
url: href.startsWith("http") ? href : `${BASE}${href}`,
contentHash: contentHash(`${partNumber}:${price}:${currency}`),
contentHash: contentHash({ partNumber, price, currency: currency || "USD" }),
});
if (isNew) newItems++;
total++;

79
packages/scraper/src/utils/db.ts
View File

@ -18,6 +18,39 @@ export const pool = new Pool({
// Alias — some scrapers import { db } instead of { pool }
export const db = pool;
// Per-form-factor price bounds [min, max] in USD equivalent
const PRICE_BOUNDS: Record<string, [number, number]> = {
"SFP": [2, 3000],
"SFP+": [4, 5000],
"SFP28": [10, 8000],
"SFP56": [20, 10000],
"SFP-DD": [30, 12000],
"QSFP+": [15, 6000],
"QSFP28": [20, 10000],
"QSFP56": [50, 15000],
"QSFP-DD": [60, 20000],
"QSFP112": [80, 25000],
"OSFP": [100, 35000],
"OSFP112": [150, 40000],
"OSFP224": [200, 60000],
"CFP": [100, 30000],
"CFP2": [100, 30000],
"XFP": [10, 5000],
"GBIC": [2, 2000],
};
async function isPriceAnomalous(transceiverId: string, priceUsd: number): Promise<boolean> {
const row = await pool.query(
`SELECT form_factor FROM transceivers WHERE id = $1`,
[transceiverId]
);
const formFactor = row.rows[0]?.form_factor as string | undefined;
if (!formFactor) return false;
const bounds = PRICE_BOUNDS[formFactor];
if (!bounds) return false;
return priceUsd < bounds[0] || priceUsd > bounds[1];
}
export async function upsertPriceObservation(params: {
transceiverId: string;
sourceVendorId: string;
@ -29,6 +62,16 @@ export async function upsertPriceObservation(params: {
url?: string;
contentHash: string;
}): Promise<boolean> {
// Normalize price to USD for sanity check (rough conversion)
const priceUsd = params.currency === "EUR" ? params.price * 1.09
: params.currency === "GBP" ? params.price * 1.27
: params.price;
const anomalous = await isPriceAnomalous(params.transceiverId, priceUsd);
if (anomalous) {
return false; // Reject price outside form-factor bounds
}
// Check if price changed via content hash
const existing = await pool.query(
`SELECT content_hash FROM price_observations
@ -37,10 +80,20 @@ export async function upsertPriceObservation(params: {
[params.transceiverId, params.sourceVendorId]
);
// Check if vendor is a competitor (non-Flexoptix) for competitor_verified flag
const vendorRow = await pool.query(
`SELECT is_competitor FROM vendors WHERE id = $1`,
[params.sourceVendorId]
);
const isCompetitor = vendorRow.rows[0]?.is_competitor === true;
if (existing.rows.length > 0 && existing.rows[0].content_hash === params.contentHash) {
// Price unchanged — but still ensure price_verified is set (in case it wasn't before)
// Price unchanged — still ensure verified flags are current
await pool.query(
`UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`,
`UPDATE transceivers SET
price_verified = true
${isCompetitor ? ", competitor_verified = true, competitor_verified_at = COALESCE(competitor_verified_at, NOW())" : ""}
WHERE id = $1 AND (price_verified IS NULL OR price_verified = false OR ${isCompetitor ? "competitor_verified IS NULL OR competitor_verified = false" : "false"})`,
[params.transceiverId]
);
return false; // No change
@ -61,11 +114,23 @@ export async function upsertPriceObservation(params: {
params.contentHash,
]
);
// Mark the transceiver as price-verified whenever we successfully record a price
await pool.query(
`UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`,
[params.transceiverId]
);
// Mark price_verified always; competitor_verified only for non-Flexoptix vendors
if (isCompetitor) {
await pool.query(
`UPDATE transceivers SET
price_verified = true,
competitor_verified = true,
competitor_verified_at = COALESCE(competitor_verified_at, NOW())
WHERE id = $1`,
[params.transceiverId]
);
} else {
await pool.query(
`UPDATE transceivers SET price_verified = true WHERE id = $1 AND (price_verified IS NULL OR price_verified = false)`,
[params.transceiverId]
);
}
return true; // New observation written
}

8
packages/scraper/src/utils/hash.ts
View File

@ -2,10 +2,12 @@ import { createHash } from "crypto";
/**
* Generate SHA-256 content hash for change detection.
* Only hashes the fields that matter (price, stock, quantity).
* Accepts an object (preferred) or a plain string (legacy scrapers).
*/
export function contentHash(data: Record<string, unknown>): string {
const normalized = JSON.stringify(data, Object.keys(data).sort());
export function contentHash(data: Record<string, unknown> | string): string {
const normalized = typeof data === "string"
? data
: JSON.stringify(data, Object.keys(data).sort());
return createHash("sha256").update(normalized).digest("hex").slice(0, 16);
}

44
scripts/migrate.ts
View File

@ -19,16 +19,50 @@ async function migrate(): Promise<void> {
.filter((f) => f.endsWith(".sql"))
.sort();
console.log(`Found ${files.length} migration files`);
const client = await pool.connect();
try {
for (const file of files) {
// Create migration tracker if it doesn't exist
await client.query(`
CREATE TABLE IF NOT EXISTS _migrations (
filename TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
// Get already-applied migrations
const { rows } = await client.query(
"SELECT filename FROM _migrations ORDER BY filename"
);
const applied = new Set(rows.map((r: { filename: string }) => r.filename));
const pending = files.filter((f) => !applied.has(f));
if (pending.length === 0) {
console.log("All migrations already applied. Nothing to do.");
return;
}
console.log(
`${applied.size} already applied, running ${pending.length} pending migrations`
);
for (const file of pending) {
const sql = readFileSync(join(sqlDir, file), "utf-8");
console.log(`Running: ${file}...`);
await client.query(sql);
console.log(` Done: ${file}`);
await client.query("BEGIN");
try {
await client.query(sql);
await client.query(
"INSERT INTO _migrations (filename) VALUES ($1)",
[file]
);
await client.query("COMMIT");
console.log(` Done: ${file}`);
} catch (err) {
await client.query("ROLLBACK");
throw err;
}
}
console.log("\nAll migrations completed successfully.");
} catch (err) {
console.error("Migration failed:", err);