diff --git a/packages/api/src/routes/scrapers.ts b/packages/api/src/routes/scrapers.ts index 0cfe6e7..0f4eea5 100644 --- a/packages/api/src/routes/scrapers.ts +++ b/packages/api/src/routes/scrapers.ts @@ -33,16 +33,16 @@ const SCRAPERS = [ // GET /api/scrapers/status — Overview of all scrapers + DB stats scraperRouter.get("/status", async (_req: Request, res: Response) => { try { - // DB counts per source + // DB counts per vendor (via vendor join) const sourceStats = await pool.query(` SELECT - vendor_slug as source, - COUNT(*) as count, - MAX(updated_at) as last_updated, - MIN(updated_at) as first_seen - FROM transceivers - WHERE vendor_slug IS NOT NULL - GROUP BY vendor_slug + v.slug as source, + COUNT(t.id) as count, + MAX(t.updated_at) as last_updated, + MIN(t.updated_at) as first_seen + FROM transceivers t + JOIN vendors v ON v.id = t.vendor_id + GROUP BY v.slug ORDER BY count DESC `).catch(() => ({ rows: [] })); @@ -50,14 +50,10 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => { const priceStats = await pool.query(` SELECT COUNT(*) as total_prices, - COUNT(DISTINCT part_number) as unique_parts, - MAX(scraped_at) as last_price_update, - AVG(price_eur) as avg_price_eur - FROM ( - SELECT part_number, price_eur, scraped_at FROM competitor_prices - UNION ALL - SELECT part_number, price_eur, scraped_at FROM ebay_listings - ) combined + COUNT(DISTINCT transceiver_id) as unique_parts, + MAX(time) as last_price_update, + AVG(price) as avg_price_eur + FROM price_observations `).catch(() => ({ rows: [{}] })); // Overall DB stats @@ -67,8 +63,8 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => { (SELECT COUNT(*) FROM vendors) as vendors, (SELECT COUNT(*) FROM switches) as switches, (SELECT COUNT(*) FROM news_articles) as news_articles, - (SELECT COUNT(*) FROM market_intelligence) as knowledge_base_entries, - (SELECT COUNT(*) FROM competitor_prices) as competitor_prices, + (SELECT COUNT(*) FROM knowledge_base) as knowledge_base_entries, + (SELECT COUNT(*) FROM price_observations) as competitor_prices, (SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size `).catch(() => ({ rows: [{}] })); @@ -169,7 +165,7 @@ scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => { MAX(created_at) as latest FROM market_intelligence GROUP BY intel_type - ORDER BY top_relevance DESC NULLS LAST + ORDER BY MAX(relevance_score) DESC NULLS LAST `).catch(() => ({ rows: [] })), // Recent News pool.query(`