fix: scrapers status endpoint — correct table and column names

- Replace vendor_slug → vendors.slug via JOIN (column doesn't exist on transceivers)
- Replace competitor_prices + ebay_listings → price_observations (correct table)
- Replace market_intelligence → knowledge_base in status DB stats (table exists, 73 rows)
- price_observations columns: transceiver_id, price, time (not part_number/price_eur/scraped_at)
- Result: 5602 transceivers, 359 vendors, 465 switches, 4252 prices, 73 KB entries all visible
This commit is contained in:
Rene Fichtmueller 2026-04-02 15:24:18 +02:00
parent 238ee8c0a4
commit 41266fa0d4

View File

@ -33,16 +33,16 @@ const SCRAPERS = [
// GET /api/scrapers/status — Overview of all scrapers + DB stats // GET /api/scrapers/status — Overview of all scrapers + DB stats
scraperRouter.get("/status", async (_req: Request, res: Response) => { scraperRouter.get("/status", async (_req: Request, res: Response) => {
try { try {
// DB counts per source // DB counts per vendor (via vendor join)
const sourceStats = await pool.query(` const sourceStats = await pool.query(`
SELECT SELECT
vendor_slug as source, v.slug as source,
COUNT(*) as count, COUNT(t.id) as count,
MAX(updated_at) as last_updated, MAX(t.updated_at) as last_updated,
MIN(updated_at) as first_seen MIN(t.updated_at) as first_seen
FROM transceivers FROM transceivers t
WHERE vendor_slug IS NOT NULL JOIN vendors v ON v.id = t.vendor_id
GROUP BY vendor_slug GROUP BY v.slug
ORDER BY count DESC ORDER BY count DESC
`).catch(() => ({ rows: [] })); `).catch(() => ({ rows: [] }));
@ -50,14 +50,10 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
const priceStats = await pool.query(` const priceStats = await pool.query(`
SELECT SELECT
COUNT(*) as total_prices, COUNT(*) as total_prices,
COUNT(DISTINCT part_number) as unique_parts, COUNT(DISTINCT transceiver_id) as unique_parts,
MAX(scraped_at) as last_price_update, MAX(time) as last_price_update,
AVG(price_eur) as avg_price_eur AVG(price) as avg_price_eur
FROM ( FROM price_observations
SELECT part_number, price_eur, scraped_at FROM competitor_prices
UNION ALL
SELECT part_number, price_eur, scraped_at FROM ebay_listings
) combined
`).catch(() => ({ rows: [{}] })); `).catch(() => ({ rows: [{}] }));
// Overall DB stats // Overall DB stats
@ -67,8 +63,8 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
(SELECT COUNT(*) FROM vendors) as vendors, (SELECT COUNT(*) FROM vendors) as vendors,
(SELECT COUNT(*) FROM switches) as switches, (SELECT COUNT(*) FROM switches) as switches,
(SELECT COUNT(*) FROM news_articles) as news_articles, (SELECT COUNT(*) FROM news_articles) as news_articles,
(SELECT COUNT(*) FROM market_intelligence) as knowledge_base_entries, (SELECT COUNT(*) FROM knowledge_base) as knowledge_base_entries,
(SELECT COUNT(*) FROM competitor_prices) as competitor_prices, (SELECT COUNT(*) FROM price_observations) as competitor_prices,
(SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size (SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size
`).catch(() => ({ rows: [{}] })); `).catch(() => ({ rows: [{}] }));
@ -169,7 +165,7 @@ scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => {
MAX(created_at) as latest MAX(created_at) as latest
FROM market_intelligence FROM market_intelligence
GROUP BY intel_type GROUP BY intel_type
ORDER BY top_relevance DESC NULLS LAST ORDER BY MAX(relevance_score) DESC NULLS LAST
`).catch(() => ({ rows: [] })), `).catch(() => ({ rows: [] })),
// Recent News // Recent News
pool.query(` pool.query(`