rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: scrapers status endpoint — correct table and column names

Browse Source 
- Replace vendor_slug → vendors.slug via JOIN (column doesn't exist on transceivers)
- Replace competitor_prices + ebay_listings → price_observations (correct table)
- Replace market_intelligence → knowledge_base in status DB stats (table exists, 73 rows)
- price_observations columns: transceiver_id, price, time (not part_number/price_eur/scraped_at)
- Result: 5602 transceivers, 359 vendors, 465 switches, 4252 prices, 73 KB entries all visible
This commit is contained in:
Rene Fichtmueller 2026-04-02 15:24:18 +02:00
parent 238ee8c0a4
commit 41266fa0d4
1 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/api/src/routes/scrapers.ts
View File

@ -33,16 +33,16 @@ const SCRAPERS = [
// GET /api/scrapers/status — Overview of all scrapers + DB stats
scraperRouter.get("/status", async (_req: Request, res: Response) => {
try {
// DB counts per source
// DB counts per vendor (via vendor join)
const sourceStats = await pool.query(`
SELECT
vendor_slug as source,
COUNT(*) as count,
MAX(updated_at) as last_updated,
MIN(updated_at) as first_seen
FROM transceivers
WHERE vendor_slug IS NOT NULL
GROUP BY vendor_slug
v.slug as source,
COUNT(t.id) as count,
MAX(t.updated_at) as last_updated,
MIN(t.updated_at) as first_seen
FROM transceivers t
JOIN vendors v ON v.id = t.vendor_id
GROUP BY v.slug
ORDER BY count DESC
`).catch(() => ({ rows: [] }));
@ -50,14 +50,10 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
const priceStats = await pool.query(`
SELECT
COUNT(*) as total_prices,
COUNT(DISTINCT part_number) as unique_parts,
MAX(scraped_at) as last_price_update,
AVG(price_eur) as avg_price_eur
FROM (
SELECT part_number, price_eur, scraped_at FROM competitor_prices
UNION ALL
SELECT part_number, price_eur, scraped_at FROM ebay_listings
) combined
COUNT(DISTINCT transceiver_id) as unique_parts,
MAX(time) as last_price_update,
AVG(price) as avg_price_eur
FROM price_observations
`).catch(() => ({ rows: [{}] }));
// Overall DB stats
@ -67,8 +63,8 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
(SELECT COUNT(*) FROM vendors) as vendors,
(SELECT COUNT(*) FROM switches) as switches,
(SELECT COUNT(*) FROM news_articles) as news_articles,
(SELECT COUNT(*) FROM market_intelligence) as knowledge_base_entries,
(SELECT COUNT(*) FROM competitor_prices) as competitor_prices,
(SELECT COUNT(*) FROM knowledge_base) as knowledge_base_entries,
(SELECT COUNT(*) FROM price_observations) as competitor_prices,
(SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size
`).catch(() => ({ rows: [{}] }));
@ -169,7 +165,7 @@ scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => {
MAX(created_at) as latest
FROM market_intelligence
GROUP BY intel_type
ORDER BY top_relevance DESC NULLS LAST
ORDER BY MAX(relevance_score) DESC NULLS LAST
`).catch(() => ({ rows: [] })),
// Recent News
pool.query(`