fix: scrapers status endpoint — correct table and column names
- Replace vendor_slug → vendors.slug via JOIN (column doesn't exist on transceivers) - Replace competitor_prices + ebay_listings → price_observations (correct table) - Replace market_intelligence → knowledge_base in status DB stats (table exists, 73 rows) - price_observations columns: transceiver_id, price, time (not part_number/price_eur/scraped_at) - Result: 5602 transceivers, 359 vendors, 465 switches, 4252 prices, 73 KB entries all visible
This commit is contained in:
parent
e3b53343d4
commit
69ef507d8a
@ -33,16 +33,16 @@ const SCRAPERS = [
|
|||||||
// GET /api/scrapers/status — Overview of all scrapers + DB stats
|
// GET /api/scrapers/status — Overview of all scrapers + DB stats
|
||||||
scraperRouter.get("/status", async (_req: Request, res: Response) => {
|
scraperRouter.get("/status", async (_req: Request, res: Response) => {
|
||||||
try {
|
try {
|
||||||
// DB counts per source
|
// DB counts per vendor (via vendor join)
|
||||||
const sourceStats = await pool.query(`
|
const sourceStats = await pool.query(`
|
||||||
SELECT
|
SELECT
|
||||||
vendor_slug as source,
|
v.slug as source,
|
||||||
COUNT(*) as count,
|
COUNT(t.id) as count,
|
||||||
MAX(updated_at) as last_updated,
|
MAX(t.updated_at) as last_updated,
|
||||||
MIN(updated_at) as first_seen
|
MIN(t.updated_at) as first_seen
|
||||||
FROM transceivers
|
FROM transceivers t
|
||||||
WHERE vendor_slug IS NOT NULL
|
JOIN vendors v ON v.id = t.vendor_id
|
||||||
GROUP BY vendor_slug
|
GROUP BY v.slug
|
||||||
ORDER BY count DESC
|
ORDER BY count DESC
|
||||||
`).catch(() => ({ rows: [] }));
|
`).catch(() => ({ rows: [] }));
|
||||||
|
|
||||||
@ -50,14 +50,10 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
|
|||||||
const priceStats = await pool.query(`
|
const priceStats = await pool.query(`
|
||||||
SELECT
|
SELECT
|
||||||
COUNT(*) as total_prices,
|
COUNT(*) as total_prices,
|
||||||
COUNT(DISTINCT part_number) as unique_parts,
|
COUNT(DISTINCT transceiver_id) as unique_parts,
|
||||||
MAX(scraped_at) as last_price_update,
|
MAX(time) as last_price_update,
|
||||||
AVG(price_eur) as avg_price_eur
|
AVG(price) as avg_price_eur
|
||||||
FROM (
|
FROM price_observations
|
||||||
SELECT part_number, price_eur, scraped_at FROM competitor_prices
|
|
||||||
UNION ALL
|
|
||||||
SELECT part_number, price_eur, scraped_at FROM ebay_listings
|
|
||||||
) combined
|
|
||||||
`).catch(() => ({ rows: [{}] }));
|
`).catch(() => ({ rows: [{}] }));
|
||||||
|
|
||||||
// Overall DB stats
|
// Overall DB stats
|
||||||
@ -67,8 +63,8 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
|
|||||||
(SELECT COUNT(*) FROM vendors) as vendors,
|
(SELECT COUNT(*) FROM vendors) as vendors,
|
||||||
(SELECT COUNT(*) FROM switches) as switches,
|
(SELECT COUNT(*) FROM switches) as switches,
|
||||||
(SELECT COUNT(*) FROM news_articles) as news_articles,
|
(SELECT COUNT(*) FROM news_articles) as news_articles,
|
||||||
(SELECT COUNT(*) FROM market_intelligence) as knowledge_base_entries,
|
(SELECT COUNT(*) FROM knowledge_base) as knowledge_base_entries,
|
||||||
(SELECT COUNT(*) FROM competitor_prices) as competitor_prices,
|
(SELECT COUNT(*) FROM price_observations) as competitor_prices,
|
||||||
(SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size
|
(SELECT pg_size_pretty(pg_database_size(current_database()))) as db_size
|
||||||
`).catch(() => ({ rows: [{}] }));
|
`).catch(() => ({ rows: [{}] }));
|
||||||
|
|
||||||
@ -169,7 +165,7 @@ scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => {
|
|||||||
MAX(created_at) as latest
|
MAX(created_at) as latest
|
||||||
FROM market_intelligence
|
FROM market_intelligence
|
||||||
GROUP BY intel_type
|
GROUP BY intel_type
|
||||||
ORDER BY top_relevance DESC NULLS LAST
|
ORDER BY MAX(relevance_score) DESC NULLS LAST
|
||||||
`).catch(() => ({ rows: [] })),
|
`).catch(() => ({ rows: [] })),
|
||||||
// Recent News
|
// Recent News
|
||||||
pool.query(`
|
pool.query(`
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user