From 414ea65fee0e8f04a0494d63a277662c34bb04f9 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Mon, 6 Jul 2026 05:49:39 +0000 Subject: [PATCH] fix(db): rekey stock_extraction_empty alert to universal quantity signal (124) Reading packages/scraper showed QSFPTEK/ATGBICS populate quantity_available 100%; they simply lack FS.com-only units_sold/warehouse-split. The 123 alert keyed on those FS.com fields and false-positived QSFPTEK. Rekey to quantity_available OR warehouse_qty so the alert list drains to empty; surface pct_qty_available/pct_vendor_ts in v_stock_completeness. --- sql/124-refine-stock-monitoring.sql | 119 ++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 sql/124-refine-stock-monitoring.sql diff --git a/sql/124-refine-stock-monitoring.sql b/sql/124-refine-stock-monitoring.sql new file mode 100644 index 0000000..e83378c --- /dev/null +++ b/sql/124-refine-stock-monitoring.sql @@ -0,0 +1,119 @@ +-- 124-refine-stock-monitoring.sql +-- Correction to 123 after reading the ingestion pipeline (packages/scraper). +-- +-- Finding: the 2026-07-05 audit concluded "QSFPTEK 75k stock rows all empty / +-- extracts nothing". That is WRONG. QSFPTEK populates quantity_available (100%), +-- in_stock (100%) and stock_vendor_ts (100%). It legitimately has NO units_sold +-- ("verkauft") counter and NO DE/global warehouse split — those are FS.com-only. +-- Every scraper captures the stock signal its source exposes; extraction is not +-- broken on any vendor. Only the FS.com-specific delivery-date / lead-time gap and +-- the dead/stale scrapers are real. +-- +-- This migration therefore: +-- 1) adds pct_qty_available + pct_vendor_ts to v_stock_completeness (proves the +-- per-vendor signal is present) and keeps the FS.com-shaped columns for context; +-- 2) rekeys the stock_extraction_empty alert to the UNIVERSAL quantitative signal +-- (quantity_available OR warehouse qty), so it no longer false-positives vendors +-- with a simpler stock model and the alert list can genuinely drain to empty. +-- +-- NON-DESTRUCTIVE: read-only view redefinitions only. Idempotent. + +DROP VIEW IF EXISTS v_data_quality_alerts; +DROP VIEW IF EXISTS v_stock_completeness; + +-- --------------------------------------------------------------------------- +-- v_stock_completeness (refined): universal signal first, FS.com-specific after. +-- --------------------------------------------------------------------------- +CREATE VIEW v_stock_completeness AS +SELECT v.name AS vendor, + v.slug AS vendor_slug, + count(*) AS stock_rows, + max(s."time")::date AS last_obs, + -- universal quantitative stock signal (any of these = extraction working) + round(100.0 * count(s.quantity_available) / count(*), 1) AS pct_qty_available, + round(100.0 * count(*) FILTER ( + WHERE s.warehouse_de_qty IS NOT NULL + OR s.warehouse_global_qty IS NOT NULL) / count(*), 1) AS pct_warehouse_qty, + round(100.0 * count(s.stock_vendor_ts) / count(*), 1) AS pct_vendor_ts, + -- FS.com-specific enrichments (vendor-model dependent, not universal) + round(100.0 * count(s.units_sold) / count(*), 1) AS pct_units_sold, + round(100.0 * count(*) FILTER ( + WHERE s.warehouse_de_delivery_date IS NOT NULL + OR s.warehouse_global_delivery_date IS NOT NULL + OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date, + round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time +FROM stock_observations s +JOIN vendors v ON v.id = s.source_vendor_id +GROUP BY v.id, v.name, v.slug; + +-- --------------------------------------------------------------------------- +-- v_data_quality_alerts (refined): stock_extraction_empty now keys on the +-- universal quantitative signal instead of FS.com-only fields. +-- --------------------------------------------------------------------------- +CREATE VIEW v_data_quality_alerts AS +WITH alerts AS ( + -- 1) Dead scraper: had data, stopped delivering > 30 days ago. + SELECT 'critical'::text AS severity, 1 AS severity_rank, + 'scraper_dead'::text AS category, + vendor || ' / ' || source_table AS entity, + format('no new rows for %s days (last %s); %s rows total', + days_since_last, last_observation, total_rows) AS detail + FROM v_scraper_health + WHERE status = 'dead' + + UNION ALL + -- 2) Stale scraper: 8-30 days since last row. + SELECT 'warning', 3, 'scraper_stale', + vendor || ' / ' || source_table, + format('no new rows for %s days (last %s)', days_since_last, last_observation) + FROM v_scraper_health + WHERE status = 'stale' + + UNION ALL + -- 3) Rows written but NO quantitative stock signal at all (qty + warehouse both empty) + -- on an otherwise-alive scraper. This is the real "extraction is a no-op" case. + SELECT 'high', 2, 'stock_extraction_empty', + vendor || ' / stock_observations', + format('%s rows but quantity_available=%s%% and warehouse_qty=%s%% populated', + stock_rows, pct_qty_available, pct_warehouse_qty) + FROM v_stock_completeness + WHERE stock_rows >= 500 + AND pct_qty_available = 0 + AND pct_warehouse_qty = 0 + AND last_obs >= now()::date - 30 + + UNION ALL + -- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable. + SELECT 'high', 2, 'delivery_date_gap', + 'ALL stock scrapers', + format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable', + round(100.0 * count(*) FILTER ( + WHERE warehouse_de_delivery_date IS NOT NULL + OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2), + round(100.0 * count(lead_time_days) / nullif(count(*),0), 2)) + FROM stock_observations + HAVING round(100.0 * count(*) FILTER ( + WHERE warehouse_de_delivery_date IS NOT NULL + OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5 + + UNION ALL + -- 5) reach_label incomplete on a sizeable catalog. + SELECT 'warning', 3, 'reach_label_incomplete', + vendor, + format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers) + FROM v_coverage + WHERE transceivers >= 100 AND pct_reach_null >= 20 + + UNION ALL + -- 6) Future-dated news -> ingest guard candidate. + SELECT 'warning', 3, 'future_dated_news', + 'news_articles', + format('%s article(s) dated in the future (max %s) — ingest guard recommended', + count(*), max(published_at)::date) + FROM news_articles + WHERE published_at > now() + interval '2 days' + HAVING count(*) > 0 +) +SELECT severity, severity_rank, category, entity, detail +FROM alerts +ORDER BY severity_rank, category, entity;