transceiver-db/sql/124-refine-stock-monitoring.sql

-- 124-refine-stock-monitoring.sql
-- Correction to 123 after reading the ingestion pipeline (packages/scraper).
--
-- Finding: the 2026-07-05 audit concluded "QSFPTEK 75k stock rows all empty /
-- extracts nothing". That is WRONG. QSFPTEK populates quantity_available (100%),
-- in_stock (100%) and stock_vendor_ts (100%). It legitimately has NO units_sold
-- ("verkauft") counter and NO DE/global warehouse split — those are FS.com-only.
-- Every scraper captures the stock signal its source exposes; extraction is not
-- broken on any vendor. Only the FS.com-specific delivery-date / lead-time gap and
-- the dead/stale scrapers are real.
--
-- This migration therefore:
--   1) adds pct_qty_available + pct_vendor_ts to v_stock_completeness (proves the
--      per-vendor signal is present) and keeps the FS.com-shaped columns for context;
--   2) rekeys the stock_extraction_empty alert to the UNIVERSAL quantitative signal
--      (quantity_available OR warehouse qty), so it no longer false-positives vendors
--      with a simpler stock model and the alert list can genuinely drain to empty.
--
-- NON-DESTRUCTIVE: read-only view redefinitions only. Idempotent.

DROP VIEW IF EXISTS v_data_quality_alerts;
DROP VIEW IF EXISTS v_stock_completeness;

-- ---------------------------------------------------------------------------
-- v_stock_completeness (refined): universal signal first, FS.com-specific after.
-- ---------------------------------------------------------------------------
CREATE VIEW v_stock_completeness AS
SELECT v.name AS vendor,
       v.slug AS vendor_slug,
       count(*)             AS stock_rows,
       max(s."time")::date AS last_obs,
       -- universal quantitative stock signal (any of these = extraction working)
       round(100.0 * count(s.quantity_available) / count(*), 1) AS pct_qty_available,
       round(100.0 * count(*) FILTER (
               WHERE s.warehouse_de_qty IS NOT NULL
                  OR s.warehouse_global_qty IS NOT NULL)    / count(*), 1) AS pct_warehouse_qty,
       round(100.0 * count(s.stock_vendor_ts) / count(*), 1) AS pct_vendor_ts,
       -- FS.com-specific enrichments (vendor-model dependent, not universal)
       round(100.0 * count(s.units_sold)     / count(*), 1) AS pct_units_sold,
       round(100.0 * count(*) FILTER (
               WHERE s.warehouse_de_delivery_date IS NOT NULL
                  OR s.warehouse_global_delivery_date IS NOT NULL
                  OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date,
       round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time
FROM stock_observations s
JOIN vendors v ON v.id = s.source_vendor_id
GROUP BY v.id, v.name, v.slug;

-- ---------------------------------------------------------------------------
-- v_data_quality_alerts (refined): stock_extraction_empty now keys on the
-- universal quantitative signal instead of FS.com-only fields.
-- ---------------------------------------------------------------------------
CREATE VIEW v_data_quality_alerts AS
WITH alerts AS (
  -- 1) Dead scraper: had data, stopped delivering > 30 days ago.
  SELECT 'critical'::text AS severity, 1 AS severity_rank,
         'scraper_dead'::text AS category,
         vendor || ' / ' || source_table AS entity,
         format('no new rows for %s days (last %s); %s rows total',
                days_since_last, last_observation, total_rows) AS detail
  FROM v_scraper_health
  WHERE status = 'dead'

  UNION ALL
  -- 2) Stale scraper: 8-30 days since last row.
  SELECT 'warning', 3, 'scraper_stale',
         vendor || ' / ' || source_table,
         format('no new rows for %s days (last %s)', days_since_last, last_observation)
  FROM v_scraper_health
  WHERE status = 'stale'

  UNION ALL
  -- 3) Rows written but NO quantitative stock signal at all (qty + warehouse both empty)
  --    on an otherwise-alive scraper. This is the real "extraction is a no-op" case.
  SELECT 'high', 2, 'stock_extraction_empty',
         vendor || ' / stock_observations',
         format('%s rows but quantity_available=%s%% and warehouse_qty=%s%% populated',
                stock_rows, pct_qty_available, pct_warehouse_qty)
  FROM v_stock_completeness
  WHERE stock_rows >= 500
    AND pct_qty_available = 0
    AND pct_warehouse_qty = 0
    AND last_obs >= now()::date - 30

  UNION ALL
  -- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable.
  SELECT 'high', 2, 'delivery_date_gap',
         'ALL stock scrapers',
         format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable',
                round(100.0 * count(*) FILTER (
                    WHERE warehouse_de_delivery_date IS NOT NULL
                       OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2),
                round(100.0 * count(lead_time_days) / nullif(count(*),0), 2))
  FROM stock_observations
  HAVING round(100.0 * count(*) FILTER (
             WHERE warehouse_de_delivery_date IS NOT NULL
                OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5

  UNION ALL
  -- 5) reach_label incomplete on a sizeable catalog.
  SELECT 'warning', 3, 'reach_label_incomplete',
         vendor,
         format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers)
  FROM v_coverage
  WHERE transceivers >= 100 AND pct_reach_null >= 20

  UNION ALL
  -- 6) Future-dated news -> ingest guard candidate.
  SELECT 'warning', 3, 'future_dated_news',
         'news_articles',
         format('%s article(s) dated in the future (max %s) — ingest guard recommended',
                count(*), max(published_at)::date)
  FROM news_articles
  WHERE published_at > now() + interval '2 days'
  HAVING count(*) > 0
)
SELECT severity, severity_rank, category, entity, detail
FROM alerts
ORDER BY severity_rank, category, entity;