transceiver-db/sql/125-switch-quality.sql

-- sql/125-switch-quality.sql
-- Switch / compatibility data-quality remediation + monitoring layer.
--
-- Audit 2026-07-06 (read-only) found systematic errors in the switch database:
--   * 5.686+ physically-impossible compatibility rows (transceiver faster than the
--     switch's fastest port). Root cause: the form-factor fallback in
--     packages/scraper/src/scrapers/flexoptix-compat.ts matched by form factor ONLY,
--     ignoring speed -> a 400G-OSFP port matched every OSFP transceiver incl. 1.6T.
--   * 420/687 switches with NULL max_speed_gbps (ports_config present -> derivable).
--   * 528 transceivers with speed_gbps <= 0 (blocks compat validation; they silently
--     pass as "compatible" with everything).
--   * coding-pattern coverage as low as ~13% at 100G (vendor_compat = [] for most).
--
-- DESIGN (why this is safe at the "no errors" bar):
--   The naive ceiling (ports_config-derived max) is UNRELIABLE because some switches'
--   ports_config is understated (e.g. Cisco C9500-32QC lists 40G but does 100G; NCS 540
--   "Z" ports list 10G but are 25G-capable; A9K-20HG-FLEX lists 100G but does 400G).
--   Flagging by that ceiling would create FALSE "incompatible" flags on curated rows.
--   So the effective ceiling is evidence-based:
--     effective_max = GREATEST(stored max_speed_gbps,
--                              ports_config-derived max,
--                              fastest curated-compatible transceiver for the switch)
--   and ONLY algorithmic 'spec_match' fallback rows are ever auto-flagged; curated
--   ('vendor_matrix'/'vendor_compat') rows are never touched (they instead RAISE the
--   ceiling, protecting understated switches). Result: 0 curated false positives.
--
-- Idempotent + safe to re-run. Applied with psql --single-transaction.

-- ═══════════════════════════════════════════════════════════════════════════
-- 0) Ports-config -> max port speed (pure helper) + evidence-based switch ceiling
-- ═══════════════════════════════════════════════════════════════════════════

create or replace function fn_ports_max_gbps(cfg jsonb) returns numeric
language sql immutable as $fn$
  select case when jsonb_typeof(cfg) = 'object'
              then (select max(substring(k from '([0-9]+)G')::numeric)
                    from jsonb_object_keys(cfg) k)
              else null end
$fn$;

comment on function fn_ports_max_gbps(jsonb) is
  'Max port speed (Gbps) parsed from a switch ports_config JSONB object, e.g. {"400G_QSFP-DD":4,"100G_QSFP28":2} -> 400. NULL if config is empty/non-object.';

create or replace view v_switch_effective_max as
select s.id as switch_id,
       s.max_speed_gbps                    as stored_max_gbps,
       fn_ports_max_gbps(s.ports_config)   as ports_max_gbps,
       (select max(t.speed_gbps)
          from compatibility c join transceivers t on t.id = c.transceiver_id
         where c.switch_id = s.id and c.status = 'compatible'
           and c.verification_method in ('vendor_matrix','vendor_compat')
           and t.speed_gbps > 0)           as curated_max_gbps,
       greatest(
         coalesce(s.max_speed_gbps, 0),
         coalesce(fn_ports_max_gbps(s.ports_config), 0),
         coalesce((select max(t.speed_gbps)
                     from compatibility c join transceivers t on t.id = c.transceiver_id
                    where c.switch_id = s.id and c.status = 'compatible'
                      and c.verification_method in ('vendor_matrix','vendor_compat')
                      and t.speed_gbps > 0), 0)
       )                                    as effective_max_gbps
from switches s;

comment on view v_switch_effective_max is
  'Evidence-based max port speed per switch (greatest of stored max_speed_gbps, ports_config-derived, and fastest curated-compatible transceiver). Used to validate compatibility without false-flagging switches whose ports_config is understated.';

-- ═══════════════════════════════════════════════════════════════════════════
-- REMEDIATION A) Backfill max_speed_gbps from ports_config (additive, nulls only).
--   Highest leverage: unblocks validation for switches that currently pass no check.
--   Only fills NULLs; never overwrites a curated stored value (61/267 stored values
--   disagree with ports-derived in both directions -> curated values are preserved).
--   Switches with an empty {} ports_config stay NULL (need a datasheet -> see alerts).
-- ═══════════════════════════════════════════════════════════════════════════

update switches s
   set max_speed_gbps = fn_ports_max_gbps(s.ports_config)
 where s.max_speed_gbps is null
   and fn_ports_max_gbps(s.ports_config) is not null
   and fn_ports_max_gbps(s.ports_config) > 0;

-- ═══════════════════════════════════════════════════════════════════════════
-- REMEDIATION C) Re-derive speed_gbps for transceivers with speed <= 0, HIGH
--   confidence only. Speed comes from the manufacturer's standardized speed token
--   in the part number (e.g. SFP-10G-SR -> 10, QDD-2X400G -> 800, QSFP-4SFP25G -> 100),
--   and is applied ONLY when it is <= the form factor's physical maximum. Parts whose
--   token exceeds the form-factor cap (usually a wrong form_factor, e.g. QSFP-400G-AOC
--   mislabelled 'SFP') or that have no speed token are LEFT flagged for datasheet work.
--   No blind update: FastEthernet "100FX"/"100BASE" (no 'G') never matches.
-- ═══════════════════════════════════════════════════════════════════════════

with cand as (
  select id,
    case
      when part_number ~* '([0-9]+)X([0-9]+)G'
        then (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[1]::numeric
           * (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[2]::numeric
      when part_number ~* '([0-9]+)SFP([0-9]+)G'
        then (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[1]::numeric
           * (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[2]::numeric
      when part_number ~* '([0-9]+)G'
        then (regexp_match(part_number,'([0-9]+)G','i'))[1]::numeric
    end as derived,
    case upper(coalesce(form_factor,''))
      when 'SFP' then 1  when 'SFP+' then 16  when 'SFP28' then 32  when 'SFP56' then 56
      when 'XFP' then 10 when 'QSFP+' then 40 when 'QSFP28' then 100 when 'QSFP56' then 200
      when 'QSFP-DD' then 800 when 'QSFP-DD800' then 800
      when 'OSFP' then 1600 when 'OSFP224' then 1600
      when 'CFP' then 100 when 'CFP2' then 400 when 'CFP4' then 100 when 'CPAK' then 100
      else 99999 end as ff_cap
  from transceivers
  where speed_gbps is null or speed_gbps <= 0)
update transceivers t
   set speed_gbps = c.derived
  from cand c
 where t.id = c.id
   and c.derived is not null and c.derived > 0
   and c.derived <= c.ff_cap;

-- ═══════════════════════════════════════════════════════════════════════════
-- REMEDIATION B) Flag physically-impossible compatibility rows.
--   Only 'spec_match' (algorithmic form-factor fallback) rows are flagged; curated
--   rows are never touched. status='incompatible' (constraint-allowed value) removes
--   them from every consumer that filters status='compatible' (e.g. queries.ts:302).
--   Runs AFTER A + C so it uses backfilled ceilings and corrected transceiver speeds.
-- ═══════════════════════════════════════════════════════════════════════════

with em as materialized (
  select switch_id, effective_max_gbps from v_switch_effective_max)
update compatibility c
   set status = 'incompatible',
       notes  = case when c.notes is null or c.notes = ''
                     then 'auto(125): transceiver speed exceeds switch max port speed'
                     else c.notes || ' | auto(125): transceiver speed exceeds switch max port speed' end
  from em e, transceivers t
 where e.switch_id = c.switch_id
   and t.id = c.transceiver_id
   and c.status = 'compatible'
   and c.verification_method = 'spec_match'
   and e.effective_max_gbps > 0
   and t.speed_gbps is not null
   and t.speed_gbps > e.effective_max_gbps;

-- ═══════════════════════════════════════════════════════════════════════════
-- MONITORING VIEWS (read-only). v_switch_quality_alerts is the task list.
-- ═══════════════════════════════════════════════════════════════════════════

-- 1) Impossible rows still CLAIMING compatibility (drains to 0 after remediation B;
--    lights up again if a future scrape inserts a new impossible spec_match row).
create or replace view v_compat_speed_errors as
select c.id                                     as compat_id,
       s.model                                  as switch,
       e.effective_max_gbps                     as switch_max_gbps,
       t.part_number                            as transceiver,
       t.speed_gbps                             as transceiver_gbps,
       round(t.speed_gbps / nullif(e.effective_max_gbps,0), 1) as over_ratio
from compatibility c
join v_switch_effective_max e on e.switch_id = c.switch_id
join switches s     on s.id = c.switch_id
join transceivers t on t.id = c.transceiver_id
where c.status = 'compatible'
  and c.verification_method = 'spec_match'
  and e.effective_max_gbps > 0
  and t.speed_gbps is not null
  and t.speed_gbps > e.effective_max_gbps;

-- 2) Understated switch port data: a CURATED source pairs a transceiver faster than the
--    switch's stored/ports max. These are NOT errors to auto-fix -- they flag switches
--    whose ports_config needs datasheet enrichment (or a wrong curated claim to review).
create or replace view v_switch_understated_ports as
select c.id                                  as compat_id,
       v.name                                as manufacturer,
       s.model                               as switch,
       s.max_speed_gbps                      as stored_max_gbps,
       fn_ports_max_gbps(s.ports_config)     as ports_max_gbps,
       t.part_number                         as transceiver,
       t.speed_gbps                          as transceiver_gbps,
       c.verification_method
from compatibility c
join switches s     on s.id = c.switch_id
join vendors v      on v.id = s.vendor_id
join transceivers t on t.id = c.transceiver_id
where c.status = 'compatible'
  and c.verification_method in ('vendor_matrix','vendor_compat')
  and t.speed_gbps is not null
  and (coalesce(s.max_speed_gbps,0) > 0 or coalesce(fn_ports_max_gbps(s.ports_config),0) > 0)
  and t.speed_gbps > greatest(coalesce(s.max_speed_gbps,0), coalesce(fn_ports_max_gbps(s.ports_config),0));

-- 3) Coding-pattern coverage per speed tier (module only) -- data-SOURCE gap, pipeline task.
create or replace view v_coding_coverage as
select t.speed_gbps::numeric as tier,
       count(*)::int          as modules,
       count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)::int as with_coding,
       round(100.0 * count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)
             / count(*), 1)   as coverage_pct
from transceivers t
where t.product_type = 'module' and t.speed_gbps >= 100
group by 1 order by 1;

-- 4) Transceivers with an impossible speed (<=0). 100 Mbps (0.1) and FC/SONET rates
--    (0.155/0.622/2.488/8/8.5/14/28/32) are legitimate and intentionally NOT flagged.
create or replace view v_transceiver_speed_errors as
select id, part_number, speed_gbps, form_factor
from transceivers
where speed_gbps is null or speed_gbps <= 0;

-- 5) Switches that cannot validate compatibility (missing max_speed / total_ports).
create or replace view v_switch_completeness as
select s.id, v.name as manufacturer, s.model, s.max_speed_gbps, s.total_ports,
       (jsonb_typeof(s.ports_config) = 'object' and s.ports_config <> '{}'::jsonb) as has_ports_config
from switches s join vendors v on v.id = s.vendor_id
where s.max_speed_gbps is null or s.total_ports is null;

-- 6) One-line alerts = the switch/coding data-quality task list.
create or replace view v_switch_quality_alerts as
select 'compatibility: ' || count(*) || ' physically-impossible spec_match rows (transceiver faster than switch)' as alert,
       'critical' as severity from v_compat_speed_errors
union all
select 'coding matrix: ' || tier || 'G only ' || coverage_pct || '% populated (' || (modules - with_coding) || ' modules without coding patterns)',
       case when coverage_pct < 25 then 'high' else 'warning' end
from v_coding_coverage where coverage_pct < 70
union all
select 'transceivers: ' || count(*) || ' with speed_gbps <= 0 (need form_factor/datasheet)', 'high'
from v_transceiver_speed_errors
union all
select 'switches: ' || count(*) || ' missing max_speed_gbps (empty ports_config -> needs datasheet)', 'warning'
from v_switch_completeness where max_speed_gbps is null
union all
select 'switches: ' || count(*) || ' with understated port data (curated compat faster than ports_config)', 'warning'
from v_switch_understated_ports;

-- Usage:  select * from v_switch_quality_alerts order by severity;
--         select * from v_compat_speed_errors order by over_ratio desc limit 50;
--         select * from v_switch_understated_ports order by transceiver_gbps desc;

insert into _migrations (filename, applied_at)
select '125-switch-quality.sql', now()
where not exists (select 1 from _migrations where filename = '125-switch-quality.sql');