transceiver-db/sql/135-modulation-fec-enrichment-and-speed-guard.sql

-- 135  Modulation/FEC enrichment + speed plausibility guard for transceivers.
--
-- Derived and adversarially verified via workflow wf_dc5bab06 (4 optical-standards
-- lenses -> synthesis -> 4 skeptical reviewers). All CONFIRMED reviewer findings are
-- folded in here: coherent >=100G speed floor (10G-ZR is NRZ, not DCO), form_factor
-- DCO/ACO scan (CFP2-DCO evidence lives in form_factor), compact DAC/AOC/AEC suffix
-- exclusion, bare-QSFP28-no-token ABSTAINS instead of guessing NRZ, CR4 dropped from
-- the NRZ set, anchored MTP/MPO, NON-COHERENT guard, search_path pinned.
--
-- FILL-ONLY: never overwrites a non-null value, never touches data_confidence='vendor_verified'.
-- Idempotent: safe to re-run; a fully enriched table yields 0 rows touched.

-- ---------------------------------------------------------------------------
-- Speed plausibility. Flags only UNAMBIGUOUS junk: sub-0.09 fragments (incl. <=0;
-- the lowest real optical rate is 100M=0.1) and >1600 (cable-part-number integers
-- like 2.0e16). The 0.09..1600 band is deliberately kept intact: it holds legit
-- non-Ethernet rates (FC line rates 2.125/4.25/8.5/14.025, SDI 3/6/12, CPRI 1.2288,
-- OTN 10.7) that must not be discarded. NULL is treated as NOT implausible.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION tip_speed_is_implausible(v numeric)
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT v IS NOT NULL AND (v < 0.09 OR v > 1600);
$$;

-- Separate, isolated speed sanitizer. Distinct from tip_transceivers_biu() (which
-- recomputes product_type/form_factor and never touches speed). Sanitize-only:
-- NULLs junk speed on write, never raises, never blocks ingestion.
CREATE OR REPLACE FUNCTION tip_speed_plausibility_biu()
RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
BEGIN
  IF NEW.speed_gbps IS NOT NULL AND tip_speed_is_implausible(NEW.speed_gbps) THEN
    NEW.speed_gbps := NULL;
  END IF;
  RETURN NEW;
END;
$$;

DROP TRIGGER IF EXISTS trg_speed_plausibility ON transceivers;
CREATE TRIGGER trg_speed_plausibility
  BEFORE INSERT OR UPDATE ON transceivers
  FOR EACH ROW EXECUTE FUNCTION tip_speed_plausibility_biu();

-- ---------------------------------------------------------------------------
-- Enrichment scope: real optics only. The part_number cable/trunk veto applies to
-- BOTH the product_type='module' and product_type IS NULL branches, so a cable
-- mis-classified 'module' upstream still cannot be enriched.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION tip_in_optics_scope(p_product_type text, p_form_factor text, p_part_number text)
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT
    coalesce(p_product_type,'') NOT IN ('dac','aoc','aec','accessory','switch','nic')
    -- compact suffix forms (-AOC3M, -DAC1M, -AEC2M), Flexoptix cable schemes (M4.*, S2.*),
    -- loopbacks, and explicit cable keywords are never optics. MTP/MPO is deliberately
    -- NOT vetoed here: parallel optics (DR4/SR4/DR8) legitimately carry an -MPO12 connector
    -- suffix, and real fiber trunks are already caught by ^M[0-9]./^S[0-9]./CABLE.
    AND coalesce(p_part_number,'') !~* '([-_](DAC|AOC|AEC)([-_0-9]|$)|CAB[-_]|CABLE|FN-CABLE|LOOPBACK|[-_]LB[0-9]|^M[0-9]\.|^S[0-9]\.)'
    AND (
      p_product_type = 'module'
      OR (
        p_product_type IS NULL
        AND upper(coalesce(p_form_factor,'')) IN
          ('QSFP28','QSFP56','QSFP-DD','QSFP-DD800','OSFP','OSFP800','CFP2','CFP2-DCO','CFP2-ACO','CFP8','SFP-DD','SFP112','DSFP')
      )
    );
$$;

-- ---------------------------------------------------------------------------
-- Coherent detection. Regex-derived triggers are floored at >=100G (100ZR is the
-- lowest coherent pluggable; 10G/40G-ZR are NRZ reach optics). A pre-existing
-- coherent=true is trusted and NOT speed-floored. NON-COHERENT is excluded.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION tip_is_coherent(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT
    p_coherent IS TRUE
    OR (
      p_speed >= 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')||' '||coalesce(p_ff,'')) !~* 'NON-?COHERENT'
      AND (
           (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])(100|200|300|400|600|800|1600)?G?-?ZR(\+|P)?([^A-Za-z0-9]|$)'
        OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])DCO([^A-Za-z0-9]|$)'
        OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* 'OPEN-?ZR'
        OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])COHERENT([^A-Za-z0-9]|$)'
        OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(16QAM|8QAM|64QAM|QPSK)'
        -- CFP2-DCO/ACO carry the only coherent evidence in form_factor (standard_name ~1% populated).
        OR upper(coalesce(p_ff,'')) ~ '(DCO|ACO|COHERENT)'
      )
    );
$$;

-- Modulation. NULL = ABSTAIN (unknown). Order: coherent -> PAM4-by-speed ->
-- 100G single-lambda PAM4 -> 100G 4x25 NRZ -> abstain. Bare QSFP28 with no reach
-- token abstains (never defaulted to NRZ). (?![0-9]) keeps SR4 from matching SR40
-- and DR/FR from matching DR4/FR4.
CREATE OR REPLACE FUNCTION tip_derive_modulation(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT CASE
    WHEN tip_is_coherent(p_std, p_pn, p_ff, p_speed, p_coherent) THEN CASE
        WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?QPSK' THEN 'DP-QPSK'
        WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?16QAM' THEN 'DP-16QAM'
        WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(8QAM|64QAM)' THEN 'Coherent'
        WHEN p_speed >= 400 THEN 'DP-16QAM'   -- 400ZR/800ZR/DCO default
        WHEN p_speed = 100 THEN 'DP-QPSK'     -- 100ZR is QPSK
        ELSE 'Coherent'                       -- 200G/300G constellation ambiguous -> generic
      END
    WHEN p_speed IN (200,400,800,1600) THEN 'PAM4'
    WHEN p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
    THEN NULL   -- conflicting single-lambda AND 4x25 tokens -> abstain
    WHEN p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
    THEN 'PAM4'
    WHEN p_speed = 100
      AND upper(coalesce(p_ff,'')) IN ('SFP-DD','SFP112','DSFP')
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) !~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
    THEN 'PAM4'
    WHEN p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
    THEN 'NRZ'
    ELSE NULL   -- bare QSFP28 / no token / sub-100G / non-standard speed -> abstain
  END;
$$;

-- FEC type. Reads modulation. Abstains (NULL) for 100G NRZ LR4/ER4/CWDM4/CLR4/4WDM
-- (no mandatory FEC); that assertion is carried by inbuilt_fec=false, not a fake fec_type.
CREATE OR REPLACE FUNCTION tip_derive_fec_type(p_modulation text, p_std text, p_pn text, p_speed numeric)
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT CASE
    WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN 'SD-FEC'
    WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN 'RS-FEC (RS(544,514) KP4)'
    WHEN p_modulation = 'NRZ' AND p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN 'RS-FEC (RS(528,514) CL91)'
    ELSE NULL
  END;
$$;

CREATE OR REPLACE FUNCTION tip_derive_inbuilt_fec(p_modulation text, p_std text, p_pn text, p_speed numeric)
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
SET search_path = pg_catalog, public AS $$
  SELECT CASE
    WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN true
    WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN true
    WHEN p_modulation = 'NRZ' AND p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN true
    WHEN p_modulation = 'NRZ' AND p_speed = 100
      AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(LR4|ER4L?|CWDM4|CLR4|4WDM)(?![0-9])' THEN false
    ELSE NULL
  END;
$$;

-- ---------------------------------------------------------------------------
-- Idempotent fill-only enrichment. Returns total per-statement rows touched
-- (a row filled across all three steps counts up to 3x). Fully enriched -> 0.
-- One SELECT call runs all three steps in a single transaction, so APPLY-3 reads
-- the modulation APPLY-2 just set.
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION fn_tip_enrich_modulation_fec()
RETURNS integer LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
DECLARE n1 int := 0; n2 int := 0; n3 int := 0;
BEGIN
  -- 1) coherent flag (set true only; never flip an existing true->false)
  UPDATE transceivers t
  SET coherent = true,
      enriched_at = now(),
      enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['coherent']) e),
      data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
  WHERE coalesce(t.data_confidence,'') <> 'vendor_verified'
    AND t.speed_gbps IS NOT NULL
    AND t.coherent IS DISTINCT FROM true
    AND tip_in_optics_scope(t.product_type, t.form_factor, t.part_number)
    AND tip_is_coherent(t.standard_name, t.part_number, t.form_factor, t.speed_gbps, t.coherent);
  GET DIAGNOSTICS n1 = ROW_COUNT;

  -- 2) modulation (fill-only)
  UPDATE transceivers t
  SET modulation = m.val,
      enriched_at = now(),
      enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['modulation']) e),
      data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
  FROM (
    SELECT ctid AS cid, tip_derive_modulation(standard_name, part_number, form_factor, speed_gbps, coherent) AS val
    FROM transceivers
    WHERE modulation IS NULL
      AND coalesce(data_confidence,'') <> 'vendor_verified'
      AND speed_gbps IS NOT NULL
      AND tip_in_optics_scope(product_type, form_factor, part_number)
  ) m
  WHERE t.ctid = m.cid AND m.val IS NOT NULL;
  GET DIAGNOSTICS n2 = ROW_COUNT;

  -- 3) fec_type + inbuilt_fec (reads modulation set in step 2)
  UPDATE transceivers t
  SET fec_type    = CASE WHEN t.fec_type IS NULL    AND f.fec IS NOT NULL THEN f.fec ELSE t.fec_type END,
      inbuilt_fec = CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN f.inb ELSE t.inbuilt_fec END,
      enriched_at = now(),
      enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(
          coalesce(t.enriched_fields,'{}'::text[])
          || CASE WHEN t.fec_type IS NULL    AND f.fec IS NOT NULL THEN ARRAY['fec_type']    ELSE '{}'::text[] END
          || CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN ARRAY['inbuilt_fec'] ELSE '{}'::text[] END) e),
      data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
  FROM (
    SELECT ctid AS cid,
           tip_derive_fec_type(modulation, standard_name, part_number, speed_gbps)    AS fec,
           tip_derive_inbuilt_fec(modulation, standard_name, part_number, speed_gbps) AS inb
    FROM transceivers
    WHERE coalesce(data_confidence,'') <> 'vendor_verified'
      AND speed_gbps IS NOT NULL
      AND (fec_type IS NULL OR inbuilt_fec IS NULL)
      AND tip_in_optics_scope(product_type, form_factor, part_number)
  ) f
  WHERE t.ctid = f.cid
    AND ((t.fec_type IS NULL AND f.fec IS NOT NULL) OR (t.inbuilt_fec IS NULL AND f.inb IS NOT NULL));
  GET DIAGNOSTICS n3 = ROW_COUNT;

  RETURN n1 + n2 + n3;
END;
$$;

-- Run enrichment:  SELECT fn_tip_enrich_modulation_fec();