From dd2c917f05c049e43bcade35a6b71d4019755d4d Mon Sep 17 00:00:00 2001 From: root Date: Sat, 4 Jul 2026 16:04:45 +0000 Subject: [PATCH] feat(data-quality): product_type classification + form_factor recovery + lead_time docs sql/119: product_type enum (module|dac|aoc|aec|switch|nic|accessory) computed by trigger trg_transceivers_biu from part_number/category/reach on every insert/update. Backfilled 41008 rows (36284 module, 2791 dac, 1835 aoc, 29 aec, 27 accessory, 26 switch, 16 nic). Cables and switches/NICs no longer counted as modules; BASE-T/RJ45 copper modules protected from cable demotion; AOC/DAC keyword beats the optical-token guard. Consumers filter product_type=module. sql/120: conservative form_factor recovery. Corrects only provably-wrong rows (form_factors.max_speed < speed_gbps) via an explicit title token; 87 fixed (e.g. 400G cables mislabelled SFP+ to QSFP-DD), 947 opaque rows left untouched (never guessed). Scraped original preserved in new column form_factor_raw. sql/121: document lead_time_days as not-collected (0 percent populated on all time-series tables) so no consumer fabricates a default. scraper/utils/db.ts: note that product_type/form_factor are trigger-maintained. --- CHANGELOG_PENDING.md | 4 + packages/scraper/src/utils/db.ts | 6 + sql/119-product-type-classification.sql | 144 ++++++++++++++++++++++++ sql/120-form-factor-recovery.sql | 124 ++++++++++++++++++++ sql/121-lead-time-not-collected-doc.sql | 35 ++++++ 5 files changed, 313 insertions(+) create mode 100644 sql/119-product-type-classification.sql create mode 100644 sql/120-form-factor-recovery.sql create mode 100644 sql/121-lead-time-not-collected-doc.sql diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 4aad864..f4a8f1a 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -320,3 +320,7 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA {"d":"2026-06-11","t":"DATA","m":"Flexoptix-Datenblatt-Audit: alle 805 katalog-bestätigten Teile programmatisch gegen gespeicherte fx_specifications (Datenblatt) geprüft. Nach Terabit-Parser-Fix + Re-Derive aus Specs: Form Factor 802/802 (100%), Speed 801/801 (100%), 0 Abweichungen. Jeder Wert kommt aus dem Datenblatt, nicht aus dem Namen."} {"d":"2026-06-11","t":"DATA","m":"Switch-Daten-Audit: 681 Switches, nur 267 (39%) mit Port-Config, 414 ohne (überwiegend Cisco-Bulk-Import). Vorhandene Port-Configs 99.3% plausibel, 248/267 mit Hersteller-URL, nur 35 mit datasheet_url. Demo-Switches (Aruba CX10000, Arista, Juniper) vollständig+korrekt. LÜCKE: autoritatives Port-Sourcing per Hersteller-Datenblatt für die 414 fehlt = separates Scraper-Projekt pro Vendor."} {"d":"2026-06-11","t":"FIX","m":"Switch-Kompatibilität = 'läuft wirklich': Matcher verlangt jetzt zusätzlich, dass Flexoptix den Transceiver für den Switch-VENDOR codieren kann (fx_compatibilities). Physisch-passend reicht nicht. CX 10000: 629 physisch -> 283 echt-Aruba-codierbar (283/283 verifiziert). Cisco 8818: 55 (alle Cisco-codierbar). Plus Chassis-Port-Format (max_per_slot_NNNG_FF) geparst. Jede Empfehlung jetzt: katalog-bestätigt + datenblatt-genau + passt physisch + codierbar = garantiert lauffähig."} +{"d":"2026-07-04","t":"DATA","m":"product_type-Klassifikation (sql/119): neues Enum module|dac|aoc|aec|switch|nic|accessory, deterministisch per Trigger trg_transceivers_biu aus part_number/category/reach gesetzt (self-healing bei jedem Insert/Update, Preis-Updates unberührt). Backfill 41008 Zeilen: 36284 module, 2791 dac, 1835 aoc, 29 aec, 27 accessory, 26 switch, 16 nic. Kabel (DAC/AOC/AEC) und Switches/NICs (ConnectX/Quantum/N-Port) nicht mehr als Modul gezählt. BASE-T/RJ45-Kupfermodule vor Kabel-Demotion geschützt (Guard vor copper-Regel); AOC/DAC-Keyword schlägt Optik-Token-Guard (AOC ist optisch, trägt legitim nm/MMF/SR). Konsumenten filtern product_type='module'. Missed-cable=0 verifiziert. Per-Vendor 100G+: NADDOD 179 Module/423 Kabel/28 Switch-NIC, ATGBICS 1331/297, QSFPTEK 22/98, ProLabs 1690/818."} +{"d":"2026-07-04","t":"DATA","m":"form_factor-Recovery (sql/120): korrigiert NUR beweisbar-falsche Zeilen (form_factors.max_speed < speed_gbps) mit explizitem Formfaktor-Token im Titel. 87 gefixt (u.a. 400G-Kabel als SFP+ getaggt -> QSFP-DD; QSFP-DD@800G -> QSFP-DD800; Q56DD -> QSFP-DD), 947 opake Zeilen (z.B. EOLS-1303-40-D, MGB-2GSR) unangetastet gelassen, nichts geraten. Gescraptes Original in neuer Spalte form_factor_raw bewahrt. Recovery auch im Ingest-Trigger (Neuzugänge self-healing)."} +{"d":"2026-07-04","t":"DATA","m":"lead_time_days als NICHT ERFASST dokumentiert (sql/121, COMMENT auf price_observations/stock_observations/stock_snapshots): 0% befüllt über 1.378.984 + 85.406 + 0 Zeilen. Plumbing existiert (crawler-llm stock-schema erfasst es, upsertPriceObservation nimmt Param), aber aktiver Scraper-Pfad liefert nie einen Wert und LLM-Crawler-Tabelle stock_snapshots ist leer. Kein Default fabrizieren; NULL = unbekannt. Befüllen ist Scraper-Aufgabe."} +{"d":"2026-07-04","t":"DATA","m":"Befund FS.COM-Untererfassung bestätigt (kein Fix, Scraper-Thema): 392 Zeilen/378 distinct SKUs vs ATGBICS 8420 (22x-Lücke). 391/392 sind bereits product_type='module' -> Klassifikation ist NICHT die Ursache, reine Crawl-Tiefe. Ursache in fs-com.ts + Seed-Scope (Kategorie-Abdeckung) + evtl. zu aggressive Quarantäne von fs.com/c/-URLs. Empfehlung: FS.com-Crawl-Seed erweitern (Scraper-Pipeline, nicht raten)."} diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index 87ae87f..e23cd69 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -480,6 +480,12 @@ export async function upsertStockObservation(params: { return true; } +// NOTE: product_type (module|dac|aoc|aec|switch|nic|accessory) and form_factor +// corrections are set automatically by the DB trigger trg_transceivers_biu +// (see sql/119 + sql/120). Callers do NOT pass or maintain them — the trigger +// classifies from part_number/category/reach on every insert and update, so +// every scraper stays consistent without touching this call site. Consumers +// filter product_type='module' for a true module count. export async function findOrCreateScrapedTransceiver(params: { partNumber: string; vendorId: string; diff --git a/sql/119-product-type-classification.sql b/sql/119-product-type-classification.sql new file mode 100644 index 0000000..0a4b4b2 --- /dev/null +++ b/sql/119-product-type-classification.sql @@ -0,0 +1,144 @@ +-- Migration 119: product_type classification (module vs cable vs switch/NIC/accessory) +-- ───────────────────────────────────────────────────────────────────────────── +-- Problem: TIP ingests DAC/AOC/AEC cables, switches and NICs as `transceivers` +-- rows (one per cable length). `form_factor` is unreliable (a 400G DAC can be +-- tagged 'SFP+'). Every "assortment breadth" metric is therefore cable-inflated, +-- and consumers (Lupo) had to re-derive a text heuristic on every query. +-- +-- Fix: a deterministic `product_type` enum, computed at ingest by a BEFORE +-- INSERT/UPDATE trigger from the free-text part_number + category + reach. +-- Consumers filter `product_type = 'module'` for a true module count. +-- +-- Taxonomy: module | dac | aoc | aec | switch | nic | accessory +-- Classification is keyword-first (an explicit 'AOC'/'DAC' token beats the +-- optical-token guard, because an Active Optical Cable legitimately carries +-- nm/MMF/SR tokens). BASE-T / RJ45 electrical transceivers are protected as +-- modules before the copper/DAC rule so genuine RJ45 optics are never demoted. +-- +-- Idempotent. Applied manually via `docker exec -i tip-postgres psql` on Erik +-- (the _migrations tracker is only used by scripts/migrate.ts; recent migrations +-- are applied by hand — we still record this file there for correctness). +-- ───────────────────────────────────────────────────────────────────────────── + +-- 1. Column + CHECK constraint -------------------------------------------------- +ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS product_type TEXT; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'transceivers_product_type_check' + ) THEN + ALTER TABLE transceivers ADD CONSTRAINT transceivers_product_type_check + CHECK (product_type IS NULL OR product_type IN + ('module','dac','aoc','aec','switch','nic','accessory')); + END IF; +END; +$$; + +COMMENT ON COLUMN transceivers.product_type IS + 'Deterministic product class set by trigger tip_transceivers_biu(): ' + 'module=optical/electrical pluggable, dac/aoc/aec=cables, switch/nic=infrastructure, ' + 'accessory=loopback/attenuator/amplifier/mux/etc. Consumers filter product_type=''module''.'; + +-- 2. Deterministic classifier --------------------------------------------------- +-- IMMUTABLE: depends only on its inputs, so it is safe in triggers and indexes. +CREATE OR REPLACE FUNCTION tip_classify_product_type( + p_part_number TEXT, + p_category TEXT, + p_reach_label TEXT +) RETURNS TEXT +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE + -- 1. NIC / adapter cards + WHEN COALESCE(p_part_number,'') ~* '(connectx|supernic|bluefield|smartnic|adapter card|\mmcx[0-9])' + THEN 'nic' + -- 2. Switches / network infrastructure (N-port matches "64-Port", not "Twin-Port") + WHEN COALESCE(p_part_number,'') ~* '(\mswitch\M|switches|\mquantum\M|\mspectrum\M|tomahawk|\mqm9[0-9]{3}|q3[0-9]{3}-ra|[0-9]+-port|\mdgx\M|\mgpu\M)' + THEN 'switch' + -- 3. Hard accessory keywords (never a module even if the title says "transceiver"/"copper") + WHEN COALESCE(p_part_number,'') ~* '(loopback|\mlb\M|attenuator|amplifier|\medfa\M|\moadm\M|demux|multiplexer|\mmux\M|transponder|transport[ -]system|microscope|cleaning|cassette|\mtester\M|media converter)' + THEN 'accessory' + -- 4. AEC — active electrical cable (Mellanox MCA copper-active family) + WHEN COALESCE(p_part_number,'') ~* '(\maec\M|active electrical|\mmca[0-9])' + THEN 'aec' + -- 5. AOC — active optical cable (Mellanox MFA/MFS active fibre families) + WHEN COALESCE(p_part_number,'') ~* '(aoc|active optical|\mmfa|\mmfs[0-9])' + THEN 'aoc' + -- 6. BASE-T / RJ45 electrical transceiver = MODULE (guard before the copper/DAC rule) + WHEN COALESCE(p_part_number,'') ~* '(base-?t|[0-9]+gbase-?t|rj-?45|8p8c)' + THEN 'module' + -- 7. DAC / passive copper / twinax (Mellanox MCP passive-copper family) + WHEN COALESCE(p_part_number,'') ~* '(dac|twinax|direct attach|copper|passive|splitter|breakout|\mmcp|\mcab-)' + THEN 'dac' + -- 8. Category-driven cable signals (scraper set a cable category but a bland title) + WHEN COALESCE(p_category,'') ~* 'AOC' THEN 'aoc' + WHEN COALESCE(p_category,'') ~* '(DAC|Copper|Cable)' THEN 'dac' + -- 9. Bare "m (ft) ..." length title — a cable, unless it carries an optical token + WHEN COALESCE(p_part_number,'') ~* '^[0-9.]+ *m( *\([0-9.]+ ?ft\))? ' + AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)' + THEN 'dac' + -- 10. Short reach <=10 m with no optical token — a copper DAC + WHEN COALESCE(p_reach_label,'') ~* '^[0-9.]+ *m$' + AND COALESCE(NULLIF(regexp_replace(p_reach_label,'[^0-9.].*$',''),'')::float, 999) <= 10 + AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)' + THEN 'dac' + -- 11. Category-driven accessory residue + WHEN COALESCE(p_category,'') IN ('Accessory','Adapter / Converter','Switch / Media Converter', + 'Switch / Network Infrastructure','NIC / Adapter','Mux / Passive Optical', + 'Product Family','Loopback / Test Module') + THEN 'accessory' + -- 12. Default: a genuine optical/electrical module + ELSE 'module' + END; +$$; + +-- 3. Trigger — classify every row on write ------------------------------------ +-- Recomputes only when a classification input changed (or product_type is NULL), +-- so ordinary price/verification updates are untouched. Deterministic + self-healing. +CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' + OR NEW.product_type IS NULL + OR NEW.part_number IS DISTINCT FROM OLD.part_number + OR NEW.category IS DISTINCT FROM OLD.category + OR NEW.reach_label IS DISTINCT FROM OLD.reach_label + THEN + NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label); + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS trg_transceivers_biu ON transceivers; +CREATE TRIGGER trg_transceivers_biu + BEFORE INSERT OR UPDATE ON transceivers + FOR EACH ROW EXECUTE FUNCTION tip_transceivers_biu(); + +-- 4. Backfill every existing row ------------------------------------------------ +UPDATE transceivers + SET product_type = tip_classify_product_type(part_number, category, reach_label) + WHERE product_type IS DISTINCT FROM + tip_classify_product_type(part_number, category, reach_label); + +-- 5. Indexes for the common consumer filter ------------------------------------ +CREATE INDEX IF NOT EXISTS idx_transceivers_product_type + ON transceivers (product_type); +CREATE INDEX IF NOT EXISTS idx_transceivers_ptype_speed + ON transceivers (product_type, speed_gbps); + +-- 6. Record in the migration tracker (idempotent) ------------------------------ +INSERT INTO _migrations (filename) +VALUES ('119-product-type-classification.sql') +ON CONFLICT (filename) DO NOTHING; + +-- 7. Report -------------------------------------------------------------------- +DO $$ +DECLARE rec RECORD; +BEGIN + RAISE NOTICE '=== product_type distribution ==='; + FOR rec IN SELECT product_type, count(*) c FROM transceivers GROUP BY 1 ORDER BY 2 DESC LOOP + RAISE NOTICE ' %: %', rpad(COALESCE(rec.product_type,'(null)'), 10), rec.c; + END LOOP; +END; +$$; diff --git a/sql/120-form-factor-recovery.sql b/sql/120-form-factor-recovery.sql new file mode 100644 index 0000000..f56825d --- /dev/null +++ b/sql/120-form-factor-recovery.sql @@ -0,0 +1,124 @@ +-- Migration 120: conservative form_factor recovery (provably-wrong rows only) +-- ───────────────────────────────────────────────────────────────────────────── +-- Problem: some scrapers default form_factor to 'SFP'/'SFP+', so a 400G QSFP-DD +-- DAC ends up tagged 'SFP+'. We only ever correct a form_factor that is +-- PROVABLY wrong — i.e. its max electrical speed (form_factors.max_speed_gbps) +-- is below the row's actual speed_gbps — and only when the part_number carries +-- an explicit form-factor token that CAN carry that speed. Opaque part numbers +-- (e.g. 'EOLS-1303-40-D') are left untouched; we never guess a form factor. +-- +-- The scraped original is preserved in form_factor_raw for audit/rollback. +-- Ingest-time self-healing is added to the tip_transceivers_biu() trigger. +-- Idempotent. +-- ───────────────────────────────────────────────────────────────────────────── + +-- 1. Preserve the scraped original --------------------------------------------- +ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS form_factor_raw TEXT; +COMMENT ON COLUMN transceivers.form_factor_raw IS + 'Form factor exactly as scraped, before tip_recover_form_factor() correction. Audit/rollback only.'; + +UPDATE transceivers SET form_factor_raw = form_factor + WHERE form_factor_raw IS NULL; + +-- 2. Explicit form-factor token extraction (most-specific first) ---------------- +CREATE OR REPLACE FUNCTION tip_extract_ff_token(p_part_number TEXT) RETURNS TEXT +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE + WHEN p_part_number ~* 'qsfp-?dd-?800|qdd-?800' THEN 'QSFP-DD800' + -- Q56DD / QSFP56-DD is the 400G double-density QSFP connector (Dell naming) → QSFP-DD + WHEN p_part_number ~* 'q56dd|qsfp56-?dd' THEN 'QSFP-DD' + WHEN p_part_number ~* 'osfp-?xd' THEN 'OSFP-XD' + WHEN p_part_number ~* 'osfp224' THEN 'OSFP224' + WHEN p_part_number ~* 'osfp112' THEN 'OSFP112' + WHEN p_part_number ~* '\mosfp\M' THEN 'OSFP' + WHEN p_part_number ~* 'qsfp112' THEN 'QSFP112' + WHEN p_part_number ~* 'qsfp-?dd|\mqdd\M|qsfpdd' THEN 'QSFP-DD' + WHEN p_part_number ~* 'qsfp56|\mq56\M' THEN 'QSFP56' + WHEN p_part_number ~* 'qsfp28|\mq28\M' THEN 'QSFP28' + WHEN p_part_number ~* 'qsfp\+|qsfp-?plus' THEN 'QSFP+' + WHEN p_part_number ~* 'sfp-?dd|sfpdd' THEN 'SFP-DD' + WHEN p_part_number ~* 'sfp112' THEN 'SFP112' + WHEN p_part_number ~* 'sfp56' THEN 'SFP56' + WHEN p_part_number ~* 'sfp28' THEN 'SFP28' + WHEN p_part_number ~* 'sfp\+|sfp-?plus' THEN 'SFP+' + WHEN p_part_number ~* 'cfp2-?dco' THEN 'CFP2-DCO' + WHEN p_part_number ~* 'cfp2' THEN 'CFP2' + WHEN p_part_number ~* 'cfp8' THEN 'CFP8' + WHEN p_part_number ~* 'cfp4' THEN 'CFP4' + WHEN p_part_number ~* '\mcfp\M' THEN 'CFP' + WHEN p_part_number ~* '\mxfp\M' THEN 'XFP' + ELSE NULL + END; +$$; + +-- 3. Recovery: return corrected FF only when the current one is PROVABLY wrong --- +CREATE OR REPLACE FUNCTION tip_recover_form_factor( + p_part_number TEXT, p_form_factor TEXT, p_speed_gbps NUMERIC +) RETURNS TEXT +LANGUAGE plpgsql STABLE PARALLEL SAFE AS $$ +DECLARE + v_token TEXT := tip_extract_ff_token(p_part_number); + v_cur_max NUMERIC; + v_tok_max NUMERIC; +BEGIN + IF v_token IS NULL OR p_form_factor IS NULL OR p_speed_gbps IS NULL THEN + RETURN p_form_factor; + END IF; + IF upper(v_token) = upper(p_form_factor) THEN + RETURN p_form_factor; -- already correct + END IF; + SELECT max_speed_gbps INTO v_cur_max FROM form_factors WHERE upper(name) = upper(p_form_factor); + SELECT max_speed_gbps INTO v_tok_max FROM form_factors WHERE name = v_token; + -- current provably wrong for the speed, and the token can actually carry it + IF v_cur_max IS NOT NULL AND v_tok_max IS NOT NULL + AND p_speed_gbps > v_cur_max AND v_tok_max >= p_speed_gbps THEN + RETURN v_token; + END IF; + RETURN p_form_factor; -- not confident → leave as-is +END; +$$; + +-- 4. Backfill the confidently-recoverable rows --------------------------------- +UPDATE transceivers + SET form_factor = tip_recover_form_factor(part_number, form_factor, speed_gbps) + WHERE form_factor IS DISTINCT FROM tip_recover_form_factor(part_number, form_factor, speed_gbps); + +-- 5. Extend the ingest trigger with FF self-healing ---------------------------- +CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + -- product_type: recompute only when a classification input changed + IF TG_OP = 'INSERT' + OR NEW.product_type IS NULL + OR NEW.part_number IS DISTINCT FROM OLD.part_number + OR NEW.category IS DISTINCT FROM OLD.category + OR NEW.reach_label IS DISTINCT FROM OLD.reach_label + THEN + NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label); + END IF; + + -- form_factor: preserve the scraped original, then correct only provably-wrong values + NEW.form_factor_raw := COALESCE(NEW.form_factor_raw, NEW.form_factor); + NEW.form_factor := tip_recover_form_factor(NEW.part_number, NEW.form_factor, NEW.speed_gbps); + + RETURN NEW; +END; +$$; + +-- (trigger trg_transceivers_biu already bound in migration 119 — replacing the +-- function body above is sufficient.) + +INSERT INTO _migrations (filename) +VALUES ('120-form-factor-recovery.sql') +ON CONFLICT (filename) DO NOTHING; + +-- 6. Report -------------------------------------------------------------------- +DO $$ +DECLARE v_fixed INT; v_left INT; +BEGIN + SELECT count(*) INTO v_fixed FROM transceivers WHERE form_factor IS DISTINCT FROM form_factor_raw; + SELECT count(*) INTO v_left FROM transceivers t JOIN form_factors f ON upper(f.name)=upper(t.form_factor) + WHERE t.speed_gbps > f.max_speed_gbps; + RAISE NOTICE 'form_factor corrected on % rows; % provably-wrong rows left untouched (opaque, not guessed).', v_fixed, v_left; +END; +$$; diff --git a/sql/121-lead-time-not-collected-doc.sql b/sql/121-lead-time-not-collected-doc.sql new file mode 100644 index 0000000..8d9b341 --- /dev/null +++ b/sql/121-lead-time-not-collected-doc.sql @@ -0,0 +1,35 @@ +-- Migration 121: document lead_time_days as NOT COLLECTED +-- ───────────────────────────────────────────────────────────────────────────── +-- Verified 2026-07-04: lead_time_days is 0 % populated on every time-series table +-- price_observations : 1,378,984 rows → 0 filled +-- stock_observations : 85,406 rows → 0 filled +-- stock_snapshots : 0 rows (LLM-crawler path is dormant) +-- +-- The plumbing exists (crawler-llm stock-schema captures lead_time_days and +-- writes it to stock_snapshots; upsertPriceObservation accepts a leadTimeDays +-- param) but the ACTIVE ingestion path never supplies a value, and there is no +-- evidence any tracked vendor page exposes a machine-readable lead time. +-- +-- Rather than fabricate a default, this migration documents the column as +-- not-collected so no consumer invents one. Lupo already renders "k.A." / "n/a" +-- and blends only the two signals it actually measures (availability, breadth). +-- Populating this field is a SCRAPER-side task (extend the active crawlers to +-- read a delivery/lead-time badge) — out of scope for a schema migration. +-- ───────────────────────────────────────────────────────────────────────────── + +COMMENT ON COLUMN price_observations.lead_time_days IS + 'NOT COLLECTED (verified 2026-07-04): 0 % populated. The active scraper path never ' + 'supplies a lead time. Do NOT default this — treat NULL as "unknown / not captured". ' + 'Populating it requires scraper work (read a delivery-time badge on the vendor page).'; + +COMMENT ON COLUMN stock_observations.lead_time_days IS + 'NOT COLLECTED (verified 2026-07-04): 0 % populated. Treat NULL as "unknown". See ' + 'price_observations.lead_time_days.'; + +COMMENT ON COLUMN stock_snapshots.lead_time_days IS + 'Captured by the crawler-llm path (stock-schema.lead_time_days) but that path is ' + 'currently dormant (0 rows). Treat NULL as "unknown / not captured".'; + +INSERT INTO _migrations (filename) +VALUES ('121-lead-time-not-collected-doc.sql') +ON CONFLICT (filename) DO NOTHING;