diff --git a/packages/api/src/routes/scrapers.ts b/packages/api/src/routes/scrapers.ts index 92de8b1..f03a56d 100644 --- a/packages/api/src/routes/scrapers.ts +++ b/packages/api/src/routes/scrapers.ts @@ -141,6 +141,53 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => { } }); +// GET /api/scrapers/jobs — Live pg-boss job queue status +scraperRouter.get("/jobs", async (_req: Request, res: Response) => { + try { + const [active, recent, queues] = await Promise.all([ + // Currently active (running) jobs + pool.query(` + SELECT name, id, created_on, started_on, output + FROM pgboss.job + WHERE state = 'active' + ORDER BY started_on DESC + LIMIT 20 + `).catch(() => ({ rows: [] })), + + // Recent completions and failures (last 4 hours) + pool.query(` + SELECT name, state, created_on, started_on, completed_on, + EXTRACT(EPOCH FROM (completed_on - started_on))::int AS duration_sec + FROM pgboss.job + WHERE state IN ('completed', 'failed', 'cancelled') + AND completed_on > NOW() - INTERVAL '4 hours' + ORDER BY completed_on DESC + LIMIT 50 + `).catch(() => ({ rows: [] })), + + // Queue summary: count per job name and state (last 24h) + pool.query(` + SELECT name, state, COUNT(*) as count, + MAX(completed_on) as last_completed, + MAX(started_on) as last_started + FROM pgboss.job + WHERE created_on > NOW() - INTERVAL '24 hours' + GROUP BY name, state + ORDER BY name, state + `).catch(() => ({ rows: [] })), + ]); + + res.json({ + success: true, + active: active.rows, + recent: recent.rows, + queues: queues.rows, + }); + } catch (err) { + res.status(503).json({ success: false, error: String(err) }); + } +}); + // GET /api/scrapers/llm-insights — What the crawler LLM has learned scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => { try { diff --git a/packages/dashboard/index.html b/packages/dashboard/index.html index d820681..ace08aa 100644 --- a/packages/dashboard/index.html +++ b/packages/dashboard/index.html @@ -1384,6 +1384,21 @@ + +
+
+

⚡ Live Job Queue

+ + Loading… + +
+
Loading job queue…
+
+

Recent (last 2h)

+
Loading…
+
+
+

Scraper Status

@@ -4835,7 +4850,7 @@ function renderSignals(filterSig) { + '
' + imgHtml + '
' - + '
' + esc(productName) + (r.is_demo ? demoBadgeHtml : '') + '
' + + '
' + esc(productName) + (r.is_demo_data || r.is_demo ? demoBadgeHtml : '') + '
' + '
' + esc(r.form_factor || '') + (r.speed_gbps ? ' · ' + r.speed_gbps + 'G' : '') + (r.vendor_name ? ' · ' + esc(r.vendor_name) : '') + '
' + '
' + '
' @@ -5001,6 +5016,7 @@ loadChangelog(); // ── CRAWLER INTELLIGENCE ──────────────────────────────────────────── async function loadCrawlerStatus() { + loadCrawlerJobs(); // load live job queue in parallel var token = (window.loadToken ? window.loadToken() : '') || ''; var status = null; var insights = null; @@ -5108,6 +5124,89 @@ async function loadCrawlerStatus() { } +/* ── Crawler Jobs (Live Queue) ──────────────────────────────────────────── */ +async function loadCrawlerJobs() { + var token = (window.loadToken ? window.loadToken() : '') || ''; + var data = null; + try { + var r = await fetch('/api/scrapers/jobs', { headers: { 'Authorization': 'Bearer ' + token } }); + data = await r.json(); + } catch(e) {} + + var active = (data && data.active) || []; + var recent = (data && data.recent) || []; + var dotEl = el('cr-live-dot'); + var countEl = el('cr-active-jobs-count'); + + if (active.length > 0) { + if (dotEl) { dotEl.style.background = '#22c55e'; dotEl.style.boxShadow = '0 0 8px #22c55e'; } + if (countEl) countEl.textContent = active.length + ' job' + (active.length !== 1 ? 's' : '') + ' running'; + } else { + if (dotEl) { dotEl.style.background = '#64748b'; dotEl.style.boxShadow = 'none'; } + if (countEl) countEl.textContent = 'Idle — waiting for next schedule'; + } + + var stateColor = { completed: '#22c55e', failed: '#ef4444', cancelled: '#f59e0b' }; + var liveEl = el('cr-live-jobs'); + if (liveEl) { + if (active.length > 0) { + var liveRows = active.map(function(j) { + var since = (j.started_on || j.startedon) ? Math.round((Date.now() - new Date(j.started_on || j.startedon).getTime()) / 1000) + 's' : '—'; + var row = document.createElement('div'); + row.style.cssText = 'background:rgba(34,197,94,0.08);border:1px solid rgba(34,197,94,0.3);border-radius:6px;padding:0.6rem 0.9rem;display:flex;align-items:center;gap:0.75rem;margin-bottom:0.3rem'; + var dot = document.createElement('span'); + dot.style.cssText = 'width:8px;height:8px;border-radius:50%;background:#22c55e;flex-shrink:0'; + var name = document.createElement('span'); + name.style.cssText = 'font-size:0.82rem;font-weight:600;color:var(--text-bright);flex:1'; + name.textContent = j.name; + var dur = document.createElement('span'); + dur.style.cssText = 'font-size:0.72rem;color:var(--text-dim)'; + dur.textContent = 'running ' + since; + row.appendChild(dot); row.appendChild(name); row.appendChild(dur); + return row; + }); + liveEl.replaceChildren.apply(liveEl, liveRows); + } else { + liveEl.textContent = 'No jobs currently active.'; + liveEl.style.color = 'var(--text-dim)'; + liveEl.style.fontSize = '0.82rem'; + } + } + + var recentEl = el('cr-recent-jobs'); + if (recentEl) { + if (recent.length > 0) { + var rows = recent.slice(0, 20).map(function(j) { + var when = (j.completed_on || j.completedon) ? new Date(j.completed_on || j.completedon).toLocaleTimeString('de-DE') : '—'; + var color = stateColor[j.state] || '#64748b'; + var dur = j.duration_sec != null ? j.duration_sec + 's' : ''; + var row = document.createElement('div'); + row.style.cssText = 'display:flex;align-items:center;gap:0.6rem;font-size:0.75rem;padding:0.35rem 0.6rem;border-radius:4px;background:var(--surface2);border:1px solid var(--border);margin-bottom:0.25rem'; + var dot = document.createElement('span'); + dot.style.cssText = 'width:7px;height:7px;border-radius:50%;background:' + color + ';flex-shrink:0'; + var name = document.createElement('span'); + name.style.cssText = 'flex:1;color:var(--text-bright);font-weight:500'; + name.textContent = j.name; + var durSpan = document.createElement('span'); + durSpan.style.color = 'var(--text-dim)'; + durSpan.textContent = dur; + var state = document.createElement('span'); + state.style.cssText = 'color:' + color + ';font-weight:600;min-width:70px;text-align:right'; + state.textContent = j.state; + var whenSpan = document.createElement('span'); + whenSpan.style.cssText = 'color:var(--text-dim);min-width:55px;text-align:right'; + whenSpan.textContent = when; + row.appendChild(dot); row.appendChild(name); row.appendChild(durSpan); + row.appendChild(state); row.appendChild(whenSpan); + return row; + }); + recentEl.replaceChildren.apply(recentEl, rows); + } else { + recentEl.textContent = 'No recent completions in the last 2 hours.'; + } + } +} + /* ── Smart Tooltips ─────────────────────────────────────────────────────── */ function initSmartTooltips() { var tip = document.createElement('div'); diff --git a/sql/029-seed-standards.sql b/sql/029-seed-standards.sql new file mode 100644 index 0000000..db57c1f --- /dev/null +++ b/sql/029-seed-standards.sql @@ -0,0 +1,137 @@ +-- Migration 029: Seed IEEE/OIF/MSA Standards +-- Authoritative standards for optical transceivers +-- Applied: 2026-04-08 + +INSERT INTO standards (name, ieee_reference, body, speed, speed_gbps, form_factors, max_reach_meters, fiber_type, wavelength, status, year_ratified, notes) +VALUES + +-- 1G +('1000BASE-SX', 'IEEE 802.3z', 'IEEE', '1G', 1, '{SFP,SFP+}', 550, 'MMF', '850nm', 'ratified', 1998, + '850nm VCSEL, 550m on OM2. Legacy GbE, still dominant in campus LAN.'), + +('1000BASE-LX', 'IEEE 802.3z', 'IEEE', '1G', 1, '{SFP,SFP+}', 10000, 'SMF', '1310nm', 'ratified', 1998, + '1310nm DFB, 10km SMF. Mode conditioning patch cable for MMF (550m).'), + +('1000BASE-LX10', 'IEEE 802.3ah', 'IEEE', '1G', 1, '{SFP}', 10000, 'SMF', '1310nm', 'ratified', 2004, + 'EFM standard. Identical to LX but formally defined for access networks.'), + +('1000BASE-BX10', 'IEEE 802.3ah', 'IEEE', '1G', 1, '{SFP}', 10000, 'SMF', '1490nm', 'ratified', 2004, + 'BiDi GbE over single SMF strand. Tx 1310/Rx 1490nm or reverse. FTTH/FTTA.'), + +-- 10G +('10GBASE-SR', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 300, 'MMF', '850nm', 'ratified', 2002, + '300m OM3, 400m OM4. Standard short-reach 10G in data centers.'), + +('10GBASE-LR', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 10000, 'SMF', '1310nm', 'ratified', 2002, + 'DFB laser, 10km SMF. Most common long-reach 10G interface.'), + +('10GBASE-ER', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 40000, 'SMF', '1550nm', 'ratified', 2002, + 'EML or DFB + APD receiver, 40km. Extended reach 10G.'), + +('10GBASE-ZR', NULL, 'de_facto', '10G', 10, '{SFP+,XFP}', 80000, 'SMF', '1550nm', 'ratified', 2003, + 'Vendor de facto, 80km. EDFA-compatible power levels. Not IEEE standardized.'), + +('10GBASE-LRM', 'IEEE 802.3aq', 'IEEE', '10G', 10, '{SFP+,XFP}', 220, 'MMF', '1310nm', 'ratified', 2006, + '220m on OM1/OM2 legacy fiber using electronic dispersion compensation.'), + +('10GBASE-T', 'IEEE 802.3an', 'IEEE', '10G', 10, '{RJ45}', 100, 'copper', NULL, 'ratified', 2006, + '100m on Cat6A copper. High power vs. optics but leverages existing cabling.'), + +-- 25G +('25GBASE-SR', 'IEEE 802.3by', 'IEEE', '25G', 25, '{SFP28}', 100, 'MMF', '850nm', 'ratified', 2016, + '70m OM3, 100m OM4. Dominant server NIC interconnect since 2017.'), + +('25GBASE-LR', 'IEEE 802.3cc', 'IEEE', '25G', 25, '{SFP28}', 10000, 'SMF', '1310nm', 'ratified', 2017, + '10km SMF. Used for ToR-to-spine where longer reach is needed.'), + +('25GBASE-ER', 'IEEE 802.3cc', 'IEEE', '25G', 25, '{SFP28}', 40000, 'SMF', '1310nm', 'ratified', 2017, + '40km SMF extended reach.'), + +-- 40G +('40GBASE-SR4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 150, 'MMF', '850nm', 'ratified', 2010, + '4x10G, 8-fiber MPO. 100m OM3, 150m OM4. Parallel optics for 40G aggregation.'), + +('40GBASE-LR4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 10000, 'SMF', '1310nm', 'ratified', 2010, + '4x10G WDM lanes (1271/1291/1311/1331nm), 10km duplex LC.'), + +('40GBASE-ER4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 40000, 'SMF', '1310nm', 'ratified', 2010, + '40km SMF, same 4x WDM as LR4 with higher power EML.'), + +('40GBASE-PLR4', NULL, 'MSA', '40G', 40, '{QSFP+}', 10000, 'SMF', '1310nm', 'ratified', 2012, + 'PSM4 parallel SMF, 8-fiber MPO. Lower cost than LR4 for parallel fiber runs.'), + +-- 100G +('100GBASE-SR4', 'IEEE 802.3bm', 'IEEE', '100G', 100, '{QSFP28}', 100, 'MMF', '850nm', 'ratified', 2015, + '4x25G, 8-fiber MPO. 70m OM3, 100m OM4. Standard hyperscaler server-to-ToR.'), + +('100GBASE-LR4', 'IEEE 802.3ba', 'IEEE', '100G', 100, '{QSFP28,CFP,CFP2,CFP4}', 10000, 'SMF', '1295-1310nm', 'ratified', 2010, + '4x25G CWDM WDM lanes. 10km duplex LC. Standard DCI and metro link.'), + +('100GBASE-ER4', 'IEEE 802.3ba', 'IEEE', '100G', 100, '{CFP,CFP2,QSFP28}', 40000, 'SMF', '1295-1310nm', 'ratified', 2010, + '40km, 4x WDM EML lasers. Higher power than LR4.'), + +('100GBASE-PSM4', NULL, 'MSA', '100G', 100, '{QSFP28}', 500, 'SMF', '1310nm', 'ratified', 2014, + 'Parallel SMF 4-lane, 500m on 8-fiber MPO. Cost-effective campus/DCI.'), + +('100GBASE-CWDM4', NULL, 'MSA', '100G', 100, '{QSFP28}', 2000, 'SMF', '1310nm', 'ratified', 2015, + '4x25G CWDM over duplex SMF, 2km. Popular inter-building DCI alternative to LR4.'), + +('100GBASE-DR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 500, 'SMF', '1310nm', 'ratified', 2021, + 'Single-lambda PAM4, 500m duplex LC. Simpler than LR4, gaining traction.'), + +('100GBASE-FR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 2000, 'SMF', '1310nm', 'ratified', 2021, + 'Single-lambda PAM4, 2km. Between DR (500m) and LR (10km).'), + +('100GBASE-LR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 10000, 'SMF', '1310nm', 'ratified', 2021, + 'Single-lambda PAM4, 10km. Simpler than LR4, no WDM mux needed.'), + +('100G-ZR (OIF)', NULL, 'OIF', '100G', 100, '{CFP,CFP2}', 1000000, 'SMF', '1550nm', 'ratified', 2016, + 'DP-QPSK coherent. 1000km+ backbone. Soft-decision FEC.'), + +-- 400G +('400GBASE-SR8', 'IEEE 802.3cm', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 100, 'MMF', '850nm', 'ratified', 2020, + '8x50G SR PAM4, 16-fiber MPO. 50m OM3, 100m OM4/OM5.'), + +('400GBASE-SR4.2', 'IEEE 802.3cm', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 150, 'MMF', '850/910nm', 'ratified', 2020, + 'BiDi 850/910nm, 8-fiber MPO. 150m OM5. Cost-effective MMF 400G upgrade.'), + +('400GBASE-DR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP,CFP8}', 500, 'SMF', '1310nm', 'ratified', 2018, + '4x100G PAM4, 8-fiber MPO SMF. 500m. Hyperscaler cluster fabric.'), + +('400GBASE-LR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 10000, 'SMF', '1310nm', 'ratified', 2018, + '4x100G WDM, 10km duplex SMF. Requires EML lasers.'), + +('400GBASE-FR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 2000, 'SMF', '1310nm', 'ratified', 2018, + '4x100G WDM, 2km. Data center interconnect.'), + +('400G-ZR (OIF)', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 120000, 'SMF', '1550nm', 'ratified', 2020, + 'DP-16QAM coherent, 120km DWDM span. Pluggable coherent revolutionized DCI economics.'), + +('400G-ZR+', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 3000000, 'SMF', '1550nm', 'ratified', 2022, + 'Extended coherent, 3000km+. Higher OSNR than ZR. Submarine/long-haul capable.'), + +-- 800G +('800GBASE-SR8', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 100, 'MMF', '850nm', 'ratified', 2023, + '8x100G SR PAM4, 16-fiber MPO. 100m OM4. First 800G reaching market in 2024.'), + +('800GBASE-DR8', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 500, 'SMF', '1310nm', 'ratified', 2023, + '8x100G PAM4, 16-fiber MPO SMF, 500m. GPU cluster interconnect target.'), + +('800GBASE-LR4', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 10000, 'SMF', '1310nm', 'ratified', 2023, + '4x200G WDM lanes, 10km duplex SMF. Requires 200G-per-lane DSPs.'), + +('800G-ZR (OIF)', NULL, 'OIF', '800G', 800, '{OSFP,QSFP-DD800}', 120000, 'SMF', '1550nm', 'ratified', 2024, + 'DP-64QAM or DP-32QAM coherent. 120km DWDM. Production starting 2025.'), + +-- PON +('XGS-PON', 'ITU-T G.9807.1', 'IEEE', '10G', 10, '{SFP+}', 20000, 'SMF', '1270/1577nm', 'ratified', 2016, + '10G symmetric PON. 1270nm upstream, 1577nm downstream. Dominant FTTH 10G standard.'), + +-- DWDM +('100G DWDM Tunable', NULL, 'OIF', '100G', 100, '{CFP,CFP2}', 1000000, 'SMF', 'C-band', 'ratified', 2014, + 'Tunable coherent 100G. 50GHz ITU C-band grid, 96 channels. Metro/long-haul transport.'), + +('400G DWDM Tunable', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 1000000, 'SMF', 'C-band', 'ratified', 2021, + 'Tunable 400G coherent over ITU C-band DWDM grid. Packet-optical transport.') + +ON CONFLICT (name) DO NOTHING; diff --git a/sql/030-blog-linkedin-columns.sql b/sql/030-blog-linkedin-columns.sql new file mode 100644 index 0000000..5015d4c --- /dev/null +++ b/sql/030-blog-linkedin-columns.sql @@ -0,0 +1,6 @@ +-- Migration 030: Add LinkedIn post columns to blog_drafts +-- Required by fo-blog-pipeline-v5 (linkedin post generation at step 16) + +ALTER TABLE blog_drafts + ADD COLUMN IF NOT EXISTS linkedin_post TEXT, + ADD COLUMN IF NOT EXISTS linkedin_char_count INTEGER; diff --git a/sql/032-switches-columns-verification-fix.sql b/sql/032-switches-columns-verification-fix.sql new file mode 100644 index 0000000..2b11db3 --- /dev/null +++ b/sql/032-switches-columns-verification-fix.sql @@ -0,0 +1,179 @@ +/** + * Migration 032 — Switches column additions + verification fix + demo data flag + * + * Adds: + * - switches: description, features, use_cases, system_type, is_linecard, + * chassis_model, slot_type, flexbox_compat_mode, flexbox_notes + * - procurement tables: is_demo_data flag for DEMO DATA badge + * - Fix compute_transceiver_verification: 'unknown' confidence with populated + * core fields counts as details_verified (scraper seeded data is valid) + */ + +-- ============================================================ +-- 1. Add missing columns to switches table +-- ============================================================ +ALTER TABLE switches + ADD COLUMN IF NOT EXISTS description text, + ADD COLUMN IF NOT EXISTS features jsonb DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS use_cases text[] DEFAULT '{}'::text[], + ADD COLUMN IF NOT EXISTS system_type text DEFAULT 'fixed', + ADD COLUMN IF NOT EXISTS is_linecard boolean DEFAULT false, + ADD COLUMN IF NOT EXISTS chassis_model text, + ADD COLUMN IF NOT EXISTS slot_type text, + ADD COLUMN IF NOT EXISTS flexbox_compat_mode text, + ADD COLUMN IF NOT EXISTS flexbox_notes text; + +-- Check constraint for system_type +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'switches_system_type_check' + ) THEN + ALTER TABLE switches ADD CONSTRAINT switches_system_type_check + CHECK (system_type IN ('fixed', 'modular', 'stackable')); + END IF; +END $$; + +-- Index for linecard lookups +CREATE INDEX IF NOT EXISTS idx_switches_is_linecard ON switches (is_linecard) WHERE is_linecard = true; +CREATE INDEX IF NOT EXISTS idx_switches_chassis_model ON switches (chassis_model) WHERE chassis_model IS NOT NULL; + +-- ============================================================ +-- 2. Add is_demo_data flag to procurement tables +-- ============================================================ +ALTER TABLE reorder_signals + ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false; + +ALTER TABLE abc_classification + ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false; + +ALTER TABLE stock_snapshots + ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false; + +ALTER TABLE market_intelligence + ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false; + +-- Mark existing demo data (seeded from migration 021) +-- These were seeded as static demo rows - mark them so frontend can badge them +UPDATE reorder_signals SET is_demo_data = true +WHERE source IS NULL OR source IN ('demo', 'seed', 'synthetic'); + +UPDATE abc_classification SET is_demo_data = true +WHERE classification_source IS NULL OR classification_source IN ('demo', 'seed', 'synthetic'); + +-- Market intelligence seeded rows (OFC 2026, AWS capex, etc. from migration 019) +UPDATE market_intelligence SET is_demo_data = true +WHERE source IN ('manual', 'seed', 'OFC 2026', 'demo') + OR (source IS NULL AND created_at < '2026-04-09'::date); + +-- ============================================================ +-- 3. Fix details_verified: accept 'unknown' confidence when +-- core fields (form_factor, speed_gbps, reach_label, part_number) +-- are all populated — seed data from npm package is valid +-- ============================================================ +CREATE OR REPLACE FUNCTION compute_transceiver_verification() +RETURNS void AS $$ +DECLARE + v_rec RECORD; + v_price_row RECORD; + v_price_eur NUMERIC; + v_price_usd NUMERIC; + v_price_verified BOOLEAN; + v_image_verified BOOLEAN; + v_details_verified BOOLEAN; +BEGIN + FOR v_rec IN SELECT id FROM transceivers LOOP + -- Price: any real price observation in last 60 days + SELECT price, currency, time INTO v_price_row + FROM price_observations + WHERE transceiver_id = v_rec.id + AND price > 0 + AND time > NOW() - INTERVAL '60 days' + ORDER BY price DESC, time DESC + LIMIT 1; + + v_price_verified := v_price_row IS NOT NULL; + + IF v_price_verified THEN + CASE v_price_row.currency + WHEN 'EUR' THEN + v_price_eur := v_price_row.price; + v_price_usd := NULL; + WHEN 'USD' THEN + v_price_usd := v_price_row.price; + v_price_eur := NULL; + WHEN 'GBP' THEN + v_price_eur := v_price_row.price * 1.17; + v_price_usd := NULL; + ELSE + v_price_eur := NULL; + v_price_usd := NULL; + END CASE; + ELSE + v_price_eur := NULL; + v_price_usd := NULL; + END IF; + + -- Image: has any image URL + v_image_verified := EXISTS ( + SELECT 1 FROM transceivers + WHERE id = v_rec.id + AND image_url IS NOT NULL + AND image_url != '' + ); + + -- Details verified: + -- EITHER confidence is 'good' (scraped/verified/official) AND has connector or wavelength + -- OR all core fields (form_factor, speed_gbps, reach_label, part_number) are populated + -- (seed data from npm package counts — 'unknown' confidence with full spec = valid details) + v_details_verified := EXISTS ( + SELECT 1 FROM transceivers t2 + WHERE t2.id = v_rec.id + AND t2.data_confidence NOT IN ('garbage', '') + AND t2.data_confidence IS NOT NULL + AND ( + -- Scraped / official data with technical details + ( + t2.data_confidence NOT IN ('unknown') + AND (t2.connector IS NOT NULL OR t2.wavelengths IS NOT NULL OR t2.fiber_type IS NOT NULL) + ) + OR + -- Seed data with all core spec fields populated + ( + t2.form_factor IS NOT NULL + AND t2.speed_gbps IS NOT NULL + AND t2.reach_label IS NOT NULL + AND t2.part_number IS NOT NULL + AND t2.fiber_type IS NOT NULL + ) + ) + ); + + UPDATE transceivers SET + price_verified = v_price_verified, + price_verified_eur = v_price_eur, + street_price_usd = v_price_usd, + image_verified = v_image_verified, + details_verified = v_details_verified, + fully_verified = v_price_verified AND v_image_verified AND v_details_verified, + updated_at = NOW() + WHERE id = v_rec.id; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- Run verification refresh +SELECT compute_transceiver_verification(); + +-- ============================================================ +-- 4. Report +-- ============================================================ +SELECT + COUNT(*) AS total, + SUM(CASE WHEN price_verified THEN 1 ELSE 0 END) AS price_verified, + SUM(CASE WHEN image_verified THEN 1 ELSE 0 END) AS image_verified, + SUM(CASE WHEN details_verified THEN 1 ELSE 0 END) AS details_verified, + SUM(CASE WHEN fully_verified THEN 1 ELSE 0 END) AS fully_verified, + ROUND(100.0 * SUM(CASE WHEN details_verified THEN 1 ELSE 0 END) / COUNT(*), 1) AS details_pct, + ROUND(100.0 * SUM(CASE WHEN fully_verified THEN 1 ELSE 0 END) / COUNT(*), 1) AS fully_pct +FROM transceivers;