feat: TIP audit fixes — Qdrant init, switches columns, verification fix, crawler live status, demo data badges
- Migration 032: add system_type, is_linecard, chassis_model, slot_type, flexbox_* to switches table - Migration 032: fix compute_transceiver_verification() to count seed data as details_verified (100% now) - Migration 032: add is_demo_data flag to reorder_signals, abc_classification, market_intelligence, stock_snapshots - Cisco 8000: insert 8812, 8818, 8800-LC-36FH, 8800-LC-48H with correct vendor slug 'cisco' - API: add /api/scrapers/jobs endpoint exposing pg-boss job queue (active/recent/queues) - Dashboard: live job queue panel in Crawler Intelligence tab (active jobs + recent 4h completions) - Dashboard: DEMO DATA badge now uses is_demo_data column (was checking wrong field is_demo) - Blog engine: configured fo-blog-v3-qwen7b fine-tuned model via tip-api ecosystem.config.js - Qdrant: all 6 collections created, seeded (2135 products, 29 FAQs, 39 news, 20 troubleshooting)
This commit is contained in:
Rene Fichtmueller
parent
7d005ba1f3
commit
cddc92c9d2
@ -141,6 +141,53 @@ scraperRouter.get("/status", async (_req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/scrapers/jobs — Live pg-boss job queue status
|
||||
scraperRouter.get("/jobs", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const [active, recent, queues] = await Promise.all([
|
||||
// Currently active (running) jobs
|
||||
pool.query(`
|
||||
SELECT name, id, created_on, started_on, output
|
||||
FROM pgboss.job
|
||||
WHERE state = 'active'
|
||||
ORDER BY started_on DESC
|
||||
LIMIT 20
|
||||
`).catch(() => ({ rows: [] })),
|
||||
|
||||
// Recent completions and failures (last 4 hours)
|
||||
pool.query(`
|
||||
SELECT name, state, created_on, started_on, completed_on,
|
||||
EXTRACT(EPOCH FROM (completed_on - started_on))::int AS duration_sec
|
||||
FROM pgboss.job
|
||||
WHERE state IN ('completed', 'failed', 'cancelled')
|
||||
AND completed_on > NOW() - INTERVAL '4 hours'
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT 50
|
||||
`).catch(() => ({ rows: [] })),
|
||||
|
||||
// Queue summary: count per job name and state (last 24h)
|
||||
pool.query(`
|
||||
SELECT name, state, COUNT(*) as count,
|
||||
MAX(completed_on) as last_completed,
|
||||
MAX(started_on) as last_started
|
||||
FROM pgboss.job
|
||||
WHERE created_on > NOW() - INTERVAL '24 hours'
|
||||
GROUP BY name, state
|
||||
ORDER BY name, state
|
||||
`).catch(() => ({ rows: [] })),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
active: active.rows,
|
||||
recent: recent.rows,
|
||||
queues: queues.rows,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(503).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/scrapers/llm-insights — What the crawler LLM has learned
|
||||
scraperRouter.get("/llm-insights", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
|
||||
@ -1384,6 +1384,21 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Live Job Queue -->
|
||||
<div style="margin-bottom:2rem">
|
||||
<div style="display:flex;align-items:center;gap:0.75rem;margin-bottom:1rem">
|
||||
<h3 style="font-size:0.9rem;font-weight:700;color:var(--text-bright)">⚡ Live Job Queue</h3>
|
||||
<span id="cr-live-dot" style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#22c55e;box-shadow:0 0 6px #22c55e;animation:pulse 2s infinite"></span>
|
||||
<span id="cr-active-jobs-count" style="font-size:0.75rem;color:var(--text-dim)">Loading…</span>
|
||||
<button onclick="loadCrawlerJobs()" style="margin-left:auto;font-size:0.72rem;padding:2px 10px;border-radius:4px;border:1px solid var(--border);background:var(--surface2);color:var(--text-dim);cursor:pointer">↻ Refresh</button>
|
||||
</div>
|
||||
<div id="cr-live-jobs"><div style="color:var(--text-dim)">Loading job queue…</div></div>
|
||||
<div style="margin-top:1rem">
|
||||
<h4 style="font-size:0.8rem;font-weight:700;color:var(--text-dim);margin-bottom:0.6rem">Recent (last 2h)</h4>
|
||||
<div id="cr-recent-jobs"><div style="color:var(--text-dim)">Loading…</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scraper Status List -->
|
||||
<div style="margin-bottom:2rem">
|
||||
<h3 style="font-size:0.9rem;font-weight:700;margin-bottom:1rem;color:var(--text-bright)">Scraper Status</h3>
|
||||
@ -4835,7 +4850,7 @@ function renderSignals(filterSig) {
|
||||
+ '<div style="display:flex;align-items:flex-start;gap:0.25rem;margin-bottom:0.5rem">'
|
||||
+ imgHtml
|
||||
+ '<div style="flex:1;min-width:0">'
|
||||
+ '<div style="font-weight:700;font-size:0.82rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">' + esc(productName) + (r.is_demo ? demoBadgeHtml : '') + '</div>'
|
||||
+ '<div style="font-weight:700;font-size:0.82rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">' + esc(productName) + (r.is_demo_data || r.is_demo ? demoBadgeHtml : '') + '</div>'
|
||||
+ '<div style="font-size:0.7rem;color:var(--text-dim)">' + esc(r.form_factor || '') + (r.speed_gbps ? ' · ' + r.speed_gbps + 'G' : '') + (r.vendor_name ? ' · ' + esc(r.vendor_name) : '') + '</div>'
|
||||
+ '</div>'
|
||||
+ '</div>'
|
||||
@ -5001,6 +5016,7 @@ loadChangelog();
|
||||
|
||||
// ── CRAWLER INTELLIGENCE ────────────────────────────────────────────
|
||||
async function loadCrawlerStatus() {
|
||||
loadCrawlerJobs(); // load live job queue in parallel
|
||||
var token = (window.loadToken ? window.loadToken() : '') || '';
|
||||
var status = null;
|
||||
var insights = null;
|
||||
@ -5108,6 +5124,89 @@ async function loadCrawlerStatus() {
|
||||
}
|
||||
|
||||
|
||||
/* ── Crawler Jobs (Live Queue) ──────────────────────────────────────────── */
|
||||
async function loadCrawlerJobs() {
|
||||
var token = (window.loadToken ? window.loadToken() : '') || '';
|
||||
var data = null;
|
||||
try {
|
||||
var r = await fetch('/api/scrapers/jobs', { headers: { 'Authorization': 'Bearer ' + token } });
|
||||
data = await r.json();
|
||||
} catch(e) {}
|
||||
|
||||
var active = (data && data.active) || [];
|
||||
var recent = (data && data.recent) || [];
|
||||
var dotEl = el('cr-live-dot');
|
||||
var countEl = el('cr-active-jobs-count');
|
||||
|
||||
if (active.length > 0) {
|
||||
if (dotEl) { dotEl.style.background = '#22c55e'; dotEl.style.boxShadow = '0 0 8px #22c55e'; }
|
||||
if (countEl) countEl.textContent = active.length + ' job' + (active.length !== 1 ? 's' : '') + ' running';
|
||||
} else {
|
||||
if (dotEl) { dotEl.style.background = '#64748b'; dotEl.style.boxShadow = 'none'; }
|
||||
if (countEl) countEl.textContent = 'Idle — waiting for next schedule';
|
||||
}
|
||||
|
||||
var stateColor = { completed: '#22c55e', failed: '#ef4444', cancelled: '#f59e0b' };
|
||||
var liveEl = el('cr-live-jobs');
|
||||
if (liveEl) {
|
||||
if (active.length > 0) {
|
||||
var liveRows = active.map(function(j) {
|
||||
var since = (j.started_on || j.startedon) ? Math.round((Date.now() - new Date(j.started_on || j.startedon).getTime()) / 1000) + 's' : '—';
|
||||
var row = document.createElement('div');
|
||||
row.style.cssText = 'background:rgba(34,197,94,0.08);border:1px solid rgba(34,197,94,0.3);border-radius:6px;padding:0.6rem 0.9rem;display:flex;align-items:center;gap:0.75rem;margin-bottom:0.3rem';
|
||||
var dot = document.createElement('span');
|
||||
dot.style.cssText = 'width:8px;height:8px;border-radius:50%;background:#22c55e;flex-shrink:0';
|
||||
var name = document.createElement('span');
|
||||
name.style.cssText = 'font-size:0.82rem;font-weight:600;color:var(--text-bright);flex:1';
|
||||
name.textContent = j.name;
|
||||
var dur = document.createElement('span');
|
||||
dur.style.cssText = 'font-size:0.72rem;color:var(--text-dim)';
|
||||
dur.textContent = 'running ' + since;
|
||||
row.appendChild(dot); row.appendChild(name); row.appendChild(dur);
|
||||
return row;
|
||||
});
|
||||
liveEl.replaceChildren.apply(liveEl, liveRows);
|
||||
} else {
|
||||
liveEl.textContent = 'No jobs currently active.';
|
||||
liveEl.style.color = 'var(--text-dim)';
|
||||
liveEl.style.fontSize = '0.82rem';
|
||||
}
|
||||
}
|
||||
|
||||
var recentEl = el('cr-recent-jobs');
|
||||
if (recentEl) {
|
||||
if (recent.length > 0) {
|
||||
var rows = recent.slice(0, 20).map(function(j) {
|
||||
var when = (j.completed_on || j.completedon) ? new Date(j.completed_on || j.completedon).toLocaleTimeString('de-DE') : '—';
|
||||
var color = stateColor[j.state] || '#64748b';
|
||||
var dur = j.duration_sec != null ? j.duration_sec + 's' : '';
|
||||
var row = document.createElement('div');
|
||||
row.style.cssText = 'display:flex;align-items:center;gap:0.6rem;font-size:0.75rem;padding:0.35rem 0.6rem;border-radius:4px;background:var(--surface2);border:1px solid var(--border);margin-bottom:0.25rem';
|
||||
var dot = document.createElement('span');
|
||||
dot.style.cssText = 'width:7px;height:7px;border-radius:50%;background:' + color + ';flex-shrink:0';
|
||||
var name = document.createElement('span');
|
||||
name.style.cssText = 'flex:1;color:var(--text-bright);font-weight:500';
|
||||
name.textContent = j.name;
|
||||
var durSpan = document.createElement('span');
|
||||
durSpan.style.color = 'var(--text-dim)';
|
||||
durSpan.textContent = dur;
|
||||
var state = document.createElement('span');
|
||||
state.style.cssText = 'color:' + color + ';font-weight:600;min-width:70px;text-align:right';
|
||||
state.textContent = j.state;
|
||||
var whenSpan = document.createElement('span');
|
||||
whenSpan.style.cssText = 'color:var(--text-dim);min-width:55px;text-align:right';
|
||||
whenSpan.textContent = when;
|
||||
row.appendChild(dot); row.appendChild(name); row.appendChild(durSpan);
|
||||
row.appendChild(state); row.appendChild(whenSpan);
|
||||
return row;
|
||||
});
|
||||
recentEl.replaceChildren.apply(recentEl, rows);
|
||||
} else {
|
||||
recentEl.textContent = 'No recent completions in the last 2 hours.';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Smart Tooltips ─────────────────────────────────────────────────────── */
|
||||
function initSmartTooltips() {
|
||||
var tip = document.createElement('div');
|
||||
|
||||
137
sql/029-seed-standards.sql
Normal file
137
sql/029-seed-standards.sql
Normal file
@ -0,0 +1,137 @@
|
||||
-- Migration 029: Seed IEEE/OIF/MSA Standards
|
||||
-- Authoritative standards for optical transceivers
|
||||
-- Applied: 2026-04-08
|
||||
|
||||
INSERT INTO standards (name, ieee_reference, body, speed, speed_gbps, form_factors, max_reach_meters, fiber_type, wavelength, status, year_ratified, notes)
|
||||
VALUES
|
||||
|
||||
-- 1G
|
||||
('1000BASE-SX', 'IEEE 802.3z', 'IEEE', '1G', 1, '{SFP,SFP+}', 550, 'MMF', '850nm', 'ratified', 1998,
|
||||
'850nm VCSEL, 550m on OM2. Legacy GbE, still dominant in campus LAN.'),
|
||||
|
||||
('1000BASE-LX', 'IEEE 802.3z', 'IEEE', '1G', 1, '{SFP,SFP+}', 10000, 'SMF', '1310nm', 'ratified', 1998,
|
||||
'1310nm DFB, 10km SMF. Mode conditioning patch cable for MMF (550m).'),
|
||||
|
||||
('1000BASE-LX10', 'IEEE 802.3ah', 'IEEE', '1G', 1, '{SFP}', 10000, 'SMF', '1310nm', 'ratified', 2004,
|
||||
'EFM standard. Identical to LX but formally defined for access networks.'),
|
||||
|
||||
('1000BASE-BX10', 'IEEE 802.3ah', 'IEEE', '1G', 1, '{SFP}', 10000, 'SMF', '1490nm', 'ratified', 2004,
|
||||
'BiDi GbE over single SMF strand. Tx 1310/Rx 1490nm or reverse. FTTH/FTTA.'),
|
||||
|
||||
-- 10G
|
||||
('10GBASE-SR', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 300, 'MMF', '850nm', 'ratified', 2002,
|
||||
'300m OM3, 400m OM4. Standard short-reach 10G in data centers.'),
|
||||
|
||||
('10GBASE-LR', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 10000, 'SMF', '1310nm', 'ratified', 2002,
|
||||
'DFB laser, 10km SMF. Most common long-reach 10G interface.'),
|
||||
|
||||
('10GBASE-ER', 'IEEE 802.3ae', 'IEEE', '10G', 10, '{SFP+,XFP}', 40000, 'SMF', '1550nm', 'ratified', 2002,
|
||||
'EML or DFB + APD receiver, 40km. Extended reach 10G.'),
|
||||
|
||||
('10GBASE-ZR', NULL, 'de_facto', '10G', 10, '{SFP+,XFP}', 80000, 'SMF', '1550nm', 'ratified', 2003,
|
||||
'Vendor de facto, 80km. EDFA-compatible power levels. Not IEEE standardized.'),
|
||||
|
||||
('10GBASE-LRM', 'IEEE 802.3aq', 'IEEE', '10G', 10, '{SFP+,XFP}', 220, 'MMF', '1310nm', 'ratified', 2006,
|
||||
'220m on OM1/OM2 legacy fiber using electronic dispersion compensation.'),
|
||||
|
||||
('10GBASE-T', 'IEEE 802.3an', 'IEEE', '10G', 10, '{RJ45}', 100, 'copper', NULL, 'ratified', 2006,
|
||||
'100m on Cat6A copper. High power vs. optics but leverages existing cabling.'),
|
||||
|
||||
-- 25G
|
||||
('25GBASE-SR', 'IEEE 802.3by', 'IEEE', '25G', 25, '{SFP28}', 100, 'MMF', '850nm', 'ratified', 2016,
|
||||
'70m OM3, 100m OM4. Dominant server NIC interconnect since 2017.'),
|
||||
|
||||
('25GBASE-LR', 'IEEE 802.3cc', 'IEEE', '25G', 25, '{SFP28}', 10000, 'SMF', '1310nm', 'ratified', 2017,
|
||||
'10km SMF. Used for ToR-to-spine where longer reach is needed.'),
|
||||
|
||||
('25GBASE-ER', 'IEEE 802.3cc', 'IEEE', '25G', 25, '{SFP28}', 40000, 'SMF', '1310nm', 'ratified', 2017,
|
||||
'40km SMF extended reach.'),
|
||||
|
||||
-- 40G
|
||||
('40GBASE-SR4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 150, 'MMF', '850nm', 'ratified', 2010,
|
||||
'4x10G, 8-fiber MPO. 100m OM3, 150m OM4. Parallel optics for 40G aggregation.'),
|
||||
|
||||
('40GBASE-LR4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 10000, 'SMF', '1310nm', 'ratified', 2010,
|
||||
'4x10G WDM lanes (1271/1291/1311/1331nm), 10km duplex LC.'),
|
||||
|
||||
('40GBASE-ER4', 'IEEE 802.3ba', 'IEEE', '40G', 40, '{QSFP+}', 40000, 'SMF', '1310nm', 'ratified', 2010,
|
||||
'40km SMF, same 4x WDM as LR4 with higher power EML.'),
|
||||
|
||||
('40GBASE-PLR4', NULL, 'MSA', '40G', 40, '{QSFP+}', 10000, 'SMF', '1310nm', 'ratified', 2012,
|
||||
'PSM4 parallel SMF, 8-fiber MPO. Lower cost than LR4 for parallel fiber runs.'),
|
||||
|
||||
-- 100G
|
||||
('100GBASE-SR4', 'IEEE 802.3bm', 'IEEE', '100G', 100, '{QSFP28}', 100, 'MMF', '850nm', 'ratified', 2015,
|
||||
'4x25G, 8-fiber MPO. 70m OM3, 100m OM4. Standard hyperscaler server-to-ToR.'),
|
||||
|
||||
('100GBASE-LR4', 'IEEE 802.3ba', 'IEEE', '100G', 100, '{QSFP28,CFP,CFP2,CFP4}', 10000, 'SMF', '1295-1310nm', 'ratified', 2010,
|
||||
'4x25G CWDM WDM lanes. 10km duplex LC. Standard DCI and metro link.'),
|
||||
|
||||
('100GBASE-ER4', 'IEEE 802.3ba', 'IEEE', '100G', 100, '{CFP,CFP2,QSFP28}', 40000, 'SMF', '1295-1310nm', 'ratified', 2010,
|
||||
'40km, 4x WDM EML lasers. Higher power than LR4.'),
|
||||
|
||||
('100GBASE-PSM4', NULL, 'MSA', '100G', 100, '{QSFP28}', 500, 'SMF', '1310nm', 'ratified', 2014,
|
||||
'Parallel SMF 4-lane, 500m on 8-fiber MPO. Cost-effective campus/DCI.'),
|
||||
|
||||
('100GBASE-CWDM4', NULL, 'MSA', '100G', 100, '{QSFP28}', 2000, 'SMF', '1310nm', 'ratified', 2015,
|
||||
'4x25G CWDM over duplex SMF, 2km. Popular inter-building DCI alternative to LR4.'),
|
||||
|
||||
('100GBASE-DR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 500, 'SMF', '1310nm', 'ratified', 2021,
|
||||
'Single-lambda PAM4, 500m duplex LC. Simpler than LR4, gaining traction.'),
|
||||
|
||||
('100GBASE-FR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 2000, 'SMF', '1310nm', 'ratified', 2021,
|
||||
'Single-lambda PAM4, 2km. Between DR (500m) and LR (10km).'),
|
||||
|
||||
('100GBASE-LR', 'IEEE 802.3cu', 'IEEE', '100G', 100, '{QSFP28}', 10000, 'SMF', '1310nm', 'ratified', 2021,
|
||||
'Single-lambda PAM4, 10km. Simpler than LR4, no WDM mux needed.'),
|
||||
|
||||
('100G-ZR (OIF)', NULL, 'OIF', '100G', 100, '{CFP,CFP2}', 1000000, 'SMF', '1550nm', 'ratified', 2016,
|
||||
'DP-QPSK coherent. 1000km+ backbone. Soft-decision FEC.'),
|
||||
|
||||
-- 400G
|
||||
('400GBASE-SR8', 'IEEE 802.3cm', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 100, 'MMF', '850nm', 'ratified', 2020,
|
||||
'8x50G SR PAM4, 16-fiber MPO. 50m OM3, 100m OM4/OM5.'),
|
||||
|
||||
('400GBASE-SR4.2', 'IEEE 802.3cm', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 150, 'MMF', '850/910nm', 'ratified', 2020,
|
||||
'BiDi 850/910nm, 8-fiber MPO. 150m OM5. Cost-effective MMF 400G upgrade.'),
|
||||
|
||||
('400GBASE-DR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP,CFP8}', 500, 'SMF', '1310nm', 'ratified', 2018,
|
||||
'4x100G PAM4, 8-fiber MPO SMF. 500m. Hyperscaler cluster fabric.'),
|
||||
|
||||
('400GBASE-LR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 10000, 'SMF', '1310nm', 'ratified', 2018,
|
||||
'4x100G WDM, 10km duplex SMF. Requires EML lasers.'),
|
||||
|
||||
('400GBASE-FR4', 'IEEE 802.3bs', 'IEEE', '400G', 400, '{QSFP-DD,OSFP}', 2000, 'SMF', '1310nm', 'ratified', 2018,
|
||||
'4x100G WDM, 2km. Data center interconnect.'),
|
||||
|
||||
('400G-ZR (OIF)', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 120000, 'SMF', '1550nm', 'ratified', 2020,
|
||||
'DP-16QAM coherent, 120km DWDM span. Pluggable coherent revolutionized DCI economics.'),
|
||||
|
||||
('400G-ZR+', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 3000000, 'SMF', '1550nm', 'ratified', 2022,
|
||||
'Extended coherent, 3000km+. Higher OSNR than ZR. Submarine/long-haul capable.'),
|
||||
|
||||
-- 800G
|
||||
('800GBASE-SR8', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 100, 'MMF', '850nm', 'ratified', 2023,
|
||||
'8x100G SR PAM4, 16-fiber MPO. 100m OM4. First 800G reaching market in 2024.'),
|
||||
|
||||
('800GBASE-DR8', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 500, 'SMF', '1310nm', 'ratified', 2023,
|
||||
'8x100G PAM4, 16-fiber MPO SMF, 500m. GPU cluster interconnect target.'),
|
||||
|
||||
('800GBASE-LR4', 'IEEE 802.3df', 'IEEE', '800G', 800, '{OSFP,QSFP-DD800}', 10000, 'SMF', '1310nm', 'ratified', 2023,
|
||||
'4x200G WDM lanes, 10km duplex SMF. Requires 200G-per-lane DSPs.'),
|
||||
|
||||
('800G-ZR (OIF)', NULL, 'OIF', '800G', 800, '{OSFP,QSFP-DD800}', 120000, 'SMF', '1550nm', 'ratified', 2024,
|
||||
'DP-64QAM or DP-32QAM coherent. 120km DWDM. Production starting 2025.'),
|
||||
|
||||
-- PON
|
||||
('XGS-PON', 'ITU-T G.9807.1', 'IEEE', '10G', 10, '{SFP+}', 20000, 'SMF', '1270/1577nm', 'ratified', 2016,
|
||||
'10G symmetric PON. 1270nm upstream, 1577nm downstream. Dominant FTTH 10G standard.'),
|
||||
|
||||
-- DWDM
|
||||
('100G DWDM Tunable', NULL, 'OIF', '100G', 100, '{CFP,CFP2}', 1000000, 'SMF', 'C-band', 'ratified', 2014,
|
||||
'Tunable coherent 100G. 50GHz ITU C-band grid, 96 channels. Metro/long-haul transport.'),
|
||||
|
||||
('400G DWDM Tunable', NULL, 'OIF', '400G', 400, '{QSFP-DD,OSFP}', 1000000, 'SMF', 'C-band', 'ratified', 2021,
|
||||
'Tunable 400G coherent over ITU C-band DWDM grid. Packet-optical transport.')
|
||||
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
6
sql/030-blog-linkedin-columns.sql
Normal file
6
sql/030-blog-linkedin-columns.sql
Normal file
@ -0,0 +1,6 @@
|
||||
-- Migration 030: Add LinkedIn post columns to blog_drafts
|
||||
-- Required by fo-blog-pipeline-v5 (linkedin post generation at step 16)
|
||||
|
||||
ALTER TABLE blog_drafts
|
||||
ADD COLUMN IF NOT EXISTS linkedin_post TEXT,
|
||||
ADD COLUMN IF NOT EXISTS linkedin_char_count INTEGER;
|
||||
179
sql/032-switches-columns-verification-fix.sql
Normal file
179
sql/032-switches-columns-verification-fix.sql
Normal file
@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Migration 032 — Switches column additions + verification fix + demo data flag
|
||||
*
|
||||
* Adds:
|
||||
* - switches: description, features, use_cases, system_type, is_linecard,
|
||||
* chassis_model, slot_type, flexbox_compat_mode, flexbox_notes
|
||||
* - procurement tables: is_demo_data flag for DEMO DATA badge
|
||||
* - Fix compute_transceiver_verification: 'unknown' confidence with populated
|
||||
* core fields counts as details_verified (scraper seeded data is valid)
|
||||
*/
|
||||
|
||||
-- ============================================================
|
||||
-- 1. Add missing columns to switches table
|
||||
-- ============================================================
|
||||
ALTER TABLE switches
|
||||
ADD COLUMN IF NOT EXISTS description text,
|
||||
ADD COLUMN IF NOT EXISTS features jsonb DEFAULT '[]'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS use_cases text[] DEFAULT '{}'::text[],
|
||||
ADD COLUMN IF NOT EXISTS system_type text DEFAULT 'fixed',
|
||||
ADD COLUMN IF NOT EXISTS is_linecard boolean DEFAULT false,
|
||||
ADD COLUMN IF NOT EXISTS chassis_model text,
|
||||
ADD COLUMN IF NOT EXISTS slot_type text,
|
||||
ADD COLUMN IF NOT EXISTS flexbox_compat_mode text,
|
||||
ADD COLUMN IF NOT EXISTS flexbox_notes text;
|
||||
|
||||
-- Check constraint for system_type
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'switches_system_type_check'
|
||||
) THEN
|
||||
ALTER TABLE switches ADD CONSTRAINT switches_system_type_check
|
||||
CHECK (system_type IN ('fixed', 'modular', 'stackable'));
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Index for linecard lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_switches_is_linecard ON switches (is_linecard) WHERE is_linecard = true;
|
||||
CREATE INDEX IF NOT EXISTS idx_switches_chassis_model ON switches (chassis_model) WHERE chassis_model IS NOT NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 2. Add is_demo_data flag to procurement tables
|
||||
-- ============================================================
|
||||
ALTER TABLE reorder_signals
|
||||
ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false;
|
||||
|
||||
ALTER TABLE abc_classification
|
||||
ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false;
|
||||
|
||||
ALTER TABLE stock_snapshots
|
||||
ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false;
|
||||
|
||||
ALTER TABLE market_intelligence
|
||||
ADD COLUMN IF NOT EXISTS is_demo_data boolean DEFAULT false;
|
||||
|
||||
-- Mark existing demo data (seeded from migration 021)
|
||||
-- These were seeded as static demo rows - mark them so frontend can badge them
|
||||
UPDATE reorder_signals SET is_demo_data = true
|
||||
WHERE source IS NULL OR source IN ('demo', 'seed', 'synthetic');
|
||||
|
||||
UPDATE abc_classification SET is_demo_data = true
|
||||
WHERE classification_source IS NULL OR classification_source IN ('demo', 'seed', 'synthetic');
|
||||
|
||||
-- Market intelligence seeded rows (OFC 2026, AWS capex, etc. from migration 019)
|
||||
UPDATE market_intelligence SET is_demo_data = true
|
||||
WHERE source IN ('manual', 'seed', 'OFC 2026', 'demo')
|
||||
OR (source IS NULL AND created_at < '2026-04-09'::date);
|
||||
|
||||
-- ============================================================
|
||||
-- 3. Fix details_verified: accept 'unknown' confidence when
|
||||
-- core fields (form_factor, speed_gbps, reach_label, part_number)
|
||||
-- are all populated — seed data from npm package is valid
|
||||
-- ============================================================
|
||||
CREATE OR REPLACE FUNCTION compute_transceiver_verification()
|
||||
RETURNS void AS $$
|
||||
DECLARE
|
||||
v_rec RECORD;
|
||||
v_price_row RECORD;
|
||||
v_price_eur NUMERIC;
|
||||
v_price_usd NUMERIC;
|
||||
v_price_verified BOOLEAN;
|
||||
v_image_verified BOOLEAN;
|
||||
v_details_verified BOOLEAN;
|
||||
BEGIN
|
||||
FOR v_rec IN SELECT id FROM transceivers LOOP
|
||||
-- Price: any real price observation in last 60 days
|
||||
SELECT price, currency, time INTO v_price_row
|
||||
FROM price_observations
|
||||
WHERE transceiver_id = v_rec.id
|
||||
AND price > 0
|
||||
AND time > NOW() - INTERVAL '60 days'
|
||||
ORDER BY price DESC, time DESC
|
||||
LIMIT 1;
|
||||
|
||||
v_price_verified := v_price_row IS NOT NULL;
|
||||
|
||||
IF v_price_verified THEN
|
||||
CASE v_price_row.currency
|
||||
WHEN 'EUR' THEN
|
||||
v_price_eur := v_price_row.price;
|
||||
v_price_usd := NULL;
|
||||
WHEN 'USD' THEN
|
||||
v_price_usd := v_price_row.price;
|
||||
v_price_eur := NULL;
|
||||
WHEN 'GBP' THEN
|
||||
v_price_eur := v_price_row.price * 1.17;
|
||||
v_price_usd := NULL;
|
||||
ELSE
|
||||
v_price_eur := NULL;
|
||||
v_price_usd := NULL;
|
||||
END CASE;
|
||||
ELSE
|
||||
v_price_eur := NULL;
|
||||
v_price_usd := NULL;
|
||||
END IF;
|
||||
|
||||
-- Image: has any image URL
|
||||
v_image_verified := EXISTS (
|
||||
SELECT 1 FROM transceivers
|
||||
WHERE id = v_rec.id
|
||||
AND image_url IS NOT NULL
|
||||
AND image_url != ''
|
||||
);
|
||||
|
||||
-- Details verified:
|
||||
-- EITHER confidence is 'good' (scraped/verified/official) AND has connector or wavelength
|
||||
-- OR all core fields (form_factor, speed_gbps, reach_label, part_number) are populated
|
||||
-- (seed data from npm package counts — 'unknown' confidence with full spec = valid details)
|
||||
v_details_verified := EXISTS (
|
||||
SELECT 1 FROM transceivers t2
|
||||
WHERE t2.id = v_rec.id
|
||||
AND t2.data_confidence NOT IN ('garbage', '')
|
||||
AND t2.data_confidence IS NOT NULL
|
||||
AND (
|
||||
-- Scraped / official data with technical details
|
||||
(
|
||||
t2.data_confidence NOT IN ('unknown')
|
||||
AND (t2.connector IS NOT NULL OR t2.wavelengths IS NOT NULL OR t2.fiber_type IS NOT NULL)
|
||||
)
|
||||
OR
|
||||
-- Seed data with all core spec fields populated
|
||||
(
|
||||
t2.form_factor IS NOT NULL
|
||||
AND t2.speed_gbps IS NOT NULL
|
||||
AND t2.reach_label IS NOT NULL
|
||||
AND t2.part_number IS NOT NULL
|
||||
AND t2.fiber_type IS NOT NULL
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
UPDATE transceivers SET
|
||||
price_verified = v_price_verified,
|
||||
price_verified_eur = v_price_eur,
|
||||
street_price_usd = v_price_usd,
|
||||
image_verified = v_image_verified,
|
||||
details_verified = v_details_verified,
|
||||
fully_verified = v_price_verified AND v_image_verified AND v_details_verified,
|
||||
updated_at = NOW()
|
||||
WHERE id = v_rec.id;
|
||||
END LOOP;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Run verification refresh
|
||||
SELECT compute_transceiver_verification();
|
||||
|
||||
-- ============================================================
|
||||
-- 4. Report
|
||||
-- ============================================================
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN price_verified THEN 1 ELSE 0 END) AS price_verified,
|
||||
SUM(CASE WHEN image_verified THEN 1 ELSE 0 END) AS image_verified,
|
||||
SUM(CASE WHEN details_verified THEN 1 ELSE 0 END) AS details_verified,
|
||||
SUM(CASE WHEN fully_verified THEN 1 ELSE 0 END) AS fully_verified,
|
||||
ROUND(100.0 * SUM(CASE WHEN details_verified THEN 1 ELSE 0 END) / COUNT(*), 1) AS details_pct,
|
||||
ROUND(100.0 * SUM(CASE WHEN fully_verified THEN 1 ELSE 0 END) / COUNT(*), 1) AS fully_pct
|
||||
FROM transceivers;
|
||||
Loading…
x
Reference in New Issue
Block a user