diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 507e726..59da088 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,15 +1,6 @@ # TIP Changelog Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}` -{"d":"2026-05-14","t":"FEAT","m":"Transceiver Academy: full API-backed customer and employee training platform. 5 categories (Standards, Form Factors, Switches & Compatibility, Fiber & Infrastructure, Testing & Buying), 22 detailed bilingual lessons (EN+DE), 74 quiz questions. API: GET /api/training/categories, /lessons, /lessons/:id, /quiz, /stats — public route, no auth required. Dashboard UI: language toggle EN/DE, category tabs, lesson cards with level badges, full lesson viewer (paragraphs/tables/callouts/code/formulas), per-lesson and per-category quiz engine with auto-advance dots progress, A-F grade results, localStorage progress tracking. Replaces old inline LLM-training data module."} -{"d":"2026-05-14","t":"UI","m":"Price History Chart: replaced 260×60px sparkline with full interactive SVG chart (520×200px). Features: multi-vendor colored polylines with end-point dots, Y-axis labels (USD normalized), X-axis date ticks (MM-DD), horizontal grid lines, hover cursor line + floating tooltip showing all vendor prices for hovered day, vendor legend with click-to-toggle visibility, time range selector (7d/14d/30d) with live re-fetch, current best prices table (FX-normalized to USD). FX normalization: EUR×1.08, GBP×1.27. Supports up to 8 vendors (indigo/orange/green/yellow/blue/red/purple/cyan palette). No API changes — existing GET /api/price-history/:id endpoint already returned price_max/min/avg per vendor per day."} -{"d":"2026-05-14","t":"FEAT","m":"Procurement: 5 neue Intelligence-Sektionen. (E) 🟢 Buy-Now Intel — Top buy_now Reorder Signals aus 211k preberechneten Signalen, filterbar nach Form Factor, Signalstärke-Balken, Preis/Stock-Trend, Gründe als Tooltip. API: GET /api/procurement/reorder-top. (A) 💰 Arbitrage — FX-Preis vs. Competitor-Preis für 59k Equivalenz-Paare mit Preisdaten auf beiden Seiten, normalisiert auf USD (EUR×1.08, GBP×1.27), sortiert nach Ersparnis-%. API: GET /api/procurement/arbitrage. (B) 🖥 Switch Compat — Suche nach Switch-Modell (Cisco, Juniper, Arista etc.), zeigt alle kompatiblen Transceiver mit Preis + Verifikationsmethode. 58k Compatibility-Rows, 429 Switches. API: GET /api/procurement/switch-compat?search=. (C) ⚠️ Supply Squeeze — Multi-Signal-Detektor: 4 parallele Quellen (Preis-Momentum 30d vs 60d, Hype-Phase, AI-Cluster-Transceiver-Nachfrage, Stock-Level-Verteilung). Severity: critical/warning/watch. API: GET /api/procurement/supply-squeeze. (D) 🪦 Dead Stock Revival — 7.297 Dead-Stock-SKUs gegen Hype-Cycle-Phasen: zeigt welche Lagerhüter in Technologieklassen liegen die gerade aufsteigen (ascending hype phases, score >30). API: GET /api/procurement/dead-stock-revival."} -{"d":"2026-05-14","t":"FEAT","m":"Crawler Intelligence: Data Quality panel. New GET /api/scrapers/data-quality endpoint — 4 parallel queries over 200,617 transceiver_verification_evidence rows: (1) coverage breakdown (price 11,366/18,146 = 62%, image 12,333/68%, details 17,085/94%, competitor_match 399/2%, quarantined 1,193); (2) all 10 evidence types with count + avg confidence + product count + last seen; (3) robot/scraper contributions table (17 robots ranked by output); (4) daily activity last 14 days. Dashboard Crawler Intelligence tab: new 🔬 Data Quality section with coverage progress bars (color-coded ≥80% green / ≥50% amber / red), evidence type table, SVG sparkline bar chart for 14-day activity, robot contributions table with live/stale dot indicators."} -{"d":"2026-05-14","t":"FEAT","m":"Dynamic Hype Cycle + Market Signal Engine: Hype Cycle tab is now fully data-driven. New GET /api/hype-cycle/market-signals endpoint blends 6 real data sources into a composite Market Signal Score (0–100) per technology: (1) hype_score from Norton-Bass model (30% weight), (2) hyperscaler CapEx YoY avg (Microsoft +68.8%, Alphabet +107.4%, Meta +46.8%), (3) price observation activity ratio 30d vs prior 30d, (4) AI cluster estimated transceiver demand (90d window), (5) eBay secondary market sell-through velocity, (6) internal fast-mover demand trend. Score thresholds: ≥70 green, ≥50 yellow, ≥30 orange, <30 gray. Recommendation engine: buildRecommendation(phase, signalScore, capexYoyAvg, speedGbps) maps hype phase × capex boom × speed class → Buy/Hold/Watch label with color + detail tooltip. Dashboard: Hype Cycle table shows Market Signal ● LIVE column (score + progress bar) + Recommendation column (emoji label, tooltip with reasoning). Market Context cards row above table shows Top Signal, CapEx Boom %, Fast Movers signal, eBay Velocity. New Hyperscaler CapEx panel (SEC filing data) + eBay Secondary Market panel at bottom of hype tab. Procurement: new 🛒 eBay Market sub-section with per-form-factor sell-through grid. All 6 queries run in parallel via Promise.all()."} -{"d":"2026-05-14","t":"FEAT","m":"Procurement tab: 2 new sections with real data. (1) 📦 Internal Demand — Flexoptix internal SKU velocity from flexoptix_internal_demand table (8,585 SKUs: 70 fast-movers 53k units/12M, 239 regular, 979 slow, 7,297 dead stock). Summary cards with trend %%. Filter by velocity class. API: GET /api/procurement/internal-demand?velocity_class=&limit=&sort=. (2) 🤖 AI Clusters — live AI datacenter announcements from ai_cluster_announcements table (396 in last 30 days). Shows estimated transceiver demand per build, MW scale, company, location, source link. Filter for entries with transceiver estimates. Stats: total announcements, MW, distinct companies, total estimated transceivers. API: GET /api/procurement/ai-clusters?days=&limit=. Replaced misleading DEMO DATA banners on Signals + ABC sections with informational note pointing to Internal Demand data."} - -{"d":"2026-05-14","t":"FEAT","m":"Training Module im Standards-Tab: 13 Lektionen (Form Factors, Glasfaser, IEEE 802.3, WDM, PAM4/NRZ, Link Budget mit Live-Rechner, Coherent Optics, MSA/DOM, Vendor Locking, Temperature Classes, Selection Guide, 400G/800G, Troubleshooting), 40 Quiz-Fragen mit Shuffle/Feedback/Erklaerungen/Note A-F, 4 Lernpfade (Einsteiger 5 / Netzwerk-Engineer 9 / Einkaeufer 6 / Expert 13 Lektionen), Fortschrittsbalken, localStorage-Persistenz. Kein DB-Schema-Aenderung - alles client-seitig als JS-Data-Objekte."} -{"d":"2026-05-14","t":"FEAT","m":"6 neue Dashboard-Features: (A) Price Movers Alert — GET /api/procurement/price-movers?days=N&limit=N, CTE-basiert (cur vs prior period avg per SKU+Vendor), zeigt Top-Gainers und Top-Losers (|delta_pct| >= 2%, obs >= 2). Procurement-Tab Sektion mit Period-Toggle 7d/14d/30d, Export CSV. (B) Executive Overview Pulse — 5 KPI-Karten (Buy Signals, Arbitrage Ops, Supply Alerts, Price Gainers, Losers) über `loadProcurementPulse()`, Top-Movers Mini-Tabelle im Overview, alle clickable → Procurement-Tab. (C) CSV Export — exportMoversCSV() generiert Gainers+Losers als CSV-Download. (D) Vendor Intelligence — GET /api/vendors/intelligence: per-Vendor in letzten 30d (sku_count, price_obs, avg/min/max price, last_seen), Top-6-Anbieter-Banner im Vendors-Tab. (E) Advanced Transceiver Search — Speed-Filter (1G/10G/.../800G), Fiber-Type-Filter (SMF/MMF), 'Verified Only'-Checkbox in Transceivers-Tab; searchTransceivers() übergibt speed_gbps=, fiber_type=, verified=price an GET /api/transceivers. (F) Knowledge Base Browser — neuer Tab KB, GET /api/kb?q=&category=&limit= (Full-Text ILIKE über question/answer/subcategory), Category-Pills, Entry-Cards mit Severity-Badge, Form-Factor/Speed-Tags."} {"d":"2026-05-14","t":"FEAT","m":"Equivalences Explorer: new dashboard tab '🔀 Equivalences' — search 63,362 cross-brand mappings (46 vendors, 7,516 competitor products → 846 Flexoptix alternatives, Ø 93.9% confidence). APIs: GET /api/equivalences (search), /api/equivalences/transceiver/:id (per-product), /api/equivalences/stats, /api/equivalences/top-vendors. Transceiver detail modal now shows equivalences panel (FX alternatives or competitor products) + SVG price history sparklines (30-day, per source vendor) from 392k+ price observations."} {"d":"2026-05-14","t":"FEAT","m":"LinkedIn Distribution Status: Blog tab shows DRY_RUN badge, posted/dry_run/skipped/failed counters, history table with live URN links. GET /api/blog/linkedin/history reads blog_linkedin_distribution table + detects DRY_RUN mode from ecosystem config."} {"d":"2026-05-14","t":"FEAT","m":"MCP Server: 2 new tools — find_equivalences (search 63k+ verified cross-brand mappings with confidence filter, returns FX alternatives + competitor matches formatted for LLM) + get_price_history (392k+ obs, daily series, per-vendor min/max/avg, cheapest source identification). Total: 21 MCP tools."} diff --git a/package-lock.json b/package-lock.json index 70b9cde..4329bbc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,11 @@ "workspaces": [ "packages/*" ], + "dependencies": { + "pdf-parse": "^1.1.4" + }, "devDependencies": { + "@types/pdf-parse": "^1.1.5", "tsx": "^4.19", "typescript": "^5.9.3", "xlsx": "^0.18.5" @@ -1662,6 +1666,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/multer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.1.0.tgz", + "integrity": "sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { "version": "25.5.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", @@ -1671,6 +1685,16 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/pdf-parse": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/pdf-parse/-/pdf-parse-1.1.5.tgz", + "integrity": "sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pg": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", @@ -1887,6 +1911,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -2038,6 +2068,23 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/byte-counter": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/byte-counter/-/byte-counter-0.1.0.tgz", @@ -2332,6 +2379,21 @@ "node": ">=20" } }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/content-disposition": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", @@ -4222,6 +4284,68 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multer": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz", + "integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "type-is": "^1.6.18" + }, + "engines": { + "node": ">= 10.16.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/multer/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -4255,6 +4379,12 @@ "node": ">= 0.6" } }, + "node_modules/node-ensure": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz", + "integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==", + "license": "MIT" + }, "node_modules/node-releases": { "version": "2.0.36", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", @@ -4568,6 +4698,22 @@ "through": "~2.3" } }, + "node_modules/pdf-parse": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-1.1.4.tgz", + "integrity": "sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ==", + "license": "MIT", + "dependencies": { + "node-ensure": "^0.0.0" + }, + "engines": { + "node": ">=6.8.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, "node_modules/pg": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", @@ -5363,6 +5509,14 @@ "stream-chain": "^2.2.5" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -5574,6 +5728,12 @@ "node": ">= 0.6" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -6076,12 +6236,15 @@ "express": "^5.1.0", "express-rate-limit": "^7.5.0", "helmet": "^8.0.0", + "multer": "^2.1.1", + "pdf-parse": "^1.1.4", "pg": "^8.13.1", "zod": "^3.24.2" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^5.0.0", + "@types/multer": "^2.1.0", "@types/pg": "^8.11.11", "tsx": "^4.19.0", "typescript": "^5.9.3" diff --git a/package.json b/package.json index e7f3585..45d51e8 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,12 @@ "url": "https://github.com/renefichtmueller/transceiver-db" }, "devDependencies": { + "@types/pdf-parse": "^1.1.5", "tsx": "^4.19", "typescript": "^5.9.3", "xlsx": "^0.18.5" + }, + "dependencies": { + "pdf-parse": "^1.1.4" } } diff --git a/packages/api/package.json b/packages/api/package.json index 46bd5af..d5997de 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -10,19 +10,22 @@ "start": "node dist/index.js" }, "dependencies": { - "express": "^5.1.0", - "pg": "^8.13.1", "cors": "^2.8.5", "dotenv": "^16.4.7", - "helmet": "^8.0.0", + "express": "^5.1.0", "express-rate-limit": "^7.5.0", + "helmet": "^8.0.0", + "multer": "^2.1.1", + "pdf-parse": "^1.1.4", + "pg": "^8.13.1", "zod": "^3.24.2" }, "devDependencies": { - "@types/express": "^5.0.0", - "@types/pg": "^8.11.11", "@types/cors": "^2.8.17", - "typescript": "^5.9.3", - "tsx": "^4.19.0" + "@types/express": "^5.0.0", + "@types/multer": "^2.1.0", + "@types/pg": "^8.11.11", + "tsx": "^4.19.0", + "typescript": "^5.9.3" } } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 9f485fe..4400735 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -43,6 +43,11 @@ import { vendorReliabilityRouter } from "./routes/vendor-reliability"; import { priceForecastRouter } from "./routes/price-forecast"; import { priceMatrixRouter } from "./routes/price-matrix"; import { trainingRouter } from "./routes/training"; +import { rfqRouter } from "./routes/rfq"; +import { priceAlertsRouter } from "./routes/price-alerts"; +import { winLossRouter } from "./routes/win-loss"; +import { apiKeysRouter } from "./routes/api-keys"; +import { roiRouter } from "./routes/roi"; const app = express(); @@ -52,7 +57,7 @@ app.set("trust proxy", 1); // Middleware app.use(helmet({ contentSecurityPolicy: false })); app.use(cors()); -app.use(express.json()); +app.use(express.json({ limit: "30mb" })); // 30MB to support base64-encoded PDF uploads app.use( rateLimit({ windowMs: 60 * 1000, @@ -126,6 +131,16 @@ app.use("/api/price-forecast", priceForecastRouter); app.use("/api/price-matrix", priceMatrixRouter); // Transceiver Academy — public training content (no auth required) app.use("/api/training", trainingRouter); +// RFQ Analyzer — quote vs market comparison +app.use("/api/rfq", rfqRouter); +// Price Alert Subscriptions +app.use("/api/price-alerts", priceAlertsRouter); +// Win/Loss Intelligence +app.use("/api/win-loss", winLossRouter); +// Customer API Key Management +app.use("/api/api-keys", apiKeysRouter); +// ROI Calculator +app.use("/api/roi", roiRouter); // Dashboard (static HTML) app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard"))); diff --git a/packages/api/src/llm/client.ts b/packages/api/src/llm/client.ts index 4faf13a..244e39f 100644 --- a/packages/api/src/llm/client.ts +++ b/packages/api/src/llm/client.ts @@ -33,7 +33,12 @@ const SETTINGS_FILE = join(process.env.TIP_ROOT || "/opt/tip", "blog-llm-setting const STATIC_FALLBACK_MODEL = "fo-blog-v10"; const DISCOVERY_REFRESH_MS = Number.parseInt(process.env.BLOG_LLM_DISCOVERY_REFRESH_MS || "", 10) || 10 * 60_000; -interface LlmSettings { provider: string; ollamaModel: string } +interface LlmSettings { + provider: string; + ollamaModel: string; + /** When set, auto-upgrade is disabled and this exact version is used. */ + pinnedVersion?: string; +} function loadSettingsRaw(): LlmSettings { try { @@ -42,6 +47,7 @@ function loadSettingsRaw(): LlmSettings { return { provider: raw.provider || process.env.BLOG_LLM_PROVIDER || "ollama", ollamaModel: raw.ollamaModel || process.env.OLLAMA_LLM_MODEL || STATIC_FALLBACK_MODEL, + pinnedVersion: raw.pinnedVersion || undefined, }; } } catch { /* ignore corrupt file */ } @@ -100,25 +106,46 @@ async function fetchOllamaFoBlogTags(): Promise { /** * Reconcile configured model against Ollama reality. * + * Always upgrades to the highest available fo-blog-vN BASE tag (no -r suffix). + * This ensures newly-trained versions are picked up automatically within 10 min, + * without needing to delete old Ollama tags or restart the API. + * * Priority: - * 1. Configured model (env or settings file) — if Ollama actually serves it - * 2. Highest fo-blog-v* version Ollama actually serves — auto-discovered + * 1. Highest fo-blog-vN base tag Ollama actually serves (auto-upgrade) + * 2. Configured model — if no upgrade candidate found * 3. Static fallback STATIC_FALLBACK_MODEL — last resort * * Non-blocking: any Ollama failure leaves _settings untouched. */ async function reconcileWithOllama(): Promise { + // Skip auto-upgrade when a version is explicitly pinned + if (_settings.pinnedVersion) return; + const configured = _settings.ollamaModel; if (!configured.startsWith("fo-blog-v")) return; // only manage fo-blog-* lane + const available = await fetchOllamaFoBlogTags(); if (available.length === 0) return; - if (available.includes(configured)) return; // configured model still exists + // Pick the highest base version (no -r suffix) available in Ollama const sorted = [...available].sort(compareFoBlogVersionsDesc); const winner = sorted[0]; - if (!winner || winner === configured) return; + if (!winner || winner === configured) return; // already on best, or nothing to do - console.log(`[LLM] auto-discovery: configured "${configured}" not in Ollama; switching to latest available "${winner}" (candidates: ${sorted.join(", ")})`); + // Only upgrade (never downgrade): winner must have a higher major version + const re = /^fo-blog-v(\d+)(?:-r(\d+))?$/; + const mc = re.exec(configured); + const mw = re.exec(winner); + if (mc && mw) { + const vc = Number.parseInt(mc[1], 10); + const vw = Number.parseInt(mw[1], 10); + if (vw <= vc) return; // winner is not newer — no-op + } + + const reason = available.includes(configured) + ? `newer version available` + : `"${configured}" no longer in Ollama`; + console.log(`[LLM] auto-upgrade: "${configured}" → "${winner}" (${reason}; candidates: ${sorted.join(", ")})`); _settings = { ..._settings, ollamaModel: winner }; try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } } @@ -129,11 +156,43 @@ let _settings = loadSettingsRaw(); void reconcileWithOllama(); setInterval(() => { void reconcileWithOllama(); }, DISCOVERY_REFRESH_MS).unref(); -/** Switch the active LLM provider at runtime. Persists to settings file. */ -export function setLlmProvider(provider: string, ollamaModel?: string): void { - _settings = { provider, ollamaModel: ollamaModel || _settings.ollamaModel }; +/** + * Switch the active LLM provider at runtime. Persists to settings file. + * Switching provider/model clears any existing pin so auto-upgrade can resume + * on the new provider — unless the caller explicitly passes a pinnedVersion. + */ +export function setLlmProvider(provider: string, ollamaModel?: string, pinnedVersion?: string): void { + _settings = { + ..._settings, // preserve any fields not explicitly overridden + provider, + ollamaModel: ollamaModel || _settings.ollamaModel, + pinnedVersion: pinnedVersion ?? undefined, // explicit undefined clears pin on provider switch + }; try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } - console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}`); + console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}${pinnedVersion ? ` [pinned]` : ""}`); +} + +/** + * Pin the active fo-blog version, disabling auto-upgrade. + * The model stays at `version` until explicitly unpinned. + */ +export function pinLlmVersion(version: string): void { + _settings = { ..._settings, ollamaModel: version, pinnedVersion: version }; + try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } + console.log(`[LLM] Version pinned → ${version} (auto-upgrade disabled)`); +} + +/** + * Remove the version pin — auto-upgrade resumes on next reconcile interval. + * Triggers an immediate reconcile so the highest available version is adopted + * without waiting up to DISCOVERY_REFRESH_MS. + */ +export async function unpinLlmVersion(): Promise { + _settings = { ..._settings, pinnedVersion: undefined }; + try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } + console.log("[LLM] Version unpinned — auto-upgrade re-enabled"); + await reconcileWithOllama(); + return { ..._settings }; } /** Returns the currently active provider config. */ diff --git a/packages/api/src/routes/api-keys.ts b/packages/api/src/routes/api-keys.ts new file mode 100644 index 0000000..add1c66 --- /dev/null +++ b/packages/api/src/routes/api-keys.ts @@ -0,0 +1,144 @@ +/** + * Customer API Key Management — /api/api-keys + * + * Manages externally-issued API keys for customer access to the TIP public API. + * Keys are stored as SHA-256 hashes. The actual key is shown ONCE at creation. + * + * Routes: + * POST /api/api-keys — Issue a new key (admin-only in prod) + * GET /api/api-keys — List keys (filter by email) + * DELETE /api/api-keys/:id — Revoke a key + * GET /api/api-keys/stats — Usage stats per key + * POST /api/api-keys/validate — Validate a key (internal use by middleware) + */ +import { Router, Request, Response } from "express"; +import { createHash, randomBytes } from "crypto"; +import { pool } from "../db/client"; + +export const apiKeysRouter = Router(); + +function hashKey(key: string): string { + return createHash("sha256").update(key).digest("hex"); +} + +function generateKey(): { key: string; prefix: string; hash: string } { + const raw = randomBytes(24).toString("base64url"); + const key = `tip_${raw}`; + const prefix = key.slice(0, 12); + const hash = hashKey(key); + return { key, prefix, hash }; +} + +// ── POST /api/api-keys — Issue new key ────────────────────────────────────── +apiKeysRouter.post("/", async (req: Request, res: Response) => { + const { email, label, tier = "free", rate_limit, expires_in_days } = req.body as Record; + + if (!email || !email.includes("@")) { + return res.status(400).json({ success: false, error: "Valid email required" }); + } + if (!label || typeof label !== "string") { + return res.status(400).json({ success: false, error: "label required" }); + } + + const RATE_LIMITS: Record = { free: 100, pro: 1000, enterprise: 10000 }; + const resolvedRateLimit = rate_limit ? parseInt(rate_limit) : RATE_LIMITS[tier] || 100; + const expiresAt = expires_in_days + ? new Date(Date.now() + parseInt(expires_in_days) * 86400000).toISOString() + : null; + + try { + const { key, prefix, hash } = generateKey(); + const result = await pool.query( + `INSERT INTO api_keys (key_hash, key_prefix, label, email, tier, rate_limit, expires_at) + VALUES ($1,$2,$3,$4,$5,$6,$7) + RETURNING id, key_prefix, label, email, tier, rate_limit, active, created_at, expires_at`, + [hash, prefix, label, email.toLowerCase().trim(), tier, resolvedRateLimit, expiresAt] + ); + + return res.status(201).json({ + success: true, + api_key: key, // shown ONCE — client must store it + warning: "Store this key now — it will not be shown again.", + meta: result.rows[0], + }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/api-keys?email= — List keys ──────────────────────────────────── +apiKeysRouter.get("/", async (req: Request, res: Response) => { + const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase(); + + try { + const result = await pool.query( + `SELECT id, key_prefix, label, email, tier, rate_limit, active, + last_used_at, usage_count, created_at, expires_at + FROM api_keys + WHERE ($1 = '' OR email = $1) + ORDER BY created_at DESC + LIMIT 100`, + [email] + ); + return res.json({ success: true, keys: result.rows }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── DELETE /api/api-keys/:id — Revoke key ─────────────────────────────────── +apiKeysRouter.delete("/:id", async (req: Request, res: Response) => { + try { + const result = await pool.query( + `UPDATE api_keys SET active = false WHERE id = $1 RETURNING id, key_prefix`, + [parseInt(String(req.params.id))] + ); + if (result.rowCount === 0) return res.status(404).json({ success: false, error: "Key not found" }); + return res.json({ success: true, revoked: result.rows[0] }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/api-keys/stats — Usage dashboard ─────────────────────────────── +apiKeysRouter.get("/stats", async (_req: Request, res: Response) => { + try { + const result = await pool.query(` + SELECT + tier, + COUNT(*) AS total_keys, + COUNT(*) FILTER (WHERE active) AS active_keys, + SUM(usage_count) AS total_requests, + MAX(last_used_at) AS last_activity + FROM api_keys + GROUP BY tier + ORDER BY total_requests DESC + `); + return res.json({ success: true, stats: result.rows }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── POST /api/api-keys/validate — Validate key (used by auth middleware) ──── +apiKeysRouter.post("/validate", async (req: Request, res: Response) => { + const { key } = req.body as { key?: string }; + if (!key) return res.status(400).json({ valid: false }); + + try { + const hash = hashKey(key); + const result = await pool.query( + `UPDATE api_keys + SET last_used_at = NOW(), usage_count = usage_count + 1 + WHERE key_hash = $1 + AND active = true + AND (expires_at IS NULL OR expires_at > NOW()) + RETURNING id, key_prefix, email, tier, rate_limit`, + [hash] + ); + if (result.rowCount === 0) return res.json({ valid: false }); + return res.json({ valid: true, ...result.rows[0] }); + } catch (err) { + return res.status(500).json({ valid: false, error: String(err) }); + } +}); diff --git a/packages/api/src/routes/blog.ts b/packages/api/src/routes/blog.ts index b73be9e..28e7f5c 100644 --- a/packages/api/src/routes/blog.ts +++ b/packages/api/src/routes/blog.ts @@ -10,8 +10,11 @@ * Voice: Senior optical network engineer, not marketing. */ import { Router, Request, Response } from "express"; +import * as pdfParseModule from "pdf-parse"; +const pdfParse: (buffer: Buffer) => Promise<{ text: string; numpages: number; info: Record }> = + (pdfParseModule as any).default ?? (pdfParseModule as any); import { pool } from "../db/client"; -import { setLlmProvider, getLlmProvider, refreshLlmAutoDiscovery } from "../llm/client"; +import { setLlmProvider, getLlmProvider, refreshLlmAutoDiscovery, pinLlmVersion, unpinLlmVersion } from "../llm/client"; /** In-memory pipeline progress tracker — step updates pushed here, polled via GET /api/blog/:id/progress */ const pipelineProgress = new Map(); @@ -984,6 +987,177 @@ async function processLlmQueue(): Promise { if (llmQueue.length > 0) setTimeout(() => processLlmQueue(), 3000); } +/** + * 3-pass pipeline for external content (from-url / from-pdf). + * Grounds entirely in source text — never expands from parametric knowledge, + * which prevents the "senior network engineer" persona from drifting to optics. + */ +async function runExternalContentPipeline( + draftId: string, + title: string, + selectedTopic: string, + targetAudience: string, + additionalContext: string, +): Promise { + const { + FO_BLOG_SYSTEM_PROMPT, + STEP8_KILL_AI_TONE, + STEP8c_STYLE_LOCK, + STEP_HEADLINE_GENERATION, + withCalibration, + buildFeedbackContext, + } = await import("../llm/fo-blog-pipeline"); + + const LLM_WRITE = { temperature: 0.7, maxTokens: 2000, timeoutMs: 480000 }; + const LLM_REFINE = { temperature: 0.35, maxTokens: 2000, timeoutMs: 480000 }; + + // Extract persona-neutral section of FO system prompt (keep writing style, drop optics persona) + const mindsetMarker = "YOUR MINDSET:"; + const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker); + const writingStyleRules = mindsetStart > -1 + ? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart) + : FO_BLOG_SYSTEM_PROMPT; + + // Load feedback + let feedbackContext = ""; + try { + const fbResult = await pool.query( + `SELECT score_overall, feedback_text, blog_type FROM blog_feedback + WHERE feedback_text IS NOT NULL AND feedback_text != '' + ORDER BY score_overall ASC LIMIT 20` + ); + feedbackContext = buildFeedbackContext(fbResult.rows.map(r => ({ + score: r.score_overall, feedback_text: r.feedback_text, blog_type: r.blog_type || "" + }))); + } catch { /* no feedback yet */ } + + // Extract just the PDF/URL text from additionalContext + const pdfStart = additionalContext.indexOf("--- EXTRACTED PDF CONTENT ---"); + const urlStart = additionalContext.indexOf("--- EXTRACTED PAGE CONTENT ---"); + const contentStart = pdfStart > -1 ? pdfStart : urlStart > -1 ? urlStart : -1; + const sourceText = contentStart > -1 + ? additionalContext.slice(contentStart).slice(0, 5000) + : additionalContext.slice(0, 5000); + + const externalSysPrompt = withCalibration( + `You are a senior IT infrastructure engineer and technical writer with 20+ years of experience.\n` + + `You write practical articles for IT architects, infrastructure managers, and decision-makers.\n\n` + + `ABSOLUTE RULE: You write ONLY about what the source document says. ` + + `Do NOT add optical transceivers, fiber optics, 400G, or compatible optics content ` + + `unless the source document explicitly covers it. ` + + `Your only source of facts is the document provided.\n\n` + + `━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` + + `WRITING STYLE (apply to everything you write):\n` + + `━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` + + writingStyleRules + feedbackContext + ); + + console.log(`Blog External Pipeline: Starting for ${draftId} — "${title}"`); + setProgress(draftId, 1, "Step 1/5: Extract source facts"); + + // ─── Pass 1: Extract key facts from source document ─────────────────────── + const extractPrompt = + `Extract the key factual content from the source document below.\n\n` + + `Return:\n` + + `- The core problem or challenge the document describes (2-3 sentences)\n` + + `- 5-7 specific facts, findings, or insights stated in the document\n` + + `- The main conclusion or practical recommendation\n` + + `- Who this affects and why it matters to them\n\n` + + `Do NOT add information not present in the document. Do NOT interpret or expand.\n` + + `Return only what the document actually says.\n\n` + + sourceText; + + await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 }).catch(() => {}); + const extractResult = await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 }); + + setProgress(draftId, 2, "Step 2/5: Write article draft"); + + // ─── Pass 2: Write the full article from extracted facts ────────────────── + const draftPrompt = + `Write a blog article titled "${title}".\n\n` + + `Use ONLY the facts below as your source material. Do not add any facts not listed here.\n\n` + + `SOURCE FACTS:\n${extractResult.text}\n\n` + + `ARTICLE REQUIREMENTS:\n` + + `- 600-900 words\n` + + `- Continuous narrative prose — no section headers, no bullet lists in the body\n` + + `- First-person engineering voice ("I've seen this happen...", "The problem is...")\n` + + `- Start with a specific operational scenario, not a general statement\n` + + `- Explain consequences for real infrastructure decisions\n` + + `- End with a concrete implication or call to action, not a summary\n` + + `- Apply all writing style rules from your system prompt\n\n` + + `Topic: "${title}"\n` + + `Audience: ${targetAudience} engineers and infrastructure decision-makers`; + + const draftResult = await generate(externalSysPrompt, draftPrompt, LLM_WRITE); + console.log(` Draft: ${draftResult.text.split(/\s+/).length} words`); + + setProgress(draftId, 3, "Step 3/5: Kill AI tone"); + + // ─── Pass 3: Kill AI tone ───────────────────────────────────────────────── + const step8 = await generate(externalSysPrompt, + STEP8_KILL_AI_TONE.replace("{{ARTICLE}}", draftResult.text), + LLM_REFINE + ); + + setProgress(draftId, 4, "Step 4/5: Style lock"); + + // Skip STEP8b_REDUCTION for external content — it targets 1,200-2,000 words + // with "DO NOT go below 1,000 words" which conflicts with thin source material. + // Just apply style lock for readability polish. + const step8c = await generate(externalSysPrompt, + STEP8c_STYLE_LOCK.replace("{{ARTICLE}}", step8.text), + LLM_REFINE + ); + + setProgress(draftId, 5, "Step 5/5: LinkedIn + headline"); + + // ─── LinkedIn post — topic-neutral version (no optics example) ──────────── + const externalLinkedInPrompt = + `Write a LinkedIn post for this article.\n\n` + + `FORMAT:\n` + + `Line 1-2: HOOK — a reframe or uncomfortable truth from the article. NOT an announcement.\n\n` + + `3-5 SHORT BEATS — each beat is 1-3 lines. One insight per beat. No bullet markers.\n\n` + + `Last line before hashtags: "Full breakdown in the blog — link in first comment."\n\n` + + `HASHTAGS (last line): 3-4 relevant hashtags based on the article topic. Include #Flexoptix.\n` + + ` Pick hashtags that match what the article is actually about — NOT #OpticalNetworking unless the article covers optics.\n\n` + + `RULES:\n` + + `- No emojis\n` + + `- No "I'm thrilled to share" or "Excited to announce"\n` + + `- Engineer voice — specific, blunt, useful\n` + + `- Maximum 2,800 characters\n` + + `- Return ONLY the post text. No commentary.\n\n` + + `Article:\n${step8c.text}`; + + const linkedInResult = await generate(externalSysPrompt, + externalLinkedInPrompt, + { ...LLM_REFINE, maxTokens: 600 } + ).catch(() => ({ text: "" })); + + // ─── Headline ────────────────────────────────────────────────────────────── + const headlineResult = await generate(externalSysPrompt, + STEP_HEADLINE_GENERATION.replace("{{ARTICLE}}", step8c.text), + { temperature: 0.5, maxTokens: 80, timeoutMs: 60000 } + ).catch(() => ({ text: title })); + + const finalTitle = headlineResult.text.trim().replace(/^["']|["']$/g, "").replace(/\n.*$/s, "").trim() || title; + const wordCount = step8c.text.split(/\s+/).length; + + await pool.query( + `UPDATE blog_drafts SET + title = $1, + draft_content = $2, + linkedin_post = $3, + word_count = $4, + status = 'draft', + updated_at = NOW() + WHERE id = $5`, + [finalTitle, step8c.text, linkedInResult.text || null, wordCount, draftId] + ); + + clearProgress(draftId); + console.log(`Blog External Pipeline: ${draftId} complete — ${wordCount} words, title: "${finalTitle}"`); +} + /** Run 10-Step Flexoptix Style LLM Pipeline and update draft in-place */ async function runLlmPipeline( draftId: string, @@ -993,6 +1167,12 @@ async function runLlmPipeline( data: Awaited>, additionalContext?: string, ): Promise { + // External content (from-url / from-pdf) uses a grounded 5-pass pipeline + // that never expands from parametric knowledge — prevents optics topic drift + if (additionalContext?.startsWith("⚠️ TOPIC LOCK") && additionalContext.length > 200) { + return runExternalContentPipeline(draftId, title, selectedTopic, targetAudience, additionalContext); + } + // Lazy-load the new FO pipeline const { FO_BLOG_SYSTEM_PROMPT, @@ -1040,7 +1220,48 @@ async function runLlmPipeline( }))); } catch { /* no feedback yet, that's fine */ } - const systemPrompt = withCalibration(FO_BLOG_SYSTEM_PROMPT + feedbackContext); + // For external content (from-url / from-pdf), the Flexoptix optical-networking persona + // must be replaced — otherwise every article drifts back to 400G transceivers regardless + // of what the TOPIC LOCK says. Strip the Flexoptix mandate and inject a generic persona. + const isExternalContent = additionalContext?.startsWith("⚠️ TOPIC LOCK"); + const extractedTopicName = isExternalContent + ? (additionalContext?.match(/TOPIC LOCK[^"]*"([^"]+)"/) || [])[1] || title + : title; + + // Build the section of FO_BLOG_SYSTEM_PROMPT that's topic-neutral (writing style rules only). + // The Flexoptix persona block ends at the first ════ separator after line 65 ("YOUR MINDSET"). + const mindsetMarker = "YOUR MINDSET:"; + const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker); + const writingStyleRules = mindsetStart > -1 + ? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart) + : FO_BLOG_SYSTEM_PROMPT; + + const externalSystemPrompt = `\ +╔══════════════════════════════════════════════════════════════════════╗ +║ EXTERNAL CONTENT MODE — these rules override ALL defaults below ║ +╚══════════════════════════════════════════════════════════════════════╝ + +THIS ARTICLE IS ABOUT: "${extractedTopicName}" + +You are a senior IT infrastructure engineer and technical writer. +Your readers are IT architects, infrastructure managers, and decision-makers. + +ABSOLUTE RULE: Write ONLY about "${extractedTopicName}". +Do NOT write about optical transceivers, fiber optics, 400G, DR4, compatible optics, +Flexoptix products, or any networking hardware UNLESS the source document below +explicitly covers that topic. The source document is your sole editorial mandate. + +The Flexoptix brand rules and compatible-optics framing in this prompt DO NOT APPLY +to this article. This is a general IT/infrastructure piece, not an optics blog post. + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +WRITING STYLE (applies to all articles — keep these): +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +${writingStyleRules}`; + + const systemPrompt = isExternalContent + ? withCalibration(externalSystemPrompt + feedbackContext) + : withCalibration(FO_BLOG_SYSTEM_PROMPT + feedbackContext); // Warmup await generate("Test", "OK", { temperature: 0.1, maxTokens: 8, timeoutMs: 60000 }).catch(() => {}); @@ -1100,11 +1321,23 @@ async function runLlmPipeline( // ═══ STEP 1: Topic Expansion ═══ console.log(" Step 1/10: Topic Expansion..."); setProgress(draftId, 1, "Step 1/10: Topic Expansion"); + + // For external content: inject a hard topic anchor before the standard prompt so the + // LLM cannot drift back to optical networking when expanding the topic in Step 1. + const step1TopicPrefix = isExternalContent + ? `HARD TOPIC LOCK: This article is about "${extractedTopicName}". ` + + `It is NOT about optical transceivers, fiber optics, 400G migrations, or compatible optics. ` + + `Your expansion below must stay strictly within the topic stated above.\n\n` + : ""; + const step1 = await generate(systemPrompt, - STEP1_TOPIC_EXPANSION - .replace("{{TOPIC}}", title) + step1TopicPrefix + STEP1_TOPIC_EXPANSION + .replace("{{TOPIC}}", isExternalContent ? extractedTopicName : title) .replace("{{ADDITIONAL_CONTEXT}}", additionalContext - ? `\n\n---\nBACKGROUND REFERENCE (editorial context — use as factual direction ONLY):\n${additionalContext}\n\nCRITICAL: Do NOT copy any phrase, sentence, or wording from the above into the article or any step output. It is context for your understanding, not source material.` + ? `\n\n---\nBACKGROUND REFERENCE (use as factual direction ONLY — do not copy verbatim):\n${additionalContext.slice(0, 4000)}\n\n` + + (isExternalContent + ? `REMINDER: Write about "${extractedTopicName}" — NOT optical networking. The background above is your source material.` + : `CRITICAL: Do NOT copy any phrase, sentence, or wording from the above into the article or any step output.`) : ""), LLM_OPTS ); @@ -1531,15 +1764,34 @@ blogRouter.post("/generate", async (req: Request, res: Response) => { } }); +/** Paywall signal patterns in page HTML */ +const PAYWALL_PATTERNS = [ + /class=["'][^"']*paywall[^"']*["']/i, + /id=["'][^"']*paywall[^"']*["']/i, + /"paywall"\s*:/i, + /data-paywall/i, + /subscribe (to|now) (read|access|continue)/i, + /sign[- ]?in to (read|continue|access)/i, + /log[- ]?in to (read|continue|access)/i, + /create (a free )?account to (read|continue|access)/i, + /register (now )?to (read|continue|access)/i, + /this (article|content|paper) is (for|available to) (subscribers?|members?)/i, + /premium (content|article|paper)/i, + /access (this|the) (full )?(article|content|paper)/i, + /metered[- ]?content/i, + /subscriber[- ]?only/i, + /intel(ligence)?\.theregister\.com\/paper/i, +]; + /** Fetch a URL and extract readable text content for use as LLM context. - * Returns spaDetected=true when extracted body text is thin (< 300 chars), - * indicating a JavaScript Single Page Application where content is rendered client-side. - * In that case, metaDesc contains OG/meta description fallback text. + * Returns spaDetected=true when extracted body text is thin (< 300 chars). + * Returns paywallDetected=true when login/subscription wall signals are found. */ async function fetchUrlContent(rawUrl: string): Promise<{ pageTitle: string; text: string; spaDetected: boolean; + paywallDetected: boolean; metaDesc: string; }> { const response = await fetch(rawUrl, { @@ -1616,8 +1868,11 @@ async function fetchUrlContent(rawUrl: string): Promise<{ // Detect SPA: very little body text means JS renders the real content const spaDetected = text.length < 300; - // When SPA detected, enrich text with what we could extract from meta tags - if (spaDetected && (metaDesc || ogSiteName)) { + // Detect paywall: check raw HTML for subscription/login wall signals + const paywallDetected = PAYWALL_PATTERNS.some(p => p.test(html.slice(0, 20000))); + + // When SPA/paywall detected, enrich text with what we could extract from meta tags + if ((spaDetected || paywallDetected) && (metaDesc || ogSiteName)) { const parts: string[] = []; if (ogSiteName) parts.push(`Site: ${ogSiteName}`); if (pageTitle) parts.push(`Title: ${pageTitle}`); @@ -1625,7 +1880,7 @@ async function fetchUrlContent(rawUrl: string): Promise<{ text = parts.join("\n"); } - return { pageTitle, text, spaDetected, metaDesc }; + return { pageTitle, text, spaDetected, paywallDetected, metaDesc }; } // POST /api/blog/from-url — Fetch URL, extract content, generate a blog from it @@ -1656,13 +1911,35 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => { try { // Fetch page content server-side (no CORS issues) - const { pageTitle, text: extractedText, spaDetected, metaDesc } = await fetchUrlContent(url); + const { pageTitle, text: extractedText, spaDetected, paywallDetected, metaDesc } = await fetchUrlContent(url); console.log( `Blog from-url: fetched "${pageTitle}" from ${parsedUrl.hostname} ` + - `(${extractedText.length} chars${spaDetected ? ", SPA detected" : ""})` + `(${extractedText.length} chars${spaDetected ? ", SPA" : ""}${paywallDetected ? ", PAYWALL" : ""})` ); + // Paywall or inaccessible content — return signal so client can prompt for PDF upload. + // Also catches meta-refresh redirects and other "content gatekeeping" patterns where + // the fetched HTML is tiny and yields no usable text or metadata. + const contentBlocked = + paywallDetected || + (spaDetected && extractedText.length < 50 && !metaDesc && !pageTitle); + + if (contentBlocked) { + res.json({ + success: false, + paywall_detected: true, + page_title: pageTitle, + meta_desc: metaDesc, + source_url: url, + topic: selectedTopic, + error: paywallDetected + ? "Paywall erkannt — bitte PDF hochladen" + : "Seite nicht zugänglich — bitte PDF hochladen", + }); + return; + } + // Build a rich additional_context from the URL content. // When a SPA is detected (JS-rendered), body text is a shell — we rely on meta tags instead. const spaWarning = spaDetected @@ -1672,6 +1949,8 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => { : ""; const additionalContext = + `⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${pageTitle || parsedUrl.hostname}"\n` + + `The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source article explicitly covers them.\n\n` + `SOURCE URL: ${url}\n` + `PAGE TITLE: ${pageTitle}\n` + `HOSTNAME: ${parsedUrl.hostname}\n` + @@ -1687,20 +1966,22 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => { const title = pageTitle || parsedUrl.hostname; const template = templates[Math.floor(Math.random() * templates.length)]; - // When SPA detected, skip optical transceiver product injection — it pollutes the LLM context - // with irrelevant product data and causes the model to default to its fine-tuning domain. - // Use empty data so the pipeline focuses purely on the URL context provided above. - const keywords = spaDetected - ? [parsedUrl.hostname.replace(/^www\./, ""), pageTitle].filter(Boolean) - : [...template.seo_keywords, "optical transceiver", "networking"].filter(Boolean); + // For from-url flow: ALWAYS use empty data — no transceiver product injection. + // The URL content IS the data. Injecting transceiver products would cause the + // fine-tuned model to ignore the source article and write a generic 400G post. + const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] }; - const data = spaDetected - ? { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] } - : await gatherBlogData(keywords, selectedTopic); - - const draftContent = generateTemplateDraft(title, selectedTopic, data); + // Use a minimal placeholder draft — generateTemplateDraft produces transceiver-specific + // skeleton content (NOC scenarios, DOM readings) that pollutes the LLM context. + const date = new Date().toISOString().split("T")[0]; + const draftContent = + `# ${title}\n\n` + + `*Generated from URL: ${url} on ${date}*\n\n` + + `> **Status**: Pending LLM enhancement — source article loaded.\n\n` + + `**Source**: ${url}\n` + + (metaDesc ? `**Summary**: ${metaDesc}\n` : ""); const wordCount = draftContent.split(/\s+/).length; - const initialIssues = validateArticle(draftContent); + const initialIssues: string[] = []; const activeModel = getLlmProvider(); const generatedBy = `tip-blog-from-url-${activeModel.ollamaModel || activeModel.provider || "llm"}`; @@ -1761,6 +2042,147 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => { } }); +// POST /api/blog/from-pdf — Upload a PDF, extract text, generate blog from it +// Accepts JSON body: { pdf_base64: string, filename: string, url?: string, topic?: string, page_title?: string } +// Using base64 JSON instead of multipart to avoid Cloudflare WAF blocking binary uploads. +blogRouter.post("/from-pdf", async (req: Request, res: Response) => { + const { pdf_base64, filename, url, topic, page_title } = req.body as { + pdf_base64?: string; + filename?: string; + url?: string; + topic?: string; + page_title?: string; + }; + + if (!pdf_base64) { + res.status(400).json({ success: false, error: "Keine PDF-Daten empfangen (pdf_base64 fehlt)" }); + return; + } + + // Validate base64 + decode + let fileBuffer: Buffer; + let fileSize: number; + try { + fileBuffer = Buffer.from(pdf_base64, "base64"); + fileSize = fileBuffer.length; + if (fileSize < 100) throw new Error("Datei zu klein"); + if (fileSize > 20 * 1024 * 1024) throw new Error("Datei zu groß (max 20 MB)"); + } catch (err) { + res.status(400).json({ success: false, error: `Ungültige PDF-Daten: ${(err as Error).message}` }); + return; + } + + const originalName = filename || "upload.pdf"; + const selectedTopic = topic || "technology_deep_dive"; + const templates = BLOG_TEMPLATES[selectedTopic]; + if (!templates) { + res.status(400).json({ success: false, error: `Ungültiger Blog-Typ. Gültig: ${Object.keys(BLOG_TEMPLATES).join(", ")}` }); + return; + } + + try { + // Extract text from PDF + const pdfData = await pdfParse(fileBuffer); + let extractedText = pdfData.text + .split("\n").map((l: string) => l.trim()).filter((l: string) => l.length > 20).join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + + if (extractedText.length < 100) { + res.status(422).json({ success: false, error: "PDF enthält zu wenig lesbaren Text (ggf. gescannt/bildbasiert)" }); + return; + } + + // Limit to ~6000 chars for LLM context + if (extractedText.length > 6000) { + extractedText = extractedText.slice(0, 6000) + "\n[… PDF content truncated for LLM context …]"; + } + + const title: string = page_title || (typeof pdfData.info?.Title === "string" ? pdfData.info.Title : null) || originalName.replace(/\.pdf$/i, "") || "Artikel aus PDF"; + + console.log(`Blog from-pdf: "${title}" — ${extractedText.length} chars from ${originalName} (${(fileSize / 1024).toFixed(0)} KB)`); + + const additionalContext = + `⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${title}"\n` + + `The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source document explicitly covers them.\n\n` + + (url ? `SOURCE URL: ${url}\n` : "") + + `SOURCE FILE: ${originalName}\n` + + `PAGE TITLE: ${title}\n` + + `\n--- EXTRACTED PDF CONTENT ---\n` + + `${extractedText}\n` + + `--- END PDF CONTENT ---\n\n` + + `IMPORTANT: Use this content as factual background and editorial direction. ` + + `The blog MUST be about the topic described above. ` + + `Do NOT copy sentences verbatim. Write a Flexoptix-voice blog article using these facts and insights.`; + + const template = templates[Math.floor(Math.random() * templates.length)]; + const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] }; + + const date = new Date().toISOString().split("T")[0]; + const draftContent = + `# ${title}\n\n` + + `*Generated from PDF: ${originalName} on ${date}*\n\n` + + `> **Status**: Pending LLM enhancement — PDF content loaded.\n\n` + + (url ? `**Source URL**: ${url}\n` : "") + + `**Source file**: ${originalName} (${(fileSize / 1024).toFixed(0)} KB, ${pdfData.numpages} pages)\n`; + + const wordCount = draftContent.split(/\s+/).length; + const activeModel = getLlmProvider(); + const generatedBy = `tip-blog-from-pdf-${activeModel.ollamaModel || activeModel.provider || "llm"}`; + + const result = await pool.query( + `INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords) + VALUES ($1, $2, $3, $4, $5, $6, 'draft', $7, $8, $9) + RETURNING id, created_at`, + [ + title, + selectedTopic, + template.target_audience, + JSON.stringify({ generation_method: "from-pdf", source_url: url || null, source_file: originalName, pdf_pages: pdfData.numpages }), + draftContent, + JSON.stringify({ source_url: url || null, source_file: originalName, extracted_chars: extractedText.length, pdf_pages: pdfData.numpages }), + generatedBy, + wordCount, + template.seo_keywords, + ], + ); + + const draftId = result.rows[0].id; + + const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" })); + let llmStarted = false; + if (health.ok) { + llmStarted = true; + enqueueLlmPipeline(draftId, title, selectedTopic, template.target_audience, data, additionalContext).catch((err) => { + console.error(`Blog from-pdf LLM pipeline error: ${(err as Error).message}`); + }); + } + + res.json({ + success: true, + source_file: originalName, + source_url: url || null, + page_title: title, + extracted_chars: extractedText.length, + pdf_pages: pdfData.numpages, + draft: { + id: draftId, + title, + topic: selectedTopic, + target_audience: template.target_audience, + word_count: wordCount, + generation_method: "from-pdf", + llm_enhancing: llmStarted, + created_at: result.rows[0].created_at, + }, + }); + } catch (err) { + const msg = (err as Error).message; + console.error(`Blog from-pdf error: ${msg}`); + res.status(500).json({ success: false, error: `PDF konnte nicht verarbeitet werden: ${msg}` }); + } +}); + // GET /api/blog — List all drafts blogRouter.get("/", async (_req: Request, res: Response) => { try { @@ -1790,6 +2212,102 @@ blogRouter.post("/llm/reset-queue", (_req: Request, res: Response) => { res.json({ success: true, message: "Ollama queue reset — stuck requests cleared" }); }); +// GET /api/blog/llm/model-info — Training metadata for the active fo-blog model +// Returns Ollama model details + training manifest stats (pairs, eval, base model, revision, etc.) +blogRouter.get("/llm/model-info", async (_req: Request, res: Response) => { + try { + const settings = getLlmProvider(); + const ollamaUrl = process.env.OLLAMA_URL || "http://localhost:11434"; + + // Only meaningful for fo-blog Ollama models + const modelName = settings.ollamaModel || ""; + const isFoBlog = /^fo-blog-v\d+/.test(modelName); + + // 1. Ollama /api/show — model metadata + let ollamaInfo: Record | null = null; + if (settings.provider === "ollama" && modelName) { + try { + const r = await fetch(`${ollamaUrl}/api/show`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: modelName }), + signal: AbortSignal.timeout(6000), + }); + if (r.ok) ollamaInfo = (await r.json()) as Record; + } catch { /* non-fatal */ } + } + + // 2. Ollama /api/tags — count available fo-blog-vX revisions + let availableVersions: string[] = []; + try { + const r = await fetch(`${ollamaUrl}/api/tags`, { signal: AbortSignal.timeout(5000) }); + if (r.ok) { + const data = (await r.json()) as { models?: { name: string; modified_at?: string }[] }; + availableVersions = (data.models || []) + .map((m) => m.name) + .filter((n) => /^fo-blog-v\d+(?:-r\d+)?:/.test(n)); + } + } catch { /* non-fatal */ } + + // 3. Training manifest on disk (training-data/runpod/blog_llm/manifest.json) + let manifest: Record | null = null; + try { + const { readFileSync } = await import("fs"); + const { resolve } = await import("path"); + const manifestPath = resolve(process.cwd(), "training-data/runpod/blog_llm/manifest.json"); + manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + } catch { /* manifest may not exist on this deployment */ } + + // Extract structured fields from Ollama response + const details = (ollamaInfo?.details ?? {}) as Record; + const modelInfo = (ollamaInfo?.model_info ?? {}) as Record; + const params = (ollamaInfo?.parameters ?? "") as string; + + // Parse revision from parent_model e.g. "fo-blog-v13-r16:latest" → r16 + const parentModel = (details?.parent_model ?? "") as string; + const revMatch = parentModel.match(/-r(\d+)/); + const revision = revMatch ? `r${revMatch[1]}` : null; + + // Parse context length from parameters string + const ctxMatch = params.match(/num_ctx\s+(\d+)/); + const contextLength = ctxMatch ? Number.parseInt(ctxMatch[1], 10) : null; + + // Count major versions (fo-blog-vX:latest, not revisions like fo-blog-vX-rY:latest) + const majorVersions = availableVersions.filter((n) => !/v\d+-r\d+:/.test(n)); + const allRevisions = availableVersions.filter((n) => /v\d+-r\d+:/.test(n)); + + res.json({ + success: true, + provider: settings.provider, + model: modelName, + is_fo_blog: isFoBlog, + revision, + parent_model: parentModel || null, + trained_at: (ollamaInfo?.modified_at ?? null) as string | null, + quantization: (details?.quantization_level ?? null) as string | null, + parameter_size: (details?.parameter_size ?? null) as string | null, + base_model: (modelInfo?.["general.basename"] ?? null) as string | null, + finetune_id: (modelInfo?.["general.finetune"] ?? null) as string | null, + parameter_count: (modelInfo?.["general.parameter_count"] ?? null) as number | null, + context_length: contextLength ?? ((modelInfo?.["qwen2.context_length"] ?? null) as number | null), + temperature: Number.parseFloat(params.match(/temperature\s+([\d.]+)/)?.[1] ?? "NaN") || null, + // Training data stats from manifest + training_pairs: (manifest?.training_pairs ?? null) as number | null, + train_pairs: (manifest?.train_pairs ?? null) as number | null, + eval_pairs: (manifest?.eval_pairs ?? null) as number | null, + raw_pairs: (manifest?.raw_pairs ?? null) as number | null, + // Version availability + major_versions_available: majorVersions.length, + total_revisions_available: allRevisions.length, + available_versions: majorVersions, + // Pin status + pinned_version: settings.pinnedVersion ?? null, + }); + } catch (err) { + res.status(500).json({ success: false, error: (err as Error).message }); + } +}); + // POST /api/blog/llm/refresh-discovery — Force auto-discovery to pick up newly-trained fo-blog-v* versions // Useful right after Magatama adopts a new fo-blog-vN model. Otherwise runs every 10 min by itself. blogRouter.post("/llm/refresh-discovery", async (_req: Request, res: Response) => { @@ -1828,6 +2346,45 @@ blogRouter.post("/llm/switch", (req: Request, res: Response) => { }); }); +// POST /api/blog/llm/pin — Pin a specific fo-blog-vN model, disabling auto-upgrade +// Body: { version: "fo-blog-v13" } — omit version to pin the current model +blogRouter.post("/llm/pin", (req: Request, res: Response) => { + const { version } = req.body as { version?: string }; + const current = getLlmProvider(); + const target = version || current.ollamaModel; + + if (!target.startsWith("fo-blog-v")) { + res.status(400).json({ success: false, error: "Only fo-blog-v* models can be pinned" }); + return; + } + + pinLlmVersion(target); + const next = getLlmProvider(); + console.log(`[blog/llm/pin] pinned to ${target}`); + res.json({ + success: true, + pinned: target, + active: { provider: next.provider, model: next.ollamaModel, pinnedVersion: next.pinnedVersion }, + message: `Pinned to ${target} — auto-upgrade disabled`, + }); +}); + +// POST /api/blog/llm/unpin — Remove version pin, re-enable auto-upgrade +// Immediately reconciles against Ollama so the highest available version is adopted. +blogRouter.post("/llm/unpin", async (_req: Request, res: Response) => { + try { + const active = await unpinLlmVersion(); + console.log(`[blog/llm/unpin] unpinned, active → ${active.ollamaModel}`); + res.json({ + success: true, + active: { provider: active.provider, model: active.ollamaModel }, + message: `Unpinned — auto-upgrade re-enabled. Active: ${active.ollamaModel}`, + }); + } catch (err) { + res.status(500).json({ success: false, error: (err as Error).message }); + } +}); + // GET /api/blog/:id — Get a specific draft with full content // GET /api/blog/:id/progress — Real-time pipeline step progress (in-memory) blogRouter.get("/:id/progress", (req: Request, res: Response) => { diff --git a/packages/api/src/routes/bulk-price.ts b/packages/api/src/routes/bulk-price.ts index f8d5cc3..b8711b8 100644 --- a/packages/api/src/routes/bulk-price.ts +++ b/packages/api/src/routes/bulk-price.ts @@ -52,7 +52,7 @@ bulkPriceRouter.post("/", async (req: Request, res: Response) => { observed_at: Date; }>( `WITH matched AS ( - SELECT id, part_number, model_name, form_factor, speed_gbps + SELECT id, part_number, COALESCE(standard_name, part_number, '') AS model_name, form_factor, speed_gbps FROM transceivers WHERE part_number ILIKE ANY (ARRAY[${placeholders}]) ), diff --git a/packages/api/src/routes/kb.ts b/packages/api/src/routes/kb.ts index 000d639..b63a703 100644 --- a/packages/api/src/routes/kb.ts +++ b/packages/api/src/routes/kb.ts @@ -1,24 +1,27 @@ import { Router, Request, Response } from "express"; import { pool } from "../db/client"; +import { semanticSearch } from "../embeddings/client"; export const kbRouter = Router(); // GET /api/kb — Knowledge base browser: FAQ + troubleshooting entries -// ?q=search&category=faq|troubleshooting|known_issue&limit=50 +// ?q=search&category=faq|troubleshooting|known_issue&limit=50&semantic=1 +// Falls back to Qdrant semantic search when ILIKE returns 0 results kbRouter.get("/", async (req: Request, res: Response) => { const q = ((req.query.q as string) || "").trim(); const category = (req.query.category as string) || ""; const limit = Math.min(parseInt((req.query.limit as string) || "60"), 200); + const forceSemantic = req.query.semantic === "1"; try { - const [entries, cats] = await Promise.all([ + const [textEntries, cats] = await Promise.all([ pool.query( `SELECT id, category, subcategory, question, answer, applies_to_form_factors, applies_to_speeds, severity, tags FROM knowledge_base WHERE ($1 = '' OR category = $1) - AND ($2 = '' OR question ILIKE '%' || $2 || '%' - OR answer ILIKE '%' || $2 || '%' + AND ($2 = '' OR question ILIKE '%' || $2 || '%' + OR answer ILIKE '%' || $2 || '%' OR subcategory ILIKE '%' || $2 || '%') ORDER BY CASE WHEN $2 != '' AND question ILIKE '%' || $2 || '%' THEN 0 ELSE 1 END, @@ -34,12 +37,80 @@ kbRouter.get("/", async (req: Request, res: Response) => { ), ]); - res.json({ + // If text search found results and semantic not forced, return them + if (textEntries.rows.length > 0 && !forceSemantic) { + return res.json({ + success: true, + entries: textEntries.rows, + categories: cats.rows, + total: textEntries.rows.length, + query: q, + search_mode: "text", + }); + } + + // Semantic fallback — only when query is provided and text search returned nothing + if (q.length > 2) { + try { + const collections: Array<"faq_embeddings" | "troubleshooting_embeddings"> = + category === "faq" ? ["faq_embeddings"] : + category === "troubleshooting" ? ["troubleshooting_embeddings"] : + ["faq_embeddings", "troubleshooting_embeddings"]; + + const semanticHits = ( + await Promise.all( + collections.map(col => + semanticSearch(col, q, Math.ceil(limit / collections.length)) + .catch(() => [] as Array<{ id: string; score: number; payload: Record }>) + ) + ) + ).flat().sort((a, b) => b.score - a.score); + + // Deduplicate by kb id from payload, then fetch full rows from DB + const kbIds = [...new Set( + semanticHits + .filter(h => h.score >= 0.5 && h.payload.kb_id) + .slice(0, limit) + .map(h => h.payload.kb_id as string) + )]; + + if (kbIds.length > 0) { + const semanticRows = await pool.query( + `SELECT id, category, subcategory, question, answer, + applies_to_form_factors, applies_to_speeds, severity, tags + FROM knowledge_base + WHERE id = ANY($1::int[])`, + [kbIds.map(Number).filter(n => !isNaN(n))] + ); + + // Sort results by semantic score order + const scoreMap = new Map(semanticHits.map(h => [String(h.payload.kb_id), h.score])); + const sorted = semanticRows.rows.sort( + (a, b) => (scoreMap.get(String(b.id)) || 0) - (scoreMap.get(String(a.id)) || 0) + ); + + return res.json({ + success: true, + entries: sorted, + categories: cats.rows, + total: sorted.length, + query: q, + search_mode: "semantic", + }); + } + } catch (_semErr) { + // Semantic search unavailable — fall through to text results + } + } + + // Final fallback: return text results (even if empty) + return res.json({ success: true, - entries: entries.rows, + entries: textEntries.rows, categories: cats.rows, - total: entries.rows.length, + total: textEntries.rows.length, query: q, + search_mode: "text", }); } catch (err) { res.status(500).json({ success: false, error: String(err) }); diff --git a/packages/api/src/routes/price-alerts.ts b/packages/api/src/routes/price-alerts.ts new file mode 100644 index 0000000..237bedf --- /dev/null +++ b/packages/api/src/routes/price-alerts.ts @@ -0,0 +1,188 @@ +/** + * Price Alert Subscriptions — /api/price-alerts + * + * Users subscribe to price thresholds for specific SKUs or form factor/speed combos. + * A background checker (called by the scraper scheduler) evaluates active subscriptions + * against the latest price_observations and queues email delivery. + * + * Routes: + * POST /api/price-alerts — Create subscription + * GET /api/price-alerts?email= — List subscriptions for an email + * DELETE /api/price-alerts/:id — Cancel subscription + * POST /api/price-alerts/check — Internal: evaluate + queue alerts (scheduler) + * GET /api/price-alerts/triggered — Recent triggered alerts + */ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; + +export const priceAlertsRouter = Router(); + +// ── POST /api/price-alerts — Create a price alert subscription ─────────────── +priceAlertsRouter.post("/", async (req: Request, res: Response) => { + const { + email, transceiver_id, form_factor, speed_gbps, + threshold_price, currency = "USD", direction = "below", vendor_id, + } = req.body as Record; + + if (!email || typeof email !== "string" || !email.includes("@")) { + return res.status(400).json({ success: false, error: "Valid email required" }); + } + if (!threshold_price || isNaN(parseFloat(threshold_price))) { + return res.status(400).json({ success: false, error: "threshold_price required" }); + } + if (!transceiver_id && !form_factor && !speed_gbps) { + return res.status(400).json({ success: false, error: "At least one of: transceiver_id, form_factor, speed_gbps" }); + } + + try { + const result = await pool.query( + `INSERT INTO price_alert_subscriptions + (email, transceiver_id, form_factor, speed_gbps, threshold_price, currency, direction, vendor_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING id, email, threshold_price, currency, direction, created_at`, + [ + email.toLowerCase().trim(), + transceiver_id || null, + form_factor || null, + speed_gbps ? parseFloat(speed_gbps) : null, + parseFloat(threshold_price), + currency.toUpperCase(), + direction, + vendor_id || null, + ] + ); + + return res.status(201).json({ success: true, subscription: result.rows[0] }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/price-alerts?email= — List subscriptions ──────────────────────── +priceAlertsRouter.get("/", async (req: Request, res: Response) => { + const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase(); + if (!email) return res.status(400).json({ success: false, error: "email required" }); + + try { + const result = await pool.query( + `SELECT pas.*, + t.standard_name, t.form_factor AS tx_form_factor, t.speed_gbps AS tx_speed, + v.name AS vendor_name + FROM price_alert_subscriptions pas + LEFT JOIN transceivers t ON t.id = pas.transceiver_id + LEFT JOIN vendors v ON v.id = pas.vendor_id + WHERE pas.email = $1 + ORDER BY pas.created_at DESC`, + [email] + ); + return res.json({ success: true, subscriptions: result.rows }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── DELETE /api/price-alerts/:id — Cancel subscription ─────────────────────── +priceAlertsRouter.delete("/:id", async (req: Request, res: Response) => { + const id = String(req.params.id); + const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase(); + + try { + const result = await pool.query( + `UPDATE price_alert_subscriptions SET active = false + WHERE id = $1 AND ($2 = '' OR email = $2) + RETURNING id`, + [parseInt(id), email] + ); + if (result.rowCount === 0) { + return res.status(404).json({ success: false, error: "Subscription not found" }); + } + return res.json({ success: true, cancelled: parseInt(id) }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/price-alerts/triggered — Recent triggered alerts ───────────────── +priceAlertsRouter.get("/triggered", async (_req: Request, res: Response) => { + try { + const result = await pool.query(` + SELECT pal.*, + t.standard_name, t.form_factor, + v.name AS vendor_name + FROM price_alert_log pal + LEFT JOIN transceivers t ON t.id = pal.transceiver_id + LEFT JOIN vendors v ON v.id = pal.vendor_id + ORDER BY pal.created_at DESC + LIMIT 100 + `); + return res.json({ success: true, alerts: result.rows }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── POST /api/price-alerts/check — Evaluate all active subscriptions ────────── +// Called by the scraper scheduler periodically. Finds triggered conditions, +// inserts into price_alert_log, and marks last_triggered on subscription. +priceAlertsRouter.post("/check", async (_req: Request, res: Response) => { + try { + // Find subscriptions where latest price crosses the threshold + const triggered = await pool.query(` + WITH latest_prices AS ( + SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id) + po.transceiver_id, po.source_vendor_id AS vendor_id, + po.price, po.currency, po.time + FROM price_observations po + WHERE po.price > 0 AND COALESCE(po.is_anomalous, false) = false + ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC + ), + matched AS ( + SELECT + pas.id AS subscription_id, + pas.email, pas.threshold_price, pas.currency, pas.direction, + lp.transceiver_id, lp.vendor_id, lp.price AS triggered_price + FROM price_alert_subscriptions pas + JOIN latest_prices lp ON ( + (pas.transceiver_id IS NULL OR lp.transceiver_id = pas.transceiver_id) + AND lp.currency = pas.currency + AND (pas.vendor_id IS NULL OR lp.vendor_id = pas.vendor_id) + ) + JOIN transceivers t ON t.id = lp.transceiver_id + WHERE pas.active = true + AND (pas.form_factor IS NULL OR t.form_factor = pas.form_factor) + AND (pas.speed_gbps IS NULL OR t.speed_gbps = pas.speed_gbps) + AND ( + (pas.direction = 'below' AND lp.price < pas.threshold_price) + OR + (pas.direction = 'above' AND lp.price > pas.threshold_price) + ) + -- Don't re-trigger more than once per 24h per subscription + AND (pas.last_triggered IS NULL OR pas.last_triggered < NOW() - INTERVAL '24 hours') + ) + SELECT * FROM matched + LIMIT 200 + `); + + let queued = 0; + for (const row of triggered.rows) { + await pool.query( + `INSERT INTO price_alert_log + (subscription_id, transceiver_id, vendor_id, triggered_price, threshold_price, currency, email, delivery_status) + VALUES ($1,$2,$3,$4,$5,$6,$7,'pending')`, + [row.subscription_id, row.transceiver_id, row.vendor_id, + row.triggered_price, row.threshold_price, row.currency, row.email] + ); + await pool.query( + `UPDATE price_alert_subscriptions + SET last_triggered = NOW(), trigger_count = trigger_count + 1 + WHERE id = $1`, + [row.subscription_id] + ); + queued++; + } + + return res.json({ success: true, checked: triggered.rowCount, queued }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); diff --git a/packages/api/src/routes/price-comparison.ts b/packages/api/src/routes/price-comparison.ts index 7c4ad08..57b9673 100644 --- a/packages/api/src/routes/price-comparison.ts +++ b/packages/api/src/routes/price-comparison.ts @@ -11,6 +11,7 @@ */ import { Router, Request, Response } from "express"; import { pool } from "../db/client"; +import { sendCSV } from "../utils/csv"; export const priceComparisonRouter = Router(); @@ -96,9 +97,10 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => { // ─── GET /api/price-comparison ─────────────────────────────────────────────── /** * Top 50 transceivers ranked by number of vendors tracking them. - * Shows price spread across vendors — the more vendors, the better the comparison. + * Add ?format=csv to download as CSV. */ -priceComparisonRouter.get("/", async (_req: Request, res: Response) => { +priceComparisonRouter.get("/", async (req: Request, res: Response) => { + const fmt = req.query.format as string | undefined; try { const result = await pool.query(` WITH latest AS ( @@ -138,6 +140,9 @@ priceComparisonRouter.get("/", async (_req: Request, res: Response) => { LIMIT 50 `); + if (fmt === "csv") { + return sendCSV(res, result.rows, `tip-price-comparison-${new Date().toISOString().slice(0,10)}.csv`); + } res.json({ success: true, data: result.rows, diff --git a/packages/api/src/routes/price-matrix.ts b/packages/api/src/routes/price-matrix.ts index ff23183..9cbb021 100644 --- a/packages/api/src/routes/price-matrix.ts +++ b/packages/api/src/routes/price-matrix.ts @@ -65,7 +65,7 @@ priceMatrixRouter.get("/", async (req: Request, res: Response) => { form_factor: string; speed_gbps: number; }>( - `SELECT id, model_name, part_number, form_factor, speed_gbps + `SELECT id, COALESCE(standard_name, part_number, '') AS model_name, part_number, form_factor, speed_gbps FROM transceivers WHERE id IN (${placeholders}) ORDER BY id`, diff --git a/packages/api/src/routes/procurement.ts b/packages/api/src/routes/procurement.ts index d3af6f3..3405b0d 100644 --- a/packages/api/src/routes/procurement.ts +++ b/packages/api/src/routes/procurement.ts @@ -14,6 +14,7 @@ */ import { Router, Request, Response } from "express"; import { pool } from "../db/client"; +import { sendCSV } from "../utils/csv"; export const procurementRouter = Router(); @@ -24,11 +25,13 @@ procurementRouter.get("/overview", async (_req: Request, res: Response) => { try { const [signals, abc, intel, lifecycle] = await Promise.all([ pool.query(` - SELECT signal, COUNT(*) AS count - FROM reorder_signals - WHERE expires_at > NOW() - AND computed_at = (SELECT MAX(r2.computed_at) FROM reorder_signals r2 WHERE r2.transceiver_id = reorder_signals.transceiver_id) - GROUP BY signal + WITH latest AS ( + SELECT DISTINCT ON (transceiver_id) signal + FROM reorder_signals + WHERE expires_at > NOW() + ORDER BY transceiver_id, computed_at DESC + ) + SELECT signal, COUNT(*) AS count FROM latest GROUP BY signal `), pool.query(` SELECT abc_class, COUNT(*) AS count FROM abc_classification GROUP BY abc_class ORDER BY abc_class @@ -72,31 +75,47 @@ procurementRouter.get("/signals", async (req: Request, res: Response) => { limit = "50", offset = "0" } = req.query; - let sql = ` + // Use DISTINCT ON with the existing idx_reorder_transceiver index instead of + // a correlated subquery that would run once per active row (108k+ scans). + const params: any[] = []; + let idx = 1; + + const signalFilter = signal ? ` AND rs.signal = $${idx++}` : ""; + if (signal) params.push(signal); + const abcFilter = abc_class ? ` AND ac.abc_class = $${idx++}` : ""; + if (abc_class) params.push(abc_class); + const ffFilter = form_factor ? ` AND t.form_factor = $${idx++}` : ""; + if (form_factor) params.push(form_factor); + const speedFilter = speed_gbps ? ` AND t.speed_gbps = $${idx++}` : ""; + if (speed_gbps) params.push(parseFloat(speed_gbps as string)); + + params.push(parseInt(limit as string), parseInt(offset as string)); + const limitIdx = idx; idx++; + const offsetIdx = idx; + + const sql = ` + WITH latest AS ( + SELECT DISTINCT ON (transceiver_id) + id, transceiver_id, signal, signal_strength, reasons, + stock_trend, price_trend, lead_time_weeks, hype_phase, + computed_at, expires_at, is_demo_data + FROM reorder_signals + WHERE expires_at > NOW() + ORDER BY transceiver_id, computed_at DESC + ) SELECT rs.*, t.part_number, t.standard_name, t.form_factor, t.speed_gbps, t.reach_label, t.image_url, t.image_r2_key, ac.abc_class, ac.demand_score, ac.supply_risk, v.name AS vendor_name - FROM reorder_signals rs + FROM latest rs JOIN transceivers t ON rs.transceiver_id = t.id LEFT JOIN abc_classification ac ON ac.transceiver_id = t.id LEFT JOIN vendors v ON t.vendor_id = v.id - WHERE rs.expires_at > NOW() - AND rs.computed_at = ( - SELECT MAX(r2.computed_at) FROM reorder_signals r2 WHERE r2.transceiver_id = rs.transceiver_id - ) + WHERE 1=1${signalFilter}${abcFilter}${ffFilter}${speedFilter} + ORDER BY rs.signal_strength DESC + LIMIT $${limitIdx} OFFSET $${offsetIdx} `; - const params: any[] = []; - let idx = 1; - - if (signal) { sql += ` AND rs.signal = $${idx}`; params.push(signal); idx++; } - if (abc_class) { sql += ` AND ac.abc_class = $${idx}`; params.push(abc_class); idx++; } - if (form_factor) { sql += ` AND t.form_factor = $${idx}`; params.push(form_factor); idx++; } - if (speed_gbps) { sql += ` AND t.speed_gbps = $${idx}`; params.push(parseFloat(speed_gbps as string)); idx++; } - - sql += ` ORDER BY rs.signal_strength DESC LIMIT $${idx} OFFSET $${idx + 1}`; - params.push(parseInt(limit as string), parseInt(offset as string)); const result = await pool.query(sql, params); res.json({ data: result.rows, total: result.rowCount }); @@ -195,6 +214,9 @@ procurementRouter.get("/abc", async (req: Request, res: Response) => { params.push(parseInt(limit as string), parseInt(offset as string)); const result = await pool.query(sql, params); + if ((req.query.format as string) === "csv") { + return sendCSV(res, result.rows, `tip-abc-classification-${new Date().toISOString().slice(0,10)}.csv`); + } res.json({ data: result.rows, total: result.rowCount }); } catch (err) { console.error("ABC error:", err); @@ -560,11 +582,12 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => { // Search for switches matching query, return their compatible transceivers const switches = await pool.query(` SELECT DISTINCT ON (sw.id) - sw.id, sw.vendor AS sw_vendor, sw.model AS sw_model, sw.series AS sw_series, + sw.id, v.name AS sw_vendor, sw.model AS sw_model, sw.series AS sw_series, COUNT(c.transceiver_id) OVER (PARTITION BY sw.id)::int AS compat_count FROM switches sw JOIN compatibility c ON c.switch_id = sw.id - WHERE sw.model ILIKE $1 OR sw.vendor ILIKE $1 OR sw.series ILIKE $1 + LEFT JOIN vendors v ON v.id = sw.vendor_id + WHERE sw.model ILIKE $1 OR COALESCE(v.name,'') ILIKE $1 OR sw.series ILIKE $1 ORDER BY sw.id, compat_count DESC LIMIT $2 `, [`%${search}%`, limitNum]); @@ -582,8 +605,7 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => { v.name AS vendor_name, c.verification_method, c.status, (SELECT ROUND(MIN(po.price)::numeric,2) FROM price_observations po - WHERE po.transceiver_id = t.id AND po.price > 0 - ORDER BY po.time DESC LIMIT 1) AS min_price, + WHERE po.transceiver_id = t.id AND po.price > 0) AS min_price, (SELECT po.currency FROM price_observations po WHERE po.transceiver_id = t.id AND po.price > 0 ORDER BY po.time DESC LIMIT 1) AS currency @@ -605,12 +627,13 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => { // No search — return top switches by compat count const top = await pool.query(` - SELECT sw.vendor, sw.model, sw.series, + SELECT v.name AS vendor, sw.model, sw.series, COUNT(c.transceiver_id)::int AS compat_count FROM switches sw JOIN compatibility c ON c.switch_id = sw.id + LEFT JOIN vendors v ON v.id = sw.vendor_id WHERE c.status = 'compatible' - GROUP BY sw.id, sw.vendor, sw.model, sw.series + GROUP BY sw.id, v.name, sw.model, sw.series ORDER BY compat_count DESC LIMIT $1 `, [limitNum]); @@ -700,7 +723,7 @@ procurementRouter.get("/dead-stock-revival", async (_req: Request, res: Response pool.query(` SELECT fid.transceiver_id, - fid.part_number_raw AS part_number, + fid.sku AS part_number, fid.velocity_class, fid.demand_12m, fid.demand_trend_pct, @@ -779,16 +802,16 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) => pool.query(` SELECT CASE - WHEN description ILIKE '%800G%' THEN 800 - WHEN description ILIKE '%400G%' THEN 400 - WHEN description ILIKE '%100G%' THEN 100 + WHEN COALESCE(network_speed, title, summary, '') ILIKE '%800G%' THEN 800 + WHEN COALESCE(network_speed, title, summary, '') ILIKE '%400G%' THEN 400 + WHEN COALESCE(network_speed, title, summary, '') ILIKE '%100G%' THEN 100 ELSE 0 END AS speed_tier, COALESCE(SUM(estimated_transceivers),0)::int AS total_tx, COUNT(*)::int AS cluster_count FROM ai_cluster_announcements WHERE announced_date >= NOW() - INTERVAL '90 days' - GROUP BY speed_tier + GROUP BY 1 HAVING COALESCE(SUM(estimated_transceivers),0) > 0 `), // Hype phase per technology @@ -873,6 +896,185 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) => } }); +// ───────────────────────────────────────────────────────────────────────────── +// GET /api/procurement/lead-times — Rolling lead-time trends per vendor/speed +// Query params: form_factor, speed_gbps, days (default 90), limit (default 20) +// ───────────────────────────────────────────────────────────────────────────── +procurementRouter.get("/lead-times", async (req: Request, res: Response) => { + const days = Math.min(parseInt(req.query.days as string) || 90, 365); + const limit = Math.min(parseInt(req.query.limit as string) || 20, 50); + const ff = req.query.form_factor as string | undefined; + const spd = req.query.speed_gbps as string | undefined; + + try { + const [weekly, summary, concentration] = await Promise.all([ + // Weekly avg lead time per vendor × speed tier + pool.query(` + SELECT + v.name AS vendor_name, + v.id::text AS vendor_id, + t.speed_gbps::text, + t.form_factor, + DATE_TRUNC('week', ss.scraped_at)::date AS week, + ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_lead_days, + ROUND(MIN(ss.lead_time_days)::numeric, 1) AS min_lead_days, + ROUND(MAX(ss.lead_time_days)::numeric, 1) AS max_lead_days, + COUNT(*)::int AS observations + FROM stock_snapshots ss + JOIN transceivers t ON t.id = ss.transceiver_id + JOIN vendors v ON v.id = ss.source_vendor_id + WHERE ss.scraped_at >= NOW() - INTERVAL '${days} days' + AND ss.lead_time_days IS NOT NULL + AND ss.lead_time_days > 0 + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + GROUP BY v.name, v.id, t.speed_gbps, t.form_factor, DATE_TRUNC('week', ss.scraped_at) + ORDER BY week DESC, avg_lead_days DESC + LIMIT 500 + `), + + // Overall summary: current vs prior-period avg per vendor + pool.query(` + WITH cur AS ( + SELECT + v.name AS vendor_name, v.id::text AS vendor_id, + ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_days, + COUNT(*)::int AS obs + FROM stock_snapshots ss + JOIN vendors v ON v.id = ss.source_vendor_id + JOIN transceivers t ON t.id = ss.transceiver_id + WHERE ss.scraped_at >= NOW() - INTERVAL '30 days' + AND ss.lead_time_days > 0 + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + GROUP BY v.name, v.id + ), + prior AS ( + SELECT v.id::text AS vendor_id, + ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_days + FROM stock_snapshots ss + JOIN vendors v ON v.id = ss.source_vendor_id + JOIN transceivers t ON t.id = ss.transceiver_id + WHERE ss.scraped_at >= NOW() - INTERVAL '60 days' + AND ss.scraped_at < NOW() - INTERVAL '30 days' + AND ss.lead_time_days > 0 + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + GROUP BY v.id + ) + SELECT + c.vendor_name, c.vendor_id, + c.avg_days AS current_30d_avg, + p.avg_days AS prior_30d_avg, + ROUND((c.avg_days - COALESCE(p.avg_days, c.avg_days))::numeric, 1) AS delta_days, + c.obs + FROM cur c + LEFT JOIN prior p ON p.vendor_id = c.vendor_id + ORDER BY c.avg_days DESC + LIMIT ${limit} + `), + + // Speed-tier breakdown — which form factors have longest lead times right now + pool.query(` + SELECT + t.speed_gbps::text, t.form_factor, + ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_lead_days, + COUNT(DISTINCT ss.source_vendor_id)::int AS vendors_reporting, + COUNT(*)::int AS total_obs + FROM stock_snapshots ss + JOIN transceivers t ON t.id = ss.transceiver_id + WHERE ss.scraped_at >= NOW() - INTERVAL '30 days' + AND ss.lead_time_days > 0 + GROUP BY t.speed_gbps, t.form_factor + HAVING COUNT(*) >= 3 + ORDER BY avg_lead_days DESC + LIMIT 20 + `), + ]); + + res.json({ + success: true, + filters: { days, form_factor: ff || null, speed_gbps: spd || null }, + weekly_trend: weekly.rows, + vendor_summary: summary.rows, + speed_tier_breakdown: concentration.rows, + }); + } catch (err) { + res.status(500).json({ error: String(err) }); + } +}); + +// ───────────────────────────────────────────────────────────────────────────── +// GET /api/procurement/supply-concentration — Single-vendor dependency risk +// Flags SKUs where >70% of price observations come from one vendor (30d) +// ───────────────────────────────────────────────────────────────────────────── +procurementRouter.get("/supply-concentration", async (_req: Request, res: Response) => { + try { + const result = await pool.query(` + WITH obs_30d AS ( + SELECT + po.transceiver_id, + po.source_vendor_id, + COUNT(*) AS vendor_obs + FROM price_observations po + WHERE po.time >= NOW() - INTERVAL '30 days' + AND po.price > 0 + AND COALESCE(po.is_anomalous, false) = false + GROUP BY po.transceiver_id, po.source_vendor_id + ), + totals AS ( + SELECT transceiver_id, SUM(vendor_obs) AS total_obs + FROM obs_30d GROUP BY transceiver_id + ), + ranked AS ( + SELECT + o.transceiver_id, + o.source_vendor_id, + o.vendor_obs, + t.total_obs, + ROUND((o.vendor_obs::numeric / NULLIF(t.total_obs,0)) * 100, 1) AS share_pct, + ROW_NUMBER() OVER (PARTITION BY o.transceiver_id ORDER BY o.vendor_obs DESC) AS rnk + FROM obs_30d o JOIN totals t ON t.transceiver_id = o.transceiver_id + ) + SELECT + tx.id::text, tx.part_number, tx.form_factor, + tx.speed_gbps::text, + tx.standard_name, + v.name AS dominant_vendor, + r.share_pct, + r.total_obs::int, + r.vendor_obs::int AS dominant_obs, + CASE + WHEN r.share_pct >= 90 THEN 'critical' + WHEN r.share_pct >= 75 THEN 'high' + ELSE 'medium' + END AS risk_level + FROM ranked r + JOIN transceivers tx ON tx.id = r.transceiver_id + JOIN vendors v ON v.id = r.source_vendor_id + WHERE r.rnk = 1 + AND r.share_pct >= 70 + AND r.total_obs >= 5 + ORDER BY r.share_pct DESC + LIMIT 50 + `); + + const rows = result.rows; + res.json({ + success: true, + concentrated: rows, + stats: { + total_at_risk: rows.length, + critical: rows.filter(r => r.risk_level === "critical").length, + high: rows.filter(r => r.risk_level === "high").length, + medium: rows.filter(r => r.risk_level === "medium").length, + }, + }); + } catch (err) { + res.status(500).json({ error: String(err) }); + } +}); + // GET /api/procurement/price-movers — SKUs with biggest price delta vs prior period procurementRouter.get("/price-movers", async (req: Request, res: Response) => { const days = Math.min(parseInt(req.query.days as string) || 7, 90); diff --git a/packages/api/src/routes/rfq.ts b/packages/api/src/routes/rfq.ts new file mode 100644 index 0000000..24c27c2 --- /dev/null +++ b/packages/api/src/routes/rfq.ts @@ -0,0 +1,215 @@ +/** + * RFQ Analyzer — POST /api/rfq/analyze + * + * Paste a vendor quote (list of part numbers + quantities + prices) and + * get back: current market rates, cheapest alternative via equivalences, + * total savings opportunity, and per-line delta. + * + * Request body: + * { + * lines: [ + * { part_number: string, quantity: number, unit_price: number, currency?: string } + * ], + * currency?: "USD" | "EUR" + * } + */ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; +import { sendCSV } from "../utils/csv"; + +export const rfqRouter = Router(); + +interface RfqLine { + part_number: string; + quantity: number; + unit_price: number; + currency?: string; +} + +// POST /api/rfq/analyze +rfqRouter.post("/analyze", async (req: Request, res: Response) => { + const { lines, currency: preferredCurrency = "USD" } = req.body as { + lines: RfqLine[]; + currency?: string; + }; + + if (!Array.isArray(lines) || lines.length === 0) { + return res.status(400).json({ success: false, error: "lines[] required" }); + } + if (lines.length > 200) { + return res.status(400).json({ success: false, error: "Max 200 lines per RFQ" }); + } + + try { + const results = await Promise.all( + lines.map(async (line) => { + const pn = String(line.part_number || "").trim(); + const qty = Math.max(1, Number(line.quantity) || 1); + const quotedPrice = Number(line.unit_price) || 0; + + if (!pn) { + return { part_number: pn, error: "Empty part_number", resolved: false }; + } + + // 1. Resolve transceiver by part_number or standard_name + const txResult = await pool.query( + `SELECT id, part_number, standard_name, form_factor, speed_gbps, speed + FROM transceivers + WHERE part_number ILIKE $1 OR standard_name ILIKE $1 + LIMIT 1`, + [pn] + ); + + if (txResult.rows.length === 0) { + return { part_number: pn, quantity: qty, quoted_unit_price: quotedPrice, resolved: false, error: "Not found in catalog" }; + } + + const tx = txResult.rows[0]; + + // 2. Get current market prices for this transceiver + const marketResult = await pool.query( + `SELECT + v.name AS vendor_name, v.website, + po.price, po.currency, po.stock_level, po.url, po.time AS observed_at + FROM ( + SELECT DISTINCT ON (source_vendor_id) + source_vendor_id, price, currency, stock_level, url, time + FROM price_observations + WHERE transceiver_id = $1 + AND price > 0 + AND COALESCE(is_anomalous, false) = false + ORDER BY source_vendor_id, time DESC + ) po + JOIN vendors v ON v.id = po.source_vendor_id + WHERE po.currency = $2 OR po.currency IS NULL + ORDER BY po.price ASC + LIMIT 10`, + [tx.id, preferredCurrency] + ); + + const prices = marketResult.rows.map(r => parseFloat(r.price)).filter(p => p > 0); + const marketMin = prices.length ? Math.min(...prices) : null; + const marketAvg = prices.length ? Math.round(prices.reduce((a, b) => a + b) / prices.length * 100) / 100 : null; + const cheapestVendor = marketResult.rows[0] || null; + + // 3. Find cheapest equivalent via transceiver_equivalences + const equivResult = await pool.query( + `SELECT + te.competitor_id, t2.part_number AS equiv_part_number, + t2.standard_name AS equiv_standard_name, + te.confidence, te.match_basis, + ( + SELECT po2.price FROM price_observations po2 + WHERE po2.transceiver_id = te.competitor_id + AND po2.currency = $2 + AND po2.price > 0 + AND COALESCE(po2.is_anomalous, false) = false + ORDER BY po2.time DESC LIMIT 1 + ) AS equiv_price, + ( + SELECT v2.name FROM price_observations po2 + JOIN vendors v2 ON v2.id = po2.source_vendor_id + WHERE po2.transceiver_id = te.competitor_id + AND po2.currency = $2 AND po2.price > 0 + ORDER BY po2.price ASC, po2.time DESC LIMIT 1 + ) AS equiv_cheapest_vendor + FROM transceiver_equivalences te + JOIN transceivers t2 ON t2.id = te.competitor_id + WHERE (te.flexoptix_id = $1 OR te.competitor_id = $1) + AND te.status = 'approved' + AND te.confidence >= 0.7 + ORDER BY te.confidence DESC + LIMIT 5`, + [tx.id, preferredCurrency] + ); + + const equivalents = equivResult.rows + .filter(e => e.equiv_price !== null) + .map(e => ({ + part_number: e.equiv_part_number, + standard_name: e.equiv_standard_name, + confidence: parseFloat(e.confidence), + match_basis: e.match_basis, + unit_price: parseFloat(e.equiv_price), + vendor: e.equiv_cheapest_vendor, + })); + + const cheapestEquiv = equivalents.sort((a, b) => a.unit_price - b.unit_price)[0] || null; + + // 4. Calculate savings + const savingsVsMarketMin = marketMin !== null && quotedPrice > 0 + ? Math.round((quotedPrice - marketMin) * qty * 100) / 100 + : null; + const savingsVsEquiv = cheapestEquiv && quotedPrice > 0 + ? Math.round((quotedPrice - cheapestEquiv.unit_price) * qty * 100) / 100 + : null; + + return { + part_number: pn, + resolved: true, + transceiver: { + id: tx.id, + standard_name: tx.standard_name, + form_factor: tx.form_factor, + speed: tx.speed, + }, + quantity: qty, + quoted_unit_price: quotedPrice, + quoted_total: Math.round(quotedPrice * qty * 100) / 100, + market: { + min_price: marketMin, + avg_price: marketAvg, + vendor_count: prices.length, + cheapest_vendor: cheapestVendor ? { + name: cheapestVendor.vendor_name, + price: parseFloat(cheapestVendor.price), + stock_level: cheapestVendor.stock_level, + url: cheapestVendor.url, + } : null, + }, + equivalents, + cheapest_equivalent: cheapestEquiv, + savings: { + vs_market_min: savingsVsMarketMin, + vs_equiv: savingsVsEquiv, + best_saving: Math.max(savingsVsMarketMin || 0, savingsVsEquiv || 0) || null, + }, + currency: preferredCurrency, + }; + }) + ); + + // Aggregate totals + const resolved = results.filter(r => r.resolved); + const totalQuoted = resolved.reduce((s, r) => s + (r.quoted_total as number || 0), 0); + const totalSavingsMarket = resolved.reduce((s, r) => { + const sv = (r.savings as any)?.vs_market_min; + return s + (sv && sv > 0 ? sv : 0); + }, 0); + const totalSavingsEquiv = resolved.reduce((s, r) => { + const sv = (r.savings as any)?.vs_equiv; + return s + (sv && sv > 0 ? sv : 0); + }, 0); + + res.json({ + success: true, + currency: preferredCurrency, + line_count: lines.length, + resolved_count: resolved.length, + lines: results, + totals: { + quoted: Math.round(totalQuoted * 100) / 100, + potential_savings_vs_market: Math.round(totalSavingsMarket * 100) / 100, + potential_savings_vs_equiv: Math.round(totalSavingsEquiv * 100) / 100, + best_total_saving: Math.round(Math.max(totalSavingsMarket, totalSavingsEquiv) * 100) / 100, + }, + }); + } catch (err) { + res.status(500).json({ success: false, error: String(err) }); + } +}); + +// GET /api/rfq/export — Export last RFQ result as CSV (pass lines as query params) +rfqRouter.get("/export", async (req: Request, res: Response) => { + res.status(405).json({ error: "Use POST /api/rfq/analyze with ?format=csv to export" }); +}); diff --git a/packages/api/src/routes/roi.ts b/packages/api/src/routes/roi.ts new file mode 100644 index 0000000..2046d75 --- /dev/null +++ b/packages/api/src/routes/roi.ts @@ -0,0 +1,178 @@ +/** + * ROI Calculator — /api/roi + * + * Calculates total cost of ownership and switching savings for transceiver decisions. + * + * POST /api/roi/calculate + * Input: + * { + * ports: number, — Number of ports to equip + * current_price_per_port: number, + * target_form_factor?: string, + * target_speed_gbps?: number, + * years?: number, — TCO horizon (default 3) + * switch_cost?: number, — One-time switching cost (labor, downtime est.) + * currency?: string + * } + * Output: + * - Current total cost (ports × price) + * - Market min/avg for target spec + * - 1/2/3 year TCO at current vs market-min price + * - Savings over TCO horizon + * - Break-even months + * - Top 5 cheapest vendors for the target spec + */ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; + +export const roiRouter = Router(); + +roiRouter.post("/calculate", async (req: Request, res: Response) => { + const { + ports, + current_price_per_port, + target_form_factor, + target_speed_gbps, + years = 3, + switch_cost = 0, + currency = "USD", + } = req.body as Record; + + if (!ports || isNaN(parseInt(ports)) || parseInt(ports) <= 0) { + return res.status(400).json({ success: false, error: "ports must be a positive integer" }); + } + if (!current_price_per_port || isNaN(parseFloat(current_price_per_port))) { + return res.status(400).json({ success: false, error: "current_price_per_port required" }); + } + if (!target_form_factor && !target_speed_gbps) { + return res.status(400).json({ success: false, error: "Provide target_form_factor and/or target_speed_gbps" }); + } + + const portCount = parseInt(ports); + const currentPrice = parseFloat(current_price_per_port); + const switchingCost = parseFloat(switch_cost) || 0; + const tcoYears = Math.min(Math.max(parseInt(years) || 3, 1), 10); + const curr = String(currency).toUpperCase(); + + try { + // Find market prices for target spec + const marketResult = await pool.query(` + WITH latest AS ( + SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id) + po.transceiver_id, po.source_vendor_id, po.price, po.currency, po.stock_level, po.url, po.time + FROM price_observations po + WHERE po.price > 0 + AND COALESCE(po.is_anomalous, false) = false + AND po.currency = $1 + ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC + ) + SELECT + v.name AS vendor_name, + v.website, + t.standard_name, + t.form_factor, + t.speed_gbps::text, + ROUND(MIN(l.price)::numeric, 2) AS min_price, + ROUND(AVG(l.price)::numeric, 2) AS avg_price, + COUNT(DISTINCT l.source_vendor_id)::int AS vendor_count, + l.price AS vendor_price, + l.stock_level, + l.url + FROM latest l + JOIN transceivers t ON t.id = l.transceiver_id + JOIN vendors v ON v.id = l.source_vendor_id + WHERE ($2::text IS NULL OR t.form_factor = $2) + AND ($3::numeric IS NULL OR t.speed_gbps = $3) + GROUP BY v.name, v.website, t.standard_name, t.form_factor, t.speed_gbps, + l.price, l.stock_level, l.url + ORDER BY l.price ASC + LIMIT 50 + `, [curr, target_form_factor || null, target_speed_gbps ? parseFloat(target_speed_gbps) : null]); + + const allPrices = marketResult.rows.map(r => parseFloat(r.vendor_price)).filter(p => p > 0); + const marketMin = allPrices.length ? Math.min(...allPrices) : null; + const marketAvg = allPrices.length ? Math.round(allPrices.reduce((a,b) => a+b) / allPrices.length * 100) / 100 : null; + + // Top 5 cheapest vendors (distinct vendors) + const vendorPrices = new Map(); + for (const r of marketResult.rows) { + if (!vendorPrices.has(r.vendor_name)) { + vendorPrices.set(r.vendor_name, { + vendor_name: r.vendor_name, + price: parseFloat(r.vendor_price), + stock_level: r.stock_level, + url: r.url, + standard_name: r.standard_name, + }); + } + } + const top5Vendors = [...vendorPrices.values()].slice(0, 5); + + // TCO calculations + const currentTotal = Math.round(portCount * currentPrice * 100) / 100; + const marketMinTotal = marketMin !== null ? Math.round(portCount * marketMin * 100) / 100 : null; + const marketAvgTotal = marketAvg !== null ? Math.round(portCount * marketAvg * 100) / 100 : null; + + // Annual OpEx: assume 15% of hardware cost for maintenance/replacement (industry standard) + const annualOpExFactor = 0.15; + const currentTCO = Math.round((currentTotal + (currentTotal * annualOpExFactor * tcoYears)) * 100) / 100; + const targetTCOMin = marketMinTotal !== null + ? Math.round((marketMinTotal + switchingCost + (marketMinTotal * annualOpExFactor * tcoYears)) * 100) / 100 + : null; + const targetTCOAvg = marketAvgTotal !== null + ? Math.round((marketAvgTotal + switchingCost + (marketAvgTotal * annualOpExFactor * tcoYears)) * 100) / 100 + : null; + + const savingsMin = targetTCOMin !== null ? Math.round((currentTCO - targetTCOMin) * 100) / 100 : null; + const savingsAvg = targetTCOAvg !== null ? Math.round((currentTCO - targetTCOAvg) * 100) / 100 : null; + + // Break-even: months until switching cost is recovered from per-unit savings + const monthlyHardwareSavingsMin = marketMin !== null + ? Math.round(portCount * (currentPrice - marketMin) / 12 * 100) / 100 + : null; + const breakEvenMonths = monthlyHardwareSavingsMin && monthlyHardwareSavingsMin > 0 && switchingCost > 0 + ? Math.ceil(switchingCost / monthlyHardwareSavingsMin) + : 0; + + return res.json({ + success: true, + input: { + ports: portCount, current_price_per_port: currentPrice, + target_form_factor: target_form_factor || null, + target_speed_gbps: target_speed_gbps ? parseFloat(target_speed_gbps) : null, + tco_years: tcoYears, switch_cost: switchingCost, currency: curr, + }, + current: { + total_hardware: currentTotal, + tco_estimate: currentTCO, + price_per_port: currentPrice, + }, + market: { + min_price: marketMin, + avg_price: marketAvg, + vendor_count: allPrices.length, + min_total_hardware: marketMinTotal, + avg_total_hardware: marketAvgTotal, + }, + tco_comparison: { + current: currentTCO, + target_min: targetTCOMin, + target_avg: targetTCOAvg, + savings_vs_min: savingsMin, + savings_vs_avg: savingsAvg, + savings_pct_min: savingsMin && currentTCO > 0 + ? Math.round(savingsMin / currentTCO * 1000) / 10 + : null, + }, + switching: { + cost: switchingCost, + break_even_months: breakEvenMonths || null, + monthly_savings: monthlyHardwareSavingsMin, + }, + top_vendors: top5Vendors, + currency: curr, + }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); diff --git a/packages/api/src/routes/stock.ts b/packages/api/src/routes/stock.ts index 9753bb5..00dde82 100644 --- a/packages/api/src/routes/stock.ts +++ b/packages/api/src/routes/stock.ts @@ -461,7 +461,7 @@ stockRouter.get("/velocity/:id", async (req: Request, res: Response) => { const [transceiver, velocity, events] = await Promise.all([ pool.query( `SELECT t.*, v.name AS brand_name - FROM transceivers t LEFT JOIN vendors v ON v.id = t.vendor_id + FROM transceivers t LEFT JOIN vendors v ON v.id = t.brand_vendor_id WHERE t.id = $1`, [transceiverUuid] ), @@ -562,7 +562,7 @@ stockRouter.get("/:id", async (req: Request, res: Response) => { const [transceiver, observations] = await Promise.all([ pool.query( `SELECT t.*, v.name AS brand_name - FROM transceivers t LEFT JOIN vendors v ON v.id = t.vendor_id + FROM transceivers t LEFT JOIN vendors v ON v.id = t.brand_vendor_id WHERE t.id = $1`, [transceiverUuid] ), diff --git a/packages/api/src/routes/vendor-reliability.ts b/packages/api/src/routes/vendor-reliability.ts index 889ade0..981e335 100644 --- a/packages/api/src/routes/vendor-reliability.ts +++ b/packages/api/src/routes/vendor-reliability.ts @@ -1,83 +1,192 @@ /** - * Vendor Reliability Scores + * Vendor Reliability Scores — Redesigned (v2) + * + * Scoring methodology (100 pts total): + * 30 pts — Data Freshness: How recently were prices scraped? + * 25 pts — SKU Coverage: How many unique transceivers covered in 60d? + * 25 pts — Price Consistency: Price anomaly rate (low anomalies = reliable) + * 20 pts — Stock Accuracy: Does vendor report stock status? How often in_stock? * * Routes: - * GET /api/vendor-reliability — Reliability score 0–100 per vendor + * GET /api/vendor-reliability — All vendor scores + * GET /api/vendor-reliability/:id — Single vendor detail + breakdown */ import { Router, Request, Response } from "express"; import { pool } from "../db/client"; export const vendorReliabilityRouter = Router(); -// ─── GET /api/vendor-reliability ───────────────────────────────────────────── +async function computeReliability() { + const result = await pool.query(` + WITH + -- Freshness + volume (30 pts) + freshness AS ( + SELECT + po.source_vendor_id AS vendor_id, + MAX(po.time) AS last_seen, + EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 86400.0 AS days_since_last, + COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d, + COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '7 days') AS obs_7d + FROM price_observations po + WHERE po.time >= NOW() - INTERVAL '90 days' + GROUP BY po.source_vendor_id + ), + -- SKU coverage breadth (25 pts) + coverage AS ( + SELECT + po.source_vendor_id AS vendor_id, + COUNT(DISTINCT po.transceiver_id) AS skus_60d, + COUNT(DISTINCT po.transceiver_id) + FILTER (WHERE po.time >= NOW() - INTERVAL '7 days') AS skus_7d + FROM price_observations po + WHERE po.time >= NOW() - INTERVAL '60 days' + AND po.price > 0 + GROUP BY po.source_vendor_id + ), + -- Price consistency: anomaly rate (25 pts) + consistency AS ( + SELECT + po.source_vendor_id AS vendor_id, + COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS total_30d, + COUNT(*) FILTER ( + WHERE po.time >= NOW() - INTERVAL '30 days' + AND COALESCE(po.is_anomalous, false) = true + ) AS anomalies_30d, + -- Price variance: std dev / mean (coefficient of variation, lower = more stable) + CASE WHEN AVG(po.price) > 0 THEN + ROUND((STDDEV(po.price) / AVG(po.price) * 100)::numeric, 1) + END AS price_cv_pct + FROM price_observations po + WHERE po.time >= NOW() - INTERVAL '30 days' + AND po.price > 0 + GROUP BY po.source_vendor_id + ), + -- Stock accuracy: does vendor report stock? reliability of in_stock flag (20 pts) + stock_acc AS ( + SELECT + so.source_vendor_id AS vendor_id, + COUNT(*)::int AS stock_obs, + COUNT(*) FILTER (WHERE so.in_stock = true)::int AS in_stock_count, + COUNT(*) FILTER (WHERE so.in_stock = false)::int AS out_of_stock_count + FROM stock_observations so + WHERE so.time >= NOW() - INTERVAL '30 days' + GROUP BY so.source_vendor_id + ) + SELECT + v.id::text AS vendor_id, + v.name AS vendor_name, + v.type, + v.website, + -- Raw metrics + f.last_seen, + f.days_since_last, + f.obs_30d, + f.obs_7d, + c.skus_60d, + c.skus_7d, + co.total_30d, + co.anomalies_30d, + co.price_cv_pct, + s.stock_obs, + s.in_stock_count, + s.out_of_stock_count + FROM freshness f + JOIN vendors v ON v.id = f.vendor_id + LEFT JOIN coverage c ON c.vendor_id = f.vendor_id + LEFT JOIN consistency co ON co.vendor_id = f.vendor_id + LEFT JOIN stock_acc s ON s.vendor_id = f.vendor_id + ORDER BY f.last_seen DESC + `); + + return result.rows.map(row => { + // ── Freshness score (30 pts) ────────────────────────────────────────── + const days = parseFloat(row.days_since_last || "999"); + const freshnessScore = + days <= 1 ? 30 : + days <= 3 ? 27 : + days <= 7 ? 22 : + days <= 14 ? 15 : + days <= 30 ? 8 : 0; + + // ── Coverage score (25 pts) ─────────────────────────────────────────── + const skus60d = parseInt(row.skus_60d || "0"); + const MAX_SKUS = 1000; + const coverageScore = Math.min(Math.round((skus60d / MAX_SKUS) * 25), 25); + + // ── Consistency score (25 pts) — low anomaly rate = good ───────────── + const total30d = parseInt(row.total_30d || "0"); + const anomalies30d = parseInt(row.anomalies_30d || "0"); + const anomalyRate = total30d > 0 ? anomalies30d / total30d : 0; + const consistencyScore = + total30d === 0 ? 0 : + anomalyRate <= 0.01 ? 25 : + anomalyRate <= 0.03 ? 20 : + anomalyRate <= 0.05 ? 15 : + anomalyRate <= 0.10 ? 10 : 5; + + // ── Stock accuracy score (20 pts) ───────────────────────────────────── + const stockObs = parseInt(row.stock_obs || "0"); + const stockScore = Math.min(Math.round((stockObs / 100) * 20), 20); + + const totalScore = freshnessScore + coverageScore + consistencyScore + stockScore; + const grade = + totalScore >= 85 ? "A" : + totalScore >= 70 ? "B" : + totalScore >= 50 ? "C" : + totalScore >= 30 ? "D" : "F"; + + return { + vendor_id: row.vendor_id, + vendor_name: row.vendor_name, + type: row.type, + website: row.website, + reliability_score: totalScore, + grade, + breakdown: { + freshness: { score: freshnessScore, max: 30, days_since_last: Math.round(days * 10) / 10 }, + coverage: { score: coverageScore, max: 25, skus_60d: skus60d, skus_7d: parseInt(row.skus_7d || "0") }, + consistency: { score: consistencyScore, max: 25, anomaly_rate_pct: Math.round(anomalyRate * 1000) / 10, obs_30d: total30d }, + stock_accuracy: { score: stockScore, max: 20, stock_obs_30d: stockObs, in_stock: parseInt(row.in_stock_count || "0") }, + }, + last_seen: row.last_seen, + obs_30d: parseInt(row.obs_30d || "0"), + }; + }); +} + +// ── GET /api/vendor-reliability ──────────────────────────────────────────── vendorReliabilityRouter.get("/", async (_req: Request, res: Response) => { try { - const result = await pool.query<{ - vendor_id: number; - vendor_name: string; - last_observation: Date; - obs_30d: string; - distinct_skus_60d: string; - days_since_last: string; - }>(` - WITH base AS ( - SELECT - po.source_vendor_id AS vendor_id, - MAX(po.time) AS last_observation, - COUNT(*) FILTER (WHERE po.time > NOW() - INTERVAL '30 days') - AS obs_30d, - COUNT(DISTINCT po.transceiver_id) - FILTER (WHERE po.time > NOW() - INTERVAL '60 days') - AS distinct_skus_60d - FROM price_observations po - WHERE po.time > NOW() - INTERVAL '90 days' - GROUP BY po.source_vendor_id - ) - SELECT - b.vendor_id, - v.name AS vendor_name, - b.last_observation, - b.obs_30d, - b.distinct_skus_60d, - EXTRACT(EPOCH FROM (NOW() - b.last_observation)) / 86400.0 AS days_since_last - FROM base b - JOIN vendors v ON v.id = b.vendor_id - ORDER BY b.last_observation DESC - `); - - const vendors = result.rows.map((row) => { - const days = parseFloat(row.days_since_last); - const obs30d = parseInt(row.obs_30d, 10); - const skus60d = parseInt(row.distinct_skus_60d, 10); - - const freshnessScore = - days <= 7 ? 40 : - days <= 14 ? 30 : - days <= 30 ? 20 : - days <= 60 ? 10 : 0; - - const frequencyScore = Math.min(Math.round((obs30d / 100) * 30), 30); - const coverageScore = Math.min(Math.round((skus60d / 500) * 30), 30); - const reliabilityScore = freshnessScore + frequencyScore + coverageScore; - - return { - vendor_id: row.vendor_id, - vendor_name: row.vendor_name, - reliability_score: reliabilityScore, - freshness_score: freshnessScore, - frequency_score: frequencyScore, - coverage_score: coverageScore, - last_observation: row.last_observation.toISOString().slice(0, 10), - obs_30d: obs30d, - distinct_skus_60d: skus60d, - }; - }); - + const vendors = await computeReliability(); vendors.sort((a, b) => b.reliability_score - a.reliability_score); - - res.json({ success: true, vendors }); + res.json({ success: true, vendors, scored_at: new Date().toISOString() }); + } catch (err) { + res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/vendor-reliability/:id — Single vendor deep-dive ────────────── +vendorReliabilityRouter.get("/:id", async (req: Request, res: Response) => { + try { + const all = await computeReliability(); + const vendor = all.find(v => v.vendor_id === req.params.id); + if (!vendor) return res.status(404).json({ success: false, error: "Vendor not found" }); + + // Price history for sparkline + const priceHistory = await pool.query(` + SELECT DATE_TRUNC('week', time)::date AS week, + ROUND(AVG(price)::numeric, 2) AS avg_price, + COUNT(*)::int AS observations + FROM price_observations + WHERE source_vendor_id = $1::uuid + AND time >= NOW() - INTERVAL '90 days' + AND price > 0 + GROUP BY DATE_TRUNC('week', time) + ORDER BY week ASC + `, [req.params.id]).catch(() => ({ rows: [] })); + + res.json({ success: true, vendor, price_history: priceHistory.rows }); } catch (err) { - console.error("GET /api/vendor-reliability error:", err); res.status(500).json({ success: false, error: String(err) }); } }); diff --git a/packages/api/src/routes/vendors.ts b/packages/api/src/routes/vendors.ts index 13c937a..0a9ead1 100644 --- a/packages/api/src/routes/vendors.ts +++ b/packages/api/src/routes/vendors.ts @@ -137,6 +137,132 @@ vendorRouter.get("/:id", async (req: Request, res: Response) => { } }); +// ───────────────────────────────────────────────────────────────────────────── +// GET /api/vendors/market-share — Weekly SKU-coverage share per vendor over time +// Shows which vendors are gaining/losing market presence +// Query params: speed_gbps, form_factor, days (default 90) +// ───────────────────────────────────────────────────────────────────────────── +vendorRouter.get("/market-share", async (req: Request, res: Response) => { + const days = Math.min(parseInt(req.query.days as string) || 90, 365); + const spd = req.query.speed_gbps as string | undefined; + const ff = req.query.form_factor as string | undefined; + + try { + const [weekly, current, momentum] = await Promise.all([ + // Weekly SKU count per vendor — shows growth/shrink trends + pool.query(` + SELECT + DATE_TRUNC('week', po.time)::date AS week, + v.id::text AS vendor_id, + v.name AS vendor_name, + COUNT(DISTINCT po.transceiver_id)::int AS sku_count + FROM price_observations po + JOIN vendors v ON v.id = po.source_vendor_id + JOIN transceivers t ON t.id = po.transceiver_id + WHERE po.time >= NOW() - INTERVAL '${days} days' + AND po.price > 0 + AND COALESCE(po.is_anomalous, false) = false + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + GROUP BY DATE_TRUNC('week', po.time), v.id, v.name + ORDER BY week ASC, sku_count DESC + `), + + // Current snapshot: SKU share % per vendor (last 30d) + pool.query(` + WITH totals AS ( + SELECT COUNT(DISTINCT transceiver_id)::float AS total + FROM price_observations + WHERE time >= NOW() - INTERVAL '30 days' + AND price > 0 AND COALESCE(is_anomalous, false) = false + ) + SELECT + v.id::text AS vendor_id, + v.name AS vendor_name, + v.type, + COUNT(DISTINCT po.transceiver_id)::int AS sku_count, + ROUND((COUNT(DISTINCT po.transceiver_id)::numeric / NULLIF(t.total,0)) * 100, 1) AS market_share_pct, + COUNT(po.id)::int AS total_obs, + MAX(po.time) AS last_seen + FROM price_observations po + JOIN vendors v ON v.id = po.source_vendor_id + JOIN transceivers tx ON tx.id = po.transceiver_id + CROSS JOIN totals t + WHERE po.time >= NOW() - INTERVAL '30 days' + AND po.price > 0 + AND COALESCE(po.is_anomalous, false) = false + ${spd ? `AND tx.speed_gbps = ${parseFloat(spd)}` : ""} + ${ff ? `AND tx.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + GROUP BY v.id, v.name, v.type, t.total + ORDER BY sku_count DESC + LIMIT 30 + `), + + // Momentum: compare last 30d vs prior 30d SKU count per vendor + pool.query(` + WITH cur AS ( + SELECT source_vendor_id, COUNT(DISTINCT transceiver_id)::int AS sku_count + FROM price_observations po + JOIN transceivers t ON t.id = po.transceiver_id + WHERE po.time >= NOW() - INTERVAL '30 days' + AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + GROUP BY source_vendor_id + ), + prior AS ( + SELECT source_vendor_id, COUNT(DISTINCT transceiver_id)::int AS sku_count + FROM price_observations po + JOIN transceivers t ON t.id = po.transceiver_id + WHERE po.time >= NOW() - INTERVAL '60 days' + AND po.time < NOW() - INTERVAL '30 days' + AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false + ${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""} + ${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""} + GROUP BY source_vendor_id + ) + SELECT + v.name AS vendor_name, v.id::text AS vendor_id, + c.sku_count AS current_skus, + COALESCE(p.sku_count, 0) AS prior_skus, + (c.sku_count - COALESCE(p.sku_count, 0)) AS delta_skus, + CASE + WHEN COALESCE(p.sku_count, 0) = 0 THEN NULL + ELSE ROUND(((c.sku_count - p.sku_count)::numeric / p.sku_count) * 100, 1) + END AS delta_pct + FROM cur c + JOIN vendors v ON v.id = c.source_vendor_id + LEFT JOIN prior p ON p.source_vendor_id = c.source_vendor_id + ORDER BY delta_skus DESC + LIMIT 20 + `), + ]); + + // Compute share % per week for chart (normalize across vendors per week) + const weekTotals = new Map(); + for (const row of weekly.rows) { + const k = row.week; + weekTotals.set(k, (weekTotals.get(k) || 0) + row.sku_count); + } + const weeklyWithShare = weekly.rows.map(r => ({ + ...r, + share_pct: weekTotals.get(r.week) + ? Math.round((r.sku_count / weekTotals.get(r.week)!) * 1000) / 10 + : 0, + })); + + res.json({ + success: true, + filters: { days, speed_gbps: spd || null, form_factor: ff || null }, + weekly_trend: weeklyWithShare, + current_share: current.rows, + momentum: momentum.rows, + }); + } catch (err) { + res.status(500).json({ success: false, error: String(err) }); + } +}); + // GET /api/vendors/intelligence — per-vendor price + SKU market stats (last 30d) vendorRouter.get("/intelligence", async (_req: Request, res: Response) => { try { diff --git a/packages/api/src/routes/win-loss.ts b/packages/api/src/routes/win-loss.ts new file mode 100644 index 0000000..77dfecd --- /dev/null +++ b/packages/api/src/routes/win-loss.ts @@ -0,0 +1,189 @@ +/** + * Win/Loss Intelligence — /api/win-loss + * + * Record and analyze deal outcomes: who won, who lost, at what price, in which segment. + * + * Routes: + * POST /api/win-loss — Record a win/loss event + * GET /api/win-loss — List events (filterable) + * GET /api/win-loss/summary — Aggregate win rate, avg price delta, segments + * GET /api/win-loss/competitors — Ranking by competitor vendor (loss analysis) + */ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; +import { sendCSV } from "../utils/csv"; + +export const winLossRouter = Router(); + +// ── POST /api/win-loss — Record a deal outcome ────────────────────────────── +winLossRouter.post("/", async (req: Request, res: Response) => { + const { + outcome, transceiver_id, competitor_vendor, + our_price, competitor_price, currency = "USD", + quantity, customer_segment, deal_source, + form_factor, speed_gbps, notes, deal_date, + } = req.body as Record; + + if (!outcome || !["won","lost","unknown"].includes(outcome)) { + return res.status(400).json({ success: false, error: "outcome must be: won | lost | unknown" }); + } + + try { + const result = await pool.query( + `INSERT INTO win_loss_events + (outcome, transceiver_id, competitor_vendor, our_price, competitor_price, + currency, quantity, customer_segment, deal_source, form_factor, speed_gbps, notes, deal_date) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, + COALESCE($13::date, CURRENT_DATE)) + RETURNING *`, + [ + outcome, + transceiver_id || null, + competitor_vendor || null, + our_price ? parseFloat(our_price) : null, + competitor_price ? parseFloat(competitor_price) : null, + currency, + quantity ? parseInt(quantity) : null, + customer_segment || null, + deal_source || null, + form_factor || null, + speed_gbps ? parseFloat(speed_gbps) : null, + notes || null, + deal_date || null, + ] + ); + return res.status(201).json({ success: true, event: result.rows[0] }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/win-loss — List events ───────────────────────────────────────── +winLossRouter.get("/", async (req: Request, res: Response) => { + const outcome = req.query.outcome as string | undefined; + const segment = req.query.customer_segment as string | undefined; + const days = Math.min(parseInt(req.query.days as string) || 90, 730); + const limit = Math.min(parseInt(req.query.limit as string) || 50, 200); + const fmt = req.query.format as string | undefined; + + try { + const result = await pool.query(` + SELECT wl.*, + t.standard_name, t.form_factor AS tx_form_factor, t.speed_gbps AS tx_speed + FROM win_loss_events wl + LEFT JOIN transceivers t ON t.id = wl.transceiver_id + WHERE wl.deal_date >= CURRENT_DATE - INTERVAL '${days} days' + ${outcome ? `AND wl.outcome = '${outcome.replace(/'/g,"''")}'` : ""} + ${segment ? `AND wl.customer_segment = '${segment.replace(/'/g,"''")}'` : ""} + ORDER BY wl.deal_date DESC + LIMIT ${limit} + `); + + if (fmt === "csv") { + return sendCSV(res, result.rows, `tip-win-loss-${new Date().toISOString().slice(0,10)}.csv`); + } + return res.json({ success: true, events: result.rows, count: result.rows.length }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/win-loss/summary — Aggregate analytics ───────────────────────── +winLossRouter.get("/summary", async (req: Request, res: Response) => { + const days = Math.min(parseInt(req.query.days as string) || 90, 730); + + try { + const [overall, bySegment, byFormFactor, priceDeltas] = await Promise.all([ + pool.query(` + SELECT + COUNT(*) AS total_events, + COUNT(*) FILTER (WHERE outcome = 'won') AS won, + COUNT(*) FILTER (WHERE outcome = 'lost') AS lost, + ROUND( + COUNT(*) FILTER (WHERE outcome = 'won')::numeric + / NULLIF(COUNT(*) FILTER (WHERE outcome IN ('won','lost')), 0) * 100, 1 + ) AS win_rate_pct, + ROUND(AVG(our_price) FILTER (WHERE outcome = 'won')::numeric, 2) AS avg_win_price, + ROUND(AVG(our_price) FILTER (WHERE outcome = 'lost')::numeric, 2) AS avg_loss_price + FROM win_loss_events + WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days' + `), + pool.query(` + SELECT customer_segment, + COUNT(*) AS total, + COUNT(*) FILTER (WHERE outcome = 'won') AS won, + COUNT(*) FILTER (WHERE outcome = 'lost') AS lost, + ROUND(COUNT(*) FILTER (WHERE outcome = 'won')::numeric + / NULLIF(COUNT(*) FILTER (WHERE outcome IN ('won','lost')),0)*100,1) AS win_rate_pct + FROM win_loss_events + WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days' + AND customer_segment IS NOT NULL + GROUP BY customer_segment + ORDER BY total DESC + `), + pool.query(` + SELECT COALESCE(wl.form_factor, tx.form_factor) AS form_factor, + COUNT(*) AS total, + COUNT(*) FILTER (WHERE outcome = 'won') AS won, + COUNT(*) FILTER (WHERE outcome = 'lost') AS lost + FROM win_loss_events wl + LEFT JOIN transceivers tx ON tx.id = wl.transceiver_id + WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days' + GROUP BY COALESCE(wl.form_factor, tx.form_factor) + HAVING COALESCE(wl.form_factor, tx.form_factor) IS NOT NULL + ORDER BY total DESC + `), + // Price delta analysis: where we lost — how far off were we? + pool.query(` + SELECT + ROUND(AVG(competitor_price - our_price)::numeric, 2) AS avg_price_gap, + ROUND(AVG((competitor_price - our_price) / NULLIF(our_price,0) * 100)::numeric, 1) AS avg_gap_pct, + COUNT(*) AS events_with_prices + FROM win_loss_events + WHERE outcome = 'lost' + AND our_price IS NOT NULL AND competitor_price IS NOT NULL + AND deal_date >= CURRENT_DATE - INTERVAL '${days} days' + `), + ]); + + return res.json({ + success: true, + days, + overall: overall.rows[0], + by_segment: bySegment.rows, + by_form_factor: byFormFactor.rows, + price_delta: priceDeltas.rows[0], + }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); + +// ── GET /api/win-loss/competitors — Competitor ranking ─────────────────────── +winLossRouter.get("/competitors", async (req: Request, res: Response) => { + const days = Math.min(parseInt(req.query.days as string) || 90, 730); + + try { + const result = await pool.query(` + SELECT + competitor_vendor, + COUNT(*) AS encounters, + COUNT(*) FILTER (WHERE outcome = 'lost') AS losses_to, + COUNT(*) FILTER (WHERE outcome = 'won') AS wins_against, + ROUND(AVG(competitor_price - our_price) + FILTER (WHERE outcome = 'lost' AND our_price IS NOT NULL AND competitor_price IS NOT NULL) + ::numeric, 2) AS avg_price_advantage, -- negative = they beat us on price + ROUND(AVG(competitor_price)::numeric, 2) AS avg_competitor_price + FROM win_loss_events + WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days' + AND competitor_vendor IS NOT NULL + GROUP BY competitor_vendor + ORDER BY losses_to DESC, encounters DESC + LIMIT 30 + `); + + return res.json({ success: true, competitors: result.rows }); + } catch (err) { + return res.status(500).json({ success: false, error: String(err) }); + } +}); diff --git a/packages/api/src/utils/csv.ts b/packages/api/src/utils/csv.ts new file mode 100644 index 0000000..1f242a1 --- /dev/null +++ b/packages/api/src/utils/csv.ts @@ -0,0 +1,35 @@ +/** + * Minimal CSV serializer — no external dependencies. + * Converts an array of flat objects to RFC 4180-compliant CSV text. + */ + +function escapeCell(value: unknown): string { + if (value === null || value === undefined) return ""; + const str = String(value); + // Quote if contains comma, quote, newline, or leading/trailing whitespace + if (/[",\n\r]/.test(str) || str !== str.trim()) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +export function toCSV(rows: Record[]): string { + if (rows.length === 0) return ""; + const headers = Object.keys(rows[0]); + const lines = [ + headers.join(","), + ...rows.map(row => headers.map(h => escapeCell(row[h])).join(",")), + ]; + return lines.join("\r\n"); +} + +/** + * Send a CSV response with proper headers. + */ +import type { Response } from "express"; +export function sendCSV(res: Response, rows: Record[], filename: string): void { + const csv = toCSV(rows); + res.setHeader("Content-Type", "text/csv; charset=utf-8"); + res.setHeader("Content-Disposition", `attachment; filename="${filename}"`); + res.send("" + csv); // BOM for Excel UTF-8 compatibility +} diff --git a/packages/dashboard/index.html b/packages/dashboard/index.html index 7049ae4..dbadcbe 100644 --- a/packages/dashboard/index.html +++ b/packages/dashboard/index.html @@ -129,17 +129,25 @@ .tabs { display: flex; gap: 0; border-bottom: 1px solid var(--border); - padding: 0 clamp(1.5rem, 4vw, 5rem); + padding: 0 clamp(1rem, 3vw, 4rem); background: var(--surface); + overflow-x: auto; + scrollbar-width: none; + -ms-overflow-style: none; + white-space: nowrap; + flex-wrap: nowrap; } + .tabs::-webkit-scrollbar { display: none; } .tab { - padding: 0.75rem 1.25rem; + padding: 0.65rem 0.9rem; cursor: pointer; border-bottom: 2px solid transparent; color: var(--text-dim); - font-size: 0.8rem; font-weight: 600; + font-size: 0.77rem; font-weight: 600; transition: all 0.2s ease; - letter-spacing: 0.02em; + letter-spacing: 0.01em; + flex-shrink: 0; + white-space: nowrap; } .tab:hover { color: var(--text-bright); } .tab.active { @@ -794,20 +802,25 @@
Transceivers
Vendors
Standards
-
🎓 Academy
+
Academy
Switches
News
Finder
Blog Engine
-
Procurement Intelligence
-
🕷 Crawler Intelligence
-
🌐 Network
-
✎ Review
-
🏭 Stock
-
💲 Price Comparison
-
🔀 Equivalences
-
📚 KB
-
🧾 Bulk
+
Procurement
+
Crawlers
+
Network
+
Review
+
Stock
+
Price Comparison
+
Equivalences
+
KB
+
Bulk
+
RFQ
+
Alerts
+
Win/Loss
+
API Keys
+
ROI Calc
@@ -1488,6 +1501,27 @@
✓ Lokal / keine API-Kosten
⚠ Qualitaet laeuft schon, Feinschliff folgt
+ +
BLOG_LLM_PROVIDER=ollama
OLLAMA_LLM_MODEL=fo-blog-v10
@@ -1569,10 +1603,12 @@
🔗 Blog aus URL generieren
Link eingeben → Inhalt wird automatisch extrahiert → BlogLLM schreibt einen Artikel daraus
+ +
-
@@ -1590,13 +1626,46 @@
-
+ + +
+ + + + + +
+ +
@@ -2556,7 +2625,7 @@