Compare commits
No commits in common. "main" and "main-pre-reconcile-2026-06-04" have entirely different histories.
main
...
main-pre-r
105
.github/scripts/changelog-draft.py
vendored
105
.github/scripts/changelog-draft.py
vendored
@ -1,105 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Draft CHANGELOG_PENDING.md entries from Conventional Commit messages
|
||||
pushed since the last run. Deterministic -- no LLM call, no network access.
|
||||
Human review still required before folding an entry into CHANGELOG.md.
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
INCLUDE_TYPES = {
|
||||
"feat": "Added",
|
||||
"fix": "Fixed",
|
||||
"refactor": "Changed",
|
||||
"perf": "Changed",
|
||||
"security": "Fixed",
|
||||
}
|
||||
SKIP_TYPES = {"chore", "ci", "test", "docs", "style", "merge"}
|
||||
|
||||
# Anything mentioning internal infra by name/address is dropped rather than
|
||||
# drafted -- better a missing entry (caught in human review) than a leaked one.
|
||||
INTERNAL_PATTERN = re.compile(
|
||||
r"\b(erik|gitea\.context-x|192\.168\.|10\.10\.0\.|opnsense|ssh |systemd|pm2 |crontab)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
PENDING_HEADER = (
|
||||
"# Pending Changelog Entries\n\n"
|
||||
"Drafted automatically from commit messages since the last processed "
|
||||
"push. Review, edit as needed, and fold into CHANGELOG.md -- then clear "
|
||||
"this file.\n\n"
|
||||
"<!-- PENDING_ENTRIES -->\n"
|
||||
)
|
||||
|
||||
|
||||
def run(cmd):
|
||||
return subprocess.check_output(cmd, shell=True, text=True).strip()
|
||||
|
||||
|
||||
def main():
|
||||
before = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
after = sys.argv[2] if len(sys.argv) > 2 else "HEAD"
|
||||
if not before or set(before) == {"0"}:
|
||||
print("No usable commit range (first push on branch) -- skipping.")
|
||||
return
|
||||
|
||||
try:
|
||||
log = run("git log {}..{} --format='%s'".format(before, after))
|
||||
except subprocess.CalledProcessError:
|
||||
print("Could not diff commit range -- skipping.")
|
||||
return
|
||||
if not log:
|
||||
print("No new commits.")
|
||||
return
|
||||
|
||||
buckets = {"Added": [], "Fixed": [], "Changed": []}
|
||||
for subject in log.splitlines():
|
||||
m = re.match(r"^(\w+)(\([\w./-]+\))?!?:\s*(.+)$", subject)
|
||||
if not m:
|
||||
continue
|
||||
ctype, _, desc = m.groups()
|
||||
ctype = ctype.lower()
|
||||
if ctype in SKIP_TYPES or ctype not in INCLUDE_TYPES:
|
||||
continue
|
||||
if INTERNAL_PATTERN.search(desc):
|
||||
continue
|
||||
buckets[INCLUDE_TYPES[ctype]].append(desc.strip())
|
||||
|
||||
if not any(buckets.values()):
|
||||
print("Nothing changelog-worthy in this push.")
|
||||
return
|
||||
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
lines = ["## Pending -- {}\n".format(date)]
|
||||
for section in ("Added", "Fixed", "Changed"):
|
||||
if buckets[section]:
|
||||
lines.append("### {}".format(section))
|
||||
for item in buckets[section]:
|
||||
lines.append("- {}".format(item))
|
||||
lines.append("")
|
||||
entry = "\n".join(lines).rstrip() + "\n\n"
|
||||
|
||||
try:
|
||||
with open("CHANGELOG_PENDING.md", "r") as f:
|
||||
existing = f.read()
|
||||
if not existing.strip():
|
||||
existing = PENDING_HEADER # pre-existing but empty placeholder file
|
||||
except FileNotFoundError:
|
||||
existing = PENDING_HEADER
|
||||
|
||||
marker = "<!-- PENDING_ENTRIES -->\n"
|
||||
if marker in existing:
|
||||
head, tail = existing.split(marker, 1)
|
||||
updated = head + marker + entry + tail.lstrip("\n")
|
||||
else:
|
||||
updated = existing.rstrip("\n") + "\n\n" + entry
|
||||
|
||||
with open("CHANGELOG_PENDING.md", "w") as f:
|
||||
f.write(updated)
|
||||
|
||||
print("Wrote entry:\n" + entry)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
39
.github/workflows/changelog-draft.yml
vendored
39
.github/workflows/changelog-draft.yml
vendored
@ -1,39 +0,0 @@
|
||||
name: changelog-draft
|
||||
|
||||
# Drafts CHANGELOG_PENDING.md entries from Conventional Commit subjects on
|
||||
# every push to main. Deterministic -- regex-parses commit subjects only,
|
||||
# no LLM call, no network access. Buckets feat/fix/refactor/perf/security
|
||||
# into Added/Fixed/Changed; skips chore/ci/test/docs/style/merge commits and
|
||||
# anything mentioning internal infra hostnames/addresses. Never touches
|
||||
# CHANGELOG.md itself -- human review still required to fold an entry in.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
changelog-draft:
|
||||
runs-on: ubuntu-latest
|
||||
if: "!contains(github.event.head_commit.message, '[skip ci]')"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Draft pending changelog entry
|
||||
run: python3 .github/scripts/changelog-draft.py "${{ github.event.before }}" "${{ github.sha }}"
|
||||
|
||||
- name: Commit draft if changed
|
||||
run: |
|
||||
if ! git diff --quiet CHANGELOG_PENDING.md 2>/dev/null; then
|
||||
git config user.name "changelog-bot"
|
||||
git config user.email "changelog-bot@context-x.org"
|
||||
git add CHANGELOG_PENDING.md
|
||||
git commit -m "chore(changelog): draft pending entries [skip ci]"
|
||||
git push origin HEAD:main
|
||||
else
|
||||
echo "No changelog-worthy commits in this push."
|
||||
fi
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -21,8 +21,3 @@ storage-*/
|
||||
# Local credentials (never commit)
|
||||
.tip/.env
|
||||
run-fs-scraper-mac.sh.local
|
||||
|
||||
# Training / SFT datasets — bulk data + potential internal pricing/market intel, keep out of git
|
||||
*.jsonl
|
||||
training-data/runpod/**/*.jsonl
|
||||
training-data/*.jsonl
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
# transceiver-db pre-push scan allowlist. ERE, one pattern per line, matched against finding lines.
|
||||
# This repo is Gitea-internal and is never published publicly; entries below are confirmed false positives.
|
||||
# Risk note: a low-entropy secret sharing a line with an allowlisted token would be masked here.
|
||||
# High-entropy secrets are still caught by the gitleaks pass, which ignores this allowlist.
|
||||
# If this repo is ever made public, drop this file and run the internas sanitizer first.
|
||||
# --- vendor name that is legitimate market-intelligence dataset subject matter ---
|
||||
flexoptix
|
||||
# --- owner homelab infra referenced in runbooks, docs and deploy scripts ---
|
||||
82\.165\.222\.127
|
||||
192\.168\.178\.
|
||||
10\.10\.0\.
|
||||
\berik\b
|
||||
\bclaudi\b
|
||||
renefichtmueller
|
||||
fearghas
|
||||
/opt/[a-z]
|
||||
fichtmueller\.org
|
||||
context-x\.org
|
||||
# --- sibling project codenames referenced in the cross-project sync dev journal ---
|
||||
\bmagatama
|
||||
peercortex
|
||||
switchblade
|
||||
shieldx
|
||||
papercortex
|
||||
ctxevent
|
||||
ctxmeet
|
||||
nognet\.net
|
||||
\bpulso\b
|
||||
# --- env-resolved DB URIs and passwords, no literal secret present ---
|
||||
postgres(ql)?://\$\{
|
||||
\$\{DB_PASS\}
|
||||
process\.env\.PGPASSWORD
|
||||
# --- placeholder secret in example config ---
|
||||
CHANGE_ME
|
||||
# --- ML tokenizer special tokens, not credentials ---
|
||||
special_tokens
|
||||
pad_token
|
||||
@ -1,12 +1,15 @@
|
||||
# TIP Changelog
|
||||
|
||||
Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}`
|
||||
{"d":"2026-07-17","t":"FIX","m":"Equivalence pipeline audit: found 2 more sources writing straight to auto_approved without the recent-price/standard_name discrimination catalog-reconcile.ts uses (spec-matcher.ts flat-0.85, no gate) vs one that already had it right (maintenance:find-equivalences enhanced mode implicitly requires standard_name to hit its 0.85 bar and pre-filters recent price). Fixed spec-matcher.ts to insert pending with re_research_due_at=NOW() instead of bypassing review; one-time reclassification of the 1,455 pre-existing spec-basis auto_approved rows (sql/140): 972 held up, 483 demoted (some permanent mismatch, some re-research in 21d for missing evidence)."}
|
||||
{"d":"2026-07-17","t":"FEAT","m":"Switch-side drift-guard (sql/141): closed the third direction of the physically-impossible-compat invariant (compatibility writes=126, transceiver speed changes=138, now switches.max_speed_gbps/ports_config changes=141). Currently a no-op live (nothing updates those columns today outside the one-time sql/125 backfill) -- preventive, ahead of the planned switch datasheet-enrichment automation. Verified with a synthetic in-transaction test (1250 rows correctly flagged, rolled back)."}
|
||||
{"d":"2026-07-12","t":"FIX","m":"Switch-DB compat_speed_guard drift gap closed (sql/138): the sql/126 trigger only validated compatibility writes, so a later transceivers.speed_gbps update (e.g. the daily enrich-modulation-fec.sh recovery) never re-checked existing compatible spec_match rows against the switch speed ceiling. Found live: 189 rows had drifted physically-impossible again. One-time remediation (189 rows -> incompatible, backed up to compat_speed_drift_backup_20260712) plus a new AFTER UPDATE OF speed_gbps trigger on transceivers closes the gap for both write directions. check_switch_quality() passes again."}
|
||||
{"d":"2026-07-11","t":"FEAT","m":"Modulation/FEC enrichment (sql/135): IMMUTABLE helpers derive modulation (NRZ/PAM4/DP-16QAM/DP-QPSK/Coherent), fec_type and inbuilt_fec from speed_gbps + form_factor + part_number/standard_name regex, incl. coherent detection (>=100G-floored ZR/DCO/OpenZR/QAM + CFP2-DCO/ACO via form_factor). Fill-only + idempotent via fn_tip_enrich_modulation_fec(); daily self-healing cron 03:30 (scripts/enrich-modulation-fec.sh). Modulation coverage >=100G rose from 3-8 percent to 46 percent (400G 59, 800G 56, 1600G 100); 4224 rows enriched. Rules built by a 4-lens optical-standards workflow and adversarially verified — the 10G-ZR mislabel, CFP2-DCO miss, AOC/DAC scope leak and bare-QSFP28 over-guess were all caught and fixed before apply. vendor_verified rows never overwritten; enriched rows tagged data_confidence=enriched_estimated + enriched_fields."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"speed_gbps plausibility guard + cleanup (sql/136): ~22 percent of rows carried junk speed (2129 at <=0, 167 astronomical >1600 values, ~2400 sub-0.09 fragments) from cable/MTP-trunk part numbers parsed into speed. speed_gbps made nullable (unknown speed is honestly NULL); tip_speed_is_implausible() + BEFORE trigger trg_speed_plausibility sanitize junk to NULL on write and never abort ingestion; one-time cleanup nulled 4708 rows (id-keyed reversible snapshot in tip_speed_gbps_rollback). The 0.09..1600 gray band (~355 legit FC/SDI/CPRI/OTN line rates) is intentionally preserved. 87 percent of nulled rows are dac/aoc/accessory where speed is meaningless."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"v_transceiver_speed_errors honest metric (sql/137): after 136 the view counted 4708 NULL-speed rows, but 4112 are dac/aoc/accessory where NULL is correct. It now flags only real optics (product_type not in dac/aoc/aec/accessory/switch/nic) with NULL/<=0/>1600 speed -> 596 genuine module gaps, down from 2134 mixed."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Transceiver Academy: full API-backed customer and employee training platform. 5 categories (Standards, Form Factors, Switches & Compatibility, Fiber & Infrastructure, Testing & Buying), 22 detailed bilingual lessons (EN+DE), 74 quiz questions. API: GET /api/training/categories, /lessons, /lessons/:id, /quiz, /stats — public route, no auth required. Dashboard UI: language toggle EN/DE, category tabs, lesson cards with level badges, full lesson viewer (paragraphs/tables/callouts/code/formulas), per-lesson and per-category quiz engine with auto-advance dots progress, A-F grade results, localStorage progress tracking. Replaces old inline LLM-training data module."}
|
||||
{"d":"2026-05-14","t":"UI","m":"Price History Chart: replaced 260×60px sparkline with full interactive SVG chart (520×200px). Features: multi-vendor colored polylines with end-point dots, Y-axis labels (USD normalized), X-axis date ticks (MM-DD), horizontal grid lines, hover cursor line + floating tooltip showing all vendor prices for hovered day, vendor legend with click-to-toggle visibility, time range selector (7d/14d/30d) with live re-fetch, current best prices table (FX-normalized to USD). FX normalization: EUR×1.08, GBP×1.27. Supports up to 8 vendors (indigo/orange/green/yellow/blue/red/purple/cyan palette). No API changes — existing GET /api/price-history/:id endpoint already returned price_max/min/avg per vendor per day."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Procurement: 5 neue Intelligence-Sektionen. (E) 🟢 Buy-Now Intel — Top buy_now Reorder Signals aus 211k preberechneten Signalen, filterbar nach Form Factor, Signalstärke-Balken, Preis/Stock-Trend, Gründe als Tooltip. API: GET /api/procurement/reorder-top. (A) 💰 Arbitrage — FX-Preis vs. Competitor-Preis für 59k Equivalenz-Paare mit Preisdaten auf beiden Seiten, normalisiert auf USD (EUR×1.08, GBP×1.27), sortiert nach Ersparnis-%. API: GET /api/procurement/arbitrage. (B) 🖥 Switch Compat — Suche nach Switch-Modell (Cisco, Juniper, Arista etc.), zeigt alle kompatiblen Transceiver mit Preis + Verifikationsmethode. 58k Compatibility-Rows, 429 Switches. API: GET /api/procurement/switch-compat?search=. (C) ⚠️ Supply Squeeze — Multi-Signal-Detektor: 4 parallele Quellen (Preis-Momentum 30d vs 60d, Hype-Phase, AI-Cluster-Transceiver-Nachfrage, Stock-Level-Verteilung). Severity: critical/warning/watch. API: GET /api/procurement/supply-squeeze. (D) 🪦 Dead Stock Revival — 7.297 Dead-Stock-SKUs gegen Hype-Cycle-Phasen: zeigt welche Lagerhüter in Technologieklassen liegen die gerade aufsteigen (ascending hype phases, score >30). API: GET /api/procurement/dead-stock-revival."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Crawler Intelligence: Data Quality panel. New GET /api/scrapers/data-quality endpoint — 4 parallel queries over 200,617 transceiver_verification_evidence rows: (1) coverage breakdown (price 11,366/18,146 = 62%, image 12,333/68%, details 17,085/94%, competitor_match 399/2%, quarantined 1,193); (2) all 10 evidence types with count + avg confidence + product count + last seen; (3) robot/scraper contributions table (17 robots ranked by output); (4) daily activity last 14 days. Dashboard Crawler Intelligence tab: new 🔬 Data Quality section with coverage progress bars (color-coded ≥80% green / ≥50% amber / red), evidence type table, SVG sparkline bar chart for 14-day activity, robot contributions table with live/stale dot indicators."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Dynamic Hype Cycle + Market Signal Engine: Hype Cycle tab is now fully data-driven. New GET /api/hype-cycle/market-signals endpoint blends 6 real data sources into a composite Market Signal Score (0–100) per technology: (1) hype_score from Norton-Bass model (30% weight), (2) hyperscaler CapEx YoY avg (Microsoft +68.8%, Alphabet +107.4%, Meta +46.8%), (3) price observation activity ratio 30d vs prior 30d, (4) AI cluster estimated transceiver demand (90d window), (5) eBay secondary market sell-through velocity, (6) internal fast-mover demand trend. Score thresholds: ≥70 green, ≥50 yellow, ≥30 orange, <30 gray. Recommendation engine: buildRecommendation(phase, signalScore, capexYoyAvg, speedGbps) maps hype phase × capex boom × speed class → Buy/Hold/Watch label with color + detail tooltip. Dashboard: Hype Cycle table shows Market Signal ● LIVE column (score + progress bar) + Recommendation column (emoji label, tooltip with reasoning). Market Context cards row above table shows Top Signal, CapEx Boom %, Fast Movers signal, eBay Velocity. New Hyperscaler CapEx panel (SEC filing data) + eBay Secondary Market panel at bottom of hype tab. Procurement: new 🛒 eBay Market sub-section with per-form-factor sell-through grid. All 6 queries run in parallel via Promise.all()."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Procurement tab: 2 new sections with real data. (1) 📦 Internal Demand — Flexoptix internal SKU velocity from flexoptix_internal_demand table (8,585 SKUs: 70 fast-movers 53k units/12M, 239 regular, 979 slow, 7,297 dead stock). Summary cards with trend %%. Filter by velocity class. API: GET /api/procurement/internal-demand?velocity_class=&limit=&sort=. (2) 🤖 AI Clusters — live AI datacenter announcements from ai_cluster_announcements table (396 in last 30 days). Shows estimated transceiver demand per build, MW scale, company, location, source link. Filter for entries with transceiver estimates. Stats: total announcements, MW, distinct companies, total estimated transceivers. API: GET /api/procurement/ai-clusters?days=&limit=. Replaced misleading DEMO DATA banners on Signals + ABC sections with informational note pointing to Internal Demand data."}
|
||||
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Training Module im Standards-Tab: 13 Lektionen (Form Factors, Glasfaser, IEEE 802.3, WDM, PAM4/NRZ, Link Budget mit Live-Rechner, Coherent Optics, MSA/DOM, Vendor Locking, Temperature Classes, Selection Guide, 400G/800G, Troubleshooting), 40 Quiz-Fragen mit Shuffle/Feedback/Erklaerungen/Note A-F, 4 Lernpfade (Einsteiger 5 / Netzwerk-Engineer 9 / Einkaeufer 6 / Expert 13 Lektionen), Fortschrittsbalken, localStorage-Persistenz. Kein DB-Schema-Aenderung - alles client-seitig als JS-Data-Objekte."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"6 neue Dashboard-Features: (A) Price Movers Alert — GET /api/procurement/price-movers?days=N&limit=N, CTE-basiert (cur vs prior period avg per SKU+Vendor), zeigt Top-Gainers und Top-Losers (|delta_pct| >= 2%, obs >= 2). Procurement-Tab Sektion mit Period-Toggle 7d/14d/30d, Export CSV. (B) Executive Overview Pulse — 5 KPI-Karten (Buy Signals, Arbitrage Ops, Supply Alerts, Price Gainers, Losers) über `loadProcurementPulse()`, Top-Movers Mini-Tabelle im Overview, alle clickable → Procurement-Tab. (C) CSV Export — exportMoversCSV() generiert Gainers+Losers als CSV-Download. (D) Vendor Intelligence — GET /api/vendors/intelligence: per-Vendor in letzten 30d (sku_count, price_obs, avg/min/max price, last_seen), Top-6-Anbieter-Banner im Vendors-Tab. (E) Advanced Transceiver Search — Speed-Filter (1G/10G/.../800G), Fiber-Type-Filter (SMF/MMF), 'Verified Only'-Checkbox in Transceivers-Tab; searchTransceivers() übergibt speed_gbps=, fiber_type=, verified=price an GET /api/transceivers. (F) Knowledge Base Browser — neuer Tab KB, GET /api/kb?q=&category=&limit= (Full-Text ILIKE über question/answer/subcategory), Category-Pills, Entry-Cards mit Severity-Badge, Form-Factor/Speed-Tags."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Equivalences Explorer: new dashboard tab '🔀 Equivalences' — search 63,362 cross-brand mappings (46 vendors, 7,516 competitor products → 846 Flexoptix alternatives, Ø 93.9% confidence). APIs: GET /api/equivalences (search), /api/equivalences/transceiver/:id (per-product), /api/equivalences/stats, /api/equivalences/top-vendors. Transceiver detail modal now shows equivalences panel (FX alternatives or competitor products) + SVG price history sparklines (30-day, per source vendor) from 392k+ price observations."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"LinkedIn Distribution Status: Blog tab shows DRY_RUN badge, posted/dry_run/skipped/failed counters, history table with live URN links. GET /api/blog/linkedin/history reads blog_linkedin_distribution table + detects DRY_RUN mode from ecosystem config."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"MCP Server: 2 new tools — find_equivalences (search 63k+ verified cross-brand mappings with confidence filter, returns FX alternatives + competitor matches formatted for LLM) + get_price_history (392k+ obs, daily series, per-vendor min/max/avg, cheapest source identification). Total: 21 MCP tools."}
|
||||
@ -302,49 +305,3 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA
|
||||
{"d":"2026-04-28","t":"FIX","m":"Product verification pipeline: image crawls now mark image_verified/image_verified_url, scraped product pages mark details_verified/details_source_url, maintenance reconcile backfills old product URLs/images/details, and --backfill-images exposes the existing image crawler via scraper CLI. Migration 102 reconciles existing data."}
|
||||
{"d":"2026-04-28","t":"FIX","m":"Blog Engine Hot Topics: diversified ranking with refresh shuffle/source caps/already-created-topic demotion, plus richer LLM context briefings passed into topic expansion and master-draft context via custom_title/additional_context."}
|
||||
{"d":"2026-04-29","t":"FEAT","m":"TIPLLM robot learning loop: verification robot controller writes status, TIPLLM plans, queue dry-runs/enqueues and crawler outcomes into the Gitea-backed TIP training pool; learning-pool build imports qa-pairs from TIP_TRAINING_REPO into the tip_llm lane. Removed hardcoded Gitea token fallback; existing git remotes or env tokens are used."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "Blog fo-blog LLM reachable again: routed to Mac Studio Ollama over the Erik<->home WireGuard tunnel via new BLOG_OLLAMA_URL (search/embeddings stay Erik-local on OLLAMA_URL). Exposed Mac Ollama on 0.0.0.0 via LaunchAgent plist; adopted training candidate as fo-blog-v45. Verified model discovery + live inference over WG."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Blog Ollama high-availability: dual SSH tunnels from Erik to Mac Studio loopback Ollama over independent transports (primary WireGuard localhost:11436, backup Cloudflare localhost:11435; both systemd, Restart=always, enabled on boot). Blog client auto-fails-over primary->backup (re-probed at startup + every 60s, self-heals to primary). Robust against the Mac training pipeline restarting Ollama or changing its bind."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Hype-cycle market-signals endpoint (/api/hype-cycle/market-signals): data-driven per-technology market signal score (0-100) + drivers + buy/hold recommendation from the adoption model and real SKU/vendor/price data. Powers the dashboard Market Context cards (were empty)."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "Vendor /market-share + /intelligence were shadowed by /:id (404) with latent SQL bugs (round(double,int); non-existent po.id). Routed past /:id via next() fallthrough; fixed SQL (numeric casts, COUNT(*)). Now 200 with real data."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Vendor reliability endpoint (/api/vendors/reliability): per-vendor freshness/frequency/coverage sub-scores + composite reliability score from real data (211 vendors). Populates the reliability badges on vendor cards."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "LLM-gym disk-watchdog keep-list now also protects the newest candidate per lane (not just adopted vN/-rN), fixing the churn that fully deleted fo-blog when the quality gate blocked adoption."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"Supply-Squeeze Preis-Momentum: paarweiser per-SKU Vergleich statt Aggregat — eliminiert Katalog-Komposition-Bias (400G OSFP +151%→0%, Signale 21→8). Nur SKUs in beiden Perioden, Median der per-SKU-Deltas, Nicht-Transceiver/AOC/Switch-Filter."}
|
||||
{"d":"2026-06-06","t":"AI","m":"Research Robot: Handlungsempfehlungen + Action-Buttons. API klassifiziert fehlschlagende Jobs (auth/401, network, no-handler) → Empfehlung+Aktionen. POST /action: dispatch/pause/resume (pgboss, whitelist-validiert, reversibel). Dashboard rendert Empfehlungs-Karten mit Buttons. sync:flexoptix-catalog→critical auth, bietet Token-Hilfe+Pause."}
|
||||
{"d":"2026-06-06","t":"FEAT","m":"FS.COM Wettbewerber-Lagerbestand pro Technologie in Sales-Velocity-Tabelle (/api/stock/competitor-by-tech), Ampel-Färbung vs. Monatsbedarf."}
|
||||
{"d":"2026-06-06","t":"FEAT","m":"NADDOD-Scraper: per-Warehouse Stock-Parser (us/nl/sg/cn) + warehouse_global_qty Fallback → erscheint in Competitor-Stock-Vergleich."}
|
||||
{"d":"2026-06-06","t":"UI","m":"Preis-Chart-Modal beim Klick auf Signal-Karte (180d SVG, multi-vendor). fmtSpd() bereinigt Speed-Anzeige (1.00G→1G, 1600→1.6T). Flexoptix-Preis in Competitor-Vergleich."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"sync:flexoptix-catalog 401 ROOT CAUSE: leerer FLEXOPTIX_API_TOKEN='' kurzschloss '?? null' (nur null/undefined), Script sendete leeren Bearer statt Username/Passwort-Login. Fix '|| null'. Credentials waren immer korrekt. Voller Sync 457 Produkte/345 Preis/36 Stock. Job completed, Robot-Empfehlung jetzt 'alle gesund'."}
|
||||
{"d":"2026-06-06","t":"DATA","m":"Voller Daten-Refresh: Hype-Cycle + ABC + Forecast + Reorder-Signale + Preis-Denorm neu gerechnet (alle completed). Hype-Phasen aktuell: 10G plateau, 800G peak, 1.6T innovation-trigger."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"reorder_signals Daten-Bloat behoben: 4.49M Zeilen/1.19GB → 18.175/4.5MB (99.6% frei). Writer hängte bei jedem 4h-Lauf seit 09.04. neue Zeilen an, löschte nie (24h-TTL nur read-seitig gefiltert). 4.37M waren bereits abgelaufen. Fix: delete-before-insert pro Transceiver. VACUUM FULL. Backup reorder_signals_keep_bak_20260606. Verifiziert stabil bei 18.175."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"Procurement-Pulse Buy-Signals-Karte rief 404-Endpoint /api/procurement/reorder auf → zeigte immer 0. Korrigiert auf /api/procurement/reorder-top (summary.buy_now=314)."}
|
||||
{"d":"2026-06-06","t":"INFRA","m":"System-Health-Audit: 24/24 API-Endpoints grün, keine Dauer-Failing-Jobs, Disk 35%/RAM 99Gi frei, tip-postgres healthy. reorder-signals war einzige Bloat-Quelle."}
|
||||
{"d":"2026-06-07","t":"INFRA","m":"Erik↔Gitea Reconciliation ABGESCHLOSSEN: 22 Erik-lokale Fixes + 28 Gitea-Feature-Commits gemergt (ort, 0 Konflikte automatisch). Einziger manueller Fix: doppelte /api/hype-cycle/market-signals Route dedupliziert (Erik-Handler behalten, matcht Dashboard). 3 Pakete 0 TS-Fehler, 11/11 Endpoints grün, market-signals liefert korrekte Shape. Push main b5925bc2..8c6df102, jetzt 0/0 sync. Safety: branch erik-pre-reconcile-20260607 + bundle (auch Fearghas) + bulk-price WIP in stash@{0}."}
|
||||
{"d":"2026-06-07","t":"DATA","m":"Reconciliation-Bonus: die 4 früher als 'kein Handler' deaktivierten Jobs (match:opn, match:spec, analyze:stock:velocity, enrich:flexoptix-details) wurden vom Merge selbst geheilt — Gitea-Branch lieferte Handler+Schedules, Daemon-Restart re-registrierte sie automatisch (korrekte Crons). Alle 4 test-gelaufen: completed, echte Daten (stock_velocity 262, +40 Equivalences, 100 Detail-Syncs). 284 Jobs geplant, 0 Failing. bulk-price: committe Version funktioniert, Varianten-WIP bleibt in stash@{0}."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Switch-Kompatibilität + Speed-Daten von der Quelle (kein Raten mehr). ROOT CAUSE: Flexoptix-Bulk-API hat KEIN Speed/Formfaktor-Feld -> Sync riet aus Produktnamen (FO-109010-CWDM -> 100000G). FIX: detail-enricher zieht jetzt form_factor aus Spec 'Form Factor' + speed aus 'Supported Protocols'/'Bandwidth' (autoritatives Datenblatt), überschreibt korrupte Bulk-Werte. Voll-Katalog re-enriched: 96.8% plausibel, 487 Teile, 0 Widersprüche bei Stichprobe."}
|
||||
{"d":"2026-06-11","t":"FIX","m":"getFlexoptixSuggestions: (1) physikalische Validierung per Port-Speed + Käfig-Mechanik aus ports_config (100G-QSFP28-Port -> nur <=100G QSFP-Module, nie 400G/QSFP-DD); (2) nur katalog-bestätigte Teile (fx_specifications IS NOT NULL). CX 10000-48Y6C: war 4 unmögliche Gruppen (400G/200G/128G/100T QSFP28) -> jetzt 630 Vorschläge, 0 unmöglich, 0 Phantom. Generalisiert: 400G-Switch zeigt bis 400G."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Befund: ~107 Teile waren fälschlich Vendor 'Flexoptix' zugeordnet (Bindestrich-SKUs FO-/FOT-, nicht im echten FX-Katalog der nur Punkt-SKUs nutzt). Aus 'BEI FLEXOPTIX BESTELLEN' gefiltert. Wettbewerber-Kataloge (ATGBICS/NADDOD) haben eigene Formfaktor-Parsing-Fehler -> separater Scraper-Fix nötig."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Flexoptix-Datenblatt-Audit: alle 805 katalog-bestätigten Teile programmatisch gegen gespeicherte fx_specifications (Datenblatt) geprüft. Nach Terabit-Parser-Fix + Re-Derive aus Specs: Form Factor 802/802 (100%), Speed 801/801 (100%), 0 Abweichungen. Jeder Wert kommt aus dem Datenblatt, nicht aus dem Namen."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Switch-Daten-Audit: 681 Switches, nur 267 (39%) mit Port-Config, 414 ohne (überwiegend Cisco-Bulk-Import). Vorhandene Port-Configs 99.3% plausibel, 248/267 mit Hersteller-URL, nur 35 mit datasheet_url. Demo-Switches (Aruba CX10000, Arista, Juniper) vollständig+korrekt. LÜCKE: autoritatives Port-Sourcing per Hersteller-Datenblatt für die 414 fehlt = separates Scraper-Projekt pro Vendor."}
|
||||
{"d":"2026-06-11","t":"FIX","m":"Switch-Kompatibilität = 'läuft wirklich': Matcher verlangt jetzt zusätzlich, dass Flexoptix den Transceiver für den Switch-VENDOR codieren kann (fx_compatibilities). Physisch-passend reicht nicht. CX 10000: 629 physisch -> 283 echt-Aruba-codierbar (283/283 verifiziert). Cisco 8818: 55 (alle Cisco-codierbar). Plus Chassis-Port-Format (max_per_slot_NNNG_FF) geparst. Jede Empfehlung jetzt: katalog-bestätigt + datenblatt-genau + passt physisch + codierbar = garantiert lauffähig."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"product_type-Klassifikation (sql/119): neues Enum module|dac|aoc|aec|switch|nic|accessory, deterministisch per Trigger trg_transceivers_biu aus part_number/category/reach gesetzt (self-healing bei jedem Insert/Update, Preis-Updates unberührt). Backfill 41008 Zeilen: 36284 module, 2791 dac, 1835 aoc, 29 aec, 27 accessory, 26 switch, 16 nic. Kabel (DAC/AOC/AEC) und Switches/NICs (ConnectX/Quantum/N-Port) nicht mehr als Modul gezählt. BASE-T/RJ45-Kupfermodule vor Kabel-Demotion geschützt (Guard vor copper-Regel); AOC/DAC-Keyword schlägt Optik-Token-Guard (AOC ist optisch, trägt legitim nm/MMF/SR). Konsumenten filtern product_type='module'. Missed-cable=0 verifiziert. Per-Vendor 100G+: NADDOD 179 Module/423 Kabel/28 Switch-NIC, ATGBICS 1331/297, QSFPTEK 22/98, ProLabs 1690/818."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"form_factor-Recovery (sql/120): korrigiert NUR beweisbar-falsche Zeilen (form_factors.max_speed < speed_gbps) mit explizitem Formfaktor-Token im Titel. 87 gefixt (u.a. 400G-Kabel als SFP+ getaggt -> QSFP-DD; QSFP-DD@800G -> QSFP-DD800; Q56DD -> QSFP-DD), 947 opake Zeilen (z.B. EOLS-1303-40-D, MGB-2GSR) unangetastet gelassen, nichts geraten. Gescraptes Original in neuer Spalte form_factor_raw bewahrt. Recovery auch im Ingest-Trigger (Neuzugänge self-healing)."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"lead_time_days als NICHT ERFASST dokumentiert (sql/121, COMMENT auf price_observations/stock_observations/stock_snapshots): 0% befüllt über 1.378.984 + 85.406 + 0 Zeilen. Plumbing existiert (crawler-llm stock-schema erfasst es, upsertPriceObservation nimmt Param), aber aktiver Scraper-Pfad liefert nie einen Wert und LLM-Crawler-Tabelle stock_snapshots ist leer. Kein Default fabrizieren; NULL = unbekannt. Befüllen ist Scraper-Aufgabe."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"Befund FS.COM-Untererfassung bestätigt (kein Fix, Scraper-Thema): 392 Zeilen/378 distinct SKUs vs ATGBICS 8420 (22x-Lücke). 391/392 sind bereits product_type='module' -> Klassifikation ist NICHT die Ursache, reine Crawl-Tiefe. Ursache in fs-com.ts + Seed-Scope (Kategorie-Abdeckung) + evtl. zu aggressive Quarantäne von fs.com/c/-URLs. Empfehlung: FS.com-Crawl-Seed erweitern (Scraper-Pipeline, nicht raten)."}
|
||||
{"d":"2026-07-07","t":"SECURITY","m":"Hardcodierte DB-Passwoerter aus dem Source entfernt und auf env umgestellt. tip-DB-Passwort (transceiver_db) lag in 14 Dateien im Klartext (quoted export, unquoted psql-Aufrufe, Python-env-Dicts, env-or-hardcoded-default-Fallbacks); llm_gateway-Passwort in seed-blog-training-data.py. Ersetzt durch PGPASSWORD-Env-Referenz mit Fail-Fast-Guard bzw. os.environ. training-data/*.jsonl (48 MB) aus Tracking genommen (git rm --cached, Datei bleibt) und gitignored. .security-scan-allowlist ergaenzt fuer legitime False-Positives (Vendor flexoptix, eigene Infra, sync-Journal-Codenamen, env-aufgeloeste URIs). Pre-Push-Leak-Scanner 397 auf 0 Funde, gruen ohne --no-verify. Offen: Rotation der geleakten Werte (llm_gateway zuerst, tip fleet-weit koordiniert); Git-History behaelt Altwerte."}
|
||||
{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync: 3 Bugs gefixt + erster Lauf. NCS-5000-Regex (passte nicht auf NCS-5001/5002/5011), FX/FX3-Boundary-Guard (verhindert falsche Matches der neueren Generation), source-aware Date-Routing (Format2-Announcement-Dates -> eos_date, nicht last_support_date). 13 Switches auf EoS_Announced gesetzt. Wöchentlicher Sync Sun 04:00 via pg-boss."}
|
||||
{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync Bug 3 fix: source-aware Date-Routing (Format2-Announcement-Dates → eos_date, nicht last_support_date)"}
|
||||
{"d":"2026-07-10","t":"DATA","m":"Run cisco-eol-sync for real to update 13 switches in DB (NCS-5001/5002/5011, NCS-5502/SE, NCS-55A1, N9K-C92300YC/C9272Q/C93120TX/C93180YC-FX/C9332C, ASR-9001/9901)"}
|
||||
{"d":"2026-07-11","t":"DATA","m":"Transceiver classification fields: 4 new columns added to transceivers table (dwdm_channel, cwdm_channel, cdr_type, protocol_family). Backfill from Flexoptix notes field: 1,209 standard_names enriched (up from 72, +1,579%), 1,177 cdr_types (all dual_cdr), 2,498 wdm_types (1,896 DWDM + 602 CWDM), 7,918 protocol_family values (7,645 ethernet + 273 fibre_channel). Migration 134 applied. Note: DWDM and CWDM are separate columns — never interchangeable; fibre_channel is a separate protocol family from ethernet."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"flexoptix-api-sync.ts: API sync now writes standard_name (from protocol field), wdm_type (DWDM/CWDM from optics.dwdm/cwdm), cdr_type (dual_cdr/cdr from title regex), cdr_support (true when cdr_type set), and protocol_family (ethernet/fibre_channel/sonet/infiniband from title regex) on every Flexoptix catalog sync. Previously these columns were never populated from the API — only dwdm/cwdm booleans existed. This fixes the root cause of empty standard_name on 95.5% of auto_approved equivalences."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"catalog-reconcile.ts: Hard rejects for incompatible entity types: protocol_family mismatch (ethernet≠fibre_channel), temp_range mismatch (COM≠IND), wdm_type mismatch (DWDM≠CWDM) → confidence=0, skip immediately. Standard_name mismatch now -25pts penalty (was no penalty). CDR type mismatch: -20pts + CDR match bonus +5pts. Auto-approve gate now requires fx.standard_name IS NOT NULL, blocking the 77,419/81,116 cases where FX had no standard_name. Confidence scoring schema extended with hardReject flag. Candidates and fxProducts queries now include cdr_type, temp_range, protocol_family, wdm_type."}
|
||||
|
||||
## Pending -- 2026-07-18
|
||||
|
||||
### Fixed
|
||||
- correct 14 false curated 10G-on-1G compatibility claims
|
||||
|
||||
## Pending -- 2026-07-18
|
||||
|
||||
### Fixed
|
||||
- clean 261 spec_match rows unmasked by the 144 curated-row fix
|
||||
|
||||
|
||||
163
package-lock.json
generated
163
package-lock.json
generated
@ -11,11 +11,7 @@
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
],
|
||||
"dependencies": {
|
||||
"pdf-parse": "^1.1.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pdf-parse": "^1.1.5",
|
||||
"tsx": "^4.19",
|
||||
"typescript": "^5.9.3",
|
||||
"xlsx": "^0.18.5"
|
||||
@ -1666,16 +1662,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/multer": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.1.0.tgz",
|
||||
"integrity": "sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz",
|
||||
@ -1685,16 +1671,6 @@
|
||||
"undici-types": "~7.18.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pdf-parse": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/pdf-parse/-/pdf-parse-1.1.5.tgz",
|
||||
"integrity": "sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pg": {
|
||||
"version": "8.20.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz",
|
||||
@ -1911,12 +1887,6 @@
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/append-field": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
|
||||
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/array-flatten": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
|
||||
@ -2068,23 +2038,6 @@
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-from": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
|
||||
"dependencies": {
|
||||
"streamsearch": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/byte-counter": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/byte-counter/-/byte-counter-0.1.0.tgz",
|
||||
@ -2379,21 +2332,6 @@
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/concat-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.0.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/content-disposition": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz",
|
||||
@ -4284,68 +4222,6 @@
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/multer": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz",
|
||||
"integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
"concat-stream": "^2.0.0",
|
||||
"type-is": "^1.6.18"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.16.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/media-typer": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
"integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/mime-db": {
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/mime-types": {
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
|
||||
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"media-typer": "0.3.0",
|
||||
"mime-types": "~2.1.24"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mute-stream": {
|
||||
"version": "0.0.8",
|
||||
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
|
||||
@ -4379,12 +4255,6 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-ensure": {
|
||||
"version": "0.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz",
|
||||
"integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.36",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz",
|
||||
@ -4698,22 +4568,6 @@
|
||||
"through": "~2.3"
|
||||
}
|
||||
},
|
||||
"node_modules/pdf-parse": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-1.1.4.tgz",
|
||||
"integrity": "sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-ensure": "^0.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.8.1"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/mehmet-kozan"
|
||||
}
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.20.0",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
|
||||
@ -5509,14 +5363,6 @@
|
||||
"stream-chain": "^2.2.5"
|
||||
}
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
@ -5728,12 +5574,6 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/typedarray": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
|
||||
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
@ -6236,15 +6076,12 @@
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"helmet": "^8.0.0",
|
||||
"multer": "^2.1.1",
|
||||
"pdf-parse": "^1.1.4",
|
||||
"pg": "^8.13.1",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cors": "^2.8.17",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/multer": "^2.1.0",
|
||||
"@types/pg": "^8.11.11",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.9.3"
|
||||
|
||||
@ -25,12 +25,8 @@
|
||||
"url": "https://github.com/renefichtmueller/transceiver-db"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pdf-parse": "^1.1.5",
|
||||
"tsx": "^4.19",
|
||||
"typescript": "^5.9.3",
|
||||
"xlsx": "^0.18.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"pdf-parse": "^1.1.4"
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,22 +10,19 @@
|
||||
"start": "node dist/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^5.1.0",
|
||||
"pg": "^8.13.1",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.7",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"helmet": "^8.0.0",
|
||||
"multer": "^2.1.1",
|
||||
"pdf-parse": "^1.1.4",
|
||||
"pg": "^8.13.1",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cors": "^2.8.17",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/multer": "^2.1.0",
|
||||
"@types/pg": "^8.11.11",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.9.3"
|
||||
"@types/cors": "^2.8.17",
|
||||
"typescript": "^5.9.3",
|
||||
"tsx": "^4.19.0"
|
||||
}
|
||||
}
|
||||
|
||||
@ -318,77 +318,23 @@ export async function getCompatibleTransceivers(switchId: string) {
|
||||
*/
|
||||
export async function getFlexoptixSuggestions(switchId: string) {
|
||||
const result = await pool.query(
|
||||
`WITH sw_vendor AS (
|
||||
-- Map the switch's vendor to the Flexoptix coding-matrix token. A transceiver
|
||||
-- only truly runs in the switch if Flexoptix can code it for this platform.
|
||||
SELECT
|
||||
CASE
|
||||
WHEN v.name ILIKE '%aruba%' OR v.name ILIKE '%hpe%' OR v.name ILIKE '%hewlett%' OR v.name ILIKE '%hp h3c%' OR v.name ILIKE '%hp %' THEN 'aruba'
|
||||
WHEN v.name ILIKE '%cisco%' THEN 'cisco'
|
||||
WHEN v.name ILIKE '%arista%' THEN 'arista'
|
||||
WHEN v.name ILIKE '%juniper%' THEN 'juniper'
|
||||
WHEN v.name ILIKE '%dell%' OR v.name ILIKE '%force10%' THEN 'dell'
|
||||
WHEN v.name ILIKE '%huawei%' THEN 'huawei'
|
||||
WHEN v.name ILIKE '%fortinet%' THEN 'fortinet'
|
||||
WHEN v.name ILIKE '%mikrotik%' THEN 'mikrotik'
|
||||
WHEN v.name ILIKE '%nvidia%' OR v.name ILIKE '%mellanox%' THEN 'mellanox'
|
||||
WHEN v.name ILIKE '%extreme%' OR v.name ILIKE '%enterasys%' THEN 'extreme'
|
||||
WHEN v.name ILIKE '%nokia%' OR v.name ILIKE '%alcatel%' THEN 'nokia'
|
||||
WHEN v.name ILIKE '%brocade%' OR v.name ILIKE '%ruckus%' THEN 'brocade'
|
||||
WHEN v.name ILIKE '%netgear%' THEN 'netgear'
|
||||
WHEN v.name ILIKE '%zyxel%' THEN 'zyxel'
|
||||
WHEN v.name ILIKE '%ubiquiti%' THEN 'ubiquiti'
|
||||
WHEN v.name ILIKE '%d-link%' OR v.name ILIKE '%dlink%' THEN 'd-link'
|
||||
WHEN v.name ILIKE '%allied telesis%' THEN 'allied telesis'
|
||||
WHEN v.name ILIKE '%adtran%' THEN 'adtran'
|
||||
WHEN v.name ILIKE '%ciena%' THEN 'ciena'
|
||||
ELSE NULL -- unmapped (whitebox e.g. Edgecore) -> MSA-standard fallback
|
||||
END AS vpat
|
||||
FROM switches s JOIN vendors v ON v.id = s.vendor_id WHERE s.id = $1
|
||||
),
|
||||
gate AS (
|
||||
-- Hard gate: a switch only presents compatibility if its port config has
|
||||
-- been verified against the manufacturer datasheet/source. Unverified =
|
||||
-- no claim (return nothing) rather than show potentially-wrong data.
|
||||
SELECT ports_verified FROM switches WHERE id = $1
|
||||
),
|
||||
switch_ports AS (
|
||||
-- Parse each ports_config key 'SPEED_FORMFACTOR' (e.g. '100G_QSFP28') into
|
||||
-- a numeric port speed (Gbps) + a cage family. Real transceivers are then
|
||||
-- bounded by the port speed so corrupt/oversized records cannot leak in.
|
||||
`WITH switch_form_factors AS (
|
||||
SELECT DISTINCT
|
||||
-- numeric speed prefix: 100G->100, 25G->25, 2.5G->2.5, 100M->0.1, 800G->800
|
||||
-- speed token may be at the start ('100G_QSFP28') or embedded
|
||||
-- ('max_per_slot_100G_QSFP28' on chassis switches). Match before the cage.
|
||||
CASE
|
||||
WHEN k ~ '[0-9.]+M[_A-Za-z]' THEN (substring(k from '([0-9.]+)M[_A-Za-z]')::numeric / 1000)
|
||||
WHEN k ~ '[0-9.]+T[_A-Za-z]' THEN (substring(k from '([0-9.]+)T[_A-Za-z]')::numeric * 1000)
|
||||
WHEN k ~ '[0-9.]+G[_A-Za-z]' THEN substring(k from '([0-9.]+)G[_A-Za-z]')::numeric
|
||||
WHEN k ~ '[0-9.]+G$' THEN substring(k from '([0-9.]+)G$')::numeric
|
||||
ELSE NULL
|
||||
END AS port_speed,
|
||||
-- the port's specific cage type (determines which modules mechanically seat)
|
||||
CASE
|
||||
WHEN k ILIKE '%QSFP-DD800%' THEN 'QSFP-DD800'
|
||||
WHEN k ILIKE '%QSFP-DD%' THEN 'QSFP-DD'
|
||||
WHEN k ILIKE '%QSFP112%' THEN 'QSFP112'
|
||||
WHEN k ILIKE '%QSFP56%' THEN 'QSFP56'
|
||||
WHEN k ILIKE '%OSFP224%' THEN 'OSFP224'
|
||||
WHEN k ILIKE '%OSFP%' THEN 'OSFP'
|
||||
WHEN k ILIKE '%QSFP28%' THEN 'QSFP28'
|
||||
WHEN k ILIKE '%QSFP+%' THEN 'QSFP+'
|
||||
WHEN k ILIKE '%QSFP%' THEN 'QSFP+'
|
||||
WHEN k ILIKE '%OSFP224%' THEN 'OSFP224'
|
||||
WHEN k ILIKE '%OSFP112%' THEN 'OSFP112'
|
||||
WHEN k ILIKE '%OSFP%' THEN 'OSFP'
|
||||
WHEN k ILIKE '%SFP56%' THEN 'SFP56'
|
||||
WHEN k ILIKE '%SFP28%' THEN 'SFP28'
|
||||
WHEN k ILIKE '%SFP+%' THEN 'SFP+'
|
||||
WHEN k ILIKE '%SFP%' THEN 'SFP+'
|
||||
WHEN k ILIKE '%CFP2%' THEN 'CFP2'
|
||||
WHEN k ILIKE '%CFP4%' THEN 'CFP4'
|
||||
WHEN k ILIKE '%CFP%' THEN 'CFP'
|
||||
WHEN k ILIKE '%RJ45%' OR k ILIKE '%mGig%' THEN 'RJ45'
|
||||
ELSE NULL
|
||||
END AS port_ff
|
||||
END AS form_factor
|
||||
FROM switches sw,
|
||||
jsonb_object_keys(sw.ports_config) AS k
|
||||
WHERE sw.id = $1 AND sw.ports_config IS NOT NULL
|
||||
@ -421,50 +367,8 @@ export async function getFlexoptixSuggestions(switchId: string) {
|
||||
LIMIT 1
|
||||
) so ON true
|
||||
WHERE LOWER(v.name) = 'flexoptix'
|
||||
AND (SELECT ports_verified FROM gate) IS TRUE
|
||||
-- Only parts confirmed in Flexoptix's live API catalog (have authoritative
|
||||
-- datasheet specs). Excludes phantom/misattributed parts not actually
|
||||
-- orderable from Flexoptix, and parts with no source-of-truth specs.
|
||||
AND t.fx_specifications IS NOT NULL
|
||||
AND t.speed_gbps IS NOT NULL AND t.speed_gbps > 0
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM switch_ports sp
|
||||
WHERE sp.port_ff IS NOT NULL
|
||||
AND sp.port_speed IS NOT NULL
|
||||
-- module must mechanically seat in this port cage. A cage accepts its own
|
||||
-- type plus smaller/backward-compatible modules, never a larger one:
|
||||
-- QSFP-DD/OSFP cages take QSFP-family; QSFP28/56 cages take QSFP+/28/56
|
||||
-- (NOT QSFP-DD); SFP cages take all SFP variants. No cross-family.
|
||||
AND t.form_factor = ANY (
|
||||
CASE sp.port_ff
|
||||
WHEN 'QSFP-DD800' THEN ARRAY['QSFP-DD800','QSFP-DD','QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP-DD' THEN ARRAY['QSFP-DD','QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP112' THEN ARRAY['QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP56' THEN ARRAY['QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP28' THEN ARRAY['QSFP28','QSFP+']
|
||||
WHEN 'QSFP+' THEN ARRAY['QSFP+']
|
||||
WHEN 'OSFP224' THEN ARRAY['OSFP224','OSFP112','OSFP']
|
||||
WHEN 'OSFP112' THEN ARRAY['OSFP112','OSFP']
|
||||
WHEN 'OSFP' THEN ARRAY['OSFP']
|
||||
WHEN 'SFP56' THEN ARRAY['SFP56','SFP28','SFP+','SFP']
|
||||
WHEN 'SFP28' THEN ARRAY['SFP28','SFP+','SFP']
|
||||
WHEN 'SFP+' THEN ARRAY['SFP+','SFP']
|
||||
WHEN 'CFP2' THEN ARRAY['CFP2']
|
||||
WHEN 'CFP4' THEN ARRAY['CFP4']
|
||||
WHEN 'CFP' THEN ARRAY['CFP']
|
||||
WHEN 'RJ45' THEN ARRAY['RJ45','Copper']
|
||||
ELSE ARRAY[]::text[]
|
||||
END
|
||||
)
|
||||
-- speed must not exceed the port's speed (slower module in faster cage OK)
|
||||
AND t.speed_gbps <= sp.port_speed
|
||||
)
|
||||
-- AND Flexoptix must be able to code it for THIS switch's platform, so it
|
||||
-- actually runs (not just physically fits). Whitebox/unmapped -> MSA standard.
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(t.fx_compatibilities) c, sw_vendor
|
||||
WHERE (sw_vendor.vpat IS NOT NULL AND c->>'compatible_to_vendor' ILIKE '%' || sw_vendor.vpat || '%')
|
||||
OR (sw_vendor.vpat IS NULL AND c->>'compatible_to_vendor' ILIKE '%MSA Standard%')
|
||||
AND t.form_factor IN (
|
||||
SELECT form_factor FROM switch_form_factors WHERE form_factor IS NOT NULL
|
||||
)
|
||||
ORDER BY t.speed_gbps DESC NULLS LAST, t.reach_meters ASC NULLS LAST`,
|
||||
[switchId]
|
||||
|
||||
@ -28,7 +28,6 @@ import { procurementRouter } from "./routes/procurement";
|
||||
import { changelogRouter } from "./routes/changelog";
|
||||
import { newsRouter } from "./routes/news";
|
||||
import { proxyRouter } from "./routes/proxy";
|
||||
import { researchRobotRouter } from "./routes/research-robot";
|
||||
import { reviewRouter } from "./routes/review";
|
||||
import { stockRouter } from "./routes/stock";
|
||||
import { priceComparisonRouter } from "./routes/price-comparison";
|
||||
@ -38,18 +37,12 @@ import { formFactorsRouter } from "./routes/form-factors";
|
||||
import { tipLlmRouter } from "./routes/tip-llm";
|
||||
import { equivalencesRouter } from "./routes/equivalences";
|
||||
import { priceHistoryRouter } from "./routes/price-history";
|
||||
import { stockCompetitorRouter } from "./routes/stock-competitor";
|
||||
import { kbRouter } from "./routes/kb";
|
||||
import { bulkPriceRouter } from "./routes/bulk-price";
|
||||
import { vendorReliabilityRouter } from "./routes/vendor-reliability";
|
||||
import { priceForecastRouter } from "./routes/price-forecast";
|
||||
import { priceMatrixRouter } from "./routes/price-matrix";
|
||||
import { trainingRouter } from "./routes/training";
|
||||
import { rfqRouter } from "./routes/rfq";
|
||||
import { priceAlertsRouter } from "./routes/price-alerts";
|
||||
import { winLossRouter } from "./routes/win-loss";
|
||||
import { apiKeysRouter } from "./routes/api-keys";
|
||||
import { roiRouter } from "./routes/roi";
|
||||
|
||||
const app = express();
|
||||
|
||||
@ -59,7 +52,7 @@ app.set("trust proxy", 1);
|
||||
// Middleware
|
||||
app.use(helmet({ contentSecurityPolicy: false }));
|
||||
app.use(cors());
|
||||
app.use(express.json({ limit: "30mb" })); // 30MB to support base64-encoded PDF uploads
|
||||
app.use(express.json());
|
||||
app.use(
|
||||
rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
@ -74,7 +67,6 @@ app.use("/api/auth", authRouter);
|
||||
|
||||
// Proxy public endpoints (register + heartbeat + stats + next — no auth)
|
||||
app.use("/api/proxy", proxyRouter);
|
||||
app.use("/api/research-robot", researchRobotRouter);
|
||||
|
||||
// All other API routes require a valid token
|
||||
app.use("/api", (req, res, next) => {
|
||||
@ -123,7 +115,6 @@ app.use("/api/tip-llm", tipLlmRouter);
|
||||
app.use("/api/equivalences", equivalencesRouter);
|
||||
// Price history charts
|
||||
app.use("/api/price-history", priceHistoryRouter);
|
||||
app.use("/api/stock", stockCompetitorRouter);
|
||||
app.use("/api/kb", kbRouter);
|
||||
// Bulk price lookup (G)
|
||||
app.use("/api/bulk-price", bulkPriceRouter);
|
||||
@ -135,16 +126,6 @@ app.use("/api/price-forecast", priceForecastRouter);
|
||||
app.use("/api/price-matrix", priceMatrixRouter);
|
||||
// Transceiver Academy — public training content (no auth required)
|
||||
app.use("/api/training", trainingRouter);
|
||||
// RFQ Analyzer — quote vs market comparison
|
||||
app.use("/api/rfq", rfqRouter);
|
||||
// Price Alert Subscriptions
|
||||
app.use("/api/price-alerts", priceAlertsRouter);
|
||||
// Win/Loss Intelligence
|
||||
app.use("/api/win-loss", winLossRouter);
|
||||
// Customer API Key Management
|
||||
app.use("/api/api-keys", apiKeysRouter);
|
||||
// ROI Calculator
|
||||
app.use("/api/roi", roiRouter);
|
||||
|
||||
// Dashboard (static HTML)
|
||||
app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard")));
|
||||
|
||||
@ -14,27 +14,7 @@
|
||||
import { existsSync, readFileSync, writeFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
|
||||
// -- Ollama endpoint with automatic primary->fallback failover --
|
||||
// Primary = BLOG_OLLAMA_URL (WireGuard tunnel to Mac Studio loopback Ollama),
|
||||
// fallback = BLOG_OLLAMA_URL_FALLBACK (Cloudflare tunnel). Independent transports;
|
||||
// if the primary transport drops, requests move to the backup. Re-probed every 60s.
|
||||
const OLLAMA_PRIMARY = process.env.BLOG_OLLAMA_URL || process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
const OLLAMA_FALLBACK = process.env.BLOG_OLLAMA_URL_FALLBACK || "";
|
||||
let OLLAMA_URL = OLLAMA_PRIMARY;
|
||||
async function pickOllamaUrl(): Promise<void> {
|
||||
for (const u of [OLLAMA_PRIMARY, OLLAMA_FALLBACK].filter(Boolean)) {
|
||||
try {
|
||||
const r = await fetch(`${u}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
||||
if (r.ok) {
|
||||
if (OLLAMA_URL !== u) console.log(`[blog-llm] ollama endpoint -> ${u}`);
|
||||
OLLAMA_URL = u;
|
||||
return;
|
||||
}
|
||||
} catch { /* try next */ }
|
||||
}
|
||||
}
|
||||
void pickOllamaUrl();
|
||||
setInterval(() => { void pickOllamaUrl(); }, 60000).unref();
|
||||
const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || "";
|
||||
const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-sonnet-4-20250514";
|
||||
const CLAUDE_BRIDGE_URL = process.env.CLAUDE_BRIDGE_URL || "http://localhost:3250";
|
||||
@ -53,12 +33,7 @@ const SETTINGS_FILE = join(process.env.TIP_ROOT || "/opt/tip", "blog-llm-setting
|
||||
const STATIC_FALLBACK_MODEL = "fo-blog-v10";
|
||||
const DISCOVERY_REFRESH_MS = Number.parseInt(process.env.BLOG_LLM_DISCOVERY_REFRESH_MS || "", 10) || 10 * 60_000;
|
||||
|
||||
interface LlmSettings {
|
||||
provider: string;
|
||||
ollamaModel: string;
|
||||
/** When set, auto-upgrade is disabled and this exact version is used. */
|
||||
pinnedVersion?: string;
|
||||
}
|
||||
interface LlmSettings { provider: string; ollamaModel: string }
|
||||
|
||||
function loadSettingsRaw(): LlmSettings {
|
||||
try {
|
||||
@ -67,7 +42,6 @@ function loadSettingsRaw(): LlmSettings {
|
||||
return {
|
||||
provider: raw.provider || process.env.BLOG_LLM_PROVIDER || "ollama",
|
||||
ollamaModel: raw.ollamaModel || process.env.OLLAMA_LLM_MODEL || STATIC_FALLBACK_MODEL,
|
||||
pinnedVersion: raw.pinnedVersion || undefined,
|
||||
};
|
||||
}
|
||||
} catch { /* ignore corrupt file */ }
|
||||
@ -126,46 +100,25 @@ async function fetchOllamaFoBlogTags(): Promise<string[]> {
|
||||
/**
|
||||
* Reconcile configured model against Ollama reality.
|
||||
*
|
||||
* Always upgrades to the highest available fo-blog-vN BASE tag (no -r suffix).
|
||||
* This ensures newly-trained versions are picked up automatically within 10 min,
|
||||
* without needing to delete old Ollama tags or restart the API.
|
||||
*
|
||||
* Priority:
|
||||
* 1. Highest fo-blog-vN base tag Ollama actually serves (auto-upgrade)
|
||||
* 2. Configured model — if no upgrade candidate found
|
||||
* 1. Configured model (env or settings file) — if Ollama actually serves it
|
||||
* 2. Highest fo-blog-v* version Ollama actually serves — auto-discovered
|
||||
* 3. Static fallback STATIC_FALLBACK_MODEL — last resort
|
||||
*
|
||||
* Non-blocking: any Ollama failure leaves _settings untouched.
|
||||
*/
|
||||
async function reconcileWithOllama(): Promise<void> {
|
||||
// Skip auto-upgrade when a version is explicitly pinned
|
||||
if (_settings.pinnedVersion) return;
|
||||
|
||||
const configured = _settings.ollamaModel;
|
||||
if (!configured.startsWith("fo-blog-v")) return; // only manage fo-blog-* lane
|
||||
|
||||
const available = await fetchOllamaFoBlogTags();
|
||||
if (available.length === 0) return;
|
||||
if (available.includes(configured)) return; // configured model still exists
|
||||
|
||||
// Pick the highest base version (no -r suffix) available in Ollama
|
||||
const sorted = [...available].sort(compareFoBlogVersionsDesc);
|
||||
const winner = sorted[0];
|
||||
if (!winner || winner === configured) return; // already on best, or nothing to do
|
||||
if (!winner || winner === configured) return;
|
||||
|
||||
// Only upgrade (never downgrade): winner must have a higher major version
|
||||
const re = /^fo-blog-v(\d+)(?:-r(\d+))?$/;
|
||||
const mc = re.exec(configured);
|
||||
const mw = re.exec(winner);
|
||||
if (mc && mw) {
|
||||
const vc = Number.parseInt(mc[1], 10);
|
||||
const vw = Number.parseInt(mw[1], 10);
|
||||
if (vw <= vc) return; // winner is not newer — no-op
|
||||
}
|
||||
|
||||
const reason = available.includes(configured)
|
||||
? `newer version available`
|
||||
: `"${configured}" no longer in Ollama`;
|
||||
console.log(`[LLM] auto-upgrade: "${configured}" → "${winner}" (${reason}; candidates: ${sorted.join(", ")})`);
|
||||
console.log(`[LLM] auto-discovery: configured "${configured}" not in Ollama; switching to latest available "${winner}" (candidates: ${sorted.join(", ")})`);
|
||||
_settings = { ..._settings, ollamaModel: winner };
|
||||
try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ }
|
||||
}
|
||||
@ -176,43 +129,11 @@ let _settings = loadSettingsRaw();
|
||||
void reconcileWithOllama();
|
||||
setInterval(() => { void reconcileWithOllama(); }, DISCOVERY_REFRESH_MS).unref();
|
||||
|
||||
/**
|
||||
* Switch the active LLM provider at runtime. Persists to settings file.
|
||||
* Switching provider/model clears any existing pin so auto-upgrade can resume
|
||||
* on the new provider — unless the caller explicitly passes a pinnedVersion.
|
||||
*/
|
||||
export function setLlmProvider(provider: string, ollamaModel?: string, pinnedVersion?: string): void {
|
||||
_settings = {
|
||||
..._settings, // preserve any fields not explicitly overridden
|
||||
provider,
|
||||
ollamaModel: ollamaModel || _settings.ollamaModel,
|
||||
pinnedVersion: pinnedVersion ?? undefined, // explicit undefined clears pin on provider switch
|
||||
};
|
||||
/** Switch the active LLM provider at runtime. Persists to settings file. */
|
||||
export function setLlmProvider(provider: string, ollamaModel?: string): void {
|
||||
_settings = { provider, ollamaModel: ollamaModel || _settings.ollamaModel };
|
||||
try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ }
|
||||
console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}${pinnedVersion ? ` [pinned]` : ""}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin the active fo-blog version, disabling auto-upgrade.
|
||||
* The model stays at `version` until explicitly unpinned.
|
||||
*/
|
||||
export function pinLlmVersion(version: string): void {
|
||||
_settings = { ..._settings, ollamaModel: version, pinnedVersion: version };
|
||||
try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ }
|
||||
console.log(`[LLM] Version pinned → ${version} (auto-upgrade disabled)`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the version pin — auto-upgrade resumes on next reconcile interval.
|
||||
* Triggers an immediate reconcile so the highest available version is adopted
|
||||
* without waiting up to DISCOVERY_REFRESH_MS.
|
||||
*/
|
||||
export async function unpinLlmVersion(): Promise<LlmSettings> {
|
||||
_settings = { ..._settings, pinnedVersion: undefined };
|
||||
try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ }
|
||||
console.log("[LLM] Version unpinned — auto-upgrade re-enabled");
|
||||
await reconcileWithOllama();
|
||||
return { ..._settings };
|
||||
console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}`);
|
||||
}
|
||||
|
||||
/** Returns the currently active provider config. */
|
||||
|
||||
@ -1,144 +0,0 @@
|
||||
/**
|
||||
* Customer API Key Management — /api/api-keys
|
||||
*
|
||||
* Manages externally-issued API keys for customer access to the TIP public API.
|
||||
* Keys are stored as SHA-256 hashes. The actual key is shown ONCE at creation.
|
||||
*
|
||||
* Routes:
|
||||
* POST /api/api-keys — Issue a new key (admin-only in prod)
|
||||
* GET /api/api-keys — List keys (filter by email)
|
||||
* DELETE /api/api-keys/:id — Revoke a key
|
||||
* GET /api/api-keys/stats — Usage stats per key
|
||||
* POST /api/api-keys/validate — Validate a key (internal use by middleware)
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { createHash, randomBytes } from "crypto";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const apiKeysRouter = Router();
|
||||
|
||||
function hashKey(key: string): string {
|
||||
return createHash("sha256").update(key).digest("hex");
|
||||
}
|
||||
|
||||
function generateKey(): { key: string; prefix: string; hash: string } {
|
||||
const raw = randomBytes(24).toString("base64url");
|
||||
const key = `tip_${raw}`;
|
||||
const prefix = key.slice(0, 12);
|
||||
const hash = hashKey(key);
|
||||
return { key, prefix, hash };
|
||||
}
|
||||
|
||||
// ── POST /api/api-keys — Issue new key ──────────────────────────────────────
|
||||
apiKeysRouter.post("/", async (req: Request, res: Response) => {
|
||||
const { email, label, tier = "free", rate_limit, expires_in_days } = req.body as Record<string, any>;
|
||||
|
||||
if (!email || !email.includes("@")) {
|
||||
return res.status(400).json({ success: false, error: "Valid email required" });
|
||||
}
|
||||
if (!label || typeof label !== "string") {
|
||||
return res.status(400).json({ success: false, error: "label required" });
|
||||
}
|
||||
|
||||
const RATE_LIMITS: Record<string, number> = { free: 100, pro: 1000, enterprise: 10000 };
|
||||
const resolvedRateLimit = rate_limit ? parseInt(rate_limit) : RATE_LIMITS[tier] || 100;
|
||||
const expiresAt = expires_in_days
|
||||
? new Date(Date.now() + parseInt(expires_in_days) * 86400000).toISOString()
|
||||
: null;
|
||||
|
||||
try {
|
||||
const { key, prefix, hash } = generateKey();
|
||||
const result = await pool.query(
|
||||
`INSERT INTO api_keys (key_hash, key_prefix, label, email, tier, rate_limit, expires_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||
RETURNING id, key_prefix, label, email, tier, rate_limit, active, created_at, expires_at`,
|
||||
[hash, prefix, label, email.toLowerCase().trim(), tier, resolvedRateLimit, expiresAt]
|
||||
);
|
||||
|
||||
return res.status(201).json({
|
||||
success: true,
|
||||
api_key: key, // shown ONCE — client must store it
|
||||
warning: "Store this key now — it will not be shown again.",
|
||||
meta: result.rows[0],
|
||||
});
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/api-keys?email= — List keys ────────────────────────────────────
|
||||
apiKeysRouter.get("/", async (req: Request, res: Response) => {
|
||||
const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase();
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`SELECT id, key_prefix, label, email, tier, rate_limit, active,
|
||||
last_used_at, usage_count, created_at, expires_at
|
||||
FROM api_keys
|
||||
WHERE ($1 = '' OR email = $1)
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 100`,
|
||||
[email]
|
||||
);
|
||||
return res.json({ success: true, keys: result.rows });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── DELETE /api/api-keys/:id — Revoke key ───────────────────────────────────
|
||||
apiKeysRouter.delete("/:id", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`UPDATE api_keys SET active = false WHERE id = $1 RETURNING id, key_prefix`,
|
||||
[parseInt(String(req.params.id))]
|
||||
);
|
||||
if (result.rowCount === 0) return res.status(404).json({ success: false, error: "Key not found" });
|
||||
return res.json({ success: true, revoked: result.rows[0] });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/api-keys/stats — Usage dashboard ───────────────────────────────
|
||||
apiKeysRouter.get("/stats", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
tier,
|
||||
COUNT(*) AS total_keys,
|
||||
COUNT(*) FILTER (WHERE active) AS active_keys,
|
||||
SUM(usage_count) AS total_requests,
|
||||
MAX(last_used_at) AS last_activity
|
||||
FROM api_keys
|
||||
GROUP BY tier
|
||||
ORDER BY total_requests DESC
|
||||
`);
|
||||
return res.json({ success: true, stats: result.rows });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── POST /api/api-keys/validate — Validate key (used by auth middleware) ────
|
||||
apiKeysRouter.post("/validate", async (req: Request, res: Response) => {
|
||||
const { key } = req.body as { key?: string };
|
||||
if (!key) return res.status(400).json({ valid: false });
|
||||
|
||||
try {
|
||||
const hash = hashKey(key);
|
||||
const result = await pool.query(
|
||||
`UPDATE api_keys
|
||||
SET last_used_at = NOW(), usage_count = usage_count + 1
|
||||
WHERE key_hash = $1
|
||||
AND active = true
|
||||
AND (expires_at IS NULL OR expires_at > NOW())
|
||||
RETURNING id, key_prefix, email, tier, rate_limit`,
|
||||
[hash]
|
||||
);
|
||||
if (result.rowCount === 0) return res.json({ valid: false });
|
||||
return res.json({ valid: true, ...result.rows[0] });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ valid: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -10,11 +10,8 @@
|
||||
* Voice: Senior optical network engineer, not marketing.
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import * as pdfParseModule from "pdf-parse";
|
||||
const pdfParse: (buffer: Buffer) => Promise<{ text: string; numpages: number; info: Record<string, unknown> }> =
|
||||
(pdfParseModule as any).default ?? (pdfParseModule as any);
|
||||
import { pool } from "../db/client";
|
||||
import { setLlmProvider, getLlmProvider, refreshLlmAutoDiscovery, pinLlmVersion, unpinLlmVersion } from "../llm/client";
|
||||
import { setLlmProvider, getLlmProvider, refreshLlmAutoDiscovery } from "../llm/client";
|
||||
|
||||
/** In-memory pipeline progress tracker — step updates pushed here, polled via GET /api/blog/:id/progress */
|
||||
const pipelineProgress = new Map<string, { step: number; total: number; label: string; pct: number }>();
|
||||
@ -987,177 +984,6 @@ async function processLlmQueue(): Promise<void> {
|
||||
if (llmQueue.length > 0) setTimeout(() => processLlmQueue(), 3000);
|
||||
}
|
||||
|
||||
/**
|
||||
* 3-pass pipeline for external content (from-url / from-pdf).
|
||||
* Grounds entirely in source text — never expands from parametric knowledge,
|
||||
* which prevents the "senior network engineer" persona from drifting to optics.
|
||||
*/
|
||||
async function runExternalContentPipeline(
|
||||
draftId: string,
|
||||
title: string,
|
||||
selectedTopic: string,
|
||||
targetAudience: string,
|
||||
additionalContext: string,
|
||||
): Promise<void> {
|
||||
const {
|
||||
FO_BLOG_SYSTEM_PROMPT,
|
||||
STEP8_KILL_AI_TONE,
|
||||
STEP8c_STYLE_LOCK,
|
||||
STEP_HEADLINE_GENERATION,
|
||||
withCalibration,
|
||||
buildFeedbackContext,
|
||||
} = await import("../llm/fo-blog-pipeline");
|
||||
|
||||
const LLM_WRITE = { temperature: 0.7, maxTokens: 2000, timeoutMs: 480000 };
|
||||
const LLM_REFINE = { temperature: 0.35, maxTokens: 2000, timeoutMs: 480000 };
|
||||
|
||||
// Extract persona-neutral section of FO system prompt (keep writing style, drop optics persona)
|
||||
const mindsetMarker = "YOUR MINDSET:";
|
||||
const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker);
|
||||
const writingStyleRules = mindsetStart > -1
|
||||
? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart)
|
||||
: FO_BLOG_SYSTEM_PROMPT;
|
||||
|
||||
// Load feedback
|
||||
let feedbackContext = "";
|
||||
try {
|
||||
const fbResult = await pool.query(
|
||||
`SELECT score_overall, feedback_text, blog_type FROM blog_feedback
|
||||
WHERE feedback_text IS NOT NULL AND feedback_text != ''
|
||||
ORDER BY score_overall ASC LIMIT 20`
|
||||
);
|
||||
feedbackContext = buildFeedbackContext(fbResult.rows.map(r => ({
|
||||
score: r.score_overall, feedback_text: r.feedback_text, blog_type: r.blog_type || ""
|
||||
})));
|
||||
} catch { /* no feedback yet */ }
|
||||
|
||||
// Extract just the PDF/URL text from additionalContext
|
||||
const pdfStart = additionalContext.indexOf("--- EXTRACTED PDF CONTENT ---");
|
||||
const urlStart = additionalContext.indexOf("--- EXTRACTED PAGE CONTENT ---");
|
||||
const contentStart = pdfStart > -1 ? pdfStart : urlStart > -1 ? urlStart : -1;
|
||||
const sourceText = contentStart > -1
|
||||
? additionalContext.slice(contentStart).slice(0, 5000)
|
||||
: additionalContext.slice(0, 5000);
|
||||
|
||||
const externalSysPrompt = withCalibration(
|
||||
`You are a senior IT infrastructure engineer and technical writer with 20+ years of experience.\n` +
|
||||
`You write practical articles for IT architects, infrastructure managers, and decision-makers.\n\n` +
|
||||
`ABSOLUTE RULE: You write ONLY about what the source document says. ` +
|
||||
`Do NOT add optical transceivers, fiber optics, 400G, or compatible optics content ` +
|
||||
`unless the source document explicitly covers it. ` +
|
||||
`Your only source of facts is the document provided.\n\n` +
|
||||
`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` +
|
||||
`WRITING STYLE (apply to everything you write):\n` +
|
||||
`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` +
|
||||
writingStyleRules + feedbackContext
|
||||
);
|
||||
|
||||
console.log(`Blog External Pipeline: Starting for ${draftId} — "${title}"`);
|
||||
setProgress(draftId, 1, "Step 1/5: Extract source facts");
|
||||
|
||||
// ─── Pass 1: Extract key facts from source document ───────────────────────
|
||||
const extractPrompt =
|
||||
`Extract the key factual content from the source document below.\n\n` +
|
||||
`Return:\n` +
|
||||
`- The core problem or challenge the document describes (2-3 sentences)\n` +
|
||||
`- 5-7 specific facts, findings, or insights stated in the document\n` +
|
||||
`- The main conclusion or practical recommendation\n` +
|
||||
`- Who this affects and why it matters to them\n\n` +
|
||||
`Do NOT add information not present in the document. Do NOT interpret or expand.\n` +
|
||||
`Return only what the document actually says.\n\n` +
|
||||
sourceText;
|
||||
|
||||
await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 }).catch(() => {});
|
||||
const extractResult = await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 });
|
||||
|
||||
setProgress(draftId, 2, "Step 2/5: Write article draft");
|
||||
|
||||
// ─── Pass 2: Write the full article from extracted facts ──────────────────
|
||||
const draftPrompt =
|
||||
`Write a blog article titled "${title}".\n\n` +
|
||||
`Use ONLY the facts below as your source material. Do not add any facts not listed here.\n\n` +
|
||||
`SOURCE FACTS:\n${extractResult.text}\n\n` +
|
||||
`ARTICLE REQUIREMENTS:\n` +
|
||||
`- 600-900 words\n` +
|
||||
`- Continuous narrative prose — no section headers, no bullet lists in the body\n` +
|
||||
`- First-person engineering voice ("I've seen this happen...", "The problem is...")\n` +
|
||||
`- Start with a specific operational scenario, not a general statement\n` +
|
||||
`- Explain consequences for real infrastructure decisions\n` +
|
||||
`- End with a concrete implication or call to action, not a summary\n` +
|
||||
`- Apply all writing style rules from your system prompt\n\n` +
|
||||
`Topic: "${title}"\n` +
|
||||
`Audience: ${targetAudience} engineers and infrastructure decision-makers`;
|
||||
|
||||
const draftResult = await generate(externalSysPrompt, draftPrompt, LLM_WRITE);
|
||||
console.log(` Draft: ${draftResult.text.split(/\s+/).length} words`);
|
||||
|
||||
setProgress(draftId, 3, "Step 3/5: Kill AI tone");
|
||||
|
||||
// ─── Pass 3: Kill AI tone ─────────────────────────────────────────────────
|
||||
const step8 = await generate(externalSysPrompt,
|
||||
STEP8_KILL_AI_TONE.replace("{{ARTICLE}}", draftResult.text),
|
||||
LLM_REFINE
|
||||
);
|
||||
|
||||
setProgress(draftId, 4, "Step 4/5: Style lock");
|
||||
|
||||
// Skip STEP8b_REDUCTION for external content — it targets 1,200-2,000 words
|
||||
// with "DO NOT go below 1,000 words" which conflicts with thin source material.
|
||||
// Just apply style lock for readability polish.
|
||||
const step8c = await generate(externalSysPrompt,
|
||||
STEP8c_STYLE_LOCK.replace("{{ARTICLE}}", step8.text),
|
||||
LLM_REFINE
|
||||
);
|
||||
|
||||
setProgress(draftId, 5, "Step 5/5: LinkedIn + headline");
|
||||
|
||||
// ─── LinkedIn post — topic-neutral version (no optics example) ────────────
|
||||
const externalLinkedInPrompt =
|
||||
`Write a LinkedIn post for this article.\n\n` +
|
||||
`FORMAT:\n` +
|
||||
`Line 1-2: HOOK — a reframe or uncomfortable truth from the article. NOT an announcement.\n\n` +
|
||||
`3-5 SHORT BEATS — each beat is 1-3 lines. One insight per beat. No bullet markers.\n\n` +
|
||||
`Last line before hashtags: "Full breakdown in the blog — link in first comment."\n\n` +
|
||||
`HASHTAGS (last line): 3-4 relevant hashtags based on the article topic. Include #Flexoptix.\n` +
|
||||
` Pick hashtags that match what the article is actually about — NOT #OpticalNetworking unless the article covers optics.\n\n` +
|
||||
`RULES:\n` +
|
||||
`- No emojis\n` +
|
||||
`- No "I'm thrilled to share" or "Excited to announce"\n` +
|
||||
`- Engineer voice — specific, blunt, useful\n` +
|
||||
`- Maximum 2,800 characters\n` +
|
||||
`- Return ONLY the post text. No commentary.\n\n` +
|
||||
`Article:\n${step8c.text}`;
|
||||
|
||||
const linkedInResult = await generate(externalSysPrompt,
|
||||
externalLinkedInPrompt,
|
||||
{ ...LLM_REFINE, maxTokens: 600 }
|
||||
).catch(() => ({ text: "" }));
|
||||
|
||||
// ─── Headline ──────────────────────────────────────────────────────────────
|
||||
const headlineResult = await generate(externalSysPrompt,
|
||||
STEP_HEADLINE_GENERATION.replace("{{ARTICLE}}", step8c.text),
|
||||
{ temperature: 0.5, maxTokens: 80, timeoutMs: 60000 }
|
||||
).catch(() => ({ text: title }));
|
||||
|
||||
const finalTitle = headlineResult.text.trim().replace(/^["']|["']$/g, "").replace(/\n.*$/s, "").trim() || title;
|
||||
const wordCount = step8c.text.split(/\s+/).length;
|
||||
|
||||
await pool.query(
|
||||
`UPDATE blog_drafts SET
|
||||
title = $1,
|
||||
draft_content = $2,
|
||||
linkedin_post = $3,
|
||||
word_count = $4,
|
||||
status = 'draft',
|
||||
updated_at = NOW()
|
||||
WHERE id = $5`,
|
||||
[finalTitle, step8c.text, linkedInResult.text || null, wordCount, draftId]
|
||||
);
|
||||
|
||||
clearProgress(draftId);
|
||||
console.log(`Blog External Pipeline: ${draftId} complete — ${wordCount} words, title: "${finalTitle}"`);
|
||||
}
|
||||
|
||||
/** Run 10-Step Flexoptix Style LLM Pipeline and update draft in-place */
|
||||
async function runLlmPipeline(
|
||||
draftId: string,
|
||||
@ -1167,12 +993,6 @@ async function runLlmPipeline(
|
||||
data: Awaited<ReturnType<typeof gatherBlogData>>,
|
||||
additionalContext?: string,
|
||||
): Promise<void> {
|
||||
// External content (from-url / from-pdf) uses a grounded 5-pass pipeline
|
||||
// that never expands from parametric knowledge — prevents optics topic drift
|
||||
if (additionalContext?.startsWith("⚠️ TOPIC LOCK") && additionalContext.length > 200) {
|
||||
return runExternalContentPipeline(draftId, title, selectedTopic, targetAudience, additionalContext);
|
||||
}
|
||||
|
||||
// Lazy-load the new FO pipeline
|
||||
const {
|
||||
FO_BLOG_SYSTEM_PROMPT,
|
||||
@ -1220,48 +1040,7 @@ async function runLlmPipeline(
|
||||
})));
|
||||
} catch { /* no feedback yet, that's fine */ }
|
||||
|
||||
// For external content (from-url / from-pdf), the Flexoptix optical-networking persona
|
||||
// must be replaced — otherwise every article drifts back to 400G transceivers regardless
|
||||
// of what the TOPIC LOCK says. Strip the Flexoptix mandate and inject a generic persona.
|
||||
const isExternalContent = additionalContext?.startsWith("⚠️ TOPIC LOCK");
|
||||
const extractedTopicName = isExternalContent
|
||||
? (additionalContext?.match(/TOPIC LOCK[^"]*"([^"]+)"/) || [])[1] || title
|
||||
: title;
|
||||
|
||||
// Build the section of FO_BLOG_SYSTEM_PROMPT that's topic-neutral (writing style rules only).
|
||||
// The Flexoptix persona block ends at the first ════ separator after line 65 ("YOUR MINDSET").
|
||||
const mindsetMarker = "YOUR MINDSET:";
|
||||
const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker);
|
||||
const writingStyleRules = mindsetStart > -1
|
||||
? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart)
|
||||
: FO_BLOG_SYSTEM_PROMPT;
|
||||
|
||||
const externalSystemPrompt = `\
|
||||
╔══════════════════════════════════════════════════════════════════════╗
|
||||
║ EXTERNAL CONTENT MODE — these rules override ALL defaults below ║
|
||||
╚══════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
THIS ARTICLE IS ABOUT: "${extractedTopicName}"
|
||||
|
||||
You are a senior IT infrastructure engineer and technical writer.
|
||||
Your readers are IT architects, infrastructure managers, and decision-makers.
|
||||
|
||||
ABSOLUTE RULE: Write ONLY about "${extractedTopicName}".
|
||||
Do NOT write about optical transceivers, fiber optics, 400G, DR4, compatible optics,
|
||||
Flexoptix products, or any networking hardware UNLESS the source document below
|
||||
explicitly covers that topic. The source document is your sole editorial mandate.
|
||||
|
||||
The Flexoptix brand rules and compatible-optics framing in this prompt DO NOT APPLY
|
||||
to this article. This is a general IT/infrastructure piece, not an optics blog post.
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
WRITING STYLE (applies to all articles — keep these):
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
${writingStyleRules}`;
|
||||
|
||||
const systemPrompt = isExternalContent
|
||||
? withCalibration(externalSystemPrompt + feedbackContext)
|
||||
: withCalibration(FO_BLOG_SYSTEM_PROMPT + feedbackContext);
|
||||
const systemPrompt = withCalibration(FO_BLOG_SYSTEM_PROMPT + feedbackContext);
|
||||
|
||||
// Warmup
|
||||
await generate("Test", "OK", { temperature: 0.1, maxTokens: 8, timeoutMs: 60000 }).catch(() => {});
|
||||
@ -1321,23 +1100,11 @@ ${writingStyleRules}`;
|
||||
// ═══ STEP 1: Topic Expansion ═══
|
||||
console.log(" Step 1/10: Topic Expansion...");
|
||||
setProgress(draftId, 1, "Step 1/10: Topic Expansion");
|
||||
|
||||
// For external content: inject a hard topic anchor before the standard prompt so the
|
||||
// LLM cannot drift back to optical networking when expanding the topic in Step 1.
|
||||
const step1TopicPrefix = isExternalContent
|
||||
? `HARD TOPIC LOCK: This article is about "${extractedTopicName}". ` +
|
||||
`It is NOT about optical transceivers, fiber optics, 400G migrations, or compatible optics. ` +
|
||||
`Your expansion below must stay strictly within the topic stated above.\n\n`
|
||||
: "";
|
||||
|
||||
const step1 = await generate(systemPrompt,
|
||||
step1TopicPrefix + STEP1_TOPIC_EXPANSION
|
||||
.replace("{{TOPIC}}", isExternalContent ? extractedTopicName : title)
|
||||
STEP1_TOPIC_EXPANSION
|
||||
.replace("{{TOPIC}}", title)
|
||||
.replace("{{ADDITIONAL_CONTEXT}}", additionalContext
|
||||
? `\n\n---\nBACKGROUND REFERENCE (use as factual direction ONLY — do not copy verbatim):\n${additionalContext.slice(0, 4000)}\n\n` +
|
||||
(isExternalContent
|
||||
? `REMINDER: Write about "${extractedTopicName}" — NOT optical networking. The background above is your source material.`
|
||||
: `CRITICAL: Do NOT copy any phrase, sentence, or wording from the above into the article or any step output.`)
|
||||
? `\n\n---\nBACKGROUND REFERENCE (editorial context — use as factual direction ONLY):\n${additionalContext}\n\nCRITICAL: Do NOT copy any phrase, sentence, or wording from the above into the article or any step output. It is context for your understanding, not source material.`
|
||||
: ""),
|
||||
LLM_OPTS
|
||||
);
|
||||
@ -1764,34 +1531,15 @@ blogRouter.post("/generate", async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/** Paywall signal patterns in page HTML */
|
||||
const PAYWALL_PATTERNS = [
|
||||
/class=["'][^"']*paywall[^"']*["']/i,
|
||||
/id=["'][^"']*paywall[^"']*["']/i,
|
||||
/"paywall"\s*:/i,
|
||||
/data-paywall/i,
|
||||
/subscribe (to|now) (read|access|continue)/i,
|
||||
/sign[- ]?in to (read|continue|access)/i,
|
||||
/log[- ]?in to (read|continue|access)/i,
|
||||
/create (a free )?account to (read|continue|access)/i,
|
||||
/register (now )?to (read|continue|access)/i,
|
||||
/this (article|content|paper) is (for|available to) (subscribers?|members?)/i,
|
||||
/premium (content|article|paper)/i,
|
||||
/access (this|the) (full )?(article|content|paper)/i,
|
||||
/metered[- ]?content/i,
|
||||
/subscriber[- ]?only/i,
|
||||
/intel(ligence)?\.theregister\.com\/paper/i,
|
||||
];
|
||||
|
||||
/** Fetch a URL and extract readable text content for use as LLM context.
|
||||
* Returns spaDetected=true when extracted body text is thin (< 300 chars).
|
||||
* Returns paywallDetected=true when login/subscription wall signals are found.
|
||||
* Returns spaDetected=true when extracted body text is thin (< 300 chars),
|
||||
* indicating a JavaScript Single Page Application where content is rendered client-side.
|
||||
* In that case, metaDesc contains OG/meta description fallback text.
|
||||
*/
|
||||
async function fetchUrlContent(rawUrl: string): Promise<{
|
||||
pageTitle: string;
|
||||
text: string;
|
||||
spaDetected: boolean;
|
||||
paywallDetected: boolean;
|
||||
metaDesc: string;
|
||||
}> {
|
||||
const response = await fetch(rawUrl, {
|
||||
@ -1868,11 +1616,8 @@ async function fetchUrlContent(rawUrl: string): Promise<{
|
||||
// Detect SPA: very little body text means JS renders the real content
|
||||
const spaDetected = text.length < 300;
|
||||
|
||||
// Detect paywall: check raw HTML for subscription/login wall signals
|
||||
const paywallDetected = PAYWALL_PATTERNS.some(p => p.test(html.slice(0, 20000)));
|
||||
|
||||
// When SPA/paywall detected, enrich text with what we could extract from meta tags
|
||||
if ((spaDetected || paywallDetected) && (metaDesc || ogSiteName)) {
|
||||
// When SPA detected, enrich text with what we could extract from meta tags
|
||||
if (spaDetected && (metaDesc || ogSiteName)) {
|
||||
const parts: string[] = [];
|
||||
if (ogSiteName) parts.push(`Site: ${ogSiteName}`);
|
||||
if (pageTitle) parts.push(`Title: ${pageTitle}`);
|
||||
@ -1880,7 +1625,7 @@ async function fetchUrlContent(rawUrl: string): Promise<{
|
||||
text = parts.join("\n");
|
||||
}
|
||||
|
||||
return { pageTitle, text, spaDetected, paywallDetected, metaDesc };
|
||||
return { pageTitle, text, spaDetected, metaDesc };
|
||||
}
|
||||
|
||||
// POST /api/blog/from-url — Fetch URL, extract content, generate a blog from it
|
||||
@ -1911,35 +1656,13 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => {
|
||||
|
||||
try {
|
||||
// Fetch page content server-side (no CORS issues)
|
||||
const { pageTitle, text: extractedText, spaDetected, paywallDetected, metaDesc } = await fetchUrlContent(url);
|
||||
const { pageTitle, text: extractedText, spaDetected, metaDesc } = await fetchUrlContent(url);
|
||||
|
||||
console.log(
|
||||
`Blog from-url: fetched "${pageTitle}" from ${parsedUrl.hostname} ` +
|
||||
`(${extractedText.length} chars${spaDetected ? ", SPA" : ""}${paywallDetected ? ", PAYWALL" : ""})`
|
||||
`(${extractedText.length} chars${spaDetected ? ", SPA detected" : ""})`
|
||||
);
|
||||
|
||||
// Paywall or inaccessible content — return signal so client can prompt for PDF upload.
|
||||
// Also catches meta-refresh redirects and other "content gatekeeping" patterns where
|
||||
// the fetched HTML is tiny and yields no usable text or metadata.
|
||||
const contentBlocked =
|
||||
paywallDetected ||
|
||||
(spaDetected && extractedText.length < 50 && !metaDesc && !pageTitle);
|
||||
|
||||
if (contentBlocked) {
|
||||
res.json({
|
||||
success: false,
|
||||
paywall_detected: true,
|
||||
page_title: pageTitle,
|
||||
meta_desc: metaDesc,
|
||||
source_url: url,
|
||||
topic: selectedTopic,
|
||||
error: paywallDetected
|
||||
? "Paywall erkannt — bitte PDF hochladen"
|
||||
: "Seite nicht zugänglich — bitte PDF hochladen",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Build a rich additional_context from the URL content.
|
||||
// When a SPA is detected (JS-rendered), body text is a shell — we rely on meta tags instead.
|
||||
const spaWarning = spaDetected
|
||||
@ -1949,8 +1672,6 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => {
|
||||
: "";
|
||||
|
||||
const additionalContext =
|
||||
`⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${pageTitle || parsedUrl.hostname}"\n` +
|
||||
`The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source article explicitly covers them.\n\n` +
|
||||
`SOURCE URL: ${url}\n` +
|
||||
`PAGE TITLE: ${pageTitle}\n` +
|
||||
`HOSTNAME: ${parsedUrl.hostname}\n` +
|
||||
@ -1966,22 +1687,20 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => {
|
||||
const title = pageTitle || parsedUrl.hostname;
|
||||
const template = templates[Math.floor(Math.random() * templates.length)];
|
||||
|
||||
// For from-url flow: ALWAYS use empty data — no transceiver product injection.
|
||||
// The URL content IS the data. Injecting transceiver products would cause the
|
||||
// fine-tuned model to ignore the source article and write a generic 400G post.
|
||||
const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] };
|
||||
// When SPA detected, skip optical transceiver product injection — it pollutes the LLM context
|
||||
// with irrelevant product data and causes the model to default to its fine-tuning domain.
|
||||
// Use empty data so the pipeline focuses purely on the URL context provided above.
|
||||
const keywords = spaDetected
|
||||
? [parsedUrl.hostname.replace(/^www\./, ""), pageTitle].filter(Boolean)
|
||||
: [...template.seo_keywords, "optical transceiver", "networking"].filter(Boolean);
|
||||
|
||||
// Use a minimal placeholder draft — generateTemplateDraft produces transceiver-specific
|
||||
// skeleton content (NOC scenarios, DOM readings) that pollutes the LLM context.
|
||||
const date = new Date().toISOString().split("T")[0];
|
||||
const draftContent =
|
||||
`# ${title}\n\n` +
|
||||
`*Generated from URL: ${url} on ${date}*\n\n` +
|
||||
`> **Status**: Pending LLM enhancement — source article loaded.\n\n` +
|
||||
`**Source**: ${url}\n` +
|
||||
(metaDesc ? `**Summary**: ${metaDesc}\n` : "");
|
||||
const data = spaDetected
|
||||
? { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] }
|
||||
: await gatherBlogData(keywords, selectedTopic);
|
||||
|
||||
const draftContent = generateTemplateDraft(title, selectedTopic, data);
|
||||
const wordCount = draftContent.split(/\s+/).length;
|
||||
const initialIssues: string[] = [];
|
||||
const initialIssues = validateArticle(draftContent);
|
||||
|
||||
const activeModel = getLlmProvider();
|
||||
const generatedBy = `tip-blog-from-url-${activeModel.ollamaModel || activeModel.provider || "llm"}`;
|
||||
@ -2042,147 +1761,6 @@ blogRouter.post("/from-url", async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/blog/from-pdf — Upload a PDF, extract text, generate blog from it
|
||||
// Accepts JSON body: { pdf_base64: string, filename: string, url?: string, topic?: string, page_title?: string }
|
||||
// Using base64 JSON instead of multipart to avoid Cloudflare WAF blocking binary uploads.
|
||||
blogRouter.post("/from-pdf", async (req: Request, res: Response) => {
|
||||
const { pdf_base64, filename, url, topic, page_title } = req.body as {
|
||||
pdf_base64?: string;
|
||||
filename?: string;
|
||||
url?: string;
|
||||
topic?: string;
|
||||
page_title?: string;
|
||||
};
|
||||
|
||||
if (!pdf_base64) {
|
||||
res.status(400).json({ success: false, error: "Keine PDF-Daten empfangen (pdf_base64 fehlt)" });
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate base64 + decode
|
||||
let fileBuffer: Buffer;
|
||||
let fileSize: number;
|
||||
try {
|
||||
fileBuffer = Buffer.from(pdf_base64, "base64");
|
||||
fileSize = fileBuffer.length;
|
||||
if (fileSize < 100) throw new Error("Datei zu klein");
|
||||
if (fileSize > 20 * 1024 * 1024) throw new Error("Datei zu groß (max 20 MB)");
|
||||
} catch (err) {
|
||||
res.status(400).json({ success: false, error: `Ungültige PDF-Daten: ${(err as Error).message}` });
|
||||
return;
|
||||
}
|
||||
|
||||
const originalName = filename || "upload.pdf";
|
||||
const selectedTopic = topic || "technology_deep_dive";
|
||||
const templates = BLOG_TEMPLATES[selectedTopic];
|
||||
if (!templates) {
|
||||
res.status(400).json({ success: false, error: `Ungültiger Blog-Typ. Gültig: ${Object.keys(BLOG_TEMPLATES).join(", ")}` });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract text from PDF
|
||||
const pdfData = await pdfParse(fileBuffer);
|
||||
let extractedText = pdfData.text
|
||||
.split("\n").map((l: string) => l.trim()).filter((l: string) => l.length > 20).join("\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
|
||||
if (extractedText.length < 100) {
|
||||
res.status(422).json({ success: false, error: "PDF enthält zu wenig lesbaren Text (ggf. gescannt/bildbasiert)" });
|
||||
return;
|
||||
}
|
||||
|
||||
// Limit to ~6000 chars for LLM context
|
||||
if (extractedText.length > 6000) {
|
||||
extractedText = extractedText.slice(0, 6000) + "\n[… PDF content truncated for LLM context …]";
|
||||
}
|
||||
|
||||
const title: string = page_title || (typeof pdfData.info?.Title === "string" ? pdfData.info.Title : null) || originalName.replace(/\.pdf$/i, "") || "Artikel aus PDF";
|
||||
|
||||
console.log(`Blog from-pdf: "${title}" — ${extractedText.length} chars from ${originalName} (${(fileSize / 1024).toFixed(0)} KB)`);
|
||||
|
||||
const additionalContext =
|
||||
`⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${title}"\n` +
|
||||
`The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source document explicitly covers them.\n\n` +
|
||||
(url ? `SOURCE URL: ${url}\n` : "") +
|
||||
`SOURCE FILE: ${originalName}\n` +
|
||||
`PAGE TITLE: ${title}\n` +
|
||||
`\n--- EXTRACTED PDF CONTENT ---\n` +
|
||||
`${extractedText}\n` +
|
||||
`--- END PDF CONTENT ---\n\n` +
|
||||
`IMPORTANT: Use this content as factual background and editorial direction. ` +
|
||||
`The blog MUST be about the topic described above. ` +
|
||||
`Do NOT copy sentences verbatim. Write a Flexoptix-voice blog article using these facts and insights.`;
|
||||
|
||||
const template = templates[Math.floor(Math.random() * templates.length)];
|
||||
const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] };
|
||||
|
||||
const date = new Date().toISOString().split("T")[0];
|
||||
const draftContent =
|
||||
`# ${title}\n\n` +
|
||||
`*Generated from PDF: ${originalName} on ${date}*\n\n` +
|
||||
`> **Status**: Pending LLM enhancement — PDF content loaded.\n\n` +
|
||||
(url ? `**Source URL**: ${url}\n` : "") +
|
||||
`**Source file**: ${originalName} (${(fileSize / 1024).toFixed(0)} KB, ${pdfData.numpages} pages)\n`;
|
||||
|
||||
const wordCount = draftContent.split(/\s+/).length;
|
||||
const activeModel = getLlmProvider();
|
||||
const generatedBy = `tip-blog-from-pdf-${activeModel.ollamaModel || activeModel.provider || "llm"}`;
|
||||
|
||||
const result = await pool.query(
|
||||
`INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'draft', $7, $8, $9)
|
||||
RETURNING id, created_at`,
|
||||
[
|
||||
title,
|
||||
selectedTopic,
|
||||
template.target_audience,
|
||||
JSON.stringify({ generation_method: "from-pdf", source_url: url || null, source_file: originalName, pdf_pages: pdfData.numpages }),
|
||||
draftContent,
|
||||
JSON.stringify({ source_url: url || null, source_file: originalName, extracted_chars: extractedText.length, pdf_pages: pdfData.numpages }),
|
||||
generatedBy,
|
||||
wordCount,
|
||||
template.seo_keywords,
|
||||
],
|
||||
);
|
||||
|
||||
const draftId = result.rows[0].id;
|
||||
|
||||
const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
|
||||
let llmStarted = false;
|
||||
if (health.ok) {
|
||||
llmStarted = true;
|
||||
enqueueLlmPipeline(draftId, title, selectedTopic, template.target_audience, data, additionalContext).catch((err) => {
|
||||
console.error(`Blog from-pdf LLM pipeline error: ${(err as Error).message}`);
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
source_file: originalName,
|
||||
source_url: url || null,
|
||||
page_title: title,
|
||||
extracted_chars: extractedText.length,
|
||||
pdf_pages: pdfData.numpages,
|
||||
draft: {
|
||||
id: draftId,
|
||||
title,
|
||||
topic: selectedTopic,
|
||||
target_audience: template.target_audience,
|
||||
word_count: wordCount,
|
||||
generation_method: "from-pdf",
|
||||
llm_enhancing: llmStarted,
|
||||
created_at: result.rows[0].created_at,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = (err as Error).message;
|
||||
console.error(`Blog from-pdf error: ${msg}`);
|
||||
res.status(500).json({ success: false, error: `PDF konnte nicht verarbeitet werden: ${msg}` });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/blog — List all drafts
|
||||
blogRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
@ -2212,102 +1790,6 @@ blogRouter.post("/llm/reset-queue", (_req: Request, res: Response) => {
|
||||
res.json({ success: true, message: "Ollama queue reset — stuck requests cleared" });
|
||||
});
|
||||
|
||||
// GET /api/blog/llm/model-info — Training metadata for the active fo-blog model
|
||||
// Returns Ollama model details + training manifest stats (pairs, eval, base model, revision, etc.)
|
||||
blogRouter.get("/llm/model-info", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const settings = getLlmProvider();
|
||||
const ollamaUrl = process.env.BLOG_OLLAMA_URL || process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
|
||||
// Only meaningful for fo-blog Ollama models
|
||||
const modelName = settings.ollamaModel || "";
|
||||
const isFoBlog = /^fo-blog-v\d+/.test(modelName);
|
||||
|
||||
// 1. Ollama /api/show — model metadata
|
||||
let ollamaInfo: Record<string, unknown> | null = null;
|
||||
if (settings.provider === "ollama" && modelName) {
|
||||
try {
|
||||
const r = await fetch(`${ollamaUrl}/api/show`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ name: modelName }),
|
||||
signal: AbortSignal.timeout(6000),
|
||||
});
|
||||
if (r.ok) ollamaInfo = (await r.json()) as Record<string, unknown>;
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
// 2. Ollama /api/tags — count available fo-blog-vX revisions
|
||||
let availableVersions: string[] = [];
|
||||
try {
|
||||
const r = await fetch(`${ollamaUrl}/api/tags`, { signal: AbortSignal.timeout(5000) });
|
||||
if (r.ok) {
|
||||
const data = (await r.json()) as { models?: { name: string; modified_at?: string }[] };
|
||||
availableVersions = (data.models || [])
|
||||
.map((m) => m.name)
|
||||
.filter((n) => /^fo-blog-v\d+(?:-r\d+)?:/.test(n));
|
||||
}
|
||||
} catch { /* non-fatal */ }
|
||||
|
||||
// 3. Training manifest on disk (training-data/runpod/blog_llm/manifest.json)
|
||||
let manifest: Record<string, unknown> | null = null;
|
||||
try {
|
||||
const { readFileSync } = await import("fs");
|
||||
const { resolve } = await import("path");
|
||||
const manifestPath = resolve(process.cwd(), "training-data/runpod/blog_llm/manifest.json");
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
|
||||
} catch { /* manifest may not exist on this deployment */ }
|
||||
|
||||
// Extract structured fields from Ollama response
|
||||
const details = (ollamaInfo?.details ?? {}) as Record<string, unknown>;
|
||||
const modelInfo = (ollamaInfo?.model_info ?? {}) as Record<string, unknown>;
|
||||
const params = (ollamaInfo?.parameters ?? "") as string;
|
||||
|
||||
// Parse revision from parent_model e.g. "fo-blog-v13-r16:latest" → r16
|
||||
const parentModel = (details?.parent_model ?? "") as string;
|
||||
const revMatch = parentModel.match(/-r(\d+)/);
|
||||
const revision = revMatch ? `r${revMatch[1]}` : null;
|
||||
|
||||
// Parse context length from parameters string
|
||||
const ctxMatch = params.match(/num_ctx\s+(\d+)/);
|
||||
const contextLength = ctxMatch ? Number.parseInt(ctxMatch[1], 10) : null;
|
||||
|
||||
// Count major versions (fo-blog-vX:latest, not revisions like fo-blog-vX-rY:latest)
|
||||
const majorVersions = availableVersions.filter((n) => !/v\d+-r\d+:/.test(n));
|
||||
const allRevisions = availableVersions.filter((n) => /v\d+-r\d+:/.test(n));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
provider: settings.provider,
|
||||
model: modelName,
|
||||
is_fo_blog: isFoBlog,
|
||||
revision,
|
||||
parent_model: parentModel || null,
|
||||
trained_at: (ollamaInfo?.modified_at ?? null) as string | null,
|
||||
quantization: (details?.quantization_level ?? null) as string | null,
|
||||
parameter_size: (details?.parameter_size ?? null) as string | null,
|
||||
base_model: (modelInfo?.["general.basename"] ?? null) as string | null,
|
||||
finetune_id: (modelInfo?.["general.finetune"] ?? null) as string | null,
|
||||
parameter_count: (modelInfo?.["general.parameter_count"] ?? null) as number | null,
|
||||
context_length: contextLength ?? ((modelInfo?.["qwen2.context_length"] ?? null) as number | null),
|
||||
temperature: Number.parseFloat(params.match(/temperature\s+([\d.]+)/)?.[1] ?? "NaN") || null,
|
||||
// Training data stats from manifest
|
||||
training_pairs: (manifest?.training_pairs ?? null) as number | null,
|
||||
train_pairs: (manifest?.train_pairs ?? null) as number | null,
|
||||
eval_pairs: (manifest?.eval_pairs ?? null) as number | null,
|
||||
raw_pairs: (manifest?.raw_pairs ?? null) as number | null,
|
||||
// Version availability
|
||||
major_versions_available: majorVersions.length,
|
||||
total_revisions_available: allRevisions.length,
|
||||
available_versions: majorVersions,
|
||||
// Pin status
|
||||
pinned_version: settings.pinnedVersion ?? null,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/blog/llm/refresh-discovery — Force auto-discovery to pick up newly-trained fo-blog-v* versions
|
||||
// Useful right after Magatama adopts a new fo-blog-vN model. Otherwise runs every 10 min by itself.
|
||||
blogRouter.post("/llm/refresh-discovery", async (_req: Request, res: Response) => {
|
||||
@ -2346,45 +1828,6 @@ blogRouter.post("/llm/switch", (req: Request, res: Response) => {
|
||||
});
|
||||
});
|
||||
|
||||
// POST /api/blog/llm/pin — Pin a specific fo-blog-vN model, disabling auto-upgrade
|
||||
// Body: { version: "fo-blog-v13" } — omit version to pin the current model
|
||||
blogRouter.post("/llm/pin", (req: Request, res: Response) => {
|
||||
const { version } = req.body as { version?: string };
|
||||
const current = getLlmProvider();
|
||||
const target = version || current.ollamaModel;
|
||||
|
||||
if (!target.startsWith("fo-blog-v")) {
|
||||
res.status(400).json({ success: false, error: "Only fo-blog-v* models can be pinned" });
|
||||
return;
|
||||
}
|
||||
|
||||
pinLlmVersion(target);
|
||||
const next = getLlmProvider();
|
||||
console.log(`[blog/llm/pin] pinned to ${target}`);
|
||||
res.json({
|
||||
success: true,
|
||||
pinned: target,
|
||||
active: { provider: next.provider, model: next.ollamaModel, pinnedVersion: next.pinnedVersion },
|
||||
message: `Pinned to ${target} — auto-upgrade disabled`,
|
||||
});
|
||||
});
|
||||
|
||||
// POST /api/blog/llm/unpin — Remove version pin, re-enable auto-upgrade
|
||||
// Immediately reconciles against Ollama so the highest available version is adopted.
|
||||
blogRouter.post("/llm/unpin", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const active = await unpinLlmVersion();
|
||||
console.log(`[blog/llm/unpin] unpinned, active → ${active.ollamaModel}`);
|
||||
res.json({
|
||||
success: true,
|
||||
active: { provider: active.provider, model: active.ollamaModel },
|
||||
message: `Unpinned — auto-upgrade re-enabled. Active: ${active.ollamaModel}`,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/blog/:id — Get a specific draft with full content
|
||||
// GET /api/blog/:id/progress — Real-time pipeline step progress (in-memory)
|
||||
blogRouter.get("/:id/progress", (req: Request, res: Response) => {
|
||||
|
||||
@ -52,7 +52,7 @@ bulkPriceRouter.post("/", async (req: Request, res: Response) => {
|
||||
observed_at: Date;
|
||||
}>(
|
||||
`WITH matched AS (
|
||||
SELECT id, part_number, COALESCE(standard_name, part_number, '') AS model_name, form_factor, speed_gbps
|
||||
SELECT id, part_number, model_name, form_factor, speed_gbps
|
||||
FROM transceivers
|
||||
WHERE part_number ILIKE ANY (ARRAY[${placeholders}])
|
||||
),
|
||||
|
||||
@ -50,7 +50,6 @@ hotTopicsRouter.get("/", async (req, res) => {
|
||||
JOIN vendors v ON pc.vendor_id = v.id
|
||||
JOIN transceivers t ON pc.transceiver_id = t.id
|
||||
WHERE pc.delta_pct < -10 AND pc.detected_at > NOW() - INTERVAL '14 days'
|
||||
AND v.name != 'Flexoptix'
|
||||
ORDER BY pc.delta_pct ASC LIMIT 5
|
||||
`).catch(() => ({ rows: [] }));
|
||||
|
||||
|
||||
@ -215,6 +215,192 @@ function buildRecommendation(
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/hype-cycle/market-signals — Multi-source demand intelligence
|
||||
hypeCycleRouter.get("/market-signals", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { pool } = await import("../db/client");
|
||||
|
||||
// ── Parallel data fetch ──────────────────────────────────────────────────
|
||||
const [hypeRows, priceRows, capexRows, aiRows, ebayRows, internalRows] = await Promise.all([
|
||||
// Latest hype cycle per technology
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (technology)
|
||||
technology, hype_phase, hype_score, asp_current_usd,
|
||||
r_squared, computed_at, current_share, years_to_next_phase
|
||||
FROM hype_cycle_analysis
|
||||
ORDER BY technology, computed_at DESC
|
||||
`),
|
||||
|
||||
// Price observation activity: last 30d vs prior 30d (scraping frequency = demand proxy)
|
||||
pool.query(`
|
||||
SELECT
|
||||
t.speed_gbps,
|
||||
t.form_factor,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
AND po.time < NOW() - INTERVAL '30 days') AS obs_prior_30d
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
GROUP BY t.speed_gbps, t.form_factor
|
||||
`),
|
||||
|
||||
// Hyperscaler capex — most recent per company
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (company)
|
||||
company, period_label, capex_usd_millions,
|
||||
dc_capex_est_millions, yoy_growth_pct
|
||||
FROM hyperscaler_capex
|
||||
WHERE yoy_growth_pct IS NOT NULL
|
||||
ORDER BY company, period_end DESC
|
||||
`),
|
||||
|
||||
// AI cluster demand last 90 days
|
||||
pool.query(`
|
||||
SELECT
|
||||
COALESCE(SUM(estimated_transceivers), 0) AS total_tx,
|
||||
COUNT(*) AS cluster_count,
|
||||
COUNT(*) FILTER (WHERE estimated_transceivers > 0) AS with_estimates
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
`),
|
||||
|
||||
// eBay marketplace velocity
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (form_factor, speed_label)
|
||||
marketplace, keyword, form_factor, speed_label,
|
||||
sold_count_30d, active_listings, avg_sold_price
|
||||
FROM marketplace_velocity
|
||||
WHERE sold_count_30d IS NOT NULL
|
||||
ORDER BY form_factor, speed_label, scraped_at DESC
|
||||
`),
|
||||
|
||||
// Internal demand: fast-mover trend
|
||||
pool.query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE velocity_class = 'fast_mover' AND demand_trend_pct > 0) AS fast_mover_pos,
|
||||
COUNT(*) FILTER (WHERE velocity_class = 'fast_mover' AND demand_trend_pct < 0) AS fast_mover_neg,
|
||||
AVG(demand_trend_pct) FILTER (WHERE velocity_class = 'fast_mover') AS avg_fast_trend,
|
||||
AVG(demand_trend_pct) FILTER (WHERE velocity_class = 'regular') AS avg_regular_trend
|
||||
FROM flexoptix_internal_demand
|
||||
WHERE is_internal = true
|
||||
`),
|
||||
]);
|
||||
|
||||
// ── Pre-compute global signals ───────────────────────────────────────────
|
||||
const capexYoyValues = capexRows.rows
|
||||
.map((r) => parseFloat(r.yoy_growth_pct))
|
||||
.filter((v) => !isNaN(v));
|
||||
const capexYoyAvg = capexYoyValues.length
|
||||
? capexYoyValues.reduce((a, b) => a + b, 0) / capexYoyValues.length
|
||||
: 0;
|
||||
|
||||
const aiData = aiRows.rows[0];
|
||||
const totalAiTx = parseInt(aiData?.total_tx ?? "0") || 0;
|
||||
|
||||
const internalData = internalRows.rows[0];
|
||||
const avgFastTrend = parseFloat(internalData?.avg_fast_trend ?? "0") || 0;
|
||||
|
||||
// Build price activity map: speedGbps+formFactor → ratio (30d/prior30d)
|
||||
const priceMap = new Map<string, number>();
|
||||
for (const row of priceRows.rows) {
|
||||
const obs30 = parseInt(row.obs_30d) || 0;
|
||||
const obsPrior = parseInt(row.obs_prior_30d) || 1;
|
||||
const key = `${row.speed_gbps}__${row.form_factor ?? ""}`;
|
||||
priceMap.set(key, obs30 / obsPrior);
|
||||
}
|
||||
|
||||
// Build eBay map: speedLabel → sold_count_30d
|
||||
const ebayMap = new Map<string, number>();
|
||||
for (const row of ebayRows.rows) {
|
||||
const speed = (row.speed_label ?? "").replace(/\s+/g, "");
|
||||
ebayMap.set(speed, parseInt(row.sold_count_30d) || 0);
|
||||
}
|
||||
|
||||
// ── Per-technology signals ───────────────────────────────────────────────
|
||||
const technologies = hypeRows.rows.map((r) => {
|
||||
const tech = TECH_SIGNAL_MAP.find((t) => t.label === r.technology);
|
||||
const phase = (r.hype_phase ?? "innovation_trigger") as PhaseKey;
|
||||
const hypeScore = parseInt(r.hype_score) || 0;
|
||||
|
||||
// Price activity ratio: average across matching speed+formFactor combos
|
||||
let priceRatios: number[] = [];
|
||||
if (tech) {
|
||||
for (const ff of tech.formFactors) {
|
||||
const key = `${tech.speedGbps}__${ff}`;
|
||||
const ratio = priceMap.get(key);
|
||||
if (ratio !== undefined) priceRatios.push(ratio);
|
||||
}
|
||||
}
|
||||
const priceActivityRatio = priceRatios.length
|
||||
? priceRatios.reduce((a, b) => a + b, 0) / priceRatios.length
|
||||
: 1;
|
||||
|
||||
// eBay velocity for this speed
|
||||
const ebayVelocity = tech ? (ebayMap.get(tech.speedLabel) ?? 0) : 0;
|
||||
|
||||
// AI cluster: allocate demand proportionally for high-speed techs
|
||||
const aiBoostTx = tech && tech.speedGbps >= 400 ? totalAiTx : 0;
|
||||
|
||||
// Compute composite score (0–100)
|
||||
let score = hypeScore * 0.3;
|
||||
const capexBoostPts = capexYoyAvg > 100 ? 18 : capexYoyAvg > 50 ? 12 : capexYoyAvg > 20 ? 5 : 0;
|
||||
const priceBoostPts = priceActivityRatio > 1.3 ? 10 : priceActivityRatio > 1.0 ? 5 : -3;
|
||||
const aiBoostPts = aiBoostTx > 100000 ? 14 : aiBoostTx > 50000 ? 9 : aiBoostTx > 10000 ? 4 : 0;
|
||||
const ebayBoostPts = ebayVelocity > 200 ? 8 : ebayVelocity > 100 ? 5 : ebayVelocity > 50 ? 2 : 0;
|
||||
const intlBoostPts = avgFastTrend > 10 ? 6 : avgFastTrend > 0 ? 3 : avgFastTrend < -20 ? -5 : 0;
|
||||
|
||||
score += capexBoostPts + priceBoostPts + aiBoostPts + ebayBoostPts + intlBoostPts;
|
||||
const marketSignalScore = Math.max(0, Math.min(100, Math.round(score)));
|
||||
|
||||
const rec = buildRecommendation(phase, marketSignalScore, capexYoyAvg, tech?.speedGbps ?? 0);
|
||||
|
||||
// Signal drivers list for tooltip
|
||||
const drivers: string[] = [];
|
||||
if (capexBoostPts > 0) drivers.push(`Hyperscaler CapEx +${capexYoyAvg.toFixed(0)}% YoY avg`);
|
||||
if (priceActivityRatio > 1.1) drivers.push(`Price obs +${((priceActivityRatio - 1) * 100).toFixed(0)}% MoM activity`);
|
||||
if (aiBoostTx > 0) drivers.push(`~${(aiBoostTx / 1000).toFixed(0)}k transceivers in AI cluster builds`);
|
||||
if (ebayVelocity > 0) drivers.push(`${ebayVelocity} units sold on secondary market (30d)`);
|
||||
if (intlBoostPts > 0) drivers.push(`Internal fast-movers trending ${avgFastTrend > 0 ? "+" : ""}${avgFastTrend.toFixed(1)}%`);
|
||||
|
||||
return {
|
||||
technology: r.technology,
|
||||
phase,
|
||||
hypeScore,
|
||||
aspCurrentUsd: r.asp_current_usd,
|
||||
marketSignalScore,
|
||||
recommendation: rec,
|
||||
drivers,
|
||||
speedGbps: tech?.speedGbps,
|
||||
priceActivityRatio: Math.round(priceActivityRatio * 100) / 100,
|
||||
ebayVelocity,
|
||||
computedAt: r.computed_at,
|
||||
};
|
||||
});
|
||||
|
||||
// ── Global context ───────────────────────────────────────────────────────
|
||||
const globalContext = {
|
||||
hyperscalerCapex: capexRows.rows.map((r) => ({
|
||||
company: r.company,
|
||||
periodLabel: r.period_label,
|
||||
capexMillions: parseFloat(r.capex_usd_millions),
|
||||
dcCapexMillions: parseFloat(r.dc_capex_est_millions),
|
||||
yoyGrowthPct: parseFloat(r.yoy_growth_pct),
|
||||
})),
|
||||
capexYoyAvg: Math.round(capexYoyAvg),
|
||||
capexBoom: capexYoyAvg > 50,
|
||||
totalAiClusterTx90d: totalAiTx,
|
||||
aiClusterCount90d: parseInt(aiData?.cluster_count ?? "0") || 0,
|
||||
internalFastMoverTrend: Math.round(avgFastTrend * 10) / 10,
|
||||
};
|
||||
|
||||
res.json({ success: true, technologies, globalContext, computed_at: new Date().toISOString() });
|
||||
} catch (err) {
|
||||
console.error("Market signals error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute market signals" });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/hype-cycle/analysis — Bass-fitted results from DB (hype_cycle_analysis table)
|
||||
hypeCycleRouter.get("/analysis", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
@ -248,74 +434,6 @@ hypeCycleRouter.get("/analysis", async (_req: Request, res: Response) => {
|
||||
});
|
||||
|
||||
// GET /api/hype-cycle/:tech — Specific technology detail (must be last!)
|
||||
// GET /api/hype-cycle/market-signals — data-driven per-technology market signal score + drivers + recommendation
|
||||
hypeCycleRouter.get("/market-signals", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const yearParam = q("year", _req);
|
||||
const year = yearParam ? parseInt(yearParam) : new Date().getFullYear();
|
||||
const overridesMap = await getDataDrivenOverrides();
|
||||
const speedMetrics = await getSpeedClassMetrics();
|
||||
const metricsBySpeed = new Map(speedMetrics.map((m) => [m.speedGbps, m]));
|
||||
const maxSku = Math.max(1, ...speedMetrics.map((m) => m.skuCount));
|
||||
const allTechs = [...TECH_GENERATIONS, ...SPECIAL_TECHS];
|
||||
|
||||
const technologies = allTechs.map((tech) => {
|
||||
const r = computeHypeCycle(tech, year, overridesMap.get(tech.speedGbps));
|
||||
const sm = metricsBySpeed.get(tech.speedGbps);
|
||||
const adoption = Math.max(0, Math.round(r.adoptionPct));
|
||||
const composite = Math.max(0, Math.round(r.compositeScore));
|
||||
const depth = sm ? Math.round(100 * Math.min(1, sm.skuCount / maxSku)) : 0;
|
||||
const marketSignalScore = Math.max(0, Math.min(100, Math.round(0.4 * adoption + 0.3 * composite + 0.3 * depth)));
|
||||
|
||||
const drivers: string[] = [];
|
||||
drivers.push(adoption + "% modeled adoption");
|
||||
if (sm && sm.skuCount) drivers.push(sm.skuCount + " SKUs / " + sm.vendorCount + " vendors");
|
||||
if (sm && sm.avgPrice) drivers.push("avg $" + Math.round(sm.avgPrice));
|
||||
drivers.push("phase: " + r.phaseLabel);
|
||||
|
||||
const phase = String(r.phaseLabel || "");
|
||||
let recommendation = { label: "Monitor", detail: "Insufficient market momentum — keep watching.", color: "#94a3b8" };
|
||||
if (/Plateau/i.test(phase)) recommendation = { label: "Commodity — negotiate", detail: "Mature, broad supply (" + (sm ? sm.vendorCount : 0) + " vendors). Push for price.", color: "#0ea5e9" };
|
||||
else if (/Slope/i.test(phase)) recommendation = { label: "Mainstream — buy", detail: "Mainstreaming with falling prices and rising supply.", color: "#16a34a" };
|
||||
else if (/Trough/i.test(phase)) recommendation = { label: "Maturing — opportunity", detail: "Past the hype; early mainstream pricing forming.", color: "#ca8a04" };
|
||||
else if (/Peak/i.test(phase)) recommendation = { label: "Hype peak — caution", detail: "Elevated expectations; supply and price still volatile.", color: "#f97316" };
|
||||
else if (/Innovation|Trigger/i.test(phase)) recommendation = { label: "Emerging — pilot", detail: "Early window; limited supply, premium pricing.", color: "#ca8a04" };
|
||||
|
||||
return {
|
||||
technology: tech.name,
|
||||
phase: r.phaseLabel,
|
||||
marketSignalScore,
|
||||
adoptionPct: adoption,
|
||||
compositeScore: composite,
|
||||
skuCount: sm ? sm.skuCount : 0,
|
||||
vendorCount: sm ? sm.vendorCount : 0,
|
||||
avgPrice: sm && sm.avgPrice ? Math.round(sm.avgPrice * 100) / 100 : null,
|
||||
drivers,
|
||||
recommendation,
|
||||
};
|
||||
});
|
||||
|
||||
technologies.sort((a, b) => b.marketSignalScore - a.marketSignalScore);
|
||||
|
||||
const totalSkus = speedMetrics.reduce((acc, m) => acc + m.skuCount, 0);
|
||||
const totalVendors = speedMetrics.length ? Math.max(...speedMetrics.map((m) => m.vendorCount)) : 0;
|
||||
const avgSignal = technologies.length ? Math.round(technologies.reduce((acc, t) => acc + t.marketSignalScore, 0) / technologies.length) : 0;
|
||||
const globalContext = {
|
||||
totalSkus,
|
||||
totalVendors,
|
||||
avgMarketSignal: avgSignal,
|
||||
strongest: technologies.length ? technologies[0].technology : null,
|
||||
strongestScore: technologies.length ? technologies[0].marketSignalScore : null,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
res.json({ success: true, year, technologies, globalContext });
|
||||
} catch (err) {
|
||||
console.error("market-signals error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute market signals" });
|
||||
}
|
||||
});
|
||||
|
||||
hypeCycleRouter.get("/:tech", (req: Request, res: Response) => {
|
||||
const techQuery = String(req.params.tech);
|
||||
const yearParam = q("year", req);
|
||||
|
||||
@ -1,27 +1,24 @@
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { semanticSearch } from "../embeddings/client";
|
||||
|
||||
export const kbRouter = Router();
|
||||
|
||||
// GET /api/kb — Knowledge base browser: FAQ + troubleshooting entries
|
||||
// ?q=search&category=faq|troubleshooting|known_issue&limit=50&semantic=1
|
||||
// Falls back to Qdrant semantic search when ILIKE returns 0 results
|
||||
// ?q=search&category=faq|troubleshooting|known_issue&limit=50
|
||||
kbRouter.get("/", async (req: Request, res: Response) => {
|
||||
const q = ((req.query.q as string) || "").trim();
|
||||
const category = (req.query.category as string) || "";
|
||||
const limit = Math.min(parseInt((req.query.limit as string) || "60"), 200);
|
||||
const forceSemantic = req.query.semantic === "1";
|
||||
|
||||
try {
|
||||
const [textEntries, cats] = await Promise.all([
|
||||
const [entries, cats] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT id, category, subcategory, question, answer,
|
||||
applies_to_form_factors, applies_to_speeds, severity, tags
|
||||
FROM knowledge_base
|
||||
WHERE ($1 = '' OR category = $1)
|
||||
AND ($2 = '' OR question ILIKE '%' || $2 || '%'
|
||||
OR answer ILIKE '%' || $2 || '%'
|
||||
AND ($2 = '' OR question ILIKE '%' || $2 || '%'
|
||||
OR answer ILIKE '%' || $2 || '%'
|
||||
OR subcategory ILIKE '%' || $2 || '%')
|
||||
ORDER BY
|
||||
CASE WHEN $2 != '' AND question ILIKE '%' || $2 || '%' THEN 0 ELSE 1 END,
|
||||
@ -37,80 +34,12 @@ kbRouter.get("/", async (req: Request, res: Response) => {
|
||||
),
|
||||
]);
|
||||
|
||||
// If text search found results and semantic not forced, return them
|
||||
if (textEntries.rows.length > 0 && !forceSemantic) {
|
||||
return res.json({
|
||||
success: true,
|
||||
entries: textEntries.rows,
|
||||
categories: cats.rows,
|
||||
total: textEntries.rows.length,
|
||||
query: q,
|
||||
search_mode: "text",
|
||||
});
|
||||
}
|
||||
|
||||
// Semantic fallback — only when query is provided and text search returned nothing
|
||||
if (q.length > 2) {
|
||||
try {
|
||||
const collections: Array<"faq_embeddings" | "troubleshooting_embeddings"> =
|
||||
category === "faq" ? ["faq_embeddings"] :
|
||||
category === "troubleshooting" ? ["troubleshooting_embeddings"] :
|
||||
["faq_embeddings", "troubleshooting_embeddings"];
|
||||
|
||||
const semanticHits = (
|
||||
await Promise.all(
|
||||
collections.map(col =>
|
||||
semanticSearch(col, q, Math.ceil(limit / collections.length))
|
||||
.catch(() => [] as Array<{ id: string; score: number; payload: Record<string, unknown> }>)
|
||||
)
|
||||
)
|
||||
).flat().sort((a, b) => b.score - a.score);
|
||||
|
||||
// Deduplicate by kb id from payload, then fetch full rows from DB
|
||||
const kbIds = [...new Set(
|
||||
semanticHits
|
||||
.filter(h => h.score >= 0.5 && h.payload.kb_id)
|
||||
.slice(0, limit)
|
||||
.map(h => h.payload.kb_id as string)
|
||||
)];
|
||||
|
||||
if (kbIds.length > 0) {
|
||||
const semanticRows = await pool.query(
|
||||
`SELECT id, category, subcategory, question, answer,
|
||||
applies_to_form_factors, applies_to_speeds, severity, tags
|
||||
FROM knowledge_base
|
||||
WHERE id = ANY($1::int[])`,
|
||||
[kbIds.map(Number).filter(n => !isNaN(n))]
|
||||
);
|
||||
|
||||
// Sort results by semantic score order
|
||||
const scoreMap = new Map(semanticHits.map(h => [String(h.payload.kb_id), h.score]));
|
||||
const sorted = semanticRows.rows.sort(
|
||||
(a, b) => (scoreMap.get(String(b.id)) || 0) - (scoreMap.get(String(a.id)) || 0)
|
||||
);
|
||||
|
||||
return res.json({
|
||||
success: true,
|
||||
entries: sorted,
|
||||
categories: cats.rows,
|
||||
total: sorted.length,
|
||||
query: q,
|
||||
search_mode: "semantic",
|
||||
});
|
||||
}
|
||||
} catch (_semErr) {
|
||||
// Semantic search unavailable — fall through to text results
|
||||
}
|
||||
}
|
||||
|
||||
// Final fallback: return text results (even if empty)
|
||||
return res.json({
|
||||
res.json({
|
||||
success: true,
|
||||
entries: textEntries.rows,
|
||||
entries: entries.rows,
|
||||
categories: cats.rows,
|
||||
total: textEntries.rows.length,
|
||||
total: entries.rows.length,
|
||||
query: q,
|
||||
search_mode: "text",
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
|
||||
@ -1,188 +0,0 @@
|
||||
/**
|
||||
* Price Alert Subscriptions — /api/price-alerts
|
||||
*
|
||||
* Users subscribe to price thresholds for specific SKUs or form factor/speed combos.
|
||||
* A background checker (called by the scraper scheduler) evaluates active subscriptions
|
||||
* against the latest price_observations and queues email delivery.
|
||||
*
|
||||
* Routes:
|
||||
* POST /api/price-alerts — Create subscription
|
||||
* GET /api/price-alerts?email= — List subscriptions for an email
|
||||
* DELETE /api/price-alerts/:id — Cancel subscription
|
||||
* POST /api/price-alerts/check — Internal: evaluate + queue alerts (scheduler)
|
||||
* GET /api/price-alerts/triggered — Recent triggered alerts
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const priceAlertsRouter = Router();
|
||||
|
||||
// ── POST /api/price-alerts — Create a price alert subscription ───────────────
|
||||
priceAlertsRouter.post("/", async (req: Request, res: Response) => {
|
||||
const {
|
||||
email, transceiver_id, form_factor, speed_gbps,
|
||||
threshold_price, currency = "USD", direction = "below", vendor_id,
|
||||
} = req.body as Record<string, any>;
|
||||
|
||||
if (!email || typeof email !== "string" || !email.includes("@")) {
|
||||
return res.status(400).json({ success: false, error: "Valid email required" });
|
||||
}
|
||||
if (!threshold_price || isNaN(parseFloat(threshold_price))) {
|
||||
return res.status(400).json({ success: false, error: "threshold_price required" });
|
||||
}
|
||||
if (!transceiver_id && !form_factor && !speed_gbps) {
|
||||
return res.status(400).json({ success: false, error: "At least one of: transceiver_id, form_factor, speed_gbps" });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`INSERT INTO price_alert_subscriptions
|
||||
(email, transceiver_id, form_factor, speed_gbps, threshold_price, currency, direction, vendor_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING id, email, threshold_price, currency, direction, created_at`,
|
||||
[
|
||||
email.toLowerCase().trim(),
|
||||
transceiver_id || null,
|
||||
form_factor || null,
|
||||
speed_gbps ? parseFloat(speed_gbps) : null,
|
||||
parseFloat(threshold_price),
|
||||
currency.toUpperCase(),
|
||||
direction,
|
||||
vendor_id || null,
|
||||
]
|
||||
);
|
||||
|
||||
return res.status(201).json({ success: true, subscription: result.rows[0] });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/price-alerts?email= — List subscriptions ────────────────────────
|
||||
priceAlertsRouter.get("/", async (req: Request, res: Response) => {
|
||||
const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase();
|
||||
if (!email) return res.status(400).json({ success: false, error: "email required" });
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`SELECT pas.*,
|
||||
t.standard_name, t.form_factor AS tx_form_factor, t.speed_gbps AS tx_speed,
|
||||
v.name AS vendor_name
|
||||
FROM price_alert_subscriptions pas
|
||||
LEFT JOIN transceivers t ON t.id = pas.transceiver_id
|
||||
LEFT JOIN vendors v ON v.id = pas.vendor_id
|
||||
WHERE pas.email = $1
|
||||
ORDER BY pas.created_at DESC`,
|
||||
[email]
|
||||
);
|
||||
return res.json({ success: true, subscriptions: result.rows });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── DELETE /api/price-alerts/:id — Cancel subscription ───────────────────────
|
||||
priceAlertsRouter.delete("/:id", async (req: Request, res: Response) => {
|
||||
const id = String(req.params.id);
|
||||
const email = String(Array.isArray(req.query.email) ? req.query.email[0] ?? "" : req.query.email ?? "").trim().toLowerCase();
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`UPDATE price_alert_subscriptions SET active = false
|
||||
WHERE id = $1 AND ($2 = '' OR email = $2)
|
||||
RETURNING id`,
|
||||
[parseInt(id), email]
|
||||
);
|
||||
if (result.rowCount === 0) {
|
||||
return res.status(404).json({ success: false, error: "Subscription not found" });
|
||||
}
|
||||
return res.json({ success: true, cancelled: parseInt(id) });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/price-alerts/triggered — Recent triggered alerts ─────────────────
|
||||
priceAlertsRouter.get("/triggered", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT pal.*,
|
||||
t.standard_name, t.form_factor,
|
||||
v.name AS vendor_name
|
||||
FROM price_alert_log pal
|
||||
LEFT JOIN transceivers t ON t.id = pal.transceiver_id
|
||||
LEFT JOIN vendors v ON v.id = pal.vendor_id
|
||||
ORDER BY pal.created_at DESC
|
||||
LIMIT 100
|
||||
`);
|
||||
return res.json({ success: true, alerts: result.rows });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── POST /api/price-alerts/check — Evaluate all active subscriptions ──────────
|
||||
// Called by the scraper scheduler periodically. Finds triggered conditions,
|
||||
// inserts into price_alert_log, and marks last_triggered on subscription.
|
||||
priceAlertsRouter.post("/check", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
// Find subscriptions where latest price crosses the threshold
|
||||
const triggered = await pool.query(`
|
||||
WITH latest_prices AS (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id, po.source_vendor_id AS vendor_id,
|
||||
po.price, po.currency, po.time
|
||||
FROM price_observations po
|
||||
WHERE po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
),
|
||||
matched AS (
|
||||
SELECT
|
||||
pas.id AS subscription_id,
|
||||
pas.email, pas.threshold_price, pas.currency, pas.direction,
|
||||
lp.transceiver_id, lp.vendor_id, lp.price AS triggered_price
|
||||
FROM price_alert_subscriptions pas
|
||||
JOIN latest_prices lp ON (
|
||||
(pas.transceiver_id IS NULL OR lp.transceiver_id = pas.transceiver_id)
|
||||
AND lp.currency = pas.currency
|
||||
AND (pas.vendor_id IS NULL OR lp.vendor_id = pas.vendor_id)
|
||||
)
|
||||
JOIN transceivers t ON t.id = lp.transceiver_id
|
||||
WHERE pas.active = true
|
||||
AND (pas.form_factor IS NULL OR t.form_factor = pas.form_factor)
|
||||
AND (pas.speed_gbps IS NULL OR t.speed_gbps = pas.speed_gbps)
|
||||
AND (
|
||||
(pas.direction = 'below' AND lp.price < pas.threshold_price)
|
||||
OR
|
||||
(pas.direction = 'above' AND lp.price > pas.threshold_price)
|
||||
)
|
||||
-- Don't re-trigger more than once per 24h per subscription
|
||||
AND (pas.last_triggered IS NULL OR pas.last_triggered < NOW() - INTERVAL '24 hours')
|
||||
)
|
||||
SELECT * FROM matched
|
||||
LIMIT 200
|
||||
`);
|
||||
|
||||
let queued = 0;
|
||||
for (const row of triggered.rows) {
|
||||
await pool.query(
|
||||
`INSERT INTO price_alert_log
|
||||
(subscription_id, transceiver_id, vendor_id, triggered_price, threshold_price, currency, email, delivery_status)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,'pending')`,
|
||||
[row.subscription_id, row.transceiver_id, row.vendor_id,
|
||||
row.triggered_price, row.threshold_price, row.currency, row.email]
|
||||
);
|
||||
await pool.query(
|
||||
`UPDATE price_alert_subscriptions
|
||||
SET last_triggered = NOW(), trigger_count = trigger_count + 1
|
||||
WHERE id = $1`,
|
||||
[row.subscription_id]
|
||||
);
|
||||
queued++;
|
||||
}
|
||||
|
||||
return res.json({ success: true, checked: triggered.rowCount, queued });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -11,7 +11,6 @@
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { sendCSV } from "../utils/csv";
|
||||
|
||||
export const priceComparisonRouter = Router();
|
||||
|
||||
@ -36,7 +35,6 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -56,7 +54,6 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -99,10 +96,9 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
// ─── GET /api/price-comparison ───────────────────────────────────────────────
|
||||
/**
|
||||
* Top 50 transceivers ranked by number of vendors tracking them.
|
||||
* Add ?format=csv to download as CSV.
|
||||
* Shows price spread across vendors — the more vendors, the better the comparison.
|
||||
*/
|
||||
priceComparisonRouter.get("/", async (req: Request, res: Response) => {
|
||||
const fmt = req.query.format as string | undefined;
|
||||
priceComparisonRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
WITH latest AS (
|
||||
@ -112,7 +108,6 @@ priceComparisonRouter.get("/", async (req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -143,9 +138,6 @@ priceComparisonRouter.get("/", async (req: Request, res: Response) => {
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
if (fmt === "csv") {
|
||||
return sendCSV(res, result.rows, `tip-price-comparison-${new Date().toISOString().slice(0,10)}.csv`);
|
||||
}
|
||||
res.json({
|
||||
success: true,
|
||||
data: result.rows,
|
||||
@ -219,7 +211,6 @@ priceComparisonRouter.get("/:sku", async (req: Request, res: Response) => {
|
||||
po.time
|
||||
FROM price_observations po
|
||||
WHERE po.transceiver_id = $1
|
||||
AND po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
) po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
|
||||
@ -65,7 +65,7 @@ priceMatrixRouter.get("/", async (req: Request, res: Response) => {
|
||||
form_factor: string;
|
||||
speed_gbps: number;
|
||||
}>(
|
||||
`SELECT id, COALESCE(standard_name, part_number, '') AS model_name, part_number, form_factor, speed_gbps
|
||||
`SELECT id, model_name, part_number, form_factor, speed_gbps
|
||||
FROM transceivers
|
||||
WHERE id IN (${placeholders})
|
||||
ORDER BY id`,
|
||||
|
||||
@ -14,7 +14,6 @@
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { sendCSV } from "../utils/csv";
|
||||
|
||||
export const procurementRouter = Router();
|
||||
|
||||
@ -25,13 +24,11 @@ procurementRouter.get("/overview", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const [signals, abc, intel, lifecycle] = await Promise.all([
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (transceiver_id) signal
|
||||
FROM reorder_signals
|
||||
WHERE expires_at > NOW()
|
||||
ORDER BY transceiver_id, computed_at DESC
|
||||
)
|
||||
SELECT signal, COUNT(*) AS count FROM latest GROUP BY signal
|
||||
SELECT signal, COUNT(*) AS count
|
||||
FROM reorder_signals
|
||||
WHERE expires_at > NOW()
|
||||
AND computed_at = (SELECT MAX(r2.computed_at) FROM reorder_signals r2 WHERE r2.transceiver_id = reorder_signals.transceiver_id)
|
||||
GROUP BY signal
|
||||
`),
|
||||
pool.query(`
|
||||
SELECT abc_class, COUNT(*) AS count FROM abc_classification GROUP BY abc_class ORDER BY abc_class
|
||||
@ -75,47 +72,31 @@ procurementRouter.get("/signals", async (req: Request, res: Response) => {
|
||||
limit = "50", offset = "0"
|
||||
} = req.query;
|
||||
|
||||
// Use DISTINCT ON with the existing idx_reorder_transceiver index instead of
|
||||
// a correlated subquery that would run once per active row (108k+ scans).
|
||||
const params: any[] = [];
|
||||
let idx = 1;
|
||||
|
||||
const signalFilter = signal ? ` AND rs.signal = $${idx++}` : "";
|
||||
if (signal) params.push(signal);
|
||||
const abcFilter = abc_class ? ` AND ac.abc_class = $${idx++}` : "";
|
||||
if (abc_class) params.push(abc_class);
|
||||
const ffFilter = form_factor ? ` AND t.form_factor = $${idx++}` : "";
|
||||
if (form_factor) params.push(form_factor);
|
||||
const speedFilter = speed_gbps ? ` AND t.speed_gbps = $${idx++}` : "";
|
||||
if (speed_gbps) params.push(parseFloat(speed_gbps as string));
|
||||
|
||||
params.push(parseInt(limit as string), parseInt(offset as string));
|
||||
const limitIdx = idx; idx++;
|
||||
const offsetIdx = idx;
|
||||
|
||||
const sql = `
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (transceiver_id)
|
||||
id, transceiver_id, signal, signal_strength, reasons,
|
||||
stock_trend, price_trend, lead_time_weeks, hype_phase,
|
||||
computed_at, expires_at, is_demo_data
|
||||
FROM reorder_signals
|
||||
WHERE expires_at > NOW()
|
||||
ORDER BY transceiver_id, computed_at DESC
|
||||
)
|
||||
let sql = `
|
||||
SELECT rs.*,
|
||||
t.part_number, t.standard_name, t.form_factor, t.speed_gbps,
|
||||
t.reach_label, t.image_url, t.image_r2_key,
|
||||
ac.abc_class, ac.demand_score, ac.supply_risk,
|
||||
v.name AS vendor_name
|
||||
FROM latest rs
|
||||
FROM reorder_signals rs
|
||||
JOIN transceivers t ON rs.transceiver_id = t.id
|
||||
LEFT JOIN abc_classification ac ON ac.transceiver_id = t.id
|
||||
LEFT JOIN vendors v ON t.vendor_id = v.id
|
||||
WHERE 1=1${signalFilter}${abcFilter}${ffFilter}${speedFilter}
|
||||
ORDER BY rs.signal_strength DESC
|
||||
LIMIT $${limitIdx} OFFSET $${offsetIdx}
|
||||
WHERE rs.expires_at > NOW()
|
||||
AND rs.computed_at = (
|
||||
SELECT MAX(r2.computed_at) FROM reorder_signals r2 WHERE r2.transceiver_id = rs.transceiver_id
|
||||
)
|
||||
`;
|
||||
const params: any[] = [];
|
||||
let idx = 1;
|
||||
|
||||
if (signal) { sql += ` AND rs.signal = $${idx}`; params.push(signal); idx++; }
|
||||
if (abc_class) { sql += ` AND ac.abc_class = $${idx}`; params.push(abc_class); idx++; }
|
||||
if (form_factor) { sql += ` AND t.form_factor = $${idx}`; params.push(form_factor); idx++; }
|
||||
if (speed_gbps) { sql += ` AND t.speed_gbps = $${idx}`; params.push(parseFloat(speed_gbps as string)); idx++; }
|
||||
|
||||
sql += ` ORDER BY rs.signal_strength DESC LIMIT $${idx} OFFSET $${idx + 1}`;
|
||||
params.push(parseInt(limit as string), parseInt(offset as string));
|
||||
|
||||
const result = await pool.query(sql, params);
|
||||
res.json({ data: result.rows, total: result.rowCount });
|
||||
@ -214,9 +195,6 @@ procurementRouter.get("/abc", async (req: Request, res: Response) => {
|
||||
params.push(parseInt(limit as string), parseInt(offset as string));
|
||||
|
||||
const result = await pool.query(sql, params);
|
||||
if ((req.query.format as string) === "csv") {
|
||||
return sendCSV(res, result.rows, `tip-abc-classification-${new Date().toISOString().slice(0,10)}.csv`);
|
||||
}
|
||||
res.json({ data: result.rows, total: result.rowCount });
|
||||
} catch (err) {
|
||||
console.error("ABC error:", err);
|
||||
@ -582,12 +560,11 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => {
|
||||
// Search for switches matching query, return their compatible transceivers
|
||||
const switches = await pool.query(`
|
||||
SELECT DISTINCT ON (sw.id)
|
||||
sw.id, v.name AS sw_vendor, sw.model AS sw_model, sw.series AS sw_series,
|
||||
sw.id, sw.vendor AS sw_vendor, sw.model AS sw_model, sw.series AS sw_series,
|
||||
COUNT(c.transceiver_id) OVER (PARTITION BY sw.id)::int AS compat_count
|
||||
FROM switches sw
|
||||
JOIN compatibility c ON c.switch_id = sw.id
|
||||
LEFT JOIN vendors v ON v.id = sw.vendor_id
|
||||
WHERE sw.model ILIKE $1 OR COALESCE(v.name,'') ILIKE $1 OR sw.series ILIKE $1
|
||||
WHERE sw.model ILIKE $1 OR sw.vendor ILIKE $1 OR sw.series ILIKE $1
|
||||
ORDER BY sw.id, compat_count DESC
|
||||
LIMIT $2
|
||||
`, [`%${search}%`, limitNum]);
|
||||
@ -605,7 +582,8 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => {
|
||||
v.name AS vendor_name,
|
||||
c.verification_method, c.status,
|
||||
(SELECT ROUND(MIN(po.price)::numeric,2) FROM price_observations po
|
||||
WHERE po.transceiver_id = t.id AND po.price > 0) AS min_price,
|
||||
WHERE po.transceiver_id = t.id AND po.price > 0
|
||||
ORDER BY po.time DESC LIMIT 1) AS min_price,
|
||||
(SELECT po.currency FROM price_observations po
|
||||
WHERE po.transceiver_id = t.id AND po.price > 0
|
||||
ORDER BY po.time DESC LIMIT 1) AS currency
|
||||
@ -627,13 +605,12 @@ procurementRouter.get("/switch-compat", async (req: Request, res: Response) => {
|
||||
|
||||
// No search — return top switches by compat count
|
||||
const top = await pool.query(`
|
||||
SELECT v.name AS vendor, sw.model, sw.series,
|
||||
SELECT sw.vendor, sw.model, sw.series,
|
||||
COUNT(c.transceiver_id)::int AS compat_count
|
||||
FROM switches sw
|
||||
JOIN compatibility c ON c.switch_id = sw.id
|
||||
LEFT JOIN vendors v ON v.id = sw.vendor_id
|
||||
WHERE c.status = 'compatible'
|
||||
GROUP BY sw.id, v.name, sw.model, sw.series
|
||||
GROUP BY sw.id, sw.vendor, sw.model, sw.series
|
||||
ORDER BY compat_count DESC
|
||||
LIMIT $1
|
||||
`, [limitNum]);
|
||||
@ -723,7 +700,7 @@ procurementRouter.get("/dead-stock-revival", async (_req: Request, res: Response
|
||||
pool.query(`
|
||||
SELECT
|
||||
fid.transceiver_id,
|
||||
fid.sku AS part_number,
|
||||
fid.part_number_raw AS part_number,
|
||||
fid.velocity_class,
|
||||
fid.demand_12m,
|
||||
fid.demand_trend_pct,
|
||||
@ -781,198 +758,38 @@ procurementRouter.get("/dead-stock-revival", async (_req: Request, res: Response
|
||||
});
|
||||
|
||||
// ─── C: GET /api/procurement/supply-squeeze ──────────────────────────────────
|
||||
|
||||
// ─── GET /api/procurement/availability ───────────────────────────────────────
|
||||
// Data-grounded supply-availability per speed tier. Derived from real supplier
|
||||
// diversity (distinct source vendors offering it) + stock coverage. Surfaces the
|
||||
// 400G/800G/1.6T supply tightening that exists in the data but was not shown.
|
||||
procurementRouter.get("/availability", async (_req: Request, res: Response) => {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
// Disable parallel workers for this aggregate — Docker /dev/shm (64MB) is too
|
||||
// small for parallel hash and the query is fast enough single-threaded.
|
||||
await client.query('BEGIN');
|
||||
await client.query('SET LOCAL max_parallel_workers_per_gather = 0');
|
||||
const result = await client.query(`
|
||||
WITH sup AS (
|
||||
SELECT t.speed_gbps,
|
||||
COUNT(DISTINCT po.source_vendor_id) FILTER (WHERE po.time > NOW() - INTERVAL '45 days') AS suppliers,
|
||||
COUNT(DISTINCT po.source_vendor_id) FILTER (WHERE po.time <= NOW() - INTERVAL '45 days') AS suppliers_prior
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600) AND po.time > NOW() - INTERVAL '90 days'
|
||||
GROUP BY t.speed_gbps
|
||||
),
|
||||
sk AS (
|
||||
SELECT speed_gbps, COUNT(*) AS sku_count
|
||||
FROM transceivers WHERE speed_gbps IN (100,200,400,800,1600)
|
||||
GROUP BY speed_gbps HAVING COUNT(*) >= 3
|
||||
),
|
||||
st AS (
|
||||
SELECT t.speed_gbps,
|
||||
COUNT(*) FILTER (WHERE s.in_stock IS TRUE) AS skus_in_stock,
|
||||
COUNT(*) AS skus_with_stock
|
||||
FROM transceivers t
|
||||
JOIN LATERAL (
|
||||
SELECT in_stock FROM stock_observations so
|
||||
WHERE so.transceiver_id = t.id ORDER BY so.time DESC LIMIT 1
|
||||
) s ON true
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600)
|
||||
GROUP BY t.speed_gbps
|
||||
),
|
||||
pm AS (
|
||||
-- per-SKU paired price momentum (30d vs prior 30d), bias-free: only SKUs in
|
||||
-- both windows; aggregate the median per-SKU pct delta per speed tier.
|
||||
SELECT speed_gbps,
|
||||
ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (med_now - med_prior) / NULLIF(med_prior,0) * 100)::numeric, 1) AS price_delta_pct
|
||||
FROM (
|
||||
SELECT t.speed_gbps, t.id,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS med_now,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS med_prior,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS n_now,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS n_prior
|
||||
FROM price_observations po JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600) AND po.price > 0 AND COALESCE(po.is_anomalous,false) = false
|
||||
GROUP BY t.speed_gbps, t.id
|
||||
) per
|
||||
WHERE med_now IS NOT NULL AND med_prior IS NOT NULL AND med_prior > 0 AND n_now >= 2 AND n_prior >= 2
|
||||
GROUP BY speed_gbps
|
||||
)
|
||||
SELECT
|
||||
sk.speed_gbps,
|
||||
sk.sku_count::int,
|
||||
COALESCE(sup.suppliers,0)::int AS suppliers,
|
||||
COALESCE(sup.suppliers_prior,0)::int AS suppliers_prior,
|
||||
COALESCE(st.skus_in_stock,0)::int AS skus_in_stock,
|
||||
COALESCE(st.skus_with_stock,0)::int AS skus_with_stock,
|
||||
CASE WHEN COALESCE(st.skus_with_stock,0) > 0
|
||||
THEN ROUND(100.0 * st.skus_in_stock / st.skus_with_stock)::int ELSE NULL END AS in_stock_pct,
|
||||
pm.price_delta_pct
|
||||
FROM sk
|
||||
LEFT JOIN sup ON sup.speed_gbps = sk.speed_gbps
|
||||
LEFT JOIN st ON st.speed_gbps = sk.speed_gbps
|
||||
LEFT JOIN pm ON pm.speed_gbps = sk.speed_gbps
|
||||
ORDER BY sk.speed_gbps DESC
|
||||
`);
|
||||
|
||||
type Row = { speed_gbps: string; sku_count: number; suppliers: number; suppliers_prior: number; skus_in_stock: number; skus_with_stock: number; in_stock_pct: number | null; price_delta_pct: string | null };
|
||||
const tiers = (result.rows as Row[]).map((r) => {
|
||||
const sup = r.suppliers ?? 0;
|
||||
const inStockPct = r.in_stock_pct ?? null;
|
||||
// availability class from supplier diversity + stock coverage
|
||||
let availability: "scarce" | "constrained" | "moderate" | "abundant";
|
||||
if (sup <= 3 || (inStockPct !== null && inStockPct < 20)) availability = "scarce";
|
||||
else if (sup <= 6 || (inStockPct !== null && inStockPct < 45)) availability = "constrained";
|
||||
else if (sup <= 9) availability = "moderate";
|
||||
else availability = "abundant";
|
||||
// tightening trend: supplier diversity dropped vs prior window
|
||||
const trend = r.suppliers_prior > 0
|
||||
? (sup < r.suppliers_prior ? "tightening" : sup > r.suppliers_prior ? "loosening" : "stable")
|
||||
: "stable";
|
||||
const priceDelta = r.price_delta_pct != null ? parseFloat(r.price_delta_pct) : null;
|
||||
const priceTrend = priceDelta === null ? "unknown" : priceDelta > 3 ? "rising" : priceDelta < -3 ? "falling" : "stable";
|
||||
const drivers: string[] = [`${sup} active suppliers`];
|
||||
if (trend === "tightening") drivers.push(`supplier base shrank ${r.suppliers_prior}→${sup} vs prior 45d`);
|
||||
else if (trend === "loosening") drivers.push(`supplier base grew ${r.suppliers_prior}→${sup} vs prior 45d`);
|
||||
if (inStockPct !== null) drivers.push(`${inStockPct}% of tracked SKUs in stock`);
|
||||
if (priceDelta !== null) drivers.push(`price ${priceDelta >= 0 ? "+" : ""}${priceDelta}% (30d, same-SKU median)`);
|
||||
drivers.push(`${r.sku_count} SKUs catalogued`);
|
||||
// Plain-language WHY, built only from real per-speed signals
|
||||
const reasons: string[] = [];
|
||||
if (availability === "scarce") reasons.push(sup <= 3 ? `only ${sup} supplier(s) offer it` : `near-zero stock coverage`);
|
||||
else if (availability === "constrained") reasons.push(`limited supplier base (${sup})`);
|
||||
if (trend === "tightening") reasons.push(`supplier base is shrinking`);
|
||||
if (priceTrend === "rising") reasons.push(`prices rising ${priceDelta}% on the same parts`);
|
||||
if (priceTrend === "falling") reasons.push(`prices easing ${priceDelta}%`);
|
||||
if (inStockPct !== null && inStockPct < 50) reasons.push(`${inStockPct}% in-stock coverage`);
|
||||
const why = reasons.length ? reasons.join("; ") : `broad supply: ${sup} suppliers, ${inStockPct ?? "n/a"}% in stock, stable pricing`;
|
||||
return {
|
||||
speed_gbps: parseFloat(r.speed_gbps),
|
||||
sku_count: r.sku_count,
|
||||
suppliers: sup,
|
||||
in_stock_pct: inStockPct,
|
||||
price_delta_pct: priceDelta,
|
||||
price_trend: priceTrend,
|
||||
availability,
|
||||
trend,
|
||||
why,
|
||||
drivers,
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ success: true, tiers });
|
||||
} catch (err) {
|
||||
try { await client.query('ROLLBACK'); } catch { /* ignore */ }
|
||||
console.error("availability error:", err);
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// Multi-signal supply constraint detector
|
||||
procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const [priceSignals, aiDemand, hypeData, stockData] = await Promise.all([
|
||||
// Price momentum: 30d vs 60d avg by speed/form_factor
|
||||
pool.query(`
|
||||
-- Per-SKU paired comparison: only transceivers present in BOTH periods.
|
||||
-- This eliminates catalog-composition bias (new expensive SKUs entering a
|
||||
-- speed/form-factor bucket would otherwise fake a huge price jump).
|
||||
WITH per_sku AS (
|
||||
SELECT
|
||||
t.id, t.speed_gbps, t.form_factor,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price)
|
||||
FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS med_now,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price)
|
||||
FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS med_prior,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_now,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS obs_prior
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.price > 5 AND po.currency = 'USD'
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
AND t.form_factor IN ('SFP','SFP+','SFP28','SFP56','QSFP+','QSFP28','QSFP56','QSFP-DD','QSFP-DD800','OSFP','OSFP-XD','XFP','CFP','CFP2','CFP4','CDFP','DSFP')
|
||||
AND po.price < 15000
|
||||
AND t.part_number NOT ILIKE '%AOC%'
|
||||
AND t.part_number NOT ILIKE '%-DAC-%'
|
||||
AND (t.standard_name IS NULL OR (t.standard_name NOT ILIKE '%Switch%' AND t.standard_name NOT ILIKE '%InfiniBand%'))
|
||||
AND t.speed_gbps > 0
|
||||
GROUP BY t.id, t.speed_gbps, t.form_factor
|
||||
)
|
||||
SELECT
|
||||
speed_gbps, form_factor,
|
||||
-- avg_30d / avg_prior_30d kept as column names for downstream compatibility,
|
||||
-- but they now carry MEDIAN-of-matched-SKU prices (only SKUs in both periods)
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY med_now) AS avg_30d,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY med_prior) AS avg_prior_30d,
|
||||
-- The real signal: median of per-SKU percentage deltas
|
||||
ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (med_now - med_prior) / NULLIF(med_prior,0) * 100)::numeric, 1) AS sku_median_delta_pct,
|
||||
COUNT(*) AS obs_30d
|
||||
FROM per_sku
|
||||
WHERE med_now IS NOT NULL AND med_prior IS NOT NULL AND med_prior > 0
|
||||
AND obs_now >= 2 AND obs_prior >= 2
|
||||
GROUP BY speed_gbps, form_factor
|
||||
HAVING COUNT(*) >= 3
|
||||
t.speed_gbps, t.form_factor,
|
||||
ROUND(AVG(po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days')::numeric,2) AS avg_30d,
|
||||
ROUND(AVG(po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days')::numeric,2) AS avg_prior_30d,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.price > 5 AND po.currency = 'USD'
|
||||
GROUP BY t.speed_gbps, t.form_factor
|
||||
HAVING COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') >= 3
|
||||
`),
|
||||
// AI cluster demand by speed tier (speed_tier = 0 excluded — unclassified announcements)
|
||||
// AI cluster demand by speed tier
|
||||
pool.query(`
|
||||
SELECT speed_tier, total_tx, cluster_count FROM (
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%800G%' THEN 800
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%400G%' THEN 400
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%100G%' THEN 100
|
||||
ELSE 0
|
||||
END AS speed_tier,
|
||||
COALESCE(SUM(estimated_transceivers),0)::int AS total_tx,
|
||||
COUNT(*)::int AS cluster_count
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
GROUP BY 1
|
||||
HAVING COALESCE(SUM(estimated_transceivers),0) > 0
|
||||
) t WHERE speed_tier > 0
|
||||
SELECT
|
||||
CASE
|
||||
WHEN description ILIKE '%800G%' THEN 800
|
||||
WHEN description ILIKE '%400G%' THEN 400
|
||||
WHEN description ILIKE '%100G%' THEN 100
|
||||
ELSE 0
|
||||
END AS speed_tier,
|
||||
COALESCE(SUM(estimated_transceivers),0)::int AS total_tx,
|
||||
COUNT(*)::int AS cluster_count
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
GROUP BY speed_tier
|
||||
HAVING COALESCE(SUM(estimated_transceivers),0) > 0
|
||||
`),
|
||||
// Hype phase per technology
|
||||
pool.query(`
|
||||
@ -995,7 +812,7 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
`).catch(() => ({ rows: [] })),
|
||||
]);
|
||||
|
||||
type PriceRow = { speed_gbps: string; form_factor: string; avg_30d: string; avg_prior_30d: string; obs_30d: string; sku_median_delta_pct: string | null };
|
||||
type PriceRow = { speed_gbps: string; form_factor: string; avg_30d: string; avg_prior_30d: string; obs_30d: string };
|
||||
type HypeRow = { technology: string; hype_phase: string; hype_score: string };
|
||||
type AiRow = { speed_tier: string; total_tx: string; cluster_count: string };
|
||||
type StockRow = { speed_gbps: string; form_factor: string; out_of_stock: string; in_stock: string; total_obs: string };
|
||||
@ -1021,12 +838,9 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
const signals = (priceSignals.rows as PriceRow[])
|
||||
.map((r) => {
|
||||
const speed = parseFloat(r.speed_gbps);
|
||||
// Prefer the per-SKU median delta (composition-bias-free); fall back to aggregate
|
||||
const priceUp = r.sku_median_delta_pct != null
|
||||
? parseFloat(r.sku_median_delta_pct)
|
||||
: (r.avg_30d && r.avg_prior_30d
|
||||
? ((parseFloat(r.avg_30d) - parseFloat(r.avg_prior_30d)) / parseFloat(r.avg_prior_30d)) * 100
|
||||
: 0);
|
||||
const priceUp = r.avg_30d && r.avg_prior_30d
|
||||
? ((parseFloat(r.avg_30d) - parseFloat(r.avg_prior_30d)) / parseFloat(r.avg_prior_30d)) * 100
|
||||
: 0;
|
||||
const hype = speedToHype.get(speed);
|
||||
const ai = aiBySpeed.get(speed);
|
||||
const stock = stockByKey.get(`${r.speed_gbps}:${r.form_factor}`);
|
||||
@ -1050,7 +864,7 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
activeSignals, severity, reasons,
|
||||
};
|
||||
})
|
||||
.filter((r) => r.activeSignals >= 1 && parseFloat(String(r.speed_gbps)) > 0)
|
||||
.filter((r) => r.activeSignals >= 1)
|
||||
.sort((a, b) => b.activeSignals - a.activeSignals || b.price_momentum_pct - a.price_momentum_pct);
|
||||
|
||||
res.json({ success: true, signals, criticalCount: signals.filter(s => s.severity === "critical").length });
|
||||
@ -1059,185 +873,6 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// GET /api/procurement/lead-times — Rolling lead-time trends per vendor/speed
|
||||
// Query params: form_factor, speed_gbps, days (default 90), limit (default 20)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
procurementRouter.get("/lead-times", async (req: Request, res: Response) => {
|
||||
const days = Math.min(parseInt(req.query.days as string) || 90, 365);
|
||||
const limit = Math.min(parseInt(req.query.limit as string) || 20, 50);
|
||||
const ff = req.query.form_factor as string | undefined;
|
||||
const spd = req.query.speed_gbps as string | undefined;
|
||||
|
||||
try {
|
||||
const [weekly, summary, concentration] = await Promise.all([
|
||||
// Weekly avg lead time per vendor × speed tier
|
||||
pool.query(`
|
||||
SELECT
|
||||
v.name AS vendor_name,
|
||||
v.id::text AS vendor_id,
|
||||
t.speed_gbps::text,
|
||||
t.form_factor,
|
||||
DATE_TRUNC('week', ss.scraped_at)::date AS week,
|
||||
ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_lead_days,
|
||||
ROUND(MIN(ss.lead_time_days)::numeric, 1) AS min_lead_days,
|
||||
ROUND(MAX(ss.lead_time_days)::numeric, 1) AS max_lead_days,
|
||||
COUNT(*)::int AS observations
|
||||
FROM stock_snapshots ss
|
||||
JOIN transceivers t ON t.id = ss.transceiver_id
|
||||
JOIN vendors v ON v.id = ss.source_vendor_id
|
||||
WHERE ss.scraped_at >= NOW() - INTERVAL '${days} days'
|
||||
AND ss.lead_time_days IS NOT NULL
|
||||
AND ss.lead_time_days > 0
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
GROUP BY v.name, v.id, t.speed_gbps, t.form_factor, DATE_TRUNC('week', ss.scraped_at)
|
||||
ORDER BY week DESC, avg_lead_days DESC
|
||||
LIMIT 500
|
||||
`),
|
||||
|
||||
// Overall summary: current vs prior-period avg per vendor
|
||||
pool.query(`
|
||||
WITH cur AS (
|
||||
SELECT
|
||||
v.name AS vendor_name, v.id::text AS vendor_id,
|
||||
ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_days,
|
||||
COUNT(*)::int AS obs
|
||||
FROM stock_snapshots ss
|
||||
JOIN vendors v ON v.id = ss.source_vendor_id
|
||||
JOIN transceivers t ON t.id = ss.transceiver_id
|
||||
WHERE ss.scraped_at >= NOW() - INTERVAL '30 days'
|
||||
AND ss.lead_time_days > 0
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
GROUP BY v.name, v.id
|
||||
),
|
||||
prior AS (
|
||||
SELECT v.id::text AS vendor_id,
|
||||
ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_days
|
||||
FROM stock_snapshots ss
|
||||
JOIN vendors v ON v.id = ss.source_vendor_id
|
||||
JOIN transceivers t ON t.id = ss.transceiver_id
|
||||
WHERE ss.scraped_at >= NOW() - INTERVAL '60 days'
|
||||
AND ss.scraped_at < NOW() - INTERVAL '30 days'
|
||||
AND ss.lead_time_days > 0
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
GROUP BY v.id
|
||||
)
|
||||
SELECT
|
||||
c.vendor_name, c.vendor_id,
|
||||
c.avg_days AS current_30d_avg,
|
||||
p.avg_days AS prior_30d_avg,
|
||||
ROUND((c.avg_days - COALESCE(p.avg_days, c.avg_days))::numeric, 1) AS delta_days,
|
||||
c.obs
|
||||
FROM cur c
|
||||
LEFT JOIN prior p ON p.vendor_id = c.vendor_id
|
||||
ORDER BY c.avg_days DESC
|
||||
LIMIT ${limit}
|
||||
`),
|
||||
|
||||
// Speed-tier breakdown — which form factors have longest lead times right now
|
||||
pool.query(`
|
||||
SELECT
|
||||
t.speed_gbps::text, t.form_factor,
|
||||
ROUND(AVG(ss.lead_time_days)::numeric, 1) AS avg_lead_days,
|
||||
COUNT(DISTINCT ss.source_vendor_id)::int AS vendors_reporting,
|
||||
COUNT(*)::int AS total_obs
|
||||
FROM stock_snapshots ss
|
||||
JOIN transceivers t ON t.id = ss.transceiver_id
|
||||
WHERE ss.scraped_at >= NOW() - INTERVAL '30 days'
|
||||
AND ss.lead_time_days > 0
|
||||
GROUP BY t.speed_gbps, t.form_factor
|
||||
HAVING COUNT(*) >= 3
|
||||
ORDER BY avg_lead_days DESC
|
||||
LIMIT 20
|
||||
`),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
filters: { days, form_factor: ff || null, speed_gbps: spd || null },
|
||||
weekly_trend: weekly.rows,
|
||||
vendor_summary: summary.rows,
|
||||
speed_tier_breakdown: concentration.rows,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// GET /api/procurement/supply-concentration — Single-vendor dependency risk
|
||||
// Flags SKUs where >70% of price observations come from one vendor (30d)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
procurementRouter.get("/supply-concentration", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
WITH obs_30d AS (
|
||||
SELECT
|
||||
po.transceiver_id,
|
||||
po.source_vendor_id,
|
||||
COUNT(*) AS vendor_obs
|
||||
FROM price_observations po
|
||||
WHERE po.time >= NOW() - INTERVAL '30 days'
|
||||
AND po.price > 0
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
GROUP BY po.transceiver_id, po.source_vendor_id
|
||||
),
|
||||
totals AS (
|
||||
SELECT transceiver_id, SUM(vendor_obs) AS total_obs
|
||||
FROM obs_30d GROUP BY transceiver_id
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
o.transceiver_id,
|
||||
o.source_vendor_id,
|
||||
o.vendor_obs,
|
||||
t.total_obs,
|
||||
ROUND((o.vendor_obs::numeric / NULLIF(t.total_obs,0)) * 100, 1) AS share_pct,
|
||||
ROW_NUMBER() OVER (PARTITION BY o.transceiver_id ORDER BY o.vendor_obs DESC) AS rnk
|
||||
FROM obs_30d o JOIN totals t ON t.transceiver_id = o.transceiver_id
|
||||
)
|
||||
SELECT
|
||||
tx.id::text, tx.part_number, tx.form_factor,
|
||||
tx.speed_gbps::text,
|
||||
tx.standard_name,
|
||||
v.name AS dominant_vendor,
|
||||
r.share_pct,
|
||||
r.total_obs::int,
|
||||
r.vendor_obs::int AS dominant_obs,
|
||||
CASE
|
||||
WHEN r.share_pct >= 90 THEN 'critical'
|
||||
WHEN r.share_pct >= 75 THEN 'high'
|
||||
ELSE 'medium'
|
||||
END AS risk_level
|
||||
FROM ranked r
|
||||
JOIN transceivers tx ON tx.id = r.transceiver_id
|
||||
JOIN vendors v ON v.id = r.source_vendor_id
|
||||
WHERE r.rnk = 1
|
||||
AND r.share_pct >= 70
|
||||
AND r.total_obs >= 5
|
||||
ORDER BY r.share_pct DESC
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
const rows = result.rows;
|
||||
res.json({
|
||||
success: true,
|
||||
concentrated: rows,
|
||||
stats: {
|
||||
total_at_risk: rows.length,
|
||||
critical: rows.filter(r => r.risk_level === "critical").length,
|
||||
high: rows.filter(r => r.risk_level === "high").length,
|
||||
medium: rows.filter(r => r.risk_level === "medium").length,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/procurement/price-movers — SKUs with biggest price delta vs prior period
|
||||
procurementRouter.get("/price-movers", async (req: Request, res: Response) => {
|
||||
const days = Math.min(parseInt(req.query.days as string) || 7, 90);
|
||||
@ -1246,68 +881,41 @@ procurementRouter.get("/price-movers", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
WITH cur AS (
|
||||
-- Group by part_number+source_vendor+currency to avoid duplicates from multiple
|
||||
-- vendor-OEM transceiver_ids with the same part number.
|
||||
-- Use PERCENTILE_CONT (median) to suppress multi-tier list-price noise
|
||||
-- (e.g. Mouser 1x/10x/100x tiers appearing as price swings).
|
||||
SELECT t.part_number, po.source_vendor_id, po.currency,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) AS med_price,
|
||||
SELECT transceiver_id, source_vendor_id, currency,
|
||||
AVG(price) AS avg_price,
|
||||
COUNT(*) AS obs
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '${days} days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
GROUP BY t.part_number, po.source_vendor_id, po.currency
|
||||
FROM price_observations
|
||||
WHERE time >= NOW() - INTERVAL '${days} days'
|
||||
AND price > 0 AND COALESCE(is_anomalous, false) = false
|
||||
GROUP BY transceiver_id, source_vendor_id, currency
|
||||
),
|
||||
prior AS (
|
||||
SELECT t.part_number, po.source_vendor_id,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) AS med_price,
|
||||
COUNT(*) AS obs
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND po.time < NOW() - INTERVAL '${days} days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
GROUP BY t.part_number, po.source_vendor_id
|
||||
),
|
||||
ref_tx AS (
|
||||
-- pick one canonical transceiver_id per part_number for metadata
|
||||
SELECT DISTINCT ON (part_number) id, part_number, form_factor, speed_gbps, standard_name
|
||||
FROM transceivers ORDER BY part_number, id
|
||||
SELECT transceiver_id, source_vendor_id,
|
||||
AVG(price) AS avg_price
|
||||
FROM price_observations
|
||||
WHERE time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND time < NOW() - INTERVAL '${days} days'
|
||||
AND price > 0 AND COALESCE(is_anomalous, false) = false
|
||||
GROUP BY transceiver_id, source_vendor_id
|
||||
)
|
||||
SELECT
|
||||
ref.id, ref.part_number, ref.form_factor,
|
||||
ref.speed_gbps::text AS speed_gbps,
|
||||
ref.standard_name,
|
||||
t.id, t.part_number, t.form_factor,
|
||||
t.speed_gbps::text AS speed_gbps,
|
||||
t.standard_name,
|
||||
sv.name AS vendor_name,
|
||||
ROUND(c.med_price::numeric, 2) AS current_avg,
|
||||
ROUND(p.med_price::numeric, 2) AS prior_avg,
|
||||
ROUND(((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100)::numeric, 1) AS delta_pct,
|
||||
ROUND(c.avg_price::numeric, 2) AS current_avg,
|
||||
ROUND(p.avg_price::numeric, 2) AS prior_avg,
|
||||
ROUND(((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100)::numeric, 1) AS delta_pct,
|
||||
c.currency,
|
||||
(c.obs + p.obs)::int AS observations
|
||||
c.obs::int AS observations
|
||||
FROM cur c
|
||||
JOIN prior p ON p.part_number = c.part_number
|
||||
AND p.source_vendor_id = c.source_vendor_id
|
||||
JOIN ref_tx ref ON ref.part_number = c.part_number
|
||||
JOIN vendors sv ON sv.id = c.source_vendor_id
|
||||
-- cv_filter: exclude SKUs where the source has high price variance across the full
|
||||
-- 2*days window (e.g. Mouser quantity-tier noise). CV > 0.35 = unreliable source.
|
||||
JOIN (
|
||||
SELECT t2.part_number, po2.source_vendor_id,
|
||||
STDDEV(po2.price) / NULLIF(AVG(po2.price), 0) AS cv
|
||||
FROM price_observations po2
|
||||
JOIN transceivers t2 ON t2.id = po2.transceiver_id
|
||||
WHERE po2.time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND po2.price > 0 AND COALESCE(po2.is_anomalous, false) = false
|
||||
GROUP BY t2.part_number, po2.source_vendor_id
|
||||
HAVING COUNT(*) >= 2
|
||||
) cv_filter
|
||||
ON cv_filter.part_number = c.part_number
|
||||
AND cv_filter.source_vendor_id = c.source_vendor_id
|
||||
WHERE ABS((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100) >= 2
|
||||
AND (c.obs + p.obs) >= 4
|
||||
AND COALESCE(cv_filter.cv, 0) <= 0.35
|
||||
ORDER BY ABS((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100) DESC
|
||||
JOIN prior p ON p.transceiver_id = c.transceiver_id
|
||||
AND p.source_vendor_id = c.source_vendor_id
|
||||
JOIN transceivers t ON t.id = c.transceiver_id
|
||||
JOIN vendors sv ON sv.id = c.source_vendor_id
|
||||
WHERE ABS((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100) >= 2
|
||||
AND c.obs::int >= 2
|
||||
ORDER BY ABS((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100) DESC
|
||||
LIMIT ${limit * 2}
|
||||
`);
|
||||
|
||||
|
||||
@ -1,210 +0,0 @@
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const researchRobotRouter = Router();
|
||||
|
||||
// Known job queues are validated against pgboss.queue before any action.
|
||||
async function isKnownQueue(name: string): Promise<boolean> {
|
||||
const r = await pool.query("SELECT 1 FROM pgboss.queue WHERE name = $1", [name]);
|
||||
return (r.rowCount ?? 0) > 0;
|
||||
}
|
||||
|
||||
type Recommendation = {
|
||||
job: string;
|
||||
severity: "critical" | "warning" | "info";
|
||||
title: string;
|
||||
detail: string;
|
||||
cause: string;
|
||||
// Which actions the UI should offer for this job
|
||||
actions: Array<"dispatch" | "pause" | "resume" | "token_help">;
|
||||
failed_count?: number;
|
||||
last_error?: string;
|
||||
};
|
||||
|
||||
// Classify a job error message into a human cause + recommended actions.
|
||||
function classifyError(jobName: string, errorText: string): Omit<Recommendation, "job" | "failed_count" | "last_error"> {
|
||||
const e = (errorText || "").toLowerCase();
|
||||
|
||||
if (/http 401|http 403|unauthorized|forbidden|invalid.*token|api.*key/.test(e)) {
|
||||
return {
|
||||
severity: "critical",
|
||||
title: `${jobName}: Authentifizierung fehlgeschlagen`,
|
||||
detail: "Der Job erreicht die Quelle, wird aber abgewiesen (401/403). Meist ein fehlender oder abgelaufener API-Token. Ohne gültigen Token reiht der Robot den Job endlos neu ein und er schlägt jedes Mal fehl.",
|
||||
cause: "auth",
|
||||
actions: ["token_help", "pause"],
|
||||
};
|
||||
}
|
||||
if (/timeout|etimedout|econnrefused|enotfound|network|socket hang up|fetch failed/.test(e)) {
|
||||
return {
|
||||
severity: "warning",
|
||||
title: `${jobName}: Quelle nicht erreichbar`,
|
||||
detail: "Netzwerk-/Verbindungsfehler beim Abruf. Oft temporär (Ziel-Server kurz down, Rate-Limit). Manuell erneut auslösen oder pausieren, falls die Quelle dauerhaft weg ist.",
|
||||
cause: "network",
|
||||
actions: ["dispatch", "pause"],
|
||||
};
|
||||
}
|
||||
if (/no handler|not registered|unknown job|kein handler/.test(e)) {
|
||||
return {
|
||||
severity: "critical",
|
||||
title: `${jobName}: Kein Handler registriert`,
|
||||
detail: "Der Job ist im Schedule, aber es gibt keinen Code-Handler dafür. Er kann nie laufen — entweder Handler nachrüsten oder den Job aus dem Schedule entfernen.",
|
||||
cause: "no_handler",
|
||||
actions: ["pause"],
|
||||
};
|
||||
}
|
||||
return {
|
||||
severity: "warning",
|
||||
title: `${jobName}: Wiederholt fehlgeschlagen`,
|
||||
detail: "Der Job schlägt regelmäßig fehl. Letzte Fehlermeldung siehe unten. Manuell auslösen zum erneuten Versuch, oder pausieren falls überflüssig.",
|
||||
cause: "other",
|
||||
actions: ["dispatch", "pause"],
|
||||
};
|
||||
}
|
||||
|
||||
// GET /api/research-robot — letzte Robot-Laeufe + abgeleitete Handlungsempfehlungen
|
||||
researchRobotRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = await pool.query(
|
||||
"SELECT run_at, freshness, decision, dispatched, model FROM research_robot_runs ORDER BY run_at DESC LIMIT 20"
|
||||
);
|
||||
|
||||
// Build recommendations from LIVE pgboss state (more accurate than the stored summary).
|
||||
const recommendations: Recommendation[] = [];
|
||||
try {
|
||||
const failing = await pool.query<{ name: string; failed: string; last_error: string | null }>(`
|
||||
SELECT j.name,
|
||||
COUNT(*) FILTER (WHERE j.state = 'failed') AS failed,
|
||||
(SELECT jj.output::text FROM pgboss.job jj
|
||||
WHERE jj.name = j.name AND jj.state = 'failed' AND jj.output IS NOT NULL
|
||||
ORDER BY jj.created_on DESC LIMIT 1) AS last_error
|
||||
FROM pgboss.job j
|
||||
WHERE j.created_on > NOW() - INTERVAL '3 days'
|
||||
GROUP BY j.name
|
||||
HAVING COUNT(*) FILTER (WHERE j.state = 'failed') >= 2
|
||||
AND COUNT(*) FILTER (WHERE j.state = 'completed') = 0
|
||||
ORDER BY failed DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
// Which jobs are currently scheduled (so we can offer pause vs resume correctly)
|
||||
const scheduled = await pool.query<{ name: string }>("SELECT name FROM pgboss.schedule");
|
||||
const scheduledSet = new Set(scheduled.rows.map((s) => s.name));
|
||||
|
||||
for (const row of failing.rows) {
|
||||
let errMsg = "";
|
||||
try {
|
||||
const parsed = JSON.parse(row.last_error || "{}") as { stack?: string; message?: string };
|
||||
errMsg = (parsed.stack || parsed.message || row.last_error || "").slice(0, 300);
|
||||
} catch {
|
||||
errMsg = (row.last_error || "").slice(0, 300);
|
||||
}
|
||||
const cls = classifyError(row.name, errMsg);
|
||||
// If the job isn't scheduled, swap pause→resume in the offered actions
|
||||
const actions = cls.actions.map((a) =>
|
||||
a === "pause" && !scheduledSet.has(row.name) ? "resume" : a
|
||||
) as Recommendation["actions"];
|
||||
recommendations.push({
|
||||
job: row.name,
|
||||
severity: cls.severity,
|
||||
title: cls.title,
|
||||
detail: cls.detail,
|
||||
cause: cls.cause,
|
||||
actions: [...new Set(actions)],
|
||||
failed_count: parseInt(row.failed, 10),
|
||||
last_error: errMsg.split("\n")[0].slice(0, 160),
|
||||
});
|
||||
}
|
||||
} catch { /* pgboss introspection best-effort */ }
|
||||
|
||||
if (recommendations.length === 0) {
|
||||
recommendations.push({
|
||||
job: "",
|
||||
severity: "info",
|
||||
title: "Alle Scraper-Jobs gesund",
|
||||
detail: "Keine dauerhaft fehlschlagenden Jobs erkannt. Keine Aktion nötig.",
|
||||
cause: "healthy",
|
||||
actions: [],
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ success: true, runs: r.rows, recommendations });
|
||||
} catch (e) {
|
||||
res.json({ success: true, runs: [], recommendations: [] });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/research-robot/action — { action, job }
|
||||
// dispatch → Job sofort in die Queue stellen; pause → aus Schedule entfernen (Backup);
|
||||
// resume → aus Backup wiederherstellen.
|
||||
researchRobotRouter.post("/action", async (req: Request, res: Response) => {
|
||||
const action = String((req.body?.action ?? "")).trim();
|
||||
const job = String((req.body?.job ?? "")).trim();
|
||||
|
||||
if (!["dispatch", "pause", "resume"].includes(action)) {
|
||||
res.status(400).json({ success: false, error: "Unbekannte Aktion." });
|
||||
return;
|
||||
}
|
||||
if (!job || !(await isKnownQueue(job))) {
|
||||
res.status(400).json({ success: false, error: "Unbekannter oder ungültiger Job." });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (action === "dispatch") {
|
||||
// Enqueue one job run immediately (pg-boss v10 standard policy, mirrors a scheduled send)
|
||||
await pool.query(
|
||||
`INSERT INTO pgboss.job (id, name, data, priority, retry_limit, retry_delay, expire_in, keep_until, policy, start_after)
|
||||
VALUES (gen_random_uuid(), $1, '{}'::jsonb, 0, 2, 0, INTERVAL '15 minutes', NOW() + INTERVAL '14 days', 'standard', NOW())`,
|
||||
[job]
|
||||
);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' wurde in die Queue gestellt und läuft beim nächsten Worker-Tick.` });
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure backup table for paused schedules exists
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS research_robot_paused_schedules (
|
||||
name TEXT PRIMARY KEY,
|
||||
cron TEXT, timezone TEXT, data JSONB, options JSONB,
|
||||
paused_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)`);
|
||||
|
||||
if (action === "pause") {
|
||||
const sched = await pool.query("SELECT name, cron, timezone, data, options FROM pgboss.schedule WHERE name = $1", [job]);
|
||||
if (sched.rowCount === 0) {
|
||||
res.json({ success: true, action, job, message: `Job '${job}' war nicht im Schedule — bereits pausiert.` });
|
||||
return;
|
||||
}
|
||||
const s = sched.rows[0];
|
||||
await pool.query(
|
||||
`INSERT INTO research_robot_paused_schedules (name, cron, timezone, data, options, paused_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (name) DO UPDATE SET cron=$2, timezone=$3, data=$4, options=$5, paused_at=NOW()`,
|
||||
[s.name, s.cron, s.timezone, s.data, s.options]
|
||||
);
|
||||
await pool.query("DELETE FROM pgboss.schedule WHERE name = $1", [job]);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' pausiert (aus Schedule entfernt). Der Robot reiht ihn nicht mehr neu ein. Reversibel über 'Fortsetzen'.` });
|
||||
return;
|
||||
}
|
||||
|
||||
if (action === "resume") {
|
||||
const bak = await pool.query("SELECT name, cron, timezone, data, options FROM research_robot_paused_schedules WHERE name = $1", [job]);
|
||||
if (bak.rowCount === 0) {
|
||||
res.status(404).json({ success: false, error: `Kein pausierter Schedule-Eintrag für '${job}' gefunden.` });
|
||||
return;
|
||||
}
|
||||
const s = bak.rows[0];
|
||||
await pool.query(
|
||||
`INSERT INTO pgboss.schedule (name, cron, timezone, data, options, created_on, updated_on)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW(),NOW())
|
||||
ON CONFLICT (name) DO UPDATE SET cron=$2, timezone=$3, data=$4, options=$5, updated_on=NOW()`,
|
||||
[s.name, s.cron, s.timezone, s.data, s.options]
|
||||
);
|
||||
await pool.query("DELETE FROM research_robot_paused_schedules WHERE name = $1", [job]);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' wieder aktiviert (zurück im Schedule).` });
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: (e as Error).message });
|
||||
}
|
||||
});
|
||||
@ -1,215 +0,0 @@
|
||||
/**
|
||||
* RFQ Analyzer — POST /api/rfq/analyze
|
||||
*
|
||||
* Paste a vendor quote (list of part numbers + quantities + prices) and
|
||||
* get back: current market rates, cheapest alternative via equivalences,
|
||||
* total savings opportunity, and per-line delta.
|
||||
*
|
||||
* Request body:
|
||||
* {
|
||||
* lines: [
|
||||
* { part_number: string, quantity: number, unit_price: number, currency?: string }
|
||||
* ],
|
||||
* currency?: "USD" | "EUR"
|
||||
* }
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { sendCSV } from "../utils/csv";
|
||||
|
||||
export const rfqRouter = Router();
|
||||
|
||||
interface RfqLine {
|
||||
part_number: string;
|
||||
quantity: number;
|
||||
unit_price: number;
|
||||
currency?: string;
|
||||
}
|
||||
|
||||
// POST /api/rfq/analyze
|
||||
rfqRouter.post("/analyze", async (req: Request, res: Response) => {
|
||||
const { lines, currency: preferredCurrency = "USD" } = req.body as {
|
||||
lines: RfqLine[];
|
||||
currency?: string;
|
||||
};
|
||||
|
||||
if (!Array.isArray(lines) || lines.length === 0) {
|
||||
return res.status(400).json({ success: false, error: "lines[] required" });
|
||||
}
|
||||
if (lines.length > 200) {
|
||||
return res.status(400).json({ success: false, error: "Max 200 lines per RFQ" });
|
||||
}
|
||||
|
||||
try {
|
||||
const results = await Promise.all(
|
||||
lines.map(async (line) => {
|
||||
const pn = String(line.part_number || "").trim();
|
||||
const qty = Math.max(1, Number(line.quantity) || 1);
|
||||
const quotedPrice = Number(line.unit_price) || 0;
|
||||
|
||||
if (!pn) {
|
||||
return { part_number: pn, error: "Empty part_number", resolved: false };
|
||||
}
|
||||
|
||||
// 1. Resolve transceiver by part_number or standard_name
|
||||
const txResult = await pool.query(
|
||||
`SELECT id, part_number, standard_name, form_factor, speed_gbps, speed
|
||||
FROM transceivers
|
||||
WHERE part_number ILIKE $1 OR standard_name ILIKE $1
|
||||
LIMIT 1`,
|
||||
[pn]
|
||||
);
|
||||
|
||||
if (txResult.rows.length === 0) {
|
||||
return { part_number: pn, quantity: qty, quoted_unit_price: quotedPrice, resolved: false, error: "Not found in catalog" };
|
||||
}
|
||||
|
||||
const tx = txResult.rows[0];
|
||||
|
||||
// 2. Get current market prices for this transceiver
|
||||
const marketResult = await pool.query(
|
||||
`SELECT
|
||||
v.name AS vendor_name, v.website,
|
||||
po.price, po.currency, po.stock_level, po.url, po.time AS observed_at
|
||||
FROM (
|
||||
SELECT DISTINCT ON (source_vendor_id)
|
||||
source_vendor_id, price, currency, stock_level, url, time
|
||||
FROM price_observations
|
||||
WHERE transceiver_id = $1
|
||||
AND price > 0
|
||||
AND COALESCE(is_anomalous, false) = false
|
||||
ORDER BY source_vendor_id, time DESC
|
||||
) po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
WHERE po.currency = $2 OR po.currency IS NULL
|
||||
ORDER BY po.price ASC
|
||||
LIMIT 10`,
|
||||
[tx.id, preferredCurrency]
|
||||
);
|
||||
|
||||
const prices = marketResult.rows.map(r => parseFloat(r.price)).filter(p => p > 0);
|
||||
const marketMin = prices.length ? Math.min(...prices) : null;
|
||||
const marketAvg = prices.length ? Math.round(prices.reduce((a, b) => a + b) / prices.length * 100) / 100 : null;
|
||||
const cheapestVendor = marketResult.rows[0] || null;
|
||||
|
||||
// 3. Find cheapest equivalent via transceiver_equivalences
|
||||
const equivResult = await pool.query(
|
||||
`SELECT
|
||||
te.competitor_id, t2.part_number AS equiv_part_number,
|
||||
t2.standard_name AS equiv_standard_name,
|
||||
te.confidence, te.match_basis,
|
||||
(
|
||||
SELECT po2.price FROM price_observations po2
|
||||
WHERE po2.transceiver_id = te.competitor_id
|
||||
AND po2.currency = $2
|
||||
AND po2.price > 0
|
||||
AND COALESCE(po2.is_anomalous, false) = false
|
||||
ORDER BY po2.time DESC LIMIT 1
|
||||
) AS equiv_price,
|
||||
(
|
||||
SELECT v2.name FROM price_observations po2
|
||||
JOIN vendors v2 ON v2.id = po2.source_vendor_id
|
||||
WHERE po2.transceiver_id = te.competitor_id
|
||||
AND po2.currency = $2 AND po2.price > 0
|
||||
ORDER BY po2.price ASC, po2.time DESC LIMIT 1
|
||||
) AS equiv_cheapest_vendor
|
||||
FROM transceiver_equivalences te
|
||||
JOIN transceivers t2 ON t2.id = te.competitor_id
|
||||
WHERE (te.flexoptix_id = $1 OR te.competitor_id = $1)
|
||||
AND te.status = 'approved'
|
||||
AND te.confidence >= 0.7
|
||||
ORDER BY te.confidence DESC
|
||||
LIMIT 5`,
|
||||
[tx.id, preferredCurrency]
|
||||
);
|
||||
|
||||
const equivalents = equivResult.rows
|
||||
.filter(e => e.equiv_price !== null)
|
||||
.map(e => ({
|
||||
part_number: e.equiv_part_number,
|
||||
standard_name: e.equiv_standard_name,
|
||||
confidence: parseFloat(e.confidence),
|
||||
match_basis: e.match_basis,
|
||||
unit_price: parseFloat(e.equiv_price),
|
||||
vendor: e.equiv_cheapest_vendor,
|
||||
}));
|
||||
|
||||
const cheapestEquiv = equivalents.sort((a, b) => a.unit_price - b.unit_price)[0] || null;
|
||||
|
||||
// 4. Calculate savings
|
||||
const savingsVsMarketMin = marketMin !== null && quotedPrice > 0
|
||||
? Math.round((quotedPrice - marketMin) * qty * 100) / 100
|
||||
: null;
|
||||
const savingsVsEquiv = cheapestEquiv && quotedPrice > 0
|
||||
? Math.round((quotedPrice - cheapestEquiv.unit_price) * qty * 100) / 100
|
||||
: null;
|
||||
|
||||
return {
|
||||
part_number: pn,
|
||||
resolved: true,
|
||||
transceiver: {
|
||||
id: tx.id,
|
||||
standard_name: tx.standard_name,
|
||||
form_factor: tx.form_factor,
|
||||
speed: tx.speed,
|
||||
},
|
||||
quantity: qty,
|
||||
quoted_unit_price: quotedPrice,
|
||||
quoted_total: Math.round(quotedPrice * qty * 100) / 100,
|
||||
market: {
|
||||
min_price: marketMin,
|
||||
avg_price: marketAvg,
|
||||
vendor_count: prices.length,
|
||||
cheapest_vendor: cheapestVendor ? {
|
||||
name: cheapestVendor.vendor_name,
|
||||
price: parseFloat(cheapestVendor.price),
|
||||
stock_level: cheapestVendor.stock_level,
|
||||
url: cheapestVendor.url,
|
||||
} : null,
|
||||
},
|
||||
equivalents,
|
||||
cheapest_equivalent: cheapestEquiv,
|
||||
savings: {
|
||||
vs_market_min: savingsVsMarketMin,
|
||||
vs_equiv: savingsVsEquiv,
|
||||
best_saving: Math.max(savingsVsMarketMin || 0, savingsVsEquiv || 0) || null,
|
||||
},
|
||||
currency: preferredCurrency,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
// Aggregate totals
|
||||
const resolved = results.filter(r => r.resolved);
|
||||
const totalQuoted = resolved.reduce((s, r) => s + (r.quoted_total as number || 0), 0);
|
||||
const totalSavingsMarket = resolved.reduce((s, r) => {
|
||||
const sv = (r.savings as any)?.vs_market_min;
|
||||
return s + (sv && sv > 0 ? sv : 0);
|
||||
}, 0);
|
||||
const totalSavingsEquiv = resolved.reduce((s, r) => {
|
||||
const sv = (r.savings as any)?.vs_equiv;
|
||||
return s + (sv && sv > 0 ? sv : 0);
|
||||
}, 0);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
currency: preferredCurrency,
|
||||
line_count: lines.length,
|
||||
resolved_count: resolved.length,
|
||||
lines: results,
|
||||
totals: {
|
||||
quoted: Math.round(totalQuoted * 100) / 100,
|
||||
potential_savings_vs_market: Math.round(totalSavingsMarket * 100) / 100,
|
||||
potential_savings_vs_equiv: Math.round(totalSavingsEquiv * 100) / 100,
|
||||
best_total_saving: Math.round(Math.max(totalSavingsMarket, totalSavingsEquiv) * 100) / 100,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/rfq/export — Export last RFQ result as CSV (pass lines as query params)
|
||||
rfqRouter.get("/export", async (req: Request, res: Response) => {
|
||||
res.status(405).json({ error: "Use POST /api/rfq/analyze with ?format=csv to export" });
|
||||
});
|
||||
@ -1,178 +0,0 @@
|
||||
/**
|
||||
* ROI Calculator — /api/roi
|
||||
*
|
||||
* Calculates total cost of ownership and switching savings for transceiver decisions.
|
||||
*
|
||||
* POST /api/roi/calculate
|
||||
* Input:
|
||||
* {
|
||||
* ports: number, — Number of ports to equip
|
||||
* current_price_per_port: number,
|
||||
* target_form_factor?: string,
|
||||
* target_speed_gbps?: number,
|
||||
* years?: number, — TCO horizon (default 3)
|
||||
* switch_cost?: number, — One-time switching cost (labor, downtime est.)
|
||||
* currency?: string
|
||||
* }
|
||||
* Output:
|
||||
* - Current total cost (ports × price)
|
||||
* - Market min/avg for target spec
|
||||
* - 1/2/3 year TCO at current vs market-min price
|
||||
* - Savings over TCO horizon
|
||||
* - Break-even months
|
||||
* - Top 5 cheapest vendors for the target spec
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const roiRouter = Router();
|
||||
|
||||
roiRouter.post("/calculate", async (req: Request, res: Response) => {
|
||||
const {
|
||||
ports,
|
||||
current_price_per_port,
|
||||
target_form_factor,
|
||||
target_speed_gbps,
|
||||
years = 3,
|
||||
switch_cost = 0,
|
||||
currency = "USD",
|
||||
} = req.body as Record<string, any>;
|
||||
|
||||
if (!ports || isNaN(parseInt(ports)) || parseInt(ports) <= 0) {
|
||||
return res.status(400).json({ success: false, error: "ports must be a positive integer" });
|
||||
}
|
||||
if (!current_price_per_port || isNaN(parseFloat(current_price_per_port))) {
|
||||
return res.status(400).json({ success: false, error: "current_price_per_port required" });
|
||||
}
|
||||
if (!target_form_factor && !target_speed_gbps) {
|
||||
return res.status(400).json({ success: false, error: "Provide target_form_factor and/or target_speed_gbps" });
|
||||
}
|
||||
|
||||
const portCount = parseInt(ports);
|
||||
const currentPrice = parseFloat(current_price_per_port);
|
||||
const switchingCost = parseFloat(switch_cost) || 0;
|
||||
const tcoYears = Math.min(Math.max(parseInt(years) || 3, 1), 10);
|
||||
const curr = String(currency).toUpperCase();
|
||||
|
||||
try {
|
||||
// Find market prices for target spec
|
||||
const marketResult = await pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id, po.source_vendor_id, po.price, po.currency, po.stock_level, po.url, po.time
|
||||
FROM price_observations po
|
||||
WHERE po.price > 0
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
AND po.currency = $1
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
v.name AS vendor_name,
|
||||
v.website,
|
||||
t.standard_name,
|
||||
t.form_factor,
|
||||
t.speed_gbps::text,
|
||||
ROUND(MIN(l.price)::numeric, 2) AS min_price,
|
||||
ROUND(AVG(l.price)::numeric, 2) AS avg_price,
|
||||
COUNT(DISTINCT l.source_vendor_id)::int AS vendor_count,
|
||||
l.price AS vendor_price,
|
||||
l.stock_level,
|
||||
l.url
|
||||
FROM latest l
|
||||
JOIN transceivers t ON t.id = l.transceiver_id
|
||||
JOIN vendors v ON v.id = l.source_vendor_id
|
||||
WHERE ($2::text IS NULL OR t.form_factor = $2)
|
||||
AND ($3::numeric IS NULL OR t.speed_gbps = $3)
|
||||
GROUP BY v.name, v.website, t.standard_name, t.form_factor, t.speed_gbps,
|
||||
l.price, l.stock_level, l.url
|
||||
ORDER BY l.price ASC
|
||||
LIMIT 50
|
||||
`, [curr, target_form_factor || null, target_speed_gbps ? parseFloat(target_speed_gbps) : null]);
|
||||
|
||||
const allPrices = marketResult.rows.map(r => parseFloat(r.vendor_price)).filter(p => p > 0);
|
||||
const marketMin = allPrices.length ? Math.min(...allPrices) : null;
|
||||
const marketAvg = allPrices.length ? Math.round(allPrices.reduce((a,b) => a+b) / allPrices.length * 100) / 100 : null;
|
||||
|
||||
// Top 5 cheapest vendors (distinct vendors)
|
||||
const vendorPrices = new Map<string, { vendor_name: string; price: number; stock_level: string; url: string; standard_name: string }>();
|
||||
for (const r of marketResult.rows) {
|
||||
if (!vendorPrices.has(r.vendor_name)) {
|
||||
vendorPrices.set(r.vendor_name, {
|
||||
vendor_name: r.vendor_name,
|
||||
price: parseFloat(r.vendor_price),
|
||||
stock_level: r.stock_level,
|
||||
url: r.url,
|
||||
standard_name: r.standard_name,
|
||||
});
|
||||
}
|
||||
}
|
||||
const top5Vendors = [...vendorPrices.values()].slice(0, 5);
|
||||
|
||||
// TCO calculations
|
||||
const currentTotal = Math.round(portCount * currentPrice * 100) / 100;
|
||||
const marketMinTotal = marketMin !== null ? Math.round(portCount * marketMin * 100) / 100 : null;
|
||||
const marketAvgTotal = marketAvg !== null ? Math.round(portCount * marketAvg * 100) / 100 : null;
|
||||
|
||||
// Annual OpEx: assume 15% of hardware cost for maintenance/replacement (industry standard)
|
||||
const annualOpExFactor = 0.15;
|
||||
const currentTCO = Math.round((currentTotal + (currentTotal * annualOpExFactor * tcoYears)) * 100) / 100;
|
||||
const targetTCOMin = marketMinTotal !== null
|
||||
? Math.round((marketMinTotal + switchingCost + (marketMinTotal * annualOpExFactor * tcoYears)) * 100) / 100
|
||||
: null;
|
||||
const targetTCOAvg = marketAvgTotal !== null
|
||||
? Math.round((marketAvgTotal + switchingCost + (marketAvgTotal * annualOpExFactor * tcoYears)) * 100) / 100
|
||||
: null;
|
||||
|
||||
const savingsMin = targetTCOMin !== null ? Math.round((currentTCO - targetTCOMin) * 100) / 100 : null;
|
||||
const savingsAvg = targetTCOAvg !== null ? Math.round((currentTCO - targetTCOAvg) * 100) / 100 : null;
|
||||
|
||||
// Break-even: months until switching cost is recovered from per-unit savings
|
||||
const monthlyHardwareSavingsMin = marketMin !== null
|
||||
? Math.round(portCount * (currentPrice - marketMin) / 12 * 100) / 100
|
||||
: null;
|
||||
const breakEvenMonths = monthlyHardwareSavingsMin && monthlyHardwareSavingsMin > 0 && switchingCost > 0
|
||||
? Math.ceil(switchingCost / monthlyHardwareSavingsMin)
|
||||
: 0;
|
||||
|
||||
return res.json({
|
||||
success: true,
|
||||
input: {
|
||||
ports: portCount, current_price_per_port: currentPrice,
|
||||
target_form_factor: target_form_factor || null,
|
||||
target_speed_gbps: target_speed_gbps ? parseFloat(target_speed_gbps) : null,
|
||||
tco_years: tcoYears, switch_cost: switchingCost, currency: curr,
|
||||
},
|
||||
current: {
|
||||
total_hardware: currentTotal,
|
||||
tco_estimate: currentTCO,
|
||||
price_per_port: currentPrice,
|
||||
},
|
||||
market: {
|
||||
min_price: marketMin,
|
||||
avg_price: marketAvg,
|
||||
vendor_count: allPrices.length,
|
||||
min_total_hardware: marketMinTotal,
|
||||
avg_total_hardware: marketAvgTotal,
|
||||
},
|
||||
tco_comparison: {
|
||||
current: currentTCO,
|
||||
target_min: targetTCOMin,
|
||||
target_avg: targetTCOAvg,
|
||||
savings_vs_min: savingsMin,
|
||||
savings_vs_avg: savingsAvg,
|
||||
savings_pct_min: savingsMin && currentTCO > 0
|
||||
? Math.round(savingsMin / currentTCO * 1000) / 10
|
||||
: null,
|
||||
},
|
||||
switching: {
|
||||
cost: switchingCost,
|
||||
break_even_months: breakEvenMonths || null,
|
||||
monthly_savings: monthlyHardwareSavingsMin,
|
||||
},
|
||||
top_vendors: top5Vendors,
|
||||
currency: curr,
|
||||
});
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -1,58 +0,0 @@
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const stockCompetitorRouter = Router();
|
||||
|
||||
// GET /api/stock/competitor-by-tech
|
||||
// Returns publicly-available competitor stock levels aggregated by technology (form_factor+speed_gbps).
|
||||
// Used by Warehouse Stock Intelligence to benchmark Flexoptix demand against competitor availability.
|
||||
stockCompetitorRouter.get("/competitor-by-tech", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
t.form_factor,
|
||||
t.speed_gbps::numeric AS speed_gbps,
|
||||
sv.name AS vendor_name,
|
||||
COUNT(DISTINCT so.transceiver_id)::int AS skus,
|
||||
COALESCE(SUM(so.warehouse_de_qty),0)::int AS de_stock,
|
||||
COALESCE(SUM(so.warehouse_global_qty),0)::int AS global_stock,
|
||||
COALESCE(SUM(so.quantity_available),0)::int AS qty_available,
|
||||
MAX(so.time)::date AS last_seen
|
||||
FROM stock_observations so
|
||||
JOIN transceivers t ON t.id = so.transceiver_id
|
||||
JOIN vendors sv ON sv.id = so.source_vendor_id
|
||||
WHERE so.time > NOW() - INTERVAL '14 days'
|
||||
AND sv.name IN ('FS.COM','ATGBICS','FiberMall','NADDOD','Prolabs','Blue Optics','Skylane Optics')
|
||||
GROUP BY t.form_factor, t.speed_gbps, sv.name
|
||||
ORDER BY sv.name, global_stock DESC NULLS LAST
|
||||
`);
|
||||
|
||||
// Pivot: { "1G SFP": { "FS.COM": { de:..., global:... }, ... }, ... }
|
||||
const byTech: Record<string, Record<string, { de: number; global: number; skus: number; last_seen: string }>> = {};
|
||||
const vendors = new Set<string>();
|
||||
|
||||
for (const row of result.rows) {
|
||||
const n = parseFloat(row.speed_gbps);
|
||||
const spd = n >= 1000
|
||||
? ((n / 1000 * 10) % 10 === 0 ? Math.round(n / 1000).toString() : (n / 1000).toFixed(1)) + "T"
|
||||
: ((n * 10) % 10 === 0 ? Math.round(n).toString() : String(n)) + "G";
|
||||
const key = spd + " " + (row.form_factor || "?");
|
||||
if (!byTech[key]) byTech[key] = {};
|
||||
byTech[key][row.vendor_name] = {
|
||||
de: parseInt(row.de_stock) || 0,
|
||||
global: parseInt(row.global_stock) || parseInt(row.qty_available) || 0,
|
||||
skus: parseInt(row.skus) || 0,
|
||||
last_seen: row.last_seen,
|
||||
};
|
||||
vendors.add(row.vendor_name);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
vendors: [...vendors].sort(),
|
||||
by_tech: byTech,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -461,7 +461,7 @@ stockRouter.get("/velocity/:id", async (req: Request, res: Response) => {
|
||||
const [transceiver, velocity, events] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT t.*, v.name AS brand_name
|
||||
FROM transceivers t LEFT JOIN vendors v ON v.id = t.brand_vendor_id
|
||||
FROM transceivers t LEFT JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE t.id = $1`,
|
||||
[transceiverUuid]
|
||||
),
|
||||
@ -518,54 +518,6 @@ stockRouter.get("/velocity/:id", async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// ─── GET /api/stock/competitor-by-tech ───────────────────────────────────────
|
||||
// Competitor stock levels from publicly-scraped data, aggregated by technology.
|
||||
stockRouter.get("/competitor-by-tech", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
t.form_factor,
|
||||
t.speed_gbps::numeric AS speed_gbps,
|
||||
sv.name AS vendor_name,
|
||||
COUNT(DISTINCT so.transceiver_id)::int AS skus,
|
||||
COALESCE(SUM(so.warehouse_de_qty),0)::int AS de_stock,
|
||||
COALESCE(SUM(so.warehouse_global_qty),0)::int AS global_stock,
|
||||
COALESCE(SUM(so.quantity_available),0)::int AS qty_available,
|
||||
MAX(so.time)::date AS last_seen
|
||||
FROM stock_observations so
|
||||
JOIN transceivers t ON t.id = so.transceiver_id
|
||||
JOIN vendors sv ON sv.id = so.source_vendor_id
|
||||
WHERE so.time > NOW() - INTERVAL '14 days'
|
||||
AND sv.name IN ('FS.COM','ATGBICS','FiberMall','NADDOD','Prolabs','Blue Optics','Skylane Optics')
|
||||
GROUP BY t.form_factor, t.speed_gbps, sv.name
|
||||
ORDER BY sv.name, global_stock DESC NULLS LAST
|
||||
`);
|
||||
|
||||
const byTech: Record<string, Record<string, { de: number; global: number; skus: number; last_seen: string }>> = {};
|
||||
const vendors = new Set<string>();
|
||||
for (const row of result.rows) {
|
||||
const n = parseFloat(row.speed_gbps);
|
||||
const spd = n >= 1000
|
||||
? ((n / 1000 * 10) % 10 === 0 ? String(Math.round(n / 1000)) : (n / 1000).toFixed(1)) + "T"
|
||||
: ((n * 10) % 10 === 0 ? String(Math.round(n)) : String(n)) + "G";
|
||||
const key = spd + " " + (row.form_factor || "?");
|
||||
if (!byTech[key]) byTech[key] = {};
|
||||
byTech[key][row.vendor_name] = {
|
||||
de: parseInt(row.de_stock) || 0,
|
||||
global: parseInt(row.global_stock) || parseInt(row.qty_available) || 0,
|
||||
skus: parseInt(row.skus) || 0,
|
||||
last_seen: row.last_seen,
|
||||
};
|
||||
vendors.add(row.vendor_name);
|
||||
}
|
||||
res.json({ success: true, vendors: [...vendors].sort(), by_tech: byTech });
|
||||
} catch (err) {
|
||||
console.error("stock/competitor-by-tech error:", err);
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /api/stock/:id ──────────────────────────────────────────────────────
|
||||
/**
|
||||
* Full observation history for one transceiver.
|
||||
@ -610,7 +562,7 @@ stockRouter.get("/:id", async (req: Request, res: Response) => {
|
||||
const [transceiver, observations] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT t.*, v.name AS brand_name
|
||||
FROM transceivers t LEFT JOIN vendors v ON v.id = t.brand_vendor_id
|
||||
FROM transceivers t LEFT JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE t.id = $1`,
|
||||
[transceiverUuid]
|
||||
),
|
||||
|
||||
@ -1,192 +1,83 @@
|
||||
/**
|
||||
* Vendor Reliability Scores — Redesigned (v2)
|
||||
*
|
||||
* Scoring methodology (100 pts total):
|
||||
* 30 pts — Data Freshness: How recently were prices scraped?
|
||||
* 25 pts — SKU Coverage: How many unique transceivers covered in 60d?
|
||||
* 25 pts — Price Consistency: Price anomaly rate (low anomalies = reliable)
|
||||
* 20 pts — Stock Accuracy: Does vendor report stock status? How often in_stock?
|
||||
* Vendor Reliability Scores
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/vendor-reliability — All vendor scores
|
||||
* GET /api/vendor-reliability/:id — Single vendor detail + breakdown
|
||||
* GET /api/vendor-reliability — Reliability score 0–100 per vendor
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const vendorReliabilityRouter = Router();
|
||||
|
||||
async function computeReliability() {
|
||||
const result = await pool.query(`
|
||||
WITH
|
||||
-- Freshness + volume (30 pts)
|
||||
freshness AS (
|
||||
SELECT
|
||||
po.source_vendor_id AS vendor_id,
|
||||
MAX(po.time) AS last_seen,
|
||||
EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 86400.0 AS days_since_last,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '7 days') AS obs_7d
|
||||
FROM price_observations po
|
||||
WHERE po.time >= NOW() - INTERVAL '90 days'
|
||||
GROUP BY po.source_vendor_id
|
||||
),
|
||||
-- SKU coverage breadth (25 pts)
|
||||
coverage AS (
|
||||
SELECT
|
||||
po.source_vendor_id AS vendor_id,
|
||||
COUNT(DISTINCT po.transceiver_id) AS skus_60d,
|
||||
COUNT(DISTINCT po.transceiver_id)
|
||||
FILTER (WHERE po.time >= NOW() - INTERVAL '7 days') AS skus_7d
|
||||
FROM price_observations po
|
||||
WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
AND po.price > 0
|
||||
GROUP BY po.source_vendor_id
|
||||
),
|
||||
-- Price consistency: anomaly rate (25 pts)
|
||||
consistency AS (
|
||||
SELECT
|
||||
po.source_vendor_id AS vendor_id,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS total_30d,
|
||||
COUNT(*) FILTER (
|
||||
WHERE po.time >= NOW() - INTERVAL '30 days'
|
||||
AND COALESCE(po.is_anomalous, false) = true
|
||||
) AS anomalies_30d,
|
||||
-- Price variance: std dev / mean (coefficient of variation, lower = more stable)
|
||||
CASE WHEN AVG(po.price) > 0 THEN
|
||||
ROUND((STDDEV(po.price) / AVG(po.price) * 100)::numeric, 1)
|
||||
END AS price_cv_pct
|
||||
FROM price_observations po
|
||||
WHERE po.time >= NOW() - INTERVAL '30 days'
|
||||
AND po.price > 0
|
||||
GROUP BY po.source_vendor_id
|
||||
),
|
||||
-- Stock accuracy: does vendor report stock? reliability of in_stock flag (20 pts)
|
||||
stock_acc AS (
|
||||
SELECT
|
||||
so.source_vendor_id AS vendor_id,
|
||||
COUNT(*)::int AS stock_obs,
|
||||
COUNT(*) FILTER (WHERE so.in_stock = true)::int AS in_stock_count,
|
||||
COUNT(*) FILTER (WHERE so.in_stock = false)::int AS out_of_stock_count
|
||||
FROM stock_observations so
|
||||
WHERE so.time >= NOW() - INTERVAL '30 days'
|
||||
GROUP BY so.source_vendor_id
|
||||
)
|
||||
SELECT
|
||||
v.id::text AS vendor_id,
|
||||
v.name AS vendor_name,
|
||||
v.type,
|
||||
v.website,
|
||||
-- Raw metrics
|
||||
f.last_seen,
|
||||
f.days_since_last,
|
||||
f.obs_30d,
|
||||
f.obs_7d,
|
||||
c.skus_60d,
|
||||
c.skus_7d,
|
||||
co.total_30d,
|
||||
co.anomalies_30d,
|
||||
co.price_cv_pct,
|
||||
s.stock_obs,
|
||||
s.in_stock_count,
|
||||
s.out_of_stock_count
|
||||
FROM freshness f
|
||||
JOIN vendors v ON v.id = f.vendor_id
|
||||
LEFT JOIN coverage c ON c.vendor_id = f.vendor_id
|
||||
LEFT JOIN consistency co ON co.vendor_id = f.vendor_id
|
||||
LEFT JOIN stock_acc s ON s.vendor_id = f.vendor_id
|
||||
ORDER BY f.last_seen DESC
|
||||
`);
|
||||
|
||||
return result.rows.map(row => {
|
||||
// ── Freshness score (30 pts) ──────────────────────────────────────────
|
||||
const days = parseFloat(row.days_since_last || "999");
|
||||
const freshnessScore =
|
||||
days <= 1 ? 30 :
|
||||
days <= 3 ? 27 :
|
||||
days <= 7 ? 22 :
|
||||
days <= 14 ? 15 :
|
||||
days <= 30 ? 8 : 0;
|
||||
|
||||
// ── Coverage score (25 pts) ───────────────────────────────────────────
|
||||
const skus60d = parseInt(row.skus_60d || "0");
|
||||
const MAX_SKUS = 1000;
|
||||
const coverageScore = Math.min(Math.round((skus60d / MAX_SKUS) * 25), 25);
|
||||
|
||||
// ── Consistency score (25 pts) — low anomaly rate = good ─────────────
|
||||
const total30d = parseInt(row.total_30d || "0");
|
||||
const anomalies30d = parseInt(row.anomalies_30d || "0");
|
||||
const anomalyRate = total30d > 0 ? anomalies30d / total30d : 0;
|
||||
const consistencyScore =
|
||||
total30d === 0 ? 0 :
|
||||
anomalyRate <= 0.01 ? 25 :
|
||||
anomalyRate <= 0.03 ? 20 :
|
||||
anomalyRate <= 0.05 ? 15 :
|
||||
anomalyRate <= 0.10 ? 10 : 5;
|
||||
|
||||
// ── Stock accuracy score (20 pts) ─────────────────────────────────────
|
||||
const stockObs = parseInt(row.stock_obs || "0");
|
||||
const stockScore = Math.min(Math.round((stockObs / 100) * 20), 20);
|
||||
|
||||
const totalScore = freshnessScore + coverageScore + consistencyScore + stockScore;
|
||||
const grade =
|
||||
totalScore >= 85 ? "A" :
|
||||
totalScore >= 70 ? "B" :
|
||||
totalScore >= 50 ? "C" :
|
||||
totalScore >= 30 ? "D" : "F";
|
||||
|
||||
return {
|
||||
vendor_id: row.vendor_id,
|
||||
vendor_name: row.vendor_name,
|
||||
type: row.type,
|
||||
website: row.website,
|
||||
reliability_score: totalScore,
|
||||
grade,
|
||||
breakdown: {
|
||||
freshness: { score: freshnessScore, max: 30, days_since_last: Math.round(days * 10) / 10 },
|
||||
coverage: { score: coverageScore, max: 25, skus_60d: skus60d, skus_7d: parseInt(row.skus_7d || "0") },
|
||||
consistency: { score: consistencyScore, max: 25, anomaly_rate_pct: Math.round(anomalyRate * 1000) / 10, obs_30d: total30d },
|
||||
stock_accuracy: { score: stockScore, max: 20, stock_obs_30d: stockObs, in_stock: parseInt(row.in_stock_count || "0") },
|
||||
},
|
||||
last_seen: row.last_seen,
|
||||
obs_30d: parseInt(row.obs_30d || "0"),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/vendor-reliability ────────────────────────────────────────────
|
||||
// ─── GET /api/vendor-reliability ─────────────────────────────────────────────
|
||||
vendorReliabilityRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const vendors = await computeReliability();
|
||||
const result = await pool.query<{
|
||||
vendor_id: number;
|
||||
vendor_name: string;
|
||||
last_observation: Date;
|
||||
obs_30d: string;
|
||||
distinct_skus_60d: string;
|
||||
days_since_last: string;
|
||||
}>(`
|
||||
WITH base AS (
|
||||
SELECT
|
||||
po.source_vendor_id AS vendor_id,
|
||||
MAX(po.time) AS last_observation,
|
||||
COUNT(*) FILTER (WHERE po.time > NOW() - INTERVAL '30 days')
|
||||
AS obs_30d,
|
||||
COUNT(DISTINCT po.transceiver_id)
|
||||
FILTER (WHERE po.time > NOW() - INTERVAL '60 days')
|
||||
AS distinct_skus_60d
|
||||
FROM price_observations po
|
||||
WHERE po.time > NOW() - INTERVAL '90 days'
|
||||
GROUP BY po.source_vendor_id
|
||||
)
|
||||
SELECT
|
||||
b.vendor_id,
|
||||
v.name AS vendor_name,
|
||||
b.last_observation,
|
||||
b.obs_30d,
|
||||
b.distinct_skus_60d,
|
||||
EXTRACT(EPOCH FROM (NOW() - b.last_observation)) / 86400.0 AS days_since_last
|
||||
FROM base b
|
||||
JOIN vendors v ON v.id = b.vendor_id
|
||||
ORDER BY b.last_observation DESC
|
||||
`);
|
||||
|
||||
const vendors = result.rows.map((row) => {
|
||||
const days = parseFloat(row.days_since_last);
|
||||
const obs30d = parseInt(row.obs_30d, 10);
|
||||
const skus60d = parseInt(row.distinct_skus_60d, 10);
|
||||
|
||||
const freshnessScore =
|
||||
days <= 7 ? 40 :
|
||||
days <= 14 ? 30 :
|
||||
days <= 30 ? 20 :
|
||||
days <= 60 ? 10 : 0;
|
||||
|
||||
const frequencyScore = Math.min(Math.round((obs30d / 100) * 30), 30);
|
||||
const coverageScore = Math.min(Math.round((skus60d / 500) * 30), 30);
|
||||
const reliabilityScore = freshnessScore + frequencyScore + coverageScore;
|
||||
|
||||
return {
|
||||
vendor_id: row.vendor_id,
|
||||
vendor_name: row.vendor_name,
|
||||
reliability_score: reliabilityScore,
|
||||
freshness_score: freshnessScore,
|
||||
frequency_score: frequencyScore,
|
||||
coverage_score: coverageScore,
|
||||
last_observation: row.last_observation.toISOString().slice(0, 10),
|
||||
obs_30d: obs30d,
|
||||
distinct_skus_60d: skus60d,
|
||||
};
|
||||
});
|
||||
|
||||
vendors.sort((a, b) => b.reliability_score - a.reliability_score);
|
||||
res.json({ success: true, vendors, scored_at: new Date().toISOString() });
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/vendor-reliability/:id — Single vendor deep-dive ──────────────
|
||||
vendorReliabilityRouter.get("/:id", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const all = await computeReliability();
|
||||
const vendor = all.find(v => v.vendor_id === req.params.id);
|
||||
if (!vendor) return res.status(404).json({ success: false, error: "Vendor not found" });
|
||||
|
||||
// Price history for sparkline
|
||||
const priceHistory = await pool.query(`
|
||||
SELECT DATE_TRUNC('week', time)::date AS week,
|
||||
ROUND(AVG(price)::numeric, 2) AS avg_price,
|
||||
COUNT(*)::int AS observations
|
||||
FROM price_observations
|
||||
WHERE source_vendor_id = $1::uuid
|
||||
AND time >= NOW() - INTERVAL '90 days'
|
||||
AND price > 0
|
||||
GROUP BY DATE_TRUNC('week', time)
|
||||
ORDER BY week ASC
|
||||
`, [req.params.id]).catch(() => ({ rows: [] }));
|
||||
|
||||
res.json({ success: true, vendor, price_history: priceHistory.rows });
|
||||
|
||||
res.json({ success: true, vendors });
|
||||
} catch (err) {
|
||||
console.error("GET /api/vendor-reliability error:", err);
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Router, Request, Response, NextFunction } from "express";
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { listVendors } from "../db/queries";
|
||||
|
||||
@ -114,41 +114,7 @@ vendorRouter.post("/", async (req: Request, res: Response) => {
|
||||
});
|
||||
|
||||
// GET /api/vendors/:id — Get single vendor with full stats
|
||||
// GET /api/vendors/reliability — per-vendor data-reliability scores (freshness/frequency/coverage)
|
||||
vendorRouter.get("/reliability", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT v.id::text AS vendor_id, v.name AS vendor_name,
|
||||
COUNT(DISTINCT t.id)::int AS sku_count,
|
||||
COUNT(po.transceiver_id)::int AS obs_count,
|
||||
MAX(po.time) AS last_obs
|
||||
FROM vendors v
|
||||
JOIN transceivers t ON t.vendor_id = v.id
|
||||
LEFT JOIN price_observations po ON po.transceiver_id = t.id
|
||||
GROUP BY v.id, v.name
|
||||
HAVING COUNT(DISTINCT t.id) > 0`);
|
||||
const rows = result.rows as Array<{ vendor_id: string; vendor_name: string; sku_count: number; obs_count: number; last_obs: string | null }>;
|
||||
const maxSku = Math.max(1, ...rows.map((r) => r.sku_count));
|
||||
const maxObs = Math.max(1, ...rows.map((r) => r.obs_count));
|
||||
const now = Date.now();
|
||||
const vendors = rows.map((r) => {
|
||||
const daysStale = r.last_obs ? Math.max(0, (now - new Date(r.last_obs).getTime()) / 86400000) : 999;
|
||||
const freshness_score = Math.max(0, Math.min(100, Math.round(100 - daysStale * 3)));
|
||||
const frequency_score = Math.round(100 * Math.min(1, Math.log10(r.obs_count + 1) / Math.log10(maxObs + 1)));
|
||||
const coverage_score = Math.round(100 * Math.min(1, r.sku_count / maxSku));
|
||||
const reliability_score = Math.round(0.4 * freshness_score + 0.3 * frequency_score + 0.3 * coverage_score);
|
||||
return { vendor_id: r.vendor_id, vendor_name: r.vendor_name, sku_count: r.sku_count, obs_count: r.obs_count, last_obs: r.last_obs, freshness_score, frequency_score, coverage_score, reliability_score };
|
||||
});
|
||||
vendors.sort((a, b) => b.reliability_score - a.reliability_score);
|
||||
res.json({ success: true, count: vendors.length, vendors });
|
||||
} catch (err) {
|
||||
console.error("vendor reliability error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute vendor reliability" });
|
||||
}
|
||||
});
|
||||
|
||||
vendorRouter.get("/:id", async (req: Request, res: Response, next: NextFunction) => {
|
||||
if (req.params.id === "market-share" || req.params.id === "intelligence") return next();
|
||||
vendorRouter.get("/:id", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const result = await pool.query(
|
||||
@ -171,132 +137,6 @@ vendorRouter.get("/:id", async (req: Request, res: Response, next: NextFunction)
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// GET /api/vendors/market-share — Weekly SKU-coverage share per vendor over time
|
||||
// Shows which vendors are gaining/losing market presence
|
||||
// Query params: speed_gbps, form_factor, days (default 90)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
vendorRouter.get("/market-share", async (req: Request, res: Response) => {
|
||||
const days = Math.min(parseInt(req.query.days as string) || 90, 365);
|
||||
const spd = req.query.speed_gbps as string | undefined;
|
||||
const ff = req.query.form_factor as string | undefined;
|
||||
|
||||
try {
|
||||
const [weekly, current, momentum] = await Promise.all([
|
||||
// Weekly SKU count per vendor — shows growth/shrink trends
|
||||
pool.query(`
|
||||
SELECT
|
||||
DATE_TRUNC('week', po.time)::date AS week,
|
||||
v.id::text AS vendor_id,
|
||||
v.name AS vendor_name,
|
||||
COUNT(DISTINCT po.transceiver_id)::int AS sku_count
|
||||
FROM price_observations po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '${days} days'
|
||||
AND po.price > 0
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
GROUP BY DATE_TRUNC('week', po.time), v.id, v.name
|
||||
ORDER BY week ASC, sku_count DESC
|
||||
`),
|
||||
|
||||
// Current snapshot: SKU share % per vendor (last 30d)
|
||||
pool.query(`
|
||||
WITH totals AS (
|
||||
SELECT COUNT(DISTINCT transceiver_id)::float AS total
|
||||
FROM price_observations
|
||||
WHERE time >= NOW() - INTERVAL '30 days'
|
||||
AND price > 0 AND COALESCE(is_anomalous, false) = false
|
||||
)
|
||||
SELECT
|
||||
v.id::text AS vendor_id,
|
||||
v.name AS vendor_name,
|
||||
v.type,
|
||||
COUNT(DISTINCT po.transceiver_id)::int AS sku_count,
|
||||
ROUND((COUNT(DISTINCT po.transceiver_id)::numeric / NULLIF(t.total,0)::numeric) * 100, 1) AS market_share_pct,
|
||||
COUNT(*)::int AS total_obs,
|
||||
MAX(po.time) AS last_seen
|
||||
FROM price_observations po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
JOIN transceivers tx ON tx.id = po.transceiver_id
|
||||
CROSS JOIN totals t
|
||||
WHERE po.time >= NOW() - INTERVAL '30 days'
|
||||
AND po.price > 0
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
${spd ? `AND tx.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
${ff ? `AND tx.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
GROUP BY v.id, v.name, v.type, t.total
|
||||
ORDER BY sku_count DESC
|
||||
LIMIT 30
|
||||
`),
|
||||
|
||||
// Momentum: compare last 30d vs prior 30d SKU count per vendor
|
||||
pool.query(`
|
||||
WITH cur AS (
|
||||
SELECT source_vendor_id, COUNT(DISTINCT transceiver_id)::int AS sku_count
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '30 days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
GROUP BY source_vendor_id
|
||||
),
|
||||
prior AS (
|
||||
SELECT source_vendor_id, COUNT(DISTINCT transceiver_id)::int AS sku_count
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
AND po.time < NOW() - INTERVAL '30 days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
${spd ? `AND t.speed_gbps = ${parseFloat(spd)}` : ""}
|
||||
${ff ? `AND t.form_factor = '${ff.replace(/'/g,"''")}'` : ""}
|
||||
GROUP BY source_vendor_id
|
||||
)
|
||||
SELECT
|
||||
v.name AS vendor_name, v.id::text AS vendor_id,
|
||||
c.sku_count AS current_skus,
|
||||
COALESCE(p.sku_count, 0) AS prior_skus,
|
||||
(c.sku_count - COALESCE(p.sku_count, 0)) AS delta_skus,
|
||||
CASE
|
||||
WHEN COALESCE(p.sku_count, 0) = 0 THEN NULL
|
||||
ELSE ROUND(((c.sku_count - p.sku_count)::numeric / p.sku_count::numeric) * 100, 1)
|
||||
END AS delta_pct
|
||||
FROM cur c
|
||||
JOIN vendors v ON v.id = c.source_vendor_id
|
||||
LEFT JOIN prior p ON p.source_vendor_id = c.source_vendor_id
|
||||
ORDER BY delta_skus DESC
|
||||
LIMIT 20
|
||||
`),
|
||||
]);
|
||||
|
||||
// Compute share % per week for chart (normalize across vendors per week)
|
||||
const weekTotals = new Map<string, number>();
|
||||
for (const row of weekly.rows) {
|
||||
const k = row.week;
|
||||
weekTotals.set(k, (weekTotals.get(k) || 0) + row.sku_count);
|
||||
}
|
||||
const weeklyWithShare = weekly.rows.map(r => ({
|
||||
...r,
|
||||
share_pct: weekTotals.get(r.week)
|
||||
? Math.round((r.sku_count / weekTotals.get(r.week)!) * 1000) / 10
|
||||
: 0,
|
||||
}));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
filters: { days, speed_gbps: spd || null, form_factor: ff || null },
|
||||
weekly_trend: weeklyWithShare,
|
||||
current_share: current.rows,
|
||||
momentum: momentum.rows,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/vendors/intelligence — per-vendor price + SKU market stats (last 30d)
|
||||
vendorRouter.get("/intelligence", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
@ -307,7 +147,7 @@ vendorRouter.get("/intelligence", async (_req: Request, res: Response) => {
|
||||
v.type,
|
||||
v.website,
|
||||
COUNT(DISTINCT po.transceiver_id)::int AS sku_count,
|
||||
COUNT(*)::int AS price_obs,
|
||||
COUNT(po.id)::int AS price_obs,
|
||||
ROUND(AVG(po.price)::numeric, 2) AS avg_price,
|
||||
ROUND(MIN(po.price)::numeric, 2) AS min_price,
|
||||
ROUND(MAX(po.price)::numeric, 2) AS max_price,
|
||||
|
||||
@ -1,189 +0,0 @@
|
||||
/**
|
||||
* Win/Loss Intelligence — /api/win-loss
|
||||
*
|
||||
* Record and analyze deal outcomes: who won, who lost, at what price, in which segment.
|
||||
*
|
||||
* Routes:
|
||||
* POST /api/win-loss — Record a win/loss event
|
||||
* GET /api/win-loss — List events (filterable)
|
||||
* GET /api/win-loss/summary — Aggregate win rate, avg price delta, segments
|
||||
* GET /api/win-loss/competitors — Ranking by competitor vendor (loss analysis)
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { sendCSV } from "../utils/csv";
|
||||
|
||||
export const winLossRouter = Router();
|
||||
|
||||
// ── POST /api/win-loss — Record a deal outcome ──────────────────────────────
|
||||
winLossRouter.post("/", async (req: Request, res: Response) => {
|
||||
const {
|
||||
outcome, transceiver_id, competitor_vendor,
|
||||
our_price, competitor_price, currency = "USD",
|
||||
quantity, customer_segment, deal_source,
|
||||
form_factor, speed_gbps, notes, deal_date,
|
||||
} = req.body as Record<string, any>;
|
||||
|
||||
if (!outcome || !["won","lost","unknown"].includes(outcome)) {
|
||||
return res.status(400).json({ success: false, error: "outcome must be: won | lost | unknown" });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`INSERT INTO win_loss_events
|
||||
(outcome, transceiver_id, competitor_vendor, our_price, competitor_price,
|
||||
currency, quantity, customer_segment, deal_source, form_factor, speed_gbps, notes, deal_date)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,
|
||||
COALESCE($13::date, CURRENT_DATE))
|
||||
RETURNING *`,
|
||||
[
|
||||
outcome,
|
||||
transceiver_id || null,
|
||||
competitor_vendor || null,
|
||||
our_price ? parseFloat(our_price) : null,
|
||||
competitor_price ? parseFloat(competitor_price) : null,
|
||||
currency,
|
||||
quantity ? parseInt(quantity) : null,
|
||||
customer_segment || null,
|
||||
deal_source || null,
|
||||
form_factor || null,
|
||||
speed_gbps ? parseFloat(speed_gbps) : null,
|
||||
notes || null,
|
||||
deal_date || null,
|
||||
]
|
||||
);
|
||||
return res.status(201).json({ success: true, event: result.rows[0] });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/win-loss — List events ─────────────────────────────────────────
|
||||
winLossRouter.get("/", async (req: Request, res: Response) => {
|
||||
const outcome = req.query.outcome as string | undefined;
|
||||
const segment = req.query.customer_segment as string | undefined;
|
||||
const days = Math.min(parseInt(req.query.days as string) || 90, 730);
|
||||
const limit = Math.min(parseInt(req.query.limit as string) || 50, 200);
|
||||
const fmt = req.query.format as string | undefined;
|
||||
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT wl.*,
|
||||
t.standard_name, t.form_factor AS tx_form_factor, t.speed_gbps AS tx_speed
|
||||
FROM win_loss_events wl
|
||||
LEFT JOIN transceivers t ON t.id = wl.transceiver_id
|
||||
WHERE wl.deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
${outcome ? `AND wl.outcome = '${outcome.replace(/'/g,"''")}'` : ""}
|
||||
${segment ? `AND wl.customer_segment = '${segment.replace(/'/g,"''")}'` : ""}
|
||||
ORDER BY wl.deal_date DESC
|
||||
LIMIT ${limit}
|
||||
`);
|
||||
|
||||
if (fmt === "csv") {
|
||||
return sendCSV(res, result.rows, `tip-win-loss-${new Date().toISOString().slice(0,10)}.csv`);
|
||||
}
|
||||
return res.json({ success: true, events: result.rows, count: result.rows.length });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/win-loss/summary — Aggregate analytics ─────────────────────────
|
||||
winLossRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
const days = Math.min(parseInt(req.query.days as string) || 90, 730);
|
||||
|
||||
try {
|
||||
const [overall, bySegment, byFormFactor, priceDeltas] = await Promise.all([
|
||||
pool.query(`
|
||||
SELECT
|
||||
COUNT(*) AS total_events,
|
||||
COUNT(*) FILTER (WHERE outcome = 'won') AS won,
|
||||
COUNT(*) FILTER (WHERE outcome = 'lost') AS lost,
|
||||
ROUND(
|
||||
COUNT(*) FILTER (WHERE outcome = 'won')::numeric
|
||||
/ NULLIF(COUNT(*) FILTER (WHERE outcome IN ('won','lost')), 0) * 100, 1
|
||||
) AS win_rate_pct,
|
||||
ROUND(AVG(our_price) FILTER (WHERE outcome = 'won')::numeric, 2) AS avg_win_price,
|
||||
ROUND(AVG(our_price) FILTER (WHERE outcome = 'lost')::numeric, 2) AS avg_loss_price
|
||||
FROM win_loss_events
|
||||
WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
`),
|
||||
pool.query(`
|
||||
SELECT customer_segment,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE outcome = 'won') AS won,
|
||||
COUNT(*) FILTER (WHERE outcome = 'lost') AS lost,
|
||||
ROUND(COUNT(*) FILTER (WHERE outcome = 'won')::numeric
|
||||
/ NULLIF(COUNT(*) FILTER (WHERE outcome IN ('won','lost')),0)*100,1) AS win_rate_pct
|
||||
FROM win_loss_events
|
||||
WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
AND customer_segment IS NOT NULL
|
||||
GROUP BY customer_segment
|
||||
ORDER BY total DESC
|
||||
`),
|
||||
pool.query(`
|
||||
SELECT COALESCE(wl.form_factor, tx.form_factor) AS form_factor,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE outcome = 'won') AS won,
|
||||
COUNT(*) FILTER (WHERE outcome = 'lost') AS lost
|
||||
FROM win_loss_events wl
|
||||
LEFT JOIN transceivers tx ON tx.id = wl.transceiver_id
|
||||
WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
GROUP BY COALESCE(wl.form_factor, tx.form_factor)
|
||||
HAVING COALESCE(wl.form_factor, tx.form_factor) IS NOT NULL
|
||||
ORDER BY total DESC
|
||||
`),
|
||||
// Price delta analysis: where we lost — how far off were we?
|
||||
pool.query(`
|
||||
SELECT
|
||||
ROUND(AVG(competitor_price - our_price)::numeric, 2) AS avg_price_gap,
|
||||
ROUND(AVG((competitor_price - our_price) / NULLIF(our_price,0) * 100)::numeric, 1) AS avg_gap_pct,
|
||||
COUNT(*) AS events_with_prices
|
||||
FROM win_loss_events
|
||||
WHERE outcome = 'lost'
|
||||
AND our_price IS NOT NULL AND competitor_price IS NOT NULL
|
||||
AND deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
`),
|
||||
]);
|
||||
|
||||
return res.json({
|
||||
success: true,
|
||||
days,
|
||||
overall: overall.rows[0],
|
||||
by_segment: bySegment.rows,
|
||||
by_form_factor: byFormFactor.rows,
|
||||
price_delta: priceDeltas.rows[0],
|
||||
});
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/win-loss/competitors — Competitor ranking ───────────────────────
|
||||
winLossRouter.get("/competitors", async (req: Request, res: Response) => {
|
||||
const days = Math.min(parseInt(req.query.days as string) || 90, 730);
|
||||
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
competitor_vendor,
|
||||
COUNT(*) AS encounters,
|
||||
COUNT(*) FILTER (WHERE outcome = 'lost') AS losses_to,
|
||||
COUNT(*) FILTER (WHERE outcome = 'won') AS wins_against,
|
||||
ROUND(AVG(competitor_price - our_price)
|
||||
FILTER (WHERE outcome = 'lost' AND our_price IS NOT NULL AND competitor_price IS NOT NULL)
|
||||
::numeric, 2) AS avg_price_advantage, -- negative = they beat us on price
|
||||
ROUND(AVG(competitor_price)::numeric, 2) AS avg_competitor_price
|
||||
FROM win_loss_events
|
||||
WHERE deal_date >= CURRENT_DATE - INTERVAL '${days} days'
|
||||
AND competitor_vendor IS NOT NULL
|
||||
GROUP BY competitor_vendor
|
||||
ORDER BY losses_to DESC, encounters DESC
|
||||
LIMIT 30
|
||||
`);
|
||||
|
||||
return res.json({ success: true, competitors: result.rows });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -1,35 +0,0 @@
|
||||
/**
|
||||
* Minimal CSV serializer — no external dependencies.
|
||||
* Converts an array of flat objects to RFC 4180-compliant CSV text.
|
||||
*/
|
||||
|
||||
function escapeCell(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
const str = String(value);
|
||||
// Quote if contains comma, quote, newline, or leading/trailing whitespace
|
||||
if (/[",\n\r]/.test(str) || str !== str.trim()) {
|
||||
return `"${str.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
export function toCSV(rows: Record<string, unknown>[]): string {
|
||||
if (rows.length === 0) return "";
|
||||
const headers = Object.keys(rows[0]);
|
||||
const lines = [
|
||||
headers.join(","),
|
||||
...rows.map(row => headers.map(h => escapeCell(row[h])).join(",")),
|
||||
];
|
||||
return lines.join("\r\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a CSV response with proper headers.
|
||||
*/
|
||||
import type { Response } from "express";
|
||||
export function sendCSV(res: Response, rows: Record<string, unknown>[], filename: string): void {
|
||||
const csv = toCSV(rows);
|
||||
res.setHeader("Content-Type", "text/csv; charset=utf-8");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
|
||||
res.send("" + csv); // BOM for Excel UTF-8 compatibility
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -33,10 +33,7 @@
|
||||
"scrape:optcore": "tsx src/scrapers/optcore.ts",
|
||||
"scrape:news": "tsx src/scrapers/news.ts",
|
||||
"scrape:all": "tsx src/index.ts --all",
|
||||
"robots:verification": "tsx src/robots/verification-robots.ts",
|
||||
"dqo": "tsx src/orchestrator/index.ts",
|
||||
"dqo:once": "tsx src/orchestrator/index.ts --once",
|
||||
"dqo:test": "node --import tsx --test src/orchestrator/self-heal-guards.test.ts"
|
||||
"robots:verification": "tsx src/robots/verification-robots.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"crawlee": "^3.12.0",
|
||||
|
||||
@ -69,7 +69,7 @@ export interface CrawlExtraction {
|
||||
}
|
||||
|
||||
export interface RobotExperience {
|
||||
event: "status" | "tipllm_plan" | "queue_plan" | "queue_enqueued" | "queue_rejected" | "crawler_result" | "dqo_dispatch" | "dqo_verify" | "dqo_escalation";
|
||||
event: "status" | "tipllm_plan" | "queue_plan" | "queue_enqueued" | "queue_rejected" | "crawler_result";
|
||||
observed_at: string;
|
||||
actor: string;
|
||||
profile?: string;
|
||||
|
||||
@ -1,394 +0,0 @@
|
||||
/**
|
||||
* tip-dqo control loop — the closed loop TIP was missing.
|
||||
*
|
||||
* DETECT read v_dqo_targets (v_scraper_health joined onto source_registry) — DATA
|
||||
* freshness of the observation TABLES, never pg-boss job cadence.
|
||||
* PRIORITIZE the view is pre-ranked (dead>stale, weighted); split staleness vs
|
||||
* code_fix/blocked. Only 'staleness' is dispatch-eligible.
|
||||
* DISPATCH guardrails (self-heal-guards) + plan-mode + profile cap, then the shared
|
||||
* verification-robots dispatchQueues() — exactly one boss.send path.
|
||||
* VERIFY re-read the source's freshness for pending dispatch rows: did fresh rows
|
||||
* land? healed / ineffective(+strike). Catches completed-but-wrote-0 jobs.
|
||||
* ESCALATE fresh=silent, stale/1-2 strikes=digest, breaker/dead=page (delta-gated by
|
||||
* acknowledged_exceptions and once-per-24h so the day-1 backlog never spams).
|
||||
*
|
||||
* INVARIANT: VERIFY + DETECT key off observation-table freshness (v_scraper_health), NOT
|
||||
* pgboss.job='completed' — that is the exact blind spot that lets a green-queue/zero-row
|
||||
* scraper look healthy. Do not "optimize" this to read pgboss.job.
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { loadavg } from "os";
|
||||
import { requireDbPassword } from "../utils/db-connection";
|
||||
import {
|
||||
dispatchQueues,
|
||||
queuesForProfile,
|
||||
defaultMaxQueues,
|
||||
type RobotProfile,
|
||||
} from "../robots/verification-robots";
|
||||
import { canDispatch, type GuardConfig, type LedgerFacts } from "./self-heal-guards";
|
||||
import { notify } from "./escalation-sink";
|
||||
import { writeRobotExperience } from "../crawler-llm/training-data-writer";
|
||||
|
||||
const PLAN_ONLY = process.env.SELF_HEAL_PLAN_ONLY !== "false"; // default true — dispatches NOTHING
|
||||
const MAX_LOAD = Number.parseFloat(process.env.DQO_MAX_LOAD || "2.5");
|
||||
const SETTLE_MINUTES = Number.parseInt(process.env.DQO_VERIFY_SETTLE_MINUTES || "20", 10);
|
||||
const ESCALATE_REPAGE_HOURS = 24;
|
||||
|
||||
let poolSingleton: Pool | null = null;
|
||||
function getPool(): Pool {
|
||||
if (!poolSingleton) {
|
||||
poolSingleton = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
max: 2, // read-mostly; kept small so the orchestrator never contends the daemon pool
|
||||
idleTimeoutMillis: 10_000,
|
||||
connectionTimeoutMillis: 5_000,
|
||||
});
|
||||
}
|
||||
return poolSingleton;
|
||||
}
|
||||
|
||||
export async function closePool(): Promise<void> {
|
||||
if (poolSingleton) {
|
||||
await poolSingleton.end();
|
||||
poolSingleton = null;
|
||||
}
|
||||
}
|
||||
|
||||
interface Target {
|
||||
source_key: string;
|
||||
vendor: string;
|
||||
source_table: string;
|
||||
status: string;
|
||||
days_since_last: number;
|
||||
rows_30d: number;
|
||||
last_observation: string | null;
|
||||
root_cause_class: "staleness" | "code_fix" | "blocked";
|
||||
queue_names: string[];
|
||||
profile: RobotProfile;
|
||||
dead_days: number;
|
||||
cooldown_minutes: number;
|
||||
budget_per_day: number;
|
||||
max_backoff_hours: number;
|
||||
circuit_breaker_strikes: number;
|
||||
score: number;
|
||||
}
|
||||
|
||||
function guardConfig(t: Target): GuardConfig {
|
||||
return {
|
||||
cooldown_minutes: t.cooldown_minutes,
|
||||
budget_per_day: t.budget_per_day,
|
||||
max_backoff_hours: t.max_backoff_hours,
|
||||
circuit_breaker_strikes: t.circuit_breaker_strikes,
|
||||
max_load: MAX_LOAD,
|
||||
};
|
||||
}
|
||||
|
||||
// ── kill-switch + global daily ceiling ────────────────────────────────────────
|
||||
async function globalGate(pool: Pool): Promise<{ armed: boolean; reason: string; globalBudget: number }> {
|
||||
const { rows } = await pool.query(
|
||||
`select enabled, budget_per_day from source_registry where source_key = '__global__'`,
|
||||
);
|
||||
if (rows.length === 0) return { armed: true, reason: "no __global__ row (default armed)", globalBudget: 9999 };
|
||||
if (!rows[0].enabled) return { armed: false, reason: "kill-switch: __global__.enabled = false", globalBudget: 0 };
|
||||
return { armed: true, reason: "armed", globalBudget: rows[0].budget_per_day };
|
||||
}
|
||||
|
||||
async function globalDispatchesToday(pool: Pool): Promise<number> {
|
||||
const { rows } = await pool.query(
|
||||
`select count(*)::int n from self_heal_dispatch
|
||||
where mode = 'dispatch' and dispatched_at >= date_trunc('day', now())`,
|
||||
);
|
||||
return rows[0]?.n ?? 0;
|
||||
}
|
||||
|
||||
// ── DETECT ────────────────────────────────────────────────────────────────────
|
||||
export async function detect(pool: Pool): Promise<Target[]> {
|
||||
const { rows } = await pool.query(`select * from v_dqo_targets`);
|
||||
return rows as Target[];
|
||||
}
|
||||
|
||||
async function ledgerFacts(pool: Pool, sourceKey: string): Promise<LedgerFacts> {
|
||||
const { rows } = await pool.query(
|
||||
`select
|
||||
(select max(dispatched_at) from self_heal_dispatch where source_key = $1 and mode = 'dispatch') as last_dispatch_at,
|
||||
(select coalesce(strikes, 0) from self_heal_dispatch where source_key = $1 and mode = 'dispatch' order by dispatched_at desc limit 1) as last_strikes,
|
||||
(select count(*)::int from self_heal_dispatch where source_key = $1 and mode = 'dispatch' and dispatched_at >= date_trunc('day', now())) as dispatches_today`,
|
||||
[sourceKey],
|
||||
);
|
||||
const r = rows[0] ?? {};
|
||||
return {
|
||||
last_dispatch_at: r.last_dispatch_at ? new Date(r.last_dispatch_at) : null,
|
||||
last_strikes: Number(r.last_strikes ?? 0),
|
||||
dispatches_today: Number(r.dispatches_today ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function insertLedger(
|
||||
pool: Pool,
|
||||
row: {
|
||||
source_key: string;
|
||||
queues: string[];
|
||||
reason: string;
|
||||
mode: "plan" | "dispatch";
|
||||
rows_before: number | null;
|
||||
last_obs_before: string | null;
|
||||
outcome: "pending" | "skipped" | "escalated";
|
||||
strikes: number;
|
||||
run_id: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
await pool.query(
|
||||
`insert into self_heal_dispatch
|
||||
(source_key, queues, reason, mode, rows_before, last_obs_before, outcome, strikes, run_id)
|
||||
values ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
|
||||
[row.source_key, row.queues, row.reason, row.mode, row.rows_before, row.last_obs_before, row.outcome, row.strikes, row.run_id],
|
||||
);
|
||||
}
|
||||
|
||||
// ── DISPATCH (one self-heal tick) ─────────────────────────────────────────────
|
||||
export interface TickResult {
|
||||
planOnly: boolean;
|
||||
detected: number;
|
||||
planned: number;
|
||||
dispatched: number;
|
||||
skipped: number;
|
||||
escalated: number;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export async function runTick(): Promise<TickResult> {
|
||||
const pool = getPool();
|
||||
const runId = `dqo-self-heal-${new Date().toISOString()}`;
|
||||
const load1 = loadavg()[0];
|
||||
const res: TickResult = { planOnly: PLAN_ONLY, detected: 0, planned: 0, dispatched: 0, skipped: 0, escalated: 0, lines: [] };
|
||||
|
||||
const gate = await globalGate(pool);
|
||||
if (!gate.armed) {
|
||||
res.lines.push(`[HALT] ${gate.reason}`);
|
||||
return res;
|
||||
}
|
||||
const globalToday = await globalDispatchesToday(pool);
|
||||
const globalRemaining = gate.globalBudget - globalToday;
|
||||
|
||||
const targets = await detect(pool);
|
||||
res.detected = targets.length;
|
||||
if (targets.length === 0) {
|
||||
res.lines.push("no stale/dead sources — nothing to do");
|
||||
return res;
|
||||
}
|
||||
|
||||
for (const t of targets) {
|
||||
// Root-cause routing: only staleness is auto-dispatched. code_fix/blocked escalate.
|
||||
if (t.root_cause_class !== "staleness") {
|
||||
await escalateSource(pool, t, `root_cause=${t.root_cause_class} (${t.status} ${t.days_since_last}d) — a re-scrape cannot fix this; human required`, runId);
|
||||
res.escalated++;
|
||||
res.lines.push(` ESCALATE ${t.source_key} — ${t.root_cause_class} (${t.status} ${t.days_since_last}d)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// A dead staleness source past dead_days also escalates (retrying won't help a truly dead source).
|
||||
if (t.status === "dead" && t.days_since_last > t.dead_days) {
|
||||
await escalateSource(pool, t, `dead ${t.days_since_last}d > dead_days ${t.dead_days} — escalating`, runId);
|
||||
res.escalated++;
|
||||
res.lines.push(` ESCALATE ${t.source_key} — dead ${t.days_since_last}d`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const facts = await ledgerFacts(pool, t.source_key);
|
||||
const verdict = canDispatch(facts, guardConfig(t), new Date(), load1);
|
||||
|
||||
// Profile filter + cap on the queues.
|
||||
const eligible = queuesForProfile(t.queue_names, t.profile).slice(0, defaultMaxQueues(t.profile));
|
||||
|
||||
if (eligible.length === 0) {
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: [], reason: `no queue eligible under profile '${t.profile}'`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "skipped", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.skipped++;
|
||||
res.lines.push(` SKIP ${t.source_key} — no queue eligible under '${t.profile}'`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!verdict.allow) {
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `guard: ${verdict.reason}`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "skipped", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.skipped++;
|
||||
res.lines.push(` SKIP ${t.source_key} — ${verdict.reason}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (PLAN_ONLY || globalRemaining <= 0) {
|
||||
const why = PLAN_ONLY ? "plan-mode (SELF_HEAL_PLAN_ONLY)" : `global budget exhausted (${globalToday}/${gate.globalBudget})`;
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `WOULD DISPATCH → ${eligible.join(", ")} [${why}]`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "pending", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.planned++;
|
||||
res.lines.push(` PLAN ${t.source_key} → ${eligible.join(", ")} [${why}]`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_dispatch", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Plan: would dispatch ${t.source_key} (${t.status} ${t.days_since_last}d)`,
|
||||
input: { source_key: t.source_key, status: t.status, days_since_last: t.days_since_last, eligible, why },
|
||||
decision: { mode: "plan", dispatched: false }, safety_notes: ["Plan mode — no jobs enqueued."],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// ARMED: real dispatch.
|
||||
await dispatchQueues(eligible, { source: "dqo:self-heal", source_key: t.source_key, run_id: runId, reason: `${t.status} ${t.days_since_last}d`, enqueued_at: new Date().toISOString() });
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `dispatched (${t.status} ${t.days_since_last}d)`,
|
||||
mode: "dispatch", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "pending", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.dispatched++;
|
||||
res.lines.push(` DISPATCH ${t.source_key} → ${eligible.join(", ")}`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_dispatch", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Dispatched ${t.source_key} (${t.status} ${t.days_since_last}d)`,
|
||||
input: { source_key: t.source_key, eligible }, decision: { mode: "dispatch", dispatched: true },
|
||||
safety_notes: ["Armed dispatch within guardrails."],
|
||||
}, { flush: true });
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// ── VERIFY (one verify tick) ──────────────────────────────────────────────────
|
||||
export interface VerifyResult {
|
||||
checked: number;
|
||||
healed: number;
|
||||
ineffective: number;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export async function runVerifyTick(): Promise<VerifyResult> {
|
||||
const pool = getPool();
|
||||
const res: VerifyResult = { checked: 0, healed: 0, ineffective: 0, lines: [] };
|
||||
|
||||
// pending REAL dispatches past the settle window
|
||||
const { rows: pending } = await pool.query(
|
||||
`select d.id, d.source_key, d.rows_before, d.last_obs_before, d.strikes, d.run_id,
|
||||
sr.vendor, sr.source_table, sr.dead_days, sr.circuit_breaker_strikes
|
||||
from self_heal_dispatch d
|
||||
join source_registry sr on sr.source_key = d.source_key
|
||||
where d.mode = 'dispatch' and d.outcome = 'pending'
|
||||
and d.dispatched_at < now() - ($1 || ' minutes')::interval`,
|
||||
[String(SETTLE_MINUTES)],
|
||||
);
|
||||
// No pending dispatches → skip the expensive v_scraper_health scan entirely.
|
||||
if (pending.length === 0) return res;
|
||||
|
||||
// Compute the (expensive) freshness view ONCE and index it, instead of re-running the
|
||||
// whole aggregate per pending source. Keyed off observation TABLES, never pgboss.job.
|
||||
const { rows: healthRows } = await pool.query(
|
||||
`select vendor, source_table, rows_30d, last_observation from v_scraper_health`,
|
||||
);
|
||||
const health = new Map<string, { rows_30d: number | null; last_observation: string | null }>();
|
||||
for (const hr of healthRows) {
|
||||
health.set(`${hr.vendor}|${hr.source_table}`, { rows_30d: hr.rows_30d, last_observation: hr.last_observation });
|
||||
}
|
||||
|
||||
for (const p of pending) {
|
||||
res.checked++;
|
||||
const h = health.get(`${p.vendor}|${p.source_table}`);
|
||||
const rowsAfter: number | null = h?.rows_30d ?? null;
|
||||
const lastObsAfter: string | null = h?.last_observation ?? null;
|
||||
const advanced = lastObsAfter && p.last_obs_before && new Date(lastObsAfter) > new Date(p.last_obs_before);
|
||||
const grew = rowsAfter != null && p.rows_before != null && rowsAfter > p.rows_before;
|
||||
|
||||
if (advanced || grew) {
|
||||
await pool.query(
|
||||
`update self_heal_dispatch set outcome='healed', checked_at=now(), rows_after=$2, last_obs_after=$3, strikes=0 where id=$1`,
|
||||
[p.id, rowsAfter, lastObsAfter],
|
||||
);
|
||||
res.healed++;
|
||||
res.lines.push(` HEALED ${p.source_key} (rows ${p.rows_before}→${rowsAfter})`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_verify", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Healed ${p.source_key}`, input: { source_key: p.source_key, rows_before: p.rows_before, rows_after: rowsAfter },
|
||||
decision: { outcome: "healed" },
|
||||
});
|
||||
} else {
|
||||
const newStrikes = Number(p.strikes ?? 0) + 1;
|
||||
await pool.query(
|
||||
`update self_heal_dispatch set outcome='ineffective', checked_at=now(), rows_after=$2, last_obs_after=$3, strikes=$4 where id=$1`,
|
||||
[p.id, rowsAfter, lastObsAfter, newStrikes],
|
||||
);
|
||||
res.ineffective++;
|
||||
res.lines.push(` INEFFECTIVE ${p.source_key} (no new rows; strikes→${newStrikes})`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_verify", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Ineffective dispatch ${p.source_key} — no fresh rows landed`,
|
||||
input: { source_key: p.source_key, rows_before: p.rows_before, rows_after: rowsAfter, strikes: newStrikes },
|
||||
decision: { outcome: "ineffective" },
|
||||
});
|
||||
if (newStrikes >= Number(p.circuit_breaker_strikes ?? 3)) {
|
||||
await escalateByKey(pool, p.source_key, `circuit-breaker open: ${newStrikes} ineffective dispatches — retrying wastes resources, human required`, p.run_id);
|
||||
res.lines.push(` ESCALATE ${p.source_key} — breaker open (${newStrikes} strikes)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// ── ESCALATE (delta-gated, once-per-24h per source) ───────────────────────────
|
||||
async function alreadyEscalatedRecently(pool: Pool, sourceKey: string): Promise<boolean> {
|
||||
const { rows } = await pool.query(
|
||||
`select 1 from self_heal_dispatch
|
||||
where source_key = $1 and outcome = 'escalated'
|
||||
and dispatched_at > now() - ($2 || ' hours')::interval limit 1`,
|
||||
[sourceKey, String(ESCALATE_REPAGE_HOURS)],
|
||||
);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
async function isAcknowledged(pool: Pool, sourceKey: string): Promise<boolean> {
|
||||
const { rows } = await pool.query(
|
||||
`select 1 from acknowledged_exceptions
|
||||
where (source_key = $1 or source_key is null)
|
||||
and (ack_until is null or ack_until > now()) limit 1`,
|
||||
[sourceKey],
|
||||
);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
async function escalateSource(pool: Pool, t: Target, reason: string, runId: string): Promise<void> {
|
||||
await escalateByKey(pool, t.source_key, reason, runId, { rows: t.rows_30d, lastObs: t.last_observation, status: t.status });
|
||||
}
|
||||
|
||||
async function escalateByKey(
|
||||
pool: Pool,
|
||||
sourceKey: string,
|
||||
reason: string,
|
||||
runId: string,
|
||||
ctx?: { rows?: number; lastObs?: string | null; status?: string },
|
||||
): Promise<void> {
|
||||
// delta-gate: suppress if acknowledged, and never re-page the same source within 24h
|
||||
if (await isAcknowledged(pool, sourceKey)) return;
|
||||
if (await alreadyEscalatedRecently(pool, sourceKey)) return;
|
||||
|
||||
await insertLedger(pool, {
|
||||
source_key: sourceKey, queues: [], reason, mode: "plan",
|
||||
rows_before: ctx?.rows ?? null, last_obs_before: ctx?.lastObs ?? null, outcome: "escalated", strikes: 0, run_id: runId,
|
||||
});
|
||||
await notify("page", `TIP data-quality: ${sourceKey}`, reason);
|
||||
writeRobotExperience({
|
||||
event: "dqo_escalation", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Escalated ${sourceKey}: ${reason}`, input: { source_key: sourceKey, ...ctx },
|
||||
decision: { escalated: true }, safety_notes: ["Paged once; suppressed for 24h + honors acknowledged_exceptions."],
|
||||
}, { flush: true });
|
||||
}
|
||||
|
||||
export async function runOnce(): Promise<{ tick: TickResult; verify: VerifyResult }> {
|
||||
const tick = await runTick();
|
||||
const verify = await runVerifyTick();
|
||||
return { tick, verify };
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
/**
|
||||
* Notification sink for the tip-dqo escalation ladder.
|
||||
*
|
||||
* Default channel: self-hosted ntfy on Erik (no external secret, fits "Gitea = EVERYTHING").
|
||||
* Set NTFY_URL (e.g. https://ntfy.fichtmueller.org) and optionally NTFY_TOPIC (default
|
||||
* "tip-dqo"). If NTFY_URL is unset, notify() is a logged no-op so the loop still runs and
|
||||
* the escalation is recorded in the ledger — the channel can be wired later without a
|
||||
* code change.
|
||||
*/
|
||||
|
||||
export type EscalationLevel = "digest" | "page";
|
||||
|
||||
export interface NotifyResult {
|
||||
sent: boolean;
|
||||
channel: "ntfy" | "noop" | "error";
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export async function notify(
|
||||
level: EscalationLevel,
|
||||
title: string,
|
||||
body: string,
|
||||
): Promise<NotifyResult> {
|
||||
const base = process.env.NTFY_URL?.replace(/\/+$/, "");
|
||||
const topic = process.env.NTFY_TOPIC || "tip-dqo";
|
||||
|
||||
if (!base) {
|
||||
console.warn(`[dqo:escalation:noop] (${level}) ${title} — ${body} [set NTFY_URL to enable]`);
|
||||
return { sent: false, channel: "noop" };
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${base}/${topic}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Title: title,
|
||||
Priority: level === "page" ? "high" : "default",
|
||||
Tags: level === "page" ? "rotating_light" : "warning",
|
||||
...(process.env.NTFY_TOKEN ? { Authorization: `Bearer ${process.env.NTFY_TOKEN}` } : {}),
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
return { sent: false, channel: "error", detail: `HTTP ${resp.status}` };
|
||||
}
|
||||
return { sent: true, channel: "ntfy" };
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[dqo:escalation] ntfy failed: ${detail}`);
|
||||
return { sent: false, channel: "error", detail };
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
/**
|
||||
* tip-dqo — TIP Autonomous Data-Quality Orchestrator (entrypoint).
|
||||
*
|
||||
* A SEPARATE always-on process (PM2 app `tip-dqo`), deliberately NOT grafted into the
|
||||
* tip-scraper-daemon event loop, so its blast radius is isolated and it never contends
|
||||
* the daemon's pg Pool. Read-mostly: its only writes are to its own ledger/registry tables.
|
||||
*
|
||||
* Default (no args) : self-heal tick every DQO_SELF_HEAL_MINUTES (30), verify tick every
|
||||
* DQO_VERIFY_MINUTES (15). Runs until SIGINT/SIGTERM.
|
||||
* --once : run one self-heal + verify tick, print the plan, exit. Used to
|
||||
* PROVE the loop against the live DB without scheduling anything.
|
||||
*
|
||||
* SELF_HEAL_PLAN_ONLY defaults to "true" (dispatches NOTHING; writes plan rows a human
|
||||
* reviews). Set SELF_HEAL_PLAN_ONLY=false only to arm live dispatch.
|
||||
*/
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { runOnce, runTick, runVerifyTick, closePool } from "./control-loop";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
const SELF_HEAL_MINUTES = Number.parseInt(process.env.DQO_SELF_HEAL_MINUTES || "30", 10);
|
||||
const VERIFY_MINUTES = Number.parseInt(process.env.DQO_VERIFY_MINUTES || "15", 10);
|
||||
const PLAN_ONLY = process.env.SELF_HEAL_PLAN_ONLY !== "false";
|
||||
|
||||
function stamp(): string {
|
||||
return new Date().toISOString().replace("T", " ").slice(0, 19);
|
||||
}
|
||||
|
||||
async function once(): Promise<void> {
|
||||
console.log(`\n=== tip-dqo one-shot (${PLAN_ONLY ? "PLAN-ONLY" : "ARMED"}) ${stamp()} ===`);
|
||||
const { tick, verify } = await runOnce();
|
||||
console.log(`\n[self-heal] detected=${tick.detected} planned=${tick.planned} dispatched=${tick.dispatched} skipped=${tick.skipped} escalated=${tick.escalated}`);
|
||||
for (const l of tick.lines) console.log(l);
|
||||
console.log(`\n[verify] checked=${verify.checked} healed=${verify.healed} ineffective=${verify.ineffective}`);
|
||||
for (const l of verify.lines) console.log(l);
|
||||
console.log("");
|
||||
await closePool();
|
||||
}
|
||||
|
||||
async function loop(): Promise<void> {
|
||||
console.log(`=== tip-dqo starting (${PLAN_ONLY ? "PLAN-ONLY" : "ARMED"}) — self-heal ${SELF_HEAL_MINUTES}m / verify ${VERIFY_MINUTES}m ===`);
|
||||
|
||||
const selfHeal = async () => {
|
||||
try {
|
||||
const r = await runTick();
|
||||
console.log(`[${stamp()}] self-heal: detected=${r.detected} planned=${r.planned} dispatched=${r.dispatched} skipped=${r.skipped} escalated=${r.escalated}`);
|
||||
for (const l of r.lines) console.log(l);
|
||||
} catch (err) {
|
||||
console.error(`[${stamp()}] self-heal error:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
};
|
||||
const verify = async () => {
|
||||
try {
|
||||
const r = await runVerifyTick();
|
||||
if (r.checked > 0) {
|
||||
console.log(`[${stamp()}] verify: checked=${r.checked} healed=${r.healed} ineffective=${r.ineffective}`);
|
||||
for (const l of r.lines) console.log(l);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[${stamp()}] verify error:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
};
|
||||
|
||||
await selfHeal(); // run once on boot
|
||||
const t1 = setInterval(selfHeal, SELF_HEAL_MINUTES * 60_000);
|
||||
const t2 = setInterval(verify, VERIFY_MINUTES * 60_000);
|
||||
|
||||
const shutdown = async (sig: string) => {
|
||||
console.log(`\n[${stamp()}] ${sig} — shutting down tip-dqo`);
|
||||
clearInterval(t1);
|
||||
clearInterval(t2);
|
||||
await closePool().catch(() => {});
|
||||
process.exit(0);
|
||||
};
|
||||
process.on("SIGINT", () => void shutdown("SIGINT"));
|
||||
process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
||||
}
|
||||
|
||||
const isOnce = process.argv.includes("--once");
|
||||
(isOnce ? once() : loop()).catch(async (err) => {
|
||||
console.error("tip-dqo fatal:", err);
|
||||
await closePool().catch(() => {});
|
||||
process.exit(1);
|
||||
});
|
||||
@ -1,78 +0,0 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
isLoadAcceptable,
|
||||
minIntervalOk,
|
||||
budgetOk,
|
||||
backoffElapsed,
|
||||
breakerTripped,
|
||||
canDispatch,
|
||||
type GuardConfig,
|
||||
type LedgerFacts,
|
||||
} from "./self-heal-guards";
|
||||
|
||||
const CFG: GuardConfig = {
|
||||
cooldown_minutes: 180,
|
||||
budget_per_day: 4,
|
||||
max_backoff_hours: 24,
|
||||
circuit_breaker_strikes: 3,
|
||||
};
|
||||
const NOW = new Date("2026-07-06T12:00:00Z");
|
||||
const fresh: LedgerFacts = { last_dispatch_at: null, last_strikes: 0, dispatches_today: 0 };
|
||||
|
||||
test("load guard", () => {
|
||||
assert.equal(isLoadAcceptable(1.2).allow, true);
|
||||
assert.equal(isLoadAcceptable(3.0).allow, false);
|
||||
assert.equal(isLoadAcceptable(2.5).allow, true); // boundary: not greater than
|
||||
});
|
||||
|
||||
test("min-interval blocks within cooldown, allows after", () => {
|
||||
const recent = { ...fresh, last_dispatch_at: new Date("2026-07-06T11:00:00Z") }; // 60min ago
|
||||
assert.equal(minIntervalOk(recent, CFG, NOW).allow, false);
|
||||
const old = { ...fresh, last_dispatch_at: new Date("2026-07-06T08:00:00Z") }; // 240min ago
|
||||
assert.equal(minIntervalOk(old, CFG, NOW).allow, true);
|
||||
assert.equal(minIntervalOk(fresh, CFG, NOW).allow, true); // never dispatched
|
||||
});
|
||||
|
||||
test("daily budget", () => {
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 3 }, CFG).allow, true);
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 4 }, CFG).allow, false);
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 9 }, CFG).allow, false);
|
||||
});
|
||||
|
||||
test("exponential backoff scales with strikes and caps", () => {
|
||||
// 2 strikes → 4h backoff. Dispatched 1h ago → still backing off.
|
||||
const s2recent = { last_dispatch_at: new Date("2026-07-06T11:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(s2recent, CFG, NOW).allow, false);
|
||||
// 2 strikes → 4h backoff. Dispatched 5h ago → elapsed.
|
||||
const s2old = { last_dispatch_at: new Date("2026-07-06T07:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(s2old, CFG, NOW).allow, true);
|
||||
// 0 strikes → no backoff
|
||||
assert.equal(backoffElapsed(fresh, CFG, NOW).allow, true);
|
||||
// huge strikes → capped at max_backoff_hours (24h), dispatched 25h ago → elapsed
|
||||
const capped = { last_dispatch_at: new Date("2026-07-05T11:00:00Z"), last_strikes: 10, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(capped, CFG, NOW).allow, true);
|
||||
});
|
||||
|
||||
test("circuit breaker opens at threshold", () => {
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 2 }, CFG).allow, true);
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 3 }, CFG).allow, false);
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 5 }, CFG).allow, false);
|
||||
});
|
||||
|
||||
test("canDispatch: clean source under low load passes", () => {
|
||||
assert.equal(canDispatch(fresh, CFG, NOW, 1.0).allow, true);
|
||||
});
|
||||
|
||||
test("canDispatch: breaker is reported before other brakes", () => {
|
||||
const tripped = { last_dispatch_at: new Date("2026-07-06T11:59:00Z"), last_strikes: 3, dispatches_today: 9 };
|
||||
const v = canDispatch(tripped, CFG, NOW, 5.0);
|
||||
assert.equal(v.allow, false);
|
||||
assert.match(v.reason, /breaker OPEN/);
|
||||
});
|
||||
|
||||
test("canDispatch: high load blocks an otherwise-clean source", () => {
|
||||
const v = canDispatch(fresh, CFG, NOW, 9.0);
|
||||
assert.equal(v.allow, false);
|
||||
assert.match(v.reason, /load/);
|
||||
});
|
||||
@ -1,126 +0,0 @@
|
||||
/**
|
||||
* Cost-safety guardrails for the tip-dqo self-heal loop — PURE, network-free, unit-testable.
|
||||
*
|
||||
* The controller must NEVER hammer a source. These functions turn ledger facts +
|
||||
* the source's guardrail config into a typed {allow, reason} verdict, so every skip is
|
||||
* explainable and logged (never a silent no-op). The DB reads live in control-loop.ts;
|
||||
* the DECISIONS live here and are tested with node:test against fixture facts.
|
||||
*
|
||||
* Nine compounding brakes (this module implements 5 of them; the rest — kill-switch,
|
||||
* plan-mode, root-cause routing, profile cap — live in control-loop.ts):
|
||||
* min-interval · daily budget · exponential backoff · circuit-breaker · load-guard
|
||||
*/
|
||||
|
||||
export interface GuardVerdict {
|
||||
allow: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** The per-source guardrail contract, straight from source_registry. */
|
||||
export interface GuardConfig {
|
||||
cooldown_minutes: number;
|
||||
budget_per_day: number;
|
||||
max_backoff_hours: number;
|
||||
circuit_breaker_strikes: number;
|
||||
max_load?: number;
|
||||
}
|
||||
|
||||
/** Facts read from the self_heal_dispatch ledger for one source. */
|
||||
export interface LedgerFacts {
|
||||
/** dispatched_at of the most recent mode='dispatch' row, or null if never dispatched. */
|
||||
last_dispatch_at: Date | null;
|
||||
/** strikes as of the most recent verified dispatch (running no-progress count). */
|
||||
last_strikes: number;
|
||||
/** count of mode='dispatch' rows for this source since local midnight. */
|
||||
dispatches_today: number;
|
||||
}
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
const MIN_MS = 60_000;
|
||||
|
||||
/**
|
||||
* Load guard — do not dispatch when the box is already busy. Pure: caller passes the
|
||||
* 1-minute load average (os.loadavg()[0]) so this stays testable. Mirrors the existing
|
||||
* scheduler.ts isLoadAcceptable so self-heal never fights the load guard.
|
||||
*/
|
||||
export function isLoadAcceptable(load1: number, maxLoad = 2.5): GuardVerdict {
|
||||
if (load1 > maxLoad) {
|
||||
return { allow: false, reason: `load ${load1.toFixed(2)} > ${maxLoad}` };
|
||||
}
|
||||
return { allow: true, reason: `load ${load1.toFixed(2)} <= ${maxLoad}` };
|
||||
}
|
||||
|
||||
/** Per-source minimum interval — a source cannot be re-fired within its cooldown window. */
|
||||
export function minIntervalOk(facts: LedgerFacts, cfg: GuardConfig, now: Date): GuardVerdict {
|
||||
if (!facts.last_dispatch_at) return { allow: true, reason: "no prior dispatch" };
|
||||
const elapsedMin = (now.getTime() - facts.last_dispatch_at.getTime()) / MIN_MS;
|
||||
if (elapsedMin < cfg.cooldown_minutes) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `cooldown: ${Math.round(elapsedMin)}min < ${cfg.cooldown_minutes}min since last dispatch`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `cooldown elapsed (${Math.round(elapsedMin)}min)` };
|
||||
}
|
||||
|
||||
/** Daily budget — no more than budget_per_day dispatches for this source since midnight. */
|
||||
export function budgetOk(facts: LedgerFacts, cfg: GuardConfig): GuardVerdict {
|
||||
if (facts.dispatches_today >= cfg.budget_per_day) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `budget: ${facts.dispatches_today}/${cfg.budget_per_day} dispatches used today`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `budget ${facts.dispatches_today}/${cfg.budget_per_day}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff — a repeatedly-ineffective source backs off 2^strikes hours
|
||||
* (capped at max_backoff_hours) instead of retrying every tick.
|
||||
*/
|
||||
export function backoffElapsed(facts: LedgerFacts, cfg: GuardConfig, now: Date): GuardVerdict {
|
||||
if (facts.last_strikes <= 0 || !facts.last_dispatch_at) {
|
||||
return { allow: true, reason: "no backoff (0 strikes)" };
|
||||
}
|
||||
const backoffHours = Math.min(Math.pow(2, facts.last_strikes), cfg.max_backoff_hours);
|
||||
const readyAt = new Date(facts.last_dispatch_at.getTime() + backoffHours * HOUR_MS);
|
||||
if (now < readyAt) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `backoff: ${facts.last_strikes} strikes → wait ${backoffHours}h (ready ${readyAt.toISOString()})`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `backoff elapsed (${backoffHours}h after ${facts.last_strikes} strikes)` };
|
||||
}
|
||||
|
||||
/** Circuit-breaker — at N strikes the source is removed from the retry pool (escalated instead). */
|
||||
export function breakerTripped(facts: LedgerFacts, cfg: GuardConfig): GuardVerdict {
|
||||
if (facts.last_strikes >= cfg.circuit_breaker_strikes) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `breaker OPEN: ${facts.last_strikes} >= ${cfg.circuit_breaker_strikes} strikes`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `breaker closed (${facts.last_strikes}/${cfg.circuit_breaker_strikes})` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine all guardrails into one verdict. Returns the FIRST failing brake so the skip
|
||||
* reason is specific. load1 is the current 1-min load average.
|
||||
*/
|
||||
export function canDispatch(
|
||||
facts: LedgerFacts,
|
||||
cfg: GuardConfig,
|
||||
now: Date,
|
||||
load1: number,
|
||||
): GuardVerdict {
|
||||
const checks = [
|
||||
breakerTripped(facts, cfg), // breaker first: an open breaker means "escalate, don't retry"
|
||||
budgetOk(facts, cfg),
|
||||
minIntervalOk(facts, cfg, now),
|
||||
backoffElapsed(facts, cfg, now),
|
||||
isLoadAcceptable(load1, cfg.max_load),
|
||||
];
|
||||
const failed = checks.find((c) => !c.allow);
|
||||
return failed ?? { allow: true, reason: "all guardrails passed" };
|
||||
}
|
||||
@ -53,41 +53,23 @@ function extractFirstNm(wavelengths: string | null): number | null {
|
||||
// ── Haupt-Matching-Logik (identisch mit Nightly-Matcher) ────────────────────
|
||||
|
||||
function calcConfidence(
|
||||
fx: {
|
||||
standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null;
|
||||
cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null;
|
||||
},
|
||||
cand: {
|
||||
standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null;
|
||||
cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null;
|
||||
}
|
||||
): { confidence: number; basis: string[]; hardReject: boolean } {
|
||||
// Hard rejects: incompatible entity types — no spec similarity overrides these
|
||||
if (fx.protocol_family && cand.protocol_family && fx.protocol_family !== cand.protocol_family) {
|
||||
return { confidence: 0, basis: [`protocol_family:${fx.protocol_family}≠${cand.protocol_family}`], hardReject: true };
|
||||
}
|
||||
if (fx.temp_range && cand.temp_range && fx.temp_range !== cand.temp_range) {
|
||||
return { confidence: 0, basis: [`temp_range:${fx.temp_range}≠${cand.temp_range}`], hardReject: true };
|
||||
}
|
||||
if (fx.wdm_type && cand.wdm_type && fx.wdm_type !== cand.wdm_type) {
|
||||
return { confidence: 0, basis: [`wdm_type:${fx.wdm_type}≠${cand.wdm_type}`], hardReject: true };
|
||||
}
|
||||
|
||||
fx: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null },
|
||||
cand: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null }
|
||||
): { confidence: number; basis: string[] } {
|
||||
// Max-Score: form_factor(25) + speed_gbps(20) + standard_name(30) +
|
||||
// wavelength_nm(20) + fiber_type(10) + reach(10) = 115
|
||||
// form_factor and speed_gbps are pre-filtered by SQL.
|
||||
// Beide form_factor und speed_gbps sind bereits durch den SQL-Filter gesichert.
|
||||
let score = 0;
|
||||
const basis: string[] = [];
|
||||
|
||||
score += 25; basis.push("form_factor");
|
||||
score += 20; basis.push("speed_gbps");
|
||||
|
||||
if (fx.standard_name && cand.standard_name) {
|
||||
if (fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) {
|
||||
score += 30; basis.push("standard_name");
|
||||
} else {
|
||||
score -= 25; // known mismatch: strong negative signal
|
||||
}
|
||||
if (
|
||||
fx.standard_name && cand.standard_name &&
|
||||
fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()
|
||||
) {
|
||||
score += 30; basis.push("standard_name");
|
||||
}
|
||||
|
||||
const fxNm = extractFirstNm(fx.wavelengths);
|
||||
@ -120,17 +102,8 @@ function calcConfidence(
|
||||
score += 5; basis.push("reach_null");
|
||||
}
|
||||
|
||||
// CDR type mismatch at >=400G is a strong penalty (different product class)
|
||||
if (fx.cdr_type && cand.cdr_type) {
|
||||
if (fx.cdr_type !== cand.cdr_type) {
|
||||
score -= 20;
|
||||
} else {
|
||||
score += 5; basis.push(`cdr_${fx.cdr_type}`);
|
||||
}
|
||||
}
|
||||
|
||||
const confidence = Math.max(0, Math.min(1, score / 115));
|
||||
return { confidence, basis, hardReject: false };
|
||||
return { confidence, basis };
|
||||
}
|
||||
|
||||
// ── Haupt-Funktion ───────────────────────────────────────────────────────────
|
||||
@ -162,14 +135,9 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
fiber_type: string | null;
|
||||
reach_meters: number | null;
|
||||
wavelengths: string | null;
|
||||
cdr_type: string | null;
|
||||
temp_range: string | null;
|
||||
protocol_family: string | null;
|
||||
wdm_type: string | null;
|
||||
}>(`
|
||||
SELECT t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
|
||||
@ -201,17 +169,12 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
reach_meters: number | null;
|
||||
wavelengths: string | null;
|
||||
vendor_name: string;
|
||||
cdr_type: string | null;
|
||||
temp_range: string | null;
|
||||
protocol_family: string | null;
|
||||
wdm_type: string | null;
|
||||
last_price: Date | null;
|
||||
price_count: string;
|
||||
}>(`
|
||||
SELECT t.id AS competitor_id, t.part_number, t.standard_name,
|
||||
t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters,
|
||||
t.wavelengths, v.name AS vendor_name,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type,
|
||||
MAX(po.time) AS last_price, COUNT(*) AS price_count
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
@ -223,25 +186,19 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
AND ROUND(t.speed_gbps::NUMERIC, 2) = ROUND($2::NUMERIC, 2)
|
||||
AND t.id != $3
|
||||
GROUP BY t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name
|
||||
HAVING COUNT(*) >= 1
|
||||
`, [fx.form_factor, fx.speed_gbps, fx.id]);
|
||||
|
||||
for (const cand of candidates) {
|
||||
const { confidence, basis, hardReject } = calcConfidence(fx, cand);
|
||||
const { confidence, basis } = calcConfidence(fx, cand);
|
||||
|
||||
if (hardReject || confidence < CONFIDENCE_MIN) {
|
||||
if (confidence < CONFIDENCE_MIN) {
|
||||
result.skippedLowConfidence++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Require FX standard_name for auto_approve; if competitor also has one it must match
|
||||
const candStdOk = !cand.standard_name || !fx.standard_name ||
|
||||
cand.standard_name.trim().toUpperCase() === fx.standard_name.trim().toUpperCase();
|
||||
const canAutoApprove = fx.standard_name !== null && candStdOk;
|
||||
|
||||
const status = (confidence >= CONFIDENCE_AUTO_APPROVE && canAutoApprove) ? "auto_approved" : "pending";
|
||||
const status = confidence >= CONFIDENCE_AUTO_APPROVE ? "auto_approved" : "pending";
|
||||
const notes =
|
||||
`${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` +
|
||||
`basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` +
|
||||
@ -249,25 +206,18 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
`last_price: ${cand.last_price?.toISOString() ?? "never"} | ` +
|
||||
`source: catalog-reconcile`;
|
||||
|
||||
// Upsert — bereits approved/rejected Einträge nicht überschreiben. Pending
|
||||
// rows get re_research_due_at = NOW() so the daily
|
||||
// maintenance:re-research-equivalences job (scheduler.ts) actually drains
|
||||
// them — that job only evaluates rows where re_research_due_at IS NOT NULL,
|
||||
// and without this the backlog only grows (found live 2026-07-13: 289,681
|
||||
// pending rows had accumulated with re_research_due_at always NULL).
|
||||
const reResearchDueAt = status === "pending" ? new Date() : null;
|
||||
// Upsert — bereits approved/rejected Einträge nicht überschreiben
|
||||
const { rowCount } = await pool.query(`
|
||||
INSERT INTO transceiver_equivalences
|
||||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status, re_research_due_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
|
||||
confidence = EXCLUDED.confidence,
|
||||
match_basis = EXCLUDED.match_basis,
|
||||
match_notes = EXCLUDED.match_notes,
|
||||
re_research_due_at = COALESCE(transceiver_equivalences.re_research_due_at, EXCLUDED.re_research_due_at),
|
||||
updated_at = NOW()
|
||||
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected', 'auto_approved')
|
||||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status, reResearchDueAt]);
|
||||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]);
|
||||
|
||||
const wasInsertOrUpdate = (rowCount ?? 0) > 0;
|
||||
if (!wasInsertOrUpdate) {
|
||||
|
||||
@ -40,11 +40,8 @@ interface CatalogProduct {
|
||||
title: string;
|
||||
url: string | null;
|
||||
imageUrl: string | null;
|
||||
productType: string | null;
|
||||
price: {
|
||||
amount: number | null;
|
||||
listAmount: number | null;
|
||||
selfConfigureAmount: number | null;
|
||||
currency: string | null;
|
||||
source: "api" | "missing";
|
||||
fetchedAt: string;
|
||||
@ -66,8 +63,6 @@ interface CatalogProduct {
|
||||
bidi: boolean | null;
|
||||
dwdm: boolean | null;
|
||||
cwdm: boolean | null;
|
||||
cdrType: string | null;
|
||||
protocolFamily: string | null;
|
||||
};
|
||||
compatibility: Array<{
|
||||
vendor: string;
|
||||
@ -83,7 +78,6 @@ export interface FlexoptixSyncResult {
|
||||
skipped: number;
|
||||
priceWrites: number;
|
||||
stockWrites: number;
|
||||
mapWrites: number;
|
||||
}
|
||||
|
||||
// ── Generic helpers ────────────────────────────────────────────────────────
|
||||
@ -187,8 +181,7 @@ function parseReachMeters(value: unknown): number | null {
|
||||
|
||||
function parseSpeedGbps(value: unknown): number | null {
|
||||
const numeric = asNumber(value);
|
||||
// API returns -10 / -1 as sentinel for N/A (e.g. SFP trays, accessories)
|
||||
if (numeric !== null) return numeric < 0 ? null : numeric;
|
||||
if (numeric !== null) return numeric;
|
||||
const text = asString(value)?.toLowerCase();
|
||||
if (!text) return null;
|
||||
const gbps = text.match(/([\d.,]+)\s*(g|gb|gbps|gbit)/);
|
||||
@ -238,42 +231,12 @@ function inferWavelengthNm(...values: Array<string | null>): number | null {
|
||||
|
||||
// ── Normalization ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
function parseStandardName(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
// Extract from first segment (before first " | ")
|
||||
const seg = title.split(" | ")[0] ?? title;
|
||||
// Try full IEEE form first: 10GBASE-SR, 100GBASE-DR4, etc.
|
||||
const ieee = seg.match(/((?:\d+(?:\.\d+)?[TGM])?BASE[-/]\w+)/i);
|
||||
if (ieee) return ieee[1].toUpperCase();
|
||||
// Short-form standards — ordered longest-first to avoid partial matches
|
||||
const m = seg.match(
|
||||
/(OpenZR\+|ZR\+|ZX\+|DR4\+|LR4\+|SR10|SR8|LR8|ER8|DR8|VR8|SR4|LR4|ER4|FR4|DR4|CLR4|PLR4|PSM4|CWDM4|LRM|BX\w*|DR|LR|ER|SR|ZR|ZX|SX|LX|LH|EX|T)/
|
||||
);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
function parseCdrType(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
if (/dual[\s-]cdr/i.test(title)) return "dual_cdr";
|
||||
if (/with[\s-]?cdr/i.test(title)) return "cdr";
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseProtocolFamily(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
if (/fibre[\s-]?channel|fc\s*\d+g/i.test(title)) return "fibre_channel";
|
||||
if (/\bsonet\b|\bsdh\b|\boc-\d+\b/i.test(title)) return "sonet";
|
||||
if (/\binfiniband\b/i.test(title)) return "infiniband";
|
||||
return "ethernet";
|
||||
}
|
||||
|
||||
function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] {
|
||||
const rawCompat = row.compatibility ?? row.compatibilities ?? row.vendorCompatibility;
|
||||
const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : [];
|
||||
return rows.flatMap(entry => {
|
||||
const flat = flatLookup(entry);
|
||||
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor", "compatibleToVendor"]));
|
||||
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"]));
|
||||
if (!vendor) return [];
|
||||
return [{
|
||||
vendor,
|
||||
@ -292,10 +255,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
|
||||
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
||||
const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"]));
|
||||
const productType = asString(pick(flat, ["type", "producttype", "moduletype"]));
|
||||
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
||||
const listAmount = asNumber(pick(flat, ["listprice", "msrp", "rrp", "recommendedprice"]));
|
||||
const selfConfigureAmount = asNumber(pick(flat, ["selfconfigureprice", "selfconfigprice"]));
|
||||
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
||||
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
||||
const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"]));
|
||||
@ -317,13 +277,10 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
fetchedAt,
|
||||
sku,
|
||||
title,
|
||||
productType,
|
||||
url,
|
||||
imageUrl,
|
||||
price: {
|
||||
amount,
|
||||
listAmount: (listAmount !== null && listAmount !== amount) ? listAmount : null,
|
||||
selfConfigureAmount,
|
||||
currency,
|
||||
source: amount === null ? "missing" : "api",
|
||||
fetchedAt,
|
||||
@ -345,8 +302,6 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
bidi: asBoolean(pick(flat, ["bidi", "bidirectional"])) ?? flags.bidi,
|
||||
dwdm: asBoolean(pick(flat, ["dwdm"])) ?? flags.dwdm,
|
||||
cwdm: asBoolean(pick(flat, ["cwdm"])) ?? flags.cwdm,
|
||||
cdrType: parseCdrType(title),
|
||||
protocolFamily: parseProtocolFamily(title),
|
||||
},
|
||||
compatibility: extractCompatibility(row),
|
||||
};
|
||||
@ -354,33 +309,14 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
|
||||
// ── Import helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
function isMerchandise(product: CatalogProduct): boolean {
|
||||
// Flexoptix merchandise SKU prefixes: jackets (MJA), mugs (MMU), stickers (MST),
|
||||
// t-shirts (MSH), trolley (MTR), buff (MBU). FO-EXTRA-* and FO-PC-* are non-product.
|
||||
if (/^FO\.M(JA|MU|ST|SH|TR|BU)\d/i.test(product.sku)) return true;
|
||||
if (/^FO-(EXTRA-COMMUNITY-KIT|PC-\d)/i.test(product.sku)) return true;
|
||||
const t = product.title.toLowerCase();
|
||||
return /\b(jacket|coffee mug|t-shirt|sticker|travel trolley|buff)\b/.test(t);
|
||||
}
|
||||
|
||||
function canImportProduct(product: CatalogProduct): boolean {
|
||||
// Minimal gate: sku + title required. formFactor/speed/reach are optional
|
||||
// — findOrCreateScrapedTransceiver provides sensible defaults for accessories.
|
||||
// Skip country-of-origin variants (SKUs like "S2A.CL.10000:S2") — they are
|
||||
// duplicates of the base SKU with a different sourcing region, same price/stock.
|
||||
if (product.sku.includes(":")) return false;
|
||||
if (isMerchandise(product)) return false;
|
||||
return Boolean(product.sku && product.title);
|
||||
}
|
||||
|
||||
function formFactorFallback(product: CatalogProduct): string | undefined {
|
||||
const t = product.productType?.toLowerCase() ?? "";
|
||||
if (/accessories|adapter/i.test(t)) return "Accessory";
|
||||
if (/hardware/i.test(t)) return "Hardware";
|
||||
// Passive MTP/MPO breakout cables have no form-factor in the API response.
|
||||
const title = product.title.toLowerCase();
|
||||
if (/\bmtp\b|\bmpo\b/.test(title) && /cable|breakout/.test(title)) return "MTP";
|
||||
return undefined;
|
||||
return Boolean(
|
||||
product.sku
|
||||
&& product.title
|
||||
&& product.optics.formFactor
|
||||
&& product.optics.speedGbps !== null
|
||||
&& product.optics.reachM !== null,
|
||||
);
|
||||
}
|
||||
|
||||
function reachLabel(reachM: number | null): string | undefined {
|
||||
@ -396,14 +332,8 @@ function speedLabel(speedGbps: number | null): string | undefined {
|
||||
}
|
||||
|
||||
function categoryFor(product: CatalogProduct): string {
|
||||
const apiType = product.productType?.toLowerCase() ?? "";
|
||||
if (/accessories|adapter/i.test(apiType)) return "Accessory";
|
||||
if (/hardware/i.test(apiType)) return "Hardware";
|
||||
if (apiType.includes("cable")) return "Cable";
|
||||
const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase();
|
||||
// Passive MTP/MPO fan-out cables (e.g. M3.T10L.* "MTP female to 10x LC ... Breakout cable")
|
||||
if (/\bmtp\b.*cable|cable.*\bmtp\b|\bmpo\b.*cable|\bbreakout cable\b/.test(text)) return "Cable";
|
||||
if (/\bdac\b|direct attach|passive copper/.test(text)) return "DAC";
|
||||
if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
|
||||
if (/\baoc\b|active optical/.test(text)) return "AOC";
|
||||
if (/coherent|zr|dco/.test(text)) return "Coherent";
|
||||
return "DataCenter";
|
||||
@ -417,41 +347,26 @@ async function importProduct(
|
||||
partNumber: product.sku,
|
||||
vendorId,
|
||||
productUrl: product.url ?? undefined,
|
||||
formFactor: product.optics.formFactor ?? formFactorFallback(product),
|
||||
formFactor: product.optics.formFactor ?? undefined,
|
||||
speedGbps: product.optics.speedGbps ?? undefined,
|
||||
speed: speedLabel(product.optics.speedGbps),
|
||||
reachMeters: product.optics.reachM ?? undefined,
|
||||
reachLabel: reachLabel(product.optics.reachM) ?? undefined,
|
||||
reachLabel: reachLabel(product.optics.reachM),
|
||||
fiberType: product.optics.fiberType ?? undefined,
|
||||
wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
|
||||
category: categoryFor(product),
|
||||
});
|
||||
|
||||
// Write image_url, product_page_url, and notes (product title) from API
|
||||
if (product.imageUrl || product.url || product.title) {
|
||||
const wdmType = product.optics.dwdm ? "DWDM" : product.optics.cwdm ? "CWDM" : null;
|
||||
// Write image_url and product_page_url from bulk API response
|
||||
if (product.imageUrl || product.url) {
|
||||
await pool.query(`
|
||||
UPDATE transceivers SET
|
||||
image_url = COALESCE(NULLIF(image_url, ''), $1::text),
|
||||
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2::text),
|
||||
notes = $3::text,
|
||||
standard_name = COALESCE(NULLIF(standard_name, ''), $5::text),
|
||||
wdm_type = COALESCE(NULLIF(wdm_type, ''), $6::text),
|
||||
cdr_type = COALESCE(NULLIF(cdr_type, ''), $7::text),
|
||||
cdr_support = CASE WHEN $7::text IS NOT NULL THEN true ELSE cdr_support END,
|
||||
protocol_family = COALESCE(NULLIF(protocol_family, ''), $8::text),
|
||||
image_url = COALESCE(NULLIF(image_url, ''), $1),
|
||||
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2),
|
||||
updated_at = NOW()
|
||||
WHERE id = $4
|
||||
`, [
|
||||
product.imageUrl ?? null,
|
||||
product.url ?? null,
|
||||
product.title,
|
||||
transceiverId,
|
||||
parseStandardName(product.title) ?? product.optics.protocol ?? null,
|
||||
wdmType,
|
||||
product.optics.cdrType ?? null,
|
||||
product.optics.protocolFamily ?? null,
|
||||
]);
|
||||
WHERE id = $3
|
||||
AND ($1 IS NOT NULL OR $2 IS NOT NULL)
|
||||
`, [product.imageUrl ?? null, product.url ?? null, transceiverId]);
|
||||
}
|
||||
|
||||
let priceWritten = false;
|
||||
@ -474,61 +389,11 @@ async function importProduct(
|
||||
});
|
||||
}
|
||||
|
||||
if (product.price.selfConfigureAmount !== null && product.price.currency) {
|
||||
await upsertPriceObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price.selfConfigureAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-selfconfigure",
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
url: product.url ?? undefined,
|
||||
contentHash: contentHash({
|
||||
source: product.source,
|
||||
sku: product.sku,
|
||||
price: product.price.selfConfigureAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-selfconfigure",
|
||||
fetchedAt: product.price.fetchedAt,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
if (product.price.listAmount !== null && product.price.currency) {
|
||||
await upsertPriceObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price.listAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-msrp",
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
url: product.url ?? undefined,
|
||||
contentHash: contentHash({
|
||||
source: product.source,
|
||||
sku: product.sku,
|
||||
price: product.price.listAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-msrp",
|
||||
fetchedAt: product.price.fetchedAt,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// When status is known but no numeric quantity, synthesize a value so
|
||||
// upsertStockObservation's early-exit guard doesn't skip the write.
|
||||
const syntheticQty =
|
||||
product.stock.quantity !== null ? product.stock.quantity
|
||||
: product.stock.status === "in_stock" ? 1
|
||||
: product.stock.status === "out_of_stock" ? 0
|
||||
: undefined;
|
||||
|
||||
const stockWritten = await upsertStockObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: syntheticQty,
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
priceNet: product.price.amount ?? undefined,
|
||||
productUrl: product.url ?? undefined,
|
||||
priceCurrency: product.price.currency ?? undefined,
|
||||
@ -538,22 +403,6 @@ async function importProduct(
|
||||
return { priceWritten, stockWritten };
|
||||
}
|
||||
|
||||
async function writeProductMapEntries(product: CatalogProduct): Promise<number> {
|
||||
const entries = product.compatibility
|
||||
.filter(c => c.coding !== null && c.vendor !== null)
|
||||
.map(c => ({
|
||||
oemPartNumber: c.coding!,
|
||||
oemVendor: c.vendor,
|
||||
flexoptixSku: product.sku,
|
||||
priceEur: product.price.amount,
|
||||
formFactor: product.optics.formFactor,
|
||||
speedGbps: product.optics.speedGbps,
|
||||
reachLabel: reachLabel(product.optics.reachM) ?? null,
|
||||
fiberType: product.optics.fiberType,
|
||||
}));
|
||||
return batchUpsertProductMap(entries);
|
||||
}
|
||||
|
||||
// ── API client ─────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } {
|
||||
@ -561,7 +410,7 @@ function validateEnv(): { baseUrl: string; username: string | null; password: st
|
||||
if (!baseUrl) {
|
||||
throw new Error("FLEXOPTIX_API_BASE_URL is required for Flexoptix API sync");
|
||||
}
|
||||
const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() || null; // empty string -> null so bearer-login fallback fires
|
||||
const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() ?? null;
|
||||
const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim() ?? null;
|
||||
const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim() ?? null;
|
||||
if (!token && (!username || !password)) {
|
||||
@ -624,6 +473,8 @@ async function fetchAllProducts(baseUrl: string, headers: Record<string, string>
|
||||
if (rows.length === 0) break;
|
||||
|
||||
allRows.push(...rows);
|
||||
|
||||
if (rows.length < (Number.isFinite(limit) ? limit : 500)) break;
|
||||
}
|
||||
|
||||
return allRows;
|
||||
@ -641,94 +492,6 @@ function extractRows(payload: unknown): JsonRecord[] {
|
||||
|
||||
// ── Main export ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
// ── Self-configure price enrichment ───────────────────────────────────────
|
||||
// The bulk paginated endpoint omits self_configure_price.
|
||||
// A targeted sku[] fetch returns it. Batch 50 SKUs per call (~10 calls total).
|
||||
|
||||
async function fetchSelfConfigurePrices(
|
||||
skus: string[],
|
||||
baseUrl: string,
|
||||
headers: Record<string, string>,
|
||||
currency: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Map<string, number>> {
|
||||
const BATCH_SIZE = 50;
|
||||
const priceMap = new Map<string, number>();
|
||||
const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products";
|
||||
|
||||
for (let i = 0; i < skus.length; i += BATCH_SIZE) {
|
||||
const batch = skus.slice(i, i + BATCH_SIZE);
|
||||
const url = buildUrl(baseUrl, productPath);
|
||||
url.searchParams.set("currency", currency);
|
||||
batch.forEach(sku => url.searchParams.append("sku[]", sku));
|
||||
|
||||
try {
|
||||
const payload = await fetchJson(url, { headers }, timeoutMs);
|
||||
for (const row of extractRows(payload)) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sku = asString(row["sku"] ?? row["SKU"]);
|
||||
const flat = flatLookup(row);
|
||||
const scPrice = asNumber(flat["selfconfigureprice"] ?? flat["selfconfigprice"] ?? null);
|
||||
if (sku && scPrice !== null) priceMap.set(sku, scPrice);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[${new Date().toISOString()}] Self-configure enrichment batch failed: ${msg.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return priceMap;
|
||||
}
|
||||
|
||||
|
||||
// ── Product map batch upsert ─────────────────────────────────────────────────
|
||||
// Writes OEM vendor coding entries into flexoptix_product_map.
|
||||
// One row per (oem_part_number, oem_vendor) pair — ON CONFLICT updates specs.
|
||||
|
||||
async function batchUpsertProductMap(entries: Array<{
|
||||
oemPartNumber: string;
|
||||
oemVendor: string;
|
||||
flexoptixSku: string;
|
||||
priceEur: number | null;
|
||||
formFactor: string | null;
|
||||
speedGbps: number | null;
|
||||
reachLabel: string | null;
|
||||
fiberType: string | null;
|
||||
}>): Promise<number> {
|
||||
if (entries.length === 0) return 0;
|
||||
const BATCH_SIZE = 500;
|
||||
let written = 0;
|
||||
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
|
||||
const chunk = entries.slice(i, i + BATCH_SIZE);
|
||||
const placeholders: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let n = 1;
|
||||
for (const e of chunk) {
|
||||
placeholders.push(`($${n},$${n+1},$${n+2},$${n+3},$${n+4},$${n+5},$${n+6},$${n+7},'exact',NOW())`);
|
||||
params.push(e.oemPartNumber, e.oemVendor, e.flexoptixSku, e.priceEur, e.formFactor, e.speedGbps, e.reachLabel, e.fiberType);
|
||||
n += 8;
|
||||
}
|
||||
await pool.query(`
|
||||
INSERT INTO flexoptix_product_map
|
||||
(oem_part_number, oem_vendor, flexoptix_sku, flexoptix_price_eur,
|
||||
form_factor, speed_gbps, reach_label, fiber_type, match_type, last_verified)
|
||||
VALUES ${placeholders.join(',')}
|
||||
ON CONFLICT (oem_part_number, oem_vendor) DO UPDATE SET
|
||||
flexoptix_sku = EXCLUDED.flexoptix_sku,
|
||||
flexoptix_price_eur = EXCLUDED.flexoptix_price_eur,
|
||||
form_factor = EXCLUDED.form_factor,
|
||||
speed_gbps = EXCLUDED.speed_gbps,
|
||||
reach_label = EXCLUDED.reach_label,
|
||||
fiber_type = EXCLUDED.fiber_type,
|
||||
last_verified = EXCLUDED.last_verified,
|
||||
updated_at = NOW()
|
||||
`, params);
|
||||
written += chunk.length;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
const { baseUrl, username, password, token } = validateEnv();
|
||||
const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
|
||||
@ -754,45 +517,25 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
const importable = products.filter(canImportProduct);
|
||||
const skipped = products.length - importable.length;
|
||||
|
||||
// Enrich importable products with self_configure_price (not in bulk response)
|
||||
const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR";
|
||||
const scPrices = await fetchSelfConfigurePrices(
|
||||
importable.map(p => p.sku),
|
||||
baseUrl,
|
||||
headers,
|
||||
currency,
|
||||
timeoutMs,
|
||||
);
|
||||
let scEnriched = 0;
|
||||
for (const product of importable) {
|
||||
const scPrice = scPrices.get(product.sku);
|
||||
if (scPrice !== undefined) {
|
||||
product.price.selfConfigureAmount = scPrice;
|
||||
scEnriched++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped} | self-configure enriched: ${scEnriched}`);
|
||||
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`);
|
||||
|
||||
const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
|
||||
|
||||
let priceWrites = 0;
|
||||
let stockWrites = 0;
|
||||
let mapWrites = 0;
|
||||
|
||||
for (const product of importable) {
|
||||
try {
|
||||
const result = await importProduct(product, vendorId);
|
||||
if (result.priceWritten) priceWrites++;
|
||||
if (result.stockWritten) stockWrites++;
|
||||
mapWrites += await writeProductMapEntries(product);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes, ${mapWrites} product-map upserts`);
|
||||
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`);
|
||||
|
||||
return {
|
||||
fetched: rawRows.length,
|
||||
@ -800,6 +543,5 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
skipped,
|
||||
priceWrites,
|
||||
stockWrites,
|
||||
mapWrites,
|
||||
};
|
||||
}
|
||||
|
||||
@ -69,8 +69,6 @@ interface FxApiProduct {
|
||||
}
|
||||
|
||||
interface ParsedSpecs {
|
||||
formFactor: string | null;
|
||||
speedGbps: number | null;
|
||||
complianceCode: string | null;
|
||||
laserType: string | null;
|
||||
receiverType: string | null;
|
||||
@ -188,73 +186,6 @@ function parseModulation(text: string | null): string | null {
|
||||
return match ? match[1].toUpperCase().replace("PAM-4", "PAM4") : text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative form factor from the structured "Form Factor" spec (NOT the name).
|
||||
*/
|
||||
function normalizeFormFactorSpec(v: string | null): string | null {
|
||||
if (!v) return null;
|
||||
const t = v.toUpperCase();
|
||||
const order = ["QSFP-DD800","QSFP-DD","QSFP112","QSFP56","QSFP28","QSFP+",
|
||||
"OSFP224","OSFP112","OSFP","SFP56","SFP28","CSFP","SFP+","SFP",
|
||||
"XFP","CFP4","CFP2","CFP","GBIC"];
|
||||
for (const ff of order) if (t.includes(ff)) return ff === "CSFP" ? "SFP" : ff;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Map a datasheet line rate (Gbit/s) to the nominal standard speed. */
|
||||
function mapLineRateToNominal(u: number): number {
|
||||
const table: Array<[number, number]> = [
|
||||
[0.2, 0.1], [1.5, 1], [3.5, 2.5], [6, 5], [13, 10], [20, 16],
|
||||
[33, 25], [46, 40], [60, 50], [120, 100], [240, 200], [480, 400],
|
||||
[960, 800], [2000, 1600],
|
||||
];
|
||||
for (const [ceil, nom] of table) if (u <= ceil) return nom;
|
||||
return Math.round(u);
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative nominal speed (Gbps) sourced from the datasheet specs only:
|
||||
* primary = highest Ethernet rate named in "Supported Protocols";
|
||||
* fallback = upper bound of "Bandwidth" line rate mapped to nominal.
|
||||
* Never derived from the product name.
|
||||
*/
|
||||
function nominalSpeedFromSpecs(specs: FxApiSpec[]): number | null {
|
||||
const protocols = specArray(specs, "Supported Protocols");
|
||||
let best: number | null = null;
|
||||
for (const p of protocols) {
|
||||
const t = p.toLowerCase();
|
||||
let sp: number | null = null;
|
||||
if (/\bfast ethernet\b/.test(t)) sp = 0.1;
|
||||
else if (/\bgigabit ethernet\b/.test(t)) sp = 1;
|
||||
else {
|
||||
// Terabit first ("1.6t ethernet" -> 1600), then Gigabit. Aggregate "Nx MMMg"
|
||||
// twin-port labels resolve via the explicit total (e.g. "1.6T Ethernet").
|
||||
const mt = t.match(/\b(\d+(?:\.\d+)?)\s*t(?:b|be|ig)?\s*ethernet\b/);
|
||||
if (mt) sp = parseFloat(mt[1]) * 1000;
|
||||
else {
|
||||
const m = t.match(/(?:^|[^x\d])(\d+(?:\.\d+)?)\s*g(?:b|be|ig)?\s*ethernet\b/) || t.match(/\b(\d+)\s*gbe\b/);
|
||||
if (m) sp = parseFloat(m[1]);
|
||||
}
|
||||
}
|
||||
if (sp !== null && (best === null || sp > best)) best = sp;
|
||||
}
|
||||
if (best !== null) return best;
|
||||
const bw = specValue(specs, "Bandwidth");
|
||||
if (bw) {
|
||||
const gbits = Array.from(bw.matchAll(/([\d.]+)\s*gbit\/s/gi)).map(m => parseFloat(m[1]));
|
||||
if (gbits.length) return mapLineRateToNominal(Math.max(...gbits));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Format a nominal Gbps value as a display label ("1G", "100G", "1.6T"). */
|
||||
function speedLabelGbps(g: number | null): string | null {
|
||||
if (g === null) return null;
|
||||
if (g >= 1000) return `${g / 1000}T`;
|
||||
if (g < 1) return `${Math.round(g * 1000)}M`;
|
||||
return `${g}G`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the flat specifications array into structured fields.
|
||||
*/
|
||||
@ -263,8 +194,6 @@ function parseSpecs(specs: FxApiSpec[]): ParsedSpecs {
|
||||
const rxPowers = parseDbm(specValue(specs, "Receiver min/max per lane"));
|
||||
|
||||
return {
|
||||
formFactor: normalizeFormFactorSpec(specValue(specs, "Form Factor")),
|
||||
speedGbps: nominalSpeedFromSpecs(specs),
|
||||
complianceCode: specValue(specs, "Compliance Code"),
|
||||
laserType: specValue(specs, "Laser"),
|
||||
receiverType: specValue(specs, "Receiver Type"),
|
||||
@ -420,9 +349,6 @@ async function writeDetails(
|
||||
|
||||
await pool.query(`
|
||||
UPDATE transceivers SET
|
||||
form_factor = COALESCE($23, form_factor),
|
||||
speed_gbps = COALESCE($24, speed_gbps),
|
||||
speed = COALESCE($25, speed),
|
||||
fx_specifications = $1,
|
||||
fx_compatibilities = $2,
|
||||
compliance_code = COALESCE(compliance_code, $3),
|
||||
@ -470,9 +396,6 @@ async function writeDetails(
|
||||
product.image ?? null, // $20
|
||||
product.url ?? null, // $21
|
||||
transceiverId, // $22
|
||||
parsed.formFactor, // $23
|
||||
parsed.speedGbps, // $24
|
||||
speedLabelGbps(parsed.speedGbps), // $25
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@ -14,18 +14,9 @@
|
||||
* - Max 30 competitor matches per FX product (safety cap)
|
||||
*
|
||||
* Match quality:
|
||||
* confidence = 0.85 (starting prior only)
|
||||
* confidence = 0.85
|
||||
* match_basis = '{spec}'
|
||||
* status = 'pending' — unlike OPN matches (manufacturer-confirmed ground
|
||||
* truth), spec matches are a coarse JOIN with no standard_name / fiber_type /
|
||||
* recent-price discrimination, so they route into the same
|
||||
* maintenance:re-research-equivalences pipeline as catalog-reconcile.ts's
|
||||
* pending rows (evaluateEquivalenceResearch(), scheduler.ts) for real
|
||||
* confidence scoring before ever being served as an approved equivalence.
|
||||
* Found live 2026-07-17: of the equivalences this matcher had previously put
|
||||
* straight into 'auto_approved', ~19% failed a recent-competitor-price check
|
||||
* and ~8% had a conflicting standard_name — i.e. real false-positive risk
|
||||
* was going live unreviewed.
|
||||
* status = 'auto_approved'
|
||||
*/
|
||||
|
||||
import { pool } from "../utils/db";
|
||||
@ -48,14 +39,13 @@ const INSERT_SPEC_MATCHES = `
|
||||
match_basis,
|
||||
match_notes,
|
||||
created_at,
|
||||
updated_at,
|
||||
re_research_due_at
|
||||
updated_at
|
||||
)
|
||||
SELECT DISTINCT
|
||||
fx.id AS flexoptix_id,
|
||||
comp.id AS competitor_id,
|
||||
0.85 AS confidence,
|
||||
'pending' AS status,
|
||||
'auto_approved' AS status,
|
||||
ARRAY['spec'] AS match_basis,
|
||||
'Spec match: ' || fx.form_factor || ' ' || fx.speed_gbps || 'G ' ||
|
||||
CASE WHEN fx.reach_meters <= 300 THEN 'SR'
|
||||
@ -67,8 +57,7 @@ const INSERT_SPEC_MATCHES = `
|
||||
THEN ' @' || tip_extract_wavelength_nm(fx.wavelengths) || 'nm'
|
||||
ELSE '' END AS match_notes,
|
||||
NOW() AS created_at,
|
||||
NOW() AS updated_at,
|
||||
NOW() AS re_research_due_at
|
||||
NOW() AS updated_at
|
||||
FROM transceivers fx
|
||||
JOIN vendors vfx ON vfx.id = fx.vendor_id AND UPPER(vfx.name) LIKE '%FLEXOPTIX%'
|
||||
JOIN transceivers comp
|
||||
|
||||
@ -20,12 +20,11 @@ import { join } from "path";
|
||||
import PgBoss from "pg-boss";
|
||||
import { pool } from "../utils/db";
|
||||
import { writeRobotExperience } from "../crawler-llm/training-data-writer";
|
||||
import { dbConnectionString } from "../utils/db-connection";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
type RobotWave = "details-fast-lane" | "priority-vendors" | "all";
|
||||
export type RobotProfile = "erik-safe" | "pi-fetch" | "proxmox-heavy";
|
||||
type RobotProfile = "erik-safe" | "pi-fetch" | "proxmox-heavy";
|
||||
|
||||
interface VendorBlockerRow {
|
||||
vendor: string;
|
||||
@ -50,7 +49,7 @@ interface TipLlmResearchPlanResponse {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const connectionString = dbConnectionString();
|
||||
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
|
||||
const TIP_API_URL = process.env.TIP_API_URL || "http://127.0.0.1:3201";
|
||||
const TIP_API_TOKEN = process.env.TIP_API_TOKEN || process.env.TIP_TOKEN || "";
|
||||
|
||||
@ -65,7 +64,7 @@ const DETAILS_FAST_LANE_QUEUES = [
|
||||
"maintenance:reconcile-verification",
|
||||
];
|
||||
|
||||
export const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
{ match: /juniper/i, queues: ["scrape:catalog:juniper-oem", "scrape:catalog:juniper-mx-oem", "scrape:catalog:juniper-qfx-oem", "scrape:compat:juniper", "discover:vendor:juniper"] },
|
||||
{ match: /cisco/i, queues: ["scrape:catalog:cisco-nexus-oem", "scrape:catalog:cisco-catalyst-oem", "scrape:catalog:cisco-asr-oem", "scrape:compat:cisco", "discover:vendor:cisco-tmg"] },
|
||||
{ match: /fs\.?com/i, queues: ["scrape:pricing:fs", "discover:vendor:fs-com"] },
|
||||
@ -84,7 +83,7 @@ export const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
{ match: /ciena/i, queues: ["scrape:catalog:ciena-oem", "scrape:catalog:ciena-waveserver-oem"] },
|
||||
];
|
||||
|
||||
export const HEAVY_QUEUES = new Set([
|
||||
const HEAVY_QUEUES = new Set([
|
||||
"scrape:pricing:fs",
|
||||
"scrape:pricing:10gtek",
|
||||
"scrape:pricing:prolabs",
|
||||
@ -94,7 +93,7 @@ export const HEAVY_QUEUES = new Set([
|
||||
"scrape:compat:edgecore",
|
||||
]);
|
||||
|
||||
export const ERIK_SAFE_QUEUES = new Set([
|
||||
const ERIK_SAFE_QUEUES = new Set([
|
||||
"scrape:pricing:flexoptix",
|
||||
"scrape:pricing:fibermall",
|
||||
"scrape:pricing:atgbics",
|
||||
@ -105,11 +104,11 @@ export const ERIK_SAFE_QUEUES = new Set([
|
||||
"maintenance:reconcile-verification",
|
||||
]);
|
||||
|
||||
export function isDiscoveryQueue(queue: string): boolean {
|
||||
function isDiscoveryQueue(queue: string): boolean {
|
||||
return queue.startsWith("discover:");
|
||||
}
|
||||
|
||||
export function queuesForProfile(queues: string[], profile: RobotProfile): string[] {
|
||||
function queuesForProfile(queues: string[], profile: RobotProfile): string[] {
|
||||
if (profile === "proxmox-heavy") return queues;
|
||||
|
||||
if (profile === "erik-safe") {
|
||||
@ -119,7 +118,7 @@ export function queuesForProfile(queues: string[], profile: RobotProfile): strin
|
||||
return queues.filter((queue) => !HEAVY_QUEUES.has(queue) && !isDiscoveryQueue(queue));
|
||||
}
|
||||
|
||||
export function defaultMaxQueues(profile: RobotProfile): number {
|
||||
function defaultMaxQueues(profile: RobotProfile): number {
|
||||
if (profile === "erik-safe") return 3;
|
||||
if (profile === "pi-fetch") return 10;
|
||||
return 30;
|
||||
@ -133,7 +132,7 @@ function unique(items: string[]): string[] {
|
||||
return [...new Set(items)];
|
||||
}
|
||||
|
||||
export async function createBoss(): Promise<PgBoss> {
|
||||
async function createBoss(): Promise<PgBoss> {
|
||||
const boss = new PgBoss({
|
||||
connectionString,
|
||||
retryLimit: 1,
|
||||
@ -146,32 +145,6 @@ export async function createBoss(): Promise<PgBoss> {
|
||||
return boss;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single shared dispatch primitive: create each queue if needed, then send one job per
|
||||
* queue with the standard safety options (retryLimit 1, 2h expiry). Used by BOTH
|
||||
* enqueueRobotWave and the tip-dqo orchestrator so there is exactly ONE boss.send path
|
||||
* with identical safety options across the whole platform.
|
||||
*/
|
||||
export async function dispatchQueues(
|
||||
queues: string[],
|
||||
data: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
if (queues.length === 0) return;
|
||||
const boss = await createBoss();
|
||||
try {
|
||||
for (const queue of queues) {
|
||||
await boss.createQueue(queue).catch(() => {});
|
||||
await (boss as unknown as { send: (name: string, data?: object, options?: object) => Promise<string | null> }).send(
|
||||
queue,
|
||||
data,
|
||||
{ retryLimit: 1, expireInSeconds: 7200 },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await boss.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export async function getVerificationStatus(): Promise<{ summary: Record<string, string>; vendors: VendorBlockerRow[] }> {
|
||||
const summaryResult = await pool.query(`
|
||||
SELECT
|
||||
@ -287,12 +260,19 @@ export async function enqueueRobotWave(
|
||||
return queues;
|
||||
}
|
||||
|
||||
await dispatchQueues(queues, {
|
||||
source: "verification-robots",
|
||||
wave,
|
||||
run_id: runId,
|
||||
enqueued_at: new Date().toISOString(),
|
||||
});
|
||||
const boss = await createBoss();
|
||||
try {
|
||||
for (const queue of queues) {
|
||||
await boss.createQueue(queue).catch(() => {});
|
||||
await (boss as unknown as { send: (name: string, data?: object, options?: object) => Promise<string | null> }).send(
|
||||
queue,
|
||||
{ source: "verification-robots", wave, run_id: runId, enqueued_at: new Date().toISOString() },
|
||||
{ retryLimit: 1, expireInSeconds: 7200 },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await boss.stop();
|
||||
}
|
||||
|
||||
console.log("\nJobs enqueued.");
|
||||
writeRobotExperience({
|
||||
|
||||
@ -22,7 +22,6 @@ import PgBoss from "pg-boss";
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { loadavg } from "os";
|
||||
import { dbConnectionString } from "./utils/db-connection";
|
||||
|
||||
// withIsolatedStorage removed — all Crawlee scrapers now use makeCrawleeConfig()
|
||||
// for instance-level storage isolation. See packages/scraper/src/utils/crawlee-config.ts
|
||||
@ -43,7 +42,7 @@ function isLoadAcceptable(maxLoad = 2.5): boolean {
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", ".env") });
|
||||
|
||||
const connectionString = dbConnectionString();
|
||||
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
|
||||
|
||||
type EquivalenceProduct = {
|
||||
part_number?: string | null;
|
||||
@ -3182,13 +3181,6 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
);
|
||||
|
||||
if (research.decision === "reject") {
|
||||
const rejectReason = research.rejectReason || "automated research: rejected";
|
||||
// Data-gap rejects (missing wavelength/fiber/reach evidence, not a
|
||||
// confirmed mismatch) get a future re-check — once a scraper fills the
|
||||
// gap, re-research picks it back up. Confirmed mismatches / no-price /
|
||||
// low-confidence rejects get re_research_due_at=NULL (permanent; nothing
|
||||
// will change on a re-check with the same stored specs).
|
||||
const isDataGap = rejectReason.includes("insufficient technical evidence");
|
||||
await pool.query(`
|
||||
UPDATE transceiver_equivalences
|
||||
SET status = 'rejected',
|
||||
@ -3197,7 +3189,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
reject_reason = $4,
|
||||
reviewed_by = 'automated-research',
|
||||
reviewed_at = NOW(),
|
||||
re_research_due_at = CASE WHEN $6 THEN NOW() + INTERVAL '21 days' ELSE NULL END,
|
||||
re_research_due_at = NULL,
|
||||
re_researched_at = NOW(),
|
||||
match_notes = CONCAT(
|
||||
COALESCE(match_notes, ''),
|
||||
@ -3209,9 +3201,8 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
eq.id,
|
||||
research.confidence,
|
||||
research.basis,
|
||||
rejectReason,
|
||||
research.rejectReason || "automated research: rejected",
|
||||
research.reasons.join("; "),
|
||||
isDataGap,
|
||||
]);
|
||||
|
||||
await pool.query(`
|
||||
|
||||
@ -14,9 +14,7 @@
|
||||
*
|
||||
* Fallback (when search returns nothing):
|
||||
* Form-factor matching — for each port type in switch.ports_config,
|
||||
* find Flexoptix transceivers matching that form factor AND whose speed does
|
||||
* not exceed the switch's fastest port (a transceiver faster than the port
|
||||
* cannot link at rated speed — e.g. an 800G module in a 400G OSFP port).
|
||||
* find Flexoptix transceivers matching that form factor.
|
||||
* Used as spec_match rather than vendor_compat — shown in dashboard
|
||||
* as "by form factor" rather than "explicitly verified".
|
||||
*
|
||||
@ -65,18 +63,6 @@ function portsConfigToFormFactors(portsConfig: Record<string, number>): string[]
|
||||
return [...factors];
|
||||
}
|
||||
|
||||
// Max port speed (Gbps) implied by a ports_config, e.g. {"400G_QSFP-DD":4} → 400.
|
||||
// The switch cannot host a transceiver faster than its fastest port.
|
||||
function maxSpeedFromPortsConfig(portsConfig: Record<string, number> | null): number {
|
||||
if (!portsConfig) return 0;
|
||||
let max = 0;
|
||||
for (const key of Object.keys(portsConfig)) {
|
||||
const m = key.match(/(\d+)G/i);
|
||||
if (m) max = Math.max(max, parseInt(m[1], 10));
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
// ── Flexoptix search API ────────────────────────────────────────────────────
|
||||
|
||||
interface FlexoptixSuggestion {
|
||||
@ -119,9 +105,8 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
model: string;
|
||||
vendor_name: string;
|
||||
ports_config: Record<string, number> | null;
|
||||
max_speed_gbps: number | null;
|
||||
}>(
|
||||
`SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config, sw.max_speed_gbps
|
||||
`SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config
|
||||
FROM switches sw
|
||||
JOIN vendors v ON v.id = sw.vendor_id
|
||||
ORDER BY sw.max_speed_gbps DESC`,
|
||||
@ -197,15 +182,7 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
// ── Strategy 2: Form-factor fallback ─────────────────────────────────
|
||||
if (sw.ports_config) {
|
||||
const formFactors = portsConfigToFormFactors(sw.ports_config);
|
||||
// Physical speed ceiling: a transceiver faster than the switch's fastest
|
||||
// port cannot link at its rated speed, so it is NOT compatible-by-form-factor.
|
||||
// (ceiling <= 0 → the query below matches nothing, i.e. no fallback when the
|
||||
// switch has no derivable port speed.)
|
||||
const ceiling = Math.max(
|
||||
maxSpeedFromPortsConfig(sw.ports_config),
|
||||
sw.max_speed_gbps ?? 0,
|
||||
);
|
||||
console.log(` Form factors: ${formFactors.join(", ")} (≤ ${ceiling}G)`);
|
||||
console.log(` Form factors: ${formFactors.join(", ")}`);
|
||||
|
||||
for (const ff of formFactors) {
|
||||
const { rows: txRows } = await pool.query<{ id: string }>(
|
||||
@ -213,14 +190,11 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
FROM transceivers t
|
||||
WHERE t.vendor_id = $1
|
||||
AND t.form_factor = $2
|
||||
AND t.speed_gbps IS NOT NULL
|
||||
AND t.speed_gbps > 0
|
||||
AND t.speed_gbps <= $4
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM compatibility c
|
||||
WHERE c.switch_id = $3 AND c.transceiver_id = t.id
|
||||
)`,
|
||||
[flexoptixVendorId, ff, sw.id, ceiling],
|
||||
[flexoptixVendorId, ff, sw.id],
|
||||
);
|
||||
|
||||
for (const tx of txRows) {
|
||||
|
||||
@ -1,138 +0,0 @@
|
||||
/**
|
||||
* Unit tests for the FS.com pure parsers. Run with:
|
||||
* npx tsx --test src/scrapers/fs-com-parse.test.ts
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
parseGermanQty,
|
||||
parseGermanDate,
|
||||
parseGermanPrice,
|
||||
extractDeliveryDate,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
sanitizeUnitsSold,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── parseGermanQty — the K/M decimal bug ─────────────────────────────────────
|
||||
|
||||
test("parseGermanQty: plain integers use '.'/',' as thousands", () => {
|
||||
assert.equal(parseGermanQty("4.895"), 4895);
|
||||
assert.equal(parseGermanQty("1.587"), 1587);
|
||||
assert.equal(parseGermanQty("1.501"), 1501);
|
||||
assert.equal(parseGermanQty("2"), 2);
|
||||
assert.equal(parseGermanQty("1627"), 1627);
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => {
|
||||
// Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier).
|
||||
assert.equal(parseGermanQty("4.8K"), 4800);
|
||||
assert.equal(parseGermanQty("210.9K"), 210900);
|
||||
assert.equal(parseGermanQty("504.4K"), 504400);
|
||||
assert.equal(parseGermanQty("1.4M"), 1_400_000);
|
||||
assert.equal(parseGermanQty("1.2M"), 1_200_000);
|
||||
assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged
|
||||
assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M with German comma decimal still works", () => {
|
||||
assert.equal(parseGermanQty("4,8K"), 4800);
|
||||
assert.equal(parseGermanQty("1,4M"), 1_400_000);
|
||||
});
|
||||
|
||||
// ── parseGermanDate ──────────────────────────────────────────────────────────
|
||||
|
||||
test("parseGermanDate: word and numeric forms", () => {
|
||||
assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08");
|
||||
assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17");
|
||||
assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("28. März 2026"), "2026-03-28");
|
||||
assert.equal(parseGermanDate("20.04.2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("nonsense"), undefined);
|
||||
});
|
||||
|
||||
// ── extractDeliveryDate ──────────────────────────────────────────────────────
|
||||
|
||||
test("extractDeliveryDate: finds bare word-date, no keyword needed", () => {
|
||||
assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08");
|
||||
assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17");
|
||||
});
|
||||
|
||||
test("extractDeliveryDate: rejects stray/garbage years and empty", () => {
|
||||
assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined);
|
||||
assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined);
|
||||
});
|
||||
|
||||
// ── parseWarehouseStock — the real screenshot block ──────────────────────────
|
||||
|
||||
const SCREENSHOT_BLOCK = [
|
||||
"2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar",
|
||||
"1.627 Stk. im Global-Lager, 17. Juli 2026",
|
||||
"1 in Nachlieferung, 28. Juli 2026",
|
||||
"4.8K verkauft",
|
||||
].join("\n");
|
||||
|
||||
test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => {
|
||||
const s = parseWarehouseStock(SCREENSHOT_BLOCK);
|
||||
assert.equal(s.deQty, 2);
|
||||
assert.equal(s.deDeliveryDate, "2026-07-08");
|
||||
assert.equal(s.globalQty, 1627);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
assert.equal(s.backorderQty, 1);
|
||||
assert.equal(s.backorderDate, "2026-07-28");
|
||||
assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => {
|
||||
const block = [
|
||||
"5 Stk. im DE-Lager", // no date — available now
|
||||
"800 Stk. im Global-Lager, 17. Juli 2026",
|
||||
].join("\n");
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 5);
|
||||
assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17
|
||||
assert.equal(s.globalQty, 800);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => {
|
||||
const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft";
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 12);
|
||||
assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: empty when no warehouse text present", () => {
|
||||
assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {});
|
||||
});
|
||||
|
||||
// ── sanitizeUnitsSold ────────────────────────────────────────────────────────
|
||||
|
||||
test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => {
|
||||
assert.equal(sanitizeUnitsSold(4800), 4800);
|
||||
assert.equal(sanitizeUnitsSold(504400), 504400);
|
||||
assert.equal(sanitizeUnitsSold(14_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(-5), undefined);
|
||||
assert.equal(sanitizeUnitsSold(undefined), undefined);
|
||||
assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000);
|
||||
});
|
||||
|
||||
// ── computeLeadTimeDays ──────────────────────────────────────────────────────
|
||||
|
||||
test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => {
|
||||
const now = new Date("2026-07-04T09:00:00");
|
||||
assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4);
|
||||
assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1);
|
||||
assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative
|
||||
assert.equal(computeLeadTimeDays([], now), undefined);
|
||||
assert.equal(computeLeadTimeDays([null, undefined], now), undefined);
|
||||
});
|
||||
|
||||
// ── parseGermanPrice (unchanged behaviour, regression guard) ─────────────────
|
||||
|
||||
test("parseGermanPrice: net EUR parsing", () => {
|
||||
assert.equal(parseGermanPrice("42,50"), 42.5);
|
||||
assert.equal(parseGermanPrice("1.063,02 €"), 1063.02);
|
||||
assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1);
|
||||
});
|
||||
@ -1,290 +0,0 @@
|
||||
/**
|
||||
* FS.com parsing helpers — pure, side-effect-free, unit-tested.
|
||||
*
|
||||
* Extracted from fs-com.ts so the number/date/warehouse parsing can be tested
|
||||
* in isolation (fs-com.ts pulls in Crawlee + Playwright + a pg Pool, which we
|
||||
* do not want to load in a unit test).
|
||||
*
|
||||
* The FS.com German storefront (www.fs.com/de/) renders per-warehouse
|
||||
* availability as a short line per warehouse, e.g.:
|
||||
*
|
||||
* 2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar
|
||||
* 1.627 Stk. im Global-Lager, 17. Juli 2026
|
||||
* 1 in Nachlieferung, 28. Juli 2026
|
||||
* 4.8K verkauft
|
||||
*
|
||||
* The quantity part was already parsed correctly; the delivery-date part was
|
||||
* not, because the previous regex only looked for a "Lieferung:"/"erwartet:"
|
||||
* keyword prefix or a numeric dd.mm.yyyy date — neither of which matches the
|
||||
* bare word-date ("8. Juli 2026") that FS.com actually renders.
|
||||
*/
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Upper bound for a plausible per-SKU lifetime "verkauft" (units-sold) counter.
|
||||
* Values above this are treated as unknown (dropped), not stored. FS.com's
|
||||
* biggest sellers are in the hundreds of thousands; multi-million values are
|
||||
* a mis-parse (see parseGermanQty K/M bug) or vendor marketing inflation.
|
||||
* Override with FS_MAX_UNITS_SOLD.
|
||||
*/
|
||||
export const DEFAULT_MAX_UNITS_SOLD = 2_000_000;
|
||||
|
||||
/** How far past a warehouse line to look for its delivery date (characters). */
|
||||
const DATE_WINDOW = 200;
|
||||
|
||||
// ── Number parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Parse the mantissa of a K/M-abbreviated number ("4.8" in "4.8K").
|
||||
*
|
||||
* K/M abbreviations carry a small decimal, never a grouped thousands value, so
|
||||
* a lone "." is the decimal point ("4.8K" = 4 800). German pages may instead
|
||||
* use "," as the decimal ("4,8K"); a "." alongside a "," is then a thousands
|
||||
* grouping and is stripped.
|
||||
*/
|
||||
function parseAbbrevMantissa(s: string): number {
|
||||
if (s.includes(",")) return parseFloat(s.replace(/\./g, "").replace(",", "."));
|
||||
return parseFloat(s); // "." is the decimal separator
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in a plain integer)
|
||||
* "1.587" → 1587
|
||||
* "4.8K" → 4800 (period = decimal in a K/M abbreviation)
|
||||
* "210.9K" → 210900
|
||||
* "1.4M" → 1400000
|
||||
* "63K" → 63000
|
||||
*/
|
||||
export function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseAbbrevMantissa(kMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseAbbrevMantissa(mMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
// Plain integer: "." and "," are both thousands separators here.
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German date to ISO "YYYY-MM-DD".
|
||||
* "8. Juli 2026" → "2026-07-08"
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
export function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German price to a EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
export function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
|
||||
// ── Units-sold sanity gate ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Clamp an implausible units-sold value to `undefined` (unknown). Returns the
|
||||
* value unchanged when it is a finite, non-negative number at or below `cap`.
|
||||
*/
|
||||
export function sanitizeUnitsSold(
|
||||
raw: number | undefined,
|
||||
cap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): number | undefined {
|
||||
if (raw === undefined) return undefined;
|
||||
if (!Number.isFinite(raw) || raw < 0) return undefined;
|
||||
if (raw > cap) return undefined;
|
||||
return raw;
|
||||
}
|
||||
|
||||
// ── Delivery-date extraction ─────────────────────────────────────────────────
|
||||
|
||||
/** dd.mm.yyyy OR "8. Juli 2026" / "20 Apr., 2026" (day + month word + year). */
|
||||
const DELIVERY_DATE_RE =
|
||||
/(\d{1,2}\.\d{1,2}\.\d{4})|(\d{1,2}\.?\s+[A-Za-zÄÖÜäöüß]{3,}\.?,?\s*\d{4})/;
|
||||
|
||||
/**
|
||||
* Find the first plausible calendar date inside a bounded text block (the slice
|
||||
* that belongs to one warehouse line) and return it as ISO "YYYY-MM-DD".
|
||||
* The date can appear with or without a "Lieferbar"/"Lieferung" keyword — FS.com
|
||||
* renders it bare ("…DE-Lager, 8. Juli 2026 Lieferbar"), so no keyword is required.
|
||||
*/
|
||||
export function extractDeliveryDate(block: string): string | undefined {
|
||||
const m = block.match(DELIVERY_DATE_RE);
|
||||
if (!m) return undefined;
|
||||
const iso = parseGermanDate(m[0]);
|
||||
if (!iso) return undefined;
|
||||
const year = Number(iso.slice(0, 4));
|
||||
if (year < 2024 || year > 2100) return undefined; // reject stray/garbage years
|
||||
return iso;
|
||||
}
|
||||
|
||||
// ── Warehouse block parsing ──────────────────────────────────────────────────
|
||||
|
||||
export interface WarehouseStock {
|
||||
deQty?: number;
|
||||
deDeliveryDate?: string;
|
||||
globalQty?: number;
|
||||
globalDeliveryDate?: string;
|
||||
backorderQty?: number;
|
||||
backorderDate?: string;
|
||||
unitsSold?: number;
|
||||
}
|
||||
|
||||
const DE_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i,
|
||||
];
|
||||
const GLOBAL_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i,
|
||||
/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i,
|
||||
];
|
||||
const BACKORDER_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i,
|
||||
/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i,
|
||||
];
|
||||
const UNITS_SOLD_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/,
|
||||
/([\d.,KkMm]+)\+?\s*sold/i,
|
||||
];
|
||||
|
||||
interface SectionMatch { value: string; index: number; }
|
||||
|
||||
function firstMatch(text: string, res: RegExp[]): SectionMatch | undefined {
|
||||
for (const re of res) {
|
||||
const m = text.match(re);
|
||||
if (m && m[1] != null && m.index != null) return { value: m[1], index: m.index };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the full warehouse-availability block out of a product page's body
|
||||
* text: DE-Lager, Global-Lager and Nachlieferung quantities, each with the
|
||||
* delivery date rendered on its line, plus the "verkauft"/"sold" counter.
|
||||
*
|
||||
* A section's delivery date is looked for only in the window between that
|
||||
* section's quantity and the start of the next section, so a warehouse with no
|
||||
* date never picks up a neighbouring warehouse's date.
|
||||
*/
|
||||
export function parseWarehouseStock(
|
||||
bodyText: string,
|
||||
unitsSoldCap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): WarehouseStock {
|
||||
const t = bodyText;
|
||||
const de = firstMatch(t, DE_QTY_RE);
|
||||
const gl = firstMatch(t, GLOBAL_QTY_RE);
|
||||
const bo = firstMatch(t, BACKORDER_QTY_RE);
|
||||
const sold = firstMatch(t, UNITS_SOLD_RE);
|
||||
|
||||
const starts = [de, gl, bo, sold]
|
||||
.filter((s): s is SectionMatch => s !== undefined)
|
||||
.map((s) => s.index);
|
||||
|
||||
const dateFor = (sec: SectionMatch | undefined): string | undefined => {
|
||||
if (!sec) return undefined;
|
||||
const later = starts.filter((b) => b > sec.index);
|
||||
const end = later.length ? Math.min(sec.index + DATE_WINDOW, ...later) : sec.index + DATE_WINDOW;
|
||||
return extractDeliveryDate(t.slice(sec.index, end));
|
||||
};
|
||||
|
||||
const result: WarehouseStock = {};
|
||||
|
||||
if (de) {
|
||||
const q = parseGermanQty(de.value);
|
||||
if (q !== undefined) result.deQty = q;
|
||||
const d = dateFor(de);
|
||||
if (d) result.deDeliveryDate = d;
|
||||
}
|
||||
if (gl) {
|
||||
const q = parseGermanQty(gl.value);
|
||||
if (q !== undefined) result.globalQty = q;
|
||||
const d = dateFor(gl);
|
||||
if (d) result.globalDeliveryDate = d;
|
||||
}
|
||||
if (bo) {
|
||||
const q = parseGermanQty(bo.value);
|
||||
if (q !== undefined) result.backorderQty = q;
|
||||
const d = dateFor(bo);
|
||||
if (d) result.backorderDate = d;
|
||||
}
|
||||
if (sold) {
|
||||
const s = sanitizeUnitsSold(parseGermanQty(sold.value), unitsSoldCap);
|
||||
if (s !== undefined) result.unitsSold = s;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Lead-time derivation ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Derive lead-time in days as the soonest availability across the warehouses
|
||||
* that quote a delivery date: min(date − today), floored at 0. Returns
|
||||
* `undefined` when no date is known (never fabricate a 0).
|
||||
*/
|
||||
export function computeLeadTimeDays(
|
||||
dates: Array<string | null | undefined>,
|
||||
now: Date = new Date()
|
||||
): number | undefined {
|
||||
const todayUtc = Date.UTC(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
let min: number | undefined;
|
||||
for (const d of dates) {
|
||||
if (!d) continue;
|
||||
const m = d.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (!m) continue;
|
||||
const dUtc = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
|
||||
const days = Math.max(0, Math.round((dUtc - todayUtc) / 86_400_000));
|
||||
if (min === undefined || days < min) min = days;
|
||||
}
|
||||
return min;
|
||||
}
|
||||
@ -63,12 +63,6 @@ import {
|
||||
} from "../utils/db";
|
||||
import { contentHash } from "../utils/hash";
|
||||
import { updateVerifiedSpecs, parseSpecTable } from "../utils/spec-updater";
|
||||
import {
|
||||
parseGermanPrice,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
DEFAULT_MAX_UNITS_SOLD,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@ -81,8 +75,6 @@ const FORCE_REVALIDATE = process.env["TIP_FORCE_REVALIDATE"] === "1";
|
||||
const ONLY_MISSING_IMAGES = process.env["FS_ONLY_MISSING_IMAGES"] === "1";
|
||||
const DB_DETAIL_ONLY = process.env["FS_DB_DETAIL_ONLY"] === "1";
|
||||
const URL_DISCOVERY_ONLY = process.env["FS_URL_DISCOVERY_ONLY"] === "1";
|
||||
// Drop implausible "verkauft" (units-sold) counters at ingest (see fs-com-parse).
|
||||
const FS_MAX_UNITS_SOLD = parseInt(process.env["FS_MAX_UNITS_SOLD"] ?? String(DEFAULT_MAX_UNITS_SOLD), 10);
|
||||
|
||||
const PROXY_URLS = (process.env["PROXY_URLS"] ?? "")
|
||||
.split(",")
|
||||
@ -121,7 +113,83 @@ const DE_COOKIES = [
|
||||
{ name: "country", value: "DE", domain: ".fs.com", path: "/" },
|
||||
];
|
||||
|
||||
// German number/date/warehouse parsers live in ./fs-com-parse (pure + unit-tested).
|
||||
// ── German locale parsers ──────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in German)
|
||||
* "210.9K" → 210900
|
||||
* "1.2M" → 1200000
|
||||
*/
|
||||
function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseFloat(kMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseFloat(mMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German date to ISO "YYYY-MM-DD".
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German price to EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
|
||||
// ── Stock level helper ─────────────────────────────────────────────────────────
|
||||
|
||||
@ -580,18 +648,64 @@ async function scrapeProductDetails(
|
||||
}
|
||||
}
|
||||
|
||||
// ── Warehouse availability (qty + per-warehouse delivery date + units sold) ──
|
||||
// FS.com renders each warehouse on its own line, e.g.
|
||||
// "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar". parseWarehouseStock
|
||||
// extracts the bare word-date per warehouse and gates units_sold.
|
||||
const wh = parseWarehouseStock(t, FS_MAX_UNITS_SOLD);
|
||||
const deQty = wh.deQty;
|
||||
const deDeliveryDate = wh.deDeliveryDate;
|
||||
const globalQty = wh.globalQty;
|
||||
const globalDeliveryDate = wh.globalDeliveryDate;
|
||||
const backorderQty = wh.backorderQty;
|
||||
const backorderDate = wh.backorderDate;
|
||||
const unitsSold = wh.unitsSold;
|
||||
// ── DE-Lager ───────────────────────────────────────────────────────────
|
||||
let deQty: number | undefined;
|
||||
let deDeliveryDate: string | undefined;
|
||||
const deM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i);
|
||||
if (deM?.[1]) {
|
||||
deQty = parseGermanQty(deM[1]);
|
||||
const idx = t.indexOf(deM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) deDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Global-Lager ───────────────────────────────────────────────────────
|
||||
let globalQty: number | undefined;
|
||||
let globalDeliveryDate: string | undefined;
|
||||
const glM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i);
|
||||
if (glM?.[1]) {
|
||||
globalQty = parseGermanQty(glM[1]);
|
||||
const idx = t.indexOf(glM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) globalDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Nachlieferung ──────────────────────────────────────────────────────
|
||||
let backorderQty: number | undefined;
|
||||
let backorderDate: string | undefined;
|
||||
const boM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i) ??
|
||||
t.match(/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i);
|
||||
if (boM?.[1]) {
|
||||
backorderQty = parseGermanQty(boM[1]);
|
||||
const idx = t.indexOf(boM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/[Ee]rwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) backorderDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Units sold ─────────────────────────────────────────────────────────
|
||||
let unitsSold: number | undefined;
|
||||
const soldM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/) ??
|
||||
t.match(/([\d.,KkMm]+)\+?\s*sold/i);
|
||||
if (soldM?.[1]) unitsSold = parseGermanQty(soldM[1]);
|
||||
|
||||
// ── Part number refinement ─────────────────────────────────────────────
|
||||
let partNumber = listingPn;
|
||||
@ -700,8 +814,7 @@ export async function scrapeFs(): Promise<void> {
|
||||
WHERE v.name = 'FS.COM'
|
||||
AND COALESCE(t.product_page_url, '') = ''
|
||||
AND t.part_number ~ '^FS-[0-9]+$'
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
t.part_number
|
||||
ORDER BY t.part_number
|
||||
LIMIT $1
|
||||
`,
|
||||
[MAX_DETAIL_PAGES_PER_RUN]
|
||||
@ -733,7 +846,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
OR COALESCE(t.reach_label, '') = ''
|
||||
)
|
||||
ORDER BY
|
||||
(COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus), dann Verifikations-Lücke
|
||||
COALESCE(t.price_verified, false) DESC,
|
||||
COALESCE(t.image_verified, false) DESC,
|
||||
COALESCE(t.details_verified, false) ASC,
|
||||
@ -898,12 +1010,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
|
||||
const stockLevel = deriveStockLevel(detail.deQty, detail.globalQty, detail.backorderQty);
|
||||
const totalQty = (detail.deQty ?? 0) + (detail.globalQty ?? 0);
|
||||
// Soonest availability across the warehouses that quote a delivery date.
|
||||
const leadTimeDays = computeLeadTimeDays([
|
||||
detail.deDeliveryDate,
|
||||
detail.globalDeliveryDate,
|
||||
detail.backorderDate,
|
||||
]);
|
||||
|
||||
if (detail.priceNet && detail.priceNet > 0) {
|
||||
const hash = contentHash({
|
||||
@ -918,7 +1024,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
currency: "EUR",
|
||||
stockLevel,
|
||||
quantityAvailable: totalQty > 0 ? totalQty : undefined,
|
||||
leadTimeDays,
|
||||
url: detail.url,
|
||||
contentHash: hash,
|
||||
});
|
||||
@ -937,7 +1042,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
backorderQty: detail.backorderQty,
|
||||
backorderEstimatedDate: detail.backorderDate ?? null,
|
||||
unitsSold: detail.unitsSold,
|
||||
leadTimeDays,
|
||||
compatibleBrands: detail.compatibleBrands,
|
||||
priceNet: detail.priceNet,
|
||||
productUrl: detail.url,
|
||||
|
||||
@ -279,9 +279,6 @@ export async function computeReorderSignals(): Promise<void> {
|
||||
|
||||
if (reasons.length === 0) reasons.push("Insufficient data for strong signal");
|
||||
|
||||
// Keep exactly one (latest) signal per transceiver — delete prior rows first.
|
||||
// Without this the table grew to 4.5M rows (24h-TTL never cleaned up old runs).
|
||||
await pool.query(`DELETE FROM reorder_signals WHERE transceiver_id = $1`, [row.id]);
|
||||
await pool.query(
|
||||
`INSERT INTO reorder_signals
|
||||
(transceiver_id, signal, signal_strength, reasons, stock_trend, price_trend, lead_time_weeks)
|
||||
|
||||
@ -193,45 +193,6 @@ function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | nul
|
||||
return { confidence: 1 }; // fallback: boolean
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse per-warehouse stock breakdown from NADDOD HTML.
|
||||
* The data lives as HTML-entity-encoded JSON in a hydration payload:
|
||||
* "warehouse_stock":[0,{"us":[0,543],"nl":[0,211],"sg":[0,0],"cn":[0,0]}]
|
||||
* Format per region: [signalBit, quantity]
|
||||
* Mapping: us → US, nl → NL/EU (closest to DE), sg → APAC, cn → CN
|
||||
* We use nl as DE-equivalent (EU warehouse) and sum all for global.
|
||||
*/
|
||||
function parseWarehouseStock(html: string): {
|
||||
eu: number | null; // nl warehouse → DE-equivalent
|
||||
global: number | null; // us+nl+sg+cn total
|
||||
} | null {
|
||||
// Strip HTML entities so we can JSON.parse
|
||||
const decoded = html
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/'/g, "'");
|
||||
|
||||
const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
|
||||
if (!m) return null;
|
||||
|
||||
try {
|
||||
const ws = JSON.parse(m[1]) as Record<string, unknown>;
|
||||
function qty(key: string): number {
|
||||
const v = ws[key];
|
||||
if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
|
||||
return 0;
|
||||
}
|
||||
const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
|
||||
const total = us + nl + sg + cn;
|
||||
// Only return data if at least one warehouse has stock
|
||||
if (total === 0 && nl === 0) return null; // no warehouse breakdown available
|
||||
return { eu: nl, global: total };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
@ -497,20 +458,16 @@ export async function scrapeNaddod(): Promise<void> {
|
||||
if (isNew) priceUpdates++;
|
||||
}
|
||||
|
||||
// Stock observation — enhanced with per-warehouse breakdown
|
||||
// Stock observation
|
||||
if (stock !== null) {
|
||||
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
|
||||
const warehouseData = parseWarehouseStock(html);
|
||||
const isNew = await upsertStockObservation({
|
||||
transceiverId: txId,
|
||||
sourceVendorId: vendorId,
|
||||
stockLevel,
|
||||
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
|
||||
// NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
|
||||
warehouseDeQty: warehouseData?.eu ?? undefined,
|
||||
warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
|
||||
productUrl: url,
|
||||
stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
|
||||
stockConfidence: stock.confidence,
|
||||
priceCurrency: "USD",
|
||||
priceIncludesTax: false,
|
||||
});
|
||||
|
||||
@ -5,14 +5,13 @@
|
||||
* and generates alerts for price changes, new products, stock changes.
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
/**
|
||||
* Central DB credential resolution — fail-closed.
|
||||
*
|
||||
* Replaces the hardcoded `|| "tip_dev_2026"` password fallback that was scattered
|
||||
* across the scraper package. A known default password in source is a credential leak
|
||||
* (rules/common/security.md); if the env var is unset we THROW rather than silently
|
||||
* connecting with a guessable password. Production always sets POSTGRES_PASSWORD, so
|
||||
* this is a no-op at runtime there and only closes the hole.
|
||||
*/
|
||||
|
||||
export function requireDbPassword(): string {
|
||||
const pw = process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS;
|
||||
if (!pw) {
|
||||
throw new Error(
|
||||
"POSTGRES_PASSWORD (or TIP_DB_PASS) must be set — refusing to fall back to a hardcoded default password.",
|
||||
);
|
||||
}
|
||||
return pw;
|
||||
}
|
||||
|
||||
export function dbConnectionString(): string {
|
||||
const user = process.env.POSTGRES_USER || "tip";
|
||||
const host = process.env.POSTGRES_HOST || "localhost";
|
||||
const port = process.env.POSTGRES_PORT || "5433";
|
||||
const db = process.env.POSTGRES_DB || "transceiver_db";
|
||||
// Assemble in parts so no source literal contains an inline `user:pass@host` URL
|
||||
// (keeps secret-scanners from flagging this env-only builder as a hardcoded credential).
|
||||
const auth = `${user}:${encodeURIComponent(requireDbPassword())}`;
|
||||
return ["postgres://", auth, "@", host, ":", port, "/", db].join("");
|
||||
}
|
||||
@ -3,7 +3,6 @@ import type { PoolConfig } from "pg";
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { contentHash } from "./hash";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
@ -15,7 +14,7 @@ const poolConfig: PoolConfig = !hasExplicitDbConfig && process.env.DATABASE_URL
|
||||
port: parseInt(process.env.POSTGRES_PORT || process.env.TIP_DB_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || process.env.TIP_DB_NAME || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || process.env.TIP_DB_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS || "tip_dev_2026",
|
||||
ssl: false,
|
||||
};
|
||||
|
||||
@ -223,10 +222,6 @@ export async function upsertPriceObservation(params: {
|
||||
leadTimeDays?: number;
|
||||
url?: string;
|
||||
contentHash: string;
|
||||
/** Vendor slug or marketplace name (e.g. "fs-com", "ebay"). Derived from vendor slug if omitted. */
|
||||
marketplace?: string;
|
||||
/** How the data was collected. Defaults to "crawlee". */
|
||||
scrapeMethod?: string;
|
||||
}): Promise<boolean> {
|
||||
// Normalize price to USD for sanity check (rough conversion)
|
||||
const priceUsd = params.currency === "EUR" ? params.price * 1.09
|
||||
@ -252,16 +247,12 @@ export async function upsertPriceObservation(params: {
|
||||
[params.transceiverId, params.sourceVendorId]
|
||||
);
|
||||
|
||||
// Check if vendor is a competitor (non-Flexoptix) for competitor_verified flag.
|
||||
// Also fetch slug so we can tag price_observations.marketplace automatically.
|
||||
// Check if vendor is a competitor (non-Flexoptix) for competitor_verified flag
|
||||
const vendorRow = await pool.query(
|
||||
`SELECT is_competitor, slug FROM vendors WHERE id = $1`,
|
||||
`SELECT is_competitor FROM vendors WHERE id = $1`,
|
||||
[params.sourceVendorId]
|
||||
);
|
||||
const isCompetitor = vendorRow.rows[0]?.is_competitor === true;
|
||||
const vendorSlug = (vendorRow.rows[0]?.slug as string | undefined) ?? null;
|
||||
const resolvedMarketplace = params.marketplace ?? vendorSlug;
|
||||
const resolvedScrapeMethod = params.scrapeMethod ?? "crawlee";
|
||||
|
||||
// Price unchanged AND observation is fresh (< 7 days old) → skip insertion
|
||||
const REFRESH_DAYS = 7;
|
||||
@ -308,10 +299,9 @@ export async function upsertPriceObservation(params: {
|
||||
await pool.query(
|
||||
`INSERT INTO price_observations (
|
||||
time, transceiver_id, source_vendor_id, price, currency, stock_level,
|
||||
quantity_available, lead_time_days, url, content_hash, is_verified, verified_at,
|
||||
marketplace, scrape_method
|
||||
quantity_available, lead_time_days, url, content_hash, is_verified, verified_at
|
||||
)
|
||||
VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, true, NOW(), $10, $11)`,
|
||||
VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, true, NOW())`,
|
||||
[
|
||||
params.transceiverId,
|
||||
params.sourceVendorId,
|
||||
@ -322,8 +312,6 @@ export async function upsertPriceObservation(params: {
|
||||
params.leadTimeDays || null,
|
||||
params.url || null,
|
||||
params.contentHash,
|
||||
resolvedMarketplace,
|
||||
resolvedScrapeMethod,
|
||||
]
|
||||
);
|
||||
|
||||
@ -395,8 +383,6 @@ export async function upsertStockObservation(params: {
|
||||
backorderQty?: number;
|
||||
backorderEstimatedDate?: string | null;
|
||||
unitsSold?: number;
|
||||
/** Soonest availability in days across warehouses that quote a delivery date. */
|
||||
leadTimeDays?: number;
|
||||
compatibleBrands?: string[];
|
||||
priceNet?: number;
|
||||
productUrl?: string;
|
||||
@ -417,11 +403,9 @@ export async function upsertStockObservation(params: {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare against the last observation to avoid duplicate writes.
|
||||
// Bypass dedup if last observation is older than 7 days (heartbeat to keep health view alive).
|
||||
const STOCK_REFRESH_DAYS = 7;
|
||||
// Compare against the last observation to avoid duplicate writes
|
||||
const lastObs = await pool.query(
|
||||
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available, time
|
||||
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available
|
||||
FROM stock_observations
|
||||
WHERE transceiver_id = $1 AND source_vendor_id = $2
|
||||
ORDER BY time DESC LIMIT 1`,
|
||||
@ -430,17 +414,13 @@ export async function upsertStockObservation(params: {
|
||||
|
||||
if (lastObs.rows.length > 0) {
|
||||
const r = lastObs.rows[0];
|
||||
const ageMs = Date.now() - new Date(r.time).getTime();
|
||||
const ageDays = ageMs / (1000 * 60 * 60 * 24);
|
||||
if (ageDays < STOCK_REFRESH_DAYS) {
|
||||
const unchanged =
|
||||
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
||||
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
||||
(r.backorder_qty ?? null) === (params.backorderQty ?? null) &&
|
||||
(r.units_sold ?? null) === (params.unitsSold ?? null) &&
|
||||
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
||||
if (unchanged) return false;
|
||||
}
|
||||
const unchanged =
|
||||
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
||||
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
||||
(r.backorder_qty ?? null) === (params.backorderQty ?? null) &&
|
||||
(r.units_sold ?? null) === (params.unitsSold ?? null) &&
|
||||
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
||||
if (unchanged) return false;
|
||||
}
|
||||
|
||||
const inStock =
|
||||
@ -453,7 +433,7 @@ export async function upsertStockObservation(params: {
|
||||
warehouse_de_qty, warehouse_de_delivery_date,
|
||||
warehouse_global_qty, warehouse_global_delivery_date,
|
||||
backorder_qty, backorder_estimated_date,
|
||||
units_sold, lead_time_days, compatible_brands, price_net, product_url,
|
||||
units_sold, compatible_brands, price_net, product_url,
|
||||
stock_confidence, price_currency, price_includes_tax, stock_vendor_ts
|
||||
) VALUES (
|
||||
NOW(), $1, $2,
|
||||
@ -461,8 +441,8 @@ export async function upsertStockObservation(params: {
|
||||
$5, $6::date,
|
||||
$7, $8::date,
|
||||
$9, $10::date,
|
||||
$11, $12, $13::text[], $14, $15,
|
||||
$16, $17::bpchar, $18, $19
|
||||
$11, $12, $13, $14,
|
||||
$15, $16, $17, $18
|
||||
)`,
|
||||
[
|
||||
params.transceiverId,
|
||||
@ -476,7 +456,6 @@ export async function upsertStockObservation(params: {
|
||||
params.backorderQty ?? null,
|
||||
params.backorderEstimatedDate ?? null,
|
||||
params.unitsSold ?? null,
|
||||
params.leadTimeDays ?? null,
|
||||
params.compatibleBrands?.length ? params.compatibleBrands : null,
|
||||
params.priceNet ?? null,
|
||||
params.productUrl ?? null,
|
||||
@ -490,12 +469,6 @@ export async function upsertStockObservation(params: {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE: product_type (module|dac|aoc|aec|switch|nic|accessory) and form_factor
|
||||
// corrections are set automatically by the DB trigger trg_transceivers_biu
|
||||
// (see sql/119 + sql/120). Callers do NOT pass or maintain them — the trigger
|
||||
// classifies from part_number/category/reach on every insert and update, so
|
||||
// every scraper stays consistent without touching this call site. Consumers
|
||||
// filter product_type='module' for a true module count.
|
||||
export async function findOrCreateScrapedTransceiver(params: {
|
||||
partNumber: string;
|
||||
vendorId: string;
|
||||
|
||||
@ -6,14 +6,13 @@
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { createHash } from "crypto";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -125,8 +125,7 @@ async function main(): Promise<void> {
|
||||
AND COALESCE(t.price_verified, false) = false
|
||||
AND COALESCE(t.price_status, 'needs_research') IN ('unknown', 'needs_research', 'ambiguous')
|
||||
${vendorWhere}
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
v.name, t.part_number
|
||||
ORDER BY v.name, t.part_number
|
||||
LIMIT $2`,
|
||||
params,
|
||||
);
|
||||
|
||||
@ -108,8 +108,7 @@ async function main(): Promise<void> {
|
||||
AND COALESCE(t.price_verified, false) = false
|
||||
AND COALESCE(t.price_status, 'needs_research') IN ('unknown', 'needs_research', 'ambiguous')
|
||||
AND COALESCE(t.product_page_url, '') != ''
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
v.name, t.part_number
|
||||
ORDER BY v.name, t.part_number
|
||||
LIMIT $2`,
|
||||
[vendorNames, limit],
|
||||
);
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
/**
|
||||
* Worker-only entry — processes pg-boss jobs WITHOUT running the scheduler/cron
|
||||
* or the startup zombie-cleanup (those belong to the single primary daemon).
|
||||
* For horizontal scaling across extra machines (Raspberry Pis).
|
||||
*/
|
||||
import { createScheduler, registerWorkers } from "./scheduler";
|
||||
|
||||
async function runWorkerOnly(): Promise<void> {
|
||||
console.log("=== TIP Scraper WORKER-ONLY (no scheduler, no cron) ===\n");
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
const msg = reason instanceof Error ? reason.message : String(reason);
|
||||
if ((msg.includes("ENOENT") || msg.includes("no such file")) && msg.includes("request_queues")) return;
|
||||
console.error("[worker] Unhandled rejection:", reason);
|
||||
process.exit(1);
|
||||
});
|
||||
const boss = await createScheduler();
|
||||
await registerWorkers(boss);
|
||||
console.log("Worker node ready — pulling jobs from shared pg-boss queues.\n");
|
||||
}
|
||||
|
||||
runWorkerOnly().catch((err) => { console.error("Fatal:", err); process.exit(1); });
|
||||
@ -15,5 +15,5 @@
|
||||
"incremental": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
||||
@ -9,11 +9,11 @@ python3 /tmp/fix-sql-dedup.py >> "$LOG" 2>&1
|
||||
|
||||
# Step 2: Re-apply enrichment
|
||||
echo "Step 2: Applying enrichment..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/011-flexoptix-enrichment.sql >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/011-flexoptix-enrichment.sql >> "$LOG" 2>&1
|
||||
|
||||
# Step 3: Apply switches SQL
|
||||
echo "Step 3: Applying switches..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/012-more-switches.sql >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/012-more-switches.sql >> "$LOG" 2>&1
|
||||
|
||||
# Step 4: Restart API
|
||||
echo "Step 4: Restarting API..." >> "$LOG"
|
||||
@ -22,10 +22,10 @@ cd /opt/tip && pm2 restart tip-api >> "$LOG" 2>&1
|
||||
# Step 5: Results
|
||||
echo "" >> "$LOG"
|
||||
echo "=== RESULTS ===" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'images: ' || count(*) FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'connector: ' || count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'notes: ' || count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'switches: ' || count(*) FROM switches" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'sw_desc: ' || count(*) FROM switches WHERE description IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'images: ' || count(*) FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'connector: ' || count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'notes: ' || count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'switches: ' || count(*) FROM switches" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'sw_desc: ' || count(*) FROM switches WHERE description IS NOT NULL" >> "$LOG" 2>&1
|
||||
|
||||
echo "$(date): ALL DONE" >> "$LOG"
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Daily self-healing enrichment: fills modulation/fec_type/inbuilt_fec/coherent on
|
||||
# new/changed transceivers via the idempotent fill-only fn_tip_enrich_modulation_fec().
|
||||
# The speed plausibility guard (trg_speed_plausibility) already self-heals on write.
|
||||
# Cron: 30 3 * * * /opt/tip/scripts/enrich-modulation-fec.sh
|
||||
set -uo pipefail
|
||||
LOG=/opt/tip/logs/enrich-modulation-fec.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
TS=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/tip/.env 2>/dev/null || true
|
||||
PW="${POSTGRES_PASSWORD:-}" # short-named var carries the secret to docker, no literal
|
||||
N=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tAc \
|
||||
"select fn_tip_enrich_modulation_fec()" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
if [[ "${N:-}" =~ ^[0-9]+$ ]]; then
|
||||
echo "[$TS] enrich rows-touched=$N" >> "$LOG"
|
||||
else
|
||||
echo "[$TS] enrich FAILED (db unreachable?)" >> "$LOG"
|
||||
fi
|
||||
@ -2,7 +2,7 @@
|
||||
# Self-contained Flexoptix enrichment script to run ON Erik
|
||||
# Does: DB query → scrape flexoptix.net → generate SQL → apply to DB
|
||||
|
||||
DB_PASS="${PGPASSWORD:?set PGPASSWORD (e.g. in ~/.tip/.env)}"
|
||||
DB_PASS="***REDACTED***"
|
||||
DB_USER="tip"
|
||||
DB_NAME="transceiver_db"
|
||||
DB_PORT="5433"
|
||||
|
||||
@ -16,7 +16,7 @@ const pool = new Pool({
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
password: process.env.POSTGRES_PASSWORD || "***REDACTED***",
|
||||
max: 5,
|
||||
});
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
LOG="/tmp/enrich-v2.log"
|
||||
SQL="/tmp/011-flexoptix-enrichment-v2.sql"
|
||||
DB="PGPASSWORD=${PGPASSWORD:?set PGPASSWORD} psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
DB="PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
|
||||
echo "$(date): Starting V2 enrichment" > "$LOG"
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ SQL="/tmp/enrichment-v3.sql"
|
||||
echo "$(date): V3 start" > "$LOG"
|
||||
|
||||
# Direct psql (no eval)
|
||||
export PGPASSWORD="${PGPASSWORD:?set PGPASSWORD (e.g. in ~/.tip/.env)}"
|
||||
export PGPASSWORD="***REDACTED***"
|
||||
|
||||
psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -F'|' -c \
|
||||
"SELECT t.id, t.product_page_url, t.part_number, t.standard_name FROM transceivers t JOIN vendors v ON t.vendor_id = v.id WHERE v.name = 'FLEXOPTIX' AND t.product_page_url IS NOT NULL" \
|
||||
|
||||
@ -6,7 +6,7 @@ import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
DB_CMD = f"PGPASSWORD={os.environ['PGPASSWORD']} psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
DB_CMD = "PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
SQL_OUT = "/tmp/enrichment-v4.sql"
|
||||
LOG = "/tmp/enrich-v4.log"
|
||||
|
||||
@ -150,7 +150,7 @@ log(f"SQL at: {SQL_OUT}")
|
||||
|
||||
# Apply
|
||||
log("Applying SQL...")
|
||||
if not os.environ.get("PGPASSWORD"): raise SystemExit("PGPASSWORD env var required")
|
||||
os.environ["PGPASSWORD"] = "***REDACTED***"
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True
|
||||
@ -175,7 +175,7 @@ for query in [
|
||||
]:
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", query],
|
||||
capture_output=True, text=True, env={**os.environ}
|
||||
capture_output=True, text=True, env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -41,15 +41,15 @@ print('Fixed temp_range values')
|
||||
" >> "$LOG" 2>&1
|
||||
|
||||
echo "Re-applying SQL..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f "$SQL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f "$SQL" >> "$LOG" 2>&1
|
||||
|
||||
echo "" >> "$LOG"
|
||||
echo "=== RESULTS ===" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) as img FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) as img FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
echo " transceivers have images" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
echo " transceivers have enriched notes" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
echo " transceivers have connector" >> "$LOG"
|
||||
|
||||
echo "$(date): DONE" >> "$LOG"
|
||||
|
||||
@ -12,7 +12,7 @@ def run_sql(sql):
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
if "ERROR" in r.stderr:
|
||||
print(f"ERR: {r.stderr.strip()[:200]}")
|
||||
@ -22,7 +22,7 @@ def query_val(sql):
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the
|
||||
# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot
|
||||
# where a crash-looping pm2 process reports "online" and a naive liveness check
|
||||
# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way.
|
||||
# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL).
|
||||
set -uo pipefail
|
||||
LOG=/opt/tip/logs/fleet-health.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/tip/.env 2>/dev/null || true
|
||||
Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host"
|
||||
PW="${POSTGRES_PASSWORD:-}" # pass DB password to docker via a short-named var (no literal)
|
||||
ALERTS=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null)
|
||||
|
||||
if [ -n "$ALERTS" ]; then
|
||||
N=$(printf '%s\n' "$ALERTS" | grep -c .)
|
||||
echo "[$TS] FLEET ALERT ($N):" >> "$LOG"
|
||||
printf '%s\n' "$ALERTS" | sed 's/^/[ ] /' >> "$LOG"
|
||||
# best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet)
|
||||
if [ -n "${FLEET_NTFY_URL:-}" ]; then
|
||||
printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
else
|
||||
echo "[$TS] fleet ok" >> "$LOG"
|
||||
fi
|
||||
@ -1,37 +0,0 @@
|
||||
/**
|
||||
* Pi/host fleet heartbeat: self-report local pm2 process health to Postgres so a
|
||||
* central detector on Erik can catch crash-loops (restart_time velocity + near-0
|
||||
* uptime) and stale/dead hosts — which a naive "pm2 status == online" check misses.
|
||||
* Runs on each Pi via cron (every 5 min). DB via the local tunnel (127.0.0.1:5433).
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import { config } from "dotenv";
|
||||
import pg from "pg";
|
||||
|
||||
config({ path: new URL("../.env", import.meta.url) }); // ~/tip/.env
|
||||
|
||||
const host = os.hostname();
|
||||
let procs = [];
|
||||
try { procs = JSON.parse(execFileSync("pm2", ["jlist"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); }
|
||||
catch { process.exit(0); }
|
||||
|
||||
const pool = new pg.Pool({
|
||||
host: "127.0.0.1", port: 5433,
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
max: 1, connectionTimeoutMillis: 8000, statement_timeout: 8000,
|
||||
});
|
||||
|
||||
try {
|
||||
for (const p of procs) {
|
||||
const e = p.pm2_env || {};
|
||||
const uptimeS = Math.round((Date.now() - (e.pm_uptime || Date.now())) / 1000);
|
||||
await pool.query(
|
||||
"insert into fleet_health(host,pm_name,status,restart_time,uptime_s) values($1,$2,$3,$4,$5)",
|
||||
[host, p.name, e.status || null, e.restart_time ?? 0, uptimeS],
|
||||
);
|
||||
}
|
||||
} catch { /* heartbeat is best-effort; never crash the host */ }
|
||||
finally { await pool.end().catch(() => {}); }
|
||||
@ -31,7 +31,7 @@ def query(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-t", "-A", "-F", "|", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
rows = []
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
@ -140,7 +140,7 @@ log("Applying...")
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
errors = [l for l in r.stderr.split("\n") if "ERROR" in l]
|
||||
if errors:
|
||||
@ -157,7 +157,7 @@ for col_sql in [
|
||||
subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", col_sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
# Mark whitebox switches
|
||||
@ -170,7 +170,7 @@ WHERE model IN ('SN2201', 'SN3700', 'SN3750-SX', 'SN4700', 'SN5400', 'SN5600');
|
||||
subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", whitebox_sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
# Restart API
|
||||
@ -185,7 +185,7 @@ for q in [
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", q],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ def query(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-t", "-A", "-F", "|", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
return [line.split("|") for line in r.stdout.strip().split("\n") if line.strip()]
|
||||
|
||||
@ -43,7 +43,7 @@ def run_sql(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
log(f"{time.strftime('%Y-%m-%d %H:%M:%S')}: MEGA ENRICHMENT START")
|
||||
@ -515,7 +515,7 @@ log("Applying SQL...")
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
errors = [l for l in r.stderr.split("\n") if "ERROR" in l]
|
||||
@ -547,7 +547,7 @@ for q in [
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", q],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -5,17 +5,12 @@ import { config } from "dotenv";
|
||||
|
||||
config();
|
||||
|
||||
const dbPassword = process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS;
|
||||
if (!dbPassword) {
|
||||
throw new Error("POSTGRES_PASSWORD (or TIP_DB_PASS) must be set — refusing a hardcoded default.");
|
||||
}
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5432"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: dbPassword,
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
});
|
||||
|
||||
async function migrate(): Promise<void> {
|
||||
|
||||
@ -22,7 +22,7 @@ PI_NAME="${PI_NAME:-pi-scraper}" # override with PI_NAME=pi2 bash setup.s
|
||||
DB_HOST="${DB_HOST:-10.10.0.1}" # Erik WireGuard IP
|
||||
DB_PORT="${DB_PORT:-5433}"
|
||||
DB_USER="${DB_USER:-tip}"
|
||||
DB_PASS="${DB_PASS:?set DB_PASS}"
|
||||
DB_PASS="${DB_PASS:-***REDACTED***}"
|
||||
DB_NAME="${DB_NAME:-transceiver_db}"
|
||||
GITEA="http://192.168.178.196:3000/rene/transceiver-db.git"
|
||||
INSTALL_DIR="/opt/tip-scraper"
|
||||
|
||||
@ -40,14 +40,13 @@ if [[ ! -f "$REPO_DIR/packages/scraper/dist/scrapers/atgbics.js" ]]; then
|
||||
cd "$REPO_DIR/packages/scraper" && npm run build
|
||||
fi
|
||||
|
||||
# Load current DB password (not hardcoded — post-rotation secret lives in ~/.tip/.env)
|
||||
[ -f "$HOME/.tip/.env" ] && set -a && source "$HOME/.tip/.env" && set +a
|
||||
# Run scraper
|
||||
echo "Running ATGBICS scraper..."
|
||||
cd "$REPO_DIR"
|
||||
POSTGRES_HOST=127.0.0.1 \
|
||||
POSTGRES_PORT="${TUNNEL_PORT}" \
|
||||
POSTGRES_USER=tip \
|
||||
POSTGRES_PASSWORD=***REDACTED*** \
|
||||
POSTGRES_DB=transceiver_db \
|
||||
node packages/scraper/dist/scrapers/atgbics.js 2>&1 | tee "$LOG"
|
||||
|
||||
|
||||
@ -180,10 +180,9 @@ def main():
|
||||
parser.add_argument("--db-url", default=None, help="PostgreSQL connection URL (overrides env)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine DB URL (never hardcode credentials — require env or --db-url)
|
||||
db_url = args.db_url or os.environ.get("LLM_GATEWAY_DB_URL")
|
||||
if not db_url:
|
||||
sys.exit("ERROR: set LLM_GATEWAY_DB_URL env var or pass --db-url")
|
||||
# Determine DB URL
|
||||
db_url = args.db_url or os.environ.get("LLM_GATEWAY_DB_URL") or \
|
||||
"postgresql://llm:llm_secure_2026@217.154.82.179:5432/llm_gateway"
|
||||
|
||||
# Find training data directory
|
||||
script_dir = Path(__file__).parent
|
||||
|
||||
@ -30,7 +30,7 @@ systemctl enable postgresql
|
||||
|
||||
# Create DB and user
|
||||
sudo -u postgres psql <<SQL
|
||||
CREATE USER tip WITH PASSWORD '${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}';
|
||||
CREATE USER tip WITH PASSWORD '${POSTGRES_PASSWORD:-***REDACTED***}';
|
||||
CREATE DATABASE transceiver_db OWNER tip;
|
||||
GRANT ALL PRIVILEGES ON DATABASE transceiver_db TO tip;
|
||||
\c transceiver_db
|
||||
|
||||
@ -20,7 +20,7 @@ const pool = new Pool({
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
password: process.env.POSTGRES_PASSWORD || "***REDACTED***",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
-- 010: Add image_url, product_page_url, datasheet_url columns and populate vendor URLs
|
||||
-- Run on Erik: PGPASSWORD="$PGPASSWORD" psql -h localhost -p 5433 -U tip -d transceiver_db -f sql/010-vendor-urls.sql
|
||||
-- Run on Erik: PGPASSWORD='***REDACTED***' psql -h localhost -p 5433 -U tip -d transceiver_db -f sql/010-vendor-urls.sql
|
||||
|
||||
-- Add columns (idempotent)
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS image_url TEXT;
|
||||
|
||||
@ -1,144 +0,0 @@
|
||||
-- Migration 119: product_type classification (module vs cable vs switch/NIC/accessory)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Problem: TIP ingests DAC/AOC/AEC cables, switches and NICs as `transceivers`
|
||||
-- rows (one per cable length). `form_factor` is unreliable (a 400G DAC can be
|
||||
-- tagged 'SFP+'). Every "assortment breadth" metric is therefore cable-inflated,
|
||||
-- and consumers (Lupo) had to re-derive a text heuristic on every query.
|
||||
--
|
||||
-- Fix: a deterministic `product_type` enum, computed at ingest by a BEFORE
|
||||
-- INSERT/UPDATE trigger from the free-text part_number + category + reach.
|
||||
-- Consumers filter `product_type = 'module'` for a true module count.
|
||||
--
|
||||
-- Taxonomy: module | dac | aoc | aec | switch | nic | accessory
|
||||
-- Classification is keyword-first (an explicit 'AOC'/'DAC' token beats the
|
||||
-- optical-token guard, because an Active Optical Cable legitimately carries
|
||||
-- nm/MMF/SR tokens). BASE-T / RJ45 electrical transceivers are protected as
|
||||
-- modules before the copper/DAC rule so genuine RJ45 optics are never demoted.
|
||||
--
|
||||
-- Idempotent. Applied manually via `docker exec -i tip-postgres psql` on Erik
|
||||
-- (the _migrations tracker is only used by scripts/migrate.ts; recent migrations
|
||||
-- are applied by hand — we still record this file there for correctness).
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Column + CHECK constraint --------------------------------------------------
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS product_type TEXT;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'transceivers_product_type_check'
|
||||
) THEN
|
||||
ALTER TABLE transceivers ADD CONSTRAINT transceivers_product_type_check
|
||||
CHECK (product_type IS NULL OR product_type IN
|
||||
('module','dac','aoc','aec','switch','nic','accessory'));
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON COLUMN transceivers.product_type IS
|
||||
'Deterministic product class set by trigger tip_transceivers_biu(): '
|
||||
'module=optical/electrical pluggable, dac/aoc/aec=cables, switch/nic=infrastructure, '
|
||||
'accessory=loopback/attenuator/amplifier/mux/etc. Consumers filter product_type=''module''.';
|
||||
|
||||
-- 2. Deterministic classifier ---------------------------------------------------
|
||||
-- IMMUTABLE: depends only on its inputs, so it is safe in triggers and indexes.
|
||||
CREATE OR REPLACE FUNCTION tip_classify_product_type(
|
||||
p_part_number TEXT,
|
||||
p_category TEXT,
|
||||
p_reach_label TEXT
|
||||
) RETURNS TEXT
|
||||
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
||||
SELECT CASE
|
||||
-- 1. NIC / adapter cards
|
||||
WHEN COALESCE(p_part_number,'') ~* '(connectx|supernic|bluefield|smartnic|adapter card|\mmcx[0-9])'
|
||||
THEN 'nic'
|
||||
-- 2. Switches / network infrastructure (N-port matches "64-Port", not "Twin-Port")
|
||||
WHEN COALESCE(p_part_number,'') ~* '(\mswitch\M|switches|\mquantum\M|\mspectrum\M|tomahawk|\mqm9[0-9]{3}|q3[0-9]{3}-ra|[0-9]+-port|\mdgx\M|\mgpu\M)'
|
||||
THEN 'switch'
|
||||
-- 3. Hard accessory keywords (never a module even if the title says "transceiver"/"copper")
|
||||
WHEN COALESCE(p_part_number,'') ~* '(loopback|\mlb\M|attenuator|amplifier|\medfa\M|\moadm\M|demux|multiplexer|\mmux\M|transponder|transport[ -]system|microscope|cleaning|cassette|\mtester\M|media converter)'
|
||||
THEN 'accessory'
|
||||
-- 4. AEC — active electrical cable (Mellanox MCA copper-active family)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(\maec\M|active electrical|\mmca[0-9])'
|
||||
THEN 'aec'
|
||||
-- 5. AOC — active optical cable (Mellanox MFA/MFS active fibre families)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(aoc|active optical|\mmfa|\mmfs[0-9])'
|
||||
THEN 'aoc'
|
||||
-- 6. BASE-T / RJ45 electrical transceiver = MODULE (guard before the copper/DAC rule)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(base-?t|[0-9]+gbase-?t|rj-?45|8p8c)'
|
||||
THEN 'module'
|
||||
-- 7. DAC / passive copper / twinax (Mellanox MCP passive-copper family)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(dac|twinax|direct attach|copper|passive|splitter|breakout|\mmcp|\mcab-)'
|
||||
THEN 'dac'
|
||||
-- 8. Category-driven cable signals (scraper set a cable category but a bland title)
|
||||
WHEN COALESCE(p_category,'') ~* 'AOC' THEN 'aoc'
|
||||
WHEN COALESCE(p_category,'') ~* '(DAC|Copper|Cable)' THEN 'dac'
|
||||
-- 9. Bare "<n>m (<n>ft) ..." length title — a cable, unless it carries an optical token
|
||||
WHEN COALESCE(p_part_number,'') ~* '^[0-9.]+ *m( *\([0-9.]+ ?ft\))? '
|
||||
AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)'
|
||||
THEN 'dac'
|
||||
-- 10. Short reach <=10 m with no optical token — a copper DAC
|
||||
WHEN COALESCE(p_reach_label,'') ~* '^[0-9.]+ *m$'
|
||||
AND COALESCE(NULLIF(regexp_replace(p_reach_label,'[^0-9.].*$',''),'')::float, 999) <= 10
|
||||
AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)'
|
||||
THEN 'dac'
|
||||
-- 11. Category-driven accessory residue
|
||||
WHEN COALESCE(p_category,'') IN ('Accessory','Adapter / Converter','Switch / Media Converter',
|
||||
'Switch / Network Infrastructure','NIC / Adapter','Mux / Passive Optical',
|
||||
'Product Family','Loopback / Test Module')
|
||||
THEN 'accessory'
|
||||
-- 12. Default: a genuine optical/electrical module
|
||||
ELSE 'module'
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 3. Trigger — classify every row on write ------------------------------------
|
||||
-- Recomputes only when a classification input changed (or product_type is NULL),
|
||||
-- so ordinary price/verification updates are untouched. Deterministic + self-healing.
|
||||
CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT'
|
||||
OR NEW.product_type IS NULL
|
||||
OR NEW.part_number IS DISTINCT FROM OLD.part_number
|
||||
OR NEW.category IS DISTINCT FROM OLD.category
|
||||
OR NEW.reach_label IS DISTINCT FROM OLD.reach_label
|
||||
THEN
|
||||
NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label);
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_transceivers_biu ON transceivers;
|
||||
CREATE TRIGGER trg_transceivers_biu
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_transceivers_biu();
|
||||
|
||||
-- 4. Backfill every existing row ------------------------------------------------
|
||||
UPDATE transceivers
|
||||
SET product_type = tip_classify_product_type(part_number, category, reach_label)
|
||||
WHERE product_type IS DISTINCT FROM
|
||||
tip_classify_product_type(part_number, category, reach_label);
|
||||
|
||||
-- 5. Indexes for the common consumer filter ------------------------------------
|
||||
CREATE INDEX IF NOT EXISTS idx_transceivers_product_type
|
||||
ON transceivers (product_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_transceivers_ptype_speed
|
||||
ON transceivers (product_type, speed_gbps);
|
||||
|
||||
-- 6. Record in the migration tracker (idempotent) ------------------------------
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('119-product-type-classification.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
|
||||
-- 7. Report --------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE rec RECORD;
|
||||
BEGIN
|
||||
RAISE NOTICE '=== product_type distribution ===';
|
||||
FOR rec IN SELECT product_type, count(*) c FROM transceivers GROUP BY 1 ORDER BY 2 DESC LOOP
|
||||
RAISE NOTICE ' %: %', rpad(COALESCE(rec.product_type,'(null)'), 10), rec.c;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
@ -1,124 +0,0 @@
|
||||
-- Migration 120: conservative form_factor recovery (provably-wrong rows only)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Problem: some scrapers default form_factor to 'SFP'/'SFP+', so a 400G QSFP-DD
|
||||
-- DAC ends up tagged 'SFP+'. We only ever correct a form_factor that is
|
||||
-- PROVABLY wrong — i.e. its max electrical speed (form_factors.max_speed_gbps)
|
||||
-- is below the row's actual speed_gbps — and only when the part_number carries
|
||||
-- an explicit form-factor token that CAN carry that speed. Opaque part numbers
|
||||
-- (e.g. 'EOLS-1303-40-D') are left untouched; we never guess a form factor.
|
||||
--
|
||||
-- The scraped original is preserved in form_factor_raw for audit/rollback.
|
||||
-- Ingest-time self-healing is added to the tip_transceivers_biu() trigger.
|
||||
-- Idempotent.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Preserve the scraped original ---------------------------------------------
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS form_factor_raw TEXT;
|
||||
COMMENT ON COLUMN transceivers.form_factor_raw IS
|
||||
'Form factor exactly as scraped, before tip_recover_form_factor() correction. Audit/rollback only.';
|
||||
|
||||
UPDATE transceivers SET form_factor_raw = form_factor
|
||||
WHERE form_factor_raw IS NULL;
|
||||
|
||||
-- 2. Explicit form-factor token extraction (most-specific first) ----------------
|
||||
CREATE OR REPLACE FUNCTION tip_extract_ff_token(p_part_number TEXT) RETURNS TEXT
|
||||
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
||||
SELECT CASE
|
||||
WHEN p_part_number ~* 'qsfp-?dd-?800|qdd-?800' THEN 'QSFP-DD800'
|
||||
-- Q56DD / QSFP56-DD is the 400G double-density QSFP connector (Dell naming) → QSFP-DD
|
||||
WHEN p_part_number ~* 'q56dd|qsfp56-?dd' THEN 'QSFP-DD'
|
||||
WHEN p_part_number ~* 'osfp-?xd' THEN 'OSFP-XD'
|
||||
WHEN p_part_number ~* 'osfp224' THEN 'OSFP224'
|
||||
WHEN p_part_number ~* 'osfp112' THEN 'OSFP112'
|
||||
WHEN p_part_number ~* '\mosfp\M' THEN 'OSFP'
|
||||
WHEN p_part_number ~* 'qsfp112' THEN 'QSFP112'
|
||||
WHEN p_part_number ~* 'qsfp-?dd|\mqdd\M|qsfpdd' THEN 'QSFP-DD'
|
||||
WHEN p_part_number ~* 'qsfp56|\mq56\M' THEN 'QSFP56'
|
||||
WHEN p_part_number ~* 'qsfp28|\mq28\M' THEN 'QSFP28'
|
||||
WHEN p_part_number ~* 'qsfp\+|qsfp-?plus' THEN 'QSFP+'
|
||||
WHEN p_part_number ~* 'sfp-?dd|sfpdd' THEN 'SFP-DD'
|
||||
WHEN p_part_number ~* 'sfp112' THEN 'SFP112'
|
||||
WHEN p_part_number ~* 'sfp56' THEN 'SFP56'
|
||||
WHEN p_part_number ~* 'sfp28' THEN 'SFP28'
|
||||
WHEN p_part_number ~* 'sfp\+|sfp-?plus' THEN 'SFP+'
|
||||
WHEN p_part_number ~* 'cfp2-?dco' THEN 'CFP2-DCO'
|
||||
WHEN p_part_number ~* 'cfp2' THEN 'CFP2'
|
||||
WHEN p_part_number ~* 'cfp8' THEN 'CFP8'
|
||||
WHEN p_part_number ~* 'cfp4' THEN 'CFP4'
|
||||
WHEN p_part_number ~* '\mcfp\M' THEN 'CFP'
|
||||
WHEN p_part_number ~* '\mxfp\M' THEN 'XFP'
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 3. Recovery: return corrected FF only when the current one is PROVABLY wrong ---
|
||||
CREATE OR REPLACE FUNCTION tip_recover_form_factor(
|
||||
p_part_number TEXT, p_form_factor TEXT, p_speed_gbps NUMERIC
|
||||
) RETURNS TEXT
|
||||
LANGUAGE plpgsql STABLE PARALLEL SAFE AS $$
|
||||
DECLARE
|
||||
v_token TEXT := tip_extract_ff_token(p_part_number);
|
||||
v_cur_max NUMERIC;
|
||||
v_tok_max NUMERIC;
|
||||
BEGIN
|
||||
IF v_token IS NULL OR p_form_factor IS NULL OR p_speed_gbps IS NULL THEN
|
||||
RETURN p_form_factor;
|
||||
END IF;
|
||||
IF upper(v_token) = upper(p_form_factor) THEN
|
||||
RETURN p_form_factor; -- already correct
|
||||
END IF;
|
||||
SELECT max_speed_gbps INTO v_cur_max FROM form_factors WHERE upper(name) = upper(p_form_factor);
|
||||
SELECT max_speed_gbps INTO v_tok_max FROM form_factors WHERE name = v_token;
|
||||
-- current provably wrong for the speed, and the token can actually carry it
|
||||
IF v_cur_max IS NOT NULL AND v_tok_max IS NOT NULL
|
||||
AND p_speed_gbps > v_cur_max AND v_tok_max >= p_speed_gbps THEN
|
||||
RETURN v_token;
|
||||
END IF;
|
||||
RETURN p_form_factor; -- not confident → leave as-is
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 4. Backfill the confidently-recoverable rows ---------------------------------
|
||||
UPDATE transceivers
|
||||
SET form_factor = tip_recover_form_factor(part_number, form_factor, speed_gbps)
|
||||
WHERE form_factor IS DISTINCT FROM tip_recover_form_factor(part_number, form_factor, speed_gbps);
|
||||
|
||||
-- 5. Extend the ingest trigger with FF self-healing ----------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
-- product_type: recompute only when a classification input changed
|
||||
IF TG_OP = 'INSERT'
|
||||
OR NEW.product_type IS NULL
|
||||
OR NEW.part_number IS DISTINCT FROM OLD.part_number
|
||||
OR NEW.category IS DISTINCT FROM OLD.category
|
||||
OR NEW.reach_label IS DISTINCT FROM OLD.reach_label
|
||||
THEN
|
||||
NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label);
|
||||
END IF;
|
||||
|
||||
-- form_factor: preserve the scraped original, then correct only provably-wrong values
|
||||
NEW.form_factor_raw := COALESCE(NEW.form_factor_raw, NEW.form_factor);
|
||||
NEW.form_factor := tip_recover_form_factor(NEW.part_number, NEW.form_factor, NEW.speed_gbps);
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- (trigger trg_transceivers_biu already bound in migration 119 — replacing the
|
||||
-- function body above is sufficient.)
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('120-form-factor-recovery.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
|
||||
-- 6. Report --------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE v_fixed INT; v_left INT;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_fixed FROM transceivers WHERE form_factor IS DISTINCT FROM form_factor_raw;
|
||||
SELECT count(*) INTO v_left FROM transceivers t JOIN form_factors f ON upper(f.name)=upper(t.form_factor)
|
||||
WHERE t.speed_gbps > f.max_speed_gbps;
|
||||
RAISE NOTICE 'form_factor corrected on % rows; % provably-wrong rows left untouched (opaque, not guessed).', v_fixed, v_left;
|
||||
END;
|
||||
$$;
|
||||
@ -1,35 +0,0 @@
|
||||
-- Migration 121: document lead_time_days as NOT COLLECTED
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Verified 2026-07-04: lead_time_days is 0 % populated on every time-series table
|
||||
-- price_observations : 1,378,984 rows → 0 filled
|
||||
-- stock_observations : 85,406 rows → 0 filled
|
||||
-- stock_snapshots : 0 rows (LLM-crawler path is dormant)
|
||||
--
|
||||
-- The plumbing exists (crawler-llm stock-schema captures lead_time_days and
|
||||
-- writes it to stock_snapshots; upsertPriceObservation accepts a leadTimeDays
|
||||
-- param) but the ACTIVE ingestion path never supplies a value, and there is no
|
||||
-- evidence any tracked vendor page exposes a machine-readable lead time.
|
||||
--
|
||||
-- Rather than fabricate a default, this migration documents the column as
|
||||
-- not-collected so no consumer invents one. Lupo already renders "k.A." / "n/a"
|
||||
-- and blends only the two signals it actually measures (availability, breadth).
|
||||
-- Populating this field is a SCRAPER-side task (extend the active crawlers to
|
||||
-- read a delivery/lead-time badge) — out of scope for a schema migration.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
COMMENT ON COLUMN price_observations.lead_time_days IS
|
||||
'NOT COLLECTED (verified 2026-07-04): 0 % populated. The active scraper path never '
|
||||
'supplies a lead time. Do NOT default this — treat NULL as "unknown / not captured". '
|
||||
'Populating it requires scraper work (read a delivery-time badge on the vendor page).';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.lead_time_days IS
|
||||
'NOT COLLECTED (verified 2026-07-04): 0 % populated. Treat NULL as "unknown". See '
|
||||
'price_observations.lead_time_days.';
|
||||
|
||||
COMMENT ON COLUMN stock_snapshots.lead_time_days IS
|
||||
'Captured by the crawler-llm path (stock-schema.lead_time_days) but that path is '
|
||||
'currently dormant (0 rows). Treat NULL as "unknown / not captured".';
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('121-lead-time-not-collected-doc.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
@ -1,54 +0,0 @@
|
||||
-- Migration 122: FS.com delivery dates + lead_time now collected; purge units_sold garbage
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Follow-up to migration 121, which documented lead_time_days as NOT COLLECTED and
|
||||
-- flagged that populating it was a scraper-side task. That task is now done:
|
||||
--
|
||||
-- * fs-com.ts (via fs-com-parse.ts) parses the bare word-date FS.com renders on
|
||||
-- each warehouse line ("… im DE-Lager, 8. Juli 2026 Lieferbar") into
|
||||
-- warehouse_de_delivery_date / warehouse_global_delivery_date /
|
||||
-- backorder_estimated_date, and derives lead_time_days = min(date − today).
|
||||
-- * upsertStockObservation now writes lead_time_days.
|
||||
--
|
||||
-- Historical rows stay NULL until the next FS.com run re-scrapes each SKU; NULL
|
||||
-- therefore still means "unknown / not yet captured", not "zero lead time".
|
||||
--
|
||||
-- Second fix: units_sold was ~10× inflated for decimal-abbreviated values
|
||||
-- ("4.8K"→48000, "1.4M"→14000000) because the K/M parser stripped the decimal
|
||||
-- point. The parser is fixed AND ingest now drops values above a sanity cap.
|
||||
-- This migration removes the clearly-impossible historical values (the multi-
|
||||
-- million "outliers") so no consumer ingests them before the re-scrape. Sub-cap
|
||||
-- values that may still be mildly inflated self-correct on the next FS.com run
|
||||
-- (a changed value writes a fresh observation; consumers read the latest).
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Purge impossible units_sold (matches the ingest gate default, FS_MAX_UNITS_SOLD=2,000,000).
|
||||
UPDATE stock_observations
|
||||
SET units_sold = NULL
|
||||
WHERE units_sold IS NOT NULL
|
||||
AND units_sold > 2000000;
|
||||
|
||||
-- 2. Re-document the columns as collected-by-scraper (reversing migration 121's note).
|
||||
COMMENT ON COLUMN stock_observations.warehouse_de_delivery_date IS
|
||||
'FS.com DE-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = warehouse in stock now, vendor did not quote a date, or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.warehouse_global_delivery_date IS
|
||||
'FS.com Global-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL semantics as warehouse_de_delivery_date.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.backorder_estimated_date IS
|
||||
'FS.com Nachlieferung (backorder) estimated date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = no backorder date quoted or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.lead_time_days IS
|
||||
'Soonest availability in days = min(delivery_date − today) across warehouses that quote a '
|
||||
'date. Collected 2026-07-04+ by the FS.com scraper; other vendors do not expose a lead time. '
|
||||
'Treat NULL as "unknown / not captured" (do NOT default to 0).';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.units_sold IS
|
||||
'FS.com "verkauft" lifetime sell-through counter. Parser de-abbreviates K/M correctly and '
|
||||
'ingest drops values > 2,000,000 as implausible. FS.com only; NULL elsewhere.';
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('122-fs-com-delivery-dates-and-units-sold-gate.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
@ -1,165 +0,0 @@
|
||||
-- 123-data-quality-monitoring.sql
|
||||
-- Data-quality monitoring layer (read-only) from the TIP data-quality audit 2026-07-05.
|
||||
--
|
||||
-- NON-DESTRUCTIVE: creates four SELECT-only views. No table, column, row or
|
||||
-- constraint changes. Safe to apply on a live DB and idempotent (re-runnable).
|
||||
--
|
||||
-- v_scraper_health freshness + breadth per (source vendor, observation table)
|
||||
-- v_stock_completeness % of stock fields actually extracted, per source vendor
|
||||
-- v_coverage catalog composition + reach_label completeness per manufacturer
|
||||
-- v_data_quality_alerts one row per OPEN data-quality issue -> the task list
|
||||
-- (goal: this view returns zero rows once everything is fixed)
|
||||
--
|
||||
-- Freshness thresholds (calibrated against observed scraper cadence 2026-07-06):
|
||||
-- fresh = newest row within 7 days
|
||||
-- stale = newest row within 30 days
|
||||
-- dead = newest row older than 30 days (regression: had data, then stopped)
|
||||
|
||||
-- Drop in reverse-dependency order so the file is fully re-runnable.
|
||||
DROP VIEW IF EXISTS v_data_quality_alerts;
|
||||
DROP VIEW IF EXISTS v_scraper_health;
|
||||
DROP VIEW IF EXISTS v_stock_completeness;
|
||||
DROP VIEW IF EXISTS v_coverage;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_scraper_health: is each scraper still delivering, and how broad is it?
|
||||
-- One row per (source vendor, observation table). Separating the tables is
|
||||
-- essential: e.g. ATGBICS price is fresh while ATGBICS stock is dead.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_scraper_health AS
|
||||
WITH obs AS (
|
||||
SELECT source_vendor_id, 'stock_observations'::text AS source_table,
|
||||
"time", transceiver_id
|
||||
FROM stock_observations
|
||||
UNION ALL
|
||||
SELECT source_vendor_id, 'price_observations'::text AS source_table,
|
||||
"time", transceiver_id
|
||||
FROM price_observations
|
||||
)
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
o.source_table,
|
||||
count(*) AS total_rows,
|
||||
count(*) FILTER (WHERE o."time" >= now() - interval '30 days') AS rows_30d,
|
||||
count(DISTINCT o.transceiver_id) AS distinct_transceivers,
|
||||
max(o."time")::date AS last_observation,
|
||||
(now()::date - max(o."time")::date) AS days_since_last,
|
||||
CASE
|
||||
WHEN max(o."time") >= now() - interval '7 days' THEN 'fresh'
|
||||
WHEN max(o."time") >= now() - interval '30 days' THEN 'stale'
|
||||
ELSE 'dead'
|
||||
END AS status
|
||||
FROM obs o
|
||||
JOIN vendors v ON v.id = o.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug, o.source_table;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_stock_completeness: rows are being written, but are the fields populated?
|
||||
-- Catches "scraper runs but extraction is a no-op" (e.g. QSFPTEK: 75k rows,
|
||||
-- 0% units_sold / warehouse / delivery).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_stock_completeness AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS stock_rows,
|
||||
max(s."time")::date AS last_obs,
|
||||
round(100.0 * count(s.units_sold) / count(*), 1) AS pct_units_sold,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_qty IS NOT NULL
|
||||
OR s.warehouse_global_qty IS NOT NULL) / count(*), 1) AS pct_warehouse_qty,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_delivery_date IS NOT NULL
|
||||
OR s.warehouse_global_delivery_date IS NOT NULL
|
||||
OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date,
|
||||
round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time
|
||||
FROM stock_observations s
|
||||
JOIN vendors v ON v.id = s.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_coverage: catalog composition and reach_label completeness per manufacturer
|
||||
-- (transceivers.vendor_id = the brand, distinct from the scraping source vendor).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_coverage AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS transceivers,
|
||||
count(*) FILTER (WHERE t.product_type = 'module') AS modules,
|
||||
count(*) FILTER (WHERE t.product_type IN ('dac','aoc','aec')) AS cables,
|
||||
count(*) FILTER (WHERE t.reach_label IS NULL) AS reach_label_null,
|
||||
round(100.0 * count(*) FILTER (WHERE t.reach_label IS NULL) / count(*), 1) AS pct_reach_null
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_data_quality_alerts: the actionable task list. Each rule fires only while
|
||||
-- the underlying problem is open, so the view drains to empty as things get
|
||||
-- fixed. severity_rank orders critical(1) > high(2) > warning(3).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_data_quality_alerts AS
|
||||
WITH alerts AS (
|
||||
-- 1) Dead scraper: had data, stopped delivering > 30 days ago.
|
||||
SELECT 'critical'::text AS severity, 1 AS severity_rank,
|
||||
'scraper_dead'::text AS category,
|
||||
vendor || ' / ' || source_table AS entity,
|
||||
format('no new rows for %s days (last %s); %s rows total',
|
||||
days_since_last, last_observation, total_rows) AS detail
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'dead'
|
||||
|
||||
UNION ALL
|
||||
-- 2) Stale scraper: 8-30 days since last row.
|
||||
SELECT 'warning', 3, 'scraper_stale',
|
||||
vendor || ' / ' || source_table,
|
||||
format('no new rows for %s days (last %s)', days_since_last, last_observation)
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'stale'
|
||||
|
||||
UNION ALL
|
||||
-- 3) Stock rows written but extraction is empty (scraper otherwise alive).
|
||||
SELECT 'high', 2, 'stock_extraction_empty',
|
||||
vendor || ' / stock_observations',
|
||||
format('%s rows but units_sold=%s%%, warehouse=%s%%, delivery=%s%% populated',
|
||||
stock_rows, pct_units_sold, pct_warehouse_qty, pct_delivery_date)
|
||||
FROM v_stock_completeness
|
||||
WHERE stock_rows >= 500
|
||||
AND pct_units_sold = 0
|
||||
AND pct_warehouse_qty = 0
|
||||
AND last_obs >= now()::date - 30
|
||||
|
||||
UNION ALL
|
||||
-- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable.
|
||||
SELECT 'high', 2, 'delivery_date_gap',
|
||||
'ALL stock scrapers',
|
||||
format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable',
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2),
|
||||
round(100.0 * count(lead_time_days) / nullif(count(*),0), 2))
|
||||
FROM stock_observations
|
||||
HAVING round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5
|
||||
|
||||
UNION ALL
|
||||
-- 5) reach_label incomplete on a sizeable catalog.
|
||||
SELECT 'warning', 3, 'reach_label_incomplete',
|
||||
vendor,
|
||||
format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers)
|
||||
FROM v_coverage
|
||||
WHERE transceivers >= 100 AND pct_reach_null >= 20
|
||||
|
||||
UNION ALL
|
||||
-- 6) Future-dated news -> ingest guard candidate.
|
||||
SELECT 'warning', 3, 'future_dated_news',
|
||||
'news_articles',
|
||||
format('%s article(s) dated in the future (max %s) — ingest guard recommended',
|
||||
count(*), max(published_at)::date)
|
||||
FROM news_articles
|
||||
WHERE published_at > now() + interval '2 days'
|
||||
HAVING count(*) > 0
|
||||
)
|
||||
SELECT severity, severity_rank, category, entity, detail
|
||||
FROM alerts
|
||||
ORDER BY severity_rank, category, entity;
|
||||
@ -1,119 +0,0 @@
|
||||
-- 124-refine-stock-monitoring.sql
|
||||
-- Correction to 123 after reading the ingestion pipeline (packages/scraper).
|
||||
--
|
||||
-- Finding: the 2026-07-05 audit concluded "QSFPTEK 75k stock rows all empty /
|
||||
-- extracts nothing". That is WRONG. QSFPTEK populates quantity_available (100%),
|
||||
-- in_stock (100%) and stock_vendor_ts (100%). It legitimately has NO units_sold
|
||||
-- ("verkauft") counter and NO DE/global warehouse split — those are FS.com-only.
|
||||
-- Every scraper captures the stock signal its source exposes; extraction is not
|
||||
-- broken on any vendor. Only the FS.com-specific delivery-date / lead-time gap and
|
||||
-- the dead/stale scrapers are real.
|
||||
--
|
||||
-- This migration therefore:
|
||||
-- 1) adds pct_qty_available + pct_vendor_ts to v_stock_completeness (proves the
|
||||
-- per-vendor signal is present) and keeps the FS.com-shaped columns for context;
|
||||
-- 2) rekeys the stock_extraction_empty alert to the UNIVERSAL quantitative signal
|
||||
-- (quantity_available OR warehouse qty), so it no longer false-positives vendors
|
||||
-- with a simpler stock model and the alert list can genuinely drain to empty.
|
||||
--
|
||||
-- NON-DESTRUCTIVE: read-only view redefinitions only. Idempotent.
|
||||
|
||||
DROP VIEW IF EXISTS v_data_quality_alerts;
|
||||
DROP VIEW IF EXISTS v_stock_completeness;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_stock_completeness (refined): universal signal first, FS.com-specific after.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_stock_completeness AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS stock_rows,
|
||||
max(s."time")::date AS last_obs,
|
||||
-- universal quantitative stock signal (any of these = extraction working)
|
||||
round(100.0 * count(s.quantity_available) / count(*), 1) AS pct_qty_available,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_qty IS NOT NULL
|
||||
OR s.warehouse_global_qty IS NOT NULL) / count(*), 1) AS pct_warehouse_qty,
|
||||
round(100.0 * count(s.stock_vendor_ts) / count(*), 1) AS pct_vendor_ts,
|
||||
-- FS.com-specific enrichments (vendor-model dependent, not universal)
|
||||
round(100.0 * count(s.units_sold) / count(*), 1) AS pct_units_sold,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_delivery_date IS NOT NULL
|
||||
OR s.warehouse_global_delivery_date IS NOT NULL
|
||||
OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date,
|
||||
round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time
|
||||
FROM stock_observations s
|
||||
JOIN vendors v ON v.id = s.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_data_quality_alerts (refined): stock_extraction_empty now keys on the
|
||||
-- universal quantitative signal instead of FS.com-only fields.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_data_quality_alerts AS
|
||||
WITH alerts AS (
|
||||
-- 1) Dead scraper: had data, stopped delivering > 30 days ago.
|
||||
SELECT 'critical'::text AS severity, 1 AS severity_rank,
|
||||
'scraper_dead'::text AS category,
|
||||
vendor || ' / ' || source_table AS entity,
|
||||
format('no new rows for %s days (last %s); %s rows total',
|
||||
days_since_last, last_observation, total_rows) AS detail
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'dead'
|
||||
|
||||
UNION ALL
|
||||
-- 2) Stale scraper: 8-30 days since last row.
|
||||
SELECT 'warning', 3, 'scraper_stale',
|
||||
vendor || ' / ' || source_table,
|
||||
format('no new rows for %s days (last %s)', days_since_last, last_observation)
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'stale'
|
||||
|
||||
UNION ALL
|
||||
-- 3) Rows written but NO quantitative stock signal at all (qty + warehouse both empty)
|
||||
-- on an otherwise-alive scraper. This is the real "extraction is a no-op" case.
|
||||
SELECT 'high', 2, 'stock_extraction_empty',
|
||||
vendor || ' / stock_observations',
|
||||
format('%s rows but quantity_available=%s%% and warehouse_qty=%s%% populated',
|
||||
stock_rows, pct_qty_available, pct_warehouse_qty)
|
||||
FROM v_stock_completeness
|
||||
WHERE stock_rows >= 500
|
||||
AND pct_qty_available = 0
|
||||
AND pct_warehouse_qty = 0
|
||||
AND last_obs >= now()::date - 30
|
||||
|
||||
UNION ALL
|
||||
-- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable.
|
||||
SELECT 'high', 2, 'delivery_date_gap',
|
||||
'ALL stock scrapers',
|
||||
format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable',
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2),
|
||||
round(100.0 * count(lead_time_days) / nullif(count(*),0), 2))
|
||||
FROM stock_observations
|
||||
HAVING round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5
|
||||
|
||||
UNION ALL
|
||||
-- 5) reach_label incomplete on a sizeable catalog.
|
||||
SELECT 'warning', 3, 'reach_label_incomplete',
|
||||
vendor,
|
||||
format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers)
|
||||
FROM v_coverage
|
||||
WHERE transceivers >= 100 AND pct_reach_null >= 20
|
||||
|
||||
UNION ALL
|
||||
-- 6) Future-dated news -> ingest guard candidate.
|
||||
SELECT 'warning', 3, 'future_dated_news',
|
||||
'news_articles',
|
||||
format('%s article(s) dated in the future (max %s) — ingest guard recommended',
|
||||
count(*), max(published_at)::date)
|
||||
FROM news_articles
|
||||
WHERE published_at > now() + interval '2 days'
|
||||
HAVING count(*) > 0
|
||||
)
|
||||
SELECT severity, severity_rank, category, entity, detail
|
||||
FROM alerts
|
||||
ORDER BY severity_rank, category, entity;
|
||||
@ -1,237 +0,0 @@
|
||||
-- sql/125-switch-quality.sql
|
||||
-- Switch / compatibility data-quality remediation + monitoring layer.
|
||||
--
|
||||
-- Audit 2026-07-06 (read-only) found systematic errors in the switch database:
|
||||
-- * 5.686+ physically-impossible compatibility rows (transceiver faster than the
|
||||
-- switch's fastest port). Root cause: the form-factor fallback in
|
||||
-- packages/scraper/src/scrapers/flexoptix-compat.ts matched by form factor ONLY,
|
||||
-- ignoring speed -> a 400G-OSFP port matched every OSFP transceiver incl. 1.6T.
|
||||
-- * 420/687 switches with NULL max_speed_gbps (ports_config present -> derivable).
|
||||
-- * 528 transceivers with speed_gbps <= 0 (blocks compat validation; they silently
|
||||
-- pass as "compatible" with everything).
|
||||
-- * coding-pattern coverage as low as ~13% at 100G (vendor_compat = [] for most).
|
||||
--
|
||||
-- DESIGN (why this is safe at the "no errors" bar):
|
||||
-- The naive ceiling (ports_config-derived max) is UNRELIABLE because some switches'
|
||||
-- ports_config is understated (e.g. Cisco C9500-32QC lists 40G but does 100G; NCS 540
|
||||
-- "Z" ports list 10G but are 25G-capable; A9K-20HG-FLEX lists 100G but does 400G).
|
||||
-- Flagging by that ceiling would create FALSE "incompatible" flags on curated rows.
|
||||
-- So the effective ceiling is evidence-based:
|
||||
-- effective_max = GREATEST(stored max_speed_gbps,
|
||||
-- ports_config-derived max,
|
||||
-- fastest curated-compatible transceiver for the switch)
|
||||
-- and ONLY algorithmic 'spec_match' fallback rows are ever auto-flagged; curated
|
||||
-- ('vendor_matrix'/'vendor_compat') rows are never touched (they instead RAISE the
|
||||
-- ceiling, protecting understated switches). Result: 0 curated false positives.
|
||||
--
|
||||
-- Idempotent + safe to re-run. Applied with psql --single-transaction.
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- 0) Ports-config -> max port speed (pure helper) + evidence-based switch ceiling
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
create or replace function fn_ports_max_gbps(cfg jsonb) returns numeric
|
||||
language sql immutable as $fn$
|
||||
select case when jsonb_typeof(cfg) = 'object'
|
||||
then (select max(substring(k from '([0-9]+)G')::numeric)
|
||||
from jsonb_object_keys(cfg) k)
|
||||
else null end
|
||||
$fn$;
|
||||
|
||||
comment on function fn_ports_max_gbps(jsonb) is
|
||||
'Max port speed (Gbps) parsed from a switch ports_config JSONB object, e.g. {"400G_QSFP-DD":4,"100G_QSFP28":2} -> 400. NULL if config is empty/non-object.';
|
||||
|
||||
create or replace view v_switch_effective_max as
|
||||
select s.id as switch_id,
|
||||
s.max_speed_gbps as stored_max_gbps,
|
||||
fn_ports_max_gbps(s.ports_config) as ports_max_gbps,
|
||||
(select max(t.speed_gbps)
|
||||
from compatibility c join transceivers t on t.id = c.transceiver_id
|
||||
where c.switch_id = s.id and c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps > 0) as curated_max_gbps,
|
||||
greatest(
|
||||
coalesce(s.max_speed_gbps, 0),
|
||||
coalesce(fn_ports_max_gbps(s.ports_config), 0),
|
||||
coalesce((select max(t.speed_gbps)
|
||||
from compatibility c join transceivers t on t.id = c.transceiver_id
|
||||
where c.switch_id = s.id and c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps > 0), 0)
|
||||
) as effective_max_gbps
|
||||
from switches s;
|
||||
|
||||
comment on view v_switch_effective_max is
|
||||
'Evidence-based max port speed per switch (greatest of stored max_speed_gbps, ports_config-derived, and fastest curated-compatible transceiver). Used to validate compatibility without false-flagging switches whose ports_config is understated.';
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION A) Backfill max_speed_gbps from ports_config (additive, nulls only).
|
||||
-- Highest leverage: unblocks validation for switches that currently pass no check.
|
||||
-- Only fills NULLs; never overwrites a curated stored value (61/267 stored values
|
||||
-- disagree with ports-derived in both directions -> curated values are preserved).
|
||||
-- Switches with an empty {} ports_config stay NULL (need a datasheet -> see alerts).
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
update switches s
|
||||
set max_speed_gbps = fn_ports_max_gbps(s.ports_config)
|
||||
where s.max_speed_gbps is null
|
||||
and fn_ports_max_gbps(s.ports_config) is not null
|
||||
and fn_ports_max_gbps(s.ports_config) > 0;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION C) Re-derive speed_gbps for transceivers with speed <= 0, HIGH
|
||||
-- confidence only. Speed comes from the manufacturer's standardized speed token
|
||||
-- in the part number (e.g. SFP-10G-SR -> 10, QDD-2X400G -> 800, QSFP-4SFP25G -> 100),
|
||||
-- and is applied ONLY when it is <= the form factor's physical maximum. Parts whose
|
||||
-- token exceeds the form-factor cap (usually a wrong form_factor, e.g. QSFP-400G-AOC
|
||||
-- mislabelled 'SFP') or that have no speed token are LEFT flagged for datasheet work.
|
||||
-- No blind update: FastEthernet "100FX"/"100BASE" (no 'G') never matches.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
with cand as (
|
||||
select id,
|
||||
case
|
||||
when part_number ~* '([0-9]+)X([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)SFP([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)G','i'))[1]::numeric
|
||||
end as derived,
|
||||
case upper(coalesce(form_factor,''))
|
||||
when 'SFP' then 1 when 'SFP+' then 16 when 'SFP28' then 32 when 'SFP56' then 56
|
||||
when 'XFP' then 10 when 'QSFP+' then 40 when 'QSFP28' then 100 when 'QSFP56' then 200
|
||||
when 'QSFP-DD' then 800 when 'QSFP-DD800' then 800
|
||||
when 'OSFP' then 1600 when 'OSFP224' then 1600
|
||||
when 'CFP' then 100 when 'CFP2' then 400 when 'CFP4' then 100 when 'CPAK' then 100
|
||||
else 99999 end as ff_cap
|
||||
from transceivers
|
||||
where speed_gbps is null or speed_gbps <= 0)
|
||||
update transceivers t
|
||||
set speed_gbps = c.derived
|
||||
from cand c
|
||||
where t.id = c.id
|
||||
and c.derived is not null and c.derived > 0
|
||||
and c.derived <= c.ff_cap;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION B) Flag physically-impossible compatibility rows.
|
||||
-- Only 'spec_match' (algorithmic form-factor fallback) rows are flagged; curated
|
||||
-- rows are never touched. status='incompatible' (constraint-allowed value) removes
|
||||
-- them from every consumer that filters status='compatible' (e.g. queries.ts:302).
|
||||
-- Runs AFTER A + C so it uses backfilled ceilings and corrected transceiver speeds.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(125): transceiver speed exceeds switch max port speed'
|
||||
else c.notes || ' | auto(125): transceiver speed exceeds switch max port speed' end
|
||||
from em e, transceivers t
|
||||
where e.switch_id = c.switch_id
|
||||
and t.id = c.transceiver_id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- MONITORING VIEWS (read-only). v_switch_quality_alerts is the task list.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- 1) Impossible rows still CLAIMING compatibility (drains to 0 after remediation B;
|
||||
-- lights up again if a future scrape inserts a new impossible spec_match row).
|
||||
create or replace view v_compat_speed_errors as
|
||||
select c.id as compat_id,
|
||||
s.model as switch,
|
||||
e.effective_max_gbps as switch_max_gbps,
|
||||
t.part_number as transceiver,
|
||||
t.speed_gbps as transceiver_gbps,
|
||||
round(t.speed_gbps / nullif(e.effective_max_gbps,0), 1) as over_ratio
|
||||
from compatibility c
|
||||
join v_switch_effective_max e on e.switch_id = c.switch_id
|
||||
join switches s on s.id = c.switch_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- 2) Understated switch port data: a CURATED source pairs a transceiver faster than the
|
||||
-- switch's stored/ports max. These are NOT errors to auto-fix -- they flag switches
|
||||
-- whose ports_config needs datasheet enrichment (or a wrong curated claim to review).
|
||||
create or replace view v_switch_understated_ports as
|
||||
select c.id as compat_id,
|
||||
v.name as manufacturer,
|
||||
s.model as switch,
|
||||
s.max_speed_gbps as stored_max_gbps,
|
||||
fn_ports_max_gbps(s.ports_config) as ports_max_gbps,
|
||||
t.part_number as transceiver,
|
||||
t.speed_gbps as transceiver_gbps,
|
||||
c.verification_method
|
||||
from compatibility c
|
||||
join switches s on s.id = c.switch_id
|
||||
join vendors v on v.id = s.vendor_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps is not null
|
||||
and (coalesce(s.max_speed_gbps,0) > 0 or coalesce(fn_ports_max_gbps(s.ports_config),0) > 0)
|
||||
and t.speed_gbps > greatest(coalesce(s.max_speed_gbps,0), coalesce(fn_ports_max_gbps(s.ports_config),0));
|
||||
|
||||
-- 3) Coding-pattern coverage per speed tier (module only) -- data-SOURCE gap, pipeline task.
|
||||
create or replace view v_coding_coverage as
|
||||
select t.speed_gbps::numeric as tier,
|
||||
count(*)::int as modules,
|
||||
count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)::int as with_coding,
|
||||
round(100.0 * count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)
|
||||
/ count(*), 1) as coverage_pct
|
||||
from transceivers t
|
||||
where t.product_type = 'module' and t.speed_gbps >= 100
|
||||
group by 1 order by 1;
|
||||
|
||||
-- 4) Transceivers with an impossible speed (<=0). 100 Mbps (0.1) and FC/SONET rates
|
||||
-- (0.155/0.622/2.488/8/8.5/14/28/32) are legitimate and intentionally NOT flagged.
|
||||
create or replace view v_transceiver_speed_errors as
|
||||
select id, part_number, speed_gbps, form_factor
|
||||
from transceivers
|
||||
where speed_gbps is null or speed_gbps <= 0;
|
||||
|
||||
-- 5) Switches that cannot validate compatibility (missing max_speed / total_ports).
|
||||
create or replace view v_switch_completeness as
|
||||
select s.id, v.name as manufacturer, s.model, s.max_speed_gbps, s.total_ports,
|
||||
(jsonb_typeof(s.ports_config) = 'object' and s.ports_config <> '{}'::jsonb) as has_ports_config
|
||||
from switches s join vendors v on v.id = s.vendor_id
|
||||
where s.max_speed_gbps is null or s.total_ports is null;
|
||||
|
||||
-- 6) One-line alerts = the switch/coding data-quality task list.
|
||||
create or replace view v_switch_quality_alerts as
|
||||
select 'compatibility: ' || count(*) || ' physically-impossible spec_match rows (transceiver faster than switch)' as alert,
|
||||
'critical' as severity from v_compat_speed_errors
|
||||
union all
|
||||
select 'coding matrix: ' || tier || 'G only ' || coverage_pct || '% populated (' || (modules - with_coding) || ' modules without coding patterns)',
|
||||
case when coverage_pct < 25 then 'high' else 'warning' end
|
||||
from v_coding_coverage where coverage_pct < 70
|
||||
union all
|
||||
select 'transceivers: ' || count(*) || ' with speed_gbps <= 0 (need form_factor/datasheet)', 'high'
|
||||
from v_transceiver_speed_errors
|
||||
union all
|
||||
select 'switches: ' || count(*) || ' missing max_speed_gbps (empty ports_config -> needs datasheet)', 'warning'
|
||||
from v_switch_completeness where max_speed_gbps is null
|
||||
union all
|
||||
select 'switches: ' || count(*) || ' with understated port data (curated compat faster than ports_config)', 'warning'
|
||||
from v_switch_understated_ports;
|
||||
|
||||
-- Usage: select * from v_switch_quality_alerts order by severity;
|
||||
-- select * from v_compat_speed_errors order by over_ratio desc limit 50;
|
||||
-- select * from v_switch_understated_ports order by transceiver_gbps desc;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '125-switch-quality.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '125-switch-quality.sql');
|
||||
@ -1,98 +0,0 @@
|
||||
-- sql/126-switch-quality-enforcement.sql
|
||||
-- Durable enforcement + on-demand validation for the switch/compatibility DB.
|
||||
-- Complements sql/125 (which cleaned the data): makes the "no physically-impossible
|
||||
-- compatibility rows" invariant structural, so it survives a scraper regression.
|
||||
--
|
||||
-- Design (same "no false positives" rule as 125):
|
||||
-- * The guard NEVER touches curated rows (vendor_matrix/vendor_compat/tested/datasheet)
|
||||
-- -- those are trusted and, on understated switches, are the source of truth.
|
||||
-- * For algorithmic rows (spec_match/community), if the transceiver is faster than the
|
||||
-- switch's fastest known port (stored max_speed_gbps OR ports_config-derived), the row
|
||||
-- is FORCED to status='incompatible' rather than dropped -- it stays recorded but is
|
||||
-- excluded by every consumer that filters status='compatible' (e.g. queries.ts:302).
|
||||
-- * Forcing (not RETURN NULL) means the guard never conflicts with the 125 flagging
|
||||
-- UPDATE and never silently loses data. Idempotent; safe to re-run.
|
||||
--
|
||||
-- Depends on sql/125 (fn_ports_max_gbps + the v_* views). Apply after 125.
|
||||
|
||||
-- ── Guard: keep physically-impossible algorithmic matches out of 'compatible' ──
|
||||
create or replace function compat_speed_guard() returns trigger
|
||||
language plpgsql as $$
|
||||
declare
|
||||
sw_ceiling numeric;
|
||||
tx_speed numeric;
|
||||
begin
|
||||
-- Trust curated/vendor/tested sources; they define reality on understated switches.
|
||||
if NEW.verification_method in ('vendor_matrix','vendor_compat','tested','datasheet') then
|
||||
return NEW;
|
||||
end if;
|
||||
if NEW.status is distinct from 'compatible' then
|
||||
return NEW; -- only guard rows claiming compatibility
|
||||
end if;
|
||||
|
||||
select greatest(coalesce(s.max_speed_gbps,0), coalesce(fn_ports_max_gbps(s.ports_config),0))
|
||||
into sw_ceiling
|
||||
from switches s where s.id = NEW.switch_id;
|
||||
|
||||
select t.speed_gbps into tx_speed
|
||||
from transceivers t where t.id = NEW.transceiver_id;
|
||||
|
||||
if sw_ceiling is not null and sw_ceiling > 0
|
||||
and tx_speed is not null and tx_speed > sw_ceiling then
|
||||
NEW.status := 'incompatible';
|
||||
if NEW.notes is null or NEW.notes = '' then
|
||||
NEW.notes := 'auto(126): transceiver speed exceeds switch max port speed';
|
||||
elsif position('auto(12' in NEW.notes) = 0 then
|
||||
NEW.notes := NEW.notes || ' | auto(126): transceiver speed exceeds switch max port speed';
|
||||
end if;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function compat_speed_guard() is
|
||||
'BEFORE INSERT/UPDATE guard on compatibility: forces algorithmic (non-curated) rows whose transceiver is faster than the switch max port speed to status=incompatible. Never alters curated rows.';
|
||||
|
||||
drop trigger if exists trg_compat_speed_guard on compatibility;
|
||||
create trigger trg_compat_speed_guard
|
||||
before insert or update on compatibility
|
||||
for each row execute function compat_speed_guard();
|
||||
|
||||
-- ── Validator: prove switch-DB quality on demand (CI / cron) ──────────────────
|
||||
-- Hard invariants have enforced=true; data-source gaps (coverage, datasheet) are
|
||||
-- reported but enforced=false so they don't fail the gate before the data exists.
|
||||
-- CI gate: select bool_and(pass) from check_switch_quality() where enforced; -- must be true
|
||||
create or replace function check_switch_quality()
|
||||
returns table(check_name text, detail text, pass boolean, enforced boolean)
|
||||
language sql stable as $$
|
||||
select 'no_impossible_spec_match_compat',
|
||||
(select count(*) from v_compat_speed_errors)::text || ' impossible rows',
|
||||
(select count(*) from v_compat_speed_errors) = 0,
|
||||
true
|
||||
union all
|
||||
select 'coding_coverage_100G_ge_50pct',
|
||||
coalesce((select coverage_pct from v_coding_coverage where tier = 100), 0)::text || '%',
|
||||
coalesce((select coverage_pct from v_coding_coverage where tier = 100), 0) >= 50,
|
||||
false
|
||||
union all
|
||||
select 'transceivers_speed_positive',
|
||||
(select count(*) from v_transceiver_speed_errors)::text || ' with speed<=0',
|
||||
(select count(*) from v_transceiver_speed_errors) = 0,
|
||||
false
|
||||
union all
|
||||
select 'switches_have_max_speed',
|
||||
(select count(*) from v_switch_completeness where max_speed_gbps is null)::text || ' missing',
|
||||
(select count(*) from v_switch_completeness where max_speed_gbps is null) = 0,
|
||||
false
|
||||
union all
|
||||
select 'no_understated_switch_ports',
|
||||
(select count(*) from v_switch_understated_ports)::text || ' understated',
|
||||
(select count(*) from v_switch_understated_ports) = 0,
|
||||
false;
|
||||
$$;
|
||||
|
||||
comment on function check_switch_quality() is
|
||||
'Switch-DB quality proof. select bool_and(pass) from check_switch_quality() where enforced; must be true (no physically-impossible compatibility rows). enforced=false rows track datasheet/data-source gaps.';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '126-switch-quality-enforcement.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '126-switch-quality-enforcement.sql');
|
||||
@ -1,32 +0,0 @@
|
||||
-- sql/127-news-future-date-guard.sql
|
||||
-- News published_at cannot be in the future: an article cannot be published after
|
||||
-- we scraped it. One future-dated row found 2026-07-06 (The Next Platform, pub
|
||||
-- 2026-07-15 but created/scraped 2026-06-19). One-time correction + ingest guard so
|
||||
-- it cannot recur (a CHECK with now() is not immutable/allowed -> BEFORE trigger).
|
||||
-- Idempotent; safe to re-run.
|
||||
|
||||
-- One-time: clamp any future published_at to the scrape date (created_at), else now().
|
||||
update news_articles
|
||||
set published_at = least(published_at, coalesce(created_at, now()))
|
||||
where published_at is not null and published_at > now();
|
||||
|
||||
create or replace function news_published_at_guard() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
if NEW.published_at is not null and NEW.published_at > now() then
|
||||
NEW.published_at := least(coalesce(NEW.created_at, now()), now());
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function news_published_at_guard() is
|
||||
'BEFORE INSERT/UPDATE on news_articles: clamps a future published_at to the scrape date (created_at) or now(). Prevents future-dated news polluting the feed.';
|
||||
|
||||
drop trigger if exists trg_news_published_at_guard on news_articles;
|
||||
create trigger trg_news_published_at_guard
|
||||
before insert or update on news_articles
|
||||
for each row execute function news_published_at_guard();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '127-news-future-date-guard.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '127-news-future-date-guard.sql');
|
||||
@ -1,75 +0,0 @@
|
||||
-- sql/128-url-normalize-and-enrichment-worklist.sql
|
||||
-- Foundation for the datasheet-enrichment pipeline (Lever 3):
|
||||
-- (1) Fix malformed double-prefixed URLs (a scraper concatenated the base host in
|
||||
-- front of an already-absolute URL, e.g. "https://www.mouser.dehttps://www.mouser.de/…").
|
||||
-- 503 rows affected (details_source_url 373, product_page_url 83, image_url 47).
|
||||
-- These cannot be fetched, so they block any enrichment run.
|
||||
-- (2) A guard so the ingest bug cannot re-introduce them.
|
||||
-- (3) v_datasheet_enrichment_worklist — the pipeline's input: every transceiver/switch
|
||||
-- that needs a field filled from source, with its best source URL + type.
|
||||
-- Idempotent; safe to re-run. The regex only matches a second scheme that appears
|
||||
-- BEFORE the first path slash, so legitimate URLs with an encoded URL in the query string
|
||||
-- (…?url=https://…) are never touched.
|
||||
|
||||
create or replace function fn_strip_double_url_prefix(u text) returns text
|
||||
language sql immutable as $$
|
||||
select case when u ~ '^https?://[^/]*https?://'
|
||||
then regexp_replace(u, '^https?://[^/]*(https?://)', '\1')
|
||||
else u end
|
||||
$$;
|
||||
|
||||
comment on function fn_strip_double_url_prefix(text) is
|
||||
'Removes a duplicated leading scheme+host from a URL, e.g. https://a.comhttps://a.com/x -> https://a.com/x. Only when the 2nd scheme precedes the first path slash (so ?url=https://… is safe).';
|
||||
|
||||
-- (1) one-time cleanup
|
||||
update transceivers set
|
||||
details_source_url = fn_strip_double_url_prefix(details_source_url),
|
||||
product_page_url = fn_strip_double_url_prefix(product_page_url),
|
||||
image_url = fn_strip_double_url_prefix(image_url)
|
||||
where details_source_url ~ '^https?://[^/]*https?://'
|
||||
or product_page_url ~ '^https?://[^/]*https?://'
|
||||
or image_url ~ '^https?://[^/]*https?://';
|
||||
|
||||
-- (2) ingest guard
|
||||
create or replace function transceiver_url_normalize() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
NEW.details_source_url := fn_strip_double_url_prefix(NEW.details_source_url);
|
||||
NEW.product_page_url := fn_strip_double_url_prefix(NEW.product_page_url);
|
||||
NEW.image_url := fn_strip_double_url_prefix(NEW.image_url);
|
||||
NEW.datasheet_url := fn_strip_double_url_prefix(NEW.datasheet_url);
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
drop trigger if exists trg_transceiver_url_normalize on transceivers;
|
||||
create trigger trg_transceiver_url_normalize
|
||||
before insert or update on transceivers
|
||||
for each row execute function transceiver_url_normalize();
|
||||
|
||||
-- (3) enrichment worklist — pipeline input
|
||||
create or replace view v_datasheet_enrichment_worklist as
|
||||
select 'switch'::text as entity, s.id as entity_id, v.name as vendor, s.model as item,
|
||||
'ports_config+max_speed'::text as need,
|
||||
coalesce(s.datasheet_url, s.product_page_url) as source_url,
|
||||
case when s.datasheet_url is not null then 'datasheet'
|
||||
when s.product_page_url is not null then 'product_page'
|
||||
else 'none' end as source_type
|
||||
from switches s join vendors v on v.id = s.vendor_id
|
||||
where (s.ports_config is null or s.ports_config = '{}'::jsonb) and s.max_speed_gbps is null
|
||||
union all
|
||||
select 'transceiver', t.id, v.name, t.part_number,
|
||||
'form_factor+speed',
|
||||
coalesce(t.details_source_url, t.datasheet_url, t.product_page_url),
|
||||
case when t.datasheet_url is not null then 'datasheet'
|
||||
when t.details_source_url is not null then 'details_page'
|
||||
when t.product_page_url is not null then 'product_page'
|
||||
else 'none' end
|
||||
from transceivers t join vendors v on v.id = t.vendor_id
|
||||
where t.speed_gbps is null or t.speed_gbps <= 0;
|
||||
|
||||
comment on view v_datasheet_enrichment_worklist is
|
||||
'Input for the datasheet-enrichment pipeline: entities whose ports_config/max_speed (switch) or form_factor/speed (transceiver) must be filled from source, with the best available source URL and its type. source_type=none means URL discovery is needed first.';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '128-url-normalize-and-enrichment-worklist.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '128-url-normalize-and-enrichment-worklist.sql');
|
||||
@ -1,183 +0,0 @@
|
||||
-- sql/128-orchestrator-control-plane.sql
|
||||
-- The DATA plane for the TIP Autonomous Data-Quality Orchestrator ("tip-dqo").
|
||||
--
|
||||
-- TIP already has a world-class DETECTION layer (sql/123-126: v_scraper_health,
|
||||
-- v_data_quality_alerts, check_switch_quality) and a blind fixed-cadence dispatcher
|
||||
-- (scheduler.ts: boss.send() is never called from any health path). This migration
|
||||
-- adds the tables that let a controller close the loop DETECT -> PRIORITIZE ->
|
||||
-- DISPATCH -> VERIFY -> RECHECK against DATA freshness, with DB-ENFORCED guardrails
|
||||
-- (the ledger is the single source of truth for cooldown/budget/backoff/breaker, so
|
||||
-- the loop physically cannot hammer a source even under concurrent ticks).
|
||||
--
|
||||
-- Read-only against the existing quality views; additive only (no existing object is
|
||||
-- altered). Idempotent (IF NOT EXISTS / CREATE OR REPLACE / ON CONFLICT DO NOTHING).
|
||||
-- Applied by scripts/migrate.ts as the next numbered migration after 127.
|
||||
|
||||
-- ── 1) source_registry — the signal->runner map + per-source SLA + guardrail contract.
|
||||
-- One row per (vendor, source_table) the loop may act on, PLUS the '__global__'
|
||||
-- kill-switch / global-budget row. root_cause_class is LOAD-BEARING: only
|
||||
-- 'staleness' rows are ever auto-dispatched; 'code_fix'/'blocked' rows escalate to a
|
||||
-- human instead of looping retries (re-scraping cannot fix a silent write-path bug).
|
||||
create table if not exists source_registry (
|
||||
source_key text primary key, -- 'atgbics/stock_observations' | '__global__'
|
||||
vendor text, -- matches v_scraper_health.vendor
|
||||
source_table text, -- matches v_scraper_health.source_table
|
||||
queue_names text[] not null default '{}', -- pg-boss queues to dispatch
|
||||
value_weight numeric not null default 1, -- business priority multiplier
|
||||
warn_days int not null default 8,
|
||||
dead_days int not null default 30,
|
||||
cooldown_minutes int not null default 180, -- >= pricing cadence (2h) so no in-flight re-fire
|
||||
budget_per_day int not null default 4,
|
||||
max_backoff_hours int not null default 24,
|
||||
circuit_breaker_strikes int not null default 3,
|
||||
root_cause_class text not null default 'staleness'
|
||||
check (root_cause_class in ('staleness','code_fix','blocked')),
|
||||
profile text not null default 'erik-safe',
|
||||
enabled boolean not null default true,
|
||||
notes text,
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
-- ── 2) self_heal_dispatch — the append-only dispatch LEDGER. Single source of truth for
|
||||
-- every guardrail (cooldown/budget/backoff/breaker) and the plan/dispatch audit trail.
|
||||
-- strikes carries the running no-progress count as of this row's verification.
|
||||
create table if not exists self_heal_dispatch (
|
||||
id bigserial primary key,
|
||||
source_key text not null,
|
||||
queues text[] not null default '{}',
|
||||
reason text,
|
||||
mode text not null check (mode in ('plan','dispatch')),
|
||||
rows_before bigint,
|
||||
last_obs_before date,
|
||||
dispatched_at timestamptz not null default now(),
|
||||
checked_at timestamptz,
|
||||
rows_after bigint,
|
||||
last_obs_after date,
|
||||
outcome text not null default 'pending'
|
||||
check (outcome in ('pending','healed','ineffective','escalated','skipped')),
|
||||
strikes int not null default 0,
|
||||
run_id text
|
||||
);
|
||||
create index if not exists idx_self_heal_dispatch_source_time
|
||||
on self_heal_dispatch (source_key, dispatched_at desc);
|
||||
create index if not exists idx_self_heal_dispatch_pending
|
||||
on self_heal_dispatch (outcome, dispatched_at) where outcome = 'pending';
|
||||
|
||||
-- ── 3) scraper_run_log — the ran-but-wrote-0 detector. Lets VERIFY distinguish a silent
|
||||
-- write-path break (job 'completed', 0 rows written — the ATGBICS-stock signature)
|
||||
-- from a legitimately-no-new-rows run. Built here; scrapers can emit into it over time.
|
||||
-- VERIFY v1 falls back to v_scraper_health freshness when a source has no run_log rows.
|
||||
create table if not exists scraper_run_log (
|
||||
id bigserial primary key,
|
||||
vendor text,
|
||||
table_name text,
|
||||
run_at timestamptz not null default now(),
|
||||
rows_written int not null default 0,
|
||||
source text, -- who triggered it (e.g. 'dqo:self-heal', 'cron')
|
||||
run_id text
|
||||
);
|
||||
create index if not exists idx_scraper_run_log_vendor_time
|
||||
on scraper_run_log (vendor, table_name, run_at desc);
|
||||
|
||||
-- ── 4) acknowledged_exceptions — delta-gate suppression so day-1 known-open failures
|
||||
-- (528 speed<=0, ~3050 units_sold, 78% coding template garbage) NEVER page. The
|
||||
-- escalation ladder only fires for entities NOT covered by an active ack.
|
||||
create table if not exists acknowledged_exceptions (
|
||||
id bigserial primary key,
|
||||
check_name text, -- v_data_quality_alerts.category or a check_switch_quality name
|
||||
source_key text, -- optional: scope the ack to one source
|
||||
reason text not null,
|
||||
task_ref text, -- Gitea issue / spawned-task id
|
||||
ack_until timestamptz, -- null = indefinite
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
-- ── 5) v_dqo_targets — the single ranked SELECT the controller reads for DETECT+PRIORITIZE.
|
||||
-- Joins the live freshness signal (v_scraper_health, keyed off observation TABLES, not
|
||||
-- pg-boss job cadence) onto the registry, ranks dead-before-stale weighted by business
|
||||
-- value and how far past dead_days. Guardrail filtering (cooldown/budget/backoff/breaker)
|
||||
-- is applied in the controller against the ledger, not here, to keep the view simple.
|
||||
create or replace view v_dqo_targets as
|
||||
select
|
||||
sr.source_key,
|
||||
h.vendor,
|
||||
h.source_table,
|
||||
h.status,
|
||||
h.days_since_last,
|
||||
h.rows_30d,
|
||||
h.last_observation,
|
||||
sr.root_cause_class,
|
||||
sr.queue_names,
|
||||
sr.profile,
|
||||
sr.value_weight,
|
||||
sr.warn_days,
|
||||
sr.dead_days,
|
||||
sr.cooldown_minutes,
|
||||
sr.budget_per_day,
|
||||
sr.max_backoff_hours,
|
||||
sr.circuit_breaker_strikes,
|
||||
round(
|
||||
(case h.status when 'dead' then 3 when 'stale' then 2 else 1 end)::numeric
|
||||
* sr.value_weight
|
||||
* (1 + coalesce(h.days_since_last, 0)::numeric / nullif(sr.dead_days, 0))
|
||||
, 2) as score
|
||||
from v_scraper_health h
|
||||
join source_registry sr
|
||||
on sr.vendor = h.vendor and sr.source_table = h.source_table
|
||||
where h.status in ('stale', 'dead')
|
||||
and sr.enabled
|
||||
and sr.source_key <> '__global__'
|
||||
order by
|
||||
(case h.status when 'dead' then 3 when 'stale' then 2 else 1 end) desc,
|
||||
score desc;
|
||||
|
||||
-- ── 6) SEED — the global kill-switch/budget row + the known EXPECTED_VENDORS sources.
|
||||
-- root_cause_class is HAND-CURATED from the 2026-07-06 gap analysis, not guessed:
|
||||
-- ATGBICS/stock = code_fix (price is the single most-alive source yet stock writes
|
||||
-- 0 rows/54d = silent write-path break; escalate, never loop)
|
||||
-- Flexoptix/stock= staleness (price+catalog sync work; a re-scrape can plausibly heal it
|
||||
-- — the first source to arm out of plan-mode)
|
||||
-- Every other source defaults to 'staleness'. Extra (vendor, source_table) rows that do
|
||||
-- not match a live v_scraper_health row are harmless — the join just yields no target.
|
||||
insert into source_registry
|
||||
(source_key, vendor, source_table, queue_names, value_weight, root_cause_class, profile, notes)
|
||||
values
|
||||
-- global kill-switch + global daily dispatch ceiling (belt-and-suspenders over per-source budget)
|
||||
('__global__', null, null, '{}', 1, 'staleness', 'erik-safe',
|
||||
'Global control row: enabled=false disarms the entire loop; budget_per_day is the global daily dispatch ceiling.'),
|
||||
|
||||
-- stock sources (higher value_weight — stock freshness is core to sourcing)
|
||||
('atgbics/stock_observations', 'ATGBICS', 'stock_observations', '{scrape:pricing:atgbics}', 1.5, 'code_fix', 'erik-safe',
|
||||
'Silent stock write-path break: price fresh, stock 0 rows/54d. Escalate to a human; a re-scrape cannot fix a code bug.'),
|
||||
('flexoptix/stock_observations', 'Flexoptix', 'stock_observations', '{scrape:pricing:flexoptix}', 1.5, 'staleness', 'erik-safe',
|
||||
'First source to arm out of plan-mode: lowest-risk staleness, erik-safe queue.'),
|
||||
('fibermall/stock_observations', 'FiberMall', 'stock_observations', '{scrape:pricing:fibermall}', 1.5, 'staleness', 'erik-safe', null),
|
||||
('qsfptek/stock_observations', 'QSFPTEK', 'stock_observations', '{scrape:pricing:qsfptek}', 1.5, 'staleness', 'erik-safe',
|
||||
'The only currently-live stock writer — the reference for a working stock-write path.'),
|
||||
('naddod/stock_observations', 'NADDOD', 'stock_observations', '{scrape:pricing:naddod}', 1.5, 'staleness', 'erik-safe', null),
|
||||
('gbics/stock_observations', 'GBICS', 'stock_observations', '{scrape:pricing:gbics}', 1.5, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('sfpcables/stock_observations', 'SFPcables', 'stock_observations', '{scrape:pricing:sfpcables}', 1.5, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
|
||||
-- price sources
|
||||
('atgbics/price_observations', 'ATGBICS', 'price_observations', '{scrape:pricing:atgbics}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('flexoptix/price_observations', 'Flexoptix', 'price_observations', '{scrape:pricing:flexoptix}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('fibermall/price_observations', 'FiberMall', 'price_observations', '{scrape:pricing:fibermall}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('qsfptek/price_observations', 'QSFPTEK', 'price_observations', '{scrape:pricing:qsfptek}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('naddod/price_observations', 'NADDOD', 'price_observations', '{scrape:pricing:naddod}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('gbics/price_observations', 'GBICS', 'price_observations', '{scrape:pricing:gbics}', 1.0, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('sfpcables/price_observations', 'SFPcables', 'price_observations', '{scrape:pricing:sfpcables}', 1.0, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('10gtek/price_observations', '10Gtek', 'price_observations', '{scrape:pricing:10gtek}', 1.0, 'staleness', 'proxmox-heavy',
|
||||
'scrape:pricing:10gtek is a HEAVY queue — only the proxmox-heavy profile dispatches it.')
|
||||
on conflict (source_key) do nothing;
|
||||
|
||||
-- set the global daily dispatch ceiling on the __global__ row (soak default: 20/day)
|
||||
update source_registry set budget_per_day = 20 where source_key = '__global__';
|
||||
|
||||
-- Proof (read anytime):
|
||||
-- select * from v_dqo_targets; -- what the loop would act on right now
|
||||
-- select enabled from source_registry where source_key='__global__'; -- kill-switch state
|
||||
-- select source_key, outcome, mode, dispatched_at from self_heal_dispatch order by id desc limit 20;
|
||||
@ -1,36 +0,0 @@
|
||||
-- sql/130-adapter-pi-topology.sql
|
||||
-- Teach the TIP Adapter (tip-dqo, the data-quality orchestrator from sql/129) the REAL
|
||||
-- fetch topology, verified 2026-07-06: TIP data is fetched primarily by three residential
|
||||
-- Raspberry Pi 5 nodes — ContextX-Pi01/tashi(.204, 16GB), Pi02/portia(.212, 4GB),
|
||||
-- Pi03/ester(.211, 4GB) — all live pg-boss workers, all now Playwright-capable
|
||||
-- (chromium v1208, launch-verified). NUC(.115) is the scheduler; Erik(.127) holds DB+API.
|
||||
--
|
||||
-- The other session seeded every vendor pricing/stock source as profile='erik-safe'. That
|
||||
-- is wrong for the fetch fleet: Erik is a datacenter IP (blocked by FS.com/ATGBICS & co.),
|
||||
-- and 'erik-safe' only dispatches ERIK_SAFE_QUEUES — so gbics/sfpcables would NEVER dispatch
|
||||
-- (their own notes flagged this). The residential Pis are the correct fetchers.
|
||||
--
|
||||
-- Fix: route all vendor pricing/stock sources to profile='pi-fetch' (recognized by the
|
||||
-- orchestrator: queuesForProfile → all non-heavy/non-discovery queues; cap 10). This is
|
||||
-- CONFIG ONLY — no scraper fires (the loop stays in Plan-Mode until SELF_HEAL_PLAN_ONLY=false).
|
||||
-- 10gtek stays 'proxmox-heavy' (its queue is in HEAVY_QUEUES). __global__ stays 'erik-safe'
|
||||
-- (config row, excluded from v_dqo_targets). Idempotent.
|
||||
--
|
||||
-- OPEN (code-level, belongs to the tip-dqo owner — NOT changed here to avoid colliding with
|
||||
-- their in-flight go-live): the HEAVY Playwright queues (scrape:pricing:fs, prolabs, compat:*)
|
||||
-- are now runnable on the residential Pi cluster too (Pis have Playwright). queuesForProfile
|
||||
-- currently excludes HEAVY from pi-fetch, so heavy IP-blocked sources (FS.com) still can't
|
||||
-- route to the Pis. A code change (let pi-fetch — or a new pi-heavy profile — dispatch HEAVY
|
||||
-- to the residential cluster) would let the cluster take those over from the Mac runners.
|
||||
|
||||
update source_registry
|
||||
set profile = 'pi-fetch',
|
||||
notes = coalesce(notes || ' | ', '')
|
||||
|| 'topology-fix 130 (2026-07-06): residential Pi cluster is the fetcher, not Erik.',
|
||||
updated_at = now()
|
||||
where source_key <> '__global__'
|
||||
and profile = 'erik-safe';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '130-adapter-pi-topology.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '130-adapter-pi-topology.sql');
|
||||
@ -1,60 +0,0 @@
|
||||
-- sql/131 — Wavelength column sync (root-cause fix for equivalence review backlog)
|
||||
--
|
||||
-- Root cause: the enricher populates the structured integer column
|
||||
-- transceivers.wavelength_tx_nm, but the equivalence spec-matcher reads the TEXT
|
||||
-- column transceivers.wavelengths via tip_extract_wavelength_nm(). 27,908 rows had
|
||||
-- wavelength_tx_nm set while wavelengths was NULL, so the matcher could not confirm
|
||||
-- wavelength on either side and dumped ~3,155 genuine equivalences into manual review.
|
||||
--
|
||||
-- Fix (no fabrication — copies existing structured data into the format the matcher
|
||||
-- expects, and keeps it in sync going forward):
|
||||
-- A) one-off backfill of transceivers.wavelengths from wavelength_tx_nm
|
||||
-- B) durable BEFORE-trigger so future ingests stay in sync
|
||||
-- C) promote pending -> auto_approved ONLY where the FULL spec-matcher criteria now
|
||||
-- agree (form_factor + speed + fiber_type + reach-tier + wavelength +/-10nm).
|
||||
-- Everything with a genuine reach-tier / fiber / wavelength difference stays pending.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- A) one-off backfill (existing structured data -> text column the matcher reads)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = wavelength_tx_nm || 'nm'
|
||||
WHERE (wavelengths IS NULL OR wavelengths = '')
|
||||
AND wavelength_tx_nm IS NOT NULL;
|
||||
|
||||
-- B) durable sync trigger for future inserts/updates
|
||||
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||
BEGIN
|
||||
IF (NEW.wavelengths IS NULL OR NEW.wavelengths = '')
|
||||
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_sync_wavelengths ON transceivers;
|
||||
CREATE TRIGGER trg_sync_wavelengths
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_sync_wavelengths_from_tx();
|
||||
|
||||
-- C) valid promotion: pending -> auto_approved where full spec now confirmed
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'auto_approved',
|
||||
confidence = GREATEST(e.confidence, 0.85),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/131: full spec confirmed after wavelength column sync',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||
|
||||
COMMIT;
|
||||
@ -1,74 +0,0 @@
|
||||
-- sql/132 — Wavelength authority correction + format normalization
|
||||
--
|
||||
-- Follow-up to sql/131. Discovery: transceivers.wavelengths holds the part-derived
|
||||
-- wavelength as a BARE number ("1470") for ~16.8k rows, which tip_extract_wavelength_nm()
|
||||
-- cannot read (it requires the "nm" suffix). Meanwhile wavelength_tx_nm disagrees with
|
||||
-- that authoritative bare number 33% of the time, and 64% of those disagreements are a
|
||||
-- systematic tx_nm=1310 enricher default. So the bare-number column is authoritative;
|
||||
-- tx_nm is not. This migration prefers the authoritative source.
|
||||
--
|
||||
-- 1) normalize authoritative bare-number wavelengths -> "NNNnm" (format only, no value change)
|
||||
-- 2) clean "N/A" text -> NULL
|
||||
-- 3) correct the few CWDM rows sql/131 backfilled with a wrong tx_nm default, using the
|
||||
-- deterministic CWDM wavelength in the part number
|
||||
-- 4) re-promote pending -> auto_approved where the FULL spec gate now agrees on
|
||||
-- authoritative data (gate already protects against tx_nm noise: both sides must
|
||||
-- agree within +/-10nm, so a wrong 1310 default can never auto-approve a real 1470).
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 1) normalize authoritative bare-number wavelengths (preserve value, add unit)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = wavelengths || 'nm'
|
||||
WHERE wavelengths ~ '^\d{3,4}$';
|
||||
|
||||
-- 2) non-values -> NULL (N/A, -, empty, etc.)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = NULL
|
||||
WHERE wavelengths IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(wavelengths) IS NULL
|
||||
AND wavelengths !~ '\d';
|
||||
|
||||
-- 3) correct CWDM rows that sql/131 filled from a wrong tx_nm default:
|
||||
-- use the deterministic CWDM wavelength encoded in the part number
|
||||
UPDATE transceivers
|
||||
SET wavelengths = (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1] || 'nm'
|
||||
WHERE wavelengths = wavelength_tx_nm || 'nm'
|
||||
AND part_number ~ '(12[6-9]|13[0-9]|14[0-9]|15[0-9]|16[01])0'
|
||||
AND (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1]::int <> wavelength_tx_nm;
|
||||
|
||||
-- 3b) upgrade the sync trigger so future ingests self-normalize:
|
||||
-- prefer an authoritative bare-number wavelength (add "nm"), fall back to tx_nm only
|
||||
-- when no textual wavelength is present at all.
|
||||
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||
BEGIN
|
||||
IF NEW.wavelengths ~ '^\d{3,4}$' THEN
|
||||
NEW.wavelengths := NEW.wavelengths || 'nm';
|
||||
ELSIF (NEW.wavelengths IS NULL OR NEW.wavelengths = '' OR tip_extract_wavelength_nm(NEW.wavelengths) IS NULL)
|
||||
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
|
||||
-- 4) valid promotion on corrected/authoritative data (same full spec gate as spec-matcher)
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'auto_approved',
|
||||
confidence = GREATEST(e.confidence, 0.85),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/132: full spec confirmed on authoritative wavelength',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||
|
||||
COMMIT;
|
||||
@ -1,46 +0,0 @@
|
||||
-- sql/133 — One-directional reach-tier substitution policy
|
||||
--
|
||||
-- Decision (Rene, 2026-07-07): a higher-reach optic may satisfy a lower-reach
|
||||
-- requirement (e.g. a 40km ER can serve where a 10km LR is asked for), never the
|
||||
-- reverse — the shorter-reach part cannot close the link budget on a longer run.
|
||||
-- ZR/coherent/amplified tiers are explicitly EXCLUDED from this automatic rule:
|
||||
-- those can overload a short-haul receiver without an attenuator, which is a real
|
||||
-- power-budget calculation, not a simple tier comparison. Those stay in manual review.
|
||||
--
|
||||
-- Result is tagged as its own status (compatible_superset), never merged into
|
||||
-- auto_approved/exact-spec matches, so downstream consumers (Fossy/Beo/Pulso, pricing)
|
||||
-- can distinguish "identical spec" from "safe higher-spec substitute" — they usually
|
||||
-- differ in price and are not a like-for-like swap.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- widen the status check (additive — existing values still valid)
|
||||
ALTER TABLE transceiver_equivalences DROP CONSTRAINT transceiver_equivalences_status_check;
|
||||
ALTER TABLE transceiver_equivalences ADD CONSTRAINT transceiver_equivalences_status_check
|
||||
CHECK (status::text = ANY (ARRAY['pending','approved','rejected','auto_approved','compatible_superset']::text[]));
|
||||
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'compatible_superset',
|
||||
confidence = GREATEST(e.confidence, 0.75),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/133: ' ||
|
||||
CASE WHEN fx.reach_meters > cx.reach_meters
|
||||
THEN 'flexoptix ('||tip_reach_tier(fx.reach_meters)||') covers competitor requirement ('||tip_reach_tier(cx.reach_meters)||')'
|
||||
ELSE 'competitor ('||tip_reach_tier(cx.reach_meters)||') covers flexoptix requirement ('||tip_reach_tier(fx.reach_meters)||')'
|
||||
END || ' — reach-tier substitute, not identical spec',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10
|
||||
AND tip_reach_tier(fx.reach_meters) <> tip_reach_tier(cx.reach_meters)
|
||||
AND tip_reach_tier(fx.reach_meters) <> 'ZR'
|
||||
AND tip_reach_tier(cx.reach_meters) <> 'ZR';
|
||||
|
||||
COMMIT;
|
||||
@ -1,20 +0,0 @@
|
||||
-- sql/134-transceiver-ff-speed-errors.sql
|
||||
-- LUPO-34: data-quality curation view for physically-impossible form_factor x speed
|
||||
-- combinations. An SFP-class module (SFP/SFP+/SFP28/XFP/GBIC/CSFP) tops out at 25-28G,
|
||||
-- so speed_gbps >= 50 on such a form factor is a misclassification (surfaced in Lupo's
|
||||
-- per-SKU price panel as flagged rows). Evidence-based (physics), NOT name-derived.
|
||||
-- Read-only; the exact corrected speed needs the vendor datasheet (data-owner task).
|
||||
-- SFP-DD / SFP112 legitimately carry >=100G and are intentionally NOT flagged.
|
||||
create or replace view v_transceiver_ff_speed_errors as
|
||||
select t.id, v.name as vendor, t.part_number, t.speed_gbps, t.form_factor
|
||||
from transceivers t
|
||||
left join vendors v on v.id = t.vendor_id
|
||||
where upper(coalesce(t.form_factor, '')) in ('SFP', 'SFP+', 'SFP28', 'XFP', 'GBIC', 'CSFP')
|
||||
and t.speed_gbps >= 50;
|
||||
|
||||
comment on view v_transceiver_ff_speed_errors is
|
||||
'LUPO-34: SFP-class parts with an impossible speed (>=50G). Data-quality curation list; exact correction needs the datasheet.';
|
||||
|
||||
-- Usage:
|
||||
-- select count(*) from v_transceiver_ff_speed_errors; -- total to curate
|
||||
-- select vendor, count(*) from v_transceiver_ff_speed_errors group by 1 order by 2 desc;
|
||||
@ -1,14 +0,0 @@
|
||||
-- Migration 135: Equivalence quality cleanup
|
||||
-- Demotes auto_approved rows that no longer meet quality gates:
|
||||
-- 1. FX product has no standard_name → can't verify match quality
|
||||
-- Applied 2026-07-11 after adding standard_name gate to catalog-reconcile.
|
||||
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'pending', updated_at = NOW()
|
||||
FROM transceivers fx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.status = 'auto_approved'
|
||||
AND fx.standard_name IS NULL;
|
||||
|
||||
-- Result: 60,278 demoted from auto_approved → pending
|
||||
-- Remaining auto_approved: 20,838 (all with standard_name)
|
||||
@ -1,227 +0,0 @@
|
||||
-- 135 Modulation/FEC enrichment + speed plausibility guard for transceivers.
|
||||
--
|
||||
-- Derived and adversarially verified via workflow wf_dc5bab06 (4 optical-standards
|
||||
-- lenses -> synthesis -> 4 skeptical reviewers). All CONFIRMED reviewer findings are
|
||||
-- folded in here: coherent >=100G speed floor (10G-ZR is NRZ, not DCO), form_factor
|
||||
-- DCO/ACO scan (CFP2-DCO evidence lives in form_factor), compact DAC/AOC/AEC suffix
|
||||
-- exclusion, bare-QSFP28-no-token ABSTAINS instead of guessing NRZ, CR4 dropped from
|
||||
-- the NRZ set, anchored MTP/MPO, NON-COHERENT guard, search_path pinned.
|
||||
--
|
||||
-- FILL-ONLY: never overwrites a non-null value, never touches data_confidence='vendor_verified'.
|
||||
-- Idempotent: safe to re-run; a fully enriched table yields 0 rows touched.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Speed plausibility. Flags only UNAMBIGUOUS junk: sub-0.09 fragments (incl. <=0;
|
||||
-- the lowest real optical rate is 100M=0.1) and >1600 (cable-part-number integers
|
||||
-- like 2.0e16). The 0.09..1600 band is deliberately kept intact: it holds legit
|
||||
-- non-Ethernet rates (FC line rates 2.125/4.25/8.5/14.025, SDI 3/6/12, CPRI 1.2288,
|
||||
-- OTN 10.7) that must not be discarded. NULL is treated as NOT implausible.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_speed_is_implausible(v numeric)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT v IS NOT NULL AND (v < 0.09 OR v > 1600);
|
||||
$$;
|
||||
|
||||
-- Separate, isolated speed sanitizer. Distinct from tip_transceivers_biu() (which
|
||||
-- recomputes product_type/form_factor and never touches speed). Sanitize-only:
|
||||
-- NULLs junk speed on write, never raises, never blocks ingestion.
|
||||
CREATE OR REPLACE FUNCTION tip_speed_plausibility_biu()
|
||||
RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.speed_gbps IS NOT NULL AND tip_speed_is_implausible(NEW.speed_gbps) THEN
|
||||
NEW.speed_gbps := NULL;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_speed_plausibility ON transceivers;
|
||||
CREATE TRIGGER trg_speed_plausibility
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_speed_plausibility_biu();
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Enrichment scope: real optics only. The part_number cable/trunk veto applies to
|
||||
-- BOTH the product_type='module' and product_type IS NULL branches, so a cable
|
||||
-- mis-classified 'module' upstream still cannot be enriched.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_in_optics_scope(p_product_type text, p_form_factor text, p_part_number text)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT
|
||||
coalesce(p_product_type,'') NOT IN ('dac','aoc','aec','accessory','switch','nic')
|
||||
-- compact suffix forms (-AOC3M, -DAC1M, -AEC2M), Flexoptix cable schemes (M4.*, S2.*),
|
||||
-- loopbacks, and explicit cable keywords are never optics. MTP/MPO is deliberately
|
||||
-- NOT vetoed here: parallel optics (DR4/SR4/DR8) legitimately carry an -MPO12 connector
|
||||
-- suffix, and real fiber trunks are already caught by ^M[0-9]./^S[0-9]./CABLE.
|
||||
AND coalesce(p_part_number,'') !~* '([-_](DAC|AOC|AEC)([-_0-9]|$)|CAB[-_]|CABLE|FN-CABLE|LOOPBACK|[-_]LB[0-9]|^M[0-9]\.|^S[0-9]\.)'
|
||||
AND (
|
||||
p_product_type = 'module'
|
||||
OR (
|
||||
p_product_type IS NULL
|
||||
AND upper(coalesce(p_form_factor,'')) IN
|
||||
('QSFP28','QSFP56','QSFP-DD','QSFP-DD800','OSFP','OSFP800','CFP2','CFP2-DCO','CFP2-ACO','CFP8','SFP-DD','SFP112','DSFP')
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Coherent detection. Regex-derived triggers are floored at >=100G (100ZR is the
|
||||
-- lowest coherent pluggable; 10G/40G-ZR are NRZ reach optics). A pre-existing
|
||||
-- coherent=true is trusted and NOT speed-floored. NON-COHERENT is excluded.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_is_coherent(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT
|
||||
p_coherent IS TRUE
|
||||
OR (
|
||||
p_speed >= 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')||' '||coalesce(p_ff,'')) !~* 'NON-?COHERENT'
|
||||
AND (
|
||||
(coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])(100|200|300|400|600|800|1600)?G?-?ZR(\+|P)?([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])DCO([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* 'OPEN-?ZR'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])COHERENT([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(16QAM|8QAM|64QAM|QPSK)'
|
||||
-- CFP2-DCO/ACO carry the only coherent evidence in form_factor (standard_name ~1% populated).
|
||||
OR upper(coalesce(p_ff,'')) ~ '(DCO|ACO|COHERENT)'
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- Modulation. NULL = ABSTAIN (unknown). Order: coherent -> PAM4-by-speed ->
|
||||
-- 100G single-lambda PAM4 -> 100G 4x25 NRZ -> abstain. Bare QSFP28 with no reach
|
||||
-- token abstains (never defaulted to NRZ). (?![0-9]) keeps SR4 from matching SR40
|
||||
-- and DR/FR from matching DR4/FR4.
|
||||
CREATE OR REPLACE FUNCTION tip_derive_modulation(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
|
||||
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN tip_is_coherent(p_std, p_pn, p_ff, p_speed, p_coherent) THEN CASE
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?QPSK' THEN 'DP-QPSK'
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?16QAM' THEN 'DP-16QAM'
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(8QAM|64QAM)' THEN 'Coherent'
|
||||
WHEN p_speed >= 400 THEN 'DP-16QAM' -- 400ZR/800ZR/DCO default
|
||||
WHEN p_speed = 100 THEN 'DP-QPSK' -- 100ZR is QPSK
|
||||
ELSE 'Coherent' -- 200G/300G constellation ambiguous -> generic
|
||||
END
|
||||
WHEN p_speed IN (200,400,800,1600) THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN NULL -- conflicting single-lambda AND 4x25 tokens -> abstain
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
|
||||
THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND upper(coalesce(p_ff,'')) IN ('SFP-DD','SFP112','DSFP')
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) !~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN 'NRZ'
|
||||
ELSE NULL -- bare QSFP28 / no token / sub-100G / non-standard speed -> abstain
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- FEC type. Reads modulation. Abstains (NULL) for 100G NRZ LR4/ER4/CWDM4/CLR4/4WDM
|
||||
-- (no mandatory FEC); that assertion is carried by inbuilt_fec=false, not a fake fec_type.
|
||||
CREATE OR REPLACE FUNCTION tip_derive_fec_type(p_modulation text, p_std text, p_pn text, p_speed numeric)
|
||||
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN 'SD-FEC'
|
||||
WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN 'RS-FEC (RS(544,514) KP4)'
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN 'RS-FEC (RS(528,514) CL91)'
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION tip_derive_inbuilt_fec(p_modulation text, p_std text, p_pn text, p_speed numeric)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN true
|
||||
WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN true
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN true
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(LR4|ER4L?|CWDM4|CLR4|4WDM)(?![0-9])' THEN false
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Idempotent fill-only enrichment. Returns total per-statement rows touched
|
||||
-- (a row filled across all three steps counts up to 3x). Fully enriched -> 0.
|
||||
-- One SELECT call runs all three steps in a single transaction, so APPLY-3 reads
|
||||
-- the modulation APPLY-2 just set.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION fn_tip_enrich_modulation_fec()
|
||||
RETURNS integer LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
DECLARE n1 int := 0; n2 int := 0; n3 int := 0;
|
||||
BEGIN
|
||||
-- 1) coherent flag (set true only; never flip an existing true->false)
|
||||
UPDATE transceivers t
|
||||
SET coherent = true,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['coherent']) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
WHERE coalesce(t.data_confidence,'') <> 'vendor_verified'
|
||||
AND t.speed_gbps IS NOT NULL
|
||||
AND t.coherent IS DISTINCT FROM true
|
||||
AND tip_in_optics_scope(t.product_type, t.form_factor, t.part_number)
|
||||
AND tip_is_coherent(t.standard_name, t.part_number, t.form_factor, t.speed_gbps, t.coherent);
|
||||
GET DIAGNOSTICS n1 = ROW_COUNT;
|
||||
|
||||
-- 2) modulation (fill-only)
|
||||
UPDATE transceivers t
|
||||
SET modulation = m.val,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['modulation']) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
FROM (
|
||||
SELECT ctid AS cid, tip_derive_modulation(standard_name, part_number, form_factor, speed_gbps, coherent) AS val
|
||||
FROM transceivers
|
||||
WHERE modulation IS NULL
|
||||
AND coalesce(data_confidence,'') <> 'vendor_verified'
|
||||
AND speed_gbps IS NOT NULL
|
||||
AND tip_in_optics_scope(product_type, form_factor, part_number)
|
||||
) m
|
||||
WHERE t.ctid = m.cid AND m.val IS NOT NULL;
|
||||
GET DIAGNOSTICS n2 = ROW_COUNT;
|
||||
|
||||
-- 3) fec_type + inbuilt_fec (reads modulation set in step 2)
|
||||
UPDATE transceivers t
|
||||
SET fec_type = CASE WHEN t.fec_type IS NULL AND f.fec IS NOT NULL THEN f.fec ELSE t.fec_type END,
|
||||
inbuilt_fec = CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN f.inb ELSE t.inbuilt_fec END,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(
|
||||
coalesce(t.enriched_fields,'{}'::text[])
|
||||
|| CASE WHEN t.fec_type IS NULL AND f.fec IS NOT NULL THEN ARRAY['fec_type'] ELSE '{}'::text[] END
|
||||
|| CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN ARRAY['inbuilt_fec'] ELSE '{}'::text[] END) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
FROM (
|
||||
SELECT ctid AS cid,
|
||||
tip_derive_fec_type(modulation, standard_name, part_number, speed_gbps) AS fec,
|
||||
tip_derive_inbuilt_fec(modulation, standard_name, part_number, speed_gbps) AS inb
|
||||
FROM transceivers
|
||||
WHERE coalesce(data_confidence,'') <> 'vendor_verified'
|
||||
AND speed_gbps IS NOT NULL
|
||||
AND (fec_type IS NULL OR inbuilt_fec IS NULL)
|
||||
AND tip_in_optics_scope(product_type, form_factor, part_number)
|
||||
) f
|
||||
WHERE t.ctid = f.cid
|
||||
AND ((t.fec_type IS NULL AND f.fec IS NOT NULL) OR (t.inbuilt_fec IS NULL AND f.inb IS NOT NULL));
|
||||
GET DIAGNOSTICS n3 = ROW_COUNT;
|
||||
|
||||
RETURN n1 + n2 + n3;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Run enrichment: SELECT fn_tip_enrich_modulation_fec();
|
||||
@ -1,48 +0,0 @@
|
||||
-- 136 Make speed_gbps nullable + one-time reversible cleanup of junk speed.
|
||||
-- Requires 135 (tip_speed_is_implausible, trg_speed_plausibility) applied first.
|
||||
--
|
||||
-- speed_gbps was NOT NULL, which (a) forced meaningless values onto cables/accessories
|
||||
-- and (b) made the guard trigger unusable: NULLing junk on write would abort the insert.
|
||||
-- "Unknown speed" is honestly NULL, so we drop the constraint, then NULL the ~4708
|
||||
-- unambiguous-junk rows (sub-0.09 fragments incl. <=0, and >1600 cable-part-number
|
||||
-- integers). 87% are dac/aoc/accessory where speed is meaningless anyway. The old value
|
||||
-- is snapshotted by primary key so the change is fully reversible. The 0.09..1600 gray
|
||||
-- band (~355 legit FC/SDI/CPRI/OTN line rates) is intentionally left untouched.
|
||||
--
|
||||
-- PREVIEW before running (nothing legit should appear):
|
||||
-- SELECT speed_gbps, count(*) FROM transceivers
|
||||
-- WHERE tip_speed_is_implausible(speed_gbps) GROUP BY 1 ORDER BY 1;
|
||||
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE transceivers ALTER COLUMN speed_gbps DROP NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tip_speed_gbps_rollback (
|
||||
id uuid PRIMARY KEY,
|
||||
old_speed_gbps numeric,
|
||||
backed_up_at timestamptz DEFAULT now()
|
||||
);
|
||||
|
||||
INSERT INTO tip_speed_gbps_rollback (id, old_speed_gbps)
|
||||
SELECT id, speed_gbps
|
||||
FROM transceivers
|
||||
WHERE tip_speed_is_implausible(speed_gbps)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
UPDATE transceivers
|
||||
SET speed_gbps = NULL
|
||||
WHERE tip_speed_is_implausible(speed_gbps);
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- REVERSE (maintenance window, no concurrent writers). Disable the guard so it does
|
||||
-- not re-NULL the junk being restored:
|
||||
-- BEGIN;
|
||||
-- ALTER TABLE transceivers DISABLE TRIGGER trg_speed_plausibility;
|
||||
-- UPDATE transceivers t SET speed_gbps = r.old_speed_gbps
|
||||
-- FROM tip_speed_gbps_rollback r
|
||||
-- WHERE t.id = r.id AND t.speed_gbps IS NULL;
|
||||
-- ALTER TABLE transceivers ENABLE TRIGGER trg_speed_plausibility;
|
||||
-- -- optional, only if every row is non-null again:
|
||||
-- -- ALTER TABLE transceivers ALTER COLUMN speed_gbps SET NOT NULL;
|
||||
-- COMMIT;
|
||||
@ -1,13 +0,0 @@
|
||||
-- 137 Make v_transceiver_speed_errors count only genuine optics.
|
||||
--
|
||||
-- After 136 nulled junk speed, the old definition (speed_gbps IS NULL OR <= 0)
|
||||
-- counted 4708 rows, but 4112 of them are dac/aoc/aec/accessory where speed_gbps
|
||||
-- is meaningless and NULL is correct. A speed error is a real optic (module) with
|
||||
-- a missing or physically-impossible speed. This drops the count to the ~596 real
|
||||
-- modules that still need a recoverable speed.
|
||||
|
||||
CREATE OR REPLACE VIEW v_transceiver_speed_errors AS
|
||||
SELECT id, part_number, speed_gbps, form_factor, product_type
|
||||
FROM transceivers
|
||||
WHERE coalesce(product_type,'') NOT IN ('dac','aoc','aec','accessory','switch','nic')
|
||||
AND (speed_gbps IS NULL OR speed_gbps <= 0 OR speed_gbps > 1600);
|
||||
@ -1,83 +0,0 @@
|
||||
-- sql/138-transceiver-speed-drift-guard.sql
|
||||
-- Closes a gap in the sql/125+126 "no physically-impossible compatibility rows"
|
||||
-- invariant: compat_speed_guard() (126) only fires on INSERT/UPDATE of the
|
||||
-- `compatibility` table itself. It never re-validates existing 'compatible'
|
||||
-- spec_match rows when a TRANSCEIVER's speed_gbps changes later (e.g. the daily
|
||||
-- enrich-modulation-fec.sh self-heal, sql/135, recovering speed from part-number
|
||||
-- tokens for previously NULL/junk rows). Found live 2026-07-12: 189 compat rows
|
||||
-- had gone physically-impossible because their transceiver's speed_gbps was
|
||||
-- updated (all sampled rows dated today, i.e. by the 03:30 cron) after the
|
||||
-- compat row was already written as 'compatible'.
|
||||
--
|
||||
-- Fix mirrors 126 exactly, just triggered from the other table: AFTER UPDATE OF
|
||||
-- speed_gbps on transceivers, re-validate every 'compatible' spec_match
|
||||
-- compatibility row referencing that transceiver against the same evidence-based
|
||||
-- ceiling (v_switch_effective_max) used by v_compat_speed_errors. Curated rows
|
||||
-- (vendor_matrix/vendor_compat/tested/datasheet) are never touched. Idempotent,
|
||||
-- safe to re-run.
|
||||
|
||||
-- Backup of affected rows before remediation (audit trail, matches
|
||||
-- tip_speed_gbps_rollback precedent from sql/136).
|
||||
create table if not exists compat_speed_drift_backup_20260712 as
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
select c.*, e.effective_max_gbps, t.speed_gbps as tx_speed_gbps, now() as backed_up_at
|
||||
from compatibility c
|
||||
join em e on e.switch_id = c.switch_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- One-time remediation: flip the rows found live today (same logic as 125's
|
||||
-- Remediation B, using the full evidence-based ceiling incl. curated bump).
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)'
|
||||
else c.notes || ' | auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)' end
|
||||
from em e, transceivers t
|
||||
where e.switch_id = c.switch_id
|
||||
and t.id = c.transceiver_id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- Durable fix: re-validate on transceiver speed change.
|
||||
create or replace function compat_speed_guard_on_transceiver_change() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
if NEW.speed_gbps is distinct from OLD.speed_gbps and NEW.speed_gbps is not null then
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)'
|
||||
else c.notes || ' | auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)' end
|
||||
from v_switch_effective_max e
|
||||
where e.switch_id = c.switch_id
|
||||
and c.transceiver_id = NEW.id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and NEW.speed_gbps > e.effective_max_gbps;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function compat_speed_guard_on_transceiver_change() is
|
||||
'AFTER UPDATE OF speed_gbps on transceivers: re-validates existing compatible spec_match rows for this transceiver against the evidence-based switch ceiling, closing the drift gap left by compat_speed_guard() (126), which only fires on compatibility writes. Never touches curated rows.';
|
||||
|
||||
drop trigger if exists trg_compat_speed_guard_on_transceiver_change on transceivers;
|
||||
create trigger trg_compat_speed_guard_on_transceiver_change
|
||||
after update of speed_gbps on transceivers
|
||||
for each row execute function compat_speed_guard_on_transceiver_change();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '138-transceiver-speed-drift-guard.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '138-transceiver-speed-drift-guard.sql');
|
||||
@ -1,130 +0,0 @@
|
||||
-- sql/139-equivalence-pending-triage.sql
|
||||
-- Set-based port of scheduler.ts's evaluateEquivalenceResearch() (line ~85),
|
||||
-- applied once to the 289,681-row pending backlog in transceiver_equivalences.
|
||||
--
|
||||
-- Root cause of the backlog: the daily "maintenance:re-research-equivalences" job
|
||||
-- (scheduler.ts:3117) only evaluates rows where re_research_due_at IS NOT NULL AND
|
||||
-- <= NOW(). catalog-reconcile.ts inserts new 'pending' rows WITHOUT setting
|
||||
-- re_research_due_at (NULL by default) -> the job that was designed to drain the
|
||||
-- queue never saw fresh pending rows. This migration (a) triages the existing
|
||||
-- backlog once, set-based, using the exact same scoring rules as the JS
|
||||
-- evaluator, and (b) is paired with code fixes (catalog-reconcile.ts sets
|
||||
-- re_research_due_at on insert; scheduler.ts's reject branch preserves it for
|
||||
-- "insufficient evidence" rejects) so the daily job keeps draining future inserts
|
||||
-- on its own.
|
||||
--
|
||||
-- Decision rules (identical to evaluateEquivalenceResearch, ported 1:1):
|
||||
-- no recent competitor price observation (45d) -> reject, no re-research
|
||||
-- form_factor or speed_gbps mismatch/missing -> reject, no re-research
|
||||
-- wavelength diff >15nm / fiber mismatch / reach -> reject, no re-research
|
||||
-- ratio <0.85 (all three "hard" if evidence exists)
|
||||
-- wavelength / fiber_type / reach_meters MISSING -> reject, WITH re-research
|
||||
-- (on either side, and no hard mismatch above) in 21 days (data-gap,
|
||||
-- not a wrong match --
|
||||
-- give scrapers time to
|
||||
-- enrich, then recheck)
|
||||
-- confidence (score/115) >= 0.73 -> auto_approved
|
||||
-- else -> reject, no re-research
|
||||
--
|
||||
-- Idempotent for its WHERE clause (status='pending' AND re_research_due_at IS
|
||||
-- NULL); re-running after the code fix lands is a no-op since new pending rows
|
||||
-- will already carry a re_research_due_at.
|
||||
|
||||
create table if not exists equivalence_pending_triage_backup_20260713 as
|
||||
select *, now() as backed_up_at
|
||||
from transceiver_equivalences
|
||||
where status = 'pending' and re_research_due_at is null;
|
||||
|
||||
with normalized as (
|
||||
select eq.id,
|
||||
nullif(upper(trim(fx.form_factor)),'') as fx_form, nullif(upper(trim(cp.form_factor)),'') as cp_form,
|
||||
fx.speed_gbps as fx_speed, cp.speed_gbps as cp_speed,
|
||||
nullif(upper(trim(fx.standard_name)),'') as fx_std, nullif(upper(trim(cp.standard_name)),'') as cp_std,
|
||||
nullif(upper(trim(fx.fiber_type)),'') as fx_fiber, nullif(upper(trim(cp.fiber_type)),'') as cp_fiber,
|
||||
fx.reach_meters as fx_reach, cp.reach_meters as cp_reach,
|
||||
(regexp_match(fx.wavelengths, '(\d{3,4})'))[1]::numeric as fx_nm,
|
||||
(regexp_match(cp.wavelengths, '(\d{3,4})'))[1]::numeric as cp_nm,
|
||||
exists(select 1 from price_observations po where po.transceiver_id=eq.competitor_id and po.time > now() - interval '45 days') as has_recent_price
|
||||
from transceiver_equivalences eq
|
||||
join transceivers fx on fx.id = eq.flexoptix_id
|
||||
join transceivers cp on cp.id = eq.competitor_id
|
||||
where eq.status = 'pending' and eq.re_research_due_at is null
|
||||
),
|
||||
scored as (
|
||||
select *,
|
||||
(case when fx_form is not null and cp_form is not null and fx_form=cp_form then 25 else 0 end
|
||||
+ case when fx_speed is not null and cp_speed is not null and fx_speed=cp_speed then 20 else 0 end
|
||||
+ case when fx_std is not null and cp_std is not null and fx_std=cp_std then 30 else 0 end
|
||||
+ case when fx_nm is not null and cp_nm is not null then
|
||||
case when abs(fx_nm-cp_nm)<=15 then 20 else -20 end
|
||||
else 0 end
|
||||
+ case when fx_fiber is not null and cp_fiber is not null then
|
||||
case when fx_fiber=cp_fiber then 10 else -15 end
|
||||
else 0 end
|
||||
+ case when fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 then
|
||||
case when least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) >= 0.85 then 10 else -15 end
|
||||
else 0 end) as raw_score,
|
||||
(fx_nm is null or cp_nm is null) as wl_missing,
|
||||
(fx_fiber is null or cp_fiber is null) as fiber_missing,
|
||||
(fx_reach is null or cp_reach is null or fx_reach<=0 or cp_reach<=0) as reach_missing,
|
||||
(fx_nm is not null and cp_nm is not null and abs(fx_nm-cp_nm)>15) as wl_mismatch,
|
||||
(fx_fiber is not null and cp_fiber is not null and fx_fiber<>cp_fiber) as fiber_mismatch,
|
||||
(fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 and least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) < 0.85) as reach_mismatch,
|
||||
(fx_form is null or cp_form is null or fx_form<>cp_form) as form_bad,
|
||||
(fx_speed is null or cp_speed is null or fx_speed<>cp_speed) as speed_bad
|
||||
from normalized
|
||||
),
|
||||
decided as (
|
||||
select *,
|
||||
greatest(0, least(1, raw_score::numeric/115)) as confidence,
|
||||
(not has_recent_price) as no_price,
|
||||
(wl_mismatch or fiber_mismatch or reach_mismatch or form_bad or speed_bad) as critical_mismatch,
|
||||
(wl_missing or fiber_missing or reach_missing) as missing_evidence
|
||||
from scored
|
||||
)
|
||||
update transceiver_equivalences eq
|
||||
set status = case
|
||||
when d.no_price then 'rejected'
|
||||
when d.critical_mismatch then 'rejected'
|
||||
when d.missing_evidence then 'rejected'
|
||||
when d.confidence >= 0.73 then 'auto_approved'
|
||||
else 'rejected'
|
||||
end,
|
||||
confidence = case when d.no_price then 0 else d.confidence end,
|
||||
reviewed_by = 'automated-research-bulk-139',
|
||||
reviewed_at = now(),
|
||||
reject_reason = case
|
||||
when d.no_price then 'automated research: competitor has no recent price observation'
|
||||
when d.critical_mismatch then 'automated research: technical mismatch'
|
||||
when d.missing_evidence then 'automated research: insufficient technical evidence'
|
||||
when d.confidence >= 0.73 then null
|
||||
else 'automated research: confidence below approval threshold'
|
||||
end,
|
||||
re_research_due_at = case
|
||||
when (not d.no_price) and (not d.critical_mismatch) and d.missing_evidence and d.confidence < 0.73
|
||||
then now() + interval '21 days'
|
||||
else null
|
||||
end,
|
||||
re_researched_at = now(),
|
||||
match_notes = case when eq.match_notes is null or eq.match_notes = ''
|
||||
then 'bulk-triage(139) ' || now()::date
|
||||
else eq.match_notes || ' | bulk-triage(139) ' || now()::date end,
|
||||
updated_at = now()
|
||||
from decided d
|
||||
where eq.id = d.id;
|
||||
|
||||
-- Mirror the daily job's side effect on approve: mark the Flexoptix product as
|
||||
-- competitor_verified (matches scheduler.ts's confirm branch).
|
||||
update transceivers t
|
||||
set competitor_verified = true,
|
||||
competitor_verified_at = coalesce(t.competitor_verified_at, now())
|
||||
from transceiver_equivalences eq
|
||||
where eq.flexoptix_id = t.id
|
||||
and eq.status = 'auto_approved'
|
||||
and eq.reviewed_by = 'automated-research-bulk-139'
|
||||
and eq.reviewed_at > now() - interval '5 minutes'
|
||||
and t.competitor_verified = false;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '139-equivalence-pending-triage.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '139-equivalence-pending-triage.sql');
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user