Compare commits
No commits in common. "main" and "reconcile-2026-06-04" have entirely different histories.
main
...
reconcile-
105
.github/scripts/changelog-draft.py
vendored
105
.github/scripts/changelog-draft.py
vendored
@ -1,105 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Draft CHANGELOG_PENDING.md entries from Conventional Commit messages
|
||||
pushed since the last run. Deterministic -- no LLM call, no network access.
|
||||
Human review still required before folding an entry into CHANGELOG.md.
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
INCLUDE_TYPES = {
|
||||
"feat": "Added",
|
||||
"fix": "Fixed",
|
||||
"refactor": "Changed",
|
||||
"perf": "Changed",
|
||||
"security": "Fixed",
|
||||
}
|
||||
SKIP_TYPES = {"chore", "ci", "test", "docs", "style", "merge"}
|
||||
|
||||
# Anything mentioning internal infra by name/address is dropped rather than
|
||||
# drafted -- better a missing entry (caught in human review) than a leaked one.
|
||||
INTERNAL_PATTERN = re.compile(
|
||||
r"\b(erik|gitea\.context-x|192\.168\.|10\.10\.0\.|opnsense|ssh |systemd|pm2 |crontab)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
PENDING_HEADER = (
|
||||
"# Pending Changelog Entries\n\n"
|
||||
"Drafted automatically from commit messages since the last processed "
|
||||
"push. Review, edit as needed, and fold into CHANGELOG.md -- then clear "
|
||||
"this file.\n\n"
|
||||
"<!-- PENDING_ENTRIES -->\n"
|
||||
)
|
||||
|
||||
|
||||
def run(cmd):
|
||||
return subprocess.check_output(cmd, shell=True, text=True).strip()
|
||||
|
||||
|
||||
def main():
|
||||
before = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
after = sys.argv[2] if len(sys.argv) > 2 else "HEAD"
|
||||
if not before or set(before) == {"0"}:
|
||||
print("No usable commit range (first push on branch) -- skipping.")
|
||||
return
|
||||
|
||||
try:
|
||||
log = run("git log {}..{} --format='%s'".format(before, after))
|
||||
except subprocess.CalledProcessError:
|
||||
print("Could not diff commit range -- skipping.")
|
||||
return
|
||||
if not log:
|
||||
print("No new commits.")
|
||||
return
|
||||
|
||||
buckets = {"Added": [], "Fixed": [], "Changed": []}
|
||||
for subject in log.splitlines():
|
||||
m = re.match(r"^(\w+)(\([\w./-]+\))?!?:\s*(.+)$", subject)
|
||||
if not m:
|
||||
continue
|
||||
ctype, _, desc = m.groups()
|
||||
ctype = ctype.lower()
|
||||
if ctype in SKIP_TYPES or ctype not in INCLUDE_TYPES:
|
||||
continue
|
||||
if INTERNAL_PATTERN.search(desc):
|
||||
continue
|
||||
buckets[INCLUDE_TYPES[ctype]].append(desc.strip())
|
||||
|
||||
if not any(buckets.values()):
|
||||
print("Nothing changelog-worthy in this push.")
|
||||
return
|
||||
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
lines = ["## Pending -- {}\n".format(date)]
|
||||
for section in ("Added", "Fixed", "Changed"):
|
||||
if buckets[section]:
|
||||
lines.append("### {}".format(section))
|
||||
for item in buckets[section]:
|
||||
lines.append("- {}".format(item))
|
||||
lines.append("")
|
||||
entry = "\n".join(lines).rstrip() + "\n\n"
|
||||
|
||||
try:
|
||||
with open("CHANGELOG_PENDING.md", "r") as f:
|
||||
existing = f.read()
|
||||
if not existing.strip():
|
||||
existing = PENDING_HEADER # pre-existing but empty placeholder file
|
||||
except FileNotFoundError:
|
||||
existing = PENDING_HEADER
|
||||
|
||||
marker = "<!-- PENDING_ENTRIES -->\n"
|
||||
if marker in existing:
|
||||
head, tail = existing.split(marker, 1)
|
||||
updated = head + marker + entry + tail.lstrip("\n")
|
||||
else:
|
||||
updated = existing.rstrip("\n") + "\n\n" + entry
|
||||
|
||||
with open("CHANGELOG_PENDING.md", "w") as f:
|
||||
f.write(updated)
|
||||
|
||||
print("Wrote entry:\n" + entry)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
39
.github/workflows/changelog-draft.yml
vendored
39
.github/workflows/changelog-draft.yml
vendored
@ -1,39 +0,0 @@
|
||||
name: changelog-draft
|
||||
|
||||
# Drafts CHANGELOG_PENDING.md entries from Conventional Commit subjects on
|
||||
# every push to main. Deterministic -- regex-parses commit subjects only,
|
||||
# no LLM call, no network access. Buckets feat/fix/refactor/perf/security
|
||||
# into Added/Fixed/Changed; skips chore/ci/test/docs/style/merge commits and
|
||||
# anything mentioning internal infra hostnames/addresses. Never touches
|
||||
# CHANGELOG.md itself -- human review still required to fold an entry in.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
changelog-draft:
|
||||
runs-on: ubuntu-latest
|
||||
if: "!contains(github.event.head_commit.message, '[skip ci]')"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Draft pending changelog entry
|
||||
run: python3 .github/scripts/changelog-draft.py "${{ github.event.before }}" "${{ github.sha }}"
|
||||
|
||||
- name: Commit draft if changed
|
||||
run: |
|
||||
if ! git diff --quiet CHANGELOG_PENDING.md 2>/dev/null; then
|
||||
git config user.name "changelog-bot"
|
||||
git config user.email "changelog-bot@context-x.org"
|
||||
git add CHANGELOG_PENDING.md
|
||||
git commit -m "chore(changelog): draft pending entries [skip ci]"
|
||||
git push origin HEAD:main
|
||||
else
|
||||
echo "No changelog-worthy commits in this push."
|
||||
fi
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -21,8 +21,3 @@ storage-*/
|
||||
# Local credentials (never commit)
|
||||
.tip/.env
|
||||
run-fs-scraper-mac.sh.local
|
||||
|
||||
# Training / SFT datasets — bulk data + potential internal pricing/market intel, keep out of git
|
||||
*.jsonl
|
||||
training-data/runpod/**/*.jsonl
|
||||
training-data/*.jsonl
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
# transceiver-db pre-push scan allowlist. ERE, one pattern per line, matched against finding lines.
|
||||
# This repo is Gitea-internal and is never published publicly; entries below are confirmed false positives.
|
||||
# Risk note: a low-entropy secret sharing a line with an allowlisted token would be masked here.
|
||||
# High-entropy secrets are still caught by the gitleaks pass, which ignores this allowlist.
|
||||
# If this repo is ever made public, drop this file and run the internas sanitizer first.
|
||||
# --- vendor name that is legitimate market-intelligence dataset subject matter ---
|
||||
flexoptix
|
||||
# --- owner homelab infra referenced in runbooks, docs and deploy scripts ---
|
||||
82\.165\.222\.127
|
||||
192\.168\.178\.
|
||||
10\.10\.0\.
|
||||
\berik\b
|
||||
\bclaudi\b
|
||||
renefichtmueller
|
||||
fearghas
|
||||
/opt/[a-z]
|
||||
fichtmueller\.org
|
||||
context-x\.org
|
||||
# --- sibling project codenames referenced in the cross-project sync dev journal ---
|
||||
\bmagatama
|
||||
peercortex
|
||||
switchblade
|
||||
shieldx
|
||||
papercortex
|
||||
ctxevent
|
||||
ctxmeet
|
||||
nognet\.net
|
||||
\bpulso\b
|
||||
# --- env-resolved DB URIs and passwords, no literal secret present ---
|
||||
postgres(ql)?://\$\{
|
||||
\$\{DB_PASS\}
|
||||
process\.env\.PGPASSWORD
|
||||
# --- placeholder secret in example config ---
|
||||
CHANGE_ME
|
||||
# --- ML tokenizer special tokens, not credentials ---
|
||||
special_tokens
|
||||
pad_token
|
||||
@ -1,12 +1,6 @@
|
||||
# TIP Changelog
|
||||
|
||||
Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}`
|
||||
{"d":"2026-07-17","t":"FIX","m":"Equivalence pipeline audit: found 2 more sources writing straight to auto_approved without the recent-price/standard_name discrimination catalog-reconcile.ts uses (spec-matcher.ts flat-0.85, no gate) vs one that already had it right (maintenance:find-equivalences enhanced mode implicitly requires standard_name to hit its 0.85 bar and pre-filters recent price). Fixed spec-matcher.ts to insert pending with re_research_due_at=NOW() instead of bypassing review; one-time reclassification of the 1,455 pre-existing spec-basis auto_approved rows (sql/140): 972 held up, 483 demoted (some permanent mismatch, some re-research in 21d for missing evidence)."}
|
||||
{"d":"2026-07-17","t":"FEAT","m":"Switch-side drift-guard (sql/141): closed the third direction of the physically-impossible-compat invariant (compatibility writes=126, transceiver speed changes=138, now switches.max_speed_gbps/ports_config changes=141). Currently a no-op live (nothing updates those columns today outside the one-time sql/125 backfill) -- preventive, ahead of the planned switch datasheet-enrichment automation. Verified with a synthetic in-transaction test (1250 rows correctly flagged, rolled back)."}
|
||||
{"d":"2026-07-12","t":"FIX","m":"Switch-DB compat_speed_guard drift gap closed (sql/138): the sql/126 trigger only validated compatibility writes, so a later transceivers.speed_gbps update (e.g. the daily enrich-modulation-fec.sh recovery) never re-checked existing compatible spec_match rows against the switch speed ceiling. Found live: 189 rows had drifted physically-impossible again. One-time remediation (189 rows -> incompatible, backed up to compat_speed_drift_backup_20260712) plus a new AFTER UPDATE OF speed_gbps trigger on transceivers closes the gap for both write directions. check_switch_quality() passes again."}
|
||||
{"d":"2026-07-11","t":"FEAT","m":"Modulation/FEC enrichment (sql/135): IMMUTABLE helpers derive modulation (NRZ/PAM4/DP-16QAM/DP-QPSK/Coherent), fec_type and inbuilt_fec from speed_gbps + form_factor + part_number/standard_name regex, incl. coherent detection (>=100G-floored ZR/DCO/OpenZR/QAM + CFP2-DCO/ACO via form_factor). Fill-only + idempotent via fn_tip_enrich_modulation_fec(); daily self-healing cron 03:30 (scripts/enrich-modulation-fec.sh). Modulation coverage >=100G rose from 3-8 percent to 46 percent (400G 59, 800G 56, 1600G 100); 4224 rows enriched. Rules built by a 4-lens optical-standards workflow and adversarially verified — the 10G-ZR mislabel, CFP2-DCO miss, AOC/DAC scope leak and bare-QSFP28 over-guess were all caught and fixed before apply. vendor_verified rows never overwritten; enriched rows tagged data_confidence=enriched_estimated + enriched_fields."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"speed_gbps plausibility guard + cleanup (sql/136): ~22 percent of rows carried junk speed (2129 at <=0, 167 astronomical >1600 values, ~2400 sub-0.09 fragments) from cable/MTP-trunk part numbers parsed into speed. speed_gbps made nullable (unknown speed is honestly NULL); tip_speed_is_implausible() + BEFORE trigger trg_speed_plausibility sanitize junk to NULL on write and never abort ingestion; one-time cleanup nulled 4708 rows (id-keyed reversible snapshot in tip_speed_gbps_rollback). The 0.09..1600 gray band (~355 legit FC/SDI/CPRI/OTN line rates) is intentionally preserved. 87 percent of nulled rows are dac/aoc/accessory where speed is meaningless."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"v_transceiver_speed_errors honest metric (sql/137): after 136 the view counted 4708 NULL-speed rows, but 4112 are dac/aoc/accessory where NULL is correct. It now flags only real optics (product_type not in dac/aoc/aec/accessory/switch/nic) with NULL/<=0/>1600 speed -> 596 genuine module gaps, down from 2134 mixed."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"Equivalences Explorer: new dashboard tab '🔀 Equivalences' — search 63,362 cross-brand mappings (46 vendors, 7,516 competitor products → 846 Flexoptix alternatives, Ø 93.9% confidence). APIs: GET /api/equivalences (search), /api/equivalences/transceiver/:id (per-product), /api/equivalences/stats, /api/equivalences/top-vendors. Transceiver detail modal now shows equivalences panel (FX alternatives or competitor products) + SVG price history sparklines (30-day, per source vendor) from 392k+ price observations."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"LinkedIn Distribution Status: Blog tab shows DRY_RUN badge, posted/dry_run/skipped/failed counters, history table with live URN links. GET /api/blog/linkedin/history reads blog_linkedin_distribution table + detects DRY_RUN mode from ecosystem config."}
|
||||
{"d":"2026-05-14","t":"FEAT","m":"MCP Server: 2 new tools — find_equivalences (search 63k+ verified cross-brand mappings with confidence filter, returns FX alternatives + competitor matches formatted for LLM) + get_price_history (392k+ obs, daily series, per-vendor min/max/avg, cheapest source identification). Total: 21 MCP tools."}
|
||||
@ -302,49 +296,3 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA
|
||||
{"d":"2026-04-28","t":"FIX","m":"Product verification pipeline: image crawls now mark image_verified/image_verified_url, scraped product pages mark details_verified/details_source_url, maintenance reconcile backfills old product URLs/images/details, and --backfill-images exposes the existing image crawler via scraper CLI. Migration 102 reconciles existing data."}
|
||||
{"d":"2026-04-28","t":"FIX","m":"Blog Engine Hot Topics: diversified ranking with refresh shuffle/source caps/already-created-topic demotion, plus richer LLM context briefings passed into topic expansion and master-draft context via custom_title/additional_context."}
|
||||
{"d":"2026-04-29","t":"FEAT","m":"TIPLLM robot learning loop: verification robot controller writes status, TIPLLM plans, queue dry-runs/enqueues and crawler outcomes into the Gitea-backed TIP training pool; learning-pool build imports qa-pairs from TIP_TRAINING_REPO into the tip_llm lane. Removed hardcoded Gitea token fallback; existing git remotes or env tokens are used."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "Blog fo-blog LLM reachable again: routed to Mac Studio Ollama over the Erik<->home WireGuard tunnel via new BLOG_OLLAMA_URL (search/embeddings stay Erik-local on OLLAMA_URL). Exposed Mac Ollama on 0.0.0.0 via LaunchAgent plist; adopted training candidate as fo-blog-v45. Verified model discovery + live inference over WG."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Blog Ollama high-availability: dual SSH tunnels from Erik to Mac Studio loopback Ollama over independent transports (primary WireGuard localhost:11436, backup Cloudflare localhost:11435; both systemd, Restart=always, enabled on boot). Blog client auto-fails-over primary->backup (re-probed at startup + every 60s, self-heals to primary). Robust against the Mac training pipeline restarting Ollama or changing its bind."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Hype-cycle market-signals endpoint (/api/hype-cycle/market-signals): data-driven per-technology market signal score (0-100) + drivers + buy/hold recommendation from the adoption model and real SKU/vendor/price data. Powers the dashboard Market Context cards (were empty)."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "Vendor /market-share + /intelligence were shadowed by /:id (404) with latent SQL bugs (round(double,int); non-existent po.id). Routed past /:id via next() fallthrough; fixed SQL (numeric casts, COUNT(*)). Now 200 with real data."}
|
||||
{"d": "2026-06-04", "t": "FEAT", "m": "Vendor reliability endpoint (/api/vendors/reliability): per-vendor freshness/frequency/coverage sub-scores + composite reliability score from real data (211 vendors). Populates the reliability badges on vendor cards."}
|
||||
{"d": "2026-06-04", "t": "FIX", "m": "LLM-gym disk-watchdog keep-list now also protects the newest candidate per lane (not just adopted vN/-rN), fixing the churn that fully deleted fo-blog when the quality gate blocked adoption."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"Supply-Squeeze Preis-Momentum: paarweiser per-SKU Vergleich statt Aggregat — eliminiert Katalog-Komposition-Bias (400G OSFP +151%→0%, Signale 21→8). Nur SKUs in beiden Perioden, Median der per-SKU-Deltas, Nicht-Transceiver/AOC/Switch-Filter."}
|
||||
{"d":"2026-06-06","t":"AI","m":"Research Robot: Handlungsempfehlungen + Action-Buttons. API klassifiziert fehlschlagende Jobs (auth/401, network, no-handler) → Empfehlung+Aktionen. POST /action: dispatch/pause/resume (pgboss, whitelist-validiert, reversibel). Dashboard rendert Empfehlungs-Karten mit Buttons. sync:flexoptix-catalog→critical auth, bietet Token-Hilfe+Pause."}
|
||||
{"d":"2026-06-06","t":"FEAT","m":"FS.COM Wettbewerber-Lagerbestand pro Technologie in Sales-Velocity-Tabelle (/api/stock/competitor-by-tech), Ampel-Färbung vs. Monatsbedarf."}
|
||||
{"d":"2026-06-06","t":"FEAT","m":"NADDOD-Scraper: per-Warehouse Stock-Parser (us/nl/sg/cn) + warehouse_global_qty Fallback → erscheint in Competitor-Stock-Vergleich."}
|
||||
{"d":"2026-06-06","t":"UI","m":"Preis-Chart-Modal beim Klick auf Signal-Karte (180d SVG, multi-vendor). fmtSpd() bereinigt Speed-Anzeige (1.00G→1G, 1600→1.6T). Flexoptix-Preis in Competitor-Vergleich."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"sync:flexoptix-catalog 401 ROOT CAUSE: leerer FLEXOPTIX_API_TOKEN='' kurzschloss '?? null' (nur null/undefined), Script sendete leeren Bearer statt Username/Passwort-Login. Fix '|| null'. Credentials waren immer korrekt. Voller Sync 457 Produkte/345 Preis/36 Stock. Job completed, Robot-Empfehlung jetzt 'alle gesund'."}
|
||||
{"d":"2026-06-06","t":"DATA","m":"Voller Daten-Refresh: Hype-Cycle + ABC + Forecast + Reorder-Signale + Preis-Denorm neu gerechnet (alle completed). Hype-Phasen aktuell: 10G plateau, 800G peak, 1.6T innovation-trigger."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"reorder_signals Daten-Bloat behoben: 4.49M Zeilen/1.19GB → 18.175/4.5MB (99.6% frei). Writer hängte bei jedem 4h-Lauf seit 09.04. neue Zeilen an, löschte nie (24h-TTL nur read-seitig gefiltert). 4.37M waren bereits abgelaufen. Fix: delete-before-insert pro Transceiver. VACUUM FULL. Backup reorder_signals_keep_bak_20260606. Verifiziert stabil bei 18.175."}
|
||||
{"d":"2026-06-06","t":"FIX","m":"Procurement-Pulse Buy-Signals-Karte rief 404-Endpoint /api/procurement/reorder auf → zeigte immer 0. Korrigiert auf /api/procurement/reorder-top (summary.buy_now=314)."}
|
||||
{"d":"2026-06-06","t":"INFRA","m":"System-Health-Audit: 24/24 API-Endpoints grün, keine Dauer-Failing-Jobs, Disk 35%/RAM 99Gi frei, tip-postgres healthy. reorder-signals war einzige Bloat-Quelle."}
|
||||
{"d":"2026-06-07","t":"INFRA","m":"Erik↔Gitea Reconciliation ABGESCHLOSSEN: 22 Erik-lokale Fixes + 28 Gitea-Feature-Commits gemergt (ort, 0 Konflikte automatisch). Einziger manueller Fix: doppelte /api/hype-cycle/market-signals Route dedupliziert (Erik-Handler behalten, matcht Dashboard). 3 Pakete 0 TS-Fehler, 11/11 Endpoints grün, market-signals liefert korrekte Shape. Push main b5925bc2..8c6df102, jetzt 0/0 sync. Safety: branch erik-pre-reconcile-20260607 + bundle (auch Fearghas) + bulk-price WIP in stash@{0}."}
|
||||
{"d":"2026-06-07","t":"DATA","m":"Reconciliation-Bonus: die 4 früher als 'kein Handler' deaktivierten Jobs (match:opn, match:spec, analyze:stock:velocity, enrich:flexoptix-details) wurden vom Merge selbst geheilt — Gitea-Branch lieferte Handler+Schedules, Daemon-Restart re-registrierte sie automatisch (korrekte Crons). Alle 4 test-gelaufen: completed, echte Daten (stock_velocity 262, +40 Equivalences, 100 Detail-Syncs). 284 Jobs geplant, 0 Failing. bulk-price: committe Version funktioniert, Varianten-WIP bleibt in stash@{0}."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Switch-Kompatibilität + Speed-Daten von der Quelle (kein Raten mehr). ROOT CAUSE: Flexoptix-Bulk-API hat KEIN Speed/Formfaktor-Feld -> Sync riet aus Produktnamen (FO-109010-CWDM -> 100000G). FIX: detail-enricher zieht jetzt form_factor aus Spec 'Form Factor' + speed aus 'Supported Protocols'/'Bandwidth' (autoritatives Datenblatt), überschreibt korrupte Bulk-Werte. Voll-Katalog re-enriched: 96.8% plausibel, 487 Teile, 0 Widersprüche bei Stichprobe."}
|
||||
{"d":"2026-06-11","t":"FIX","m":"getFlexoptixSuggestions: (1) physikalische Validierung per Port-Speed + Käfig-Mechanik aus ports_config (100G-QSFP28-Port -> nur <=100G QSFP-Module, nie 400G/QSFP-DD); (2) nur katalog-bestätigte Teile (fx_specifications IS NOT NULL). CX 10000-48Y6C: war 4 unmögliche Gruppen (400G/200G/128G/100T QSFP28) -> jetzt 630 Vorschläge, 0 unmöglich, 0 Phantom. Generalisiert: 400G-Switch zeigt bis 400G."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Befund: ~107 Teile waren fälschlich Vendor 'Flexoptix' zugeordnet (Bindestrich-SKUs FO-/FOT-, nicht im echten FX-Katalog der nur Punkt-SKUs nutzt). Aus 'BEI FLEXOPTIX BESTELLEN' gefiltert. Wettbewerber-Kataloge (ATGBICS/NADDOD) haben eigene Formfaktor-Parsing-Fehler -> separater Scraper-Fix nötig."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Flexoptix-Datenblatt-Audit: alle 805 katalog-bestätigten Teile programmatisch gegen gespeicherte fx_specifications (Datenblatt) geprüft. Nach Terabit-Parser-Fix + Re-Derive aus Specs: Form Factor 802/802 (100%), Speed 801/801 (100%), 0 Abweichungen. Jeder Wert kommt aus dem Datenblatt, nicht aus dem Namen."}
|
||||
{"d":"2026-06-11","t":"DATA","m":"Switch-Daten-Audit: 681 Switches, nur 267 (39%) mit Port-Config, 414 ohne (überwiegend Cisco-Bulk-Import). Vorhandene Port-Configs 99.3% plausibel, 248/267 mit Hersteller-URL, nur 35 mit datasheet_url. Demo-Switches (Aruba CX10000, Arista, Juniper) vollständig+korrekt. LÜCKE: autoritatives Port-Sourcing per Hersteller-Datenblatt für die 414 fehlt = separates Scraper-Projekt pro Vendor."}
|
||||
{"d":"2026-06-11","t":"FIX","m":"Switch-Kompatibilität = 'läuft wirklich': Matcher verlangt jetzt zusätzlich, dass Flexoptix den Transceiver für den Switch-VENDOR codieren kann (fx_compatibilities). Physisch-passend reicht nicht. CX 10000: 629 physisch -> 283 echt-Aruba-codierbar (283/283 verifiziert). Cisco 8818: 55 (alle Cisco-codierbar). Plus Chassis-Port-Format (max_per_slot_NNNG_FF) geparst. Jede Empfehlung jetzt: katalog-bestätigt + datenblatt-genau + passt physisch + codierbar = garantiert lauffähig."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"product_type-Klassifikation (sql/119): neues Enum module|dac|aoc|aec|switch|nic|accessory, deterministisch per Trigger trg_transceivers_biu aus part_number/category/reach gesetzt (self-healing bei jedem Insert/Update, Preis-Updates unberührt). Backfill 41008 Zeilen: 36284 module, 2791 dac, 1835 aoc, 29 aec, 27 accessory, 26 switch, 16 nic. Kabel (DAC/AOC/AEC) und Switches/NICs (ConnectX/Quantum/N-Port) nicht mehr als Modul gezählt. BASE-T/RJ45-Kupfermodule vor Kabel-Demotion geschützt (Guard vor copper-Regel); AOC/DAC-Keyword schlägt Optik-Token-Guard (AOC ist optisch, trägt legitim nm/MMF/SR). Konsumenten filtern product_type='module'. Missed-cable=0 verifiziert. Per-Vendor 100G+: NADDOD 179 Module/423 Kabel/28 Switch-NIC, ATGBICS 1331/297, QSFPTEK 22/98, ProLabs 1690/818."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"form_factor-Recovery (sql/120): korrigiert NUR beweisbar-falsche Zeilen (form_factors.max_speed < speed_gbps) mit explizitem Formfaktor-Token im Titel. 87 gefixt (u.a. 400G-Kabel als SFP+ getaggt -> QSFP-DD; QSFP-DD@800G -> QSFP-DD800; Q56DD -> QSFP-DD), 947 opake Zeilen (z.B. EOLS-1303-40-D, MGB-2GSR) unangetastet gelassen, nichts geraten. Gescraptes Original in neuer Spalte form_factor_raw bewahrt. Recovery auch im Ingest-Trigger (Neuzugänge self-healing)."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"lead_time_days als NICHT ERFASST dokumentiert (sql/121, COMMENT auf price_observations/stock_observations/stock_snapshots): 0% befüllt über 1.378.984 + 85.406 + 0 Zeilen. Plumbing existiert (crawler-llm stock-schema erfasst es, upsertPriceObservation nimmt Param), aber aktiver Scraper-Pfad liefert nie einen Wert und LLM-Crawler-Tabelle stock_snapshots ist leer. Kein Default fabrizieren; NULL = unbekannt. Befüllen ist Scraper-Aufgabe."}
|
||||
{"d":"2026-07-04","t":"DATA","m":"Befund FS.COM-Untererfassung bestätigt (kein Fix, Scraper-Thema): 392 Zeilen/378 distinct SKUs vs ATGBICS 8420 (22x-Lücke). 391/392 sind bereits product_type='module' -> Klassifikation ist NICHT die Ursache, reine Crawl-Tiefe. Ursache in fs-com.ts + Seed-Scope (Kategorie-Abdeckung) + evtl. zu aggressive Quarantäne von fs.com/c/-URLs. Empfehlung: FS.com-Crawl-Seed erweitern (Scraper-Pipeline, nicht raten)."}
|
||||
{"d":"2026-07-07","t":"SECURITY","m":"Hardcodierte DB-Passwoerter aus dem Source entfernt und auf env umgestellt. tip-DB-Passwort (transceiver_db) lag in 14 Dateien im Klartext (quoted export, unquoted psql-Aufrufe, Python-env-Dicts, env-or-hardcoded-default-Fallbacks); llm_gateway-Passwort in seed-blog-training-data.py. Ersetzt durch PGPASSWORD-Env-Referenz mit Fail-Fast-Guard bzw. os.environ. training-data/*.jsonl (48 MB) aus Tracking genommen (git rm --cached, Datei bleibt) und gitignored. .security-scan-allowlist ergaenzt fuer legitime False-Positives (Vendor flexoptix, eigene Infra, sync-Journal-Codenamen, env-aufgeloeste URIs). Pre-Push-Leak-Scanner 397 auf 0 Funde, gruen ohne --no-verify. Offen: Rotation der geleakten Werte (llm_gateway zuerst, tip fleet-weit koordiniert); Git-History behaelt Altwerte."}
|
||||
{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync: 3 Bugs gefixt + erster Lauf. NCS-5000-Regex (passte nicht auf NCS-5001/5002/5011), FX/FX3-Boundary-Guard (verhindert falsche Matches der neueren Generation), source-aware Date-Routing (Format2-Announcement-Dates -> eos_date, nicht last_support_date). 13 Switches auf EoS_Announced gesetzt. Wöchentlicher Sync Sun 04:00 via pg-boss."}
|
||||
{"d":"2026-07-10","t":"DATA","m":"cisco-eol-sync Bug 3 fix: source-aware Date-Routing (Format2-Announcement-Dates → eos_date, nicht last_support_date)"}
|
||||
{"d":"2026-07-10","t":"DATA","m":"Run cisco-eol-sync for real to update 13 switches in DB (NCS-5001/5002/5011, NCS-5502/SE, NCS-55A1, N9K-C92300YC/C9272Q/C93120TX/C93180YC-FX/C9332C, ASR-9001/9901)"}
|
||||
{"d":"2026-07-11","t":"DATA","m":"Transceiver classification fields: 4 new columns added to transceivers table (dwdm_channel, cwdm_channel, cdr_type, protocol_family). Backfill from Flexoptix notes field: 1,209 standard_names enriched (up from 72, +1,579%), 1,177 cdr_types (all dual_cdr), 2,498 wdm_types (1,896 DWDM + 602 CWDM), 7,918 protocol_family values (7,645 ethernet + 273 fibre_channel). Migration 134 applied. Note: DWDM and CWDM are separate columns — never interchangeable; fibre_channel is a separate protocol family from ethernet."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"flexoptix-api-sync.ts: API sync now writes standard_name (from protocol field), wdm_type (DWDM/CWDM from optics.dwdm/cwdm), cdr_type (dual_cdr/cdr from title regex), cdr_support (true when cdr_type set), and protocol_family (ethernet/fibre_channel/sonet/infiniband from title regex) on every Flexoptix catalog sync. Previously these columns were never populated from the API — only dwdm/cwdm booleans existed. This fixes the root cause of empty standard_name on 95.5% of auto_approved equivalences."}
|
||||
{"d":"2026-07-11","t":"FIX","m":"catalog-reconcile.ts: Hard rejects for incompatible entity types: protocol_family mismatch (ethernet≠fibre_channel), temp_range mismatch (COM≠IND), wdm_type mismatch (DWDM≠CWDM) → confidence=0, skip immediately. Standard_name mismatch now -25pts penalty (was no penalty). CDR type mismatch: -20pts + CDR match bonus +5pts. Auto-approve gate now requires fx.standard_name IS NOT NULL, blocking the 77,419/81,116 cases where FX had no standard_name. Confidence scoring schema extended with hardReject flag. Candidates and fxProducts queries now include cdr_type, temp_range, protocol_family, wdm_type."}
|
||||
|
||||
## Pending -- 2026-07-18
|
||||
|
||||
### Fixed
|
||||
- correct 14 false curated 10G-on-1G compatibility claims
|
||||
|
||||
## Pending -- 2026-07-18
|
||||
|
||||
### Fixed
|
||||
- clean 261 spec_match rows unmasked by the 144 curated-row fix
|
||||
|
||||
|
||||
@ -318,77 +318,23 @@ export async function getCompatibleTransceivers(switchId: string) {
|
||||
*/
|
||||
export async function getFlexoptixSuggestions(switchId: string) {
|
||||
const result = await pool.query(
|
||||
`WITH sw_vendor AS (
|
||||
-- Map the switch's vendor to the Flexoptix coding-matrix token. A transceiver
|
||||
-- only truly runs in the switch if Flexoptix can code it for this platform.
|
||||
SELECT
|
||||
CASE
|
||||
WHEN v.name ILIKE '%aruba%' OR v.name ILIKE '%hpe%' OR v.name ILIKE '%hewlett%' OR v.name ILIKE '%hp h3c%' OR v.name ILIKE '%hp %' THEN 'aruba'
|
||||
WHEN v.name ILIKE '%cisco%' THEN 'cisco'
|
||||
WHEN v.name ILIKE '%arista%' THEN 'arista'
|
||||
WHEN v.name ILIKE '%juniper%' THEN 'juniper'
|
||||
WHEN v.name ILIKE '%dell%' OR v.name ILIKE '%force10%' THEN 'dell'
|
||||
WHEN v.name ILIKE '%huawei%' THEN 'huawei'
|
||||
WHEN v.name ILIKE '%fortinet%' THEN 'fortinet'
|
||||
WHEN v.name ILIKE '%mikrotik%' THEN 'mikrotik'
|
||||
WHEN v.name ILIKE '%nvidia%' OR v.name ILIKE '%mellanox%' THEN 'mellanox'
|
||||
WHEN v.name ILIKE '%extreme%' OR v.name ILIKE '%enterasys%' THEN 'extreme'
|
||||
WHEN v.name ILIKE '%nokia%' OR v.name ILIKE '%alcatel%' THEN 'nokia'
|
||||
WHEN v.name ILIKE '%brocade%' OR v.name ILIKE '%ruckus%' THEN 'brocade'
|
||||
WHEN v.name ILIKE '%netgear%' THEN 'netgear'
|
||||
WHEN v.name ILIKE '%zyxel%' THEN 'zyxel'
|
||||
WHEN v.name ILIKE '%ubiquiti%' THEN 'ubiquiti'
|
||||
WHEN v.name ILIKE '%d-link%' OR v.name ILIKE '%dlink%' THEN 'd-link'
|
||||
WHEN v.name ILIKE '%allied telesis%' THEN 'allied telesis'
|
||||
WHEN v.name ILIKE '%adtran%' THEN 'adtran'
|
||||
WHEN v.name ILIKE '%ciena%' THEN 'ciena'
|
||||
ELSE NULL -- unmapped (whitebox e.g. Edgecore) -> MSA-standard fallback
|
||||
END AS vpat
|
||||
FROM switches s JOIN vendors v ON v.id = s.vendor_id WHERE s.id = $1
|
||||
),
|
||||
gate AS (
|
||||
-- Hard gate: a switch only presents compatibility if its port config has
|
||||
-- been verified against the manufacturer datasheet/source. Unverified =
|
||||
-- no claim (return nothing) rather than show potentially-wrong data.
|
||||
SELECT ports_verified FROM switches WHERE id = $1
|
||||
),
|
||||
switch_ports AS (
|
||||
-- Parse each ports_config key 'SPEED_FORMFACTOR' (e.g. '100G_QSFP28') into
|
||||
-- a numeric port speed (Gbps) + a cage family. Real transceivers are then
|
||||
-- bounded by the port speed so corrupt/oversized records cannot leak in.
|
||||
`WITH switch_form_factors AS (
|
||||
SELECT DISTINCT
|
||||
-- numeric speed prefix: 100G->100, 25G->25, 2.5G->2.5, 100M->0.1, 800G->800
|
||||
-- speed token may be at the start ('100G_QSFP28') or embedded
|
||||
-- ('max_per_slot_100G_QSFP28' on chassis switches). Match before the cage.
|
||||
CASE
|
||||
WHEN k ~ '[0-9.]+M[_A-Za-z]' THEN (substring(k from '([0-9.]+)M[_A-Za-z]')::numeric / 1000)
|
||||
WHEN k ~ '[0-9.]+T[_A-Za-z]' THEN (substring(k from '([0-9.]+)T[_A-Za-z]')::numeric * 1000)
|
||||
WHEN k ~ '[0-9.]+G[_A-Za-z]' THEN substring(k from '([0-9.]+)G[_A-Za-z]')::numeric
|
||||
WHEN k ~ '[0-9.]+G$' THEN substring(k from '([0-9.]+)G$')::numeric
|
||||
ELSE NULL
|
||||
END AS port_speed,
|
||||
-- the port's specific cage type (determines which modules mechanically seat)
|
||||
CASE
|
||||
WHEN k ILIKE '%QSFP-DD800%' THEN 'QSFP-DD800'
|
||||
WHEN k ILIKE '%QSFP-DD%' THEN 'QSFP-DD'
|
||||
WHEN k ILIKE '%QSFP112%' THEN 'QSFP112'
|
||||
WHEN k ILIKE '%QSFP56%' THEN 'QSFP56'
|
||||
WHEN k ILIKE '%OSFP224%' THEN 'OSFP224'
|
||||
WHEN k ILIKE '%OSFP%' THEN 'OSFP'
|
||||
WHEN k ILIKE '%QSFP28%' THEN 'QSFP28'
|
||||
WHEN k ILIKE '%QSFP+%' THEN 'QSFP+'
|
||||
WHEN k ILIKE '%QSFP%' THEN 'QSFP+'
|
||||
WHEN k ILIKE '%OSFP224%' THEN 'OSFP224'
|
||||
WHEN k ILIKE '%OSFP112%' THEN 'OSFP112'
|
||||
WHEN k ILIKE '%OSFP%' THEN 'OSFP'
|
||||
WHEN k ILIKE '%SFP56%' THEN 'SFP56'
|
||||
WHEN k ILIKE '%SFP28%' THEN 'SFP28'
|
||||
WHEN k ILIKE '%SFP+%' THEN 'SFP+'
|
||||
WHEN k ILIKE '%SFP%' THEN 'SFP+'
|
||||
WHEN k ILIKE '%CFP2%' THEN 'CFP2'
|
||||
WHEN k ILIKE '%CFP4%' THEN 'CFP4'
|
||||
WHEN k ILIKE '%CFP%' THEN 'CFP'
|
||||
WHEN k ILIKE '%RJ45%' OR k ILIKE '%mGig%' THEN 'RJ45'
|
||||
ELSE NULL
|
||||
END AS port_ff
|
||||
END AS form_factor
|
||||
FROM switches sw,
|
||||
jsonb_object_keys(sw.ports_config) AS k
|
||||
WHERE sw.id = $1 AND sw.ports_config IS NOT NULL
|
||||
@ -421,50 +367,8 @@ export async function getFlexoptixSuggestions(switchId: string) {
|
||||
LIMIT 1
|
||||
) so ON true
|
||||
WHERE LOWER(v.name) = 'flexoptix'
|
||||
AND (SELECT ports_verified FROM gate) IS TRUE
|
||||
-- Only parts confirmed in Flexoptix's live API catalog (have authoritative
|
||||
-- datasheet specs). Excludes phantom/misattributed parts not actually
|
||||
-- orderable from Flexoptix, and parts with no source-of-truth specs.
|
||||
AND t.fx_specifications IS NOT NULL
|
||||
AND t.speed_gbps IS NOT NULL AND t.speed_gbps > 0
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM switch_ports sp
|
||||
WHERE sp.port_ff IS NOT NULL
|
||||
AND sp.port_speed IS NOT NULL
|
||||
-- module must mechanically seat in this port cage. A cage accepts its own
|
||||
-- type plus smaller/backward-compatible modules, never a larger one:
|
||||
-- QSFP-DD/OSFP cages take QSFP-family; QSFP28/56 cages take QSFP+/28/56
|
||||
-- (NOT QSFP-DD); SFP cages take all SFP variants. No cross-family.
|
||||
AND t.form_factor = ANY (
|
||||
CASE sp.port_ff
|
||||
WHEN 'QSFP-DD800' THEN ARRAY['QSFP-DD800','QSFP-DD','QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP-DD' THEN ARRAY['QSFP-DD','QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP112' THEN ARRAY['QSFP112','QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP56' THEN ARRAY['QSFP56','QSFP28','QSFP+']
|
||||
WHEN 'QSFP28' THEN ARRAY['QSFP28','QSFP+']
|
||||
WHEN 'QSFP+' THEN ARRAY['QSFP+']
|
||||
WHEN 'OSFP224' THEN ARRAY['OSFP224','OSFP112','OSFP']
|
||||
WHEN 'OSFP112' THEN ARRAY['OSFP112','OSFP']
|
||||
WHEN 'OSFP' THEN ARRAY['OSFP']
|
||||
WHEN 'SFP56' THEN ARRAY['SFP56','SFP28','SFP+','SFP']
|
||||
WHEN 'SFP28' THEN ARRAY['SFP28','SFP+','SFP']
|
||||
WHEN 'SFP+' THEN ARRAY['SFP+','SFP']
|
||||
WHEN 'CFP2' THEN ARRAY['CFP2']
|
||||
WHEN 'CFP4' THEN ARRAY['CFP4']
|
||||
WHEN 'CFP' THEN ARRAY['CFP']
|
||||
WHEN 'RJ45' THEN ARRAY['RJ45','Copper']
|
||||
ELSE ARRAY[]::text[]
|
||||
END
|
||||
)
|
||||
-- speed must not exceed the port's speed (slower module in faster cage OK)
|
||||
AND t.speed_gbps <= sp.port_speed
|
||||
)
|
||||
-- AND Flexoptix must be able to code it for THIS switch's platform, so it
|
||||
-- actually runs (not just physically fits). Whitebox/unmapped -> MSA standard.
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(t.fx_compatibilities) c, sw_vendor
|
||||
WHERE (sw_vendor.vpat IS NOT NULL AND c->>'compatible_to_vendor' ILIKE '%' || sw_vendor.vpat || '%')
|
||||
OR (sw_vendor.vpat IS NULL AND c->>'compatible_to_vendor' ILIKE '%MSA Standard%')
|
||||
AND t.form_factor IN (
|
||||
SELECT form_factor FROM switch_form_factors WHERE form_factor IS NOT NULL
|
||||
)
|
||||
ORDER BY t.speed_gbps DESC NULLS LAST, t.reach_meters ASC NULLS LAST`,
|
||||
[switchId]
|
||||
|
||||
@ -28,7 +28,6 @@ import { procurementRouter } from "./routes/procurement";
|
||||
import { changelogRouter } from "./routes/changelog";
|
||||
import { newsRouter } from "./routes/news";
|
||||
import { proxyRouter } from "./routes/proxy";
|
||||
import { researchRobotRouter } from "./routes/research-robot";
|
||||
import { reviewRouter } from "./routes/review";
|
||||
import { stockRouter } from "./routes/stock";
|
||||
import { priceComparisonRouter } from "./routes/price-comparison";
|
||||
@ -38,7 +37,6 @@ import { formFactorsRouter } from "./routes/form-factors";
|
||||
import { tipLlmRouter } from "./routes/tip-llm";
|
||||
import { equivalencesRouter } from "./routes/equivalences";
|
||||
import { priceHistoryRouter } from "./routes/price-history";
|
||||
import { stockCompetitorRouter } from "./routes/stock-competitor";
|
||||
import { kbRouter } from "./routes/kb";
|
||||
import { bulkPriceRouter } from "./routes/bulk-price";
|
||||
import { vendorReliabilityRouter } from "./routes/vendor-reliability";
|
||||
@ -74,7 +72,6 @@ app.use("/api/auth", authRouter);
|
||||
|
||||
// Proxy public endpoints (register + heartbeat + stats + next — no auth)
|
||||
app.use("/api/proxy", proxyRouter);
|
||||
app.use("/api/research-robot", researchRobotRouter);
|
||||
|
||||
// All other API routes require a valid token
|
||||
app.use("/api", (req, res, next) => {
|
||||
@ -123,7 +120,6 @@ app.use("/api/tip-llm", tipLlmRouter);
|
||||
app.use("/api/equivalences", equivalencesRouter);
|
||||
// Price history charts
|
||||
app.use("/api/price-history", priceHistoryRouter);
|
||||
app.use("/api/stock", stockCompetitorRouter);
|
||||
app.use("/api/kb", kbRouter);
|
||||
// Bulk price lookup (G)
|
||||
app.use("/api/bulk-price", bulkPriceRouter);
|
||||
|
||||
@ -14,27 +14,7 @@
|
||||
import { existsSync, readFileSync, writeFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
|
||||
// -- Ollama endpoint with automatic primary->fallback failover --
|
||||
// Primary = BLOG_OLLAMA_URL (WireGuard tunnel to Mac Studio loopback Ollama),
|
||||
// fallback = BLOG_OLLAMA_URL_FALLBACK (Cloudflare tunnel). Independent transports;
|
||||
// if the primary transport drops, requests move to the backup. Re-probed every 60s.
|
||||
const OLLAMA_PRIMARY = process.env.BLOG_OLLAMA_URL || process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
const OLLAMA_FALLBACK = process.env.BLOG_OLLAMA_URL_FALLBACK || "";
|
||||
let OLLAMA_URL = OLLAMA_PRIMARY;
|
||||
async function pickOllamaUrl(): Promise<void> {
|
||||
for (const u of [OLLAMA_PRIMARY, OLLAMA_FALLBACK].filter(Boolean)) {
|
||||
try {
|
||||
const r = await fetch(`${u}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
||||
if (r.ok) {
|
||||
if (OLLAMA_URL !== u) console.log(`[blog-llm] ollama endpoint -> ${u}`);
|
||||
OLLAMA_URL = u;
|
||||
return;
|
||||
}
|
||||
} catch { /* try next */ }
|
||||
}
|
||||
}
|
||||
void pickOllamaUrl();
|
||||
setInterval(() => { void pickOllamaUrl(); }, 60000).unref();
|
||||
const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || "";
|
||||
const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-sonnet-4-20250514";
|
||||
const CLAUDE_BRIDGE_URL = process.env.CLAUDE_BRIDGE_URL || "http://localhost:3250";
|
||||
|
||||
@ -2217,7 +2217,7 @@ blogRouter.post("/llm/reset-queue", (_req: Request, res: Response) => {
|
||||
blogRouter.get("/llm/model-info", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const settings = getLlmProvider();
|
||||
const ollamaUrl = process.env.BLOG_OLLAMA_URL || process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
const ollamaUrl = process.env.OLLAMA_URL || "http://localhost:11434";
|
||||
|
||||
// Only meaningful for fo-blog Ollama models
|
||||
const modelName = settings.ollamaModel || "";
|
||||
|
||||
@ -50,7 +50,6 @@ hotTopicsRouter.get("/", async (req, res) => {
|
||||
JOIN vendors v ON pc.vendor_id = v.id
|
||||
JOIN transceivers t ON pc.transceiver_id = t.id
|
||||
WHERE pc.delta_pct < -10 AND pc.detected_at > NOW() - INTERVAL '14 days'
|
||||
AND v.name != 'Flexoptix'
|
||||
ORDER BY pc.delta_pct ASC LIMIT 5
|
||||
`).catch(() => ({ rows: [] }));
|
||||
|
||||
|
||||
@ -215,6 +215,192 @@ function buildRecommendation(
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/hype-cycle/market-signals — Multi-source demand intelligence
|
||||
hypeCycleRouter.get("/market-signals", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { pool } = await import("../db/client");
|
||||
|
||||
// ── Parallel data fetch ──────────────────────────────────────────────────
|
||||
const [hypeRows, priceRows, capexRows, aiRows, ebayRows, internalRows] = await Promise.all([
|
||||
// Latest hype cycle per technology
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (technology)
|
||||
technology, hype_phase, hype_score, asp_current_usd,
|
||||
r_squared, computed_at, current_share, years_to_next_phase
|
||||
FROM hype_cycle_analysis
|
||||
ORDER BY technology, computed_at DESC
|
||||
`),
|
||||
|
||||
// Price observation activity: last 30d vs prior 30d (scraping frequency = demand proxy)
|
||||
pool.query(`
|
||||
SELECT
|
||||
t.speed_gbps,
|
||||
t.form_factor,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
AND po.time < NOW() - INTERVAL '30 days') AS obs_prior_30d
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '60 days'
|
||||
GROUP BY t.speed_gbps, t.form_factor
|
||||
`),
|
||||
|
||||
// Hyperscaler capex — most recent per company
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (company)
|
||||
company, period_label, capex_usd_millions,
|
||||
dc_capex_est_millions, yoy_growth_pct
|
||||
FROM hyperscaler_capex
|
||||
WHERE yoy_growth_pct IS NOT NULL
|
||||
ORDER BY company, period_end DESC
|
||||
`),
|
||||
|
||||
// AI cluster demand last 90 days
|
||||
pool.query(`
|
||||
SELECT
|
||||
COALESCE(SUM(estimated_transceivers), 0) AS total_tx,
|
||||
COUNT(*) AS cluster_count,
|
||||
COUNT(*) FILTER (WHERE estimated_transceivers > 0) AS with_estimates
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
`),
|
||||
|
||||
// eBay marketplace velocity
|
||||
pool.query(`
|
||||
SELECT DISTINCT ON (form_factor, speed_label)
|
||||
marketplace, keyword, form_factor, speed_label,
|
||||
sold_count_30d, active_listings, avg_sold_price
|
||||
FROM marketplace_velocity
|
||||
WHERE sold_count_30d IS NOT NULL
|
||||
ORDER BY form_factor, speed_label, scraped_at DESC
|
||||
`),
|
||||
|
||||
// Internal demand: fast-mover trend
|
||||
pool.query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE velocity_class = 'fast_mover' AND demand_trend_pct > 0) AS fast_mover_pos,
|
||||
COUNT(*) FILTER (WHERE velocity_class = 'fast_mover' AND demand_trend_pct < 0) AS fast_mover_neg,
|
||||
AVG(demand_trend_pct) FILTER (WHERE velocity_class = 'fast_mover') AS avg_fast_trend,
|
||||
AVG(demand_trend_pct) FILTER (WHERE velocity_class = 'regular') AS avg_regular_trend
|
||||
FROM flexoptix_internal_demand
|
||||
WHERE is_internal = true
|
||||
`),
|
||||
]);
|
||||
|
||||
// ── Pre-compute global signals ───────────────────────────────────────────
|
||||
const capexYoyValues = capexRows.rows
|
||||
.map((r) => parseFloat(r.yoy_growth_pct))
|
||||
.filter((v) => !isNaN(v));
|
||||
const capexYoyAvg = capexYoyValues.length
|
||||
? capexYoyValues.reduce((a, b) => a + b, 0) / capexYoyValues.length
|
||||
: 0;
|
||||
|
||||
const aiData = aiRows.rows[0];
|
||||
const totalAiTx = parseInt(aiData?.total_tx ?? "0") || 0;
|
||||
|
||||
const internalData = internalRows.rows[0];
|
||||
const avgFastTrend = parseFloat(internalData?.avg_fast_trend ?? "0") || 0;
|
||||
|
||||
// Build price activity map: speedGbps+formFactor → ratio (30d/prior30d)
|
||||
const priceMap = new Map<string, number>();
|
||||
for (const row of priceRows.rows) {
|
||||
const obs30 = parseInt(row.obs_30d) || 0;
|
||||
const obsPrior = parseInt(row.obs_prior_30d) || 1;
|
||||
const key = `${row.speed_gbps}__${row.form_factor ?? ""}`;
|
||||
priceMap.set(key, obs30 / obsPrior);
|
||||
}
|
||||
|
||||
// Build eBay map: speedLabel → sold_count_30d
|
||||
const ebayMap = new Map<string, number>();
|
||||
for (const row of ebayRows.rows) {
|
||||
const speed = (row.speed_label ?? "").replace(/\s+/g, "");
|
||||
ebayMap.set(speed, parseInt(row.sold_count_30d) || 0);
|
||||
}
|
||||
|
||||
// ── Per-technology signals ───────────────────────────────────────────────
|
||||
const technologies = hypeRows.rows.map((r) => {
|
||||
const tech = TECH_SIGNAL_MAP.find((t) => t.label === r.technology);
|
||||
const phase = (r.hype_phase ?? "innovation_trigger") as PhaseKey;
|
||||
const hypeScore = parseInt(r.hype_score) || 0;
|
||||
|
||||
// Price activity ratio: average across matching speed+formFactor combos
|
||||
let priceRatios: number[] = [];
|
||||
if (tech) {
|
||||
for (const ff of tech.formFactors) {
|
||||
const key = `${tech.speedGbps}__${ff}`;
|
||||
const ratio = priceMap.get(key);
|
||||
if (ratio !== undefined) priceRatios.push(ratio);
|
||||
}
|
||||
}
|
||||
const priceActivityRatio = priceRatios.length
|
||||
? priceRatios.reduce((a, b) => a + b, 0) / priceRatios.length
|
||||
: 1;
|
||||
|
||||
// eBay velocity for this speed
|
||||
const ebayVelocity = tech ? (ebayMap.get(tech.speedLabel) ?? 0) : 0;
|
||||
|
||||
// AI cluster: allocate demand proportionally for high-speed techs
|
||||
const aiBoostTx = tech && tech.speedGbps >= 400 ? totalAiTx : 0;
|
||||
|
||||
// Compute composite score (0–100)
|
||||
let score = hypeScore * 0.3;
|
||||
const capexBoostPts = capexYoyAvg > 100 ? 18 : capexYoyAvg > 50 ? 12 : capexYoyAvg > 20 ? 5 : 0;
|
||||
const priceBoostPts = priceActivityRatio > 1.3 ? 10 : priceActivityRatio > 1.0 ? 5 : -3;
|
||||
const aiBoostPts = aiBoostTx > 100000 ? 14 : aiBoostTx > 50000 ? 9 : aiBoostTx > 10000 ? 4 : 0;
|
||||
const ebayBoostPts = ebayVelocity > 200 ? 8 : ebayVelocity > 100 ? 5 : ebayVelocity > 50 ? 2 : 0;
|
||||
const intlBoostPts = avgFastTrend > 10 ? 6 : avgFastTrend > 0 ? 3 : avgFastTrend < -20 ? -5 : 0;
|
||||
|
||||
score += capexBoostPts + priceBoostPts + aiBoostPts + ebayBoostPts + intlBoostPts;
|
||||
const marketSignalScore = Math.max(0, Math.min(100, Math.round(score)));
|
||||
|
||||
const rec = buildRecommendation(phase, marketSignalScore, capexYoyAvg, tech?.speedGbps ?? 0);
|
||||
|
||||
// Signal drivers list for tooltip
|
||||
const drivers: string[] = [];
|
||||
if (capexBoostPts > 0) drivers.push(`Hyperscaler CapEx +${capexYoyAvg.toFixed(0)}% YoY avg`);
|
||||
if (priceActivityRatio > 1.1) drivers.push(`Price obs +${((priceActivityRatio - 1) * 100).toFixed(0)}% MoM activity`);
|
||||
if (aiBoostTx > 0) drivers.push(`~${(aiBoostTx / 1000).toFixed(0)}k transceivers in AI cluster builds`);
|
||||
if (ebayVelocity > 0) drivers.push(`${ebayVelocity} units sold on secondary market (30d)`);
|
||||
if (intlBoostPts > 0) drivers.push(`Internal fast-movers trending ${avgFastTrend > 0 ? "+" : ""}${avgFastTrend.toFixed(1)}%`);
|
||||
|
||||
return {
|
||||
technology: r.technology,
|
||||
phase,
|
||||
hypeScore,
|
||||
aspCurrentUsd: r.asp_current_usd,
|
||||
marketSignalScore,
|
||||
recommendation: rec,
|
||||
drivers,
|
||||
speedGbps: tech?.speedGbps,
|
||||
priceActivityRatio: Math.round(priceActivityRatio * 100) / 100,
|
||||
ebayVelocity,
|
||||
computedAt: r.computed_at,
|
||||
};
|
||||
});
|
||||
|
||||
// ── Global context ───────────────────────────────────────────────────────
|
||||
const globalContext = {
|
||||
hyperscalerCapex: capexRows.rows.map((r) => ({
|
||||
company: r.company,
|
||||
periodLabel: r.period_label,
|
||||
capexMillions: parseFloat(r.capex_usd_millions),
|
||||
dcCapexMillions: parseFloat(r.dc_capex_est_millions),
|
||||
yoyGrowthPct: parseFloat(r.yoy_growth_pct),
|
||||
})),
|
||||
capexYoyAvg: Math.round(capexYoyAvg),
|
||||
capexBoom: capexYoyAvg > 50,
|
||||
totalAiClusterTx90d: totalAiTx,
|
||||
aiClusterCount90d: parseInt(aiData?.cluster_count ?? "0") || 0,
|
||||
internalFastMoverTrend: Math.round(avgFastTrend * 10) / 10,
|
||||
};
|
||||
|
||||
res.json({ success: true, technologies, globalContext, computed_at: new Date().toISOString() });
|
||||
} catch (err) {
|
||||
console.error("Market signals error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute market signals" });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/hype-cycle/analysis — Bass-fitted results from DB (hype_cycle_analysis table)
|
||||
hypeCycleRouter.get("/analysis", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
@ -248,74 +434,6 @@ hypeCycleRouter.get("/analysis", async (_req: Request, res: Response) => {
|
||||
});
|
||||
|
||||
// GET /api/hype-cycle/:tech — Specific technology detail (must be last!)
|
||||
// GET /api/hype-cycle/market-signals — data-driven per-technology market signal score + drivers + recommendation
|
||||
hypeCycleRouter.get("/market-signals", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const yearParam = q("year", _req);
|
||||
const year = yearParam ? parseInt(yearParam) : new Date().getFullYear();
|
||||
const overridesMap = await getDataDrivenOverrides();
|
||||
const speedMetrics = await getSpeedClassMetrics();
|
||||
const metricsBySpeed = new Map(speedMetrics.map((m) => [m.speedGbps, m]));
|
||||
const maxSku = Math.max(1, ...speedMetrics.map((m) => m.skuCount));
|
||||
const allTechs = [...TECH_GENERATIONS, ...SPECIAL_TECHS];
|
||||
|
||||
const technologies = allTechs.map((tech) => {
|
||||
const r = computeHypeCycle(tech, year, overridesMap.get(tech.speedGbps));
|
||||
const sm = metricsBySpeed.get(tech.speedGbps);
|
||||
const adoption = Math.max(0, Math.round(r.adoptionPct));
|
||||
const composite = Math.max(0, Math.round(r.compositeScore));
|
||||
const depth = sm ? Math.round(100 * Math.min(1, sm.skuCount / maxSku)) : 0;
|
||||
const marketSignalScore = Math.max(0, Math.min(100, Math.round(0.4 * adoption + 0.3 * composite + 0.3 * depth)));
|
||||
|
||||
const drivers: string[] = [];
|
||||
drivers.push(adoption + "% modeled adoption");
|
||||
if (sm && sm.skuCount) drivers.push(sm.skuCount + " SKUs / " + sm.vendorCount + " vendors");
|
||||
if (sm && sm.avgPrice) drivers.push("avg $" + Math.round(sm.avgPrice));
|
||||
drivers.push("phase: " + r.phaseLabel);
|
||||
|
||||
const phase = String(r.phaseLabel || "");
|
||||
let recommendation = { label: "Monitor", detail: "Insufficient market momentum — keep watching.", color: "#94a3b8" };
|
||||
if (/Plateau/i.test(phase)) recommendation = { label: "Commodity — negotiate", detail: "Mature, broad supply (" + (sm ? sm.vendorCount : 0) + " vendors). Push for price.", color: "#0ea5e9" };
|
||||
else if (/Slope/i.test(phase)) recommendation = { label: "Mainstream — buy", detail: "Mainstreaming with falling prices and rising supply.", color: "#16a34a" };
|
||||
else if (/Trough/i.test(phase)) recommendation = { label: "Maturing — opportunity", detail: "Past the hype; early mainstream pricing forming.", color: "#ca8a04" };
|
||||
else if (/Peak/i.test(phase)) recommendation = { label: "Hype peak — caution", detail: "Elevated expectations; supply and price still volatile.", color: "#f97316" };
|
||||
else if (/Innovation|Trigger/i.test(phase)) recommendation = { label: "Emerging — pilot", detail: "Early window; limited supply, premium pricing.", color: "#ca8a04" };
|
||||
|
||||
return {
|
||||
technology: tech.name,
|
||||
phase: r.phaseLabel,
|
||||
marketSignalScore,
|
||||
adoptionPct: adoption,
|
||||
compositeScore: composite,
|
||||
skuCount: sm ? sm.skuCount : 0,
|
||||
vendorCount: sm ? sm.vendorCount : 0,
|
||||
avgPrice: sm && sm.avgPrice ? Math.round(sm.avgPrice * 100) / 100 : null,
|
||||
drivers,
|
||||
recommendation,
|
||||
};
|
||||
});
|
||||
|
||||
technologies.sort((a, b) => b.marketSignalScore - a.marketSignalScore);
|
||||
|
||||
const totalSkus = speedMetrics.reduce((acc, m) => acc + m.skuCount, 0);
|
||||
const totalVendors = speedMetrics.length ? Math.max(...speedMetrics.map((m) => m.vendorCount)) : 0;
|
||||
const avgSignal = technologies.length ? Math.round(technologies.reduce((acc, t) => acc + t.marketSignalScore, 0) / technologies.length) : 0;
|
||||
const globalContext = {
|
||||
totalSkus,
|
||||
totalVendors,
|
||||
avgMarketSignal: avgSignal,
|
||||
strongest: technologies.length ? technologies[0].technology : null,
|
||||
strongestScore: technologies.length ? technologies[0].marketSignalScore : null,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
res.json({ success: true, year, technologies, globalContext });
|
||||
} catch (err) {
|
||||
console.error("market-signals error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute market signals" });
|
||||
}
|
||||
});
|
||||
|
||||
hypeCycleRouter.get("/:tech", (req: Request, res: Response) => {
|
||||
const techQuery = String(req.params.tech);
|
||||
const yearParam = q("year", req);
|
||||
|
||||
@ -36,7 +36,6 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -56,7 +55,6 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -112,7 +110,6 @@ priceComparisonRouter.get("/", async (req: Request, res: Response) => {
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
@ -219,7 +216,6 @@ priceComparisonRouter.get("/:sku", async (req: Request, res: Response) => {
|
||||
po.time
|
||||
FROM price_observations po
|
||||
WHERE po.transceiver_id = $1
|
||||
AND po.marketplace NOT LIKE 'flexoptix%'
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
) po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
|
||||
@ -781,198 +781,38 @@ procurementRouter.get("/dead-stock-revival", async (_req: Request, res: Response
|
||||
});
|
||||
|
||||
// ─── C: GET /api/procurement/supply-squeeze ──────────────────────────────────
|
||||
|
||||
// ─── GET /api/procurement/availability ───────────────────────────────────────
|
||||
// Data-grounded supply-availability per speed tier. Derived from real supplier
|
||||
// diversity (distinct source vendors offering it) + stock coverage. Surfaces the
|
||||
// 400G/800G/1.6T supply tightening that exists in the data but was not shown.
|
||||
procurementRouter.get("/availability", async (_req: Request, res: Response) => {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
// Disable parallel workers for this aggregate — Docker /dev/shm (64MB) is too
|
||||
// small for parallel hash and the query is fast enough single-threaded.
|
||||
await client.query('BEGIN');
|
||||
await client.query('SET LOCAL max_parallel_workers_per_gather = 0');
|
||||
const result = await client.query(`
|
||||
WITH sup AS (
|
||||
SELECT t.speed_gbps,
|
||||
COUNT(DISTINCT po.source_vendor_id) FILTER (WHERE po.time > NOW() - INTERVAL '45 days') AS suppliers,
|
||||
COUNT(DISTINCT po.source_vendor_id) FILTER (WHERE po.time <= NOW() - INTERVAL '45 days') AS suppliers_prior
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600) AND po.time > NOW() - INTERVAL '90 days'
|
||||
GROUP BY t.speed_gbps
|
||||
),
|
||||
sk AS (
|
||||
SELECT speed_gbps, COUNT(*) AS sku_count
|
||||
FROM transceivers WHERE speed_gbps IN (100,200,400,800,1600)
|
||||
GROUP BY speed_gbps HAVING COUNT(*) >= 3
|
||||
),
|
||||
st AS (
|
||||
SELECT t.speed_gbps,
|
||||
COUNT(*) FILTER (WHERE s.in_stock IS TRUE) AS skus_in_stock,
|
||||
COUNT(*) AS skus_with_stock
|
||||
FROM transceivers t
|
||||
JOIN LATERAL (
|
||||
SELECT in_stock FROM stock_observations so
|
||||
WHERE so.transceiver_id = t.id ORDER BY so.time DESC LIMIT 1
|
||||
) s ON true
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600)
|
||||
GROUP BY t.speed_gbps
|
||||
),
|
||||
pm AS (
|
||||
-- per-SKU paired price momentum (30d vs prior 30d), bias-free: only SKUs in
|
||||
-- both windows; aggregate the median per-SKU pct delta per speed tier.
|
||||
SELECT speed_gbps,
|
||||
ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (med_now - med_prior) / NULLIF(med_prior,0) * 100)::numeric, 1) AS price_delta_pct
|
||||
FROM (
|
||||
SELECT t.speed_gbps, t.id,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS med_now,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS med_prior,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS n_now,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS n_prior
|
||||
FROM price_observations po JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE t.speed_gbps IN (100,200,400,800,1600) AND po.price > 0 AND COALESCE(po.is_anomalous,false) = false
|
||||
GROUP BY t.speed_gbps, t.id
|
||||
) per
|
||||
WHERE med_now IS NOT NULL AND med_prior IS NOT NULL AND med_prior > 0 AND n_now >= 2 AND n_prior >= 2
|
||||
GROUP BY speed_gbps
|
||||
)
|
||||
SELECT
|
||||
sk.speed_gbps,
|
||||
sk.sku_count::int,
|
||||
COALESCE(sup.suppliers,0)::int AS suppliers,
|
||||
COALESCE(sup.suppliers_prior,0)::int AS suppliers_prior,
|
||||
COALESCE(st.skus_in_stock,0)::int AS skus_in_stock,
|
||||
COALESCE(st.skus_with_stock,0)::int AS skus_with_stock,
|
||||
CASE WHEN COALESCE(st.skus_with_stock,0) > 0
|
||||
THEN ROUND(100.0 * st.skus_in_stock / st.skus_with_stock)::int ELSE NULL END AS in_stock_pct,
|
||||
pm.price_delta_pct
|
||||
FROM sk
|
||||
LEFT JOIN sup ON sup.speed_gbps = sk.speed_gbps
|
||||
LEFT JOIN st ON st.speed_gbps = sk.speed_gbps
|
||||
LEFT JOIN pm ON pm.speed_gbps = sk.speed_gbps
|
||||
ORDER BY sk.speed_gbps DESC
|
||||
`);
|
||||
|
||||
type Row = { speed_gbps: string; sku_count: number; suppliers: number; suppliers_prior: number; skus_in_stock: number; skus_with_stock: number; in_stock_pct: number | null; price_delta_pct: string | null };
|
||||
const tiers = (result.rows as Row[]).map((r) => {
|
||||
const sup = r.suppliers ?? 0;
|
||||
const inStockPct = r.in_stock_pct ?? null;
|
||||
// availability class from supplier diversity + stock coverage
|
||||
let availability: "scarce" | "constrained" | "moderate" | "abundant";
|
||||
if (sup <= 3 || (inStockPct !== null && inStockPct < 20)) availability = "scarce";
|
||||
else if (sup <= 6 || (inStockPct !== null && inStockPct < 45)) availability = "constrained";
|
||||
else if (sup <= 9) availability = "moderate";
|
||||
else availability = "abundant";
|
||||
// tightening trend: supplier diversity dropped vs prior window
|
||||
const trend = r.suppliers_prior > 0
|
||||
? (sup < r.suppliers_prior ? "tightening" : sup > r.suppliers_prior ? "loosening" : "stable")
|
||||
: "stable";
|
||||
const priceDelta = r.price_delta_pct != null ? parseFloat(r.price_delta_pct) : null;
|
||||
const priceTrend = priceDelta === null ? "unknown" : priceDelta > 3 ? "rising" : priceDelta < -3 ? "falling" : "stable";
|
||||
const drivers: string[] = [`${sup} active suppliers`];
|
||||
if (trend === "tightening") drivers.push(`supplier base shrank ${r.suppliers_prior}→${sup} vs prior 45d`);
|
||||
else if (trend === "loosening") drivers.push(`supplier base grew ${r.suppliers_prior}→${sup} vs prior 45d`);
|
||||
if (inStockPct !== null) drivers.push(`${inStockPct}% of tracked SKUs in stock`);
|
||||
if (priceDelta !== null) drivers.push(`price ${priceDelta >= 0 ? "+" : ""}${priceDelta}% (30d, same-SKU median)`);
|
||||
drivers.push(`${r.sku_count} SKUs catalogued`);
|
||||
// Plain-language WHY, built only from real per-speed signals
|
||||
const reasons: string[] = [];
|
||||
if (availability === "scarce") reasons.push(sup <= 3 ? `only ${sup} supplier(s) offer it` : `near-zero stock coverage`);
|
||||
else if (availability === "constrained") reasons.push(`limited supplier base (${sup})`);
|
||||
if (trend === "tightening") reasons.push(`supplier base is shrinking`);
|
||||
if (priceTrend === "rising") reasons.push(`prices rising ${priceDelta}% on the same parts`);
|
||||
if (priceTrend === "falling") reasons.push(`prices easing ${priceDelta}%`);
|
||||
if (inStockPct !== null && inStockPct < 50) reasons.push(`${inStockPct}% in-stock coverage`);
|
||||
const why = reasons.length ? reasons.join("; ") : `broad supply: ${sup} suppliers, ${inStockPct ?? "n/a"}% in stock, stable pricing`;
|
||||
return {
|
||||
speed_gbps: parseFloat(r.speed_gbps),
|
||||
sku_count: r.sku_count,
|
||||
suppliers: sup,
|
||||
in_stock_pct: inStockPct,
|
||||
price_delta_pct: priceDelta,
|
||||
price_trend: priceTrend,
|
||||
availability,
|
||||
trend,
|
||||
why,
|
||||
drivers,
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ success: true, tiers });
|
||||
} catch (err) {
|
||||
try { await client.query('ROLLBACK'); } catch { /* ignore */ }
|
||||
console.error("availability error:", err);
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// Multi-signal supply constraint detector
|
||||
procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const [priceSignals, aiDemand, hypeData, stockData] = await Promise.all([
|
||||
// Price momentum: 30d vs 60d avg by speed/form_factor
|
||||
pool.query(`
|
||||
-- Per-SKU paired comparison: only transceivers present in BOTH periods.
|
||||
-- This eliminates catalog-composition bias (new expensive SKUs entering a
|
||||
-- speed/form-factor bucket would otherwise fake a huge price jump).
|
||||
WITH per_sku AS (
|
||||
SELECT
|
||||
t.id, t.speed_gbps, t.form_factor,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price)
|
||||
FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS med_now,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price)
|
||||
FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS med_prior,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_now,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days') AS obs_prior
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.price > 5 AND po.currency = 'USD'
|
||||
AND COALESCE(po.is_anomalous, false) = false
|
||||
AND t.form_factor IN ('SFP','SFP+','SFP28','SFP56','QSFP+','QSFP28','QSFP56','QSFP-DD','QSFP-DD800','OSFP','OSFP-XD','XFP','CFP','CFP2','CFP4','CDFP','DSFP')
|
||||
AND po.price < 15000
|
||||
AND t.part_number NOT ILIKE '%AOC%'
|
||||
AND t.part_number NOT ILIKE '%-DAC-%'
|
||||
AND (t.standard_name IS NULL OR (t.standard_name NOT ILIKE '%Switch%' AND t.standard_name NOT ILIKE '%InfiniBand%'))
|
||||
AND t.speed_gbps > 0
|
||||
GROUP BY t.id, t.speed_gbps, t.form_factor
|
||||
)
|
||||
SELECT
|
||||
speed_gbps, form_factor,
|
||||
-- avg_30d / avg_prior_30d kept as column names for downstream compatibility,
|
||||
-- but they now carry MEDIAN-of-matched-SKU prices (only SKUs in both periods)
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY med_now) AS avg_30d,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY med_prior) AS avg_prior_30d,
|
||||
-- The real signal: median of per-SKU percentage deltas
|
||||
ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (med_now - med_prior) / NULLIF(med_prior,0) * 100)::numeric, 1) AS sku_median_delta_pct,
|
||||
COUNT(*) AS obs_30d
|
||||
FROM per_sku
|
||||
WHERE med_now IS NOT NULL AND med_prior IS NOT NULL AND med_prior > 0
|
||||
AND obs_now >= 2 AND obs_prior >= 2
|
||||
GROUP BY speed_gbps, form_factor
|
||||
HAVING COUNT(*) >= 3
|
||||
t.speed_gbps, t.form_factor,
|
||||
ROUND(AVG(po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days')::numeric,2) AS avg_30d,
|
||||
ROUND(AVG(po.price) FILTER (WHERE po.time >= NOW() - INTERVAL '60 days' AND po.time < NOW() - INTERVAL '30 days')::numeric,2) AS avg_prior_30d,
|
||||
COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') AS obs_30d
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.price > 5 AND po.currency = 'USD'
|
||||
GROUP BY t.speed_gbps, t.form_factor
|
||||
HAVING COUNT(*) FILTER (WHERE po.time >= NOW() - INTERVAL '30 days') >= 3
|
||||
`),
|
||||
// AI cluster demand by speed tier (speed_tier = 0 excluded — unclassified announcements)
|
||||
// AI cluster demand by speed tier
|
||||
pool.query(`
|
||||
SELECT speed_tier, total_tx, cluster_count FROM (
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%800G%' THEN 800
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%400G%' THEN 400
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%100G%' THEN 100
|
||||
ELSE 0
|
||||
END AS speed_tier,
|
||||
COALESCE(SUM(estimated_transceivers),0)::int AS total_tx,
|
||||
COUNT(*)::int AS cluster_count
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
GROUP BY 1
|
||||
HAVING COALESCE(SUM(estimated_transceivers),0) > 0
|
||||
) t WHERE speed_tier > 0
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%800G%' THEN 800
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%400G%' THEN 400
|
||||
WHEN COALESCE(network_speed, title, summary, '') ILIKE '%100G%' THEN 100
|
||||
ELSE 0
|
||||
END AS speed_tier,
|
||||
COALESCE(SUM(estimated_transceivers),0)::int AS total_tx,
|
||||
COUNT(*)::int AS cluster_count
|
||||
FROM ai_cluster_announcements
|
||||
WHERE announced_date >= NOW() - INTERVAL '90 days'
|
||||
GROUP BY 1
|
||||
HAVING COALESCE(SUM(estimated_transceivers),0) > 0
|
||||
`),
|
||||
// Hype phase per technology
|
||||
pool.query(`
|
||||
@ -995,7 +835,7 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
`).catch(() => ({ rows: [] })),
|
||||
]);
|
||||
|
||||
type PriceRow = { speed_gbps: string; form_factor: string; avg_30d: string; avg_prior_30d: string; obs_30d: string; sku_median_delta_pct: string | null };
|
||||
type PriceRow = { speed_gbps: string; form_factor: string; avg_30d: string; avg_prior_30d: string; obs_30d: string };
|
||||
type HypeRow = { technology: string; hype_phase: string; hype_score: string };
|
||||
type AiRow = { speed_tier: string; total_tx: string; cluster_count: string };
|
||||
type StockRow = { speed_gbps: string; form_factor: string; out_of_stock: string; in_stock: string; total_obs: string };
|
||||
@ -1021,12 +861,9 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
const signals = (priceSignals.rows as PriceRow[])
|
||||
.map((r) => {
|
||||
const speed = parseFloat(r.speed_gbps);
|
||||
// Prefer the per-SKU median delta (composition-bias-free); fall back to aggregate
|
||||
const priceUp = r.sku_median_delta_pct != null
|
||||
? parseFloat(r.sku_median_delta_pct)
|
||||
: (r.avg_30d && r.avg_prior_30d
|
||||
? ((parseFloat(r.avg_30d) - parseFloat(r.avg_prior_30d)) / parseFloat(r.avg_prior_30d)) * 100
|
||||
: 0);
|
||||
const priceUp = r.avg_30d && r.avg_prior_30d
|
||||
? ((parseFloat(r.avg_30d) - parseFloat(r.avg_prior_30d)) / parseFloat(r.avg_prior_30d)) * 100
|
||||
: 0;
|
||||
const hype = speedToHype.get(speed);
|
||||
const ai = aiBySpeed.get(speed);
|
||||
const stock = stockByKey.get(`${r.speed_gbps}:${r.form_factor}`);
|
||||
@ -1050,7 +887,7 @@ procurementRouter.get("/supply-squeeze", async (_req: Request, res: Response) =>
|
||||
activeSignals, severity, reasons,
|
||||
};
|
||||
})
|
||||
.filter((r) => r.activeSignals >= 1 && parseFloat(String(r.speed_gbps)) > 0)
|
||||
.filter((r) => r.activeSignals >= 1)
|
||||
.sort((a, b) => b.activeSignals - a.activeSignals || b.price_momentum_pct - a.price_momentum_pct);
|
||||
|
||||
res.json({ success: true, signals, criticalCount: signals.filter(s => s.severity === "critical").length });
|
||||
@ -1246,68 +1083,41 @@ procurementRouter.get("/price-movers", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
WITH cur AS (
|
||||
-- Group by part_number+source_vendor+currency to avoid duplicates from multiple
|
||||
-- vendor-OEM transceiver_ids with the same part number.
|
||||
-- Use PERCENTILE_CONT (median) to suppress multi-tier list-price noise
|
||||
-- (e.g. Mouser 1x/10x/100x tiers appearing as price swings).
|
||||
SELECT t.part_number, po.source_vendor_id, po.currency,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) AS med_price,
|
||||
SELECT transceiver_id, source_vendor_id, currency,
|
||||
AVG(price) AS avg_price,
|
||||
COUNT(*) AS obs
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '${days} days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
GROUP BY t.part_number, po.source_vendor_id, po.currency
|
||||
FROM price_observations
|
||||
WHERE time >= NOW() - INTERVAL '${days} days'
|
||||
AND price > 0 AND COALESCE(is_anomalous, false) = false
|
||||
GROUP BY transceiver_id, source_vendor_id, currency
|
||||
),
|
||||
prior AS (
|
||||
SELECT t.part_number, po.source_vendor_id,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY po.price) AS med_price,
|
||||
COUNT(*) AS obs
|
||||
FROM price_observations po
|
||||
JOIN transceivers t ON t.id = po.transceiver_id
|
||||
WHERE po.time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND po.time < NOW() - INTERVAL '${days} days'
|
||||
AND po.price > 0 AND COALESCE(po.is_anomalous, false) = false
|
||||
GROUP BY t.part_number, po.source_vendor_id
|
||||
),
|
||||
ref_tx AS (
|
||||
-- pick one canonical transceiver_id per part_number for metadata
|
||||
SELECT DISTINCT ON (part_number) id, part_number, form_factor, speed_gbps, standard_name
|
||||
FROM transceivers ORDER BY part_number, id
|
||||
SELECT transceiver_id, source_vendor_id,
|
||||
AVG(price) AS avg_price
|
||||
FROM price_observations
|
||||
WHERE time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND time < NOW() - INTERVAL '${days} days'
|
||||
AND price > 0 AND COALESCE(is_anomalous, false) = false
|
||||
GROUP BY transceiver_id, source_vendor_id
|
||||
)
|
||||
SELECT
|
||||
ref.id, ref.part_number, ref.form_factor,
|
||||
ref.speed_gbps::text AS speed_gbps,
|
||||
ref.standard_name,
|
||||
t.id, t.part_number, t.form_factor,
|
||||
t.speed_gbps::text AS speed_gbps,
|
||||
t.standard_name,
|
||||
sv.name AS vendor_name,
|
||||
ROUND(c.med_price::numeric, 2) AS current_avg,
|
||||
ROUND(p.med_price::numeric, 2) AS prior_avg,
|
||||
ROUND(((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100)::numeric, 1) AS delta_pct,
|
||||
ROUND(c.avg_price::numeric, 2) AS current_avg,
|
||||
ROUND(p.avg_price::numeric, 2) AS prior_avg,
|
||||
ROUND(((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100)::numeric, 1) AS delta_pct,
|
||||
c.currency,
|
||||
(c.obs + p.obs)::int AS observations
|
||||
c.obs::int AS observations
|
||||
FROM cur c
|
||||
JOIN prior p ON p.part_number = c.part_number
|
||||
AND p.source_vendor_id = c.source_vendor_id
|
||||
JOIN ref_tx ref ON ref.part_number = c.part_number
|
||||
JOIN vendors sv ON sv.id = c.source_vendor_id
|
||||
-- cv_filter: exclude SKUs where the source has high price variance across the full
|
||||
-- 2*days window (e.g. Mouser quantity-tier noise). CV > 0.35 = unreliable source.
|
||||
JOIN (
|
||||
SELECT t2.part_number, po2.source_vendor_id,
|
||||
STDDEV(po2.price) / NULLIF(AVG(po2.price), 0) AS cv
|
||||
FROM price_observations po2
|
||||
JOIN transceivers t2 ON t2.id = po2.transceiver_id
|
||||
WHERE po2.time >= NOW() - INTERVAL '${days * 2} days'
|
||||
AND po2.price > 0 AND COALESCE(po2.is_anomalous, false) = false
|
||||
GROUP BY t2.part_number, po2.source_vendor_id
|
||||
HAVING COUNT(*) >= 2
|
||||
) cv_filter
|
||||
ON cv_filter.part_number = c.part_number
|
||||
AND cv_filter.source_vendor_id = c.source_vendor_id
|
||||
WHERE ABS((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100) >= 2
|
||||
AND (c.obs + p.obs) >= 4
|
||||
AND COALESCE(cv_filter.cv, 0) <= 0.35
|
||||
ORDER BY ABS((c.med_price - p.med_price) / NULLIF(p.med_price, 0) * 100) DESC
|
||||
JOIN prior p ON p.transceiver_id = c.transceiver_id
|
||||
AND p.source_vendor_id = c.source_vendor_id
|
||||
JOIN transceivers t ON t.id = c.transceiver_id
|
||||
JOIN vendors sv ON sv.id = c.source_vendor_id
|
||||
WHERE ABS((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100) >= 2
|
||||
AND c.obs::int >= 2
|
||||
ORDER BY ABS((c.avg_price - p.avg_price) / NULLIF(p.avg_price, 0) * 100) DESC
|
||||
LIMIT ${limit * 2}
|
||||
`);
|
||||
|
||||
|
||||
@ -1,210 +0,0 @@
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const researchRobotRouter = Router();
|
||||
|
||||
// Known job queues are validated against pgboss.queue before any action.
|
||||
async function isKnownQueue(name: string): Promise<boolean> {
|
||||
const r = await pool.query("SELECT 1 FROM pgboss.queue WHERE name = $1", [name]);
|
||||
return (r.rowCount ?? 0) > 0;
|
||||
}
|
||||
|
||||
type Recommendation = {
|
||||
job: string;
|
||||
severity: "critical" | "warning" | "info";
|
||||
title: string;
|
||||
detail: string;
|
||||
cause: string;
|
||||
// Which actions the UI should offer for this job
|
||||
actions: Array<"dispatch" | "pause" | "resume" | "token_help">;
|
||||
failed_count?: number;
|
||||
last_error?: string;
|
||||
};
|
||||
|
||||
// Classify a job error message into a human cause + recommended actions.
|
||||
function classifyError(jobName: string, errorText: string): Omit<Recommendation, "job" | "failed_count" | "last_error"> {
|
||||
const e = (errorText || "").toLowerCase();
|
||||
|
||||
if (/http 401|http 403|unauthorized|forbidden|invalid.*token|api.*key/.test(e)) {
|
||||
return {
|
||||
severity: "critical",
|
||||
title: `${jobName}: Authentifizierung fehlgeschlagen`,
|
||||
detail: "Der Job erreicht die Quelle, wird aber abgewiesen (401/403). Meist ein fehlender oder abgelaufener API-Token. Ohne gültigen Token reiht der Robot den Job endlos neu ein und er schlägt jedes Mal fehl.",
|
||||
cause: "auth",
|
||||
actions: ["token_help", "pause"],
|
||||
};
|
||||
}
|
||||
if (/timeout|etimedout|econnrefused|enotfound|network|socket hang up|fetch failed/.test(e)) {
|
||||
return {
|
||||
severity: "warning",
|
||||
title: `${jobName}: Quelle nicht erreichbar`,
|
||||
detail: "Netzwerk-/Verbindungsfehler beim Abruf. Oft temporär (Ziel-Server kurz down, Rate-Limit). Manuell erneut auslösen oder pausieren, falls die Quelle dauerhaft weg ist.",
|
||||
cause: "network",
|
||||
actions: ["dispatch", "pause"],
|
||||
};
|
||||
}
|
||||
if (/no handler|not registered|unknown job|kein handler/.test(e)) {
|
||||
return {
|
||||
severity: "critical",
|
||||
title: `${jobName}: Kein Handler registriert`,
|
||||
detail: "Der Job ist im Schedule, aber es gibt keinen Code-Handler dafür. Er kann nie laufen — entweder Handler nachrüsten oder den Job aus dem Schedule entfernen.",
|
||||
cause: "no_handler",
|
||||
actions: ["pause"],
|
||||
};
|
||||
}
|
||||
return {
|
||||
severity: "warning",
|
||||
title: `${jobName}: Wiederholt fehlgeschlagen`,
|
||||
detail: "Der Job schlägt regelmäßig fehl. Letzte Fehlermeldung siehe unten. Manuell auslösen zum erneuten Versuch, oder pausieren falls überflüssig.",
|
||||
cause: "other",
|
||||
actions: ["dispatch", "pause"],
|
||||
};
|
||||
}
|
||||
|
||||
// GET /api/research-robot — letzte Robot-Laeufe + abgeleitete Handlungsempfehlungen
|
||||
researchRobotRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = await pool.query(
|
||||
"SELECT run_at, freshness, decision, dispatched, model FROM research_robot_runs ORDER BY run_at DESC LIMIT 20"
|
||||
);
|
||||
|
||||
// Build recommendations from LIVE pgboss state (more accurate than the stored summary).
|
||||
const recommendations: Recommendation[] = [];
|
||||
try {
|
||||
const failing = await pool.query<{ name: string; failed: string; last_error: string | null }>(`
|
||||
SELECT j.name,
|
||||
COUNT(*) FILTER (WHERE j.state = 'failed') AS failed,
|
||||
(SELECT jj.output::text FROM pgboss.job jj
|
||||
WHERE jj.name = j.name AND jj.state = 'failed' AND jj.output IS NOT NULL
|
||||
ORDER BY jj.created_on DESC LIMIT 1) AS last_error
|
||||
FROM pgboss.job j
|
||||
WHERE j.created_on > NOW() - INTERVAL '3 days'
|
||||
GROUP BY j.name
|
||||
HAVING COUNT(*) FILTER (WHERE j.state = 'failed') >= 2
|
||||
AND COUNT(*) FILTER (WHERE j.state = 'completed') = 0
|
||||
ORDER BY failed DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
// Which jobs are currently scheduled (so we can offer pause vs resume correctly)
|
||||
const scheduled = await pool.query<{ name: string }>("SELECT name FROM pgboss.schedule");
|
||||
const scheduledSet = new Set(scheduled.rows.map((s) => s.name));
|
||||
|
||||
for (const row of failing.rows) {
|
||||
let errMsg = "";
|
||||
try {
|
||||
const parsed = JSON.parse(row.last_error || "{}") as { stack?: string; message?: string };
|
||||
errMsg = (parsed.stack || parsed.message || row.last_error || "").slice(0, 300);
|
||||
} catch {
|
||||
errMsg = (row.last_error || "").slice(0, 300);
|
||||
}
|
||||
const cls = classifyError(row.name, errMsg);
|
||||
// If the job isn't scheduled, swap pause→resume in the offered actions
|
||||
const actions = cls.actions.map((a) =>
|
||||
a === "pause" && !scheduledSet.has(row.name) ? "resume" : a
|
||||
) as Recommendation["actions"];
|
||||
recommendations.push({
|
||||
job: row.name,
|
||||
severity: cls.severity,
|
||||
title: cls.title,
|
||||
detail: cls.detail,
|
||||
cause: cls.cause,
|
||||
actions: [...new Set(actions)],
|
||||
failed_count: parseInt(row.failed, 10),
|
||||
last_error: errMsg.split("\n")[0].slice(0, 160),
|
||||
});
|
||||
}
|
||||
} catch { /* pgboss introspection best-effort */ }
|
||||
|
||||
if (recommendations.length === 0) {
|
||||
recommendations.push({
|
||||
job: "",
|
||||
severity: "info",
|
||||
title: "Alle Scraper-Jobs gesund",
|
||||
detail: "Keine dauerhaft fehlschlagenden Jobs erkannt. Keine Aktion nötig.",
|
||||
cause: "healthy",
|
||||
actions: [],
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ success: true, runs: r.rows, recommendations });
|
||||
} catch (e) {
|
||||
res.json({ success: true, runs: [], recommendations: [] });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/research-robot/action — { action, job }
|
||||
// dispatch → Job sofort in die Queue stellen; pause → aus Schedule entfernen (Backup);
|
||||
// resume → aus Backup wiederherstellen.
|
||||
researchRobotRouter.post("/action", async (req: Request, res: Response) => {
|
||||
const action = String((req.body?.action ?? "")).trim();
|
||||
const job = String((req.body?.job ?? "")).trim();
|
||||
|
||||
if (!["dispatch", "pause", "resume"].includes(action)) {
|
||||
res.status(400).json({ success: false, error: "Unbekannte Aktion." });
|
||||
return;
|
||||
}
|
||||
if (!job || !(await isKnownQueue(job))) {
|
||||
res.status(400).json({ success: false, error: "Unbekannter oder ungültiger Job." });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (action === "dispatch") {
|
||||
// Enqueue one job run immediately (pg-boss v10 standard policy, mirrors a scheduled send)
|
||||
await pool.query(
|
||||
`INSERT INTO pgboss.job (id, name, data, priority, retry_limit, retry_delay, expire_in, keep_until, policy, start_after)
|
||||
VALUES (gen_random_uuid(), $1, '{}'::jsonb, 0, 2, 0, INTERVAL '15 minutes', NOW() + INTERVAL '14 days', 'standard', NOW())`,
|
||||
[job]
|
||||
);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' wurde in die Queue gestellt und läuft beim nächsten Worker-Tick.` });
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure backup table for paused schedules exists
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS research_robot_paused_schedules (
|
||||
name TEXT PRIMARY KEY,
|
||||
cron TEXT, timezone TEXT, data JSONB, options JSONB,
|
||||
paused_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)`);
|
||||
|
||||
if (action === "pause") {
|
||||
const sched = await pool.query("SELECT name, cron, timezone, data, options FROM pgboss.schedule WHERE name = $1", [job]);
|
||||
if (sched.rowCount === 0) {
|
||||
res.json({ success: true, action, job, message: `Job '${job}' war nicht im Schedule — bereits pausiert.` });
|
||||
return;
|
||||
}
|
||||
const s = sched.rows[0];
|
||||
await pool.query(
|
||||
`INSERT INTO research_robot_paused_schedules (name, cron, timezone, data, options, paused_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (name) DO UPDATE SET cron=$2, timezone=$3, data=$4, options=$5, paused_at=NOW()`,
|
||||
[s.name, s.cron, s.timezone, s.data, s.options]
|
||||
);
|
||||
await pool.query("DELETE FROM pgboss.schedule WHERE name = $1", [job]);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' pausiert (aus Schedule entfernt). Der Robot reiht ihn nicht mehr neu ein. Reversibel über 'Fortsetzen'.` });
|
||||
return;
|
||||
}
|
||||
|
||||
if (action === "resume") {
|
||||
const bak = await pool.query("SELECT name, cron, timezone, data, options FROM research_robot_paused_schedules WHERE name = $1", [job]);
|
||||
if (bak.rowCount === 0) {
|
||||
res.status(404).json({ success: false, error: `Kein pausierter Schedule-Eintrag für '${job}' gefunden.` });
|
||||
return;
|
||||
}
|
||||
const s = bak.rows[0];
|
||||
await pool.query(
|
||||
`INSERT INTO pgboss.schedule (name, cron, timezone, data, options, created_on, updated_on)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW(),NOW())
|
||||
ON CONFLICT (name) DO UPDATE SET cron=$2, timezone=$3, data=$4, options=$5, updated_on=NOW()`,
|
||||
[s.name, s.cron, s.timezone, s.data, s.options]
|
||||
);
|
||||
await pool.query("DELETE FROM research_robot_paused_schedules WHERE name = $1", [job]);
|
||||
res.json({ success: true, action, job, message: `Job '${job}' wieder aktiviert (zurück im Schedule).` });
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: (e as Error).message });
|
||||
}
|
||||
});
|
||||
@ -1,58 +0,0 @@
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const stockCompetitorRouter = Router();
|
||||
|
||||
// GET /api/stock/competitor-by-tech
|
||||
// Returns publicly-available competitor stock levels aggregated by technology (form_factor+speed_gbps).
|
||||
// Used by Warehouse Stock Intelligence to benchmark Flexoptix demand against competitor availability.
|
||||
stockCompetitorRouter.get("/competitor-by-tech", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
t.form_factor,
|
||||
t.speed_gbps::numeric AS speed_gbps,
|
||||
sv.name AS vendor_name,
|
||||
COUNT(DISTINCT so.transceiver_id)::int AS skus,
|
||||
COALESCE(SUM(so.warehouse_de_qty),0)::int AS de_stock,
|
||||
COALESCE(SUM(so.warehouse_global_qty),0)::int AS global_stock,
|
||||
COALESCE(SUM(so.quantity_available),0)::int AS qty_available,
|
||||
MAX(so.time)::date AS last_seen
|
||||
FROM stock_observations so
|
||||
JOIN transceivers t ON t.id = so.transceiver_id
|
||||
JOIN vendors sv ON sv.id = so.source_vendor_id
|
||||
WHERE so.time > NOW() - INTERVAL '14 days'
|
||||
AND sv.name IN ('FS.COM','ATGBICS','FiberMall','NADDOD','Prolabs','Blue Optics','Skylane Optics')
|
||||
GROUP BY t.form_factor, t.speed_gbps, sv.name
|
||||
ORDER BY sv.name, global_stock DESC NULLS LAST
|
||||
`);
|
||||
|
||||
// Pivot: { "1G SFP": { "FS.COM": { de:..., global:... }, ... }, ... }
|
||||
const byTech: Record<string, Record<string, { de: number; global: number; skus: number; last_seen: string }>> = {};
|
||||
const vendors = new Set<string>();
|
||||
|
||||
for (const row of result.rows) {
|
||||
const n = parseFloat(row.speed_gbps);
|
||||
const spd = n >= 1000
|
||||
? ((n / 1000 * 10) % 10 === 0 ? Math.round(n / 1000).toString() : (n / 1000).toFixed(1)) + "T"
|
||||
: ((n * 10) % 10 === 0 ? Math.round(n).toString() : String(n)) + "G";
|
||||
const key = spd + " " + (row.form_factor || "?");
|
||||
if (!byTech[key]) byTech[key] = {};
|
||||
byTech[key][row.vendor_name] = {
|
||||
de: parseInt(row.de_stock) || 0,
|
||||
global: parseInt(row.global_stock) || parseInt(row.qty_available) || 0,
|
||||
skus: parseInt(row.skus) || 0,
|
||||
last_seen: row.last_seen,
|
||||
};
|
||||
vendors.add(row.vendor_name);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
vendors: [...vendors].sort(),
|
||||
by_tech: byTech,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
@ -518,54 +518,6 @@ stockRouter.get("/velocity/:id", async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// ─── GET /api/stock/competitor-by-tech ───────────────────────────────────────
|
||||
// Competitor stock levels from publicly-scraped data, aggregated by technology.
|
||||
stockRouter.get("/competitor-by-tech", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
t.form_factor,
|
||||
t.speed_gbps::numeric AS speed_gbps,
|
||||
sv.name AS vendor_name,
|
||||
COUNT(DISTINCT so.transceiver_id)::int AS skus,
|
||||
COALESCE(SUM(so.warehouse_de_qty),0)::int AS de_stock,
|
||||
COALESCE(SUM(so.warehouse_global_qty),0)::int AS global_stock,
|
||||
COALESCE(SUM(so.quantity_available),0)::int AS qty_available,
|
||||
MAX(so.time)::date AS last_seen
|
||||
FROM stock_observations so
|
||||
JOIN transceivers t ON t.id = so.transceiver_id
|
||||
JOIN vendors sv ON sv.id = so.source_vendor_id
|
||||
WHERE so.time > NOW() - INTERVAL '14 days'
|
||||
AND sv.name IN ('FS.COM','ATGBICS','FiberMall','NADDOD','Prolabs','Blue Optics','Skylane Optics')
|
||||
GROUP BY t.form_factor, t.speed_gbps, sv.name
|
||||
ORDER BY sv.name, global_stock DESC NULLS LAST
|
||||
`);
|
||||
|
||||
const byTech: Record<string, Record<string, { de: number; global: number; skus: number; last_seen: string }>> = {};
|
||||
const vendors = new Set<string>();
|
||||
for (const row of result.rows) {
|
||||
const n = parseFloat(row.speed_gbps);
|
||||
const spd = n >= 1000
|
||||
? ((n / 1000 * 10) % 10 === 0 ? String(Math.round(n / 1000)) : (n / 1000).toFixed(1)) + "T"
|
||||
: ((n * 10) % 10 === 0 ? String(Math.round(n)) : String(n)) + "G";
|
||||
const key = spd + " " + (row.form_factor || "?");
|
||||
if (!byTech[key]) byTech[key] = {};
|
||||
byTech[key][row.vendor_name] = {
|
||||
de: parseInt(row.de_stock) || 0,
|
||||
global: parseInt(row.global_stock) || parseInt(row.qty_available) || 0,
|
||||
skus: parseInt(row.skus) || 0,
|
||||
last_seen: row.last_seen,
|
||||
};
|
||||
vendors.add(row.vendor_name);
|
||||
}
|
||||
res.json({ success: true, vendors: [...vendors].sort(), by_tech: byTech });
|
||||
} catch (err) {
|
||||
console.error("stock/competitor-by-tech error:", err);
|
||||
res.status(500).json({ success: false, error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /api/stock/:id ──────────────────────────────────────────────────────
|
||||
/**
|
||||
* Full observation history for one transceiver.
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Router, Request, Response, NextFunction } from "express";
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
import { listVendors } from "../db/queries";
|
||||
|
||||
@ -114,41 +114,7 @@ vendorRouter.post("/", async (req: Request, res: Response) => {
|
||||
});
|
||||
|
||||
// GET /api/vendors/:id — Get single vendor with full stats
|
||||
// GET /api/vendors/reliability — per-vendor data-reliability scores (freshness/frequency/coverage)
|
||||
vendorRouter.get("/reliability", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT v.id::text AS vendor_id, v.name AS vendor_name,
|
||||
COUNT(DISTINCT t.id)::int AS sku_count,
|
||||
COUNT(po.transceiver_id)::int AS obs_count,
|
||||
MAX(po.time) AS last_obs
|
||||
FROM vendors v
|
||||
JOIN transceivers t ON t.vendor_id = v.id
|
||||
LEFT JOIN price_observations po ON po.transceiver_id = t.id
|
||||
GROUP BY v.id, v.name
|
||||
HAVING COUNT(DISTINCT t.id) > 0`);
|
||||
const rows = result.rows as Array<{ vendor_id: string; vendor_name: string; sku_count: number; obs_count: number; last_obs: string | null }>;
|
||||
const maxSku = Math.max(1, ...rows.map((r) => r.sku_count));
|
||||
const maxObs = Math.max(1, ...rows.map((r) => r.obs_count));
|
||||
const now = Date.now();
|
||||
const vendors = rows.map((r) => {
|
||||
const daysStale = r.last_obs ? Math.max(0, (now - new Date(r.last_obs).getTime()) / 86400000) : 999;
|
||||
const freshness_score = Math.max(0, Math.min(100, Math.round(100 - daysStale * 3)));
|
||||
const frequency_score = Math.round(100 * Math.min(1, Math.log10(r.obs_count + 1) / Math.log10(maxObs + 1)));
|
||||
const coverage_score = Math.round(100 * Math.min(1, r.sku_count / maxSku));
|
||||
const reliability_score = Math.round(0.4 * freshness_score + 0.3 * frequency_score + 0.3 * coverage_score);
|
||||
return { vendor_id: r.vendor_id, vendor_name: r.vendor_name, sku_count: r.sku_count, obs_count: r.obs_count, last_obs: r.last_obs, freshness_score, frequency_score, coverage_score, reliability_score };
|
||||
});
|
||||
vendors.sort((a, b) => b.reliability_score - a.reliability_score);
|
||||
res.json({ success: true, count: vendors.length, vendors });
|
||||
} catch (err) {
|
||||
console.error("vendor reliability error:", err);
|
||||
res.status(500).json({ success: false, error: "Failed to compute vendor reliability" });
|
||||
}
|
||||
});
|
||||
|
||||
vendorRouter.get("/:id", async (req: Request, res: Response, next: NextFunction) => {
|
||||
if (req.params.id === "market-share" || req.params.id === "intelligence") return next();
|
||||
vendorRouter.get("/:id", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const result = await pool.query(
|
||||
@ -215,8 +181,8 @@ vendorRouter.get("/market-share", async (req: Request, res: Response) => {
|
||||
v.name AS vendor_name,
|
||||
v.type,
|
||||
COUNT(DISTINCT po.transceiver_id)::int AS sku_count,
|
||||
ROUND((COUNT(DISTINCT po.transceiver_id)::numeric / NULLIF(t.total,0)::numeric) * 100, 1) AS market_share_pct,
|
||||
COUNT(*)::int AS total_obs,
|
||||
ROUND((COUNT(DISTINCT po.transceiver_id)::numeric / NULLIF(t.total,0)) * 100, 1) AS market_share_pct,
|
||||
COUNT(po.id)::int AS total_obs,
|
||||
MAX(po.time) AS last_seen
|
||||
FROM price_observations po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
@ -262,7 +228,7 @@ vendorRouter.get("/market-share", async (req: Request, res: Response) => {
|
||||
(c.sku_count - COALESCE(p.sku_count, 0)) AS delta_skus,
|
||||
CASE
|
||||
WHEN COALESCE(p.sku_count, 0) = 0 THEN NULL
|
||||
ELSE ROUND(((c.sku_count - p.sku_count)::numeric / p.sku_count::numeric) * 100, 1)
|
||||
ELSE ROUND(((c.sku_count - p.sku_count)::numeric / p.sku_count) * 100, 1)
|
||||
END AS delta_pct
|
||||
FROM cur c
|
||||
JOIN vendors v ON v.id = c.source_vendor_id
|
||||
@ -307,7 +273,7 @@ vendorRouter.get("/intelligence", async (_req: Request, res: Response) => {
|
||||
v.type,
|
||||
v.website,
|
||||
COUNT(DISTINCT po.transceiver_id)::int AS sku_count,
|
||||
COUNT(*)::int AS price_obs,
|
||||
COUNT(po.id)::int AS price_obs,
|
||||
ROUND(AVG(po.price)::numeric, 2) AS avg_price,
|
||||
ROUND(MIN(po.price)::numeric, 2) AS min_price,
|
||||
ROUND(MAX(po.price)::numeric, 2) AS max_price,
|
||||
|
||||
@ -722,18 +722,6 @@
|
||||
|
||||
<!-- Auth guard — redirect to login if no valid token -->
|
||||
<script>
|
||||
// fmtSpd: clean speed display — 1.00->1G, 400.00->400G, 1600->1.6T, 2.5->2.5G
|
||||
function fmtSpd(gbps) {
|
||||
if (gbps == null || gbps === '') return '?';
|
||||
var n = parseFloat(gbps);
|
||||
if (isNaN(n) || n === 0) return '?';
|
||||
if (n >= 1000) {
|
||||
var t = n / 1000;
|
||||
return ((t * 10) % 10 === 0 ? String(Math.round(t)) : t.toFixed(1)) + 'T';
|
||||
}
|
||||
return ((n * 10) % 10 === 0 ? String(Math.round(n)) : String(n)) + 'G';
|
||||
}
|
||||
|
||||
// ── Token storage helpers — never store plaintext ──────────────────────────
|
||||
(function() {
|
||||
var _K = 'tip_v3_tk';
|
||||
@ -913,8 +901,6 @@ function fmtSpd(gbps) {
|
||||
</div>
|
||||
<div id="ov-movers-inner" class="mt" style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem"></div>
|
||||
</div>
|
||||
<!-- RESEARCH ROBOT (overview) -->
|
||||
<div class="card mb" id="research-robot-card" style="display:none"></div>
|
||||
<!-- RESEARCH STATUS -->
|
||||
<div class="card mb" id="verification-card">
|
||||
<div class="card-label">Data Research Status</div>
|
||||
@ -1770,16 +1756,6 @@ function fmtSpd(gbps) {
|
||||
<!-- PROCUREMENT INTEL TAB -->
|
||||
<div id="tab-procurement" class="hidden">
|
||||
|
||||
<!-- Transceiver-Suche -->
|
||||
<div style="margin-bottom:1.25rem;border:1px solid var(--border);border-radius:10px;padding:0.9rem 1rem;background:var(--surface2)">
|
||||
<div style="font-size:0.8rem;font-weight:700;color:var(--text-bright);margin-bottom:0.5rem">🔎 Transceiver suchen — Preise, Anbieter, Verfügbarkeit & Signale</div>
|
||||
<div style="display:flex;gap:0.5rem;flex-wrap:wrap">
|
||||
<input id="proc-tx-search" type="text" onkeydown="if(event.key==='Enter')procTxSearch()" placeholder="Part-Number oder Name, z.B. SFP-10G-LR, QSFP28-100G…" style="flex:1;min-width:240px;padding:0.5rem 0.7rem;border:1px solid var(--border);border-radius:7px;background:var(--surface);color:var(--text);font-size:0.85rem">
|
||||
<button onclick="procTxSearch()" style="padding:0.5rem 1.1rem;border:none;border-radius:7px;background:var(--accent);color:#fff;font-weight:600;font-size:0.85rem;cursor:pointer">Suchen</button>
|
||||
</div>
|
||||
<div id="proc-tx-results" style="margin-top:0.6rem"></div>
|
||||
</div>
|
||||
|
||||
<!-- Sub-nav -->
|
||||
<div style="display:flex;gap:0.5rem;margin-bottom:1.25rem;flex-wrap:wrap;align-items:center">
|
||||
<button onclick="showProcSection('signals')" id="proc-btn-signals" class="proc-btn proc-btn-active">Reorder Signals</button>
|
||||
@ -2355,12 +2331,10 @@ function fmtSpd(gbps) {
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Momentum</th>
|
||||
<th style="padding:6px 8px;text-align:center;color:var(--text-dim);font-weight:500">Trend</th>
|
||||
<th style="padding:6px 8px;text-align:center;color:var(--text-dim);font-weight:500">Fast Movers</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:#06b6d4;font-weight:500;font-size:0.7rem;white-space:nowrap" title="FS.COM DE-Lager (14-Tage-Schnitt)">FS.COM DE</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:#0ea5e9;font-weight:500;font-size:0.7rem;white-space:nowrap" title="FS.COM Global-Lager (14-Tage-Schnitt)">FS.COM Global</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="foxd-by-speed-body">
|
||||
<tr><td colspan="9" style="text-align:center;padding:2rem;color:var(--text-dim)">Lade Flexoptix Demand-Daten…</td></tr>
|
||||
<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">Lade Flexoptix Demand-Daten…</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@ -3545,7 +3519,6 @@ async function loadOverview() {
|
||||
|
||||
// Async fire — don't block overview render
|
||||
loadProcurementPulse();
|
||||
loadResearchRobot();
|
||||
}
|
||||
|
||||
// SEARCH
|
||||
@ -4634,14 +4607,11 @@ async function openTxDetail(id) {
|
||||
h += '<div style="font-size:0.72rem;color:#888;margin-bottom:0.5rem">Gleiche Spezifikationsklasse — andere Part Number</div>';
|
||||
comparPrices.forEach(function(p) {
|
||||
// Calculate price delta (EUR-normalized)
|
||||
// Use Flexoptix own verified EUR price as reference for comparison
|
||||
var myEur = t.price_verified_eur ? parseFloat(t.price_verified_eur) : null;
|
||||
if (!myEur) {
|
||||
var refPrice = directPrices.length > 0 ? directPrices[0] : null;
|
||||
if (refPrice) {
|
||||
var ra = parseFloat(refPrice.price), rc = (refPrice.currency||'USD').toUpperCase();
|
||||
myEur = rc === 'EUR' ? ra : rc === 'USD' ? ra * 0.92 : ra;
|
||||
}
|
||||
var myEur = null;
|
||||
var refPrice = directPrices.length > 0 ? directPrices[0] : null;
|
||||
if (refPrice) {
|
||||
var ra = parseFloat(refPrice.price), rc = (refPrice.currency||'USD').toUpperCase();
|
||||
myEur = rc === 'EUR' ? ra : rc === 'USD' ? ra * 0.92 : ra;
|
||||
}
|
||||
var compEur = null;
|
||||
var ca = parseFloat(p.price), cc = (p.currency||'USD').toUpperCase();
|
||||
@ -4669,8 +4639,8 @@ async function openTxDetail(id) {
|
||||
+ '<td style="color:' + compColor + ';font-size:0.7rem;padding:2px 0">' + esc(compVal || '—') + '</td></tr>';
|
||||
}
|
||||
|
||||
var mySpeed = fmtSpd(t.speed_gbps);
|
||||
var compSpeed = p.comp_speed_gbps ? fmtSpd(p.comp_speed_gbps) : null;
|
||||
var mySpeed = t.speed_gbps >= 1000 ? (t.speed_gbps / 1000).toFixed(1).replace('.0','') + 'T' : t.speed_gbps + 'G';
|
||||
var compSpeed = p.comp_speed_gbps ? (p.comp_speed_gbps >= 1000 ? (p.comp_speed_gbps/1000).toFixed(1).replace('.0','')+'T' : p.comp_speed_gbps+'G') : null;
|
||||
|
||||
h += '<div style="border:1px solid var(--border);border-radius:8px;margin-bottom:0.6rem;overflow:hidden">';
|
||||
|
||||
@ -4691,8 +4661,7 @@ async function openTxDetail(id) {
|
||||
if (savBadge) {
|
||||
h += '<div style="padding:0.3rem 0.75rem;background:rgba(255,255,255,0.02);border-bottom:1px solid var(--border);display:flex;align-items:center;gap:0.5rem">';
|
||||
h += savBadge;
|
||||
var fxPriceLabel = myEur ? 'EUR ' + myEur.toLocaleString('de-DE',{minimumFractionDigits:2,maximumFractionDigits:2}) : null;
|
||||
h += '<span style="font-size:0.68rem;color:#666">vs. Flexoptix Listenpreis' + (fxPriceLabel ? ' (' + fxPriceLabel + ')' : '') + '</span>';
|
||||
h += '<span style="font-size:0.68rem;color:#666">vs. Flexoptix Listenpreis</span>';
|
||||
h += '</div>';
|
||||
}
|
||||
|
||||
@ -5212,7 +5181,7 @@ function searchSwitches() {
|
||||
buildDOM(el('sw-table'), items.map(function(s) {
|
||||
var catColors = { DataCenter: 'b-blue', Campus: 'b-green', SP: 'b-purple', Core: 'b-orange', Edge: 'b-cyan', Industrial: 'b-yellow' };
|
||||
var statusColors = { Active: 'b-green', 'EoS_Announced': 'b-yellow', EoL: 'b-red', Legacy: 'b-neutral' };
|
||||
var maxSpd = fmtSpd(s.max_speed_gbps);
|
||||
var maxSpd = s.max_speed_gbps >= 1000 ? (s.max_speed_gbps/1000) + 'T' : s.max_speed_gbps + 'G';
|
||||
var cap = s.switching_capacity_tbps ? s.switching_capacity_tbps + ' Tbps' : '—';
|
||||
// Thumbnail — show image if available, otherwise a switch icon
|
||||
var thumb = s.image_url
|
||||
@ -5466,7 +5435,12 @@ async function openSwitchDetail(id) {
|
||||
fch += '<div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.6rem">Passend für diesen Switch — FlexBox-Codierung möglich</div>';
|
||||
|
||||
// Format speed_gbps → "1.6T", "400G", "100G" etc.
|
||||
var fmtSpeed = fmtSpd; // use global
|
||||
function fmtSpeed(gbps) {
|
||||
if (!gbps) return '?';
|
||||
var n = parseFloat(gbps);
|
||||
if (n >= 1000) return (n / 1000) + 'T';
|
||||
return Math.round(n) + 'G';
|
||||
}
|
||||
|
||||
// Group by speed class
|
||||
var foGroups = {};
|
||||
@ -5956,7 +5930,7 @@ function renderFormFactors(items) {
|
||||
var sCl = statusColors[f.status] || '#888';
|
||||
var fCl = familyColors[f.family] || '#888';
|
||||
var sLbl = statusLabels[f.status] || f.status || '';
|
||||
var maxSpd = fmtSpd(f.max_speed_gbps);
|
||||
var maxSpd = f.max_speed_gbps >= 1000 ? (f.max_speed_gbps/1000) + 'T' : (f.max_speed_gbps || '?') + 'G';
|
||||
// Description: show first part (DE) if bilingual
|
||||
var descFull = f.description || '';
|
||||
var descDE = descFull.split('//')[0].trim();
|
||||
@ -6937,7 +6911,7 @@ async function runFinder() {
|
||||
'<div style="font-size:1.1rem;font-weight:700">' + sw.vendor + ' ' + sw.model + '</div>' +
|
||||
'<div style="color:var(--text-dim);font-size:0.8rem">' +
|
||||
(sw.series ? sw.series + ' · ' : '') +
|
||||
'Max speed: ' + fmtSpd(sw.max_speed_gbps) +
|
||||
'Max speed: ' + (sw.max_speed_gbps || '?') + 'G' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'<div style="text-align:right;font-size:0.8rem;color:var(--text-dim)">' +
|
||||
@ -6953,7 +6927,7 @@ async function runFinder() {
|
||||
// Group by speed class
|
||||
var bySpeed = {};
|
||||
for (var t of transceivers) {
|
||||
var key = fmtSpd(t.speed_gbps) + ' ' + t.form_factor;
|
||||
var key = t.speed_gbps + 'G ' + t.form_factor;
|
||||
if (!bySpeed[key]) bySpeed[key] = [];
|
||||
bySpeed[key].push(t);
|
||||
}
|
||||
@ -7990,7 +7964,7 @@ async function loadReorderTop() {
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-weight:700;color:var(--text-bright);font-size:0.78rem">'+esc(r.vendor_name||'')+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-family:monospace;font-size:0.72rem;color:var(--text-dim)">'+esc(r.part_number||'')+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:center"><span style="background:rgba(99,102,241,0.15);color:#818cf8;border-radius:4px;padding:2px 6px;font-size:0.7rem">'+esc(r.form_factor||'')+'</span></td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.78rem">'+fmtSpd(r.speed_gbps)+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.78rem">'+esc(String(r.speed_gbps||''))+'G</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:center;font-weight:700;color:'+strColor+';font-family:monospace">'+str+'%</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-size:0.7rem">'+pt+' Preis · '+st+' Stock</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-size:0.7rem;color:var(--text-dim);max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="'+esc(reasons)+'">'+esc(reasons.substring(0,80))+'</td>'
|
||||
@ -8106,7 +8080,7 @@ async function loadSwitchCompat() {
|
||||
+ '<td style="padding:0.4rem 0.5rem;font-weight:600;font-size:0.75rem;color:var(--text-bright)">'+esc(t.vendor_name)+'</td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;font-family:monospace;font-size:0.7rem;color:var(--text-dim)">'+esc(t.part_number||'')+'</td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;text-align:center"><span style="background:rgba(99,102,241,0.15);color:#818cf8;border-radius:4px;padding:1px 5px;font-size:0.68rem">'+esc(t.form_factor)+'</span></td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.72rem">'+fmtSpd(t.speed_gbps)+'</td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.72rem">'+esc(String(t.speed_gbps))+'G</td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;text-align:right;color:var(--green);font-size:0.72rem">'+esc(priceStr)+'</td>'
|
||||
+ '<td style="padding:0.4rem 0.5rem;text-align:center;font-size:0.68rem;color:var(--text-dim)">'+esc(t.verification_method||'')+'</td>'
|
||||
+ '</tr>';
|
||||
@ -8129,51 +8103,12 @@ async function loadSwitchCompat() {
|
||||
}
|
||||
|
||||
/* ── C: Supply Squeeze Detector ────────────────────────────────────────── */
|
||||
|
||||
/* ── Verfügbarkeit pro Speed-Tier (mit Warum-Begründung) ────────────────── */
|
||||
async function renderAvailabilityPanel() {
|
||||
var host = el('proc-squeeze-summary');
|
||||
if (!host) return;
|
||||
try {
|
||||
var d = await api('/api/procurement/availability');
|
||||
var tiers = (d && d.tiers) || [];
|
||||
if (!tiers.length) return;
|
||||
var meta = {
|
||||
scarce: { c:'#ef4444', bg:'rgba(239,68,68,0.10)', lbl:'KNAPP' },
|
||||
constrained: { c:'#f97316', bg:'rgba(249,115,22,0.10)', lbl:'eng' },
|
||||
moderate: { c:'#f59e0b', bg:'rgba(245,158,11,0.10)', lbl:'moderat' },
|
||||
abundant: { c:'#22c55e', bg:'rgba(34,197,94,0.10)', lbl:'breit' }
|
||||
};
|
||||
function spd(g){ return g>=1000 ? (g/1000)+'T' : g+'G'; }
|
||||
var html = '<div style="margin-bottom:1rem">'
|
||||
+ '<div style="font-size:0.8rem;font-weight:700;color:var(--text-bright);margin-bottom:0.5rem">📦 Verfügbarkeit nach Geschwindigkeit <span style="font-weight:400;color:var(--text-dim);font-size:0.7rem">— aus Anbieter-Diversität, Stock-Coverage & Preis-Momentum (echte Daten)</span></div>'
|
||||
+ '<div style="display:flex;gap:0.5rem;flex-wrap:wrap">';
|
||||
tiers.forEach(function(t) {
|
||||
var m = meta[t.availability] || meta.moderate;
|
||||
var pt = t.price_trend, pd = t.price_delta_pct;
|
||||
var priceTag = pt==='rising' ? '<span style="color:#ef4444">▲ +'+pd+'%</span>' : pt==='falling' ? '<span style="color:#22c55e">▼ '+pd+'%</span>' : '<span style="color:var(--text-dim)">→ stabil</span>';
|
||||
html += '<div title="' + esc(t.why || '') + '" style="flex:1;min-width:150px;border:1px solid '+m.c+'44;background:'+m.bg+';border-radius:9px;padding:0.6rem 0.7rem">'
|
||||
+ '<div style="display:flex;align-items:baseline;justify-content:space-between">'
|
||||
+ '<span style="font-size:1.05rem;font-weight:800;color:var(--text-bright)">'+spd(t.speed_gbps)+'</span>'
|
||||
+ '<span style="font-size:0.62rem;font-weight:800;color:'+m.c+';text-transform:uppercase;letter-spacing:0.03em">'+m.lbl+'</span>'
|
||||
+ '</div>'
|
||||
+ '<div style="font-size:0.68rem;color:var(--text-dim);margin-top:2px">'+t.suppliers+' Anbieter · '+(t.in_stock_pct!=null?t.in_stock_pct+'% stock':'k.A.')+' · '+priceTag+'</div>'
|
||||
+ '<div style="font-size:0.66rem;color:var(--text-dim);margin-top:5px;line-height:1.35">'+esc((t.why||'').charAt(0).toUpperCase()+(t.why||'').slice(1))+'</div>'
|
||||
+ '</div>';
|
||||
});
|
||||
html += '</div></div>';
|
||||
// prepend to summary (before the squeeze severity badges, which get set after)
|
||||
host.insertAdjacentHTML('afterbegin', html);
|
||||
} catch(e) { /* silent */ }
|
||||
}
|
||||
|
||||
async function loadSupplySqueeze() {
|
||||
var token = (window.loadToken ? window.loadToken() : '') || '';
|
||||
var summEl = el('proc-squeeze-summary');
|
||||
var listEl = el('proc-squeeze-list');
|
||||
if (listEl) listEl.innerHTML = '<div style="color:var(--text-dim)">Analysiere Preis- & Nachfragesignale…</div>';
|
||||
try {
|
||||
renderAvailabilityPanel();
|
||||
var r = await fetch('/api/procurement/supply-squeeze', { headers: { 'Authorization': 'Bearer ' + token } });
|
||||
var d = await r.json();
|
||||
if (!d.success) throw new Error(d.error);
|
||||
@ -8194,7 +8129,7 @@ async function loadSupplySqueeze() {
|
||||
var mom = s.price_momentum_pct !== 0 ? (s.price_momentum_pct > 0 ? '+' : '') + s.price_momentum_pct + '%' : '—';
|
||||
var momColor = s.price_momentum_pct > 10 ? '#ef4444' : s.price_momentum_pct > 0 ? '#f59e0b' : '#22c55e';
|
||||
return '<tr style="border-bottom:1px solid var(--border)">'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-weight:700;font-size:0.8rem;color:'+col+'">'+icon+' '+fmtSpd(s.speed_gbps)+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-weight:700;font-size:0.8rem;color:'+col+'">'+icon+' '+esc(String(s.speed_gbps))+'G</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem"><span style="background:rgba(99,102,241,0.15);color:#818cf8;border-radius:4px;padding:2px 6px;font-size:0.7rem">'+esc(s.form_factor||'—')+'</span></td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;font-weight:700;color:'+momColor+';font-family:monospace">'+esc(mom)+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-size:0.7rem;color:var(--text-dim)">'+esc((s.hype_phase||'—').replace(/_/g,' '))+'</td>'
|
||||
@ -8246,7 +8181,7 @@ async function loadDeadStockRevival() {
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-weight:700;font-size:0.75rem;color:var(--text-bright)">'+esc(r.vendor_name||'')+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-family:monospace;font-size:0.68rem;color:var(--text-dim)">'+esc(r.part_number||'')+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:center"><span style="background:rgba(99,102,241,0.15);color:#818cf8;border-radius:4px;padding:2px 5px;font-size:0.68rem">'+esc(r.form_factor)+'</span></td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.72rem">'+fmtSpd(r.speed_gbps)+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;color:var(--blue);font-weight:700;font-size:0.72rem">'+esc(String(r.speed_gbps))+'G</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;font-size:0.72rem">'+icon+' '+esc((r.hype_phase||'').replace(/_/g,' '))+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;font-weight:700;color:'+scoreColor+';font-family:monospace">'+Math.round(r.hype_score)+'</td>'
|
||||
+ '<td style="padding:0.45rem 0.5rem;text-align:right;font-size:0.72rem;color:'+(parseFloat(trend)>0?'#22c55e':'#64748b')+'">'+esc(trend)+'</td>'
|
||||
@ -8316,12 +8251,12 @@ function renderSignals(filterSig) {
|
||||
if (r.image_r2_key) {
|
||||
imgHtml = '<img src="https://pub-placeholder.r2.dev/' + esc(r.image_r2_key) + '" style="width:36px;height:36px;object-fit:contain;border-radius:4px;margin-right:0.5rem;flex-shrink:0" onerror="this.style.display=\'none\'">';
|
||||
}
|
||||
return '<div class="signal-card ' + sigClass + '" data-txid="' + esc(String(r.transceiver_id||r.id||'')) + '" data-part="' + esc(String(r.standard_name||r.part_number||'')) + '" style="cursor:pointer" onclick="openSignalPriceChart(this.dataset.txid,this.dataset.part)">'
|
||||
return '<div class="signal-card ' + sigClass + '">'
|
||||
+ '<div style="display:flex;align-items:flex-start;gap:0.25rem;margin-bottom:0.5rem">'
|
||||
+ imgHtml
|
||||
+ '<div style="flex:1;min-width:0">'
|
||||
+ '<div style="font-weight:700;font-size:0.82rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">' + esc(productName) + (r.is_demo_data || r.is_demo ? demoBadgeHtml : '') + '</div>'
|
||||
+ '<div style="font-size:0.7rem;color:var(--text-dim)">' + esc(r.form_factor || '') + (r.speed_gbps ? ' · ' + fmtSpd(r.speed_gbps) : '') + (r.vendor_name ? ' · ' + esc(r.vendor_name) : '') + '</div>'
|
||||
+ '<div style="font-size:0.7rem;color:var(--text-dim)">' + esc(r.form_factor || '') + (r.speed_gbps ? ' · ' + r.speed_gbps + 'G' : '') + (r.vendor_name ? ' · ' + esc(r.vendor_name) : '') + '</div>'
|
||||
+ '</div>'
|
||||
+ '</div>'
|
||||
+ '<div style="display:flex;gap:0.4rem;align-items:center;margin-bottom:0.6rem;flex-wrap:wrap">'
|
||||
@ -9903,14 +9838,10 @@ async function loadStock() {
|
||||
|
||||
// ── Flexoptix Internal Demand (real data) ────────────────────────────────
|
||||
try {
|
||||
var [demandBySpeed, demandVelocity, compStockResp] = await Promise.all([
|
||||
var [demandBySpeed, demandVelocity] = await Promise.all([
|
||||
api('/api/internal/demand/by-speed').catch(function() { return null; }),
|
||||
api('/api/internal/demand/velocity').catch(function() { return null; }),
|
||||
api('/api/stock/competitor-by-tech').catch(function() { return null; })
|
||||
api('/api/internal/demand/velocity').catch(function() { return null; })
|
||||
]);
|
||||
if (compStockResp && compStockResp.success) {
|
||||
window._compStockByTech = compStockResp.by_tech || {};
|
||||
}
|
||||
|
||||
if (demandBySpeed && demandBySpeed.success && demandBySpeed.data) {
|
||||
var rows = demandBySpeed.data;
|
||||
@ -9941,7 +9872,7 @@ async function loadStock() {
|
||||
var momColor = momentum >= 1.05 ? '#22c55e' : momentum >= 0.95 ? '#f59e0b' : '#ef4444';
|
||||
var trendArrow = momentum >= 1.1 ? '▲▲' : momentum >= 1.02 ? '▲' : momentum >= 0.98 ? '→' : momentum >= 0.9 ? '▼' : '▼▼';
|
||||
var trendColor = momentum >= 1.05 ? '#22c55e' : momentum >= 0.95 ? '#f59e0b' : '#ef4444';
|
||||
var tech = fmtSpd(r.speed_gbps) + ' ' + (r.form_factor || '');
|
||||
var tech = (r.speed_gbps || '?') + 'G ' + (r.form_factor || '');
|
||||
var fastBadge = Number(r.fast_movers || 0) > 0
|
||||
? '<span style="background:#6366f122;color:#818cf8;border-radius:10px;padding:1px 7px;font-size:0.65rem;font-weight:700">' + r.fast_movers + '</span>'
|
||||
: '<span style="color:var(--text-dim)">—</span>';
|
||||
@ -9955,17 +9886,6 @@ async function loadStock() {
|
||||
+ '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center;font-size:0.85rem;color:' + trendColor + ';font-weight:700">' + trendArrow + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center">' + fastBadge + '</td>'
|
||||
+ (function() {
|
||||
if (!window._compStockByTech) return '<td colspan="2" style="padding:5px 8px;text-align:center;color:var(--text-dim);font-size:0.68rem">—</td>';
|
||||
var cs = window._compStockByTech[tech] || {};
|
||||
var fs = cs['FS.COM'] || null;
|
||||
if (!fs) return '<td style="padding:5px 8px;text-align:right;color:var(--text-dim)">—</td><td style="padding:5px 8px;text-align:right;color:var(--text-dim)">—</td>';
|
||||
var demandMonthly = Number(r.total_demand_12m || 0) / 12;
|
||||
var deColor = fs.de > demandMonthly * 3 ? '#22c55e' : fs.de > demandMonthly ? '#f59e0b' : '#ef4444';
|
||||
var glColor = fs.global > demandMonthly * 6 ? '#22c55e' : fs.global > demandMonthly * 2 ? '#f59e0b' : '#ef4444';
|
||||
return '<td style="padding:5px 8px;text-align:right;font-family:monospace;font-size:0.72rem;color:' + deColor + '">' + Number(fs.de).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;font-family:monospace;font-size:0.72rem;color:' + glColor + '">' + Number(fs.global).toLocaleString() + '</td>';
|
||||
}())
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
}
|
||||
@ -10477,144 +10397,6 @@ function exportMoversCSV() {
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// B – PROCUREMENT PULSE (overview cards)
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
async function loadResearchRobot() {
|
||||
try {
|
||||
var d = await api("/api/research-robot");
|
||||
if (!d.runs || !d.runs.length) return;
|
||||
var r = d.runs[0], f = r.freshness || {}, dec = r.decision || {};
|
||||
var card = el("research-robot-card"); if (!card) return;
|
||||
var total = f.jobsTotal || 0, stale = f.stale || 0;
|
||||
var disp = r.dispatched || [];
|
||||
var recs = d.recommendations || [];
|
||||
|
||||
var h = "<div class=\"card-label\">🤖 Research Robot <span style=\"font-weight:400;color:var(--text-dim);font-size:0.72rem\">letzter Lauf " + esc(new Date(r.run_at).toLocaleString()) + " · " + esc(r.model||"") + "</span></div>";
|
||||
h += "<div class=\"mt\" style=\"display:flex;gap:1.6rem;flex-wrap:wrap;align-items:baseline;margin-bottom:0.6rem\">";
|
||||
h += "<div><span style=\"font-size:1.5rem;font-weight:700;color:var(--green)\">" + (total-stale) + "</span> <span style=\"color:var(--text-dim);font-size:0.8rem\">/" + total + " Jobs frisch</span></div>";
|
||||
h += "<div><span style=\"font-size:1.5rem;font-weight:700;color:" + (stale>0?"#f59e0b":"var(--green)") + "\">" + stale + "</span> <span style=\"color:var(--text-dim);font-size:0.8rem\">ueberfaellig</span></div>";
|
||||
var critCount = recs.filter(function(x){return x.severity==="critical";}).length;
|
||||
if (critCount) h += "<div><span style=\"font-size:1.5rem;font-weight:700;color:#ef4444\">" + critCount + "</span> <span style=\"color:var(--text-dim);font-size:0.8rem\">Handlungsbedarf</span></div>";
|
||||
h += "</div>";
|
||||
|
||||
if (dec.assessment) h += "<div style=\"font-size:0.8rem;margin-bottom:0.6rem;color:var(--text-dim)\"><b style=\"color:var(--text)\">KI-Urteil (lokales LLM):</b> " + esc(dec.assessment) + "</div>";
|
||||
if (disp.length) h += "<div style=\"font-size:0.74rem;color:var(--text-dim);margin-bottom:0.5rem\">Automatisch neu eingereiht: " + disp.map(esc).join(", ") + "</div>";
|
||||
|
||||
// ── Recommendations with action buttons ─────────────────────────────────
|
||||
var sevMeta = {
|
||||
critical: { icon: "🔴", color: "#ef4444", bg: "rgba(239,68,68,0.07)", bd: "rgba(239,68,68,0.3)" },
|
||||
warning: { icon: "⚠️", color: "#f59e0b", bg: "rgba(245,158,11,0.07)", bd: "rgba(245,158,11,0.3)" },
|
||||
info: { icon: "✅", color: "#22c55e", bg: "rgba(34,197,94,0.07)", bd: "rgba(34,197,94,0.3)" }
|
||||
};
|
||||
var actionMeta = {
|
||||
dispatch: { label: "▶ Jetzt auslösen", color: "#6366f1", title: "Job sofort in die Queue stellen (einmaliger Lauf)" },
|
||||
pause: { label: "❚❚ Pausieren", color: "#f59e0b", title: "Aus dem Schedule nehmen — Robot reiht ihn nicht mehr neu ein (reversibel)" },
|
||||
resume: { label: "▶ Fortsetzen", color: "#22c55e", title: "Wieder in den Schedule aufnehmen" },
|
||||
token_help: { label: "🔑 Token-Anleitung", color: "#0ea5e9", title: "Wie man den fehlenden API-Token setzt" }
|
||||
};
|
||||
|
||||
h += "<div style=\"display:flex;flex-direction:column;gap:0.5rem;margin-top:0.4rem\">";
|
||||
recs.forEach(function(rec) {
|
||||
var m = sevMeta[rec.severity] || sevMeta.info;
|
||||
h += "<div style=\"border:1px solid " + m.bd + ";background:" + m.bg + ";border-radius:8px;padding:0.6rem 0.75rem\">";
|
||||
h += "<div style=\"display:flex;align-items:flex-start;gap:0.5rem;justify-content:space-between;flex-wrap:wrap\">";
|
||||
h += "<div style=\"flex:1;min-width:200px\">";
|
||||
h += "<div style=\"font-weight:700;font-size:0.84rem;color:" + m.color + "\">" + m.icon + " " + esc(rec.title) + "</div>";
|
||||
h += "<div style=\"font-size:0.76rem;color:var(--text-dim);margin-top:0.25rem;line-height:1.45\">" + esc(rec.detail) + "</div>";
|
||||
if (rec.last_error) h += "<div style=\"font-size:0.68rem;color:var(--text-dim);margin-top:0.3rem;font-family:var(--mono);opacity:0.8\">Letzter Fehler: " + esc(rec.last_error) + (rec.failed_count?" (" + rec.failed_count + "x fehlgeschlagen)":"") + "</div>";
|
||||
h += "</div>";
|
||||
// Action buttons
|
||||
if (rec.actions && rec.actions.length) {
|
||||
h += "<div style=\"display:flex;gap:0.4rem;flex-wrap:wrap;align-items:flex-start\">";
|
||||
rec.actions.forEach(function(act) {
|
||||
var am = actionMeta[act]; if (!am) return;
|
||||
if (act === "token_help") {
|
||||
h += "<button onclick=\"showTokenHelp('" + esc(rec.job) + "')\" title=\"" + am.title + "\" style=\"cursor:pointer;background:transparent;border:1px solid " + am.color + ";color:" + am.color + ";border-radius:6px;padding:4px 10px;font-size:0.72rem;font-weight:600;white-space:nowrap\">" + am.label + "</button>";
|
||||
} else {
|
||||
h += "<button onclick=\"researchRobotAction('" + act + "','" + esc(rec.job) + "',this)\" title=\"" + am.title + "\" style=\"cursor:pointer;background:transparent;border:1px solid " + am.color + ";color:" + am.color + ";border-radius:6px;padding:4px 10px;font-size:0.72rem;font-weight:600;white-space:nowrap\">" + am.label + "</button>";
|
||||
}
|
||||
});
|
||||
h += "</div>";
|
||||
}
|
||||
h += "</div></div>";
|
||||
});
|
||||
h += "</div>";
|
||||
|
||||
card.innerHTML = h; card.style.display = "";
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
// Execute a research-robot action (dispatch/pause/resume) and refresh the panel
|
||||
async function researchRobotAction(action, job, btn) {
|
||||
if (btn) { btn.disabled = true; btn.style.opacity = "0.5"; }
|
||||
try {
|
||||
var token = (window.loadToken ? window.loadToken() : "") || "";
|
||||
var resp = await fetch("/api/research-robot/action", {
|
||||
method: "POST",
|
||||
headers: { "Authorization": "Bearer " + token, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ action: action, job: job })
|
||||
});
|
||||
var d = await resp.json();
|
||||
if (d.success) {
|
||||
if (typeof showToast === "function") showToast("Aktion ausgeführt", d.message || (action + " für " + job));
|
||||
} else {
|
||||
if (typeof showToast === "function") showToast("Fehler", d.error || "Aktion fehlgeschlagen", true);
|
||||
}
|
||||
} catch(e) {
|
||||
if (typeof showToast === "function") showToast("Fehler", e.message, true);
|
||||
}
|
||||
setTimeout(loadResearchRobot, 600);
|
||||
}
|
||||
|
||||
// Show how to set a missing API token (e.g. FLEXOPTIX_API_TOKEN)
|
||||
function showTokenHelp(job) {
|
||||
var isFlexoptix = /flexoptix/i.test(job);
|
||||
var msg;
|
||||
if (isFlexoptix) {
|
||||
msg = "Der Job '" + job + "' ruft die Flexoptix REST-API auf und bekommt HTTP 401 (kein gültiger Token).\n\n"
|
||||
+ "So setzt du den Token:\n"
|
||||
+ "1. Im Flexoptix-Portal einen REST-API-Token erzeugen (Account → API).\n"
|
||||
+ "2. Auf Erik in /opt/tip/.env setzen: FLEXOPTIX_API_TOKEN=<dein-token>\n"
|
||||
+ "3. API neu starten: pm2 restart tip-api\n\n"
|
||||
+ "Bis der Token gesetzt ist, kannst du den Job pausieren, damit er nicht alle 15 Min erneut fehlschlägt.";
|
||||
} else {
|
||||
msg = "Der Job '" + job + "' wird von der Quelle mit 401/403 abgewiesen — meist fehlt ein API-Token oder er ist abgelaufen.\n\n"
|
||||
+ "Token in /opt/tip/.env setzen und 'pm2 restart tip-api'. Bis dahin Job pausieren.";
|
||||
}
|
||||
if (typeof showModal === "function") { showModal("Token-Anleitung", "<pre style=\"white-space:pre-wrap;font-size:0.8rem;line-height:1.5\">" + esc(msg) + "</pre>"); }
|
||||
else { alert(msg); }
|
||||
}
|
||||
|
||||
|
||||
/* ── Procurement: Transceiver-Suche ────────────────────────────────────── */
|
||||
async function procTxSearch() {
|
||||
var q = (el('proc-tx-search').value || '').trim();
|
||||
var box = el('proc-tx-results');
|
||||
if (!q) { box.innerHTML = '<div style="color:var(--text-dim);font-size:0.78rem">Bitte Part-Number oder Name eingeben.</div>'; return; }
|
||||
box.innerHTML = '<div style="color:var(--text-dim);font-size:0.78rem">Suche…</div>';
|
||||
try {
|
||||
var d = await api('/api/transceivers?q=' + encodeURIComponent(q) + '&limit=25');
|
||||
var rows = d.data || [];
|
||||
if (!rows.length) { box.innerHTML = '<div style="color:var(--text-dim);font-size:0.78rem">Kein Transceiver gefunden für „' + esc(q) + '".</div>'; return; }
|
||||
var seen = {};
|
||||
var html = '<div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.35rem">' + (d.total || rows.length) + ' Treffer — klicken für Details (Preise, Anbieter, Verfügbarkeit, Preisverlauf):</div>';
|
||||
html += '<div style="display:flex;flex-direction:column;gap:2px;max-height:340px;overflow-y:auto">';
|
||||
rows.forEach(function(r) {
|
||||
// dedupe by part_number (mehrere OEM-Zeilen)
|
||||
var key = (r.part_number || r.id);
|
||||
if (seen[key]) return; seen[key] = 1;
|
||||
var spd = (typeof fmtSpd === 'function') ? fmtSpd(r.speed_gbps) : (r.speed_gbps + 'G');
|
||||
html += '<div onclick="openTxDetail(\'' + esc(String(r.id)) + '\')" style="cursor:pointer;display:flex;align-items:center;gap:0.6rem;padding:6px 8px;border-radius:6px;border:1px solid var(--border)" onmouseover="this.style.background=\'var(--surface2)\'" onmouseout="this.style.background=\'\'">'
|
||||
+ '<span style="font-family:var(--mono);font-size:0.78rem;font-weight:600;color:var(--text-bright);min-width:140px">' + esc(r.part_number || r.standard_name || '—') + '</span>'
|
||||
+ '<span style="font-size:0.7rem;color:var(--text-dim)">' + esc(r.form_factor || '') + ' · ' + spd + (r.reach_label ? ' · ' + esc(r.reach_label) : '') + '</span>'
|
||||
+ '<span style="font-size:0.68rem;color:var(--text-dim);margin-left:auto">' + esc(r.vendor_name || '') + '</span>'
|
||||
+ '</div>';
|
||||
});
|
||||
html += '</div>';
|
||||
box.innerHTML = html;
|
||||
} catch (e) {
|
||||
box.innerHTML = '<div style="color:var(--text-dim);font-size:0.78rem">Suche fehlgeschlagen: ' + esc(e.message) + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadProcurementPulse() {
|
||||
var pulse = el('ov-proc-pulse');
|
||||
var moversCard = el('ov-movers-card');
|
||||
@ -10623,14 +10405,14 @@ async function loadProcurementPulse() {
|
||||
try {
|
||||
// Fire all in parallel
|
||||
var [reorder, arb, squeeze, movers] = await Promise.all([
|
||||
api('/api/procurement/reorder-top?limit=1').catch(function() { return null; }),
|
||||
api('/api/procurement/reorder?limit=1').catch(function() { return null; }),
|
||||
api('/api/procurement/arbitrage?limit=1').catch(function() { return null; }),
|
||||
api('/api/procurement/supply-squeeze?limit=1').catch(function() { return null; }),
|
||||
api('/api/procurement/price-movers?days=7&limit=10').catch(function() { return null; }),
|
||||
api('/api/equivalences?limit=1').catch(function() { return null; }),
|
||||
]);
|
||||
|
||||
var buySignals = reorder && reorder.summary ? (reorder.summary.buy_now || 0) : 0;
|
||||
var buySignals = reorder ? (reorder.total || (reorder.reorder_signals || []).length || 0) : 0;
|
||||
var arbOps = arb ? (arb.total || (arb.opportunities || []).length || 0) : 0;
|
||||
var supplyAlert = squeeze ? (squeeze.total || (squeeze.items || []).length || 0) : 0;
|
||||
var gainers = movers ? ((movers.gainers || []).length) : 0;
|
||||
@ -10854,7 +10636,7 @@ async function runBulkPrice() {
|
||||
+ '</div>'
|
||||
+ '<div style="display:flex;gap:0.4rem">'
|
||||
+ (r.form_factor ? '<span class="b b-blue">' + esc(r.form_factor) + '</span>' : '')
|
||||
+ (r.speed_gbps ? '<span class="b b-neutral">' + fmtSpd(r.speed_gbps) + '</span>' : '')
|
||||
+ (r.speed_gbps ? '<span class="b b-neutral">' + r.speed_gbps + 'G</span>' : '')
|
||||
+ '</div>'
|
||||
+ '</div>';
|
||||
if (!r.prices || !r.prices.length) {
|
||||
@ -12442,89 +12224,6 @@ function roiCard(title, val, sub, color) {
|
||||
+ '</div>';
|
||||
}
|
||||
|
||||
|
||||
/* ── PRICE HISTORY CHART (signal-card click) ─────────────────────────── */
|
||||
async function openSignalPriceChart(txId, partNumber) {
|
||||
var modal = document.createElement('div');
|
||||
modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.55);z-index:9000;display:flex;align-items:center;justify-content:center;padding:1rem';
|
||||
modal.innerHTML = '<div style="background:var(--surface);border-radius:12px;padding:1.5rem;max-width:680px;width:100%;box-shadow:0 8px 40px rgba(0,0,0,0.35);position:relative">'
|
||||
+ '<button onclick="this.closest(\'[style*=fixed]\').remove()" style="position:absolute;top:0.75rem;right:0.9rem;background:none;border:none;font-size:1.4rem;cursor:pointer;color:var(--text-dim)">x</button>'
|
||||
+ '<div style="font-weight:700;font-size:1rem;margin-bottom:1rem;color:var(--text-bright)" id="ph-title">Price History</div>'
|
||||
+ '<div id="ph-body" style="min-height:220px;display:flex;align-items:center;justify-content:center;color:var(--text-dim)">Loading...</div>'
|
||||
+ '</div>';
|
||||
modal.onclick = function(e) { if (e.target === modal) modal.remove(); };
|
||||
document.body.appendChild(modal);
|
||||
try {
|
||||
var tok = (window.loadToken ? window.loadToken() : '') || '';
|
||||
var d = await (await fetch('/api/price-history/' + txId + '?days=180', { headers: { 'Authorization': 'Bearer ' + tok } })).json();
|
||||
var tx = d.transceiver || {};
|
||||
document.getElementById('ph-title').textContent = 'Price History — ' + (tx.part_number || partNumber || txId)
|
||||
+ (tx.form_factor ? ' (' + tx.form_factor + (tx.speed_gbps ? ' · ' + fmtSpd(tx.speed_gbps) : '') + ')' : '');
|
||||
var series = d.series || [];
|
||||
if (!series.length) {
|
||||
document.getElementById('ph-body').innerHTML = '<div style="color:var(--text-dim);text-align:center;padding:2rem">No price data available.</div>';
|
||||
return;
|
||||
}
|
||||
var vendors = {}, cur = d.current_prices || [];
|
||||
series.forEach(function(r) { if (r.source_vendor) vendors[r.source_vendor] = r.currency; });
|
||||
var vendorList = Object.keys(vendors).slice(0, 6);
|
||||
var COLORS = ['#6366f1','#22c55e','#f59e0b','#ef4444','#0ea5e9','#a855f7'];
|
||||
var dates = [], seen = {};
|
||||
series.forEach(function(r) { var d2 = r.day ? r.day.slice(0,10) : ''; if (d2 && !seen[d2]) { seen[d2]=1; dates.push(d2); } });
|
||||
dates.sort();
|
||||
var vData = {};
|
||||
vendorList.forEach(function(v) { vData[v] = {}; });
|
||||
series.forEach(function(r) {
|
||||
if (r.source_vendor && vData[r.source_vendor] !== undefined && r.day)
|
||||
vData[r.source_vendor][r.day.slice(0,10)] = parseFloat(r.price_avg || r.price_min || 0);
|
||||
});
|
||||
var allP = [];
|
||||
vendorList.forEach(function(v) { Object.values(vData[v]).forEach(function(p) { allP.push(p); }); });
|
||||
var pMin = Math.min.apply(null,allP), pMax = Math.max.apply(null,allP), pRange = pMax-pMin||1;
|
||||
var W=620,H=200,PAD={l:52,r:12,t:10,b:36};
|
||||
var iW=W-PAD.l-PAD.r, iH=H-PAD.t-PAD.b;
|
||||
function xOf(i){ return PAD.l+(dates.length<2?iW/2:i/(dates.length-1)*iW); }
|
||||
function yOf(p){ return PAD.t+iH-((p-pMin)/pRange*iH); }
|
||||
var currency = series[0] ? (series[0].currency||'') : '';
|
||||
var gridSvg='';
|
||||
for(var gi=0;gi<=4;gi++){
|
||||
var gy=PAD.t+gi*iH/4, gv=pMax-gi*pRange/4;
|
||||
gridSvg+='<line x1="'+PAD.l+'" y1="'+gy.toFixed(1)+'" x2="'+(W-PAD.r)+'" y2="'+gy.toFixed(1)+'" stroke="var(--border)" stroke-width="1"/>';
|
||||
gridSvg+='<text x="'+(PAD.l-5)+'" y="'+(gy+4).toFixed(1)+'" text-anchor="end" font-size="9" fill="var(--text-dim)">'+(gv>=1000?(gv/1000).toFixed(1)+'k':gv.toFixed(0))+'</text>';
|
||||
}
|
||||
var xLabels='';
|
||||
var nXL=Math.min(6,dates.length);
|
||||
for(var xi=0;xi<nXL;xi++){
|
||||
var didx=Math.round(xi*(dates.length-1)/Math.max(nXL-1,1));
|
||||
xLabels+='<text x="'+xOf(didx).toFixed(1)+'" y="'+(H-4)+'" text-anchor="middle" font-size="9" fill="var(--text-dim)">'+dates[didx].slice(5)+'</text>';
|
||||
}
|
||||
var linesSvg='';
|
||||
vendorList.forEach(function(v,vi){
|
||||
var pts=dates.map(function(d3,di){ var p=vData[v][d3]; return p!==undefined?(xOf(di).toFixed(1)+','+yOf(p).toFixed(1)):null; }).filter(Boolean);
|
||||
if(!pts.length) return;
|
||||
linesSvg+='<polyline points="'+pts.join(' ')+'" fill="none" stroke="'+COLORS[vi%COLORS.length]+'" stroke-width="2" stroke-linejoin="round" stroke-linecap="round"/>';
|
||||
var lp=pts[pts.length-1].split(',');
|
||||
linesSvg+='<circle cx="'+lp[0]+'" cy="'+lp[1]+'" r="3" fill="'+COLORS[vi%COLORS.length]+'"/>';
|
||||
});
|
||||
var legend=vendorList.map(function(v,vi){
|
||||
var lp=Object.values(vData[v]).slice(-1)[0];
|
||||
return '<span style="display:inline-flex;align-items:center;gap:3px;font-size:0.68rem;margin-right:0.6rem">'
|
||||
+'<span style="display:inline-block;width:12px;height:3px;background:'+COLORS[vi%COLORS.length]+';border-radius:2px"></span>'
|
||||
+esc(v)+(lp?' '+lp.toFixed(0)+' '+currency:'')+'</span>';
|
||||
}).join('');
|
||||
var bestHtml='';
|
||||
if(cur.length) bestHtml='<div style="margin-top:0.6rem;font-size:0.72rem;color:var(--text-dim)">Best now: '
|
||||
+cur.slice(0,3).map(function(c){ return '<b style="color:var(--text)">'+c.best_price+' '+c.currency+'</b> @ '+esc(c.source_vendor); }).join(' · ')+'</div>';
|
||||
document.getElementById('ph-body').innerHTML=
|
||||
'<svg viewBox="0 0 '+W+' '+H+'" style="width:100%;height:auto;display:block">'+gridSvg+linesSvg+xLabels
|
||||
+'<text x="'+(W/2)+'" y="'+(H-2)+'" text-anchor="middle" font-size="9" fill="var(--text-dim)">'+currency+'</text></svg>'
|
||||
+'<div style="margin-top:0.4rem;line-height:1.8">'+legend+'</div>'+bestHtml;
|
||||
} catch(e) {
|
||||
var b=document.getElementById('ph-body');
|
||||
if(b) b.innerHTML='<div style="color:var(--text-dim);padding:1rem">Error: '+esc(e.message)+'</div>';
|
||||
}
|
||||
}
|
||||
/* ── END PRICE HISTORY CHART ─────────────────────────────────────────── */
|
||||
</script>
|
||||
<script src="/dashboard/hot-topics.js"></script>
|
||||
</body>
|
||||
|
||||
@ -33,10 +33,7 @@
|
||||
"scrape:optcore": "tsx src/scrapers/optcore.ts",
|
||||
"scrape:news": "tsx src/scrapers/news.ts",
|
||||
"scrape:all": "tsx src/index.ts --all",
|
||||
"robots:verification": "tsx src/robots/verification-robots.ts",
|
||||
"dqo": "tsx src/orchestrator/index.ts",
|
||||
"dqo:once": "tsx src/orchestrator/index.ts --once",
|
||||
"dqo:test": "node --import tsx --test src/orchestrator/self-heal-guards.test.ts"
|
||||
"robots:verification": "tsx src/robots/verification-robots.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"crawlee": "^3.12.0",
|
||||
|
||||
@ -69,7 +69,7 @@ export interface CrawlExtraction {
|
||||
}
|
||||
|
||||
export interface RobotExperience {
|
||||
event: "status" | "tipllm_plan" | "queue_plan" | "queue_enqueued" | "queue_rejected" | "crawler_result" | "dqo_dispatch" | "dqo_verify" | "dqo_escalation";
|
||||
event: "status" | "tipllm_plan" | "queue_plan" | "queue_enqueued" | "queue_rejected" | "crawler_result";
|
||||
observed_at: string;
|
||||
actor: string;
|
||||
profile?: string;
|
||||
|
||||
@ -1,394 +0,0 @@
|
||||
/**
|
||||
* tip-dqo control loop — the closed loop TIP was missing.
|
||||
*
|
||||
* DETECT read v_dqo_targets (v_scraper_health joined onto source_registry) — DATA
|
||||
* freshness of the observation TABLES, never pg-boss job cadence.
|
||||
* PRIORITIZE the view is pre-ranked (dead>stale, weighted); split staleness vs
|
||||
* code_fix/blocked. Only 'staleness' is dispatch-eligible.
|
||||
* DISPATCH guardrails (self-heal-guards) + plan-mode + profile cap, then the shared
|
||||
* verification-robots dispatchQueues() — exactly one boss.send path.
|
||||
* VERIFY re-read the source's freshness for pending dispatch rows: did fresh rows
|
||||
* land? healed / ineffective(+strike). Catches completed-but-wrote-0 jobs.
|
||||
* ESCALATE fresh=silent, stale/1-2 strikes=digest, breaker/dead=page (delta-gated by
|
||||
* acknowledged_exceptions and once-per-24h so the day-1 backlog never spams).
|
||||
*
|
||||
* INVARIANT: VERIFY + DETECT key off observation-table freshness (v_scraper_health), NOT
|
||||
* pgboss.job='completed' — that is the exact blind spot that lets a green-queue/zero-row
|
||||
* scraper look healthy. Do not "optimize" this to read pgboss.job.
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { loadavg } from "os";
|
||||
import { requireDbPassword } from "../utils/db-connection";
|
||||
import {
|
||||
dispatchQueues,
|
||||
queuesForProfile,
|
||||
defaultMaxQueues,
|
||||
type RobotProfile,
|
||||
} from "../robots/verification-robots";
|
||||
import { canDispatch, type GuardConfig, type LedgerFacts } from "./self-heal-guards";
|
||||
import { notify } from "./escalation-sink";
|
||||
import { writeRobotExperience } from "../crawler-llm/training-data-writer";
|
||||
|
||||
const PLAN_ONLY = process.env.SELF_HEAL_PLAN_ONLY !== "false"; // default true — dispatches NOTHING
|
||||
const MAX_LOAD = Number.parseFloat(process.env.DQO_MAX_LOAD || "2.5");
|
||||
const SETTLE_MINUTES = Number.parseInt(process.env.DQO_VERIFY_SETTLE_MINUTES || "20", 10);
|
||||
const ESCALATE_REPAGE_HOURS = 24;
|
||||
|
||||
let poolSingleton: Pool | null = null;
|
||||
function getPool(): Pool {
|
||||
if (!poolSingleton) {
|
||||
poolSingleton = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
max: 2, // read-mostly; kept small so the orchestrator never contends the daemon pool
|
||||
idleTimeoutMillis: 10_000,
|
||||
connectionTimeoutMillis: 5_000,
|
||||
});
|
||||
}
|
||||
return poolSingleton;
|
||||
}
|
||||
|
||||
export async function closePool(): Promise<void> {
|
||||
if (poolSingleton) {
|
||||
await poolSingleton.end();
|
||||
poolSingleton = null;
|
||||
}
|
||||
}
|
||||
|
||||
interface Target {
|
||||
source_key: string;
|
||||
vendor: string;
|
||||
source_table: string;
|
||||
status: string;
|
||||
days_since_last: number;
|
||||
rows_30d: number;
|
||||
last_observation: string | null;
|
||||
root_cause_class: "staleness" | "code_fix" | "blocked";
|
||||
queue_names: string[];
|
||||
profile: RobotProfile;
|
||||
dead_days: number;
|
||||
cooldown_minutes: number;
|
||||
budget_per_day: number;
|
||||
max_backoff_hours: number;
|
||||
circuit_breaker_strikes: number;
|
||||
score: number;
|
||||
}
|
||||
|
||||
function guardConfig(t: Target): GuardConfig {
|
||||
return {
|
||||
cooldown_minutes: t.cooldown_minutes,
|
||||
budget_per_day: t.budget_per_day,
|
||||
max_backoff_hours: t.max_backoff_hours,
|
||||
circuit_breaker_strikes: t.circuit_breaker_strikes,
|
||||
max_load: MAX_LOAD,
|
||||
};
|
||||
}
|
||||
|
||||
// ── kill-switch + global daily ceiling ────────────────────────────────────────
|
||||
async function globalGate(pool: Pool): Promise<{ armed: boolean; reason: string; globalBudget: number }> {
|
||||
const { rows } = await pool.query(
|
||||
`select enabled, budget_per_day from source_registry where source_key = '__global__'`,
|
||||
);
|
||||
if (rows.length === 0) return { armed: true, reason: "no __global__ row (default armed)", globalBudget: 9999 };
|
||||
if (!rows[0].enabled) return { armed: false, reason: "kill-switch: __global__.enabled = false", globalBudget: 0 };
|
||||
return { armed: true, reason: "armed", globalBudget: rows[0].budget_per_day };
|
||||
}
|
||||
|
||||
async function globalDispatchesToday(pool: Pool): Promise<number> {
|
||||
const { rows } = await pool.query(
|
||||
`select count(*)::int n from self_heal_dispatch
|
||||
where mode = 'dispatch' and dispatched_at >= date_trunc('day', now())`,
|
||||
);
|
||||
return rows[0]?.n ?? 0;
|
||||
}
|
||||
|
||||
// ── DETECT ────────────────────────────────────────────────────────────────────
|
||||
export async function detect(pool: Pool): Promise<Target[]> {
|
||||
const { rows } = await pool.query(`select * from v_dqo_targets`);
|
||||
return rows as Target[];
|
||||
}
|
||||
|
||||
async function ledgerFacts(pool: Pool, sourceKey: string): Promise<LedgerFacts> {
|
||||
const { rows } = await pool.query(
|
||||
`select
|
||||
(select max(dispatched_at) from self_heal_dispatch where source_key = $1 and mode = 'dispatch') as last_dispatch_at,
|
||||
(select coalesce(strikes, 0) from self_heal_dispatch where source_key = $1 and mode = 'dispatch' order by dispatched_at desc limit 1) as last_strikes,
|
||||
(select count(*)::int from self_heal_dispatch where source_key = $1 and mode = 'dispatch' and dispatched_at >= date_trunc('day', now())) as dispatches_today`,
|
||||
[sourceKey],
|
||||
);
|
||||
const r = rows[0] ?? {};
|
||||
return {
|
||||
last_dispatch_at: r.last_dispatch_at ? new Date(r.last_dispatch_at) : null,
|
||||
last_strikes: Number(r.last_strikes ?? 0),
|
||||
dispatches_today: Number(r.dispatches_today ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function insertLedger(
|
||||
pool: Pool,
|
||||
row: {
|
||||
source_key: string;
|
||||
queues: string[];
|
||||
reason: string;
|
||||
mode: "plan" | "dispatch";
|
||||
rows_before: number | null;
|
||||
last_obs_before: string | null;
|
||||
outcome: "pending" | "skipped" | "escalated";
|
||||
strikes: number;
|
||||
run_id: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
await pool.query(
|
||||
`insert into self_heal_dispatch
|
||||
(source_key, queues, reason, mode, rows_before, last_obs_before, outcome, strikes, run_id)
|
||||
values ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
|
||||
[row.source_key, row.queues, row.reason, row.mode, row.rows_before, row.last_obs_before, row.outcome, row.strikes, row.run_id],
|
||||
);
|
||||
}
|
||||
|
||||
// ── DISPATCH (one self-heal tick) ─────────────────────────────────────────────
|
||||
export interface TickResult {
|
||||
planOnly: boolean;
|
||||
detected: number;
|
||||
planned: number;
|
||||
dispatched: number;
|
||||
skipped: number;
|
||||
escalated: number;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export async function runTick(): Promise<TickResult> {
|
||||
const pool = getPool();
|
||||
const runId = `dqo-self-heal-${new Date().toISOString()}`;
|
||||
const load1 = loadavg()[0];
|
||||
const res: TickResult = { planOnly: PLAN_ONLY, detected: 0, planned: 0, dispatched: 0, skipped: 0, escalated: 0, lines: [] };
|
||||
|
||||
const gate = await globalGate(pool);
|
||||
if (!gate.armed) {
|
||||
res.lines.push(`[HALT] ${gate.reason}`);
|
||||
return res;
|
||||
}
|
||||
const globalToday = await globalDispatchesToday(pool);
|
||||
const globalRemaining = gate.globalBudget - globalToday;
|
||||
|
||||
const targets = await detect(pool);
|
||||
res.detected = targets.length;
|
||||
if (targets.length === 0) {
|
||||
res.lines.push("no stale/dead sources — nothing to do");
|
||||
return res;
|
||||
}
|
||||
|
||||
for (const t of targets) {
|
||||
// Root-cause routing: only staleness is auto-dispatched. code_fix/blocked escalate.
|
||||
if (t.root_cause_class !== "staleness") {
|
||||
await escalateSource(pool, t, `root_cause=${t.root_cause_class} (${t.status} ${t.days_since_last}d) — a re-scrape cannot fix this; human required`, runId);
|
||||
res.escalated++;
|
||||
res.lines.push(` ESCALATE ${t.source_key} — ${t.root_cause_class} (${t.status} ${t.days_since_last}d)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// A dead staleness source past dead_days also escalates (retrying won't help a truly dead source).
|
||||
if (t.status === "dead" && t.days_since_last > t.dead_days) {
|
||||
await escalateSource(pool, t, `dead ${t.days_since_last}d > dead_days ${t.dead_days} — escalating`, runId);
|
||||
res.escalated++;
|
||||
res.lines.push(` ESCALATE ${t.source_key} — dead ${t.days_since_last}d`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const facts = await ledgerFacts(pool, t.source_key);
|
||||
const verdict = canDispatch(facts, guardConfig(t), new Date(), load1);
|
||||
|
||||
// Profile filter + cap on the queues.
|
||||
const eligible = queuesForProfile(t.queue_names, t.profile).slice(0, defaultMaxQueues(t.profile));
|
||||
|
||||
if (eligible.length === 0) {
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: [], reason: `no queue eligible under profile '${t.profile}'`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "skipped", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.skipped++;
|
||||
res.lines.push(` SKIP ${t.source_key} — no queue eligible under '${t.profile}'`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!verdict.allow) {
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `guard: ${verdict.reason}`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "skipped", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.skipped++;
|
||||
res.lines.push(` SKIP ${t.source_key} — ${verdict.reason}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (PLAN_ONLY || globalRemaining <= 0) {
|
||||
const why = PLAN_ONLY ? "plan-mode (SELF_HEAL_PLAN_ONLY)" : `global budget exhausted (${globalToday}/${gate.globalBudget})`;
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `WOULD DISPATCH → ${eligible.join(", ")} [${why}]`,
|
||||
mode: "plan", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "pending", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.planned++;
|
||||
res.lines.push(` PLAN ${t.source_key} → ${eligible.join(", ")} [${why}]`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_dispatch", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Plan: would dispatch ${t.source_key} (${t.status} ${t.days_since_last}d)`,
|
||||
input: { source_key: t.source_key, status: t.status, days_since_last: t.days_since_last, eligible, why },
|
||||
decision: { mode: "plan", dispatched: false }, safety_notes: ["Plan mode — no jobs enqueued."],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// ARMED: real dispatch.
|
||||
await dispatchQueues(eligible, { source: "dqo:self-heal", source_key: t.source_key, run_id: runId, reason: `${t.status} ${t.days_since_last}d`, enqueued_at: new Date().toISOString() });
|
||||
await insertLedger(pool, {
|
||||
source_key: t.source_key, queues: eligible, reason: `dispatched (${t.status} ${t.days_since_last}d)`,
|
||||
mode: "dispatch", rows_before: t.rows_30d, last_obs_before: t.last_observation, outcome: "pending", strikes: facts.last_strikes, run_id: runId,
|
||||
});
|
||||
res.dispatched++;
|
||||
res.lines.push(` DISPATCH ${t.source_key} → ${eligible.join(", ")}`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_dispatch", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Dispatched ${t.source_key} (${t.status} ${t.days_since_last}d)`,
|
||||
input: { source_key: t.source_key, eligible }, decision: { mode: "dispatch", dispatched: true },
|
||||
safety_notes: ["Armed dispatch within guardrails."],
|
||||
}, { flush: true });
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// ── VERIFY (one verify tick) ──────────────────────────────────────────────────
|
||||
export interface VerifyResult {
|
||||
checked: number;
|
||||
healed: number;
|
||||
ineffective: number;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export async function runVerifyTick(): Promise<VerifyResult> {
|
||||
const pool = getPool();
|
||||
const res: VerifyResult = { checked: 0, healed: 0, ineffective: 0, lines: [] };
|
||||
|
||||
// pending REAL dispatches past the settle window
|
||||
const { rows: pending } = await pool.query(
|
||||
`select d.id, d.source_key, d.rows_before, d.last_obs_before, d.strikes, d.run_id,
|
||||
sr.vendor, sr.source_table, sr.dead_days, sr.circuit_breaker_strikes
|
||||
from self_heal_dispatch d
|
||||
join source_registry sr on sr.source_key = d.source_key
|
||||
where d.mode = 'dispatch' and d.outcome = 'pending'
|
||||
and d.dispatched_at < now() - ($1 || ' minutes')::interval`,
|
||||
[String(SETTLE_MINUTES)],
|
||||
);
|
||||
// No pending dispatches → skip the expensive v_scraper_health scan entirely.
|
||||
if (pending.length === 0) return res;
|
||||
|
||||
// Compute the (expensive) freshness view ONCE and index it, instead of re-running the
|
||||
// whole aggregate per pending source. Keyed off observation TABLES, never pgboss.job.
|
||||
const { rows: healthRows } = await pool.query(
|
||||
`select vendor, source_table, rows_30d, last_observation from v_scraper_health`,
|
||||
);
|
||||
const health = new Map<string, { rows_30d: number | null; last_observation: string | null }>();
|
||||
for (const hr of healthRows) {
|
||||
health.set(`${hr.vendor}|${hr.source_table}`, { rows_30d: hr.rows_30d, last_observation: hr.last_observation });
|
||||
}
|
||||
|
||||
for (const p of pending) {
|
||||
res.checked++;
|
||||
const h = health.get(`${p.vendor}|${p.source_table}`);
|
||||
const rowsAfter: number | null = h?.rows_30d ?? null;
|
||||
const lastObsAfter: string | null = h?.last_observation ?? null;
|
||||
const advanced = lastObsAfter && p.last_obs_before && new Date(lastObsAfter) > new Date(p.last_obs_before);
|
||||
const grew = rowsAfter != null && p.rows_before != null && rowsAfter > p.rows_before;
|
||||
|
||||
if (advanced || grew) {
|
||||
await pool.query(
|
||||
`update self_heal_dispatch set outcome='healed', checked_at=now(), rows_after=$2, last_obs_after=$3, strikes=0 where id=$1`,
|
||||
[p.id, rowsAfter, lastObsAfter],
|
||||
);
|
||||
res.healed++;
|
||||
res.lines.push(` HEALED ${p.source_key} (rows ${p.rows_before}→${rowsAfter})`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_verify", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Healed ${p.source_key}`, input: { source_key: p.source_key, rows_before: p.rows_before, rows_after: rowsAfter },
|
||||
decision: { outcome: "healed" },
|
||||
});
|
||||
} else {
|
||||
const newStrikes = Number(p.strikes ?? 0) + 1;
|
||||
await pool.query(
|
||||
`update self_heal_dispatch set outcome='ineffective', checked_at=now(), rows_after=$2, last_obs_after=$3, strikes=$4 where id=$1`,
|
||||
[p.id, rowsAfter, lastObsAfter, newStrikes],
|
||||
);
|
||||
res.ineffective++;
|
||||
res.lines.push(` INEFFECTIVE ${p.source_key} (no new rows; strikes→${newStrikes})`);
|
||||
writeRobotExperience({
|
||||
event: "dqo_verify", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Ineffective dispatch ${p.source_key} — no fresh rows landed`,
|
||||
input: { source_key: p.source_key, rows_before: p.rows_before, rows_after: rowsAfter, strikes: newStrikes },
|
||||
decision: { outcome: "ineffective" },
|
||||
});
|
||||
if (newStrikes >= Number(p.circuit_breaker_strikes ?? 3)) {
|
||||
await escalateByKey(pool, p.source_key, `circuit-breaker open: ${newStrikes} ineffective dispatches — retrying wastes resources, human required`, p.run_id);
|
||||
res.lines.push(` ESCALATE ${p.source_key} — breaker open (${newStrikes} strikes)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// ── ESCALATE (delta-gated, once-per-24h per source) ───────────────────────────
|
||||
async function alreadyEscalatedRecently(pool: Pool, sourceKey: string): Promise<boolean> {
|
||||
const { rows } = await pool.query(
|
||||
`select 1 from self_heal_dispatch
|
||||
where source_key = $1 and outcome = 'escalated'
|
||||
and dispatched_at > now() - ($2 || ' hours')::interval limit 1`,
|
||||
[sourceKey, String(ESCALATE_REPAGE_HOURS)],
|
||||
);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
async function isAcknowledged(pool: Pool, sourceKey: string): Promise<boolean> {
|
||||
const { rows } = await pool.query(
|
||||
`select 1 from acknowledged_exceptions
|
||||
where (source_key = $1 or source_key is null)
|
||||
and (ack_until is null or ack_until > now()) limit 1`,
|
||||
[sourceKey],
|
||||
);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
async function escalateSource(pool: Pool, t: Target, reason: string, runId: string): Promise<void> {
|
||||
await escalateByKey(pool, t.source_key, reason, runId, { rows: t.rows_30d, lastObs: t.last_observation, status: t.status });
|
||||
}
|
||||
|
||||
async function escalateByKey(
|
||||
pool: Pool,
|
||||
sourceKey: string,
|
||||
reason: string,
|
||||
runId: string,
|
||||
ctx?: { rows?: number; lastObs?: string | null; status?: string },
|
||||
): Promise<void> {
|
||||
// delta-gate: suppress if acknowledged, and never re-page the same source within 24h
|
||||
if (await isAcknowledged(pool, sourceKey)) return;
|
||||
if (await alreadyEscalatedRecently(pool, sourceKey)) return;
|
||||
|
||||
await insertLedger(pool, {
|
||||
source_key: sourceKey, queues: [], reason, mode: "plan",
|
||||
rows_before: ctx?.rows ?? null, last_obs_before: ctx?.lastObs ?? null, outcome: "escalated", strikes: 0, run_id: runId,
|
||||
});
|
||||
await notify("page", `TIP data-quality: ${sourceKey}`, reason);
|
||||
writeRobotExperience({
|
||||
event: "dqo_escalation", observed_at: new Date().toISOString(), actor: "tip-dqo",
|
||||
summary: `Escalated ${sourceKey}: ${reason}`, input: { source_key: sourceKey, ...ctx },
|
||||
decision: { escalated: true }, safety_notes: ["Paged once; suppressed for 24h + honors acknowledged_exceptions."],
|
||||
}, { flush: true });
|
||||
}
|
||||
|
||||
export async function runOnce(): Promise<{ tick: TickResult; verify: VerifyResult }> {
|
||||
const tick = await runTick();
|
||||
const verify = await runVerifyTick();
|
||||
return { tick, verify };
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
/**
|
||||
* Notification sink for the tip-dqo escalation ladder.
|
||||
*
|
||||
* Default channel: self-hosted ntfy on Erik (no external secret, fits "Gitea = EVERYTHING").
|
||||
* Set NTFY_URL (e.g. https://ntfy.fichtmueller.org) and optionally NTFY_TOPIC (default
|
||||
* "tip-dqo"). If NTFY_URL is unset, notify() is a logged no-op so the loop still runs and
|
||||
* the escalation is recorded in the ledger — the channel can be wired later without a
|
||||
* code change.
|
||||
*/
|
||||
|
||||
export type EscalationLevel = "digest" | "page";
|
||||
|
||||
export interface NotifyResult {
|
||||
sent: boolean;
|
||||
channel: "ntfy" | "noop" | "error";
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export async function notify(
|
||||
level: EscalationLevel,
|
||||
title: string,
|
||||
body: string,
|
||||
): Promise<NotifyResult> {
|
||||
const base = process.env.NTFY_URL?.replace(/\/+$/, "");
|
||||
const topic = process.env.NTFY_TOPIC || "tip-dqo";
|
||||
|
||||
if (!base) {
|
||||
console.warn(`[dqo:escalation:noop] (${level}) ${title} — ${body} [set NTFY_URL to enable]`);
|
||||
return { sent: false, channel: "noop" };
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${base}/${topic}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Title: title,
|
||||
Priority: level === "page" ? "high" : "default",
|
||||
Tags: level === "page" ? "rotating_light" : "warning",
|
||||
...(process.env.NTFY_TOKEN ? { Authorization: `Bearer ${process.env.NTFY_TOKEN}` } : {}),
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
return { sent: false, channel: "error", detail: `HTTP ${resp.status}` };
|
||||
}
|
||||
return { sent: true, channel: "ntfy" };
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[dqo:escalation] ntfy failed: ${detail}`);
|
||||
return { sent: false, channel: "error", detail };
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
/**
|
||||
* tip-dqo — TIP Autonomous Data-Quality Orchestrator (entrypoint).
|
||||
*
|
||||
* A SEPARATE always-on process (PM2 app `tip-dqo`), deliberately NOT grafted into the
|
||||
* tip-scraper-daemon event loop, so its blast radius is isolated and it never contends
|
||||
* the daemon's pg Pool. Read-mostly: its only writes are to its own ledger/registry tables.
|
||||
*
|
||||
* Default (no args) : self-heal tick every DQO_SELF_HEAL_MINUTES (30), verify tick every
|
||||
* DQO_VERIFY_MINUTES (15). Runs until SIGINT/SIGTERM.
|
||||
* --once : run one self-heal + verify tick, print the plan, exit. Used to
|
||||
* PROVE the loop against the live DB without scheduling anything.
|
||||
*
|
||||
* SELF_HEAL_PLAN_ONLY defaults to "true" (dispatches NOTHING; writes plan rows a human
|
||||
* reviews). Set SELF_HEAL_PLAN_ONLY=false only to arm live dispatch.
|
||||
*/
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { runOnce, runTick, runVerifyTick, closePool } from "./control-loop";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
const SELF_HEAL_MINUTES = Number.parseInt(process.env.DQO_SELF_HEAL_MINUTES || "30", 10);
|
||||
const VERIFY_MINUTES = Number.parseInt(process.env.DQO_VERIFY_MINUTES || "15", 10);
|
||||
const PLAN_ONLY = process.env.SELF_HEAL_PLAN_ONLY !== "false";
|
||||
|
||||
function stamp(): string {
|
||||
return new Date().toISOString().replace("T", " ").slice(0, 19);
|
||||
}
|
||||
|
||||
async function once(): Promise<void> {
|
||||
console.log(`\n=== tip-dqo one-shot (${PLAN_ONLY ? "PLAN-ONLY" : "ARMED"}) ${stamp()} ===`);
|
||||
const { tick, verify } = await runOnce();
|
||||
console.log(`\n[self-heal] detected=${tick.detected} planned=${tick.planned} dispatched=${tick.dispatched} skipped=${tick.skipped} escalated=${tick.escalated}`);
|
||||
for (const l of tick.lines) console.log(l);
|
||||
console.log(`\n[verify] checked=${verify.checked} healed=${verify.healed} ineffective=${verify.ineffective}`);
|
||||
for (const l of verify.lines) console.log(l);
|
||||
console.log("");
|
||||
await closePool();
|
||||
}
|
||||
|
||||
async function loop(): Promise<void> {
|
||||
console.log(`=== tip-dqo starting (${PLAN_ONLY ? "PLAN-ONLY" : "ARMED"}) — self-heal ${SELF_HEAL_MINUTES}m / verify ${VERIFY_MINUTES}m ===`);
|
||||
|
||||
const selfHeal = async () => {
|
||||
try {
|
||||
const r = await runTick();
|
||||
console.log(`[${stamp()}] self-heal: detected=${r.detected} planned=${r.planned} dispatched=${r.dispatched} skipped=${r.skipped} escalated=${r.escalated}`);
|
||||
for (const l of r.lines) console.log(l);
|
||||
} catch (err) {
|
||||
console.error(`[${stamp()}] self-heal error:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
};
|
||||
const verify = async () => {
|
||||
try {
|
||||
const r = await runVerifyTick();
|
||||
if (r.checked > 0) {
|
||||
console.log(`[${stamp()}] verify: checked=${r.checked} healed=${r.healed} ineffective=${r.ineffective}`);
|
||||
for (const l of r.lines) console.log(l);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[${stamp()}] verify error:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
};
|
||||
|
||||
await selfHeal(); // run once on boot
|
||||
const t1 = setInterval(selfHeal, SELF_HEAL_MINUTES * 60_000);
|
||||
const t2 = setInterval(verify, VERIFY_MINUTES * 60_000);
|
||||
|
||||
const shutdown = async (sig: string) => {
|
||||
console.log(`\n[${stamp()}] ${sig} — shutting down tip-dqo`);
|
||||
clearInterval(t1);
|
||||
clearInterval(t2);
|
||||
await closePool().catch(() => {});
|
||||
process.exit(0);
|
||||
};
|
||||
process.on("SIGINT", () => void shutdown("SIGINT"));
|
||||
process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
||||
}
|
||||
|
||||
const isOnce = process.argv.includes("--once");
|
||||
(isOnce ? once() : loop()).catch(async (err) => {
|
||||
console.error("tip-dqo fatal:", err);
|
||||
await closePool().catch(() => {});
|
||||
process.exit(1);
|
||||
});
|
||||
@ -1,78 +0,0 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
isLoadAcceptable,
|
||||
minIntervalOk,
|
||||
budgetOk,
|
||||
backoffElapsed,
|
||||
breakerTripped,
|
||||
canDispatch,
|
||||
type GuardConfig,
|
||||
type LedgerFacts,
|
||||
} from "./self-heal-guards";
|
||||
|
||||
const CFG: GuardConfig = {
|
||||
cooldown_minutes: 180,
|
||||
budget_per_day: 4,
|
||||
max_backoff_hours: 24,
|
||||
circuit_breaker_strikes: 3,
|
||||
};
|
||||
const NOW = new Date("2026-07-06T12:00:00Z");
|
||||
const fresh: LedgerFacts = { last_dispatch_at: null, last_strikes: 0, dispatches_today: 0 };
|
||||
|
||||
test("load guard", () => {
|
||||
assert.equal(isLoadAcceptable(1.2).allow, true);
|
||||
assert.equal(isLoadAcceptable(3.0).allow, false);
|
||||
assert.equal(isLoadAcceptable(2.5).allow, true); // boundary: not greater than
|
||||
});
|
||||
|
||||
test("min-interval blocks within cooldown, allows after", () => {
|
||||
const recent = { ...fresh, last_dispatch_at: new Date("2026-07-06T11:00:00Z") }; // 60min ago
|
||||
assert.equal(minIntervalOk(recent, CFG, NOW).allow, false);
|
||||
const old = { ...fresh, last_dispatch_at: new Date("2026-07-06T08:00:00Z") }; // 240min ago
|
||||
assert.equal(minIntervalOk(old, CFG, NOW).allow, true);
|
||||
assert.equal(minIntervalOk(fresh, CFG, NOW).allow, true); // never dispatched
|
||||
});
|
||||
|
||||
test("daily budget", () => {
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 3 }, CFG).allow, true);
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 4 }, CFG).allow, false);
|
||||
assert.equal(budgetOk({ ...fresh, dispatches_today: 9 }, CFG).allow, false);
|
||||
});
|
||||
|
||||
test("exponential backoff scales with strikes and caps", () => {
|
||||
// 2 strikes → 4h backoff. Dispatched 1h ago → still backing off.
|
||||
const s2recent = { last_dispatch_at: new Date("2026-07-06T11:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(s2recent, CFG, NOW).allow, false);
|
||||
// 2 strikes → 4h backoff. Dispatched 5h ago → elapsed.
|
||||
const s2old = { last_dispatch_at: new Date("2026-07-06T07:00:00Z"), last_strikes: 2, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(s2old, CFG, NOW).allow, true);
|
||||
// 0 strikes → no backoff
|
||||
assert.equal(backoffElapsed(fresh, CFG, NOW).allow, true);
|
||||
// huge strikes → capped at max_backoff_hours (24h), dispatched 25h ago → elapsed
|
||||
const capped = { last_dispatch_at: new Date("2026-07-05T11:00:00Z"), last_strikes: 10, dispatches_today: 0 };
|
||||
assert.equal(backoffElapsed(capped, CFG, NOW).allow, true);
|
||||
});
|
||||
|
||||
test("circuit breaker opens at threshold", () => {
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 2 }, CFG).allow, true);
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 3 }, CFG).allow, false);
|
||||
assert.equal(breakerTripped({ ...fresh, last_strikes: 5 }, CFG).allow, false);
|
||||
});
|
||||
|
||||
test("canDispatch: clean source under low load passes", () => {
|
||||
assert.equal(canDispatch(fresh, CFG, NOW, 1.0).allow, true);
|
||||
});
|
||||
|
||||
test("canDispatch: breaker is reported before other brakes", () => {
|
||||
const tripped = { last_dispatch_at: new Date("2026-07-06T11:59:00Z"), last_strikes: 3, dispatches_today: 9 };
|
||||
const v = canDispatch(tripped, CFG, NOW, 5.0);
|
||||
assert.equal(v.allow, false);
|
||||
assert.match(v.reason, /breaker OPEN/);
|
||||
});
|
||||
|
||||
test("canDispatch: high load blocks an otherwise-clean source", () => {
|
||||
const v = canDispatch(fresh, CFG, NOW, 9.0);
|
||||
assert.equal(v.allow, false);
|
||||
assert.match(v.reason, /load/);
|
||||
});
|
||||
@ -1,126 +0,0 @@
|
||||
/**
|
||||
* Cost-safety guardrails for the tip-dqo self-heal loop — PURE, network-free, unit-testable.
|
||||
*
|
||||
* The controller must NEVER hammer a source. These functions turn ledger facts +
|
||||
* the source's guardrail config into a typed {allow, reason} verdict, so every skip is
|
||||
* explainable and logged (never a silent no-op). The DB reads live in control-loop.ts;
|
||||
* the DECISIONS live here and are tested with node:test against fixture facts.
|
||||
*
|
||||
* Nine compounding brakes (this module implements 5 of them; the rest — kill-switch,
|
||||
* plan-mode, root-cause routing, profile cap — live in control-loop.ts):
|
||||
* min-interval · daily budget · exponential backoff · circuit-breaker · load-guard
|
||||
*/
|
||||
|
||||
export interface GuardVerdict {
|
||||
allow: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** The per-source guardrail contract, straight from source_registry. */
|
||||
export interface GuardConfig {
|
||||
cooldown_minutes: number;
|
||||
budget_per_day: number;
|
||||
max_backoff_hours: number;
|
||||
circuit_breaker_strikes: number;
|
||||
max_load?: number;
|
||||
}
|
||||
|
||||
/** Facts read from the self_heal_dispatch ledger for one source. */
|
||||
export interface LedgerFacts {
|
||||
/** dispatched_at of the most recent mode='dispatch' row, or null if never dispatched. */
|
||||
last_dispatch_at: Date | null;
|
||||
/** strikes as of the most recent verified dispatch (running no-progress count). */
|
||||
last_strikes: number;
|
||||
/** count of mode='dispatch' rows for this source since local midnight. */
|
||||
dispatches_today: number;
|
||||
}
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
const MIN_MS = 60_000;
|
||||
|
||||
/**
|
||||
* Load guard — do not dispatch when the box is already busy. Pure: caller passes the
|
||||
* 1-minute load average (os.loadavg()[0]) so this stays testable. Mirrors the existing
|
||||
* scheduler.ts isLoadAcceptable so self-heal never fights the load guard.
|
||||
*/
|
||||
export function isLoadAcceptable(load1: number, maxLoad = 2.5): GuardVerdict {
|
||||
if (load1 > maxLoad) {
|
||||
return { allow: false, reason: `load ${load1.toFixed(2)} > ${maxLoad}` };
|
||||
}
|
||||
return { allow: true, reason: `load ${load1.toFixed(2)} <= ${maxLoad}` };
|
||||
}
|
||||
|
||||
/** Per-source minimum interval — a source cannot be re-fired within its cooldown window. */
|
||||
export function minIntervalOk(facts: LedgerFacts, cfg: GuardConfig, now: Date): GuardVerdict {
|
||||
if (!facts.last_dispatch_at) return { allow: true, reason: "no prior dispatch" };
|
||||
const elapsedMin = (now.getTime() - facts.last_dispatch_at.getTime()) / MIN_MS;
|
||||
if (elapsedMin < cfg.cooldown_minutes) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `cooldown: ${Math.round(elapsedMin)}min < ${cfg.cooldown_minutes}min since last dispatch`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `cooldown elapsed (${Math.round(elapsedMin)}min)` };
|
||||
}
|
||||
|
||||
/** Daily budget — no more than budget_per_day dispatches for this source since midnight. */
|
||||
export function budgetOk(facts: LedgerFacts, cfg: GuardConfig): GuardVerdict {
|
||||
if (facts.dispatches_today >= cfg.budget_per_day) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `budget: ${facts.dispatches_today}/${cfg.budget_per_day} dispatches used today`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `budget ${facts.dispatches_today}/${cfg.budget_per_day}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff — a repeatedly-ineffective source backs off 2^strikes hours
|
||||
* (capped at max_backoff_hours) instead of retrying every tick.
|
||||
*/
|
||||
export function backoffElapsed(facts: LedgerFacts, cfg: GuardConfig, now: Date): GuardVerdict {
|
||||
if (facts.last_strikes <= 0 || !facts.last_dispatch_at) {
|
||||
return { allow: true, reason: "no backoff (0 strikes)" };
|
||||
}
|
||||
const backoffHours = Math.min(Math.pow(2, facts.last_strikes), cfg.max_backoff_hours);
|
||||
const readyAt = new Date(facts.last_dispatch_at.getTime() + backoffHours * HOUR_MS);
|
||||
if (now < readyAt) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `backoff: ${facts.last_strikes} strikes → wait ${backoffHours}h (ready ${readyAt.toISOString()})`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `backoff elapsed (${backoffHours}h after ${facts.last_strikes} strikes)` };
|
||||
}
|
||||
|
||||
/** Circuit-breaker — at N strikes the source is removed from the retry pool (escalated instead). */
|
||||
export function breakerTripped(facts: LedgerFacts, cfg: GuardConfig): GuardVerdict {
|
||||
if (facts.last_strikes >= cfg.circuit_breaker_strikes) {
|
||||
return {
|
||||
allow: false,
|
||||
reason: `breaker OPEN: ${facts.last_strikes} >= ${cfg.circuit_breaker_strikes} strikes`,
|
||||
};
|
||||
}
|
||||
return { allow: true, reason: `breaker closed (${facts.last_strikes}/${cfg.circuit_breaker_strikes})` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine all guardrails into one verdict. Returns the FIRST failing brake so the skip
|
||||
* reason is specific. load1 is the current 1-min load average.
|
||||
*/
|
||||
export function canDispatch(
|
||||
facts: LedgerFacts,
|
||||
cfg: GuardConfig,
|
||||
now: Date,
|
||||
load1: number,
|
||||
): GuardVerdict {
|
||||
const checks = [
|
||||
breakerTripped(facts, cfg), // breaker first: an open breaker means "escalate, don't retry"
|
||||
budgetOk(facts, cfg),
|
||||
minIntervalOk(facts, cfg, now),
|
||||
backoffElapsed(facts, cfg, now),
|
||||
isLoadAcceptable(load1, cfg.max_load),
|
||||
];
|
||||
const failed = checks.find((c) => !c.allow);
|
||||
return failed ?? { allow: true, reason: "all guardrails passed" };
|
||||
}
|
||||
@ -53,41 +53,23 @@ function extractFirstNm(wavelengths: string | null): number | null {
|
||||
// ── Haupt-Matching-Logik (identisch mit Nightly-Matcher) ────────────────────
|
||||
|
||||
function calcConfidence(
|
||||
fx: {
|
||||
standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null;
|
||||
cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null;
|
||||
},
|
||||
cand: {
|
||||
standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null;
|
||||
cdr_type: string | null; temp_range: string | null; protocol_family: string | null; wdm_type: string | null;
|
||||
}
|
||||
): { confidence: number; basis: string[]; hardReject: boolean } {
|
||||
// Hard rejects: incompatible entity types — no spec similarity overrides these
|
||||
if (fx.protocol_family && cand.protocol_family && fx.protocol_family !== cand.protocol_family) {
|
||||
return { confidence: 0, basis: [`protocol_family:${fx.protocol_family}≠${cand.protocol_family}`], hardReject: true };
|
||||
}
|
||||
if (fx.temp_range && cand.temp_range && fx.temp_range !== cand.temp_range) {
|
||||
return { confidence: 0, basis: [`temp_range:${fx.temp_range}≠${cand.temp_range}`], hardReject: true };
|
||||
}
|
||||
if (fx.wdm_type && cand.wdm_type && fx.wdm_type !== cand.wdm_type) {
|
||||
return { confidence: 0, basis: [`wdm_type:${fx.wdm_type}≠${cand.wdm_type}`], hardReject: true };
|
||||
}
|
||||
|
||||
fx: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null },
|
||||
cand: { standard_name: string | null; fiber_type: string | null; reach_meters: number | null; wavelengths: string | null }
|
||||
): { confidence: number; basis: string[] } {
|
||||
// Max-Score: form_factor(25) + speed_gbps(20) + standard_name(30) +
|
||||
// wavelength_nm(20) + fiber_type(10) + reach(10) = 115
|
||||
// form_factor and speed_gbps are pre-filtered by SQL.
|
||||
// Beide form_factor und speed_gbps sind bereits durch den SQL-Filter gesichert.
|
||||
let score = 0;
|
||||
const basis: string[] = [];
|
||||
|
||||
score += 25; basis.push("form_factor");
|
||||
score += 20; basis.push("speed_gbps");
|
||||
|
||||
if (fx.standard_name && cand.standard_name) {
|
||||
if (fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) {
|
||||
score += 30; basis.push("standard_name");
|
||||
} else {
|
||||
score -= 25; // known mismatch: strong negative signal
|
||||
}
|
||||
if (
|
||||
fx.standard_name && cand.standard_name &&
|
||||
fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()
|
||||
) {
|
||||
score += 30; basis.push("standard_name");
|
||||
}
|
||||
|
||||
const fxNm = extractFirstNm(fx.wavelengths);
|
||||
@ -120,17 +102,8 @@ function calcConfidence(
|
||||
score += 5; basis.push("reach_null");
|
||||
}
|
||||
|
||||
// CDR type mismatch at >=400G is a strong penalty (different product class)
|
||||
if (fx.cdr_type && cand.cdr_type) {
|
||||
if (fx.cdr_type !== cand.cdr_type) {
|
||||
score -= 20;
|
||||
} else {
|
||||
score += 5; basis.push(`cdr_${fx.cdr_type}`);
|
||||
}
|
||||
}
|
||||
|
||||
const confidence = Math.max(0, Math.min(1, score / 115));
|
||||
return { confidence, basis, hardReject: false };
|
||||
return { confidence, basis };
|
||||
}
|
||||
|
||||
// ── Haupt-Funktion ───────────────────────────────────────────────────────────
|
||||
@ -162,14 +135,9 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
fiber_type: string | null;
|
||||
reach_meters: number | null;
|
||||
wavelengths: string | null;
|
||||
cdr_type: string | null;
|
||||
temp_range: string | null;
|
||||
protocol_family: string | null;
|
||||
wdm_type: string | null;
|
||||
}>(`
|
||||
SELECT t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
|
||||
@ -201,17 +169,12 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
reach_meters: number | null;
|
||||
wavelengths: string | null;
|
||||
vendor_name: string;
|
||||
cdr_type: string | null;
|
||||
temp_range: string | null;
|
||||
protocol_family: string | null;
|
||||
wdm_type: string | null;
|
||||
last_price: Date | null;
|
||||
price_count: string;
|
||||
}>(`
|
||||
SELECT t.id AS competitor_id, t.part_number, t.standard_name,
|
||||
t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters,
|
||||
t.wavelengths, v.name AS vendor_name,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type,
|
||||
MAX(po.time) AS last_price, COUNT(*) AS price_count
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
@ -223,25 +186,19 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
AND ROUND(t.speed_gbps::NUMERIC, 2) = ROUND($2::NUMERIC, 2)
|
||||
AND t.id != $3
|
||||
GROUP BY t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name,
|
||||
t.cdr_type, t.temp_range, t.protocol_family, t.wdm_type
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths, v.name
|
||||
HAVING COUNT(*) >= 1
|
||||
`, [fx.form_factor, fx.speed_gbps, fx.id]);
|
||||
|
||||
for (const cand of candidates) {
|
||||
const { confidence, basis, hardReject } = calcConfidence(fx, cand);
|
||||
const { confidence, basis } = calcConfidence(fx, cand);
|
||||
|
||||
if (hardReject || confidence < CONFIDENCE_MIN) {
|
||||
if (confidence < CONFIDENCE_MIN) {
|
||||
result.skippedLowConfidence++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Require FX standard_name for auto_approve; if competitor also has one it must match
|
||||
const candStdOk = !cand.standard_name || !fx.standard_name ||
|
||||
cand.standard_name.trim().toUpperCase() === fx.standard_name.trim().toUpperCase();
|
||||
const canAutoApprove = fx.standard_name !== null && candStdOk;
|
||||
|
||||
const status = (confidence >= CONFIDENCE_AUTO_APPROVE && canAutoApprove) ? "auto_approved" : "pending";
|
||||
const status = confidence >= CONFIDENCE_AUTO_APPROVE ? "auto_approved" : "pending";
|
||||
const notes =
|
||||
`${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` +
|
||||
`basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` +
|
||||
@ -249,25 +206,18 @@ export async function runCatalogReconcile(): Promise<ReconcileResult> {
|
||||
`last_price: ${cand.last_price?.toISOString() ?? "never"} | ` +
|
||||
`source: catalog-reconcile`;
|
||||
|
||||
// Upsert — bereits approved/rejected Einträge nicht überschreiben. Pending
|
||||
// rows get re_research_due_at = NOW() so the daily
|
||||
// maintenance:re-research-equivalences job (scheduler.ts) actually drains
|
||||
// them — that job only evaluates rows where re_research_due_at IS NOT NULL,
|
||||
// and without this the backlog only grows (found live 2026-07-13: 289,681
|
||||
// pending rows had accumulated with re_research_due_at always NULL).
|
||||
const reResearchDueAt = status === "pending" ? new Date() : null;
|
||||
// Upsert — bereits approved/rejected Einträge nicht überschreiben
|
||||
const { rowCount } = await pool.query(`
|
||||
INSERT INTO transceiver_equivalences
|
||||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status, re_research_due_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
|
||||
confidence = EXCLUDED.confidence,
|
||||
match_basis = EXCLUDED.match_basis,
|
||||
match_notes = EXCLUDED.match_notes,
|
||||
re_research_due_at = COALESCE(transceiver_equivalences.re_research_due_at, EXCLUDED.re_research_due_at),
|
||||
updated_at = NOW()
|
||||
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected', 'auto_approved')
|
||||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status, reResearchDueAt]);
|
||||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]);
|
||||
|
||||
const wasInsertOrUpdate = (rowCount ?? 0) > 0;
|
||||
if (!wasInsertOrUpdate) {
|
||||
|
||||
@ -40,11 +40,8 @@ interface CatalogProduct {
|
||||
title: string;
|
||||
url: string | null;
|
||||
imageUrl: string | null;
|
||||
productType: string | null;
|
||||
price: {
|
||||
amount: number | null;
|
||||
listAmount: number | null;
|
||||
selfConfigureAmount: number | null;
|
||||
currency: string | null;
|
||||
source: "api" | "missing";
|
||||
fetchedAt: string;
|
||||
@ -66,8 +63,6 @@ interface CatalogProduct {
|
||||
bidi: boolean | null;
|
||||
dwdm: boolean | null;
|
||||
cwdm: boolean | null;
|
||||
cdrType: string | null;
|
||||
protocolFamily: string | null;
|
||||
};
|
||||
compatibility: Array<{
|
||||
vendor: string;
|
||||
@ -83,7 +78,6 @@ export interface FlexoptixSyncResult {
|
||||
skipped: number;
|
||||
priceWrites: number;
|
||||
stockWrites: number;
|
||||
mapWrites: number;
|
||||
}
|
||||
|
||||
// ── Generic helpers ────────────────────────────────────────────────────────
|
||||
@ -187,8 +181,7 @@ function parseReachMeters(value: unknown): number | null {
|
||||
|
||||
function parseSpeedGbps(value: unknown): number | null {
|
||||
const numeric = asNumber(value);
|
||||
// API returns -10 / -1 as sentinel for N/A (e.g. SFP trays, accessories)
|
||||
if (numeric !== null) return numeric < 0 ? null : numeric;
|
||||
if (numeric !== null) return numeric;
|
||||
const text = asString(value)?.toLowerCase();
|
||||
if (!text) return null;
|
||||
const gbps = text.match(/([\d.,]+)\s*(g|gb|gbps|gbit)/);
|
||||
@ -238,42 +231,12 @@ function inferWavelengthNm(...values: Array<string | null>): number | null {
|
||||
|
||||
// ── Normalization ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
function parseStandardName(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
// Extract from first segment (before first " | ")
|
||||
const seg = title.split(" | ")[0] ?? title;
|
||||
// Try full IEEE form first: 10GBASE-SR, 100GBASE-DR4, etc.
|
||||
const ieee = seg.match(/((?:\d+(?:\.\d+)?[TGM])?BASE[-/]\w+)/i);
|
||||
if (ieee) return ieee[1].toUpperCase();
|
||||
// Short-form standards — ordered longest-first to avoid partial matches
|
||||
const m = seg.match(
|
||||
/(OpenZR\+|ZR\+|ZX\+|DR4\+|LR4\+|SR10|SR8|LR8|ER8|DR8|VR8|SR4|LR4|ER4|FR4|DR4|CLR4|PLR4|PSM4|CWDM4|LRM|BX\w*|DR|LR|ER|SR|ZR|ZX|SX|LX|LH|EX|T)/
|
||||
);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
function parseCdrType(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
if (/dual[\s-]cdr/i.test(title)) return "dual_cdr";
|
||||
if (/with[\s-]?cdr/i.test(title)) return "cdr";
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseProtocolFamily(title: string | null): string | null {
|
||||
if (!title) return null;
|
||||
if (/fibre[\s-]?channel|fc\s*\d+g/i.test(title)) return "fibre_channel";
|
||||
if (/\bsonet\b|\bsdh\b|\boc-\d+\b/i.test(title)) return "sonet";
|
||||
if (/\binfiniband\b/i.test(title)) return "infiniband";
|
||||
return "ethernet";
|
||||
}
|
||||
|
||||
function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] {
|
||||
const rawCompat = row.compatibility ?? row.compatibilities ?? row.vendorCompatibility;
|
||||
const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : [];
|
||||
return rows.flatMap(entry => {
|
||||
const flat = flatLookup(entry);
|
||||
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor", "compatibleToVendor"]));
|
||||
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"]));
|
||||
if (!vendor) return [];
|
||||
return [{
|
||||
vendor,
|
||||
@ -292,10 +255,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
|
||||
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
||||
const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"]));
|
||||
const productType = asString(pick(flat, ["type", "producttype", "moduletype"]));
|
||||
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
||||
const listAmount = asNumber(pick(flat, ["listprice", "msrp", "rrp", "recommendedprice"]));
|
||||
const selfConfigureAmount = asNumber(pick(flat, ["selfconfigureprice", "selfconfigprice"]));
|
||||
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
||||
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
||||
const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"]));
|
||||
@ -317,13 +277,10 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
fetchedAt,
|
||||
sku,
|
||||
title,
|
||||
productType,
|
||||
url,
|
||||
imageUrl,
|
||||
price: {
|
||||
amount,
|
||||
listAmount: (listAmount !== null && listAmount !== amount) ? listAmount : null,
|
||||
selfConfigureAmount,
|
||||
currency,
|
||||
source: amount === null ? "missing" : "api",
|
||||
fetchedAt,
|
||||
@ -345,8 +302,6 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
bidi: asBoolean(pick(flat, ["bidi", "bidirectional"])) ?? flags.bidi,
|
||||
dwdm: asBoolean(pick(flat, ["dwdm"])) ?? flags.dwdm,
|
||||
cwdm: asBoolean(pick(flat, ["cwdm"])) ?? flags.cwdm,
|
||||
cdrType: parseCdrType(title),
|
||||
protocolFamily: parseProtocolFamily(title),
|
||||
},
|
||||
compatibility: extractCompatibility(row),
|
||||
};
|
||||
@ -354,33 +309,14 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
||||
|
||||
// ── Import helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
function isMerchandise(product: CatalogProduct): boolean {
|
||||
// Flexoptix merchandise SKU prefixes: jackets (MJA), mugs (MMU), stickers (MST),
|
||||
// t-shirts (MSH), trolley (MTR), buff (MBU). FO-EXTRA-* and FO-PC-* are non-product.
|
||||
if (/^FO\.M(JA|MU|ST|SH|TR|BU)\d/i.test(product.sku)) return true;
|
||||
if (/^FO-(EXTRA-COMMUNITY-KIT|PC-\d)/i.test(product.sku)) return true;
|
||||
const t = product.title.toLowerCase();
|
||||
return /\b(jacket|coffee mug|t-shirt|sticker|travel trolley|buff)\b/.test(t);
|
||||
}
|
||||
|
||||
function canImportProduct(product: CatalogProduct): boolean {
|
||||
// Minimal gate: sku + title required. formFactor/speed/reach are optional
|
||||
// — findOrCreateScrapedTransceiver provides sensible defaults for accessories.
|
||||
// Skip country-of-origin variants (SKUs like "S2A.CL.10000:S2") — they are
|
||||
// duplicates of the base SKU with a different sourcing region, same price/stock.
|
||||
if (product.sku.includes(":")) return false;
|
||||
if (isMerchandise(product)) return false;
|
||||
return Boolean(product.sku && product.title);
|
||||
}
|
||||
|
||||
function formFactorFallback(product: CatalogProduct): string | undefined {
|
||||
const t = product.productType?.toLowerCase() ?? "";
|
||||
if (/accessories|adapter/i.test(t)) return "Accessory";
|
||||
if (/hardware/i.test(t)) return "Hardware";
|
||||
// Passive MTP/MPO breakout cables have no form-factor in the API response.
|
||||
const title = product.title.toLowerCase();
|
||||
if (/\bmtp\b|\bmpo\b/.test(title) && /cable|breakout/.test(title)) return "MTP";
|
||||
return undefined;
|
||||
return Boolean(
|
||||
product.sku
|
||||
&& product.title
|
||||
&& product.optics.formFactor
|
||||
&& product.optics.speedGbps !== null
|
||||
&& product.optics.reachM !== null,
|
||||
);
|
||||
}
|
||||
|
||||
function reachLabel(reachM: number | null): string | undefined {
|
||||
@ -396,14 +332,8 @@ function speedLabel(speedGbps: number | null): string | undefined {
|
||||
}
|
||||
|
||||
function categoryFor(product: CatalogProduct): string {
|
||||
const apiType = product.productType?.toLowerCase() ?? "";
|
||||
if (/accessories|adapter/i.test(apiType)) return "Accessory";
|
||||
if (/hardware/i.test(apiType)) return "Hardware";
|
||||
if (apiType.includes("cable")) return "Cable";
|
||||
const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase();
|
||||
// Passive MTP/MPO fan-out cables (e.g. M3.T10L.* "MTP female to 10x LC ... Breakout cable")
|
||||
if (/\bmtp\b.*cable|cable.*\bmtp\b|\bmpo\b.*cable|\bbreakout cable\b/.test(text)) return "Cable";
|
||||
if (/\bdac\b|direct attach|passive copper/.test(text)) return "DAC";
|
||||
if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
|
||||
if (/\baoc\b|active optical/.test(text)) return "AOC";
|
||||
if (/coherent|zr|dco/.test(text)) return "Coherent";
|
||||
return "DataCenter";
|
||||
@ -417,41 +347,26 @@ async function importProduct(
|
||||
partNumber: product.sku,
|
||||
vendorId,
|
||||
productUrl: product.url ?? undefined,
|
||||
formFactor: product.optics.formFactor ?? formFactorFallback(product),
|
||||
formFactor: product.optics.formFactor ?? undefined,
|
||||
speedGbps: product.optics.speedGbps ?? undefined,
|
||||
speed: speedLabel(product.optics.speedGbps),
|
||||
reachMeters: product.optics.reachM ?? undefined,
|
||||
reachLabel: reachLabel(product.optics.reachM) ?? undefined,
|
||||
reachLabel: reachLabel(product.optics.reachM),
|
||||
fiberType: product.optics.fiberType ?? undefined,
|
||||
wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
|
||||
category: categoryFor(product),
|
||||
});
|
||||
|
||||
// Write image_url, product_page_url, and notes (product title) from API
|
||||
if (product.imageUrl || product.url || product.title) {
|
||||
const wdmType = product.optics.dwdm ? "DWDM" : product.optics.cwdm ? "CWDM" : null;
|
||||
// Write image_url and product_page_url from bulk API response
|
||||
if (product.imageUrl || product.url) {
|
||||
await pool.query(`
|
||||
UPDATE transceivers SET
|
||||
image_url = COALESCE(NULLIF(image_url, ''), $1::text),
|
||||
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2::text),
|
||||
notes = $3::text,
|
||||
standard_name = COALESCE(NULLIF(standard_name, ''), $5::text),
|
||||
wdm_type = COALESCE(NULLIF(wdm_type, ''), $6::text),
|
||||
cdr_type = COALESCE(NULLIF(cdr_type, ''), $7::text),
|
||||
cdr_support = CASE WHEN $7::text IS NOT NULL THEN true ELSE cdr_support END,
|
||||
protocol_family = COALESCE(NULLIF(protocol_family, ''), $8::text),
|
||||
image_url = COALESCE(NULLIF(image_url, ''), $1),
|
||||
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2),
|
||||
updated_at = NOW()
|
||||
WHERE id = $4
|
||||
`, [
|
||||
product.imageUrl ?? null,
|
||||
product.url ?? null,
|
||||
product.title,
|
||||
transceiverId,
|
||||
parseStandardName(product.title) ?? product.optics.protocol ?? null,
|
||||
wdmType,
|
||||
product.optics.cdrType ?? null,
|
||||
product.optics.protocolFamily ?? null,
|
||||
]);
|
||||
WHERE id = $3
|
||||
AND ($1 IS NOT NULL OR $2 IS NOT NULL)
|
||||
`, [product.imageUrl ?? null, product.url ?? null, transceiverId]);
|
||||
}
|
||||
|
||||
let priceWritten = false;
|
||||
@ -474,61 +389,11 @@ async function importProduct(
|
||||
});
|
||||
}
|
||||
|
||||
if (product.price.selfConfigureAmount !== null && product.price.currency) {
|
||||
await upsertPriceObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price.selfConfigureAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-selfconfigure",
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
url: product.url ?? undefined,
|
||||
contentHash: contentHash({
|
||||
source: product.source,
|
||||
sku: product.sku,
|
||||
price: product.price.selfConfigureAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-selfconfigure",
|
||||
fetchedAt: product.price.fetchedAt,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
if (product.price.listAmount !== null && product.price.currency) {
|
||||
await upsertPriceObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
price: product.price.listAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-msrp",
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
url: product.url ?? undefined,
|
||||
contentHash: contentHash({
|
||||
source: product.source,
|
||||
sku: product.sku,
|
||||
price: product.price.listAmount,
|
||||
currency: product.price.currency,
|
||||
marketplace: "flexoptix-msrp",
|
||||
fetchedAt: product.price.fetchedAt,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// When status is known but no numeric quantity, synthesize a value so
|
||||
// upsertStockObservation's early-exit guard doesn't skip the write.
|
||||
const syntheticQty =
|
||||
product.stock.quantity !== null ? product.stock.quantity
|
||||
: product.stock.status === "in_stock" ? 1
|
||||
: product.stock.status === "out_of_stock" ? 0
|
||||
: undefined;
|
||||
|
||||
const stockWritten = await upsertStockObservation({
|
||||
transceiverId,
|
||||
sourceVendorId: vendorId,
|
||||
stockLevel: product.stock.status ?? "unknown",
|
||||
quantityAvailable: syntheticQty,
|
||||
quantityAvailable: product.stock.quantity ?? undefined,
|
||||
priceNet: product.price.amount ?? undefined,
|
||||
productUrl: product.url ?? undefined,
|
||||
priceCurrency: product.price.currency ?? undefined,
|
||||
@ -538,22 +403,6 @@ async function importProduct(
|
||||
return { priceWritten, stockWritten };
|
||||
}
|
||||
|
||||
async function writeProductMapEntries(product: CatalogProduct): Promise<number> {
|
||||
const entries = product.compatibility
|
||||
.filter(c => c.coding !== null && c.vendor !== null)
|
||||
.map(c => ({
|
||||
oemPartNumber: c.coding!,
|
||||
oemVendor: c.vendor,
|
||||
flexoptixSku: product.sku,
|
||||
priceEur: product.price.amount,
|
||||
formFactor: product.optics.formFactor,
|
||||
speedGbps: product.optics.speedGbps,
|
||||
reachLabel: reachLabel(product.optics.reachM) ?? null,
|
||||
fiberType: product.optics.fiberType,
|
||||
}));
|
||||
return batchUpsertProductMap(entries);
|
||||
}
|
||||
|
||||
// ── API client ─────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } {
|
||||
@ -561,7 +410,7 @@ function validateEnv(): { baseUrl: string; username: string | null; password: st
|
||||
if (!baseUrl) {
|
||||
throw new Error("FLEXOPTIX_API_BASE_URL is required for Flexoptix API sync");
|
||||
}
|
||||
const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() || null; // empty string -> null so bearer-login fallback fires
|
||||
const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() ?? null;
|
||||
const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim() ?? null;
|
||||
const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim() ?? null;
|
||||
if (!token && (!username || !password)) {
|
||||
@ -624,6 +473,8 @@ async function fetchAllProducts(baseUrl: string, headers: Record<string, string>
|
||||
if (rows.length === 0) break;
|
||||
|
||||
allRows.push(...rows);
|
||||
|
||||
if (rows.length < (Number.isFinite(limit) ? limit : 500)) break;
|
||||
}
|
||||
|
||||
return allRows;
|
||||
@ -641,94 +492,6 @@ function extractRows(payload: unknown): JsonRecord[] {
|
||||
|
||||
// ── Main export ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
// ── Self-configure price enrichment ───────────────────────────────────────
|
||||
// The bulk paginated endpoint omits self_configure_price.
|
||||
// A targeted sku[] fetch returns it. Batch 50 SKUs per call (~10 calls total).
|
||||
|
||||
async function fetchSelfConfigurePrices(
|
||||
skus: string[],
|
||||
baseUrl: string,
|
||||
headers: Record<string, string>,
|
||||
currency: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Map<string, number>> {
|
||||
const BATCH_SIZE = 50;
|
||||
const priceMap = new Map<string, number>();
|
||||
const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products";
|
||||
|
||||
for (let i = 0; i < skus.length; i += BATCH_SIZE) {
|
||||
const batch = skus.slice(i, i + BATCH_SIZE);
|
||||
const url = buildUrl(baseUrl, productPath);
|
||||
url.searchParams.set("currency", currency);
|
||||
batch.forEach(sku => url.searchParams.append("sku[]", sku));
|
||||
|
||||
try {
|
||||
const payload = await fetchJson(url, { headers }, timeoutMs);
|
||||
for (const row of extractRows(payload)) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sku = asString(row["sku"] ?? row["SKU"]);
|
||||
const flat = flatLookup(row);
|
||||
const scPrice = asNumber(flat["selfconfigureprice"] ?? flat["selfconfigprice"] ?? null);
|
||||
if (sku && scPrice !== null) priceMap.set(sku, scPrice);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[${new Date().toISOString()}] Self-configure enrichment batch failed: ${msg.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return priceMap;
|
||||
}
|
||||
|
||||
|
||||
// ── Product map batch upsert ─────────────────────────────────────────────────
|
||||
// Writes OEM vendor coding entries into flexoptix_product_map.
|
||||
// One row per (oem_part_number, oem_vendor) pair — ON CONFLICT updates specs.
|
||||
|
||||
async function batchUpsertProductMap(entries: Array<{
|
||||
oemPartNumber: string;
|
||||
oemVendor: string;
|
||||
flexoptixSku: string;
|
||||
priceEur: number | null;
|
||||
formFactor: string | null;
|
||||
speedGbps: number | null;
|
||||
reachLabel: string | null;
|
||||
fiberType: string | null;
|
||||
}>): Promise<number> {
|
||||
if (entries.length === 0) return 0;
|
||||
const BATCH_SIZE = 500;
|
||||
let written = 0;
|
||||
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
|
||||
const chunk = entries.slice(i, i + BATCH_SIZE);
|
||||
const placeholders: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let n = 1;
|
||||
for (const e of chunk) {
|
||||
placeholders.push(`($${n},$${n+1},$${n+2},$${n+3},$${n+4},$${n+5},$${n+6},$${n+7},'exact',NOW())`);
|
||||
params.push(e.oemPartNumber, e.oemVendor, e.flexoptixSku, e.priceEur, e.formFactor, e.speedGbps, e.reachLabel, e.fiberType);
|
||||
n += 8;
|
||||
}
|
||||
await pool.query(`
|
||||
INSERT INTO flexoptix_product_map
|
||||
(oem_part_number, oem_vendor, flexoptix_sku, flexoptix_price_eur,
|
||||
form_factor, speed_gbps, reach_label, fiber_type, match_type, last_verified)
|
||||
VALUES ${placeholders.join(',')}
|
||||
ON CONFLICT (oem_part_number, oem_vendor) DO UPDATE SET
|
||||
flexoptix_sku = EXCLUDED.flexoptix_sku,
|
||||
flexoptix_price_eur = EXCLUDED.flexoptix_price_eur,
|
||||
form_factor = EXCLUDED.form_factor,
|
||||
speed_gbps = EXCLUDED.speed_gbps,
|
||||
reach_label = EXCLUDED.reach_label,
|
||||
fiber_type = EXCLUDED.fiber_type,
|
||||
last_verified = EXCLUDED.last_verified,
|
||||
updated_at = NOW()
|
||||
`, params);
|
||||
written += chunk.length;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
const { baseUrl, username, password, token } = validateEnv();
|
||||
const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
|
||||
@ -754,45 +517,25 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
const importable = products.filter(canImportProduct);
|
||||
const skipped = products.length - importable.length;
|
||||
|
||||
// Enrich importable products with self_configure_price (not in bulk response)
|
||||
const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR";
|
||||
const scPrices = await fetchSelfConfigurePrices(
|
||||
importable.map(p => p.sku),
|
||||
baseUrl,
|
||||
headers,
|
||||
currency,
|
||||
timeoutMs,
|
||||
);
|
||||
let scEnriched = 0;
|
||||
for (const product of importable) {
|
||||
const scPrice = scPrices.get(product.sku);
|
||||
if (scPrice !== undefined) {
|
||||
product.price.selfConfigureAmount = scPrice;
|
||||
scEnriched++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped} | self-configure enriched: ${scEnriched}`);
|
||||
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`);
|
||||
|
||||
const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
|
||||
|
||||
let priceWrites = 0;
|
||||
let stockWrites = 0;
|
||||
let mapWrites = 0;
|
||||
|
||||
for (const product of importable) {
|
||||
try {
|
||||
const result = await importProduct(product, vendorId);
|
||||
if (result.priceWritten) priceWrites++;
|
||||
if (result.stockWritten) stockWrites++;
|
||||
mapWrites += await writeProductMapEntries(product);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes, ${mapWrites} product-map upserts`);
|
||||
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`);
|
||||
|
||||
return {
|
||||
fetched: rawRows.length,
|
||||
@ -800,6 +543,5 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||
skipped,
|
||||
priceWrites,
|
||||
stockWrites,
|
||||
mapWrites,
|
||||
};
|
||||
}
|
||||
|
||||
@ -69,8 +69,6 @@ interface FxApiProduct {
|
||||
}
|
||||
|
||||
interface ParsedSpecs {
|
||||
formFactor: string | null;
|
||||
speedGbps: number | null;
|
||||
complianceCode: string | null;
|
||||
laserType: string | null;
|
||||
receiverType: string | null;
|
||||
@ -188,73 +186,6 @@ function parseModulation(text: string | null): string | null {
|
||||
return match ? match[1].toUpperCase().replace("PAM-4", "PAM4") : text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative form factor from the structured "Form Factor" spec (NOT the name).
|
||||
*/
|
||||
function normalizeFormFactorSpec(v: string | null): string | null {
|
||||
if (!v) return null;
|
||||
const t = v.toUpperCase();
|
||||
const order = ["QSFP-DD800","QSFP-DD","QSFP112","QSFP56","QSFP28","QSFP+",
|
||||
"OSFP224","OSFP112","OSFP","SFP56","SFP28","CSFP","SFP+","SFP",
|
||||
"XFP","CFP4","CFP2","CFP","GBIC"];
|
||||
for (const ff of order) if (t.includes(ff)) return ff === "CSFP" ? "SFP" : ff;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Map a datasheet line rate (Gbit/s) to the nominal standard speed. */
|
||||
function mapLineRateToNominal(u: number): number {
|
||||
const table: Array<[number, number]> = [
|
||||
[0.2, 0.1], [1.5, 1], [3.5, 2.5], [6, 5], [13, 10], [20, 16],
|
||||
[33, 25], [46, 40], [60, 50], [120, 100], [240, 200], [480, 400],
|
||||
[960, 800], [2000, 1600],
|
||||
];
|
||||
for (const [ceil, nom] of table) if (u <= ceil) return nom;
|
||||
return Math.round(u);
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative nominal speed (Gbps) sourced from the datasheet specs only:
|
||||
* primary = highest Ethernet rate named in "Supported Protocols";
|
||||
* fallback = upper bound of "Bandwidth" line rate mapped to nominal.
|
||||
* Never derived from the product name.
|
||||
*/
|
||||
function nominalSpeedFromSpecs(specs: FxApiSpec[]): number | null {
|
||||
const protocols = specArray(specs, "Supported Protocols");
|
||||
let best: number | null = null;
|
||||
for (const p of protocols) {
|
||||
const t = p.toLowerCase();
|
||||
let sp: number | null = null;
|
||||
if (/\bfast ethernet\b/.test(t)) sp = 0.1;
|
||||
else if (/\bgigabit ethernet\b/.test(t)) sp = 1;
|
||||
else {
|
||||
// Terabit first ("1.6t ethernet" -> 1600), then Gigabit. Aggregate "Nx MMMg"
|
||||
// twin-port labels resolve via the explicit total (e.g. "1.6T Ethernet").
|
||||
const mt = t.match(/\b(\d+(?:\.\d+)?)\s*t(?:b|be|ig)?\s*ethernet\b/);
|
||||
if (mt) sp = parseFloat(mt[1]) * 1000;
|
||||
else {
|
||||
const m = t.match(/(?:^|[^x\d])(\d+(?:\.\d+)?)\s*g(?:b|be|ig)?\s*ethernet\b/) || t.match(/\b(\d+)\s*gbe\b/);
|
||||
if (m) sp = parseFloat(m[1]);
|
||||
}
|
||||
}
|
||||
if (sp !== null && (best === null || sp > best)) best = sp;
|
||||
}
|
||||
if (best !== null) return best;
|
||||
const bw = specValue(specs, "Bandwidth");
|
||||
if (bw) {
|
||||
const gbits = Array.from(bw.matchAll(/([\d.]+)\s*gbit\/s/gi)).map(m => parseFloat(m[1]));
|
||||
if (gbits.length) return mapLineRateToNominal(Math.max(...gbits));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Format a nominal Gbps value as a display label ("1G", "100G", "1.6T"). */
|
||||
function speedLabelGbps(g: number | null): string | null {
|
||||
if (g === null) return null;
|
||||
if (g >= 1000) return `${g / 1000}T`;
|
||||
if (g < 1) return `${Math.round(g * 1000)}M`;
|
||||
return `${g}G`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the flat specifications array into structured fields.
|
||||
*/
|
||||
@ -263,8 +194,6 @@ function parseSpecs(specs: FxApiSpec[]): ParsedSpecs {
|
||||
const rxPowers = parseDbm(specValue(specs, "Receiver min/max per lane"));
|
||||
|
||||
return {
|
||||
formFactor: normalizeFormFactorSpec(specValue(specs, "Form Factor")),
|
||||
speedGbps: nominalSpeedFromSpecs(specs),
|
||||
complianceCode: specValue(specs, "Compliance Code"),
|
||||
laserType: specValue(specs, "Laser"),
|
||||
receiverType: specValue(specs, "Receiver Type"),
|
||||
@ -420,9 +349,6 @@ async function writeDetails(
|
||||
|
||||
await pool.query(`
|
||||
UPDATE transceivers SET
|
||||
form_factor = COALESCE($23, form_factor),
|
||||
speed_gbps = COALESCE($24, speed_gbps),
|
||||
speed = COALESCE($25, speed),
|
||||
fx_specifications = $1,
|
||||
fx_compatibilities = $2,
|
||||
compliance_code = COALESCE(compliance_code, $3),
|
||||
@ -470,9 +396,6 @@ async function writeDetails(
|
||||
product.image ?? null, // $20
|
||||
product.url ?? null, // $21
|
||||
transceiverId, // $22
|
||||
parsed.formFactor, // $23
|
||||
parsed.speedGbps, // $24
|
||||
speedLabelGbps(parsed.speedGbps), // $25
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@ -14,18 +14,9 @@
|
||||
* - Max 30 competitor matches per FX product (safety cap)
|
||||
*
|
||||
* Match quality:
|
||||
* confidence = 0.85 (starting prior only)
|
||||
* confidence = 0.85
|
||||
* match_basis = '{spec}'
|
||||
* status = 'pending' — unlike OPN matches (manufacturer-confirmed ground
|
||||
* truth), spec matches are a coarse JOIN with no standard_name / fiber_type /
|
||||
* recent-price discrimination, so they route into the same
|
||||
* maintenance:re-research-equivalences pipeline as catalog-reconcile.ts's
|
||||
* pending rows (evaluateEquivalenceResearch(), scheduler.ts) for real
|
||||
* confidence scoring before ever being served as an approved equivalence.
|
||||
* Found live 2026-07-17: of the equivalences this matcher had previously put
|
||||
* straight into 'auto_approved', ~19% failed a recent-competitor-price check
|
||||
* and ~8% had a conflicting standard_name — i.e. real false-positive risk
|
||||
* was going live unreviewed.
|
||||
* status = 'auto_approved'
|
||||
*/
|
||||
|
||||
import { pool } from "../utils/db";
|
||||
@ -48,14 +39,13 @@ const INSERT_SPEC_MATCHES = `
|
||||
match_basis,
|
||||
match_notes,
|
||||
created_at,
|
||||
updated_at,
|
||||
re_research_due_at
|
||||
updated_at
|
||||
)
|
||||
SELECT DISTINCT
|
||||
fx.id AS flexoptix_id,
|
||||
comp.id AS competitor_id,
|
||||
0.85 AS confidence,
|
||||
'pending' AS status,
|
||||
'auto_approved' AS status,
|
||||
ARRAY['spec'] AS match_basis,
|
||||
'Spec match: ' || fx.form_factor || ' ' || fx.speed_gbps || 'G ' ||
|
||||
CASE WHEN fx.reach_meters <= 300 THEN 'SR'
|
||||
@ -67,8 +57,7 @@ const INSERT_SPEC_MATCHES = `
|
||||
THEN ' @' || tip_extract_wavelength_nm(fx.wavelengths) || 'nm'
|
||||
ELSE '' END AS match_notes,
|
||||
NOW() AS created_at,
|
||||
NOW() AS updated_at,
|
||||
NOW() AS re_research_due_at
|
||||
NOW() AS updated_at
|
||||
FROM transceivers fx
|
||||
JOIN vendors vfx ON vfx.id = fx.vendor_id AND UPPER(vfx.name) LIKE '%FLEXOPTIX%'
|
||||
JOIN transceivers comp
|
||||
|
||||
@ -20,12 +20,11 @@ import { join } from "path";
|
||||
import PgBoss from "pg-boss";
|
||||
import { pool } from "../utils/db";
|
||||
import { writeRobotExperience } from "../crawler-llm/training-data-writer";
|
||||
import { dbConnectionString } from "../utils/db-connection";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
type RobotWave = "details-fast-lane" | "priority-vendors" | "all";
|
||||
export type RobotProfile = "erik-safe" | "pi-fetch" | "proxmox-heavy";
|
||||
type RobotProfile = "erik-safe" | "pi-fetch" | "proxmox-heavy";
|
||||
|
||||
interface VendorBlockerRow {
|
||||
vendor: string;
|
||||
@ -50,7 +49,7 @@ interface TipLlmResearchPlanResponse {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const connectionString = dbConnectionString();
|
||||
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
|
||||
const TIP_API_URL = process.env.TIP_API_URL || "http://127.0.0.1:3201";
|
||||
const TIP_API_TOKEN = process.env.TIP_API_TOKEN || process.env.TIP_TOKEN || "";
|
||||
|
||||
@ -65,7 +64,7 @@ const DETAILS_FAST_LANE_QUEUES = [
|
||||
"maintenance:reconcile-verification",
|
||||
];
|
||||
|
||||
export const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
{ match: /juniper/i, queues: ["scrape:catalog:juniper-oem", "scrape:catalog:juniper-mx-oem", "scrape:catalog:juniper-qfx-oem", "scrape:compat:juniper", "discover:vendor:juniper"] },
|
||||
{ match: /cisco/i, queues: ["scrape:catalog:cisco-nexus-oem", "scrape:catalog:cisco-catalyst-oem", "scrape:catalog:cisco-asr-oem", "scrape:compat:cisco", "discover:vendor:cisco-tmg"] },
|
||||
{ match: /fs\.?com/i, queues: ["scrape:pricing:fs", "discover:vendor:fs-com"] },
|
||||
@ -84,7 +83,7 @@ export const VENDOR_QUEUE_MAP: Array<{ match: RegExp; queues: string[] }> = [
|
||||
{ match: /ciena/i, queues: ["scrape:catalog:ciena-oem", "scrape:catalog:ciena-waveserver-oem"] },
|
||||
];
|
||||
|
||||
export const HEAVY_QUEUES = new Set([
|
||||
const HEAVY_QUEUES = new Set([
|
||||
"scrape:pricing:fs",
|
||||
"scrape:pricing:10gtek",
|
||||
"scrape:pricing:prolabs",
|
||||
@ -94,7 +93,7 @@ export const HEAVY_QUEUES = new Set([
|
||||
"scrape:compat:edgecore",
|
||||
]);
|
||||
|
||||
export const ERIK_SAFE_QUEUES = new Set([
|
||||
const ERIK_SAFE_QUEUES = new Set([
|
||||
"scrape:pricing:flexoptix",
|
||||
"scrape:pricing:fibermall",
|
||||
"scrape:pricing:atgbics",
|
||||
@ -105,11 +104,11 @@ export const ERIK_SAFE_QUEUES = new Set([
|
||||
"maintenance:reconcile-verification",
|
||||
]);
|
||||
|
||||
export function isDiscoveryQueue(queue: string): boolean {
|
||||
function isDiscoveryQueue(queue: string): boolean {
|
||||
return queue.startsWith("discover:");
|
||||
}
|
||||
|
||||
export function queuesForProfile(queues: string[], profile: RobotProfile): string[] {
|
||||
function queuesForProfile(queues: string[], profile: RobotProfile): string[] {
|
||||
if (profile === "proxmox-heavy") return queues;
|
||||
|
||||
if (profile === "erik-safe") {
|
||||
@ -119,7 +118,7 @@ export function queuesForProfile(queues: string[], profile: RobotProfile): strin
|
||||
return queues.filter((queue) => !HEAVY_QUEUES.has(queue) && !isDiscoveryQueue(queue));
|
||||
}
|
||||
|
||||
export function defaultMaxQueues(profile: RobotProfile): number {
|
||||
function defaultMaxQueues(profile: RobotProfile): number {
|
||||
if (profile === "erik-safe") return 3;
|
||||
if (profile === "pi-fetch") return 10;
|
||||
return 30;
|
||||
@ -133,7 +132,7 @@ function unique(items: string[]): string[] {
|
||||
return [...new Set(items)];
|
||||
}
|
||||
|
||||
export async function createBoss(): Promise<PgBoss> {
|
||||
async function createBoss(): Promise<PgBoss> {
|
||||
const boss = new PgBoss({
|
||||
connectionString,
|
||||
retryLimit: 1,
|
||||
@ -146,32 +145,6 @@ export async function createBoss(): Promise<PgBoss> {
|
||||
return boss;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single shared dispatch primitive: create each queue if needed, then send one job per
|
||||
* queue with the standard safety options (retryLimit 1, 2h expiry). Used by BOTH
|
||||
* enqueueRobotWave and the tip-dqo orchestrator so there is exactly ONE boss.send path
|
||||
* with identical safety options across the whole platform.
|
||||
*/
|
||||
export async function dispatchQueues(
|
||||
queues: string[],
|
||||
data: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
if (queues.length === 0) return;
|
||||
const boss = await createBoss();
|
||||
try {
|
||||
for (const queue of queues) {
|
||||
await boss.createQueue(queue).catch(() => {});
|
||||
await (boss as unknown as { send: (name: string, data?: object, options?: object) => Promise<string | null> }).send(
|
||||
queue,
|
||||
data,
|
||||
{ retryLimit: 1, expireInSeconds: 7200 },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await boss.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export async function getVerificationStatus(): Promise<{ summary: Record<string, string>; vendors: VendorBlockerRow[] }> {
|
||||
const summaryResult = await pool.query(`
|
||||
SELECT
|
||||
@ -287,12 +260,19 @@ export async function enqueueRobotWave(
|
||||
return queues;
|
||||
}
|
||||
|
||||
await dispatchQueues(queues, {
|
||||
source: "verification-robots",
|
||||
wave,
|
||||
run_id: runId,
|
||||
enqueued_at: new Date().toISOString(),
|
||||
});
|
||||
const boss = await createBoss();
|
||||
try {
|
||||
for (const queue of queues) {
|
||||
await boss.createQueue(queue).catch(() => {});
|
||||
await (boss as unknown as { send: (name: string, data?: object, options?: object) => Promise<string | null> }).send(
|
||||
queue,
|
||||
{ source: "verification-robots", wave, run_id: runId, enqueued_at: new Date().toISOString() },
|
||||
{ retryLimit: 1, expireInSeconds: 7200 },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await boss.stop();
|
||||
}
|
||||
|
||||
console.log("\nJobs enqueued.");
|
||||
writeRobotExperience({
|
||||
|
||||
@ -22,7 +22,6 @@ import PgBoss from "pg-boss";
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { loadavg } from "os";
|
||||
import { dbConnectionString } from "./utils/db-connection";
|
||||
|
||||
// withIsolatedStorage removed — all Crawlee scrapers now use makeCrawleeConfig()
|
||||
// for instance-level storage isolation. See packages/scraper/src/utils/crawlee-config.ts
|
||||
@ -43,7 +42,7 @@ function isLoadAcceptable(maxLoad = 2.5): boolean {
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", ".env") });
|
||||
|
||||
const connectionString = dbConnectionString();
|
||||
const connectionString = `postgres://${process.env.POSTGRES_USER || "tip"}:${process.env.POSTGRES_PASSWORD || "tip_dev_2026"}@${process.env.POSTGRES_HOST || "localhost"}:${process.env.POSTGRES_PORT || "5433"}/${process.env.POSTGRES_DB || "transceiver_db"}`;
|
||||
|
||||
type EquivalenceProduct = {
|
||||
part_number?: string | null;
|
||||
@ -3182,13 +3181,6 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
);
|
||||
|
||||
if (research.decision === "reject") {
|
||||
const rejectReason = research.rejectReason || "automated research: rejected";
|
||||
// Data-gap rejects (missing wavelength/fiber/reach evidence, not a
|
||||
// confirmed mismatch) get a future re-check — once a scraper fills the
|
||||
// gap, re-research picks it back up. Confirmed mismatches / no-price /
|
||||
// low-confidence rejects get re_research_due_at=NULL (permanent; nothing
|
||||
// will change on a re-check with the same stored specs).
|
||||
const isDataGap = rejectReason.includes("insufficient technical evidence");
|
||||
await pool.query(`
|
||||
UPDATE transceiver_equivalences
|
||||
SET status = 'rejected',
|
||||
@ -3197,7 +3189,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
reject_reason = $4,
|
||||
reviewed_by = 'automated-research',
|
||||
reviewed_at = NOW(),
|
||||
re_research_due_at = CASE WHEN $6 THEN NOW() + INTERVAL '21 days' ELSE NULL END,
|
||||
re_research_due_at = NULL,
|
||||
re_researched_at = NOW(),
|
||||
match_notes = CONCAT(
|
||||
COALESCE(match_notes, ''),
|
||||
@ -3209,9 +3201,8 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
eq.id,
|
||||
research.confidence,
|
||||
research.basis,
|
||||
rejectReason,
|
||||
research.rejectReason || "automated research: rejected",
|
||||
research.reasons.join("; "),
|
||||
isDataGap,
|
||||
]);
|
||||
|
||||
await pool.query(`
|
||||
|
||||
@ -14,9 +14,7 @@
|
||||
*
|
||||
* Fallback (when search returns nothing):
|
||||
* Form-factor matching — for each port type in switch.ports_config,
|
||||
* find Flexoptix transceivers matching that form factor AND whose speed does
|
||||
* not exceed the switch's fastest port (a transceiver faster than the port
|
||||
* cannot link at rated speed — e.g. an 800G module in a 400G OSFP port).
|
||||
* find Flexoptix transceivers matching that form factor.
|
||||
* Used as spec_match rather than vendor_compat — shown in dashboard
|
||||
* as "by form factor" rather than "explicitly verified".
|
||||
*
|
||||
@ -65,18 +63,6 @@ function portsConfigToFormFactors(portsConfig: Record<string, number>): string[]
|
||||
return [...factors];
|
||||
}
|
||||
|
||||
// Max port speed (Gbps) implied by a ports_config, e.g. {"400G_QSFP-DD":4} → 400.
|
||||
// The switch cannot host a transceiver faster than its fastest port.
|
||||
function maxSpeedFromPortsConfig(portsConfig: Record<string, number> | null): number {
|
||||
if (!portsConfig) return 0;
|
||||
let max = 0;
|
||||
for (const key of Object.keys(portsConfig)) {
|
||||
const m = key.match(/(\d+)G/i);
|
||||
if (m) max = Math.max(max, parseInt(m[1], 10));
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
// ── Flexoptix search API ────────────────────────────────────────────────────
|
||||
|
||||
interface FlexoptixSuggestion {
|
||||
@ -119,9 +105,8 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
model: string;
|
||||
vendor_name: string;
|
||||
ports_config: Record<string, number> | null;
|
||||
max_speed_gbps: number | null;
|
||||
}>(
|
||||
`SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config, sw.max_speed_gbps
|
||||
`SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config
|
||||
FROM switches sw
|
||||
JOIN vendors v ON v.id = sw.vendor_id
|
||||
ORDER BY sw.max_speed_gbps DESC`,
|
||||
@ -197,15 +182,7 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
// ── Strategy 2: Form-factor fallback ─────────────────────────────────
|
||||
if (sw.ports_config) {
|
||||
const formFactors = portsConfigToFormFactors(sw.ports_config);
|
||||
// Physical speed ceiling: a transceiver faster than the switch's fastest
|
||||
// port cannot link at its rated speed, so it is NOT compatible-by-form-factor.
|
||||
// (ceiling <= 0 → the query below matches nothing, i.e. no fallback when the
|
||||
// switch has no derivable port speed.)
|
||||
const ceiling = Math.max(
|
||||
maxSpeedFromPortsConfig(sw.ports_config),
|
||||
sw.max_speed_gbps ?? 0,
|
||||
);
|
||||
console.log(` Form factors: ${formFactors.join(", ")} (≤ ${ceiling}G)`);
|
||||
console.log(` Form factors: ${formFactors.join(", ")}`);
|
||||
|
||||
for (const ff of formFactors) {
|
||||
const { rows: txRows } = await pool.query<{ id: string }>(
|
||||
@ -213,14 +190,11 @@ export async function scrapeFlexoptixCompatibility(): Promise<void> {
|
||||
FROM transceivers t
|
||||
WHERE t.vendor_id = $1
|
||||
AND t.form_factor = $2
|
||||
AND t.speed_gbps IS NOT NULL
|
||||
AND t.speed_gbps > 0
|
||||
AND t.speed_gbps <= $4
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM compatibility c
|
||||
WHERE c.switch_id = $3 AND c.transceiver_id = t.id
|
||||
)`,
|
||||
[flexoptixVendorId, ff, sw.id, ceiling],
|
||||
[flexoptixVendorId, ff, sw.id],
|
||||
);
|
||||
|
||||
for (const tx of txRows) {
|
||||
|
||||
@ -1,138 +0,0 @@
|
||||
/**
|
||||
* Unit tests for the FS.com pure parsers. Run with:
|
||||
* npx tsx --test src/scrapers/fs-com-parse.test.ts
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
parseGermanQty,
|
||||
parseGermanDate,
|
||||
parseGermanPrice,
|
||||
extractDeliveryDate,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
sanitizeUnitsSold,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── parseGermanQty — the K/M decimal bug ─────────────────────────────────────
|
||||
|
||||
test("parseGermanQty: plain integers use '.'/',' as thousands", () => {
|
||||
assert.equal(parseGermanQty("4.895"), 4895);
|
||||
assert.equal(parseGermanQty("1.587"), 1587);
|
||||
assert.equal(parseGermanQty("1.501"), 1501);
|
||||
assert.equal(parseGermanQty("2"), 2);
|
||||
assert.equal(parseGermanQty("1627"), 1627);
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => {
|
||||
// Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier).
|
||||
assert.equal(parseGermanQty("4.8K"), 4800);
|
||||
assert.equal(parseGermanQty("210.9K"), 210900);
|
||||
assert.equal(parseGermanQty("504.4K"), 504400);
|
||||
assert.equal(parseGermanQty("1.4M"), 1_400_000);
|
||||
assert.equal(parseGermanQty("1.2M"), 1_200_000);
|
||||
assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged
|
||||
assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M with German comma decimal still works", () => {
|
||||
assert.equal(parseGermanQty("4,8K"), 4800);
|
||||
assert.equal(parseGermanQty("1,4M"), 1_400_000);
|
||||
});
|
||||
|
||||
// ── parseGermanDate ──────────────────────────────────────────────────────────
|
||||
|
||||
test("parseGermanDate: word and numeric forms", () => {
|
||||
assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08");
|
||||
assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17");
|
||||
assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("28. März 2026"), "2026-03-28");
|
||||
assert.equal(parseGermanDate("20.04.2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("nonsense"), undefined);
|
||||
});
|
||||
|
||||
// ── extractDeliveryDate ──────────────────────────────────────────────────────
|
||||
|
||||
test("extractDeliveryDate: finds bare word-date, no keyword needed", () => {
|
||||
assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08");
|
||||
assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17");
|
||||
});
|
||||
|
||||
test("extractDeliveryDate: rejects stray/garbage years and empty", () => {
|
||||
assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined);
|
||||
assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined);
|
||||
});
|
||||
|
||||
// ── parseWarehouseStock — the real screenshot block ──────────────────────────
|
||||
|
||||
const SCREENSHOT_BLOCK = [
|
||||
"2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar",
|
||||
"1.627 Stk. im Global-Lager, 17. Juli 2026",
|
||||
"1 in Nachlieferung, 28. Juli 2026",
|
||||
"4.8K verkauft",
|
||||
].join("\n");
|
||||
|
||||
test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => {
|
||||
const s = parseWarehouseStock(SCREENSHOT_BLOCK);
|
||||
assert.equal(s.deQty, 2);
|
||||
assert.equal(s.deDeliveryDate, "2026-07-08");
|
||||
assert.equal(s.globalQty, 1627);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
assert.equal(s.backorderQty, 1);
|
||||
assert.equal(s.backorderDate, "2026-07-28");
|
||||
assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => {
|
||||
const block = [
|
||||
"5 Stk. im DE-Lager", // no date — available now
|
||||
"800 Stk. im Global-Lager, 17. Juli 2026",
|
||||
].join("\n");
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 5);
|
||||
assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17
|
||||
assert.equal(s.globalQty, 800);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => {
|
||||
const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft";
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 12);
|
||||
assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: empty when no warehouse text present", () => {
|
||||
assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {});
|
||||
});
|
||||
|
||||
// ── sanitizeUnitsSold ────────────────────────────────────────────────────────
|
||||
|
||||
test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => {
|
||||
assert.equal(sanitizeUnitsSold(4800), 4800);
|
||||
assert.equal(sanitizeUnitsSold(504400), 504400);
|
||||
assert.equal(sanitizeUnitsSold(14_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(-5), undefined);
|
||||
assert.equal(sanitizeUnitsSold(undefined), undefined);
|
||||
assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000);
|
||||
});
|
||||
|
||||
// ── computeLeadTimeDays ──────────────────────────────────────────────────────
|
||||
|
||||
test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => {
|
||||
const now = new Date("2026-07-04T09:00:00");
|
||||
assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4);
|
||||
assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1);
|
||||
assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative
|
||||
assert.equal(computeLeadTimeDays([], now), undefined);
|
||||
assert.equal(computeLeadTimeDays([null, undefined], now), undefined);
|
||||
});
|
||||
|
||||
// ── parseGermanPrice (unchanged behaviour, regression guard) ─────────────────
|
||||
|
||||
test("parseGermanPrice: net EUR parsing", () => {
|
||||
assert.equal(parseGermanPrice("42,50"), 42.5);
|
||||
assert.equal(parseGermanPrice("1.063,02 €"), 1063.02);
|
||||
assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1);
|
||||
});
|
||||
@ -1,290 +0,0 @@
|
||||
/**
|
||||
* FS.com parsing helpers — pure, side-effect-free, unit-tested.
|
||||
*
|
||||
* Extracted from fs-com.ts so the number/date/warehouse parsing can be tested
|
||||
* in isolation (fs-com.ts pulls in Crawlee + Playwright + a pg Pool, which we
|
||||
* do not want to load in a unit test).
|
||||
*
|
||||
* The FS.com German storefront (www.fs.com/de/) renders per-warehouse
|
||||
* availability as a short line per warehouse, e.g.:
|
||||
*
|
||||
* 2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar
|
||||
* 1.627 Stk. im Global-Lager, 17. Juli 2026
|
||||
* 1 in Nachlieferung, 28. Juli 2026
|
||||
* 4.8K verkauft
|
||||
*
|
||||
* The quantity part was already parsed correctly; the delivery-date part was
|
||||
* not, because the previous regex only looked for a "Lieferung:"/"erwartet:"
|
||||
* keyword prefix or a numeric dd.mm.yyyy date — neither of which matches the
|
||||
* bare word-date ("8. Juli 2026") that FS.com actually renders.
|
||||
*/
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Upper bound for a plausible per-SKU lifetime "verkauft" (units-sold) counter.
|
||||
* Values above this are treated as unknown (dropped), not stored. FS.com's
|
||||
* biggest sellers are in the hundreds of thousands; multi-million values are
|
||||
* a mis-parse (see parseGermanQty K/M bug) or vendor marketing inflation.
|
||||
* Override with FS_MAX_UNITS_SOLD.
|
||||
*/
|
||||
export const DEFAULT_MAX_UNITS_SOLD = 2_000_000;
|
||||
|
||||
/** How far past a warehouse line to look for its delivery date (characters). */
|
||||
const DATE_WINDOW = 200;
|
||||
|
||||
// ── Number parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Parse the mantissa of a K/M-abbreviated number ("4.8" in "4.8K").
|
||||
*
|
||||
* K/M abbreviations carry a small decimal, never a grouped thousands value, so
|
||||
* a lone "." is the decimal point ("4.8K" = 4 800). German pages may instead
|
||||
* use "," as the decimal ("4,8K"); a "." alongside a "," is then a thousands
|
||||
* grouping and is stripped.
|
||||
*/
|
||||
function parseAbbrevMantissa(s: string): number {
|
||||
if (s.includes(",")) return parseFloat(s.replace(/\./g, "").replace(",", "."));
|
||||
return parseFloat(s); // "." is the decimal separator
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in a plain integer)
|
||||
* "1.587" → 1587
|
||||
* "4.8K" → 4800 (period = decimal in a K/M abbreviation)
|
||||
* "210.9K" → 210900
|
||||
* "1.4M" → 1400000
|
||||
* "63K" → 63000
|
||||
*/
|
||||
export function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseAbbrevMantissa(kMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseAbbrevMantissa(mMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
// Plain integer: "." and "," are both thousands separators here.
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German date to ISO "YYYY-MM-DD".
|
||||
* "8. Juli 2026" → "2026-07-08"
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
export function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German price to a EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
export function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
|
||||
// ── Units-sold sanity gate ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Clamp an implausible units-sold value to `undefined` (unknown). Returns the
|
||||
* value unchanged when it is a finite, non-negative number at or below `cap`.
|
||||
*/
|
||||
export function sanitizeUnitsSold(
|
||||
raw: number | undefined,
|
||||
cap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): number | undefined {
|
||||
if (raw === undefined) return undefined;
|
||||
if (!Number.isFinite(raw) || raw < 0) return undefined;
|
||||
if (raw > cap) return undefined;
|
||||
return raw;
|
||||
}
|
||||
|
||||
// ── Delivery-date extraction ─────────────────────────────────────────────────
|
||||
|
||||
/** dd.mm.yyyy OR "8. Juli 2026" / "20 Apr., 2026" (day + month word + year). */
|
||||
const DELIVERY_DATE_RE =
|
||||
/(\d{1,2}\.\d{1,2}\.\d{4})|(\d{1,2}\.?\s+[A-Za-zÄÖÜäöüß]{3,}\.?,?\s*\d{4})/;
|
||||
|
||||
/**
|
||||
* Find the first plausible calendar date inside a bounded text block (the slice
|
||||
* that belongs to one warehouse line) and return it as ISO "YYYY-MM-DD".
|
||||
* The date can appear with or without a "Lieferbar"/"Lieferung" keyword — FS.com
|
||||
* renders it bare ("…DE-Lager, 8. Juli 2026 Lieferbar"), so no keyword is required.
|
||||
*/
|
||||
export function extractDeliveryDate(block: string): string | undefined {
|
||||
const m = block.match(DELIVERY_DATE_RE);
|
||||
if (!m) return undefined;
|
||||
const iso = parseGermanDate(m[0]);
|
||||
if (!iso) return undefined;
|
||||
const year = Number(iso.slice(0, 4));
|
||||
if (year < 2024 || year > 2100) return undefined; // reject stray/garbage years
|
||||
return iso;
|
||||
}
|
||||
|
||||
// ── Warehouse block parsing ──────────────────────────────────────────────────
|
||||
|
||||
export interface WarehouseStock {
|
||||
deQty?: number;
|
||||
deDeliveryDate?: string;
|
||||
globalQty?: number;
|
||||
globalDeliveryDate?: string;
|
||||
backorderQty?: number;
|
||||
backorderDate?: string;
|
||||
unitsSold?: number;
|
||||
}
|
||||
|
||||
const DE_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i,
|
||||
];
|
||||
const GLOBAL_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i,
|
||||
/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i,
|
||||
];
|
||||
const BACKORDER_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i,
|
||||
/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i,
|
||||
];
|
||||
const UNITS_SOLD_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/,
|
||||
/([\d.,KkMm]+)\+?\s*sold/i,
|
||||
];
|
||||
|
||||
interface SectionMatch { value: string; index: number; }
|
||||
|
||||
function firstMatch(text: string, res: RegExp[]): SectionMatch | undefined {
|
||||
for (const re of res) {
|
||||
const m = text.match(re);
|
||||
if (m && m[1] != null && m.index != null) return { value: m[1], index: m.index };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the full warehouse-availability block out of a product page's body
|
||||
* text: DE-Lager, Global-Lager and Nachlieferung quantities, each with the
|
||||
* delivery date rendered on its line, plus the "verkauft"/"sold" counter.
|
||||
*
|
||||
* A section's delivery date is looked for only in the window between that
|
||||
* section's quantity and the start of the next section, so a warehouse with no
|
||||
* date never picks up a neighbouring warehouse's date.
|
||||
*/
|
||||
export function parseWarehouseStock(
|
||||
bodyText: string,
|
||||
unitsSoldCap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): WarehouseStock {
|
||||
const t = bodyText;
|
||||
const de = firstMatch(t, DE_QTY_RE);
|
||||
const gl = firstMatch(t, GLOBAL_QTY_RE);
|
||||
const bo = firstMatch(t, BACKORDER_QTY_RE);
|
||||
const sold = firstMatch(t, UNITS_SOLD_RE);
|
||||
|
||||
const starts = [de, gl, bo, sold]
|
||||
.filter((s): s is SectionMatch => s !== undefined)
|
||||
.map((s) => s.index);
|
||||
|
||||
const dateFor = (sec: SectionMatch | undefined): string | undefined => {
|
||||
if (!sec) return undefined;
|
||||
const later = starts.filter((b) => b > sec.index);
|
||||
const end = later.length ? Math.min(sec.index + DATE_WINDOW, ...later) : sec.index + DATE_WINDOW;
|
||||
return extractDeliveryDate(t.slice(sec.index, end));
|
||||
};
|
||||
|
||||
const result: WarehouseStock = {};
|
||||
|
||||
if (de) {
|
||||
const q = parseGermanQty(de.value);
|
||||
if (q !== undefined) result.deQty = q;
|
||||
const d = dateFor(de);
|
||||
if (d) result.deDeliveryDate = d;
|
||||
}
|
||||
if (gl) {
|
||||
const q = parseGermanQty(gl.value);
|
||||
if (q !== undefined) result.globalQty = q;
|
||||
const d = dateFor(gl);
|
||||
if (d) result.globalDeliveryDate = d;
|
||||
}
|
||||
if (bo) {
|
||||
const q = parseGermanQty(bo.value);
|
||||
if (q !== undefined) result.backorderQty = q;
|
||||
const d = dateFor(bo);
|
||||
if (d) result.backorderDate = d;
|
||||
}
|
||||
if (sold) {
|
||||
const s = sanitizeUnitsSold(parseGermanQty(sold.value), unitsSoldCap);
|
||||
if (s !== undefined) result.unitsSold = s;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Lead-time derivation ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Derive lead-time in days as the soonest availability across the warehouses
|
||||
* that quote a delivery date: min(date − today), floored at 0. Returns
|
||||
* `undefined` when no date is known (never fabricate a 0).
|
||||
*/
|
||||
export function computeLeadTimeDays(
|
||||
dates: Array<string | null | undefined>,
|
||||
now: Date = new Date()
|
||||
): number | undefined {
|
||||
const todayUtc = Date.UTC(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
let min: number | undefined;
|
||||
for (const d of dates) {
|
||||
if (!d) continue;
|
||||
const m = d.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (!m) continue;
|
||||
const dUtc = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
|
||||
const days = Math.max(0, Math.round((dUtc - todayUtc) / 86_400_000));
|
||||
if (min === undefined || days < min) min = days;
|
||||
}
|
||||
return min;
|
||||
}
|
||||
@ -63,12 +63,6 @@ import {
|
||||
} from "../utils/db";
|
||||
import { contentHash } from "../utils/hash";
|
||||
import { updateVerifiedSpecs, parseSpecTable } from "../utils/spec-updater";
|
||||
import {
|
||||
parseGermanPrice,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
DEFAULT_MAX_UNITS_SOLD,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@ -81,8 +75,6 @@ const FORCE_REVALIDATE = process.env["TIP_FORCE_REVALIDATE"] === "1";
|
||||
const ONLY_MISSING_IMAGES = process.env["FS_ONLY_MISSING_IMAGES"] === "1";
|
||||
const DB_DETAIL_ONLY = process.env["FS_DB_DETAIL_ONLY"] === "1";
|
||||
const URL_DISCOVERY_ONLY = process.env["FS_URL_DISCOVERY_ONLY"] === "1";
|
||||
// Drop implausible "verkauft" (units-sold) counters at ingest (see fs-com-parse).
|
||||
const FS_MAX_UNITS_SOLD = parseInt(process.env["FS_MAX_UNITS_SOLD"] ?? String(DEFAULT_MAX_UNITS_SOLD), 10);
|
||||
|
||||
const PROXY_URLS = (process.env["PROXY_URLS"] ?? "")
|
||||
.split(",")
|
||||
@ -121,7 +113,83 @@ const DE_COOKIES = [
|
||||
{ name: "country", value: "DE", domain: ".fs.com", path: "/" },
|
||||
];
|
||||
|
||||
// German number/date/warehouse parsers live in ./fs-com-parse (pure + unit-tested).
|
||||
// ── German locale parsers ──────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in German)
|
||||
* "210.9K" → 210900
|
||||
* "1.2M" → 1200000
|
||||
*/
|
||||
function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseFloat(kMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseFloat(mMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German date to ISO "YYYY-MM-DD".
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German price to EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
|
||||
// ── Stock level helper ─────────────────────────────────────────────────────────
|
||||
|
||||
@ -580,18 +648,64 @@ async function scrapeProductDetails(
|
||||
}
|
||||
}
|
||||
|
||||
// ── Warehouse availability (qty + per-warehouse delivery date + units sold) ──
|
||||
// FS.com renders each warehouse on its own line, e.g.
|
||||
// "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar". parseWarehouseStock
|
||||
// extracts the bare word-date per warehouse and gates units_sold.
|
||||
const wh = parseWarehouseStock(t, FS_MAX_UNITS_SOLD);
|
||||
const deQty = wh.deQty;
|
||||
const deDeliveryDate = wh.deDeliveryDate;
|
||||
const globalQty = wh.globalQty;
|
||||
const globalDeliveryDate = wh.globalDeliveryDate;
|
||||
const backorderQty = wh.backorderQty;
|
||||
const backorderDate = wh.backorderDate;
|
||||
const unitsSold = wh.unitsSold;
|
||||
// ── DE-Lager ───────────────────────────────────────────────────────────
|
||||
let deQty: number | undefined;
|
||||
let deDeliveryDate: string | undefined;
|
||||
const deM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i);
|
||||
if (deM?.[1]) {
|
||||
deQty = parseGermanQty(deM[1]);
|
||||
const idx = t.indexOf(deM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) deDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Global-Lager ───────────────────────────────────────────────────────
|
||||
let globalQty: number | undefined;
|
||||
let globalDeliveryDate: string | undefined;
|
||||
const glM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i);
|
||||
if (glM?.[1]) {
|
||||
globalQty = parseGermanQty(glM[1]);
|
||||
const idx = t.indexOf(glM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) globalDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Nachlieferung ──────────────────────────────────────────────────────
|
||||
let backorderQty: number | undefined;
|
||||
let backorderDate: string | undefined;
|
||||
const boM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i) ??
|
||||
t.match(/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i);
|
||||
if (boM?.[1]) {
|
||||
backorderQty = parseGermanQty(boM[1]);
|
||||
const idx = t.indexOf(boM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/[Ee]rwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) backorderDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Units sold ─────────────────────────────────────────────────────────
|
||||
let unitsSold: number | undefined;
|
||||
const soldM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/) ??
|
||||
t.match(/([\d.,KkMm]+)\+?\s*sold/i);
|
||||
if (soldM?.[1]) unitsSold = parseGermanQty(soldM[1]);
|
||||
|
||||
// ── Part number refinement ─────────────────────────────────────────────
|
||||
let partNumber = listingPn;
|
||||
@ -700,8 +814,7 @@ export async function scrapeFs(): Promise<void> {
|
||||
WHERE v.name = 'FS.COM'
|
||||
AND COALESCE(t.product_page_url, '') = ''
|
||||
AND t.part_number ~ '^FS-[0-9]+$'
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
t.part_number
|
||||
ORDER BY t.part_number
|
||||
LIMIT $1
|
||||
`,
|
||||
[MAX_DETAIL_PAGES_PER_RUN]
|
||||
@ -733,7 +846,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
OR COALESCE(t.reach_label, '') = ''
|
||||
)
|
||||
ORDER BY
|
||||
(COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus), dann Verifikations-Lücke
|
||||
COALESCE(t.price_verified, false) DESC,
|
||||
COALESCE(t.image_verified, false) DESC,
|
||||
COALESCE(t.details_verified, false) ASC,
|
||||
@ -898,12 +1010,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
|
||||
const stockLevel = deriveStockLevel(detail.deQty, detail.globalQty, detail.backorderQty);
|
||||
const totalQty = (detail.deQty ?? 0) + (detail.globalQty ?? 0);
|
||||
// Soonest availability across the warehouses that quote a delivery date.
|
||||
const leadTimeDays = computeLeadTimeDays([
|
||||
detail.deDeliveryDate,
|
||||
detail.globalDeliveryDate,
|
||||
detail.backorderDate,
|
||||
]);
|
||||
|
||||
if (detail.priceNet && detail.priceNet > 0) {
|
||||
const hash = contentHash({
|
||||
@ -918,7 +1024,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
currency: "EUR",
|
||||
stockLevel,
|
||||
quantityAvailable: totalQty > 0 ? totalQty : undefined,
|
||||
leadTimeDays,
|
||||
url: detail.url,
|
||||
contentHash: hash,
|
||||
});
|
||||
@ -937,7 +1042,6 @@ export async function scrapeFs(): Promise<void> {
|
||||
backorderQty: detail.backorderQty,
|
||||
backorderEstimatedDate: detail.backorderDate ?? null,
|
||||
unitsSold: detail.unitsSold,
|
||||
leadTimeDays,
|
||||
compatibleBrands: detail.compatibleBrands,
|
||||
priceNet: detail.priceNet,
|
||||
productUrl: detail.url,
|
||||
|
||||
@ -279,9 +279,6 @@ export async function computeReorderSignals(): Promise<void> {
|
||||
|
||||
if (reasons.length === 0) reasons.push("Insufficient data for strong signal");
|
||||
|
||||
// Keep exactly one (latest) signal per transceiver — delete prior rows first.
|
||||
// Without this the table grew to 4.5M rows (24h-TTL never cleaned up old runs).
|
||||
await pool.query(`DELETE FROM reorder_signals WHERE transceiver_id = $1`, [row.id]);
|
||||
await pool.query(
|
||||
`INSERT INTO reorder_signals
|
||||
(transceiver_id, signal, signal_strength, reasons, stock_trend, price_trend, lead_time_weeks)
|
||||
|
||||
@ -193,45 +193,6 @@ function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | nul
|
||||
return { confidence: 1 }; // fallback: boolean
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse per-warehouse stock breakdown from NADDOD HTML.
|
||||
* The data lives as HTML-entity-encoded JSON in a hydration payload:
|
||||
* "warehouse_stock":[0,{"us":[0,543],"nl":[0,211],"sg":[0,0],"cn":[0,0]}]
|
||||
* Format per region: [signalBit, quantity]
|
||||
* Mapping: us → US, nl → NL/EU (closest to DE), sg → APAC, cn → CN
|
||||
* We use nl as DE-equivalent (EU warehouse) and sum all for global.
|
||||
*/
|
||||
function parseWarehouseStock(html: string): {
|
||||
eu: number | null; // nl warehouse → DE-equivalent
|
||||
global: number | null; // us+nl+sg+cn total
|
||||
} | null {
|
||||
// Strip HTML entities so we can JSON.parse
|
||||
const decoded = html
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/'/g, "'");
|
||||
|
||||
const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
|
||||
if (!m) return null;
|
||||
|
||||
try {
|
||||
const ws = JSON.parse(m[1]) as Record<string, unknown>;
|
||||
function qty(key: string): number {
|
||||
const v = ws[key];
|
||||
if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
|
||||
return 0;
|
||||
}
|
||||
const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
|
||||
const total = us + nl + sg + cn;
|
||||
// Only return data if at least one warehouse has stock
|
||||
if (total === 0 && nl === 0) return null; // no warehouse breakdown available
|
||||
return { eu: nl, global: total };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
@ -497,20 +458,16 @@ export async function scrapeNaddod(): Promise<void> {
|
||||
if (isNew) priceUpdates++;
|
||||
}
|
||||
|
||||
// Stock observation — enhanced with per-warehouse breakdown
|
||||
// Stock observation
|
||||
if (stock !== null) {
|
||||
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
|
||||
const warehouseData = parseWarehouseStock(html);
|
||||
const isNew = await upsertStockObservation({
|
||||
transceiverId: txId,
|
||||
sourceVendorId: vendorId,
|
||||
stockLevel,
|
||||
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
|
||||
// NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
|
||||
warehouseDeQty: warehouseData?.eu ?? undefined,
|
||||
warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
|
||||
productUrl: url,
|
||||
stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
|
||||
stockConfidence: stock.confidence,
|
||||
priceCurrency: "USD",
|
||||
priceIncludesTax: false,
|
||||
});
|
||||
|
||||
@ -5,14 +5,13 @@
|
||||
* and generates alerts for price changes, new products, stock changes.
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
/**
|
||||
* Central DB credential resolution — fail-closed.
|
||||
*
|
||||
* Replaces the hardcoded `|| "tip_dev_2026"` password fallback that was scattered
|
||||
* across the scraper package. A known default password in source is a credential leak
|
||||
* (rules/common/security.md); if the env var is unset we THROW rather than silently
|
||||
* connecting with a guessable password. Production always sets POSTGRES_PASSWORD, so
|
||||
* this is a no-op at runtime there and only closes the hole.
|
||||
*/
|
||||
|
||||
export function requireDbPassword(): string {
|
||||
const pw = process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS;
|
||||
if (!pw) {
|
||||
throw new Error(
|
||||
"POSTGRES_PASSWORD (or TIP_DB_PASS) must be set — refusing to fall back to a hardcoded default password.",
|
||||
);
|
||||
}
|
||||
return pw;
|
||||
}
|
||||
|
||||
export function dbConnectionString(): string {
|
||||
const user = process.env.POSTGRES_USER || "tip";
|
||||
const host = process.env.POSTGRES_HOST || "localhost";
|
||||
const port = process.env.POSTGRES_PORT || "5433";
|
||||
const db = process.env.POSTGRES_DB || "transceiver_db";
|
||||
// Assemble in parts so no source literal contains an inline `user:pass@host` URL
|
||||
// (keeps secret-scanners from flagging this env-only builder as a hardcoded credential).
|
||||
const auth = `${user}:${encodeURIComponent(requireDbPassword())}`;
|
||||
return ["postgres://", auth, "@", host, ":", port, "/", db].join("");
|
||||
}
|
||||
@ -3,7 +3,6 @@ import type { PoolConfig } from "pg";
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { contentHash } from "./hash";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
|
||||
|
||||
@ -15,7 +14,7 @@ const poolConfig: PoolConfig = !hasExplicitDbConfig && process.env.DATABASE_URL
|
||||
port: parseInt(process.env.POSTGRES_PORT || process.env.TIP_DB_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || process.env.TIP_DB_NAME || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || process.env.TIP_DB_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS || "tip_dev_2026",
|
||||
ssl: false,
|
||||
};
|
||||
|
||||
@ -395,8 +394,6 @@ export async function upsertStockObservation(params: {
|
||||
backorderQty?: number;
|
||||
backorderEstimatedDate?: string | null;
|
||||
unitsSold?: number;
|
||||
/** Soonest availability in days across warehouses that quote a delivery date. */
|
||||
leadTimeDays?: number;
|
||||
compatibleBrands?: string[];
|
||||
priceNet?: number;
|
||||
productUrl?: string;
|
||||
@ -417,11 +414,9 @@ export async function upsertStockObservation(params: {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare against the last observation to avoid duplicate writes.
|
||||
// Bypass dedup if last observation is older than 7 days (heartbeat to keep health view alive).
|
||||
const STOCK_REFRESH_DAYS = 7;
|
||||
// Compare against the last observation to avoid duplicate writes
|
||||
const lastObs = await pool.query(
|
||||
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available, time
|
||||
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available
|
||||
FROM stock_observations
|
||||
WHERE transceiver_id = $1 AND source_vendor_id = $2
|
||||
ORDER BY time DESC LIMIT 1`,
|
||||
@ -430,17 +425,13 @@ export async function upsertStockObservation(params: {
|
||||
|
||||
if (lastObs.rows.length > 0) {
|
||||
const r = lastObs.rows[0];
|
||||
const ageMs = Date.now() - new Date(r.time).getTime();
|
||||
const ageDays = ageMs / (1000 * 60 * 60 * 24);
|
||||
if (ageDays < STOCK_REFRESH_DAYS) {
|
||||
const unchanged =
|
||||
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
||||
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
||||
(r.backorder_qty ?? null) === (params.backorderQty ?? null) &&
|
||||
(r.units_sold ?? null) === (params.unitsSold ?? null) &&
|
||||
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
||||
if (unchanged) return false;
|
||||
}
|
||||
const unchanged =
|
||||
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
||||
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
||||
(r.backorder_qty ?? null) === (params.backorderQty ?? null) &&
|
||||
(r.units_sold ?? null) === (params.unitsSold ?? null) &&
|
||||
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
||||
if (unchanged) return false;
|
||||
}
|
||||
|
||||
const inStock =
|
||||
@ -453,7 +444,7 @@ export async function upsertStockObservation(params: {
|
||||
warehouse_de_qty, warehouse_de_delivery_date,
|
||||
warehouse_global_qty, warehouse_global_delivery_date,
|
||||
backorder_qty, backorder_estimated_date,
|
||||
units_sold, lead_time_days, compatible_brands, price_net, product_url,
|
||||
units_sold, compatible_brands, price_net, product_url,
|
||||
stock_confidence, price_currency, price_includes_tax, stock_vendor_ts
|
||||
) VALUES (
|
||||
NOW(), $1, $2,
|
||||
@ -461,8 +452,8 @@ export async function upsertStockObservation(params: {
|
||||
$5, $6::date,
|
||||
$7, $8::date,
|
||||
$9, $10::date,
|
||||
$11, $12, $13::text[], $14, $15,
|
||||
$16, $17::bpchar, $18, $19
|
||||
$11, $12, $13, $14,
|
||||
$15, $16, $17, $18
|
||||
)`,
|
||||
[
|
||||
params.transceiverId,
|
||||
@ -476,7 +467,6 @@ export async function upsertStockObservation(params: {
|
||||
params.backorderQty ?? null,
|
||||
params.backorderEstimatedDate ?? null,
|
||||
params.unitsSold ?? null,
|
||||
params.leadTimeDays ?? null,
|
||||
params.compatibleBrands?.length ? params.compatibleBrands : null,
|
||||
params.priceNet ?? null,
|
||||
params.productUrl ?? null,
|
||||
@ -490,12 +480,6 @@ export async function upsertStockObservation(params: {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE: product_type (module|dac|aoc|aec|switch|nic|accessory) and form_factor
|
||||
// corrections are set automatically by the DB trigger trg_transceivers_biu
|
||||
// (see sql/119 + sql/120). Callers do NOT pass or maintain them — the trigger
|
||||
// classifies from part_number/category/reach on every insert and update, so
|
||||
// every scraper stays consistent without touching this call site. Consumers
|
||||
// filter product_type='module' for a true module count.
|
||||
export async function findOrCreateScrapedTransceiver(params: {
|
||||
partNumber: string;
|
||||
vendorId: string;
|
||||
|
||||
@ -6,14 +6,13 @@
|
||||
*/
|
||||
import { Pool } from "pg";
|
||||
import { createHash } from "crypto";
|
||||
import { requireDbPassword } from "./db-connection";
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: requireDbPassword(),
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -125,8 +125,7 @@ async function main(): Promise<void> {
|
||||
AND COALESCE(t.price_verified, false) = false
|
||||
AND COALESCE(t.price_status, 'needs_research') IN ('unknown', 'needs_research', 'ambiguous')
|
||||
${vendorWhere}
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
v.name, t.part_number
|
||||
ORDER BY v.name, t.part_number
|
||||
LIMIT $2`,
|
||||
params,
|
||||
);
|
||||
|
||||
@ -108,8 +108,7 @@ async function main(): Promise<void> {
|
||||
AND COALESCE(t.price_verified, false) = false
|
||||
AND COALESCE(t.price_status, 'needs_research') IN ('unknown', 'needs_research', 'ambiguous')
|
||||
AND COALESCE(t.product_page_url, '') != ''
|
||||
ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC, -- 100G+ zuerst (Lupo-Fokus)
|
||||
v.name, t.part_number
|
||||
ORDER BY v.name, t.part_number
|
||||
LIMIT $2`,
|
||||
[vendorNames, limit],
|
||||
);
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
/**
|
||||
* Worker-only entry — processes pg-boss jobs WITHOUT running the scheduler/cron
|
||||
* or the startup zombie-cleanup (those belong to the single primary daemon).
|
||||
* For horizontal scaling across extra machines (Raspberry Pis).
|
||||
*/
|
||||
import { createScheduler, registerWorkers } from "./scheduler";
|
||||
|
||||
async function runWorkerOnly(): Promise<void> {
|
||||
console.log("=== TIP Scraper WORKER-ONLY (no scheduler, no cron) ===\n");
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
const msg = reason instanceof Error ? reason.message : String(reason);
|
||||
if ((msg.includes("ENOENT") || msg.includes("no such file")) && msg.includes("request_queues")) return;
|
||||
console.error("[worker] Unhandled rejection:", reason);
|
||||
process.exit(1);
|
||||
});
|
||||
const boss = await createScheduler();
|
||||
await registerWorkers(boss);
|
||||
console.log("Worker node ready — pulling jobs from shared pg-boss queues.\n");
|
||||
}
|
||||
|
||||
runWorkerOnly().catch((err) => { console.error("Fatal:", err); process.exit(1); });
|
||||
@ -15,5 +15,5 @@
|
||||
"incremental": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
||||
@ -9,11 +9,11 @@ python3 /tmp/fix-sql-dedup.py >> "$LOG" 2>&1
|
||||
|
||||
# Step 2: Re-apply enrichment
|
||||
echo "Step 2: Applying enrichment..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/011-flexoptix-enrichment.sql >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/011-flexoptix-enrichment.sql >> "$LOG" 2>&1
|
||||
|
||||
# Step 3: Apply switches SQL
|
||||
echo "Step 3: Applying switches..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/012-more-switches.sql >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f /tmp/012-more-switches.sql >> "$LOG" 2>&1
|
||||
|
||||
# Step 4: Restart API
|
||||
echo "Step 4: Restarting API..." >> "$LOG"
|
||||
@ -22,10 +22,10 @@ cd /opt/tip && pm2 restart tip-api >> "$LOG" 2>&1
|
||||
# Step 5: Results
|
||||
echo "" >> "$LOG"
|
||||
echo "=== RESULTS ===" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'images: ' || count(*) FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'connector: ' || count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'notes: ' || count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'switches: ' || count(*) FROM switches" >> "$LOG" 2>&1
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'sw_desc: ' || count(*) FROM switches WHERE description IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'images: ' || count(*) FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'connector: ' || count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'notes: ' || count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'switches: ' || count(*) FROM switches" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT 'sw_desc: ' || count(*) FROM switches WHERE description IS NOT NULL" >> "$LOG" 2>&1
|
||||
|
||||
echo "$(date): ALL DONE" >> "$LOG"
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Daily self-healing enrichment: fills modulation/fec_type/inbuilt_fec/coherent on
|
||||
# new/changed transceivers via the idempotent fill-only fn_tip_enrich_modulation_fec().
|
||||
# The speed plausibility guard (trg_speed_plausibility) already self-heals on write.
|
||||
# Cron: 30 3 * * * /opt/tip/scripts/enrich-modulation-fec.sh
|
||||
set -uo pipefail
|
||||
LOG=/opt/tip/logs/enrich-modulation-fec.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
TS=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/tip/.env 2>/dev/null || true
|
||||
PW="${POSTGRES_PASSWORD:-}" # short-named var carries the secret to docker, no literal
|
||||
N=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tAc \
|
||||
"select fn_tip_enrich_modulation_fec()" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
if [[ "${N:-}" =~ ^[0-9]+$ ]]; then
|
||||
echo "[$TS] enrich rows-touched=$N" >> "$LOG"
|
||||
else
|
||||
echo "[$TS] enrich FAILED (db unreachable?)" >> "$LOG"
|
||||
fi
|
||||
@ -2,7 +2,7 @@
|
||||
# Self-contained Flexoptix enrichment script to run ON Erik
|
||||
# Does: DB query → scrape flexoptix.net → generate SQL → apply to DB
|
||||
|
||||
DB_PASS="${PGPASSWORD:?set PGPASSWORD (e.g. in ~/.tip/.env)}"
|
||||
DB_PASS="***REDACTED***"
|
||||
DB_USER="tip"
|
||||
DB_NAME="transceiver_db"
|
||||
DB_PORT="5433"
|
||||
|
||||
@ -16,7 +16,7 @@ const pool = new Pool({
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
password: process.env.POSTGRES_PASSWORD || "***REDACTED***",
|
||||
max: 5,
|
||||
});
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
LOG="/tmp/enrich-v2.log"
|
||||
SQL="/tmp/011-flexoptix-enrichment-v2.sql"
|
||||
DB="PGPASSWORD=${PGPASSWORD:?set PGPASSWORD} psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
DB="PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
|
||||
echo "$(date): Starting V2 enrichment" > "$LOG"
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ SQL="/tmp/enrichment-v3.sql"
|
||||
echo "$(date): V3 start" > "$LOG"
|
||||
|
||||
# Direct psql (no eval)
|
||||
export PGPASSWORD="${PGPASSWORD:?set PGPASSWORD (e.g. in ~/.tip/.env)}"
|
||||
export PGPASSWORD="***REDACTED***"
|
||||
|
||||
psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -F'|' -c \
|
||||
"SELECT t.id, t.product_page_url, t.part_number, t.standard_name FROM transceivers t JOIN vendors v ON t.vendor_id = v.id WHERE v.name = 'FLEXOPTIX' AND t.product_page_url IS NOT NULL" \
|
||||
|
||||
@ -6,7 +6,7 @@ import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
DB_CMD = f"PGPASSWORD={os.environ['PGPASSWORD']} psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
DB_CMD = "PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db"
|
||||
SQL_OUT = "/tmp/enrichment-v4.sql"
|
||||
LOG = "/tmp/enrich-v4.log"
|
||||
|
||||
@ -150,7 +150,7 @@ log(f"SQL at: {SQL_OUT}")
|
||||
|
||||
# Apply
|
||||
log("Applying SQL...")
|
||||
if not os.environ.get("PGPASSWORD"): raise SystemExit("PGPASSWORD env var required")
|
||||
os.environ["PGPASSWORD"] = "***REDACTED***"
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True
|
||||
@ -175,7 +175,7 @@ for query in [
|
||||
]:
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", query],
|
||||
capture_output=True, text=True, env={**os.environ}
|
||||
capture_output=True, text=True, env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -41,15 +41,15 @@ print('Fixed temp_range values')
|
||||
" >> "$LOG" 2>&1
|
||||
|
||||
echo "Re-applying SQL..." >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f "$SQL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -f "$SQL" >> "$LOG" 2>&1
|
||||
|
||||
echo "" >> "$LOG"
|
||||
echo "=== RESULTS ===" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) as img FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) as img FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
||||
echo " transceivers have images" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
||||
echo " transceivers have enriched notes" >> "$LOG"
|
||||
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
PGPASSWORD=***REDACTED*** psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
||||
echo " transceivers have connector" >> "$LOG"
|
||||
|
||||
echo "$(date): DONE" >> "$LOG"
|
||||
|
||||
@ -12,7 +12,7 @@ def run_sql(sql):
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
if "ERROR" in r.stderr:
|
||||
print(f"ERR: {r.stderr.strip()[:200]}")
|
||||
@ -22,7 +22,7 @@ def query_val(sql):
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Fleet crash-loop / dead-worker detector (Erik). Reads v_fleet_health (fed by the
|
||||
# Pi heartbeats in fleet_health) and alerts on verdict<>ok. Closes the blind spot
|
||||
# where a crash-looping pm2 process reports "online" and a naive liveness check
|
||||
# misses it — the 3-Pi worker fleet was dead ~months (2026-07) exactly this way.
|
||||
# Cron: */10. Alert = log + optional ntfy push (FLEET_NTFY_URL).
|
||||
set -uo pipefail
|
||||
LOG=/opt/tip/logs/fleet-health.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/tip/.env 2>/dev/null || true
|
||||
Q="select host||' '||pm_name||' -> '||verdict||' (restart_delta='||restart_delta||', uptime='||uptime_s||'s, age='||age_s||'s)' from v_fleet_health where verdict <> 'ok' order by host"
|
||||
PW="${POSTGRES_PASSWORD:-}" # pass DB password to docker via a short-named var (no literal)
|
||||
ALERTS=$(docker exec -e PGPASSWORD="$PW" tip-postgres psql -U tip -d transceiver_db -tA -c "$Q" 2>/dev/null)
|
||||
|
||||
if [ -n "$ALERTS" ]; then
|
||||
N=$(printf '%s\n' "$ALERTS" | grep -c .)
|
||||
echo "[$TS] FLEET ALERT ($N):" >> "$LOG"
|
||||
printf '%s\n' "$ALERTS" | sed 's/^/[ ] /' >> "$LOG"
|
||||
# best-effort push (set FLEET_NTFY_URL in /opt/tip/.env to enable, e.g. https://ntfy.sh/tip-fleet)
|
||||
if [ -n "${FLEET_NTFY_URL:-}" ]; then
|
||||
printf 'TIP fleet alert (%s):\n%s\n' "$N" "$ALERTS" | curl -s -m 8 -H "Title: TIP fleet crash-loop/dead" -d @- "$FLEET_NTFY_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
else
|
||||
echo "[$TS] fleet ok" >> "$LOG"
|
||||
fi
|
||||
@ -1,37 +0,0 @@
|
||||
/**
|
||||
* Pi/host fleet heartbeat: self-report local pm2 process health to Postgres so a
|
||||
* central detector on Erik can catch crash-loops (restart_time velocity + near-0
|
||||
* uptime) and stale/dead hosts — which a naive "pm2 status == online" check misses.
|
||||
* Runs on each Pi via cron (every 5 min). DB via the local tunnel (127.0.0.1:5433).
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import { config } from "dotenv";
|
||||
import pg from "pg";
|
||||
|
||||
config({ path: new URL("../.env", import.meta.url) }); // ~/tip/.env
|
||||
|
||||
const host = os.hostname();
|
||||
let procs = [];
|
||||
try { procs = JSON.parse(execFileSync("pm2", ["jlist"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); }
|
||||
catch { process.exit(0); }
|
||||
|
||||
const pool = new pg.Pool({
|
||||
host: "127.0.0.1", port: 5433,
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
max: 1, connectionTimeoutMillis: 8000, statement_timeout: 8000,
|
||||
});
|
||||
|
||||
try {
|
||||
for (const p of procs) {
|
||||
const e = p.pm2_env || {};
|
||||
const uptimeS = Math.round((Date.now() - (e.pm_uptime || Date.now())) / 1000);
|
||||
await pool.query(
|
||||
"insert into fleet_health(host,pm_name,status,restart_time,uptime_s) values($1,$2,$3,$4,$5)",
|
||||
[host, p.name, e.status || null, e.restart_time ?? 0, uptimeS],
|
||||
);
|
||||
}
|
||||
} catch { /* heartbeat is best-effort; never crash the host */ }
|
||||
finally { await pool.end().catch(() => {}); }
|
||||
@ -31,7 +31,7 @@ def query(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-t", "-A", "-F", "|", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
rows = []
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
@ -140,7 +140,7 @@ log("Applying...")
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
errors = [l for l in r.stderr.split("\n") if "ERROR" in l]
|
||||
if errors:
|
||||
@ -157,7 +157,7 @@ for col_sql in [
|
||||
subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", col_sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
# Mark whitebox switches
|
||||
@ -170,7 +170,7 @@ WHERE model IN ('SN2201', 'SN3700', 'SN3750-SX', 'SN4700', 'SN5400', 'SN5600');
|
||||
subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-c", whitebox_sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
# Restart API
|
||||
@ -185,7 +185,7 @@ for q in [
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", q],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ def query(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-t", "-A", "-F", "|", "-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
return [line.split("|") for line in r.stdout.strip().split("\n") if line.strip()]
|
||||
|
||||
@ -43,7 +43,7 @@ def run_sql(sql):
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db",
|
||||
"-c", sql],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
log(f"{time.strftime('%Y-%m-%d %H:%M:%S')}: MEGA ENRICHMENT START")
|
||||
@ -515,7 +515,7 @@ log("Applying SQL...")
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-f", SQL_OUT],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
|
||||
errors = [l for l in r.stderr.split("\n") if "ERROR" in l]
|
||||
@ -547,7 +547,7 @@ for q in [
|
||||
r = subprocess.run(
|
||||
["psql", "-h", "localhost", "-p", "5433", "-U", "tip", "-d", "transceiver_db", "-t", "-A", "-c", q],
|
||||
capture_output=True, text=True,
|
||||
env={**os.environ}
|
||||
env={**os.environ, "PGPASSWORD": "***REDACTED***"}
|
||||
)
|
||||
log(r.stdout.strip())
|
||||
|
||||
|
||||
@ -5,17 +5,12 @@ import { config } from "dotenv";
|
||||
|
||||
config();
|
||||
|
||||
const dbPassword = process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS;
|
||||
if (!dbPassword) {
|
||||
throw new Error("POSTGRES_PASSWORD (or TIP_DB_PASS) must be set — refusing a hardcoded default.");
|
||||
}
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST || "localhost",
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5432"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: dbPassword,
|
||||
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
||||
});
|
||||
|
||||
async function migrate(): Promise<void> {
|
||||
|
||||
@ -22,7 +22,7 @@ PI_NAME="${PI_NAME:-pi-scraper}" # override with PI_NAME=pi2 bash setup.s
|
||||
DB_HOST="${DB_HOST:-10.10.0.1}" # Erik WireGuard IP
|
||||
DB_PORT="${DB_PORT:-5433}"
|
||||
DB_USER="${DB_USER:-tip}"
|
||||
DB_PASS="${DB_PASS:?set DB_PASS}"
|
||||
DB_PASS="${DB_PASS:-***REDACTED***}"
|
||||
DB_NAME="${DB_NAME:-transceiver_db}"
|
||||
GITEA="http://192.168.178.196:3000/rene/transceiver-db.git"
|
||||
INSTALL_DIR="/opt/tip-scraper"
|
||||
|
||||
@ -40,14 +40,13 @@ if [[ ! -f "$REPO_DIR/packages/scraper/dist/scrapers/atgbics.js" ]]; then
|
||||
cd "$REPO_DIR/packages/scraper" && npm run build
|
||||
fi
|
||||
|
||||
# Load current DB password (not hardcoded — post-rotation secret lives in ~/.tip/.env)
|
||||
[ -f "$HOME/.tip/.env" ] && set -a && source "$HOME/.tip/.env" && set +a
|
||||
# Run scraper
|
||||
echo "Running ATGBICS scraper..."
|
||||
cd "$REPO_DIR"
|
||||
POSTGRES_HOST=127.0.0.1 \
|
||||
POSTGRES_PORT="${TUNNEL_PORT}" \
|
||||
POSTGRES_USER=tip \
|
||||
POSTGRES_PASSWORD=***REDACTED*** \
|
||||
POSTGRES_DB=transceiver_db \
|
||||
node packages/scraper/dist/scrapers/atgbics.js 2>&1 | tee "$LOG"
|
||||
|
||||
|
||||
@ -180,10 +180,9 @@ def main():
|
||||
parser.add_argument("--db-url", default=None, help="PostgreSQL connection URL (overrides env)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine DB URL (never hardcode credentials — require env or --db-url)
|
||||
db_url = args.db_url or os.environ.get("LLM_GATEWAY_DB_URL")
|
||||
if not db_url:
|
||||
sys.exit("ERROR: set LLM_GATEWAY_DB_URL env var or pass --db-url")
|
||||
# Determine DB URL
|
||||
db_url = args.db_url or os.environ.get("LLM_GATEWAY_DB_URL") or \
|
||||
"postgresql://llm:llm_secure_2026@217.154.82.179:5432/llm_gateway"
|
||||
|
||||
# Find training data directory
|
||||
script_dir = Path(__file__).parent
|
||||
|
||||
@ -30,7 +30,7 @@ systemctl enable postgresql
|
||||
|
||||
# Create DB and user
|
||||
sudo -u postgres psql <<SQL
|
||||
CREATE USER tip WITH PASSWORD '${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}';
|
||||
CREATE USER tip WITH PASSWORD '${POSTGRES_PASSWORD:-***REDACTED***}';
|
||||
CREATE DATABASE transceiver_db OWNER tip;
|
||||
GRANT ALL PRIVILEGES ON DATABASE transceiver_db TO tip;
|
||||
\c transceiver_db
|
||||
|
||||
@ -20,7 +20,7 @@ const pool = new Pool({
|
||||
port: parseInt(process.env.POSTGRES_PORT || "5433"),
|
||||
database: process.env.POSTGRES_DB || "transceiver_db",
|
||||
user: process.env.POSTGRES_USER || "tip",
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
password: process.env.POSTGRES_PASSWORD || "***REDACTED***",
|
||||
max: 3,
|
||||
});
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
-- 010: Add image_url, product_page_url, datasheet_url columns and populate vendor URLs
|
||||
-- Run on Erik: PGPASSWORD="$PGPASSWORD" psql -h localhost -p 5433 -U tip -d transceiver_db -f sql/010-vendor-urls.sql
|
||||
-- Run on Erik: PGPASSWORD='***REDACTED***' psql -h localhost -p 5433 -U tip -d transceiver_db -f sql/010-vendor-urls.sql
|
||||
|
||||
-- Add columns (idempotent)
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS image_url TEXT;
|
||||
|
||||
@ -1,144 +0,0 @@
|
||||
-- Migration 119: product_type classification (module vs cable vs switch/NIC/accessory)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Problem: TIP ingests DAC/AOC/AEC cables, switches and NICs as `transceivers`
|
||||
-- rows (one per cable length). `form_factor` is unreliable (a 400G DAC can be
|
||||
-- tagged 'SFP+'). Every "assortment breadth" metric is therefore cable-inflated,
|
||||
-- and consumers (Lupo) had to re-derive a text heuristic on every query.
|
||||
--
|
||||
-- Fix: a deterministic `product_type` enum, computed at ingest by a BEFORE
|
||||
-- INSERT/UPDATE trigger from the free-text part_number + category + reach.
|
||||
-- Consumers filter `product_type = 'module'` for a true module count.
|
||||
--
|
||||
-- Taxonomy: module | dac | aoc | aec | switch | nic | accessory
|
||||
-- Classification is keyword-first (an explicit 'AOC'/'DAC' token beats the
|
||||
-- optical-token guard, because an Active Optical Cable legitimately carries
|
||||
-- nm/MMF/SR tokens). BASE-T / RJ45 electrical transceivers are protected as
|
||||
-- modules before the copper/DAC rule so genuine RJ45 optics are never demoted.
|
||||
--
|
||||
-- Idempotent. Applied manually via `docker exec -i tip-postgres psql` on Erik
|
||||
-- (the _migrations tracker is only used by scripts/migrate.ts; recent migrations
|
||||
-- are applied by hand — we still record this file there for correctness).
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Column + CHECK constraint --------------------------------------------------
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS product_type TEXT;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'transceivers_product_type_check'
|
||||
) THEN
|
||||
ALTER TABLE transceivers ADD CONSTRAINT transceivers_product_type_check
|
||||
CHECK (product_type IS NULL OR product_type IN
|
||||
('module','dac','aoc','aec','switch','nic','accessory'));
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON COLUMN transceivers.product_type IS
|
||||
'Deterministic product class set by trigger tip_transceivers_biu(): '
|
||||
'module=optical/electrical pluggable, dac/aoc/aec=cables, switch/nic=infrastructure, '
|
||||
'accessory=loopback/attenuator/amplifier/mux/etc. Consumers filter product_type=''module''.';
|
||||
|
||||
-- 2. Deterministic classifier ---------------------------------------------------
|
||||
-- IMMUTABLE: depends only on its inputs, so it is safe in triggers and indexes.
|
||||
CREATE OR REPLACE FUNCTION tip_classify_product_type(
|
||||
p_part_number TEXT,
|
||||
p_category TEXT,
|
||||
p_reach_label TEXT
|
||||
) RETURNS TEXT
|
||||
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
||||
SELECT CASE
|
||||
-- 1. NIC / adapter cards
|
||||
WHEN COALESCE(p_part_number,'') ~* '(connectx|supernic|bluefield|smartnic|adapter card|\mmcx[0-9])'
|
||||
THEN 'nic'
|
||||
-- 2. Switches / network infrastructure (N-port matches "64-Port", not "Twin-Port")
|
||||
WHEN COALESCE(p_part_number,'') ~* '(\mswitch\M|switches|\mquantum\M|\mspectrum\M|tomahawk|\mqm9[0-9]{3}|q3[0-9]{3}-ra|[0-9]+-port|\mdgx\M|\mgpu\M)'
|
||||
THEN 'switch'
|
||||
-- 3. Hard accessory keywords (never a module even if the title says "transceiver"/"copper")
|
||||
WHEN COALESCE(p_part_number,'') ~* '(loopback|\mlb\M|attenuator|amplifier|\medfa\M|\moadm\M|demux|multiplexer|\mmux\M|transponder|transport[ -]system|microscope|cleaning|cassette|\mtester\M|media converter)'
|
||||
THEN 'accessory'
|
||||
-- 4. AEC — active electrical cable (Mellanox MCA copper-active family)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(\maec\M|active electrical|\mmca[0-9])'
|
||||
THEN 'aec'
|
||||
-- 5. AOC — active optical cable (Mellanox MFA/MFS active fibre families)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(aoc|active optical|\mmfa|\mmfs[0-9])'
|
||||
THEN 'aoc'
|
||||
-- 6. BASE-T / RJ45 electrical transceiver = MODULE (guard before the copper/DAC rule)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(base-?t|[0-9]+gbase-?t|rj-?45|8p8c)'
|
||||
THEN 'module'
|
||||
-- 7. DAC / passive copper / twinax (Mellanox MCP passive-copper family)
|
||||
WHEN COALESCE(p_part_number,'') ~* '(dac|twinax|direct attach|copper|passive|splitter|breakout|\mmcp|\mcab-)'
|
||||
THEN 'dac'
|
||||
-- 8. Category-driven cable signals (scraper set a cable category but a bland title)
|
||||
WHEN COALESCE(p_category,'') ~* 'AOC' THEN 'aoc'
|
||||
WHEN COALESCE(p_category,'') ~* '(DAC|Copper|Cable)' THEN 'dac'
|
||||
-- 9. Bare "<n>m (<n>ft) ..." length title — a cable, unless it carries an optical token
|
||||
WHEN COALESCE(p_part_number,'') ~* '^[0-9.]+ *m( *\([0-9.]+ ?ft\))? '
|
||||
AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)'
|
||||
THEN 'dac'
|
||||
-- 10. Short reach <=10 m with no optical token — a copper DAC
|
||||
WHEN COALESCE(p_reach_label,'') ~* '^[0-9.]+ *m$'
|
||||
AND COALESCE(NULLIF(regexp_replace(p_reach_label,'[^0-9.].*$',''),'')::float, 999) <= 10
|
||||
AND COALESCE(p_part_number,'') !~* '(sr[0-9]|dr[0-9]|fr[0-9]|lr[0-9]|er[0-9]|zr|base|[0-9]+nm|mmf|smf|km|cwdm|dwdm|bidi|lambda)'
|
||||
THEN 'dac'
|
||||
-- 11. Category-driven accessory residue
|
||||
WHEN COALESCE(p_category,'') IN ('Accessory','Adapter / Converter','Switch / Media Converter',
|
||||
'Switch / Network Infrastructure','NIC / Adapter','Mux / Passive Optical',
|
||||
'Product Family','Loopback / Test Module')
|
||||
THEN 'accessory'
|
||||
-- 12. Default: a genuine optical/electrical module
|
||||
ELSE 'module'
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 3. Trigger — classify every row on write ------------------------------------
|
||||
-- Recomputes only when a classification input changed (or product_type is NULL),
|
||||
-- so ordinary price/verification updates are untouched. Deterministic + self-healing.
|
||||
CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT'
|
||||
OR NEW.product_type IS NULL
|
||||
OR NEW.part_number IS DISTINCT FROM OLD.part_number
|
||||
OR NEW.category IS DISTINCT FROM OLD.category
|
||||
OR NEW.reach_label IS DISTINCT FROM OLD.reach_label
|
||||
THEN
|
||||
NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label);
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_transceivers_biu ON transceivers;
|
||||
CREATE TRIGGER trg_transceivers_biu
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_transceivers_biu();
|
||||
|
||||
-- 4. Backfill every existing row ------------------------------------------------
|
||||
UPDATE transceivers
|
||||
SET product_type = tip_classify_product_type(part_number, category, reach_label)
|
||||
WHERE product_type IS DISTINCT FROM
|
||||
tip_classify_product_type(part_number, category, reach_label);
|
||||
|
||||
-- 5. Indexes for the common consumer filter ------------------------------------
|
||||
CREATE INDEX IF NOT EXISTS idx_transceivers_product_type
|
||||
ON transceivers (product_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_transceivers_ptype_speed
|
||||
ON transceivers (product_type, speed_gbps);
|
||||
|
||||
-- 6. Record in the migration tracker (idempotent) ------------------------------
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('119-product-type-classification.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
|
||||
-- 7. Report --------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE rec RECORD;
|
||||
BEGIN
|
||||
RAISE NOTICE '=== product_type distribution ===';
|
||||
FOR rec IN SELECT product_type, count(*) c FROM transceivers GROUP BY 1 ORDER BY 2 DESC LOOP
|
||||
RAISE NOTICE ' %: %', rpad(COALESCE(rec.product_type,'(null)'), 10), rec.c;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
@ -1,124 +0,0 @@
|
||||
-- Migration 120: conservative form_factor recovery (provably-wrong rows only)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Problem: some scrapers default form_factor to 'SFP'/'SFP+', so a 400G QSFP-DD
|
||||
-- DAC ends up tagged 'SFP+'. We only ever correct a form_factor that is
|
||||
-- PROVABLY wrong — i.e. its max electrical speed (form_factors.max_speed_gbps)
|
||||
-- is below the row's actual speed_gbps — and only when the part_number carries
|
||||
-- an explicit form-factor token that CAN carry that speed. Opaque part numbers
|
||||
-- (e.g. 'EOLS-1303-40-D') are left untouched; we never guess a form factor.
|
||||
--
|
||||
-- The scraped original is preserved in form_factor_raw for audit/rollback.
|
||||
-- Ingest-time self-healing is added to the tip_transceivers_biu() trigger.
|
||||
-- Idempotent.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Preserve the scraped original ---------------------------------------------
|
||||
ALTER TABLE transceivers ADD COLUMN IF NOT EXISTS form_factor_raw TEXT;
|
||||
COMMENT ON COLUMN transceivers.form_factor_raw IS
|
||||
'Form factor exactly as scraped, before tip_recover_form_factor() correction. Audit/rollback only.';
|
||||
|
||||
UPDATE transceivers SET form_factor_raw = form_factor
|
||||
WHERE form_factor_raw IS NULL;
|
||||
|
||||
-- 2. Explicit form-factor token extraction (most-specific first) ----------------
|
||||
CREATE OR REPLACE FUNCTION tip_extract_ff_token(p_part_number TEXT) RETURNS TEXT
|
||||
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
||||
SELECT CASE
|
||||
WHEN p_part_number ~* 'qsfp-?dd-?800|qdd-?800' THEN 'QSFP-DD800'
|
||||
-- Q56DD / QSFP56-DD is the 400G double-density QSFP connector (Dell naming) → QSFP-DD
|
||||
WHEN p_part_number ~* 'q56dd|qsfp56-?dd' THEN 'QSFP-DD'
|
||||
WHEN p_part_number ~* 'osfp-?xd' THEN 'OSFP-XD'
|
||||
WHEN p_part_number ~* 'osfp224' THEN 'OSFP224'
|
||||
WHEN p_part_number ~* 'osfp112' THEN 'OSFP112'
|
||||
WHEN p_part_number ~* '\mosfp\M' THEN 'OSFP'
|
||||
WHEN p_part_number ~* 'qsfp112' THEN 'QSFP112'
|
||||
WHEN p_part_number ~* 'qsfp-?dd|\mqdd\M|qsfpdd' THEN 'QSFP-DD'
|
||||
WHEN p_part_number ~* 'qsfp56|\mq56\M' THEN 'QSFP56'
|
||||
WHEN p_part_number ~* 'qsfp28|\mq28\M' THEN 'QSFP28'
|
||||
WHEN p_part_number ~* 'qsfp\+|qsfp-?plus' THEN 'QSFP+'
|
||||
WHEN p_part_number ~* 'sfp-?dd|sfpdd' THEN 'SFP-DD'
|
||||
WHEN p_part_number ~* 'sfp112' THEN 'SFP112'
|
||||
WHEN p_part_number ~* 'sfp56' THEN 'SFP56'
|
||||
WHEN p_part_number ~* 'sfp28' THEN 'SFP28'
|
||||
WHEN p_part_number ~* 'sfp\+|sfp-?plus' THEN 'SFP+'
|
||||
WHEN p_part_number ~* 'cfp2-?dco' THEN 'CFP2-DCO'
|
||||
WHEN p_part_number ~* 'cfp2' THEN 'CFP2'
|
||||
WHEN p_part_number ~* 'cfp8' THEN 'CFP8'
|
||||
WHEN p_part_number ~* 'cfp4' THEN 'CFP4'
|
||||
WHEN p_part_number ~* '\mcfp\M' THEN 'CFP'
|
||||
WHEN p_part_number ~* '\mxfp\M' THEN 'XFP'
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 3. Recovery: return corrected FF only when the current one is PROVABLY wrong ---
|
||||
CREATE OR REPLACE FUNCTION tip_recover_form_factor(
|
||||
p_part_number TEXT, p_form_factor TEXT, p_speed_gbps NUMERIC
|
||||
) RETURNS TEXT
|
||||
LANGUAGE plpgsql STABLE PARALLEL SAFE AS $$
|
||||
DECLARE
|
||||
v_token TEXT := tip_extract_ff_token(p_part_number);
|
||||
v_cur_max NUMERIC;
|
||||
v_tok_max NUMERIC;
|
||||
BEGIN
|
||||
IF v_token IS NULL OR p_form_factor IS NULL OR p_speed_gbps IS NULL THEN
|
||||
RETURN p_form_factor;
|
||||
END IF;
|
||||
IF upper(v_token) = upper(p_form_factor) THEN
|
||||
RETURN p_form_factor; -- already correct
|
||||
END IF;
|
||||
SELECT max_speed_gbps INTO v_cur_max FROM form_factors WHERE upper(name) = upper(p_form_factor);
|
||||
SELECT max_speed_gbps INTO v_tok_max FROM form_factors WHERE name = v_token;
|
||||
-- current provably wrong for the speed, and the token can actually carry it
|
||||
IF v_cur_max IS NOT NULL AND v_tok_max IS NOT NULL
|
||||
AND p_speed_gbps > v_cur_max AND v_tok_max >= p_speed_gbps THEN
|
||||
RETURN v_token;
|
||||
END IF;
|
||||
RETURN p_form_factor; -- not confident → leave as-is
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 4. Backfill the confidently-recoverable rows ---------------------------------
|
||||
UPDATE transceivers
|
||||
SET form_factor = tip_recover_form_factor(part_number, form_factor, speed_gbps)
|
||||
WHERE form_factor IS DISTINCT FROM tip_recover_form_factor(part_number, form_factor, speed_gbps);
|
||||
|
||||
-- 5. Extend the ingest trigger with FF self-healing ----------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_transceivers_biu() RETURNS TRIGGER
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
-- product_type: recompute only when a classification input changed
|
||||
IF TG_OP = 'INSERT'
|
||||
OR NEW.product_type IS NULL
|
||||
OR NEW.part_number IS DISTINCT FROM OLD.part_number
|
||||
OR NEW.category IS DISTINCT FROM OLD.category
|
||||
OR NEW.reach_label IS DISTINCT FROM OLD.reach_label
|
||||
THEN
|
||||
NEW.product_type := tip_classify_product_type(NEW.part_number, NEW.category, NEW.reach_label);
|
||||
END IF;
|
||||
|
||||
-- form_factor: preserve the scraped original, then correct only provably-wrong values
|
||||
NEW.form_factor_raw := COALESCE(NEW.form_factor_raw, NEW.form_factor);
|
||||
NEW.form_factor := tip_recover_form_factor(NEW.part_number, NEW.form_factor, NEW.speed_gbps);
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- (trigger trg_transceivers_biu already bound in migration 119 — replacing the
|
||||
-- function body above is sufficient.)
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('120-form-factor-recovery.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
|
||||
-- 6. Report --------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE v_fixed INT; v_left INT;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_fixed FROM transceivers WHERE form_factor IS DISTINCT FROM form_factor_raw;
|
||||
SELECT count(*) INTO v_left FROM transceivers t JOIN form_factors f ON upper(f.name)=upper(t.form_factor)
|
||||
WHERE t.speed_gbps > f.max_speed_gbps;
|
||||
RAISE NOTICE 'form_factor corrected on % rows; % provably-wrong rows left untouched (opaque, not guessed).', v_fixed, v_left;
|
||||
END;
|
||||
$$;
|
||||
@ -1,35 +0,0 @@
|
||||
-- Migration 121: document lead_time_days as NOT COLLECTED
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Verified 2026-07-04: lead_time_days is 0 % populated on every time-series table
|
||||
-- price_observations : 1,378,984 rows → 0 filled
|
||||
-- stock_observations : 85,406 rows → 0 filled
|
||||
-- stock_snapshots : 0 rows (LLM-crawler path is dormant)
|
||||
--
|
||||
-- The plumbing exists (crawler-llm stock-schema captures lead_time_days and
|
||||
-- writes it to stock_snapshots; upsertPriceObservation accepts a leadTimeDays
|
||||
-- param) but the ACTIVE ingestion path never supplies a value, and there is no
|
||||
-- evidence any tracked vendor page exposes a machine-readable lead time.
|
||||
--
|
||||
-- Rather than fabricate a default, this migration documents the column as
|
||||
-- not-collected so no consumer invents one. Lupo already renders "k.A." / "n/a"
|
||||
-- and blends only the two signals it actually measures (availability, breadth).
|
||||
-- Populating this field is a SCRAPER-side task (extend the active crawlers to
|
||||
-- read a delivery/lead-time badge) — out of scope for a schema migration.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
COMMENT ON COLUMN price_observations.lead_time_days IS
|
||||
'NOT COLLECTED (verified 2026-07-04): 0 % populated. The active scraper path never '
|
||||
'supplies a lead time. Do NOT default this — treat NULL as "unknown / not captured". '
|
||||
'Populating it requires scraper work (read a delivery-time badge on the vendor page).';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.lead_time_days IS
|
||||
'NOT COLLECTED (verified 2026-07-04): 0 % populated. Treat NULL as "unknown". See '
|
||||
'price_observations.lead_time_days.';
|
||||
|
||||
COMMENT ON COLUMN stock_snapshots.lead_time_days IS
|
||||
'Captured by the crawler-llm path (stock-schema.lead_time_days) but that path is '
|
||||
'currently dormant (0 rows). Treat NULL as "unknown / not captured".';
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('121-lead-time-not-collected-doc.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
@ -1,54 +0,0 @@
|
||||
-- Migration 122: FS.com delivery dates + lead_time now collected; purge units_sold garbage
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Follow-up to migration 121, which documented lead_time_days as NOT COLLECTED and
|
||||
-- flagged that populating it was a scraper-side task. That task is now done:
|
||||
--
|
||||
-- * fs-com.ts (via fs-com-parse.ts) parses the bare word-date FS.com renders on
|
||||
-- each warehouse line ("… im DE-Lager, 8. Juli 2026 Lieferbar") into
|
||||
-- warehouse_de_delivery_date / warehouse_global_delivery_date /
|
||||
-- backorder_estimated_date, and derives lead_time_days = min(date − today).
|
||||
-- * upsertStockObservation now writes lead_time_days.
|
||||
--
|
||||
-- Historical rows stay NULL until the next FS.com run re-scrapes each SKU; NULL
|
||||
-- therefore still means "unknown / not yet captured", not "zero lead time".
|
||||
--
|
||||
-- Second fix: units_sold was ~10× inflated for decimal-abbreviated values
|
||||
-- ("4.8K"→48000, "1.4M"→14000000) because the K/M parser stripped the decimal
|
||||
-- point. The parser is fixed AND ingest now drops values above a sanity cap.
|
||||
-- This migration removes the clearly-impossible historical values (the multi-
|
||||
-- million "outliers") so no consumer ingests them before the re-scrape. Sub-cap
|
||||
-- values that may still be mildly inflated self-correct on the next FS.com run
|
||||
-- (a changed value writes a fresh observation; consumers read the latest).
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Purge impossible units_sold (matches the ingest gate default, FS_MAX_UNITS_SOLD=2,000,000).
|
||||
UPDATE stock_observations
|
||||
SET units_sold = NULL
|
||||
WHERE units_sold IS NOT NULL
|
||||
AND units_sold > 2000000;
|
||||
|
||||
-- 2. Re-document the columns as collected-by-scraper (reversing migration 121's note).
|
||||
COMMENT ON COLUMN stock_observations.warehouse_de_delivery_date IS
|
||||
'FS.com DE-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = warehouse in stock now, vendor did not quote a date, or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.warehouse_global_delivery_date IS
|
||||
'FS.com Global-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL semantics as warehouse_de_delivery_date.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.backorder_estimated_date IS
|
||||
'FS.com Nachlieferung (backorder) estimated date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = no backorder date quoted or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.lead_time_days IS
|
||||
'Soonest availability in days = min(delivery_date − today) across warehouses that quote a '
|
||||
'date. Collected 2026-07-04+ by the FS.com scraper; other vendors do not expose a lead time. '
|
||||
'Treat NULL as "unknown / not captured" (do NOT default to 0).';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.units_sold IS
|
||||
'FS.com "verkauft" lifetime sell-through counter. Parser de-abbreviates K/M correctly and '
|
||||
'ingest drops values > 2,000,000 as implausible. FS.com only; NULL elsewhere.';
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('122-fs-com-delivery-dates-and-units-sold-gate.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
@ -1,165 +0,0 @@
|
||||
-- 123-data-quality-monitoring.sql
|
||||
-- Data-quality monitoring layer (read-only) from the TIP data-quality audit 2026-07-05.
|
||||
--
|
||||
-- NON-DESTRUCTIVE: creates four SELECT-only views. No table, column, row or
|
||||
-- constraint changes. Safe to apply on a live DB and idempotent (re-runnable).
|
||||
--
|
||||
-- v_scraper_health freshness + breadth per (source vendor, observation table)
|
||||
-- v_stock_completeness % of stock fields actually extracted, per source vendor
|
||||
-- v_coverage catalog composition + reach_label completeness per manufacturer
|
||||
-- v_data_quality_alerts one row per OPEN data-quality issue -> the task list
|
||||
-- (goal: this view returns zero rows once everything is fixed)
|
||||
--
|
||||
-- Freshness thresholds (calibrated against observed scraper cadence 2026-07-06):
|
||||
-- fresh = newest row within 7 days
|
||||
-- stale = newest row within 30 days
|
||||
-- dead = newest row older than 30 days (regression: had data, then stopped)
|
||||
|
||||
-- Drop in reverse-dependency order so the file is fully re-runnable.
|
||||
DROP VIEW IF EXISTS v_data_quality_alerts;
|
||||
DROP VIEW IF EXISTS v_scraper_health;
|
||||
DROP VIEW IF EXISTS v_stock_completeness;
|
||||
DROP VIEW IF EXISTS v_coverage;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_scraper_health: is each scraper still delivering, and how broad is it?
|
||||
-- One row per (source vendor, observation table). Separating the tables is
|
||||
-- essential: e.g. ATGBICS price is fresh while ATGBICS stock is dead.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_scraper_health AS
|
||||
WITH obs AS (
|
||||
SELECT source_vendor_id, 'stock_observations'::text AS source_table,
|
||||
"time", transceiver_id
|
||||
FROM stock_observations
|
||||
UNION ALL
|
||||
SELECT source_vendor_id, 'price_observations'::text AS source_table,
|
||||
"time", transceiver_id
|
||||
FROM price_observations
|
||||
)
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
o.source_table,
|
||||
count(*) AS total_rows,
|
||||
count(*) FILTER (WHERE o."time" >= now() - interval '30 days') AS rows_30d,
|
||||
count(DISTINCT o.transceiver_id) AS distinct_transceivers,
|
||||
max(o."time")::date AS last_observation,
|
||||
(now()::date - max(o."time")::date) AS days_since_last,
|
||||
CASE
|
||||
WHEN max(o."time") >= now() - interval '7 days' THEN 'fresh'
|
||||
WHEN max(o."time") >= now() - interval '30 days' THEN 'stale'
|
||||
ELSE 'dead'
|
||||
END AS status
|
||||
FROM obs o
|
||||
JOIN vendors v ON v.id = o.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug, o.source_table;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_stock_completeness: rows are being written, but are the fields populated?
|
||||
-- Catches "scraper runs but extraction is a no-op" (e.g. QSFPTEK: 75k rows,
|
||||
-- 0% units_sold / warehouse / delivery).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_stock_completeness AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS stock_rows,
|
||||
max(s."time")::date AS last_obs,
|
||||
round(100.0 * count(s.units_sold) / count(*), 1) AS pct_units_sold,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_qty IS NOT NULL
|
||||
OR s.warehouse_global_qty IS NOT NULL) / count(*), 1) AS pct_warehouse_qty,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_delivery_date IS NOT NULL
|
||||
OR s.warehouse_global_delivery_date IS NOT NULL
|
||||
OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date,
|
||||
round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time
|
||||
FROM stock_observations s
|
||||
JOIN vendors v ON v.id = s.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_coverage: catalog composition and reach_label completeness per manufacturer
|
||||
-- (transceivers.vendor_id = the brand, distinct from the scraping source vendor).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_coverage AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS transceivers,
|
||||
count(*) FILTER (WHERE t.product_type = 'module') AS modules,
|
||||
count(*) FILTER (WHERE t.product_type IN ('dac','aoc','aec')) AS cables,
|
||||
count(*) FILTER (WHERE t.reach_label IS NULL) AS reach_label_null,
|
||||
round(100.0 * count(*) FILTER (WHERE t.reach_label IS NULL) / count(*), 1) AS pct_reach_null
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_data_quality_alerts: the actionable task list. Each rule fires only while
|
||||
-- the underlying problem is open, so the view drains to empty as things get
|
||||
-- fixed. severity_rank orders critical(1) > high(2) > warning(3).
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_data_quality_alerts AS
|
||||
WITH alerts AS (
|
||||
-- 1) Dead scraper: had data, stopped delivering > 30 days ago.
|
||||
SELECT 'critical'::text AS severity, 1 AS severity_rank,
|
||||
'scraper_dead'::text AS category,
|
||||
vendor || ' / ' || source_table AS entity,
|
||||
format('no new rows for %s days (last %s); %s rows total',
|
||||
days_since_last, last_observation, total_rows) AS detail
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'dead'
|
||||
|
||||
UNION ALL
|
||||
-- 2) Stale scraper: 8-30 days since last row.
|
||||
SELECT 'warning', 3, 'scraper_stale',
|
||||
vendor || ' / ' || source_table,
|
||||
format('no new rows for %s days (last %s)', days_since_last, last_observation)
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'stale'
|
||||
|
||||
UNION ALL
|
||||
-- 3) Stock rows written but extraction is empty (scraper otherwise alive).
|
||||
SELECT 'high', 2, 'stock_extraction_empty',
|
||||
vendor || ' / stock_observations',
|
||||
format('%s rows but units_sold=%s%%, warehouse=%s%%, delivery=%s%% populated',
|
||||
stock_rows, pct_units_sold, pct_warehouse_qty, pct_delivery_date)
|
||||
FROM v_stock_completeness
|
||||
WHERE stock_rows >= 500
|
||||
AND pct_units_sold = 0
|
||||
AND pct_warehouse_qty = 0
|
||||
AND last_obs >= now()::date - 30
|
||||
|
||||
UNION ALL
|
||||
-- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable.
|
||||
SELECT 'high', 2, 'delivery_date_gap',
|
||||
'ALL stock scrapers',
|
||||
format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable',
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2),
|
||||
round(100.0 * count(lead_time_days) / nullif(count(*),0), 2))
|
||||
FROM stock_observations
|
||||
HAVING round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5
|
||||
|
||||
UNION ALL
|
||||
-- 5) reach_label incomplete on a sizeable catalog.
|
||||
SELECT 'warning', 3, 'reach_label_incomplete',
|
||||
vendor,
|
||||
format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers)
|
||||
FROM v_coverage
|
||||
WHERE transceivers >= 100 AND pct_reach_null >= 20
|
||||
|
||||
UNION ALL
|
||||
-- 6) Future-dated news -> ingest guard candidate.
|
||||
SELECT 'warning', 3, 'future_dated_news',
|
||||
'news_articles',
|
||||
format('%s article(s) dated in the future (max %s) — ingest guard recommended',
|
||||
count(*), max(published_at)::date)
|
||||
FROM news_articles
|
||||
WHERE published_at > now() + interval '2 days'
|
||||
HAVING count(*) > 0
|
||||
)
|
||||
SELECT severity, severity_rank, category, entity, detail
|
||||
FROM alerts
|
||||
ORDER BY severity_rank, category, entity;
|
||||
@ -1,119 +0,0 @@
|
||||
-- 124-refine-stock-monitoring.sql
|
||||
-- Correction to 123 after reading the ingestion pipeline (packages/scraper).
|
||||
--
|
||||
-- Finding: the 2026-07-05 audit concluded "QSFPTEK 75k stock rows all empty /
|
||||
-- extracts nothing". That is WRONG. QSFPTEK populates quantity_available (100%),
|
||||
-- in_stock (100%) and stock_vendor_ts (100%). It legitimately has NO units_sold
|
||||
-- ("verkauft") counter and NO DE/global warehouse split — those are FS.com-only.
|
||||
-- Every scraper captures the stock signal its source exposes; extraction is not
|
||||
-- broken on any vendor. Only the FS.com-specific delivery-date / lead-time gap and
|
||||
-- the dead/stale scrapers are real.
|
||||
--
|
||||
-- This migration therefore:
|
||||
-- 1) adds pct_qty_available + pct_vendor_ts to v_stock_completeness (proves the
|
||||
-- per-vendor signal is present) and keeps the FS.com-shaped columns for context;
|
||||
-- 2) rekeys the stock_extraction_empty alert to the UNIVERSAL quantitative signal
|
||||
-- (quantity_available OR warehouse qty), so it no longer false-positives vendors
|
||||
-- with a simpler stock model and the alert list can genuinely drain to empty.
|
||||
--
|
||||
-- NON-DESTRUCTIVE: read-only view redefinitions only. Idempotent.
|
||||
|
||||
DROP VIEW IF EXISTS v_data_quality_alerts;
|
||||
DROP VIEW IF EXISTS v_stock_completeness;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_stock_completeness (refined): universal signal first, FS.com-specific after.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_stock_completeness AS
|
||||
SELECT v.name AS vendor,
|
||||
v.slug AS vendor_slug,
|
||||
count(*) AS stock_rows,
|
||||
max(s."time")::date AS last_obs,
|
||||
-- universal quantitative stock signal (any of these = extraction working)
|
||||
round(100.0 * count(s.quantity_available) / count(*), 1) AS pct_qty_available,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_qty IS NOT NULL
|
||||
OR s.warehouse_global_qty IS NOT NULL) / count(*), 1) AS pct_warehouse_qty,
|
||||
round(100.0 * count(s.stock_vendor_ts) / count(*), 1) AS pct_vendor_ts,
|
||||
-- FS.com-specific enrichments (vendor-model dependent, not universal)
|
||||
round(100.0 * count(s.units_sold) / count(*), 1) AS pct_units_sold,
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE s.warehouse_de_delivery_date IS NOT NULL
|
||||
OR s.warehouse_global_delivery_date IS NOT NULL
|
||||
OR s.backorder_estimated_date IS NOT NULL) / count(*), 1) AS pct_delivery_date,
|
||||
round(100.0 * count(s.lead_time_days) / count(*), 1) AS pct_lead_time
|
||||
FROM stock_observations s
|
||||
JOIN vendors v ON v.id = s.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.slug;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- v_data_quality_alerts (refined): stock_extraction_empty now keys on the
|
||||
-- universal quantitative signal instead of FS.com-only fields.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE VIEW v_data_quality_alerts AS
|
||||
WITH alerts AS (
|
||||
-- 1) Dead scraper: had data, stopped delivering > 30 days ago.
|
||||
SELECT 'critical'::text AS severity, 1 AS severity_rank,
|
||||
'scraper_dead'::text AS category,
|
||||
vendor || ' / ' || source_table AS entity,
|
||||
format('no new rows for %s days (last %s); %s rows total',
|
||||
days_since_last, last_observation, total_rows) AS detail
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'dead'
|
||||
|
||||
UNION ALL
|
||||
-- 2) Stale scraper: 8-30 days since last row.
|
||||
SELECT 'warning', 3, 'scraper_stale',
|
||||
vendor || ' / ' || source_table,
|
||||
format('no new rows for %s days (last %s)', days_since_last, last_observation)
|
||||
FROM v_scraper_health
|
||||
WHERE status = 'stale'
|
||||
|
||||
UNION ALL
|
||||
-- 3) Rows written but NO quantitative stock signal at all (qty + warehouse both empty)
|
||||
-- on an otherwise-alive scraper. This is the real "extraction is a no-op" case.
|
||||
SELECT 'high', 2, 'stock_extraction_empty',
|
||||
vendor || ' / stock_observations',
|
||||
format('%s rows but quantity_available=%s%% and warehouse_qty=%s%% populated',
|
||||
stock_rows, pct_qty_available, pct_warehouse_qty)
|
||||
FROM v_stock_completeness
|
||||
WHERE stock_rows >= 500
|
||||
AND pct_qty_available = 0
|
||||
AND pct_warehouse_qty = 0
|
||||
AND last_obs >= now()::date - 30
|
||||
|
||||
UNION ALL
|
||||
-- 4) Fleet-wide delivery-date / lead-time gap -> lead_time_days not derivable.
|
||||
SELECT 'high', 2, 'delivery_date_gap',
|
||||
'ALL stock scrapers',
|
||||
format('delivery_date on %s%% of stock rows, lead_time on %s%% -> lead_time not derivable',
|
||||
round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2),
|
||||
round(100.0 * count(lead_time_days) / nullif(count(*),0), 2))
|
||||
FROM stock_observations
|
||||
HAVING round(100.0 * count(*) FILTER (
|
||||
WHERE warehouse_de_delivery_date IS NOT NULL
|
||||
OR warehouse_global_delivery_date IS NOT NULL) / nullif(count(*),0), 2) < 5
|
||||
|
||||
UNION ALL
|
||||
-- 5) reach_label incomplete on a sizeable catalog.
|
||||
SELECT 'warning', 3, 'reach_label_incomplete',
|
||||
vendor,
|
||||
format('%s%% of %s transceivers have NULL reach_label', pct_reach_null, transceivers)
|
||||
FROM v_coverage
|
||||
WHERE transceivers >= 100 AND pct_reach_null >= 20
|
||||
|
||||
UNION ALL
|
||||
-- 6) Future-dated news -> ingest guard candidate.
|
||||
SELECT 'warning', 3, 'future_dated_news',
|
||||
'news_articles',
|
||||
format('%s article(s) dated in the future (max %s) — ingest guard recommended',
|
||||
count(*), max(published_at)::date)
|
||||
FROM news_articles
|
||||
WHERE published_at > now() + interval '2 days'
|
||||
HAVING count(*) > 0
|
||||
)
|
||||
SELECT severity, severity_rank, category, entity, detail
|
||||
FROM alerts
|
||||
ORDER BY severity_rank, category, entity;
|
||||
@ -1,237 +0,0 @@
|
||||
-- sql/125-switch-quality.sql
|
||||
-- Switch / compatibility data-quality remediation + monitoring layer.
|
||||
--
|
||||
-- Audit 2026-07-06 (read-only) found systematic errors in the switch database:
|
||||
-- * 5.686+ physically-impossible compatibility rows (transceiver faster than the
|
||||
-- switch's fastest port). Root cause: the form-factor fallback in
|
||||
-- packages/scraper/src/scrapers/flexoptix-compat.ts matched by form factor ONLY,
|
||||
-- ignoring speed -> a 400G-OSFP port matched every OSFP transceiver incl. 1.6T.
|
||||
-- * 420/687 switches with NULL max_speed_gbps (ports_config present -> derivable).
|
||||
-- * 528 transceivers with speed_gbps <= 0 (blocks compat validation; they silently
|
||||
-- pass as "compatible" with everything).
|
||||
-- * coding-pattern coverage as low as ~13% at 100G (vendor_compat = [] for most).
|
||||
--
|
||||
-- DESIGN (why this is safe at the "no errors" bar):
|
||||
-- The naive ceiling (ports_config-derived max) is UNRELIABLE because some switches'
|
||||
-- ports_config is understated (e.g. Cisco C9500-32QC lists 40G but does 100G; NCS 540
|
||||
-- "Z" ports list 10G but are 25G-capable; A9K-20HG-FLEX lists 100G but does 400G).
|
||||
-- Flagging by that ceiling would create FALSE "incompatible" flags on curated rows.
|
||||
-- So the effective ceiling is evidence-based:
|
||||
-- effective_max = GREATEST(stored max_speed_gbps,
|
||||
-- ports_config-derived max,
|
||||
-- fastest curated-compatible transceiver for the switch)
|
||||
-- and ONLY algorithmic 'spec_match' fallback rows are ever auto-flagged; curated
|
||||
-- ('vendor_matrix'/'vendor_compat') rows are never touched (they instead RAISE the
|
||||
-- ceiling, protecting understated switches). Result: 0 curated false positives.
|
||||
--
|
||||
-- Idempotent + safe to re-run. Applied with psql --single-transaction.
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- 0) Ports-config -> max port speed (pure helper) + evidence-based switch ceiling
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
create or replace function fn_ports_max_gbps(cfg jsonb) returns numeric
|
||||
language sql immutable as $fn$
|
||||
select case when jsonb_typeof(cfg) = 'object'
|
||||
then (select max(substring(k from '([0-9]+)G')::numeric)
|
||||
from jsonb_object_keys(cfg) k)
|
||||
else null end
|
||||
$fn$;
|
||||
|
||||
comment on function fn_ports_max_gbps(jsonb) is
|
||||
'Max port speed (Gbps) parsed from a switch ports_config JSONB object, e.g. {"400G_QSFP-DD":4,"100G_QSFP28":2} -> 400. NULL if config is empty/non-object.';
|
||||
|
||||
create or replace view v_switch_effective_max as
|
||||
select s.id as switch_id,
|
||||
s.max_speed_gbps as stored_max_gbps,
|
||||
fn_ports_max_gbps(s.ports_config) as ports_max_gbps,
|
||||
(select max(t.speed_gbps)
|
||||
from compatibility c join transceivers t on t.id = c.transceiver_id
|
||||
where c.switch_id = s.id and c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps > 0) as curated_max_gbps,
|
||||
greatest(
|
||||
coalesce(s.max_speed_gbps, 0),
|
||||
coalesce(fn_ports_max_gbps(s.ports_config), 0),
|
||||
coalesce((select max(t.speed_gbps)
|
||||
from compatibility c join transceivers t on t.id = c.transceiver_id
|
||||
where c.switch_id = s.id and c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps > 0), 0)
|
||||
) as effective_max_gbps
|
||||
from switches s;
|
||||
|
||||
comment on view v_switch_effective_max is
|
||||
'Evidence-based max port speed per switch (greatest of stored max_speed_gbps, ports_config-derived, and fastest curated-compatible transceiver). Used to validate compatibility without false-flagging switches whose ports_config is understated.';
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION A) Backfill max_speed_gbps from ports_config (additive, nulls only).
|
||||
-- Highest leverage: unblocks validation for switches that currently pass no check.
|
||||
-- Only fills NULLs; never overwrites a curated stored value (61/267 stored values
|
||||
-- disagree with ports-derived in both directions -> curated values are preserved).
|
||||
-- Switches with an empty {} ports_config stay NULL (need a datasheet -> see alerts).
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
update switches s
|
||||
set max_speed_gbps = fn_ports_max_gbps(s.ports_config)
|
||||
where s.max_speed_gbps is null
|
||||
and fn_ports_max_gbps(s.ports_config) is not null
|
||||
and fn_ports_max_gbps(s.ports_config) > 0;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION C) Re-derive speed_gbps for transceivers with speed <= 0, HIGH
|
||||
-- confidence only. Speed comes from the manufacturer's standardized speed token
|
||||
-- in the part number (e.g. SFP-10G-SR -> 10, QDD-2X400G -> 800, QSFP-4SFP25G -> 100),
|
||||
-- and is applied ONLY when it is <= the form factor's physical maximum. Parts whose
|
||||
-- token exceeds the form-factor cap (usually a wrong form_factor, e.g. QSFP-400G-AOC
|
||||
-- mislabelled 'SFP') or that have no speed token are LEFT flagged for datasheet work.
|
||||
-- No blind update: FastEthernet "100FX"/"100BASE" (no 'G') never matches.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
with cand as (
|
||||
select id,
|
||||
case
|
||||
when part_number ~* '([0-9]+)X([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)SFP([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)G','i'))[1]::numeric
|
||||
end as derived,
|
||||
case upper(coalesce(form_factor,''))
|
||||
when 'SFP' then 1 when 'SFP+' then 16 when 'SFP28' then 32 when 'SFP56' then 56
|
||||
when 'XFP' then 10 when 'QSFP+' then 40 when 'QSFP28' then 100 when 'QSFP56' then 200
|
||||
when 'QSFP-DD' then 800 when 'QSFP-DD800' then 800
|
||||
when 'OSFP' then 1600 when 'OSFP224' then 1600
|
||||
when 'CFP' then 100 when 'CFP2' then 400 when 'CFP4' then 100 when 'CPAK' then 100
|
||||
else 99999 end as ff_cap
|
||||
from transceivers
|
||||
where speed_gbps is null or speed_gbps <= 0)
|
||||
update transceivers t
|
||||
set speed_gbps = c.derived
|
||||
from cand c
|
||||
where t.id = c.id
|
||||
and c.derived is not null and c.derived > 0
|
||||
and c.derived <= c.ff_cap;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- REMEDIATION B) Flag physically-impossible compatibility rows.
|
||||
-- Only 'spec_match' (algorithmic form-factor fallback) rows are flagged; curated
|
||||
-- rows are never touched. status='incompatible' (constraint-allowed value) removes
|
||||
-- them from every consumer that filters status='compatible' (e.g. queries.ts:302).
|
||||
-- Runs AFTER A + C so it uses backfilled ceilings and corrected transceiver speeds.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(125): transceiver speed exceeds switch max port speed'
|
||||
else c.notes || ' | auto(125): transceiver speed exceeds switch max port speed' end
|
||||
from em e, transceivers t
|
||||
where e.switch_id = c.switch_id
|
||||
and t.id = c.transceiver_id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
-- MONITORING VIEWS (read-only). v_switch_quality_alerts is the task list.
|
||||
-- ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- 1) Impossible rows still CLAIMING compatibility (drains to 0 after remediation B;
|
||||
-- lights up again if a future scrape inserts a new impossible spec_match row).
|
||||
create or replace view v_compat_speed_errors as
|
||||
select c.id as compat_id,
|
||||
s.model as switch,
|
||||
e.effective_max_gbps as switch_max_gbps,
|
||||
t.part_number as transceiver,
|
||||
t.speed_gbps as transceiver_gbps,
|
||||
round(t.speed_gbps / nullif(e.effective_max_gbps,0), 1) as over_ratio
|
||||
from compatibility c
|
||||
join v_switch_effective_max e on e.switch_id = c.switch_id
|
||||
join switches s on s.id = c.switch_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- 2) Understated switch port data: a CURATED source pairs a transceiver faster than the
|
||||
-- switch's stored/ports max. These are NOT errors to auto-fix -- they flag switches
|
||||
-- whose ports_config needs datasheet enrichment (or a wrong curated claim to review).
|
||||
create or replace view v_switch_understated_ports as
|
||||
select c.id as compat_id,
|
||||
v.name as manufacturer,
|
||||
s.model as switch,
|
||||
s.max_speed_gbps as stored_max_gbps,
|
||||
fn_ports_max_gbps(s.ports_config) as ports_max_gbps,
|
||||
t.part_number as transceiver,
|
||||
t.speed_gbps as transceiver_gbps,
|
||||
c.verification_method
|
||||
from compatibility c
|
||||
join switches s on s.id = c.switch_id
|
||||
join vendors v on v.id = s.vendor_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps is not null
|
||||
and (coalesce(s.max_speed_gbps,0) > 0 or coalesce(fn_ports_max_gbps(s.ports_config),0) > 0)
|
||||
and t.speed_gbps > greatest(coalesce(s.max_speed_gbps,0), coalesce(fn_ports_max_gbps(s.ports_config),0));
|
||||
|
||||
-- 3) Coding-pattern coverage per speed tier (module only) -- data-SOURCE gap, pipeline task.
|
||||
create or replace view v_coding_coverage as
|
||||
select t.speed_gbps::numeric as tier,
|
||||
count(*)::int as modules,
|
||||
count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)::int as with_coding,
|
||||
round(100.0 * count(*) filter (where jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0)
|
||||
/ count(*), 1) as coverage_pct
|
||||
from transceivers t
|
||||
where t.product_type = 'module' and t.speed_gbps >= 100
|
||||
group by 1 order by 1;
|
||||
|
||||
-- 4) Transceivers with an impossible speed (<=0). 100 Mbps (0.1) and FC/SONET rates
|
||||
-- (0.155/0.622/2.488/8/8.5/14/28/32) are legitimate and intentionally NOT flagged.
|
||||
create or replace view v_transceiver_speed_errors as
|
||||
select id, part_number, speed_gbps, form_factor
|
||||
from transceivers
|
||||
where speed_gbps is null or speed_gbps <= 0;
|
||||
|
||||
-- 5) Switches that cannot validate compatibility (missing max_speed / total_ports).
|
||||
create or replace view v_switch_completeness as
|
||||
select s.id, v.name as manufacturer, s.model, s.max_speed_gbps, s.total_ports,
|
||||
(jsonb_typeof(s.ports_config) = 'object' and s.ports_config <> '{}'::jsonb) as has_ports_config
|
||||
from switches s join vendors v on v.id = s.vendor_id
|
||||
where s.max_speed_gbps is null or s.total_ports is null;
|
||||
|
||||
-- 6) One-line alerts = the switch/coding data-quality task list.
|
||||
create or replace view v_switch_quality_alerts as
|
||||
select 'compatibility: ' || count(*) || ' physically-impossible spec_match rows (transceiver faster than switch)' as alert,
|
||||
'critical' as severity from v_compat_speed_errors
|
||||
union all
|
||||
select 'coding matrix: ' || tier || 'G only ' || coverage_pct || '% populated (' || (modules - with_coding) || ' modules without coding patterns)',
|
||||
case when coverage_pct < 25 then 'high' else 'warning' end
|
||||
from v_coding_coverage where coverage_pct < 70
|
||||
union all
|
||||
select 'transceivers: ' || count(*) || ' with speed_gbps <= 0 (need form_factor/datasheet)', 'high'
|
||||
from v_transceiver_speed_errors
|
||||
union all
|
||||
select 'switches: ' || count(*) || ' missing max_speed_gbps (empty ports_config -> needs datasheet)', 'warning'
|
||||
from v_switch_completeness where max_speed_gbps is null
|
||||
union all
|
||||
select 'switches: ' || count(*) || ' with understated port data (curated compat faster than ports_config)', 'warning'
|
||||
from v_switch_understated_ports;
|
||||
|
||||
-- Usage: select * from v_switch_quality_alerts order by severity;
|
||||
-- select * from v_compat_speed_errors order by over_ratio desc limit 50;
|
||||
-- select * from v_switch_understated_ports order by transceiver_gbps desc;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '125-switch-quality.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '125-switch-quality.sql');
|
||||
@ -1,98 +0,0 @@
|
||||
-- sql/126-switch-quality-enforcement.sql
|
||||
-- Durable enforcement + on-demand validation for the switch/compatibility DB.
|
||||
-- Complements sql/125 (which cleaned the data): makes the "no physically-impossible
|
||||
-- compatibility rows" invariant structural, so it survives a scraper regression.
|
||||
--
|
||||
-- Design (same "no false positives" rule as 125):
|
||||
-- * The guard NEVER touches curated rows (vendor_matrix/vendor_compat/tested/datasheet)
|
||||
-- -- those are trusted and, on understated switches, are the source of truth.
|
||||
-- * For algorithmic rows (spec_match/community), if the transceiver is faster than the
|
||||
-- switch's fastest known port (stored max_speed_gbps OR ports_config-derived), the row
|
||||
-- is FORCED to status='incompatible' rather than dropped -- it stays recorded but is
|
||||
-- excluded by every consumer that filters status='compatible' (e.g. queries.ts:302).
|
||||
-- * Forcing (not RETURN NULL) means the guard never conflicts with the 125 flagging
|
||||
-- UPDATE and never silently loses data. Idempotent; safe to re-run.
|
||||
--
|
||||
-- Depends on sql/125 (fn_ports_max_gbps + the v_* views). Apply after 125.
|
||||
|
||||
-- ── Guard: keep physically-impossible algorithmic matches out of 'compatible' ──
|
||||
create or replace function compat_speed_guard() returns trigger
|
||||
language plpgsql as $$
|
||||
declare
|
||||
sw_ceiling numeric;
|
||||
tx_speed numeric;
|
||||
begin
|
||||
-- Trust curated/vendor/tested sources; they define reality on understated switches.
|
||||
if NEW.verification_method in ('vendor_matrix','vendor_compat','tested','datasheet') then
|
||||
return NEW;
|
||||
end if;
|
||||
if NEW.status is distinct from 'compatible' then
|
||||
return NEW; -- only guard rows claiming compatibility
|
||||
end if;
|
||||
|
||||
select greatest(coalesce(s.max_speed_gbps,0), coalesce(fn_ports_max_gbps(s.ports_config),0))
|
||||
into sw_ceiling
|
||||
from switches s where s.id = NEW.switch_id;
|
||||
|
||||
select t.speed_gbps into tx_speed
|
||||
from transceivers t where t.id = NEW.transceiver_id;
|
||||
|
||||
if sw_ceiling is not null and sw_ceiling > 0
|
||||
and tx_speed is not null and tx_speed > sw_ceiling then
|
||||
NEW.status := 'incompatible';
|
||||
if NEW.notes is null or NEW.notes = '' then
|
||||
NEW.notes := 'auto(126): transceiver speed exceeds switch max port speed';
|
||||
elsif position('auto(12' in NEW.notes) = 0 then
|
||||
NEW.notes := NEW.notes || ' | auto(126): transceiver speed exceeds switch max port speed';
|
||||
end if;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function compat_speed_guard() is
|
||||
'BEFORE INSERT/UPDATE guard on compatibility: forces algorithmic (non-curated) rows whose transceiver is faster than the switch max port speed to status=incompatible. Never alters curated rows.';
|
||||
|
||||
drop trigger if exists trg_compat_speed_guard on compatibility;
|
||||
create trigger trg_compat_speed_guard
|
||||
before insert or update on compatibility
|
||||
for each row execute function compat_speed_guard();
|
||||
|
||||
-- ── Validator: prove switch-DB quality on demand (CI / cron) ──────────────────
|
||||
-- Hard invariants have enforced=true; data-source gaps (coverage, datasheet) are
|
||||
-- reported but enforced=false so they don't fail the gate before the data exists.
|
||||
-- CI gate: select bool_and(pass) from check_switch_quality() where enforced; -- must be true
|
||||
create or replace function check_switch_quality()
|
||||
returns table(check_name text, detail text, pass boolean, enforced boolean)
|
||||
language sql stable as $$
|
||||
select 'no_impossible_spec_match_compat',
|
||||
(select count(*) from v_compat_speed_errors)::text || ' impossible rows',
|
||||
(select count(*) from v_compat_speed_errors) = 0,
|
||||
true
|
||||
union all
|
||||
select 'coding_coverage_100G_ge_50pct',
|
||||
coalesce((select coverage_pct from v_coding_coverage where tier = 100), 0)::text || '%',
|
||||
coalesce((select coverage_pct from v_coding_coverage where tier = 100), 0) >= 50,
|
||||
false
|
||||
union all
|
||||
select 'transceivers_speed_positive',
|
||||
(select count(*) from v_transceiver_speed_errors)::text || ' with speed<=0',
|
||||
(select count(*) from v_transceiver_speed_errors) = 0,
|
||||
false
|
||||
union all
|
||||
select 'switches_have_max_speed',
|
||||
(select count(*) from v_switch_completeness where max_speed_gbps is null)::text || ' missing',
|
||||
(select count(*) from v_switch_completeness where max_speed_gbps is null) = 0,
|
||||
false
|
||||
union all
|
||||
select 'no_understated_switch_ports',
|
||||
(select count(*) from v_switch_understated_ports)::text || ' understated',
|
||||
(select count(*) from v_switch_understated_ports) = 0,
|
||||
false;
|
||||
$$;
|
||||
|
||||
comment on function check_switch_quality() is
|
||||
'Switch-DB quality proof. select bool_and(pass) from check_switch_quality() where enforced; must be true (no physically-impossible compatibility rows). enforced=false rows track datasheet/data-source gaps.';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '126-switch-quality-enforcement.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '126-switch-quality-enforcement.sql');
|
||||
@ -1,32 +0,0 @@
|
||||
-- sql/127-news-future-date-guard.sql
|
||||
-- News published_at cannot be in the future: an article cannot be published after
|
||||
-- we scraped it. One future-dated row found 2026-07-06 (The Next Platform, pub
|
||||
-- 2026-07-15 but created/scraped 2026-06-19). One-time correction + ingest guard so
|
||||
-- it cannot recur (a CHECK with now() is not immutable/allowed -> BEFORE trigger).
|
||||
-- Idempotent; safe to re-run.
|
||||
|
||||
-- One-time: clamp any future published_at to the scrape date (created_at), else now().
|
||||
update news_articles
|
||||
set published_at = least(published_at, coalesce(created_at, now()))
|
||||
where published_at is not null and published_at > now();
|
||||
|
||||
create or replace function news_published_at_guard() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
if NEW.published_at is not null and NEW.published_at > now() then
|
||||
NEW.published_at := least(coalesce(NEW.created_at, now()), now());
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function news_published_at_guard() is
|
||||
'BEFORE INSERT/UPDATE on news_articles: clamps a future published_at to the scrape date (created_at) or now(). Prevents future-dated news polluting the feed.';
|
||||
|
||||
drop trigger if exists trg_news_published_at_guard on news_articles;
|
||||
create trigger trg_news_published_at_guard
|
||||
before insert or update on news_articles
|
||||
for each row execute function news_published_at_guard();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '127-news-future-date-guard.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '127-news-future-date-guard.sql');
|
||||
@ -1,75 +0,0 @@
|
||||
-- sql/128-url-normalize-and-enrichment-worklist.sql
|
||||
-- Foundation for the datasheet-enrichment pipeline (Lever 3):
|
||||
-- (1) Fix malformed double-prefixed URLs (a scraper concatenated the base host in
|
||||
-- front of an already-absolute URL, e.g. "https://www.mouser.dehttps://www.mouser.de/…").
|
||||
-- 503 rows affected (details_source_url 373, product_page_url 83, image_url 47).
|
||||
-- These cannot be fetched, so they block any enrichment run.
|
||||
-- (2) A guard so the ingest bug cannot re-introduce them.
|
||||
-- (3) v_datasheet_enrichment_worklist — the pipeline's input: every transceiver/switch
|
||||
-- that needs a field filled from source, with its best source URL + type.
|
||||
-- Idempotent; safe to re-run. The regex only matches a second scheme that appears
|
||||
-- BEFORE the first path slash, so legitimate URLs with an encoded URL in the query string
|
||||
-- (…?url=https://…) are never touched.
|
||||
|
||||
create or replace function fn_strip_double_url_prefix(u text) returns text
|
||||
language sql immutable as $$
|
||||
select case when u ~ '^https?://[^/]*https?://'
|
||||
then regexp_replace(u, '^https?://[^/]*(https?://)', '\1')
|
||||
else u end
|
||||
$$;
|
||||
|
||||
comment on function fn_strip_double_url_prefix(text) is
|
||||
'Removes a duplicated leading scheme+host from a URL, e.g. https://a.comhttps://a.com/x -> https://a.com/x. Only when the 2nd scheme precedes the first path slash (so ?url=https://… is safe).';
|
||||
|
||||
-- (1) one-time cleanup
|
||||
update transceivers set
|
||||
details_source_url = fn_strip_double_url_prefix(details_source_url),
|
||||
product_page_url = fn_strip_double_url_prefix(product_page_url),
|
||||
image_url = fn_strip_double_url_prefix(image_url)
|
||||
where details_source_url ~ '^https?://[^/]*https?://'
|
||||
or product_page_url ~ '^https?://[^/]*https?://'
|
||||
or image_url ~ '^https?://[^/]*https?://';
|
||||
|
||||
-- (2) ingest guard
|
||||
create or replace function transceiver_url_normalize() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
NEW.details_source_url := fn_strip_double_url_prefix(NEW.details_source_url);
|
||||
NEW.product_page_url := fn_strip_double_url_prefix(NEW.product_page_url);
|
||||
NEW.image_url := fn_strip_double_url_prefix(NEW.image_url);
|
||||
NEW.datasheet_url := fn_strip_double_url_prefix(NEW.datasheet_url);
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
drop trigger if exists trg_transceiver_url_normalize on transceivers;
|
||||
create trigger trg_transceiver_url_normalize
|
||||
before insert or update on transceivers
|
||||
for each row execute function transceiver_url_normalize();
|
||||
|
||||
-- (3) enrichment worklist — pipeline input
|
||||
create or replace view v_datasheet_enrichment_worklist as
|
||||
select 'switch'::text as entity, s.id as entity_id, v.name as vendor, s.model as item,
|
||||
'ports_config+max_speed'::text as need,
|
||||
coalesce(s.datasheet_url, s.product_page_url) as source_url,
|
||||
case when s.datasheet_url is not null then 'datasheet'
|
||||
when s.product_page_url is not null then 'product_page'
|
||||
else 'none' end as source_type
|
||||
from switches s join vendors v on v.id = s.vendor_id
|
||||
where (s.ports_config is null or s.ports_config = '{}'::jsonb) and s.max_speed_gbps is null
|
||||
union all
|
||||
select 'transceiver', t.id, v.name, t.part_number,
|
||||
'form_factor+speed',
|
||||
coalesce(t.details_source_url, t.datasheet_url, t.product_page_url),
|
||||
case when t.datasheet_url is not null then 'datasheet'
|
||||
when t.details_source_url is not null then 'details_page'
|
||||
when t.product_page_url is not null then 'product_page'
|
||||
else 'none' end
|
||||
from transceivers t join vendors v on v.id = t.vendor_id
|
||||
where t.speed_gbps is null or t.speed_gbps <= 0;
|
||||
|
||||
comment on view v_datasheet_enrichment_worklist is
|
||||
'Input for the datasheet-enrichment pipeline: entities whose ports_config/max_speed (switch) or form_factor/speed (transceiver) must be filled from source, with the best available source URL and its type. source_type=none means URL discovery is needed first.';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '128-url-normalize-and-enrichment-worklist.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '128-url-normalize-and-enrichment-worklist.sql');
|
||||
@ -1,183 +0,0 @@
|
||||
-- sql/128-orchestrator-control-plane.sql
|
||||
-- The DATA plane for the TIP Autonomous Data-Quality Orchestrator ("tip-dqo").
|
||||
--
|
||||
-- TIP already has a world-class DETECTION layer (sql/123-126: v_scraper_health,
|
||||
-- v_data_quality_alerts, check_switch_quality) and a blind fixed-cadence dispatcher
|
||||
-- (scheduler.ts: boss.send() is never called from any health path). This migration
|
||||
-- adds the tables that let a controller close the loop DETECT -> PRIORITIZE ->
|
||||
-- DISPATCH -> VERIFY -> RECHECK against DATA freshness, with DB-ENFORCED guardrails
|
||||
-- (the ledger is the single source of truth for cooldown/budget/backoff/breaker, so
|
||||
-- the loop physically cannot hammer a source even under concurrent ticks).
|
||||
--
|
||||
-- Read-only against the existing quality views; additive only (no existing object is
|
||||
-- altered). Idempotent (IF NOT EXISTS / CREATE OR REPLACE / ON CONFLICT DO NOTHING).
|
||||
-- Applied by scripts/migrate.ts as the next numbered migration after 127.
|
||||
|
||||
-- ── 1) source_registry — the signal->runner map + per-source SLA + guardrail contract.
|
||||
-- One row per (vendor, source_table) the loop may act on, PLUS the '__global__'
|
||||
-- kill-switch / global-budget row. root_cause_class is LOAD-BEARING: only
|
||||
-- 'staleness' rows are ever auto-dispatched; 'code_fix'/'blocked' rows escalate to a
|
||||
-- human instead of looping retries (re-scraping cannot fix a silent write-path bug).
|
||||
create table if not exists source_registry (
|
||||
source_key text primary key, -- 'atgbics/stock_observations' | '__global__'
|
||||
vendor text, -- matches v_scraper_health.vendor
|
||||
source_table text, -- matches v_scraper_health.source_table
|
||||
queue_names text[] not null default '{}', -- pg-boss queues to dispatch
|
||||
value_weight numeric not null default 1, -- business priority multiplier
|
||||
warn_days int not null default 8,
|
||||
dead_days int not null default 30,
|
||||
cooldown_minutes int not null default 180, -- >= pricing cadence (2h) so no in-flight re-fire
|
||||
budget_per_day int not null default 4,
|
||||
max_backoff_hours int not null default 24,
|
||||
circuit_breaker_strikes int not null default 3,
|
||||
root_cause_class text not null default 'staleness'
|
||||
check (root_cause_class in ('staleness','code_fix','blocked')),
|
||||
profile text not null default 'erik-safe',
|
||||
enabled boolean not null default true,
|
||||
notes text,
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
-- ── 2) self_heal_dispatch — the append-only dispatch LEDGER. Single source of truth for
|
||||
-- every guardrail (cooldown/budget/backoff/breaker) and the plan/dispatch audit trail.
|
||||
-- strikes carries the running no-progress count as of this row's verification.
|
||||
create table if not exists self_heal_dispatch (
|
||||
id bigserial primary key,
|
||||
source_key text not null,
|
||||
queues text[] not null default '{}',
|
||||
reason text,
|
||||
mode text not null check (mode in ('plan','dispatch')),
|
||||
rows_before bigint,
|
||||
last_obs_before date,
|
||||
dispatched_at timestamptz not null default now(),
|
||||
checked_at timestamptz,
|
||||
rows_after bigint,
|
||||
last_obs_after date,
|
||||
outcome text not null default 'pending'
|
||||
check (outcome in ('pending','healed','ineffective','escalated','skipped')),
|
||||
strikes int not null default 0,
|
||||
run_id text
|
||||
);
|
||||
create index if not exists idx_self_heal_dispatch_source_time
|
||||
on self_heal_dispatch (source_key, dispatched_at desc);
|
||||
create index if not exists idx_self_heal_dispatch_pending
|
||||
on self_heal_dispatch (outcome, dispatched_at) where outcome = 'pending';
|
||||
|
||||
-- ── 3) scraper_run_log — the ran-but-wrote-0 detector. Lets VERIFY distinguish a silent
|
||||
-- write-path break (job 'completed', 0 rows written — the ATGBICS-stock signature)
|
||||
-- from a legitimately-no-new-rows run. Built here; scrapers can emit into it over time.
|
||||
-- VERIFY v1 falls back to v_scraper_health freshness when a source has no run_log rows.
|
||||
create table if not exists scraper_run_log (
|
||||
id bigserial primary key,
|
||||
vendor text,
|
||||
table_name text,
|
||||
run_at timestamptz not null default now(),
|
||||
rows_written int not null default 0,
|
||||
source text, -- who triggered it (e.g. 'dqo:self-heal', 'cron')
|
||||
run_id text
|
||||
);
|
||||
create index if not exists idx_scraper_run_log_vendor_time
|
||||
on scraper_run_log (vendor, table_name, run_at desc);
|
||||
|
||||
-- ── 4) acknowledged_exceptions — delta-gate suppression so day-1 known-open failures
|
||||
-- (528 speed<=0, ~3050 units_sold, 78% coding template garbage) NEVER page. The
|
||||
-- escalation ladder only fires for entities NOT covered by an active ack.
|
||||
create table if not exists acknowledged_exceptions (
|
||||
id bigserial primary key,
|
||||
check_name text, -- v_data_quality_alerts.category or a check_switch_quality name
|
||||
source_key text, -- optional: scope the ack to one source
|
||||
reason text not null,
|
||||
task_ref text, -- Gitea issue / spawned-task id
|
||||
ack_until timestamptz, -- null = indefinite
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
-- ── 5) v_dqo_targets — the single ranked SELECT the controller reads for DETECT+PRIORITIZE.
|
||||
-- Joins the live freshness signal (v_scraper_health, keyed off observation TABLES, not
|
||||
-- pg-boss job cadence) onto the registry, ranks dead-before-stale weighted by business
|
||||
-- value and how far past dead_days. Guardrail filtering (cooldown/budget/backoff/breaker)
|
||||
-- is applied in the controller against the ledger, not here, to keep the view simple.
|
||||
create or replace view v_dqo_targets as
|
||||
select
|
||||
sr.source_key,
|
||||
h.vendor,
|
||||
h.source_table,
|
||||
h.status,
|
||||
h.days_since_last,
|
||||
h.rows_30d,
|
||||
h.last_observation,
|
||||
sr.root_cause_class,
|
||||
sr.queue_names,
|
||||
sr.profile,
|
||||
sr.value_weight,
|
||||
sr.warn_days,
|
||||
sr.dead_days,
|
||||
sr.cooldown_minutes,
|
||||
sr.budget_per_day,
|
||||
sr.max_backoff_hours,
|
||||
sr.circuit_breaker_strikes,
|
||||
round(
|
||||
(case h.status when 'dead' then 3 when 'stale' then 2 else 1 end)::numeric
|
||||
* sr.value_weight
|
||||
* (1 + coalesce(h.days_since_last, 0)::numeric / nullif(sr.dead_days, 0))
|
||||
, 2) as score
|
||||
from v_scraper_health h
|
||||
join source_registry sr
|
||||
on sr.vendor = h.vendor and sr.source_table = h.source_table
|
||||
where h.status in ('stale', 'dead')
|
||||
and sr.enabled
|
||||
and sr.source_key <> '__global__'
|
||||
order by
|
||||
(case h.status when 'dead' then 3 when 'stale' then 2 else 1 end) desc,
|
||||
score desc;
|
||||
|
||||
-- ── 6) SEED — the global kill-switch/budget row + the known EXPECTED_VENDORS sources.
|
||||
-- root_cause_class is HAND-CURATED from the 2026-07-06 gap analysis, not guessed:
|
||||
-- ATGBICS/stock = code_fix (price is the single most-alive source yet stock writes
|
||||
-- 0 rows/54d = silent write-path break; escalate, never loop)
|
||||
-- Flexoptix/stock= staleness (price+catalog sync work; a re-scrape can plausibly heal it
|
||||
-- — the first source to arm out of plan-mode)
|
||||
-- Every other source defaults to 'staleness'. Extra (vendor, source_table) rows that do
|
||||
-- not match a live v_scraper_health row are harmless — the join just yields no target.
|
||||
insert into source_registry
|
||||
(source_key, vendor, source_table, queue_names, value_weight, root_cause_class, profile, notes)
|
||||
values
|
||||
-- global kill-switch + global daily dispatch ceiling (belt-and-suspenders over per-source budget)
|
||||
('__global__', null, null, '{}', 1, 'staleness', 'erik-safe',
|
||||
'Global control row: enabled=false disarms the entire loop; budget_per_day is the global daily dispatch ceiling.'),
|
||||
|
||||
-- stock sources (higher value_weight — stock freshness is core to sourcing)
|
||||
('atgbics/stock_observations', 'ATGBICS', 'stock_observations', '{scrape:pricing:atgbics}', 1.5, 'code_fix', 'erik-safe',
|
||||
'Silent stock write-path break: price fresh, stock 0 rows/54d. Escalate to a human; a re-scrape cannot fix a code bug.'),
|
||||
('flexoptix/stock_observations', 'Flexoptix', 'stock_observations', '{scrape:pricing:flexoptix}', 1.5, 'staleness', 'erik-safe',
|
||||
'First source to arm out of plan-mode: lowest-risk staleness, erik-safe queue.'),
|
||||
('fibermall/stock_observations', 'FiberMall', 'stock_observations', '{scrape:pricing:fibermall}', 1.5, 'staleness', 'erik-safe', null),
|
||||
('qsfptek/stock_observations', 'QSFPTEK', 'stock_observations', '{scrape:pricing:qsfptek}', 1.5, 'staleness', 'erik-safe',
|
||||
'The only currently-live stock writer — the reference for a working stock-write path.'),
|
||||
('naddod/stock_observations', 'NADDOD', 'stock_observations', '{scrape:pricing:naddod}', 1.5, 'staleness', 'erik-safe', null),
|
||||
('gbics/stock_observations', 'GBICS', 'stock_observations', '{scrape:pricing:gbics}', 1.5, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('sfpcables/stock_observations', 'SFPcables', 'stock_observations', '{scrape:pricing:sfpcables}', 1.5, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
|
||||
-- price sources
|
||||
('atgbics/price_observations', 'ATGBICS', 'price_observations', '{scrape:pricing:atgbics}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('flexoptix/price_observations', 'Flexoptix', 'price_observations', '{scrape:pricing:flexoptix}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('fibermall/price_observations', 'FiberMall', 'price_observations', '{scrape:pricing:fibermall}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('qsfptek/price_observations', 'QSFPTEK', 'price_observations', '{scrape:pricing:qsfptek}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('naddod/price_observations', 'NADDOD', 'price_observations', '{scrape:pricing:naddod}', 1.0, 'staleness', 'erik-safe', null),
|
||||
('gbics/price_observations', 'GBICS', 'price_observations', '{scrape:pricing:gbics}', 1.0, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('sfpcables/price_observations', 'SFPcables', 'price_observations', '{scrape:pricing:sfpcables}', 1.0, 'staleness', 'erik-safe',
|
||||
'Queue not in ERIK_SAFE_QUEUES — needs pi-fetch/proxmox profile to actually dispatch.'),
|
||||
('10gtek/price_observations', '10Gtek', 'price_observations', '{scrape:pricing:10gtek}', 1.0, 'staleness', 'proxmox-heavy',
|
||||
'scrape:pricing:10gtek is a HEAVY queue — only the proxmox-heavy profile dispatches it.')
|
||||
on conflict (source_key) do nothing;
|
||||
|
||||
-- set the global daily dispatch ceiling on the __global__ row (soak default: 20/day)
|
||||
update source_registry set budget_per_day = 20 where source_key = '__global__';
|
||||
|
||||
-- Proof (read anytime):
|
||||
-- select * from v_dqo_targets; -- what the loop would act on right now
|
||||
-- select enabled from source_registry where source_key='__global__'; -- kill-switch state
|
||||
-- select source_key, outcome, mode, dispatched_at from self_heal_dispatch order by id desc limit 20;
|
||||
@ -1,36 +0,0 @@
|
||||
-- sql/130-adapter-pi-topology.sql
|
||||
-- Teach the TIP Adapter (tip-dqo, the data-quality orchestrator from sql/129) the REAL
|
||||
-- fetch topology, verified 2026-07-06: TIP data is fetched primarily by three residential
|
||||
-- Raspberry Pi 5 nodes — ContextX-Pi01/tashi(.204, 16GB), Pi02/portia(.212, 4GB),
|
||||
-- Pi03/ester(.211, 4GB) — all live pg-boss workers, all now Playwright-capable
|
||||
-- (chromium v1208, launch-verified). NUC(.115) is the scheduler; Erik(.127) holds DB+API.
|
||||
--
|
||||
-- The other session seeded every vendor pricing/stock source as profile='erik-safe'. That
|
||||
-- is wrong for the fetch fleet: Erik is a datacenter IP (blocked by FS.com/ATGBICS & co.),
|
||||
-- and 'erik-safe' only dispatches ERIK_SAFE_QUEUES — so gbics/sfpcables would NEVER dispatch
|
||||
-- (their own notes flagged this). The residential Pis are the correct fetchers.
|
||||
--
|
||||
-- Fix: route all vendor pricing/stock sources to profile='pi-fetch' (recognized by the
|
||||
-- orchestrator: queuesForProfile → all non-heavy/non-discovery queues; cap 10). This is
|
||||
-- CONFIG ONLY — no scraper fires (the loop stays in Plan-Mode until SELF_HEAL_PLAN_ONLY=false).
|
||||
-- 10gtek stays 'proxmox-heavy' (its queue is in HEAVY_QUEUES). __global__ stays 'erik-safe'
|
||||
-- (config row, excluded from v_dqo_targets). Idempotent.
|
||||
--
|
||||
-- OPEN (code-level, belongs to the tip-dqo owner — NOT changed here to avoid colliding with
|
||||
-- their in-flight go-live): the HEAVY Playwright queues (scrape:pricing:fs, prolabs, compat:*)
|
||||
-- are now runnable on the residential Pi cluster too (Pis have Playwright). queuesForProfile
|
||||
-- currently excludes HEAVY from pi-fetch, so heavy IP-blocked sources (FS.com) still can't
|
||||
-- route to the Pis. A code change (let pi-fetch — or a new pi-heavy profile — dispatch HEAVY
|
||||
-- to the residential cluster) would let the cluster take those over from the Mac runners.
|
||||
|
||||
update source_registry
|
||||
set profile = 'pi-fetch',
|
||||
notes = coalesce(notes || ' | ', '')
|
||||
|| 'topology-fix 130 (2026-07-06): residential Pi cluster is the fetcher, not Erik.',
|
||||
updated_at = now()
|
||||
where source_key <> '__global__'
|
||||
and profile = 'erik-safe';
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '130-adapter-pi-topology.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '130-adapter-pi-topology.sql');
|
||||
@ -1,60 +0,0 @@
|
||||
-- sql/131 — Wavelength column sync (root-cause fix for equivalence review backlog)
|
||||
--
|
||||
-- Root cause: the enricher populates the structured integer column
|
||||
-- transceivers.wavelength_tx_nm, but the equivalence spec-matcher reads the TEXT
|
||||
-- column transceivers.wavelengths via tip_extract_wavelength_nm(). 27,908 rows had
|
||||
-- wavelength_tx_nm set while wavelengths was NULL, so the matcher could not confirm
|
||||
-- wavelength on either side and dumped ~3,155 genuine equivalences into manual review.
|
||||
--
|
||||
-- Fix (no fabrication — copies existing structured data into the format the matcher
|
||||
-- expects, and keeps it in sync going forward):
|
||||
-- A) one-off backfill of transceivers.wavelengths from wavelength_tx_nm
|
||||
-- B) durable BEFORE-trigger so future ingests stay in sync
|
||||
-- C) promote pending -> auto_approved ONLY where the FULL spec-matcher criteria now
|
||||
-- agree (form_factor + speed + fiber_type + reach-tier + wavelength +/-10nm).
|
||||
-- Everything with a genuine reach-tier / fiber / wavelength difference stays pending.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- A) one-off backfill (existing structured data -> text column the matcher reads)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = wavelength_tx_nm || 'nm'
|
||||
WHERE (wavelengths IS NULL OR wavelengths = '')
|
||||
AND wavelength_tx_nm IS NOT NULL;
|
||||
|
||||
-- B) durable sync trigger for future inserts/updates
|
||||
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||
BEGIN
|
||||
IF (NEW.wavelengths IS NULL OR NEW.wavelengths = '')
|
||||
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_sync_wavelengths ON transceivers;
|
||||
CREATE TRIGGER trg_sync_wavelengths
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_sync_wavelengths_from_tx();
|
||||
|
||||
-- C) valid promotion: pending -> auto_approved where full spec now confirmed
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'auto_approved',
|
||||
confidence = GREATEST(e.confidence, 0.85),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/131: full spec confirmed after wavelength column sync',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||
|
||||
COMMIT;
|
||||
@ -1,74 +0,0 @@
|
||||
-- sql/132 — Wavelength authority correction + format normalization
|
||||
--
|
||||
-- Follow-up to sql/131. Discovery: transceivers.wavelengths holds the part-derived
|
||||
-- wavelength as a BARE number ("1470") for ~16.8k rows, which tip_extract_wavelength_nm()
|
||||
-- cannot read (it requires the "nm" suffix). Meanwhile wavelength_tx_nm disagrees with
|
||||
-- that authoritative bare number 33% of the time, and 64% of those disagreements are a
|
||||
-- systematic tx_nm=1310 enricher default. So the bare-number column is authoritative;
|
||||
-- tx_nm is not. This migration prefers the authoritative source.
|
||||
--
|
||||
-- 1) normalize authoritative bare-number wavelengths -> "NNNnm" (format only, no value change)
|
||||
-- 2) clean "N/A" text -> NULL
|
||||
-- 3) correct the few CWDM rows sql/131 backfilled with a wrong tx_nm default, using the
|
||||
-- deterministic CWDM wavelength in the part number
|
||||
-- 4) re-promote pending -> auto_approved where the FULL spec gate now agrees on
|
||||
-- authoritative data (gate already protects against tx_nm noise: both sides must
|
||||
-- agree within +/-10nm, so a wrong 1310 default can never auto-approve a real 1470).
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 1) normalize authoritative bare-number wavelengths (preserve value, add unit)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = wavelengths || 'nm'
|
||||
WHERE wavelengths ~ '^\d{3,4}$';
|
||||
|
||||
-- 2) non-values -> NULL (N/A, -, empty, etc.)
|
||||
UPDATE transceivers
|
||||
SET wavelengths = NULL
|
||||
WHERE wavelengths IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(wavelengths) IS NULL
|
||||
AND wavelengths !~ '\d';
|
||||
|
||||
-- 3) correct CWDM rows that sql/131 filled from a wrong tx_nm default:
|
||||
-- use the deterministic CWDM wavelength encoded in the part number
|
||||
UPDATE transceivers
|
||||
SET wavelengths = (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1] || 'nm'
|
||||
WHERE wavelengths = wavelength_tx_nm || 'nm'
|
||||
AND part_number ~ '(12[6-9]|13[0-9]|14[0-9]|15[0-9]|16[01])0'
|
||||
AND (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1]::int <> wavelength_tx_nm;
|
||||
|
||||
-- 3b) upgrade the sync trigger so future ingests self-normalize:
|
||||
-- prefer an authoritative bare-number wavelength (add "nm"), fall back to tx_nm only
|
||||
-- when no textual wavelength is present at all.
|
||||
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||
BEGIN
|
||||
IF NEW.wavelengths ~ '^\d{3,4}$' THEN
|
||||
NEW.wavelengths := NEW.wavelengths || 'nm';
|
||||
ELSIF (NEW.wavelengths IS NULL OR NEW.wavelengths = '' OR tip_extract_wavelength_nm(NEW.wavelengths) IS NULL)
|
||||
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
|
||||
-- 4) valid promotion on corrected/authoritative data (same full spec gate as spec-matcher)
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'auto_approved',
|
||||
confidence = GREATEST(e.confidence, 0.85),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/132: full spec confirmed on authoritative wavelength',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||
|
||||
COMMIT;
|
||||
@ -1,46 +0,0 @@
|
||||
-- sql/133 — One-directional reach-tier substitution policy
|
||||
--
|
||||
-- Decision (Rene, 2026-07-07): a higher-reach optic may satisfy a lower-reach
|
||||
-- requirement (e.g. a 40km ER can serve where a 10km LR is asked for), never the
|
||||
-- reverse — the shorter-reach part cannot close the link budget on a longer run.
|
||||
-- ZR/coherent/amplified tiers are explicitly EXCLUDED from this automatic rule:
|
||||
-- those can overload a short-haul receiver without an attenuator, which is a real
|
||||
-- power-budget calculation, not a simple tier comparison. Those stay in manual review.
|
||||
--
|
||||
-- Result is tagged as its own status (compatible_superset), never merged into
|
||||
-- auto_approved/exact-spec matches, so downstream consumers (Fossy/Beo/Pulso, pricing)
|
||||
-- can distinguish "identical spec" from "safe higher-spec substitute" — they usually
|
||||
-- differ in price and are not a like-for-like swap.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- widen the status check (additive — existing values still valid)
|
||||
ALTER TABLE transceiver_equivalences DROP CONSTRAINT transceiver_equivalences_status_check;
|
||||
ALTER TABLE transceiver_equivalences ADD CONSTRAINT transceiver_equivalences_status_check
|
||||
CHECK (status::text = ANY (ARRAY['pending','approved','rejected','auto_approved','compatible_superset']::text[]));
|
||||
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'compatible_superset',
|
||||
confidence = GREATEST(e.confidence, 0.75),
|
||||
match_notes = COALESCE(e.match_notes,'') || ' | sql/133: ' ||
|
||||
CASE WHEN fx.reach_meters > cx.reach_meters
|
||||
THEN 'flexoptix ('||tip_reach_tier(fx.reach_meters)||') covers competitor requirement ('||tip_reach_tier(cx.reach_meters)||')'
|
||||
ELSE 'competitor ('||tip_reach_tier(cx.reach_meters)||') covers flexoptix requirement ('||tip_reach_tier(fx.reach_meters)||')'
|
||||
END || ' — reach-tier substitute, not identical spec',
|
||||
updated_at = now()
|
||||
FROM transceivers fx, transceivers cx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.competitor_id = cx.id
|
||||
AND e.status = 'pending'
|
||||
AND fx.form_factor = cx.form_factor
|
||||
AND fx.speed_gbps = cx.speed_gbps
|
||||
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10
|
||||
AND tip_reach_tier(fx.reach_meters) <> tip_reach_tier(cx.reach_meters)
|
||||
AND tip_reach_tier(fx.reach_meters) <> 'ZR'
|
||||
AND tip_reach_tier(cx.reach_meters) <> 'ZR';
|
||||
|
||||
COMMIT;
|
||||
@ -1,20 +0,0 @@
|
||||
-- sql/134-transceiver-ff-speed-errors.sql
|
||||
-- LUPO-34: data-quality curation view for physically-impossible form_factor x speed
|
||||
-- combinations. An SFP-class module (SFP/SFP+/SFP28/XFP/GBIC/CSFP) tops out at 25-28G,
|
||||
-- so speed_gbps >= 50 on such a form factor is a misclassification (surfaced in Lupo's
|
||||
-- per-SKU price panel as flagged rows). Evidence-based (physics), NOT name-derived.
|
||||
-- Read-only; the exact corrected speed needs the vendor datasheet (data-owner task).
|
||||
-- SFP-DD / SFP112 legitimately carry >=100G and are intentionally NOT flagged.
|
||||
create or replace view v_transceiver_ff_speed_errors as
|
||||
select t.id, v.name as vendor, t.part_number, t.speed_gbps, t.form_factor
|
||||
from transceivers t
|
||||
left join vendors v on v.id = t.vendor_id
|
||||
where upper(coalesce(t.form_factor, '')) in ('SFP', 'SFP+', 'SFP28', 'XFP', 'GBIC', 'CSFP')
|
||||
and t.speed_gbps >= 50;
|
||||
|
||||
comment on view v_transceiver_ff_speed_errors is
|
||||
'LUPO-34: SFP-class parts with an impossible speed (>=50G). Data-quality curation list; exact correction needs the datasheet.';
|
||||
|
||||
-- Usage:
|
||||
-- select count(*) from v_transceiver_ff_speed_errors; -- total to curate
|
||||
-- select vendor, count(*) from v_transceiver_ff_speed_errors group by 1 order by 2 desc;
|
||||
@ -1,14 +0,0 @@
|
||||
-- Migration 135: Equivalence quality cleanup
|
||||
-- Demotes auto_approved rows that no longer meet quality gates:
|
||||
-- 1. FX product has no standard_name → can't verify match quality
|
||||
-- Applied 2026-07-11 after adding standard_name gate to catalog-reconcile.
|
||||
|
||||
UPDATE transceiver_equivalences e
|
||||
SET status = 'pending', updated_at = NOW()
|
||||
FROM transceivers fx
|
||||
WHERE e.flexoptix_id = fx.id
|
||||
AND e.status = 'auto_approved'
|
||||
AND fx.standard_name IS NULL;
|
||||
|
||||
-- Result: 60,278 demoted from auto_approved → pending
|
||||
-- Remaining auto_approved: 20,838 (all with standard_name)
|
||||
@ -1,227 +0,0 @@
|
||||
-- 135 Modulation/FEC enrichment + speed plausibility guard for transceivers.
|
||||
--
|
||||
-- Derived and adversarially verified via workflow wf_dc5bab06 (4 optical-standards
|
||||
-- lenses -> synthesis -> 4 skeptical reviewers). All CONFIRMED reviewer findings are
|
||||
-- folded in here: coherent >=100G speed floor (10G-ZR is NRZ, not DCO), form_factor
|
||||
-- DCO/ACO scan (CFP2-DCO evidence lives in form_factor), compact DAC/AOC/AEC suffix
|
||||
-- exclusion, bare-QSFP28-no-token ABSTAINS instead of guessing NRZ, CR4 dropped from
|
||||
-- the NRZ set, anchored MTP/MPO, NON-COHERENT guard, search_path pinned.
|
||||
--
|
||||
-- FILL-ONLY: never overwrites a non-null value, never touches data_confidence='vendor_verified'.
|
||||
-- Idempotent: safe to re-run; a fully enriched table yields 0 rows touched.
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Speed plausibility. Flags only UNAMBIGUOUS junk: sub-0.09 fragments (incl. <=0;
|
||||
-- the lowest real optical rate is 100M=0.1) and >1600 (cable-part-number integers
|
||||
-- like 2.0e16). The 0.09..1600 band is deliberately kept intact: it holds legit
|
||||
-- non-Ethernet rates (FC line rates 2.125/4.25/8.5/14.025, SDI 3/6/12, CPRI 1.2288,
|
||||
-- OTN 10.7) that must not be discarded. NULL is treated as NOT implausible.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_speed_is_implausible(v numeric)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT v IS NOT NULL AND (v < 0.09 OR v > 1600);
|
||||
$$;
|
||||
|
||||
-- Separate, isolated speed sanitizer. Distinct from tip_transceivers_biu() (which
|
||||
-- recomputes product_type/form_factor and never touches speed). Sanitize-only:
|
||||
-- NULLs junk speed on write, never raises, never blocks ingestion.
|
||||
CREATE OR REPLACE FUNCTION tip_speed_plausibility_biu()
|
||||
RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.speed_gbps IS NOT NULL AND tip_speed_is_implausible(NEW.speed_gbps) THEN
|
||||
NEW.speed_gbps := NULL;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_speed_plausibility ON transceivers;
|
||||
CREATE TRIGGER trg_speed_plausibility
|
||||
BEFORE INSERT OR UPDATE ON transceivers
|
||||
FOR EACH ROW EXECUTE FUNCTION tip_speed_plausibility_biu();
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Enrichment scope: real optics only. The part_number cable/trunk veto applies to
|
||||
-- BOTH the product_type='module' and product_type IS NULL branches, so a cable
|
||||
-- mis-classified 'module' upstream still cannot be enriched.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_in_optics_scope(p_product_type text, p_form_factor text, p_part_number text)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT
|
||||
coalesce(p_product_type,'') NOT IN ('dac','aoc','aec','accessory','switch','nic')
|
||||
-- compact suffix forms (-AOC3M, -DAC1M, -AEC2M), Flexoptix cable schemes (M4.*, S2.*),
|
||||
-- loopbacks, and explicit cable keywords are never optics. MTP/MPO is deliberately
|
||||
-- NOT vetoed here: parallel optics (DR4/SR4/DR8) legitimately carry an -MPO12 connector
|
||||
-- suffix, and real fiber trunks are already caught by ^M[0-9]./^S[0-9]./CABLE.
|
||||
AND coalesce(p_part_number,'') !~* '([-_](DAC|AOC|AEC)([-_0-9]|$)|CAB[-_]|CABLE|FN-CABLE|LOOPBACK|[-_]LB[0-9]|^M[0-9]\.|^S[0-9]\.)'
|
||||
AND (
|
||||
p_product_type = 'module'
|
||||
OR (
|
||||
p_product_type IS NULL
|
||||
AND upper(coalesce(p_form_factor,'')) IN
|
||||
('QSFP28','QSFP56','QSFP-DD','QSFP-DD800','OSFP','OSFP800','CFP2','CFP2-DCO','CFP2-ACO','CFP8','SFP-DD','SFP112','DSFP')
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Coherent detection. Regex-derived triggers are floored at >=100G (100ZR is the
|
||||
-- lowest coherent pluggable; 10G/40G-ZR are NRZ reach optics). A pre-existing
|
||||
-- coherent=true is trusted and NOT speed-floored. NON-COHERENT is excluded.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tip_is_coherent(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT
|
||||
p_coherent IS TRUE
|
||||
OR (
|
||||
p_speed >= 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')||' '||coalesce(p_ff,'')) !~* 'NON-?COHERENT'
|
||||
AND (
|
||||
(coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])(100|200|300|400|600|800|1600)?G?-?ZR(\+|P)?([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])DCO([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* 'OPEN-?ZR'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^A-Za-z0-9])COHERENT([^A-Za-z0-9]|$)'
|
||||
OR (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(16QAM|8QAM|64QAM|QPSK)'
|
||||
-- CFP2-DCO/ACO carry the only coherent evidence in form_factor (standard_name ~1% populated).
|
||||
OR upper(coalesce(p_ff,'')) ~ '(DCO|ACO|COHERENT)'
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- Modulation. NULL = ABSTAIN (unknown). Order: coherent -> PAM4-by-speed ->
|
||||
-- 100G single-lambda PAM4 -> 100G 4x25 NRZ -> abstain. Bare QSFP28 with no reach
|
||||
-- token abstains (never defaulted to NRZ). (?![0-9]) keeps SR4 from matching SR40
|
||||
-- and DR/FR from matching DR4/FR4.
|
||||
CREATE OR REPLACE FUNCTION tip_derive_modulation(p_std text, p_pn text, p_ff text, p_speed numeric, p_coherent boolean)
|
||||
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN tip_is_coherent(p_std, p_pn, p_ff, p_speed, p_coherent) THEN CASE
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?QPSK' THEN 'DP-QPSK'
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?16QAM' THEN 'DP-16QAM'
|
||||
WHEN (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(DP|PM)[-_]?(8QAM|64QAM)' THEN 'Coherent'
|
||||
WHEN p_speed >= 400 THEN 'DP-16QAM' -- 400ZR/800ZR/DCO default
|
||||
WHEN p_speed = 100 THEN 'DP-QPSK' -- 100ZR is QPSK
|
||||
ELSE 'Coherent' -- 200G/300G constellation ambiguous -> generic
|
||||
END
|
||||
WHEN p_speed IN (200,400,800,1600) THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN NULL -- conflicting single-lambda AND 4x25 tokens -> abstain
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(DR1?|FR1?|LR1|ER1|SR1|SR2|VR1?)(?![0-9])'
|
||||
THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND upper(coalesce(p_ff,'')) IN ('SFP-DD','SFP112','DSFP')
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) !~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN 'PAM4'
|
||||
WHEN p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|LR4|ER4L?|CWDM4|CLR4|PSM4|4WDM)(?![0-9])'
|
||||
THEN 'NRZ'
|
||||
ELSE NULL -- bare QSFP28 / no token / sub-100G / non-standard speed -> abstain
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- FEC type. Reads modulation. Abstains (NULL) for 100G NRZ LR4/ER4/CWDM4/CLR4/4WDM
|
||||
-- (no mandatory FEC); that assertion is carried by inbuilt_fec=false, not a fake fec_type.
|
||||
CREATE OR REPLACE FUNCTION tip_derive_fec_type(p_modulation text, p_std text, p_pn text, p_speed numeric)
|
||||
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN 'SD-FEC'
|
||||
WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN 'RS-FEC (RS(544,514) KP4)'
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN 'RS-FEC (RS(528,514) CL91)'
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION tip_derive_inbuilt_fec(p_modulation text, p_std text, p_pn text, p_speed numeric)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
SELECT CASE
|
||||
WHEN p_modulation IN ('DP-16QAM','DP-QPSK','Coherent') THEN true
|
||||
WHEN p_modulation = 'PAM4' AND p_speed IN (100,200,400,800,1600) THEN true
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(SR4|PSM4)(?![0-9])' THEN true
|
||||
WHEN p_modulation = 'NRZ' AND p_speed = 100
|
||||
AND (coalesce(p_std,'')||' '||coalesce(p_pn,'')) ~* '(^|[^0-9A-Za-z])(LR4|ER4L?|CWDM4|CLR4|4WDM)(?![0-9])' THEN false
|
||||
ELSE NULL
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Idempotent fill-only enrichment. Returns total per-statement rows touched
|
||||
-- (a row filled across all three steps counts up to 3x). Fully enriched -> 0.
|
||||
-- One SELECT call runs all three steps in a single transaction, so APPLY-3 reads
|
||||
-- the modulation APPLY-2 just set.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION fn_tip_enrich_modulation_fec()
|
||||
RETURNS integer LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
DECLARE n1 int := 0; n2 int := 0; n3 int := 0;
|
||||
BEGIN
|
||||
-- 1) coherent flag (set true only; never flip an existing true->false)
|
||||
UPDATE transceivers t
|
||||
SET coherent = true,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['coherent']) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
WHERE coalesce(t.data_confidence,'') <> 'vendor_verified'
|
||||
AND t.speed_gbps IS NOT NULL
|
||||
AND t.coherent IS DISTINCT FROM true
|
||||
AND tip_in_optics_scope(t.product_type, t.form_factor, t.part_number)
|
||||
AND tip_is_coherent(t.standard_name, t.part_number, t.form_factor, t.speed_gbps, t.coherent);
|
||||
GET DIAGNOSTICS n1 = ROW_COUNT;
|
||||
|
||||
-- 2) modulation (fill-only)
|
||||
UPDATE transceivers t
|
||||
SET modulation = m.val,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(coalesce(t.enriched_fields,'{}'::text[]) || ARRAY['modulation']) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
FROM (
|
||||
SELECT ctid AS cid, tip_derive_modulation(standard_name, part_number, form_factor, speed_gbps, coherent) AS val
|
||||
FROM transceivers
|
||||
WHERE modulation IS NULL
|
||||
AND coalesce(data_confidence,'') <> 'vendor_verified'
|
||||
AND speed_gbps IS NOT NULL
|
||||
AND tip_in_optics_scope(product_type, form_factor, part_number)
|
||||
) m
|
||||
WHERE t.ctid = m.cid AND m.val IS NOT NULL;
|
||||
GET DIAGNOSTICS n2 = ROW_COUNT;
|
||||
|
||||
-- 3) fec_type + inbuilt_fec (reads modulation set in step 2)
|
||||
UPDATE transceivers t
|
||||
SET fec_type = CASE WHEN t.fec_type IS NULL AND f.fec IS NOT NULL THEN f.fec ELSE t.fec_type END,
|
||||
inbuilt_fec = CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN f.inb ELSE t.inbuilt_fec END,
|
||||
enriched_at = now(),
|
||||
enriched_fields = (SELECT array_agg(DISTINCT e) FROM unnest(
|
||||
coalesce(t.enriched_fields,'{}'::text[])
|
||||
|| CASE WHEN t.fec_type IS NULL AND f.fec IS NOT NULL THEN ARRAY['fec_type'] ELSE '{}'::text[] END
|
||||
|| CASE WHEN t.inbuilt_fec IS NULL AND f.inb IS NOT NULL THEN ARRAY['inbuilt_fec'] ELSE '{}'::text[] END) e),
|
||||
data_confidence = CASE WHEN coalesce(t.data_confidence,'') IN ('','scraped_unverified','unknown') THEN 'enriched_estimated' ELSE t.data_confidence END
|
||||
FROM (
|
||||
SELECT ctid AS cid,
|
||||
tip_derive_fec_type(modulation, standard_name, part_number, speed_gbps) AS fec,
|
||||
tip_derive_inbuilt_fec(modulation, standard_name, part_number, speed_gbps) AS inb
|
||||
FROM transceivers
|
||||
WHERE coalesce(data_confidence,'') <> 'vendor_verified'
|
||||
AND speed_gbps IS NOT NULL
|
||||
AND (fec_type IS NULL OR inbuilt_fec IS NULL)
|
||||
AND tip_in_optics_scope(product_type, form_factor, part_number)
|
||||
) f
|
||||
WHERE t.ctid = f.cid
|
||||
AND ((t.fec_type IS NULL AND f.fec IS NOT NULL) OR (t.inbuilt_fec IS NULL AND f.inb IS NOT NULL));
|
||||
GET DIAGNOSTICS n3 = ROW_COUNT;
|
||||
|
||||
RETURN n1 + n2 + n3;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Run enrichment: SELECT fn_tip_enrich_modulation_fec();
|
||||
@ -1,48 +0,0 @@
|
||||
-- 136 Make speed_gbps nullable + one-time reversible cleanup of junk speed.
|
||||
-- Requires 135 (tip_speed_is_implausible, trg_speed_plausibility) applied first.
|
||||
--
|
||||
-- speed_gbps was NOT NULL, which (a) forced meaningless values onto cables/accessories
|
||||
-- and (b) made the guard trigger unusable: NULLing junk on write would abort the insert.
|
||||
-- "Unknown speed" is honestly NULL, so we drop the constraint, then NULL the ~4708
|
||||
-- unambiguous-junk rows (sub-0.09 fragments incl. <=0, and >1600 cable-part-number
|
||||
-- integers). 87% are dac/aoc/accessory where speed is meaningless anyway. The old value
|
||||
-- is snapshotted by primary key so the change is fully reversible. The 0.09..1600 gray
|
||||
-- band (~355 legit FC/SDI/CPRI/OTN line rates) is intentionally left untouched.
|
||||
--
|
||||
-- PREVIEW before running (nothing legit should appear):
|
||||
-- SELECT speed_gbps, count(*) FROM transceivers
|
||||
-- WHERE tip_speed_is_implausible(speed_gbps) GROUP BY 1 ORDER BY 1;
|
||||
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE transceivers ALTER COLUMN speed_gbps DROP NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tip_speed_gbps_rollback (
|
||||
id uuid PRIMARY KEY,
|
||||
old_speed_gbps numeric,
|
||||
backed_up_at timestamptz DEFAULT now()
|
||||
);
|
||||
|
||||
INSERT INTO tip_speed_gbps_rollback (id, old_speed_gbps)
|
||||
SELECT id, speed_gbps
|
||||
FROM transceivers
|
||||
WHERE tip_speed_is_implausible(speed_gbps)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
UPDATE transceivers
|
||||
SET speed_gbps = NULL
|
||||
WHERE tip_speed_is_implausible(speed_gbps);
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- REVERSE (maintenance window, no concurrent writers). Disable the guard so it does
|
||||
-- not re-NULL the junk being restored:
|
||||
-- BEGIN;
|
||||
-- ALTER TABLE transceivers DISABLE TRIGGER trg_speed_plausibility;
|
||||
-- UPDATE transceivers t SET speed_gbps = r.old_speed_gbps
|
||||
-- FROM tip_speed_gbps_rollback r
|
||||
-- WHERE t.id = r.id AND t.speed_gbps IS NULL;
|
||||
-- ALTER TABLE transceivers ENABLE TRIGGER trg_speed_plausibility;
|
||||
-- -- optional, only if every row is non-null again:
|
||||
-- -- ALTER TABLE transceivers ALTER COLUMN speed_gbps SET NOT NULL;
|
||||
-- COMMIT;
|
||||
@ -1,13 +0,0 @@
|
||||
-- 137 Make v_transceiver_speed_errors count only genuine optics.
|
||||
--
|
||||
-- After 136 nulled junk speed, the old definition (speed_gbps IS NULL OR <= 0)
|
||||
-- counted 4708 rows, but 4112 of them are dac/aoc/aec/accessory where speed_gbps
|
||||
-- is meaningless and NULL is correct. A speed error is a real optic (module) with
|
||||
-- a missing or physically-impossible speed. This drops the count to the ~596 real
|
||||
-- modules that still need a recoverable speed.
|
||||
|
||||
CREATE OR REPLACE VIEW v_transceiver_speed_errors AS
|
||||
SELECT id, part_number, speed_gbps, form_factor, product_type
|
||||
FROM transceivers
|
||||
WHERE coalesce(product_type,'') NOT IN ('dac','aoc','aec','accessory','switch','nic')
|
||||
AND (speed_gbps IS NULL OR speed_gbps <= 0 OR speed_gbps > 1600);
|
||||
@ -1,83 +0,0 @@
|
||||
-- sql/138-transceiver-speed-drift-guard.sql
|
||||
-- Closes a gap in the sql/125+126 "no physically-impossible compatibility rows"
|
||||
-- invariant: compat_speed_guard() (126) only fires on INSERT/UPDATE of the
|
||||
-- `compatibility` table itself. It never re-validates existing 'compatible'
|
||||
-- spec_match rows when a TRANSCEIVER's speed_gbps changes later (e.g. the daily
|
||||
-- enrich-modulation-fec.sh self-heal, sql/135, recovering speed from part-number
|
||||
-- tokens for previously NULL/junk rows). Found live 2026-07-12: 189 compat rows
|
||||
-- had gone physically-impossible because their transceiver's speed_gbps was
|
||||
-- updated (all sampled rows dated today, i.e. by the 03:30 cron) after the
|
||||
-- compat row was already written as 'compatible'.
|
||||
--
|
||||
-- Fix mirrors 126 exactly, just triggered from the other table: AFTER UPDATE OF
|
||||
-- speed_gbps on transceivers, re-validate every 'compatible' spec_match
|
||||
-- compatibility row referencing that transceiver against the same evidence-based
|
||||
-- ceiling (v_switch_effective_max) used by v_compat_speed_errors. Curated rows
|
||||
-- (vendor_matrix/vendor_compat/tested/datasheet) are never touched. Idempotent,
|
||||
-- safe to re-run.
|
||||
|
||||
-- Backup of affected rows before remediation (audit trail, matches
|
||||
-- tip_speed_gbps_rollback precedent from sql/136).
|
||||
create table if not exists compat_speed_drift_backup_20260712 as
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
select c.*, e.effective_max_gbps, t.speed_gbps as tx_speed_gbps, now() as backed_up_at
|
||||
from compatibility c
|
||||
join em e on e.switch_id = c.switch_id
|
||||
join transceivers t on t.id = c.transceiver_id
|
||||
where c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- One-time remediation: flip the rows found live today (same logic as 125's
|
||||
-- Remediation B, using the full evidence-based ceiling incl. curated bump).
|
||||
with em as materialized (
|
||||
select switch_id, effective_max_gbps from v_switch_effective_max)
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)'
|
||||
else c.notes || ' | auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)' end
|
||||
from em e, transceivers t
|
||||
where e.switch_id = c.switch_id
|
||||
and t.id = c.transceiver_id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > e.effective_max_gbps;
|
||||
|
||||
-- Durable fix: re-validate on transceiver speed change.
|
||||
create or replace function compat_speed_guard_on_transceiver_change() returns trigger
|
||||
language plpgsql as $$
|
||||
begin
|
||||
if NEW.speed_gbps is distinct from OLD.speed_gbps and NEW.speed_gbps is not null then
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)'
|
||||
else c.notes || ' | auto(138): transceiver speed exceeds switch max port speed (post-hoc drift)' end
|
||||
from v_switch_effective_max e
|
||||
where e.switch_id = c.switch_id
|
||||
and c.transceiver_id = NEW.id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and e.effective_max_gbps > 0
|
||||
and NEW.speed_gbps > e.effective_max_gbps;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function compat_speed_guard_on_transceiver_change() is
|
||||
'AFTER UPDATE OF speed_gbps on transceivers: re-validates existing compatible spec_match rows for this transceiver against the evidence-based switch ceiling, closing the drift gap left by compat_speed_guard() (126), which only fires on compatibility writes. Never touches curated rows.';
|
||||
|
||||
drop trigger if exists trg_compat_speed_guard_on_transceiver_change on transceivers;
|
||||
create trigger trg_compat_speed_guard_on_transceiver_change
|
||||
after update of speed_gbps on transceivers
|
||||
for each row execute function compat_speed_guard_on_transceiver_change();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '138-transceiver-speed-drift-guard.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '138-transceiver-speed-drift-guard.sql');
|
||||
@ -1,130 +0,0 @@
|
||||
-- sql/139-equivalence-pending-triage.sql
|
||||
-- Set-based port of scheduler.ts's evaluateEquivalenceResearch() (line ~85),
|
||||
-- applied once to the 289,681-row pending backlog in transceiver_equivalences.
|
||||
--
|
||||
-- Root cause of the backlog: the daily "maintenance:re-research-equivalences" job
|
||||
-- (scheduler.ts:3117) only evaluates rows where re_research_due_at IS NOT NULL AND
|
||||
-- <= NOW(). catalog-reconcile.ts inserts new 'pending' rows WITHOUT setting
|
||||
-- re_research_due_at (NULL by default) -> the job that was designed to drain the
|
||||
-- queue never saw fresh pending rows. This migration (a) triages the existing
|
||||
-- backlog once, set-based, using the exact same scoring rules as the JS
|
||||
-- evaluator, and (b) is paired with code fixes (catalog-reconcile.ts sets
|
||||
-- re_research_due_at on insert; scheduler.ts's reject branch preserves it for
|
||||
-- "insufficient evidence" rejects) so the daily job keeps draining future inserts
|
||||
-- on its own.
|
||||
--
|
||||
-- Decision rules (identical to evaluateEquivalenceResearch, ported 1:1):
|
||||
-- no recent competitor price observation (45d) -> reject, no re-research
|
||||
-- form_factor or speed_gbps mismatch/missing -> reject, no re-research
|
||||
-- wavelength diff >15nm / fiber mismatch / reach -> reject, no re-research
|
||||
-- ratio <0.85 (all three "hard" if evidence exists)
|
||||
-- wavelength / fiber_type / reach_meters MISSING -> reject, WITH re-research
|
||||
-- (on either side, and no hard mismatch above) in 21 days (data-gap,
|
||||
-- not a wrong match --
|
||||
-- give scrapers time to
|
||||
-- enrich, then recheck)
|
||||
-- confidence (score/115) >= 0.73 -> auto_approved
|
||||
-- else -> reject, no re-research
|
||||
--
|
||||
-- Idempotent for its WHERE clause (status='pending' AND re_research_due_at IS
|
||||
-- NULL); re-running after the code fix lands is a no-op since new pending rows
|
||||
-- will already carry a re_research_due_at.
|
||||
|
||||
create table if not exists equivalence_pending_triage_backup_20260713 as
|
||||
select *, now() as backed_up_at
|
||||
from transceiver_equivalences
|
||||
where status = 'pending' and re_research_due_at is null;
|
||||
|
||||
with normalized as (
|
||||
select eq.id,
|
||||
nullif(upper(trim(fx.form_factor)),'') as fx_form, nullif(upper(trim(cp.form_factor)),'') as cp_form,
|
||||
fx.speed_gbps as fx_speed, cp.speed_gbps as cp_speed,
|
||||
nullif(upper(trim(fx.standard_name)),'') as fx_std, nullif(upper(trim(cp.standard_name)),'') as cp_std,
|
||||
nullif(upper(trim(fx.fiber_type)),'') as fx_fiber, nullif(upper(trim(cp.fiber_type)),'') as cp_fiber,
|
||||
fx.reach_meters as fx_reach, cp.reach_meters as cp_reach,
|
||||
(regexp_match(fx.wavelengths, '(\d{3,4})'))[1]::numeric as fx_nm,
|
||||
(regexp_match(cp.wavelengths, '(\d{3,4})'))[1]::numeric as cp_nm,
|
||||
exists(select 1 from price_observations po where po.transceiver_id=eq.competitor_id and po.time > now() - interval '45 days') as has_recent_price
|
||||
from transceiver_equivalences eq
|
||||
join transceivers fx on fx.id = eq.flexoptix_id
|
||||
join transceivers cp on cp.id = eq.competitor_id
|
||||
where eq.status = 'pending' and eq.re_research_due_at is null
|
||||
),
|
||||
scored as (
|
||||
select *,
|
||||
(case when fx_form is not null and cp_form is not null and fx_form=cp_form then 25 else 0 end
|
||||
+ case when fx_speed is not null and cp_speed is not null and fx_speed=cp_speed then 20 else 0 end
|
||||
+ case when fx_std is not null and cp_std is not null and fx_std=cp_std then 30 else 0 end
|
||||
+ case when fx_nm is not null and cp_nm is not null then
|
||||
case when abs(fx_nm-cp_nm)<=15 then 20 else -20 end
|
||||
else 0 end
|
||||
+ case when fx_fiber is not null and cp_fiber is not null then
|
||||
case when fx_fiber=cp_fiber then 10 else -15 end
|
||||
else 0 end
|
||||
+ case when fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 then
|
||||
case when least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) >= 0.85 then 10 else -15 end
|
||||
else 0 end) as raw_score,
|
||||
(fx_nm is null or cp_nm is null) as wl_missing,
|
||||
(fx_fiber is null or cp_fiber is null) as fiber_missing,
|
||||
(fx_reach is null or cp_reach is null or fx_reach<=0 or cp_reach<=0) as reach_missing,
|
||||
(fx_nm is not null and cp_nm is not null and abs(fx_nm-cp_nm)>15) as wl_mismatch,
|
||||
(fx_fiber is not null and cp_fiber is not null and fx_fiber<>cp_fiber) as fiber_mismatch,
|
||||
(fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 and least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) < 0.85) as reach_mismatch,
|
||||
(fx_form is null or cp_form is null or fx_form<>cp_form) as form_bad,
|
||||
(fx_speed is null or cp_speed is null or fx_speed<>cp_speed) as speed_bad
|
||||
from normalized
|
||||
),
|
||||
decided as (
|
||||
select *,
|
||||
greatest(0, least(1, raw_score::numeric/115)) as confidence,
|
||||
(not has_recent_price) as no_price,
|
||||
(wl_mismatch or fiber_mismatch or reach_mismatch or form_bad or speed_bad) as critical_mismatch,
|
||||
(wl_missing or fiber_missing or reach_missing) as missing_evidence
|
||||
from scored
|
||||
)
|
||||
update transceiver_equivalences eq
|
||||
set status = case
|
||||
when d.no_price then 'rejected'
|
||||
when d.critical_mismatch then 'rejected'
|
||||
when d.missing_evidence then 'rejected'
|
||||
when d.confidence >= 0.73 then 'auto_approved'
|
||||
else 'rejected'
|
||||
end,
|
||||
confidence = case when d.no_price then 0 else d.confidence end,
|
||||
reviewed_by = 'automated-research-bulk-139',
|
||||
reviewed_at = now(),
|
||||
reject_reason = case
|
||||
when d.no_price then 'automated research: competitor has no recent price observation'
|
||||
when d.critical_mismatch then 'automated research: technical mismatch'
|
||||
when d.missing_evidence then 'automated research: insufficient technical evidence'
|
||||
when d.confidence >= 0.73 then null
|
||||
else 'automated research: confidence below approval threshold'
|
||||
end,
|
||||
re_research_due_at = case
|
||||
when (not d.no_price) and (not d.critical_mismatch) and d.missing_evidence and d.confidence < 0.73
|
||||
then now() + interval '21 days'
|
||||
else null
|
||||
end,
|
||||
re_researched_at = now(),
|
||||
match_notes = case when eq.match_notes is null or eq.match_notes = ''
|
||||
then 'bulk-triage(139) ' || now()::date
|
||||
else eq.match_notes || ' | bulk-triage(139) ' || now()::date end,
|
||||
updated_at = now()
|
||||
from decided d
|
||||
where eq.id = d.id;
|
||||
|
||||
-- Mirror the daily job's side effect on approve: mark the Flexoptix product as
|
||||
-- competitor_verified (matches scheduler.ts's confirm branch).
|
||||
update transceivers t
|
||||
set competitor_verified = true,
|
||||
competitor_verified_at = coalesce(t.competitor_verified_at, now())
|
||||
from transceiver_equivalences eq
|
||||
where eq.flexoptix_id = t.id
|
||||
and eq.status = 'auto_approved'
|
||||
and eq.reviewed_by = 'automated-research-bulk-139'
|
||||
and eq.reviewed_at > now() - interval '5 minutes'
|
||||
and t.competitor_verified = false;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '139-equivalence-pending-triage.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '139-equivalence-pending-triage.sql');
|
||||
@ -1,99 +0,0 @@
|
||||
-- sql/140-spec-matcher-reclassify.sql
|
||||
-- One-time reclassification of the 1,455 equivalences that spec-matcher.ts had
|
||||
-- inserted straight into status='auto_approved' at a flat confidence=0.85,
|
||||
-- bypassing the discrimination (standard_name, fiber_type, recent-competitor-
|
||||
-- price) that catalog-reconcile.ts's calcConfidence() applies before ever
|
||||
-- auto-approving a pending row. Same set-based scoring port as sql/139, applied
|
||||
-- here to the pre-existing 'spec'-basis auto_approved rows instead of pending
|
||||
-- ones. Paired with a code fix (commit e7299ac7) so spec-matcher.ts inserts
|
||||
-- 'pending' going forward and routes through the same reviewed pipeline.
|
||||
--
|
||||
-- OPN-basis rows (manufacturer-confirmed, fx_compatibilities ground truth) are
|
||||
-- NOT touched -- this only reclassifies 'spec'-basis rows.
|
||||
|
||||
create table if not exists spec_matcher_reclassify_backup_20260717 as
|
||||
select *, now() as backed_up_at
|
||||
from transceiver_equivalences
|
||||
where 'spec' = any(match_basis) and status = 'auto_approved';
|
||||
|
||||
with normalized as (
|
||||
select eq.id,
|
||||
nullif(upper(trim(fx.form_factor)),'') as fx_form, nullif(upper(trim(cp.form_factor)),'') as cp_form,
|
||||
fx.speed_gbps as fx_speed, cp.speed_gbps as cp_speed,
|
||||
nullif(upper(trim(fx.standard_name)),'') as fx_std, nullif(upper(trim(cp.standard_name)),'') as cp_std,
|
||||
nullif(upper(trim(fx.fiber_type)),'') as fx_fiber, nullif(upper(trim(cp.fiber_type)),'') as cp_fiber,
|
||||
fx.reach_meters as fx_reach, cp.reach_meters as cp_reach,
|
||||
(regexp_match(fx.wavelengths, '(\d{3,4})'))[1]::numeric as fx_nm,
|
||||
(regexp_match(cp.wavelengths, '(\d{3,4})'))[1]::numeric as cp_nm,
|
||||
exists(select 1 from price_observations po where po.transceiver_id=eq.competitor_id and po.time > now() - interval '45 days') as has_recent_price
|
||||
from transceiver_equivalences eq
|
||||
join transceivers fx on fx.id = eq.flexoptix_id
|
||||
join transceivers cp on cp.id = eq.competitor_id
|
||||
where 'spec' = any(eq.match_basis) and eq.status = 'auto_approved'
|
||||
),
|
||||
scored as (
|
||||
select *,
|
||||
(case when fx_form is not null and cp_form is not null and fx_form=cp_form then 25 else 0 end
|
||||
+ case when fx_speed is not null and cp_speed is not null and fx_speed=cp_speed then 20 else 0 end
|
||||
+ case when fx_std is not null and cp_std is not null and fx_std=cp_std then 30 else 0 end
|
||||
+ case when fx_nm is not null and cp_nm is not null then
|
||||
case when abs(fx_nm-cp_nm)<=15 then 20 else -20 end
|
||||
else 0 end
|
||||
+ case when fx_fiber is not null and cp_fiber is not null then
|
||||
case when fx_fiber=cp_fiber then 10 else -15 end
|
||||
else 0 end
|
||||
+ case when fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 then
|
||||
case when least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) >= 0.85 then 10 else -15 end
|
||||
else 0 end) as raw_score,
|
||||
(fx_nm is null or cp_nm is null) as wl_missing,
|
||||
(fx_fiber is null or cp_fiber is null) as fiber_missing,
|
||||
(fx_reach is null or cp_reach is null or fx_reach<=0 or cp_reach<=0) as reach_missing,
|
||||
(fx_nm is not null and cp_nm is not null and abs(fx_nm-cp_nm)>15) as wl_mismatch,
|
||||
(fx_fiber is not null and cp_fiber is not null and fx_fiber<>cp_fiber) as fiber_mismatch,
|
||||
(fx_reach is not null and cp_reach is not null and fx_reach>0 and cp_reach>0 and least(fx_reach,cp_reach)/greatest(fx_reach,cp_reach) < 0.85) as reach_mismatch,
|
||||
(fx_form is null or cp_form is null or fx_form<>cp_form) as form_bad,
|
||||
(fx_speed is null or cp_speed is null or fx_speed<>cp_speed) as speed_bad
|
||||
from normalized
|
||||
),
|
||||
decided as (
|
||||
select *,
|
||||
greatest(0, least(1, raw_score::numeric/115)) as confidence,
|
||||
(not has_recent_price) as no_price,
|
||||
(wl_mismatch or fiber_mismatch or reach_mismatch or form_bad or speed_bad) as critical_mismatch,
|
||||
(wl_missing or fiber_missing or reach_missing) as missing_evidence
|
||||
from scored
|
||||
)
|
||||
update transceiver_equivalences eq
|
||||
set status = case
|
||||
when d.no_price then 'rejected'
|
||||
when d.critical_mismatch then 'rejected'
|
||||
when d.missing_evidence then 'rejected'
|
||||
when d.confidence >= 0.73 then 'auto_approved'
|
||||
else 'rejected'
|
||||
end,
|
||||
confidence = case when d.no_price then 0 else d.confidence end,
|
||||
reviewed_by = 'automated-research-bulk-140',
|
||||
reviewed_at = now(),
|
||||
reject_reason = case
|
||||
when d.no_price then 'automated research: competitor has no recent price observation'
|
||||
when d.critical_mismatch then 'automated research: technical mismatch'
|
||||
when d.missing_evidence then 'automated research: insufficient technical evidence'
|
||||
when d.confidence >= 0.73 then null
|
||||
else 'automated research: confidence below approval threshold'
|
||||
end,
|
||||
re_research_due_at = case
|
||||
when (not d.no_price) and (not d.critical_mismatch) and d.missing_evidence and d.confidence < 0.73
|
||||
then now() + interval '21 days'
|
||||
else null
|
||||
end,
|
||||
re_researched_at = now(),
|
||||
match_notes = case when eq.match_notes is null or eq.match_notes = ''
|
||||
then 'bulk-reclassify(140) ' || now()::date
|
||||
else eq.match_notes || ' | bulk-reclassify(140) ' || now()::date end,
|
||||
updated_at = now()
|
||||
from decided d
|
||||
where eq.id = d.id;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '140-spec-matcher-reclassify.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '140-spec-matcher-reclassify.sql');
|
||||
@ -1,70 +0,0 @@
|
||||
-- sql/141-switch-speed-drift-guard.sql
|
||||
-- Preventive close of the THIRD direction of the same drift-gap class fixed in
|
||||
-- sql/126 (compat_speed_guard, guards compatibility writes) and sql/138
|
||||
-- (compat_speed_guard_on_transceiver_change, guards transceivers.speed_gbps
|
||||
-- changes). The "no physically-impossible compatibility rows" invariant is
|
||||
-- derived from THREE tables' data: compatibility.status, transceivers.speed_gbps,
|
||||
-- and switches.max_speed_gbps/ports_config (via v_switch_effective_max). Only
|
||||
-- two of those three write paths were guarded.
|
||||
--
|
||||
-- Audit 2026-07-17 found switches.max_speed_gbps/ports_config are currently
|
||||
-- write-once (set at scrape/seed time, sql/125's one-time backfill) — no live
|
||||
-- job updates them today, so this direction is NOT actively exploited yet. But
|
||||
-- the documented open item "110 switches with understated ports_config need
|
||||
-- datasheet enrichment" ([[tip-switch-database-audit-2026-07-06]]) describes
|
||||
-- exactly the future job that WOULD trigger this gap: correcting a switch's
|
||||
-- max_speed_gbps or ports_config downward after compat rows already exist would
|
||||
-- silently leave now-impossible rows marked 'compatible', same failure mode as
|
||||
-- sql/138, just from the other table. Closing it now, before that automation
|
||||
-- ships, rather than waiting for the next backlog surprise.
|
||||
--
|
||||
-- Same "no false positives" design as 126/138: only algorithmic (non-curated)
|
||||
-- rows are ever forced to incompatible; curated rows are trusted as-is.
|
||||
|
||||
create or replace function compat_speed_guard_on_switch_change() returns trigger
|
||||
language plpgsql as $$
|
||||
declare
|
||||
new_ceiling numeric;
|
||||
begin
|
||||
if NEW.max_speed_gbps is distinct from OLD.max_speed_gbps
|
||||
or NEW.ports_config is distinct from OLD.ports_config then
|
||||
select greatest(
|
||||
coalesce(NEW.max_speed_gbps, 0),
|
||||
coalesce(fn_ports_max_gbps(NEW.ports_config), 0),
|
||||
coalesce((select max(t.speed_gbps)
|
||||
from compatibility c join transceivers t on t.id = c.transceiver_id
|
||||
where c.switch_id = NEW.id and c.status = 'compatible'
|
||||
and c.verification_method in ('vendor_matrix','vendor_compat')
|
||||
and t.speed_gbps > 0), 0)
|
||||
)
|
||||
into new_ceiling;
|
||||
|
||||
if new_ceiling > 0 then
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(141): transceiver speed exceeds switch max port speed (post-hoc drift)'
|
||||
else c.notes || ' | auto(141): transceiver speed exceeds switch max port speed (post-hoc drift)' end
|
||||
from transceivers t
|
||||
where t.id = c.transceiver_id
|
||||
and c.switch_id = NEW.id
|
||||
and c.status = 'compatible'
|
||||
and c.verification_method = 'spec_match'
|
||||
and t.speed_gbps is not null
|
||||
and t.speed_gbps > new_ceiling;
|
||||
end if;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function compat_speed_guard_on_switch_change() is
|
||||
'AFTER UPDATE OF max_speed_gbps/ports_config on switches: re-validates existing compatible spec_match compatibility rows for this switch against the (re-derived) evidence-based ceiling. Third leg of the compat_speed_guard family (126=compatibility writes, 138=transceiver speed changes, this=switch capacity changes). Never touches curated rows. Currently a no-op in practice -- nothing updates these columns live yet -- this is preventive, closing the gap before the planned switch datasheet-enrichment automation ships.';
|
||||
|
||||
drop trigger if exists trg_compat_speed_guard_on_switch_change on switches;
|
||||
create trigger trg_compat_speed_guard_on_switch_change
|
||||
after update of max_speed_gbps, ports_config on switches
|
||||
for each row execute function compat_speed_guard_on_switch_change();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '141-switch-speed-drift-guard.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '141-switch-speed-drift-guard.sql');
|
||||
@ -1,44 +0,0 @@
|
||||
-- sql/142-speed-recovery-rerun.sql
|
||||
-- Re-run of sql/125's Remediation C (part-number speed token recovery) against
|
||||
-- the current genuine-optics speed gap (v_transceiver_speed_errors, 596 rows as
|
||||
-- of 2026-07-17). Same exact logic, same high-confidence-only design: derives
|
||||
-- speed_gbps from the manufacturer's standardized speed token in the part
|
||||
-- number, applied ONLY when it is <= the form factor's physical maximum. Parts
|
||||
-- whose token exceeds the form-factor cap or that have no speed token are left
|
||||
-- for datasheet work (596 -> ~248 after this, per the 2026-07-17 gap analysis:
|
||||
-- 348/596 had a regex-recoverable G-token, mostly Cisco Systems + Juniper).
|
||||
-- Idempotent, safe to re-run (no-ops on rows that already have a plausible
|
||||
-- speed).
|
||||
|
||||
with cand as (
|
||||
select id,
|
||||
case
|
||||
when part_number ~* '([0-9]+)X([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)X([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)SFP([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[1]::numeric
|
||||
* (regexp_match(part_number,'([0-9]+)SFP([0-9]+)G','i'))[2]::numeric
|
||||
when part_number ~* '([0-9]+)G'
|
||||
then (regexp_match(part_number,'([0-9]+)G','i'))[1]::numeric
|
||||
end as derived,
|
||||
case upper(coalesce(form_factor,''))
|
||||
when 'SFP' then 1 when 'SFP+' then 16 when 'SFP28' then 32 when 'SFP56' then 56
|
||||
when 'XFP' then 10 when 'QSFP+' then 40 when 'QSFP28' then 100 when 'QSFP56' then 200
|
||||
when 'QSFP-DD' then 800 when 'QSFP-DD800' then 800
|
||||
when 'OSFP' then 1600 when 'OSFP224' then 1600
|
||||
when 'CFP' then 100 when 'CFP2' then 400 when 'CFP4' then 100 when 'CPAK' then 100
|
||||
else 99999 end as ff_cap
|
||||
from transceivers
|
||||
where id in (select id from v_transceiver_speed_errors))
|
||||
update transceivers t
|
||||
set speed_gbps = c.derived,
|
||||
updated_at = now()
|
||||
from cand c
|
||||
where t.id = c.id
|
||||
and c.derived is not null and c.derived > 0
|
||||
and c.derived <= c.ff_cap;
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '142-speed-recovery-rerun.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '142-speed-recovery-rerun.sql');
|
||||
@ -1,87 +0,0 @@
|
||||
-- sql/143-vendor-compat-template-fill.sql
|
||||
-- Fills transceivers.vendor_compat (Flexoptix OEM-coding-pattern reference data,
|
||||
-- feeds v_coding_coverage / check_switch_quality's "coding matrix" metric) for
|
||||
-- Flexoptix products that are missing it.
|
||||
--
|
||||
-- Investigated 2026-07-17 (Rene: "die sind in der API alle zum auslesen drinnen"):
|
||||
-- vendor_compat is NOT derived per-SKU from fx_compatibilities (found rows with
|
||||
-- vendor_compat fully populated but fx_compatibilities completely empty, and vice
|
||||
-- versa -- no correlation). It is a SHARED REFERENCE TEMPLATE keyed by
|
||||
-- (speed_gbps, form_factor): only 11 distinct templates exist across the entire
|
||||
-- Flexoptix catalog, and every (speed, form_factor) family maps to exactly one
|
||||
-- template except 100G/SFP (2 templates -- ambiguous, deliberately excluded
|
||||
-- below). This was clearly seeded once (likely seed-from-npm.ts) for a subset of
|
||||
-- products and never propagated to new SKUs added since -- 167 of 192 missing
|
||||
-- rows (87%) have an unambiguous sibling template available to copy.
|
||||
--
|
||||
-- Fill-only, idempotent, no-false-positive design matching this project's other
|
||||
-- migrations: only touches rows with an EMPTY vendor_compat, only copies a
|
||||
-- template when exactly one distinct template exists for that (speed,
|
||||
-- form_factor) family among Flexoptix's own catalog, never invents new data.
|
||||
-- Competitor vendors are explicitly OUT OF SCOPE here -- see accompanying memory
|
||||
-- for why blind propagation to competitor products needs a separate, more
|
||||
-- careful pass.
|
||||
|
||||
with templates as (
|
||||
select speed_gbps, form_factor, (array_agg(vendor_compat))[1] as template
|
||||
from transceivers t join vendors v on v.id = t.vendor_id
|
||||
where v.name = 'Flexoptix' and jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) > 0
|
||||
group by speed_gbps, form_factor
|
||||
having count(distinct vendor_compat::text) = 1
|
||||
)
|
||||
update transceivers t
|
||||
set vendor_compat = tpl.template,
|
||||
updated_at = now()
|
||||
from vendors v, templates tpl
|
||||
where v.id = t.vendor_id
|
||||
and v.name = 'Flexoptix'
|
||||
and tpl.speed_gbps = t.speed_gbps
|
||||
and tpl.form_factor = t.form_factor
|
||||
and jsonb_array_length(coalesce(t.vendor_compat,'[]'::jsonb)) = 0;
|
||||
|
||||
-- Durable fix: new/updated Flexoptix products inherit the family template
|
||||
-- automatically once speed_gbps + form_factor are known (i.e. after the detail
|
||||
-- enricher runs), instead of relying on another one-off backfill.
|
||||
create or replace function tip_fill_vendor_compat_from_family() returns trigger
|
||||
language plpgsql as $$
|
||||
declare
|
||||
fx_vendor_id uuid;
|
||||
tpl jsonb;
|
||||
tpl_count int;
|
||||
begin
|
||||
if jsonb_array_length(coalesce(NEW.vendor_compat,'[]'::jsonb)) > 0
|
||||
or NEW.speed_gbps is null or NEW.form_factor is null then
|
||||
return NEW;
|
||||
end if;
|
||||
|
||||
select id into fx_vendor_id from vendors where upper(name) like '%FLEXOPTIX%' limit 1;
|
||||
if NEW.vendor_id is distinct from fx_vendor_id then
|
||||
return NEW;
|
||||
end if;
|
||||
|
||||
select count(distinct vendor_compat::text), (array_agg(vendor_compat))[1]
|
||||
into tpl_count, tpl
|
||||
from transceivers
|
||||
where vendor_id = fx_vendor_id
|
||||
and speed_gbps = NEW.speed_gbps
|
||||
and form_factor = NEW.form_factor
|
||||
and jsonb_array_length(coalesce(vendor_compat,'[]'::jsonb)) > 0
|
||||
and id is distinct from NEW.id;
|
||||
|
||||
if tpl_count = 1 then
|
||||
NEW.vendor_compat := tpl;
|
||||
end if;
|
||||
return NEW;
|
||||
end $$;
|
||||
|
||||
comment on function tip_fill_vendor_compat_from_family() is
|
||||
'BEFORE INSERT/UPDATE on transceivers: for Flexoptix products with empty vendor_compat, copies the OEM-coding-pattern template from an existing sibling of the same (speed_gbps, form_factor) family, if exactly one distinct template exists for that family. Never invents a template, never overwrites existing data, skips ambiguous families (>1 template).';
|
||||
|
||||
drop trigger if exists trg_fill_vendor_compat_from_family on transceivers;
|
||||
create trigger trg_fill_vendor_compat_from_family
|
||||
before insert or update of speed_gbps, form_factor, vendor_compat on transceivers
|
||||
for each row execute function tip_fill_vendor_compat_from_family();
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '143-vendor-compat-template-fill.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '143-vendor-compat-template-fill.sql');
|
||||
@ -1,49 +0,0 @@
|
||||
-- sql/144-fix-false-curated-10g-compat.sql
|
||||
-- Manual, datasheet-verified correction of 14 CURATED (verification_method=
|
||||
-- 'vendor_compat') compatibility rows that claim 10G transceivers are
|
||||
-- compatible with switches whose ports are physically 1G-only.
|
||||
--
|
||||
-- The automated guards (126/138/141) deliberately never touch curated rows --
|
||||
-- on understated switches, curated data is normally the source of truth that
|
||||
-- RAISES the ceiling. These 14 rows are the documented exception: verified
|
||||
-- 2026-07-18 against Cisco's own primary documentation:
|
||||
-- * Cisco ASR 9000 Transceiver Module Support Matrix (C78-624747-06):
|
||||
-- A9K-40GE-L/B/E -- SFP=Yes, XFP=No, SFP+=No, QSFP=No, CFP=No. Supported
|
||||
-- SFP lists are exclusively 100M/1G-class (1000BASE SFP, CWDM "Gigabit
|
||||
-- Ethernet and 1G/2G FC", DWDM 1000BASE). No 10G module type accepted.
|
||||
-- A9K-40GE-TR is the packet-transport variant of the same 40-Port GE
|
||||
-- family (SFP cages only).
|
||||
-- * Cisco Catalyst 9300 Series Data Sheet (nb-06-cat9300-ser-...):
|
||||
-- "C9300-48S 48 ports 1G SFP Modular uplinks", "C9300-24S 24 ports 1G SFP
|
||||
-- Modular uplinks" -- fixed downlinks are 1G; 10G exists only on the
|
||||
-- separate, optional uplink network module, which TIP tracks as its own
|
||||
-- switch entity (e.g. C9300-NM-8X).
|
||||
-- * C9400-LC-48S / C9400-LC-24S / C9600-LC-48S: "48-port 1G SFP line card"
|
||||
-- per Cisco 9400/9600 line-card documentation (C9600-LC-48S slot
|
||||
-- bandwidth 96 Gbps full-duplex = 48x1G).
|
||||
--
|
||||
-- Likely origin of the bad rows: flexoptix-compat.ts (vendor_compat method)
|
||||
-- matching 10G Flexoptix parts (P.1696.23.yy, X.1696.23.yy, P.1596.40:Sx,
|
||||
-- P.Czz.z) to these switch records via the configurator supported-vendors
|
||||
-- family match -- same scraper lane that produced the sql/125 spec_match
|
||||
-- cleanup, this time in the curated tier.
|
||||
--
|
||||
-- Effect: v_switch_understated_ports drops to 0 (its rows were exactly these
|
||||
-- curated-faster-than-ports claims), removing the last "understated ports"
|
||||
-- alert class for these switches. Idempotent.
|
||||
|
||||
create table if not exists curated_compat_10g_on_1g_backup_20260718 as
|
||||
select c.*, now() as backed_up_at
|
||||
from compatibility c
|
||||
where c.id in (select compat_id from v_switch_understated_ports);
|
||||
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'manual(144): 10G transceiver on 1G-only switch; verified against Cisco ASR9000 transceiver support matrix (C78-624747-06) and Catalyst 9300/9400/9600 datasheets 2026-07-18'
|
||||
else c.notes || ' | manual(144): 10G transceiver on 1G-only switch; verified against Cisco primary datasheets 2026-07-18' end
|
||||
where c.id in (select compat_id from v_switch_understated_ports);
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '144-fix-false-curated-10g-compat.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '144-fix-false-curated-10g-compat.sql');
|
||||
@ -1,25 +0,0 @@
|
||||
-- sql/145-unmasked-spec-match-cleanup.sql
|
||||
-- Follow-up to sql/144: the 14 false curated 10G-on-1G rows had been RAISING
|
||||
-- the evidence-based ceiling (v_switch_effective_max takes the fastest curated
|
||||
-- compatible transceiver as evidence) for those 1G-only switches, thereby
|
||||
-- MASKING every algorithmic spec_match row claiming >1G parts on the same
|
||||
-- switches. With the curated rows corrected, the ceiling is honestly 1G again
|
||||
-- and v_compat_speed_errors correctly flags the previously-hidden rows (261
|
||||
-- found live 2026-07-18). Same remediation as sql/125-B/138: flip algorithmic
|
||||
-- rows exceeding the ceiling to incompatible. Backup first. Idempotent.
|
||||
|
||||
create table if not exists unmasked_spec_match_backup_20260718 as
|
||||
select c.*, now() as backed_up_at
|
||||
from compatibility c
|
||||
where c.id in (select compat_id from v_compat_speed_errors);
|
||||
|
||||
update compatibility c
|
||||
set status = 'incompatible',
|
||||
notes = case when c.notes is null or c.notes = ''
|
||||
then 'auto(145): transceiver speed exceeds switch max port speed (unmasked by 144 curated-row correction)'
|
||||
else c.notes || ' | auto(145): transceiver speed exceeds switch max port speed (unmasked by 144)' end
|
||||
where c.id in (select compat_id from v_compat_speed_errors);
|
||||
|
||||
insert into _migrations (filename, applied_at)
|
||||
select '145-unmasked-spec-match-cleanup.sql', now()
|
||||
where not exists (select 1 from _migrations where filename = '145-unmasked-spec-match-cleanup.sql');
|
||||
11535
training-data/runpod/blog_llm/blog_llm-sft-all.jsonl
Normal file
11535
training-data/runpod/blog_llm/blog_llm-sft-all.jsonl
Normal file
File diff suppressed because one or more lines are too long
1154
training-data/runpod/blog_llm/blog_llm-sft-eval.jsonl
Normal file
1154
training-data/runpod/blog_llm/blog_llm-sft-eval.jsonl
Normal file
File diff suppressed because one or more lines are too long
10381
training-data/runpod/blog_llm/blog_llm-sft-train.jsonl
Normal file
10381
training-data/runpod/blog_llm/blog_llm-sft-train.jsonl
Normal file
File diff suppressed because one or more lines are too long
11999
training-data/runpod/tip_llm/tip_llm-sft-all.jsonl
Normal file
11999
training-data/runpod/tip_llm/tip_llm-sft-all.jsonl
Normal file
File diff suppressed because one or more lines are too long
1200
training-data/runpod/tip_llm/tip_llm-sft-eval.jsonl
Normal file
1200
training-data/runpod/tip_llm/tip_llm-sft-eval.jsonl
Normal file
File diff suppressed because one or more lines are too long
10799
training-data/runpod/tip_llm/tip_llm-sft-train.jsonl
Normal file
10799
training-data/runpod/tip_llm/tip_llm-sft-train.jsonl
Normal file
File diff suppressed because one or more lines are too long
69
training-data/tip-llm-capabilities-v1.jsonl
Normal file
69
training-data/tip-llm-capabilities-v1.jsonl
Normal file
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user