llm-gateway/packages/gateway/src/db/migrations/002-tokenvault-cost-tracking.sql

-- Migration: Add Tokenvault & Cost Tracking Tables
-- Created: 2026-04-19
-- Purpose: Track token compression and cost analytics

-- Enable JSON extension if not already enabled
CREATE EXTENSION IF NOT EXISTS json;

-- Table: Token compression metrics (LeanCTX, RTK)
CREATE TABLE IF NOT EXISTS tokenvault_metrics (
  id SERIAL PRIMARY KEY,
  file_path VARCHAR(255),
  mode VARCHAR(50),
  tokens_before INT NOT NULL,
  tokens_after INT NOT NULL,
  savings_pct DECIMAL(5,2) NOT NULL,
  tool_used VARCHAR(50) NOT NULL,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_tool_created (tool_used, created_at),
  INDEX idx_created (created_at)
);

-- Table: Cost analytics per task
CREATE TABLE IF NOT EXISTS cost_analytics (
  id SERIAL PRIMARY KEY,
  call_id VARCHAR(50),
  project VARCHAR(100),
  task_type VARCHAR(50),
  model VARCHAR(100),
  agent_id VARCHAR(50),
  tokens_in INT NOT NULL DEFAULT 0,
  tokens_out INT NOT NULL DEFAULT 0,
  tokens_compressed INT NOT NULL DEFAULT 0,
  cost_usd DECIMAL(10,6) NOT NULL DEFAULT 0,
  cost_saved_usd DECIMAL(10,6) NOT NULL DEFAULT 0,
  provider VARCHAR(50),
  confidence_score DECIMAL(3,2),
  request_hash VARCHAR(64),
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_project_created (project, created_at),
  INDEX idx_agent_created (agent_id, created_at),
  INDEX idx_model_created (model, created_at),
  INDEX idx_call_id (call_id)
);

-- Table: Compression savings summary (daily aggregate)
CREATE TABLE IF NOT EXISTS compression_summary (
  id SERIAL PRIMARY KEY,
  date DATE NOT NULL,
  tool VARCHAR(50) NOT NULL,
  total_tokens_before INT NOT NULL,
  total_tokens_after INT NOT NULL,
  total_savings_pct DECIMAL(5,2),
  count INT NOT NULL,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  UNIQUE KEY unique_date_tool (date, tool)
);

-- Table: Cost alerts configuration
CREATE TABLE IF NOT EXISTS cost_alert_config (
  id SERIAL PRIMARY KEY,
  user_id VARCHAR(100),
  project VARCHAR(100),
  alert_type VARCHAR(50),
  threshold DECIMAL(8,2),
  threshold_type VARCHAR(20),
  enabled BOOLEAN DEFAULT TRUE,
  weekly_budget_usd DECIMAL(10,2),
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
);

-- Table: Alert history
CREATE TABLE IF NOT EXISTS alert_log (
  id SERIAL PRIMARY KEY,
  alert_type VARCHAR(50),
  severity VARCHAR(20),
  message TEXT,
  metadata JSON,
  acknowledged BOOLEAN DEFAULT FALSE,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_severity_created (severity, created_at),
  INDEX idx_created (created_at)
);

-- Create additional indexes
CREATE INDEX IF NOT EXISTS idx_cost_analytics_week
  ON cost_analytics(created_at DESC)
  WHERE created_at > DATE_SUB(NOW(), INTERVAL 7 DAY);

CREATE INDEX IF NOT EXISTS idx_compression_daily
  ON tokenvault_metrics(created_at DESC)
  WHERE created_at > DATE_SUB(NOW(), INTERVAL 1 DAY);

-- Add cost tracking to batch_jobs table
ALTER TABLE batch_jobs
  ADD COLUMN IF NOT EXISTS total_cost_usd DECIMAL(10,6) NOT NULL DEFAULT 0,
  ADD COLUMN IF NOT EXISTS total_saved_usd DECIMAL(10,6) NOT NULL DEFAULT 0,
  ADD INDEX idx_batch_jobs_cost_created (total_cost_usd, created_at DESC);

-- Table: Fallback chain execution metrics
CREATE TABLE IF NOT EXISTS fallback_chain_metrics (
  id SERIAL PRIMARY KEY,
  call_id VARCHAR(50),
  task_type VARCHAR(50),
  primary_model VARCHAR(100) NOT NULL,
  fallback_step INT NOT NULL,
  fallback_model VARCHAR(100),
  provider VARCHAR(50),
  success BOOLEAN NOT NULL,
  tokens_in INT NOT NULL DEFAULT 0,
  tokens_out INT NOT NULL DEFAULT 0,
  latency_ms INT NOT NULL,
  reason_switched VARCHAR(50),
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_call_created (call_id, created_at),
  INDEX idx_model_created (primary_model, created_at),
  INDEX idx_task_created (task_type, created_at)
);

-- Table: Model performance metrics (for learning engine)
CREATE TABLE IF NOT EXISTS model_performance (
  id SERIAL PRIMARY KEY,
  model VARCHAR(100) NOT NULL,
  task_type VARCHAR(50),
  success_rate DECIMAL(5,2),
  avg_latency_ms INT,
  avg_tokens_in INT,
  avg_tokens_out INT,
  total_calls INT DEFAULT 0,
  total_failures INT DEFAULT 0,
  confidence_avg DECIMAL(3,2),
  last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  UNIQUE KEY unique_model_task (model, task_type),
  INDEX idx_success_rate (success_rate DESC),
  INDEX idx_latency (avg_latency_ms ASC)
);

-- Table: Learning engine cycle logs
CREATE TABLE IF NOT EXISTS learning_cycles (
  id SERIAL PRIMARY KEY,
  cycle_id VARCHAR(50) NOT NULL UNIQUE,
  cycle_duration VARCHAR(20),
  improvements_found INT DEFAULT 0,
  routing_changes INT DEFAULT 0,
  template_updates INT DEFAULT 0,
  model_rankings_updated BOOLEAN DEFAULT FALSE,
  confidence_threshold_adjusted DECIMAL(3,2),
  metrics JSON,
  status VARCHAR(50),
  started_at TIMESTAMP,
  completed_at TIMESTAMP,
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_cycle_duration (cycle_duration, completed_at DESC),
  INDEX idx_status (status),
  INDEX idx_completed (completed_at DESC)
);

-- Table: Routing decisions and performance
CREATE TABLE IF NOT EXISTS routing_decisions (
  id SERIAL PRIMARY KEY,
  call_id VARCHAR(50),
  task_type VARCHAR(50) NOT NULL,
  caller VARCHAR(100),
  routing_model VARCHAR(100) NOT NULL,
  routing_tier VARCHAR(20),
  actual_model_used VARCHAR(100),
  was_fallback BOOLEAN DEFAULT FALSE,
  success BOOLEAN NOT NULL,
  confidence_final DECIMAL(3,2),
  tokens_in INT,
  tokens_out INT,
  latency_ms INT,
  cost_usd DECIMAL(10,6),
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX idx_task_created (task_type, created_at),
  INDEX idx_routing_model (routing_model, created_at),
  INDEX idx_success (success, created_at)
);

-- Initialize default alert config for Rene
INSERT INTO cost_alert_config
  (user_id, alert_type, threshold, threshold_type, enabled, weekly_budget_usd)
VALUES
  ('rene', 'compression_below', 40, 'percent', TRUE, 50),
  ('rene', 'external_api', 0, 'usd', TRUE, NULL),
  ('rene', 'weekly_budget', 50, 'usd', TRUE, 50)
ON DUPLICATE KEY UPDATE enabled = VALUES(enabled);