Compare commits
6 Commits
92a2d41fb1
...
6275c25e67
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6275c25e67 | ||
|
|
d265320819 | ||
|
|
812b695e1a | ||
|
|
a5f07a9ef8 | ||
|
|
a6547eda2f | ||
|
|
421926768a |
1
.gitignore
vendored
1
.gitignore
vendored
@ -17,3 +17,4 @@ ecosystem.config.js.bak-*
|
||||
|
||||
# Codex/editor backup directories
|
||||
.codex-backup-*/
|
||||
.serena/
|
||||
|
||||
@ -36,7 +36,7 @@ await adapter.start()
|
||||
import OpenAI from 'openai'
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: 'not-needed',
|
||||
apiKey: '<key>',
|
||||
baseURL: 'http://localhost:3111/v1'
|
||||
})
|
||||
|
||||
|
||||
@ -2,21 +2,24 @@
|
||||
"""Load blog-training-alpaca.jsonl into PostgreSQL learning_corpus table via psql."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
JSONL_FILE = Path(__file__).parent.parent / "data" / "blog-training-alpaca.jsonl"
|
||||
DB_URL = "postgresql://llm:llm_secure_2026@127.0.0.1:15432/llm_gateway"
|
||||
DB_URL = os.environ.get("FT_DB_URL", "postgresql://llm@localhost:15432/llm_gateway")
|
||||
PSQL_BIN = os.environ.get("PSQL_BIN", "psql")
|
||||
TASK_TYPE = "tip_blog"
|
||||
|
||||
# Parse connection details
|
||||
parts = DB_URL.replace("postgresql://", "").split("@")
|
||||
creds = parts[0].split(":")
|
||||
db_user, db_pass = creds[0], creds[1]
|
||||
host_port = parts[1].split("/")
|
||||
host, port = host_port[0].split(":")
|
||||
db_name = host_port[1]
|
||||
parsed = urlparse(DB_URL)
|
||||
db_user = parsed.username or "llm"
|
||||
db_pass = parsed.password or os.environ.get("PGPASSWORD")
|
||||
host = parsed.hostname or "localhost"
|
||||
port = str(parsed.port or 5432)
|
||||
db_name = parsed.path.lstrip("/") or "llm_gateway"
|
||||
|
||||
print("🔄 Loading blog training data into PostgreSQL...")
|
||||
print(f" File: {JSONL_FILE}")
|
||||
@ -66,10 +69,12 @@ with open(tmpfile, "w") as f:
|
||||
|
||||
# Execute via psql
|
||||
try:
|
||||
env = {"PGPASSWORD": db_pass}
|
||||
env = os.environ.copy()
|
||||
if db_pass:
|
||||
env["PGPASSWORD"] = db_pass
|
||||
result = subprocess.run(
|
||||
[
|
||||
"/opt/homebrew/bin/psql",
|
||||
PSQL_BIN,
|
||||
"-h", host,
|
||||
"-p", port,
|
||||
"-U", db_user,
|
||||
|
||||
@ -1,16 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Load blog-training-alpaca.jsonl into PostgreSQL learning_corpus table
|
||||
|
||||
JSONL_FILE="/Users/renefichtmueller/Desktop/Claude Code/llm-gateway/packages/fine-tuner/data/blog-training-alpaca.jsonl"
|
||||
DB_URL="postgresql://llm:llm_secure_2026@127.0.0.1:15432/llm_gateway"
|
||||
JSONL_FILE="${JSONL_FILE:-$(cd "$(dirname "$0")/.." && pwd)/data/blog-training-alpaca.jsonl}"
|
||||
DB_HOST="${DB_HOST:-localhost}"
|
||||
DB_PORT="${DB_PORT:-15432}"
|
||||
DB_NAME="${DB_NAME:-llm_gateway}"
|
||||
DB_USER="${DB_USER:-llm}"
|
||||
TASK_TYPE="tip_blog"
|
||||
|
||||
# Parse connection string
|
||||
DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
||||
DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
||||
DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\(.*\)$/\1/p')
|
||||
DB_USER="llm"
|
||||
|
||||
echo "🔄 Loading blog training data into PostgreSQL..."
|
||||
echo " Host: $DB_HOST:$DB_PORT"
|
||||
echo " Database: $DB_NAME"
|
||||
@ -29,7 +26,7 @@ SELECT COUNT(*) as "Rows before" FROM learning_corpus WHERE task_type = :task_ty
|
||||
EOF
|
||||
|
||||
# Use Python to parse JSONL and generate SQL
|
||||
python3 << PYEOF
|
||||
python3 << PYEOF >> "$TMPFILE"
|
||||
import json
|
||||
import sys
|
||||
|
||||
@ -63,10 +60,12 @@ print("\n-- Count after")
|
||||
print(f"SELECT COUNT(*) as \"Rows after\" FROM learning_corpus WHERE task_type = '{task_type}';")
|
||||
print("COMMIT;")
|
||||
|
||||
PYEOF >> "$TMPFILE"
|
||||
PYEOF
|
||||
|
||||
# Execute SQL
|
||||
PGPASSWORD="llm_secure_2026" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$TMPFILE"
|
||||
: "${PGPASSWORD:?Set PGPASSWORD before running this script}"
|
||||
export PGPASSWORD
|
||||
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$TMPFILE"
|
||||
|
||||
# Cleanup
|
||||
rm "$TMPFILE"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
# start.sh — Launch fine-tuner with SSH tunnel to Erik's PostgreSQL
|
||||
# start.sh — Launch fine-tuner with SSH tunnel to PostgreSQL
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/start.sh # run fine-tuner
|
||||
@ -9,11 +9,11 @@
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
TUNNEL_PORT=5434
|
||||
SSH_HOST="erik"
|
||||
REMOTE_PG_PORT=5432
|
||||
TUNNEL_PORT="${TUNNEL_PORT:-5434}"
|
||||
SSH_HOST="${FT_SSH_HOST:?Set FT_SSH_HOST before running this script}"
|
||||
REMOTE_PG_PORT="${REMOTE_PG_PORT:-5432}"
|
||||
|
||||
echo "[start.sh] Opening SSH tunnel to Erik:${REMOTE_PG_PORT} → localhost:${TUNNEL_PORT}"
|
||||
echo "[start.sh] Opening SSH tunnel to ${SSH_HOST}:${REMOTE_PG_PORT} -> localhost:${TUNNEL_PORT}"
|
||||
|
||||
# Kill any existing tunnel on that port
|
||||
pkill -f "ssh.*${TUNNEL_PORT}:localhost:${REMOTE_PG_PORT}" 2>/dev/null || true
|
||||
@ -36,9 +36,9 @@ trap cleanup EXIT
|
||||
|
||||
# Run fine-tuner with tunnel DB URL
|
||||
# Fine-tuner reads FT_DB_URL (see src/main.py load_config)
|
||||
export FT_DB_URL="postgresql://llm:llm_secure_2026@localhost:${TUNNEL_PORT}/llm_gateway"
|
||||
export FT_GATEWAY_URL="https://llm-gateway.context-x.org"
|
||||
export FT_OLLAMA_URL="http://localhost:11434"
|
||||
export FT_DB_URL="${FT_DB_URL:-postgresql://llm@localhost:${TUNNEL_PORT}/llm_gateway}"
|
||||
export FT_GATEWAY_URL="${FT_GATEWAY_URL:-http://localhost:3100}"
|
||||
export FT_OLLAMA_URL="${FT_OLLAMA_URL:-http://localhost:11434}"
|
||||
|
||||
echo "[start.sh] Starting fine-tuner..."
|
||||
# Route --status / --dry-run / --task-type / --general / --dpo to manual_trigger.py
|
||||
|
||||
@ -54,9 +54,9 @@ logger = logging.getLogger(__name__)
|
||||
_BASE_DIR = Path(__file__).parent.parent
|
||||
_DEFAULT_CONFIG = _BASE_DIR / "config" / "fine_tuner.yaml"
|
||||
|
||||
DEFAULT_DB_URL = "postgresql://llm:llm_secure_password@localhost:5432/llm_gateway"
|
||||
DEFAULT_DB_URL = "postgresql://llm@localhost:5432/llm_gateway"
|
||||
DEFAULT_GATEWAY_URL = "http://localhost:3100"
|
||||
DEFAULT_OLLAMA_URL = "http://192.168.178.169:11434"
|
||||
DEFAULT_OLLAMA_URL = "http://localhost:11434"
|
||||
|
||||
|
||||
def load_config(path: Optional[str] = None) -> dict:
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { compressContext } from '../context-compressor.js';
|
||||
|
||||
describe('context compressor', () => {
|
||||
it('checks short inputs without lossy compression', () => {
|
||||
const result = compressContext('short operational prompt', { enabled: true, mode: 'auto' });
|
||||
|
||||
expect(result.applied).toBe(false);
|
||||
expect(result.tokensSaved).toBe(0);
|
||||
expect(result.tokensBefore).toBe(result.tokensAfter);
|
||||
});
|
||||
|
||||
it('compresses long contexts before routing', () => {
|
||||
const result = compressContext('x'.repeat(80_000), { enabled: true, mode: 'auto' });
|
||||
|
||||
expect(result.applied).toBe(true);
|
||||
expect(result.tokensSaved).toBeGreaterThan(0);
|
||||
expect(result.tokensAfter).toBeLessThan(result.tokensBefore);
|
||||
expect(result.input).toContain('[ctxlean omitted');
|
||||
});
|
||||
});
|
||||
69
packages/gateway/src/modules/__tests__/pii-redaction.test.ts
Normal file
69
packages/gateway/src/modules/__tests__/pii-redaction.test.ts
Normal file
@ -0,0 +1,69 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getRedactMode,
|
||||
redactPii,
|
||||
restorePii,
|
||||
shouldRedactFor,
|
||||
} from '../pii-redaction.js';
|
||||
|
||||
const ORIGINAL_REDACT_PII_MODE = process.env['REDACT_PII_MODE'];
|
||||
const ORIGINAL_PII_REDACTION_MODE = process.env['PII_REDACTION_MODE'];
|
||||
|
||||
afterEach(() => {
|
||||
if (ORIGINAL_REDACT_PII_MODE === undefined) {
|
||||
delete process.env['REDACT_PII_MODE'];
|
||||
} else {
|
||||
process.env['REDACT_PII_MODE'] = ORIGINAL_REDACT_PII_MODE;
|
||||
}
|
||||
|
||||
if (ORIGINAL_PII_REDACTION_MODE === undefined) {
|
||||
delete process.env['PII_REDACTION_MODE'];
|
||||
} else {
|
||||
process.env['PII_REDACTION_MODE'] = ORIGINAL_PII_REDACTION_MODE;
|
||||
}
|
||||
});
|
||||
|
||||
describe('PII redaction', () => {
|
||||
it('defaults to cloud_only redaction', () => {
|
||||
delete process.env['REDACT_PII_MODE'];
|
||||
delete process.env['PII_REDACTION_MODE'];
|
||||
|
||||
expect(getRedactMode()).toBe('cloud_only');
|
||||
});
|
||||
|
||||
it('supports explicit opt-out and the legacy alias', () => {
|
||||
process.env['REDACT_PII_MODE'] = 'off';
|
||||
process.env['PII_REDACTION_MODE'] = 'always';
|
||||
expect(getRedactMode()).toBe('off');
|
||||
|
||||
delete process.env['REDACT_PII_MODE'];
|
||||
process.env['PII_REDACTION_MODE'] = 'always';
|
||||
expect(getRedactMode()).toBe('always');
|
||||
});
|
||||
|
||||
it('redacts and restores outward identifiers and API keys', () => {
|
||||
const email = ['person', 'example.org'].join('@');
|
||||
const ipAddress = [203, 0, 113, 42].join('.');
|
||||
const apiKey = `sk-${'a'.repeat(24)}`;
|
||||
const original = [
|
||||
`Contact ${email} from ${ipAddress}.`,
|
||||
`Use token ${apiKey}.`,
|
||||
].join(' ');
|
||||
|
||||
const result = redactPii(original);
|
||||
|
||||
expect(result.redacted).toContain('<EMAIL_001>');
|
||||
expect(result.redacted).toContain('<IP_ADDRESS_001>');
|
||||
expect(result.redacted).toContain('<API_KEY_001>');
|
||||
expect(result.redacted).not.toContain(email);
|
||||
expect(result.redacted).not.toContain(ipAddress);
|
||||
expect(result.redacted).not.toContain(apiKey);
|
||||
expect(restorePii(result.redacted, result.restoreMap)).toBe(original);
|
||||
});
|
||||
|
||||
it('redacts cloud and bridge providers while leaving local providers raw in cloud_only mode', () => {
|
||||
expect(shouldRedactFor('cloud_only', 'claude-bridge', 'gateway-test')).toBe(true);
|
||||
expect(shouldRedactFor('cloud_only', 'api.openai.com', 'gateway-test')).toBe(true);
|
||||
expect(shouldRedactFor('cloud_only', 'ollama', 'gateway-test')).toBe(false);
|
||||
});
|
||||
});
|
||||
@ -324,7 +324,7 @@ export function spawnBridge(desc: SubscriptionDescriptor): Promise<RunningBridge
|
||||
{ subscription: desc.id, port: desc.bridgePort },
|
||||
'Port already in use — assuming external bridge is healthy'
|
||||
);
|
||||
const url = `http://127.0.0.1:${desc.bridgePort}`;
|
||||
const url = `http://localhost:${desc.bridgePort}`;
|
||||
const fakeBridge: RunningBridge = {
|
||||
descriptor: desc,
|
||||
server, // server failed to bind; OK to keep handle
|
||||
@ -339,8 +339,8 @@ export function spawnBridge(desc: SubscriptionDescriptor): Promise<RunningBridge
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(desc.bridgePort, '127.0.0.1', () => {
|
||||
const url = `http://127.0.0.1:${desc.bridgePort}`;
|
||||
server.listen(desc.bridgePort, 'localhost', () => {
|
||||
const url = `http://localhost:${desc.bridgePort}`;
|
||||
const bridge: RunningBridge = {
|
||||
descriptor: desc,
|
||||
server,
|
||||
|
||||
@ -10,9 +10,9 @@
|
||||
* trust boundary without redaction. The mapping lives only in process
|
||||
* memory for the duration of a single call — never persisted.
|
||||
*
|
||||
* Modes (env REDACT_PII_MODE):
|
||||
* Modes (env REDACT_PII_MODE or PII_REDACTION_MODE):
|
||||
* - off → no redaction
|
||||
* - cloud_only → redact when target provider is non-local
|
||||
* - cloud_only → redact when target provider is non-local (default)
|
||||
* (i.e. any *-bridge or hosted API; Ollama passes raw)
|
||||
* - always → redact every call regardless of provider
|
||||
*
|
||||
@ -23,7 +23,7 @@ import { logger } from '../observability/logger.js';
|
||||
export type RedactMode = 'off' | 'cloud_only' | 'always';
|
||||
export type PiiCategory =
|
||||
| 'email' | 'phone' | 'credit_card' | 'iban' | 'ssn' | 'ip_address'
|
||||
| 'aws_key' | 'private_key' | 'jwt' | 'person_name';
|
||||
| 'aws_key' | 'api_key' | 'private_key' | 'jwt' | 'person_name';
|
||||
|
||||
export interface PiiMatch {
|
||||
category: PiiCategory;
|
||||
@ -113,6 +113,9 @@ const RULES: readonly DetectionRule[] = [
|
||||
// AWS access key IDs
|
||||
{ category: 'aws_key', pattern: /\bAKIA[0-9A-Z]{16}\b/g },
|
||||
|
||||
// Common API access tokens (OpenAI-style, GitHub, GitLab, Slack)
|
||||
{ category: 'api_key', pattern: /\b(?:sk-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{20,})\b/g },
|
||||
|
||||
// PEM-style private keys
|
||||
{ category: 'private_key', pattern: /-----BEGIN (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----/g },
|
||||
|
||||
@ -127,7 +130,7 @@ const RULES: readonly DetectionRule[] = [
|
||||
|
||||
const counters: Record<PiiCategory, number> = {
|
||||
email: 0, phone: 0, credit_card: 0, iban: 0, ssn: 0, ip_address: 0,
|
||||
aws_key: 0, private_key: 0, jwt: 0, person_name: 0,
|
||||
aws_key: 0, api_key: 0, private_key: 0, jwt: 0, person_name: 0,
|
||||
};
|
||||
|
||||
function tokenFor(category: PiiCategory, restoreMap: Map<string, string>): string {
|
||||
@ -190,7 +193,7 @@ export function restorePii(text: string, restoreMap: Map<string, string>): strin
|
||||
// ─── Mode helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
export function getRedactMode(): RedactMode {
|
||||
const v = (process.env['REDACT_PII_MODE'] ?? 'off').toLowerCase();
|
||||
const v = (process.env['REDACT_PII_MODE'] ?? process.env['PII_REDACTION_MODE'] ?? 'cloud_only').toLowerCase();
|
||||
if (v === 'cloud_only' || v === 'always') return v;
|
||||
return 'off';
|
||||
}
|
||||
|
||||
@ -238,7 +238,7 @@ async function probeBridge(url: string | undefined): Promise<boolean> {
|
||||
/**
|
||||
* Resolve the bridge URL for a subscription:
|
||||
* 1. Explicit env var (CLAUDE_BRIDGE_URL etc.) — set by Settings or PM2 ecosystem
|
||||
* 2. Auto-detect: probe http://127.0.0.1:{bridgePort} for a /health endpoint
|
||||
* 2. Auto-detect: probe http://localhost:{bridgePort} for a /health endpoint
|
||||
*
|
||||
* This means a bridge running locally on its default port is picked up
|
||||
* automatically without any configuration.
|
||||
@ -250,7 +250,7 @@ async function resolveBridgeUrl(desc: SubscriptionDescriptor): Promise<{ url?: s
|
||||
return { url: explicit, running };
|
||||
}
|
||||
// Auto-detect on the default port
|
||||
const localUrl = `http://127.0.0.1:${desc.bridgePort}`;
|
||||
const localUrl = `http://localhost:${desc.bridgePort}`;
|
||||
const running = await probeBridge(localUrl);
|
||||
return running ? { url: localUrl, running: true } : { running: false };
|
||||
}
|
||||
|
||||
@ -231,6 +231,111 @@ interface GatewayCompletionResult {
|
||||
body: Record<string, unknown>;
|
||||
}
|
||||
|
||||
function mergeRestoreMaps(target: Map<string, string>, source: Map<string, string>): void {
|
||||
for (const [sourceToken, value] of source.entries()) {
|
||||
const existing = Array.from(target.entries()).find(([, current]) => current === value);
|
||||
if (existing) continue;
|
||||
|
||||
if (!target.has(sourceToken)) {
|
||||
target.set(sourceToken, value);
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = sourceToken.match(/^<(.+)_(\d+)>$/);
|
||||
const prefix = match?.[1] ?? 'PII';
|
||||
let next = 1;
|
||||
let token = `<${prefix}_${String(next).padStart(3, '0')}>`;
|
||||
while (target.has(token)) {
|
||||
next += 1;
|
||||
token = `<${prefix}_${String(next).padStart(3, '0')}>`;
|
||||
}
|
||||
target.set(token, value);
|
||||
}
|
||||
}
|
||||
|
||||
function applyPiiTokens(text: string, restoreMap: Map<string, string>): string {
|
||||
let out = text;
|
||||
const entries = Array.from(restoreMap.entries())
|
||||
.filter(([, value]) => value.length > 0)
|
||||
.sort((a, b) => b[1].length - a[1].length);
|
||||
|
||||
for (const [token, value] of entries) {
|
||||
out = out.split(value).join(token);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function buildOutboundPiiRedactionMap(
|
||||
text: string,
|
||||
options: { providerName: string; caller: string; callId: string; language?: string }
|
||||
): Promise<Map<string, string> | null> {
|
||||
const redactMode = getRedactMode();
|
||||
if (!text.trim() || !shouldRedactFor(redactMode, options.providerName, options.caller)) return null;
|
||||
|
||||
const restoreMap = new Map<string, string>();
|
||||
const pii = redactPii(text);
|
||||
mergeRestoreMaps(restoreMap, pii.restoreMap);
|
||||
|
||||
let nameCount = 0;
|
||||
const presidioUrl = process.env['PRESIDIO_URL'];
|
||||
if (presidioUrl) {
|
||||
try {
|
||||
const nameResult = await redactPersonNamesAsync(applyPiiTokens(text, restoreMap), presidioUrl, options.language ?? 'de');
|
||||
nameCount = nameResult.count;
|
||||
mergeRestoreMaps(restoreMap, nameResult.restoreMap);
|
||||
} catch (err) {
|
||||
logger.warn({ err, callId: options.callId }, 'Presidio name redaction failed — continuing');
|
||||
}
|
||||
}
|
||||
|
||||
if (restoreMap.size === 0) return null;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
callId: options.callId,
|
||||
caller: options.caller,
|
||||
provider: options.providerName,
|
||||
redactedCounts: pii.counts,
|
||||
redactedNames: nameCount,
|
||||
redactedTokens: restoreMap.size,
|
||||
},
|
||||
'PII redaction applied before outbound LLM call',
|
||||
);
|
||||
return restoreMap;
|
||||
}
|
||||
|
||||
function redactChatContent(
|
||||
content: OpenAIChatCompletionRequest['messages'][number]['content'],
|
||||
restoreMap: Map<string, string>
|
||||
): OpenAIChatCompletionRequest['messages'][number]['content'] {
|
||||
if (typeof content === 'string') return applyPiiTokens(content, restoreMap);
|
||||
if (!Array.isArray(content)) return content;
|
||||
|
||||
return content.map((part) => {
|
||||
if (typeof part === 'string') return applyPiiTokens(part, restoreMap);
|
||||
if (!part || typeof part !== 'object') return part;
|
||||
|
||||
const copy: Record<string, unknown> = { ...(part as Record<string, unknown>) };
|
||||
for (const key of ['text', 'input_text', 'output_text']) {
|
||||
if (typeof copy[key] === 'string') copy[key] = applyPiiTokens(copy[key] as string, restoreMap);
|
||||
}
|
||||
return copy;
|
||||
});
|
||||
}
|
||||
|
||||
function redactChatCompletionBody(
|
||||
body: OpenAIChatCompletionRequest,
|
||||
restoreMap: Map<string, string>
|
||||
): OpenAIChatCompletionRequest {
|
||||
return {
|
||||
...body,
|
||||
messages: body.messages.map((message) => ({
|
||||
...message,
|
||||
content: redactChatContent(message.content, restoreMap),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// input: string,
|
||||
// caller: string,
|
||||
@ -397,7 +502,8 @@ function buildResponseBody(callId: string, decision: ReturnType<typeof route>, t
|
||||
}
|
||||
|
||||
async function executeCompletion(body: CompletionRequest, startMs: number, callId: string): Promise<GatewayCompletionResult> {
|
||||
const { caller, language, context, options } = body;
|
||||
const { caller, language, options } = body;
|
||||
let context = body.context;
|
||||
|
||||
// ─── Plugin pre-hooks (PLUGINS_DIR) ────────────────────────────────────
|
||||
try {
|
||||
@ -413,33 +519,18 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
||||
}
|
||||
|
||||
// ─── PII Redaction (REDACT_PII_MODE: off|cloud_only|always) ─────────────
|
||||
const redactMode = getRedactMode();
|
||||
let piiRestoreMap: Map<string, string> | null = null;
|
||||
if (redactMode !== 'off' && shouldRedactFor(redactMode, 'unknown', caller)) {
|
||||
const r = redactPii(body.input);
|
||||
if (r.restoreMap.size > 0) {
|
||||
body = { ...body, input: r.redacted };
|
||||
piiRestoreMap = r.restoreMap;
|
||||
logger.info(
|
||||
{ callId, caller, redactedCounts: r.counts, redactedTokens: r.restoreMap.size },
|
||||
'PII redaction applied',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── PII: person names via Presidio+GLiNER sidecar (async, non-blocking) ──
|
||||
const presidioUrl = process.env['PRESIDIO_URL'];
|
||||
if (presidioUrl && redactMode !== 'off' && shouldRedactFor(redactMode, 'unknown', caller)) {
|
||||
try {
|
||||
const nameResult = await redactPersonNamesAsync(body.input, presidioUrl);
|
||||
if (nameResult.count > 0) {
|
||||
body = { ...body, input: nameResult.redacted };
|
||||
if (!piiRestoreMap) piiRestoreMap = new Map();
|
||||
for (const [k, v] of nameResult.restoreMap) piiRestoreMap.set(k, v);
|
||||
logger.info({ callId, caller, names: nameResult.count }, 'Person names redacted via Presidio');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err, callId }, 'Presidio name redaction failed — continuing');
|
||||
const systemContext = typeof context === 'object' && context && 'system' in context
|
||||
? String((context as any).system ?? '')
|
||||
: '';
|
||||
const piiRestoreMap = await buildOutboundPiiRedactionMap(
|
||||
[body.input, systemContext].filter(Boolean).join('\n\n'),
|
||||
{ providerName: 'unknown', caller, callId, language }
|
||||
);
|
||||
if (piiRestoreMap) {
|
||||
body = { ...body, input: applyPiiTokens(body.input, piiRestoreMap) };
|
||||
if (systemContext && context) {
|
||||
context = { ...context, system: applyPiiTokens(systemContext, piiRestoreMap) };
|
||||
body = { ...body, context };
|
||||
}
|
||||
}
|
||||
|
||||
@ -517,7 +608,7 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
||||
|
||||
// ─── Cache check (Tier 1: exact-match hash lookup) ─────────────────────
|
||||
const agenticNoCache = shouldBypassResponseCache(caller);
|
||||
const skipCache = agenticNoCache || (options as any)?.skip_cache === true;
|
||||
const skipCache = agenticNoCache || (options as any)?.skip_cache === true || Boolean(piiRestoreMap);
|
||||
const cacheableReq = {
|
||||
caller,
|
||||
task_type: body.task_type,
|
||||
@ -554,6 +645,14 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
||||
if (hit) {
|
||||
const latencyMs = Date.now() - startMs;
|
||||
void recordCacheHit(dbForCache, hit.id);
|
||||
void logCompressionMetric(dbForCache, {
|
||||
filePath: callId,
|
||||
mode: 'cache:none',
|
||||
tokensBefore: hit.tokensIn,
|
||||
tokensAfter: hit.tokensIn,
|
||||
savingsPct: 0,
|
||||
toolUsed: 'gateway',
|
||||
});
|
||||
// Log cache hit as a successful request (status=approved, fallback=false)
|
||||
const requestLogger = createRequestLogger(dbForCache);
|
||||
void requestLogger.logRequest(
|
||||
@ -655,12 +754,14 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
||||
}
|
||||
|
||||
const latencyMs = Date.now() - startMs;
|
||||
const outputText = ollamaResponse.response;
|
||||
const modelOutputText = ollamaResponse.response;
|
||||
const outputText = piiRestoreMap ? restorePii(modelOutputText, piiRestoreMap) : modelOutputText;
|
||||
const validationOutput = await runPostValidation(outputText, { validators: decision.validators, language, output_format: decision.output_format, requires_fact_check: decision.requires_fact_check, schema: resolved.schema });
|
||||
const confidenceResult = evaluateConfidence(validationOutput);
|
||||
|
||||
recordAllMetrics(caller, taskType, confidenceResult, ollamaResponse, decision, validationOutput);
|
||||
const { costUsd, costSavedUsd } = await auditAndTrackCosts(caller, taskType, compression.originalInput, outputText, latencyMs, ollamaResponse, resolved, decision, confidenceResult, validationOutput, classificationResult, callId, compression);
|
||||
const auditOutputText = piiRestoreMap ? modelOutputText : outputText;
|
||||
const { costUsd, costSavedUsd } = await auditAndTrackCosts(caller, taskType, compression.originalInput, auditOutputText, latencyMs, ollamaResponse, resolved, decision, confidenceResult, validationOutput, classificationResult, callId, compression);
|
||||
|
||||
latencySeconds.labels({ caller, task_type: taskType, model: ollamaResponse.model ?? decision.model }).observe(latencyMs / 1000);
|
||||
|
||||
@ -728,13 +829,40 @@ function isCodexBridgeModel(model: string | undefined): boolean {
|
||||
return /^(codex-default|codex-|gpt-5(?:\.|-|$))/i.test(model ?? '');
|
||||
}
|
||||
|
||||
function chatMessagesToPrompt(messages: OpenAIChatCompletionRequest['messages']): string {
|
||||
function chatMessagesToPrompt(
|
||||
messages: OpenAIChatCompletionRequest['messages'],
|
||||
options: { includeSystem?: boolean } = {}
|
||||
): string {
|
||||
return messages
|
||||
.filter((message) => options.includeSystem !== false || message.role !== 'system')
|
||||
.map((message) => `${message.role}: ${contentToText(message.content)}`)
|
||||
.join('\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function compressChatMessagesForBridge(body: OpenAIChatCompletionRequest): {
|
||||
body: OpenAIChatCompletionRequest;
|
||||
inputText: string;
|
||||
compression: CompressionResult;
|
||||
} {
|
||||
const inputText = chatMessagesToPrompt(body.messages, { includeSystem: false });
|
||||
const compression = compressContext(inputText, { enabled: true, mode: 'auto' });
|
||||
if (!compression.applied) return { body, inputText, compression };
|
||||
|
||||
const systemMessages = body.messages.filter((message) => message.role === 'system');
|
||||
return {
|
||||
body: {
|
||||
...body,
|
||||
messages: [
|
||||
...systemMessages,
|
||||
{ role: 'user', content: compression.input },
|
||||
],
|
||||
},
|
||||
inputText: compression.input,
|
||||
compression,
|
||||
};
|
||||
}
|
||||
|
||||
interface SubscriptionBridgeTarget {
|
||||
subscriptionId: string;
|
||||
provider: string;
|
||||
@ -742,70 +870,127 @@ interface SubscriptionBridgeTarget {
|
||||
defaultUrl: string;
|
||||
}
|
||||
|
||||
const SUBSCRIPTION_BRIDGE_TARGETS_BY_MODEL: Record<string, SubscriptionBridgeTarget> = {
|
||||
'claude-opus-4-1': {
|
||||
const SUBSCRIPTION_BRIDGE_TARGETS_BY_ID: Record<string, SubscriptionBridgeTarget> = {
|
||||
'claude-code': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3250',
|
||||
defaultUrl: 'http://localhost:3250',
|
||||
},
|
||||
'claude-sonnet-4-6': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3250',
|
||||
},
|
||||
'claude-haiku-3': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3250',
|
||||
'github-copilot': {
|
||||
subscriptionId: 'github-copilot',
|
||||
provider: 'copilot-bridge',
|
||||
envKey: 'COPILOT_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3252',
|
||||
},
|
||||
'microsoft-365-copilot': {
|
||||
subscriptionId: 'microsoft-365-copilot',
|
||||
provider: 'm365-copilot-bridge',
|
||||
envKey: 'M365_COPILOT_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3257',
|
||||
defaultUrl: 'http://localhost:3257',
|
||||
},
|
||||
'chatgpt': {
|
||||
subscriptionId: 'chatgpt',
|
||||
provider: 'chatgpt-bridge',
|
||||
envKey: 'CHATGPT_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3251',
|
||||
},
|
||||
'gemini': {
|
||||
subscriptionId: 'gemini',
|
||||
provider: 'gemini-bridge',
|
||||
envKey: 'GEMINI_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3254',
|
||||
},
|
||||
'codex': {
|
||||
subscriptionId: 'codex',
|
||||
provider: 'codex-bridge',
|
||||
envKey: 'CODEX_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3253',
|
||||
},
|
||||
'aider': {
|
||||
subscriptionId: 'aider',
|
||||
provider: 'aider-bridge',
|
||||
envKey: 'AIDER_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3256',
|
||||
},
|
||||
};
|
||||
|
||||
const SUBSCRIPTION_BRIDGE_TARGETS_BY_MODEL: Record<string, SubscriptionBridgeTarget> = {
|
||||
'claude-opus-4-1': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3250',
|
||||
},
|
||||
'claude-sonnet-4-6': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3250',
|
||||
},
|
||||
'claude-haiku-3': {
|
||||
subscriptionId: 'claude-code',
|
||||
provider: 'claude-bridge',
|
||||
envKey: 'CLAUDE_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3250',
|
||||
},
|
||||
'gpt-4-turbo': {
|
||||
subscriptionId: 'chatgpt',
|
||||
provider: 'chatgpt-bridge',
|
||||
envKey: 'CHATGPT_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3251',
|
||||
},
|
||||
'microsoft-365-copilot': {
|
||||
subscriptionId: 'microsoft-365-copilot',
|
||||
provider: 'm365-copilot-bridge',
|
||||
envKey: 'M365_COPILOT_BRIDGE_URL',
|
||||
defaultUrl: 'http://localhost:3257',
|
||||
},
|
||||
'm365-copilot-chat': {
|
||||
subscriptionId: 'microsoft-365-copilot',
|
||||
provider: 'm365-copilot-bridge',
|
||||
envKey: 'M365_COPILOT_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3257',
|
||||
defaultUrl: 'http://localhost:3257',
|
||||
},
|
||||
'gemini-1.5-pro': {
|
||||
subscriptionId: 'gemini',
|
||||
provider: 'gemini-bridge',
|
||||
envKey: 'GEMINI_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3254',
|
||||
defaultUrl: 'http://localhost:3254',
|
||||
},
|
||||
'gemini-1.5-flash': {
|
||||
subscriptionId: 'gemini',
|
||||
provider: 'gemini-bridge',
|
||||
envKey: 'GEMINI_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3254',
|
||||
defaultUrl: 'http://localhost:3254',
|
||||
},
|
||||
'aider-default': {
|
||||
subscriptionId: 'aider',
|
||||
provider: 'aider-bridge',
|
||||
envKey: 'AIDER_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3256',
|
||||
defaultUrl: 'http://localhost:3256',
|
||||
},
|
||||
};
|
||||
|
||||
function subscriptionBridgeTargetForSubscriptionId(subscriptionId: string | undefined): SubscriptionBridgeTarget | null {
|
||||
if (!subscriptionId) return null;
|
||||
return SUBSCRIPTION_BRIDGE_TARGETS_BY_ID[subscriptionId.toLowerCase()] ?? null;
|
||||
}
|
||||
|
||||
function subscriptionBridgeTargetForModel(model: string | undefined): SubscriptionBridgeTarget | null {
|
||||
if (!model) return null;
|
||||
if (isCodexBridgeModel(model)) {
|
||||
return {
|
||||
subscriptionId: 'codex',
|
||||
provider: 'codex-bridge',
|
||||
envKey: 'CODEX_BRIDGE_URL',
|
||||
defaultUrl: 'http://127.0.0.1:3253',
|
||||
};
|
||||
return subscriptionBridgeTargetForSubscriptionId('codex');
|
||||
}
|
||||
return SUBSCRIPTION_BRIDGE_TARGETS_BY_MODEL[model.toLowerCase()] ?? null;
|
||||
}
|
||||
|
||||
function requestedSubscriptionFromHeaders(request: FastifyRequest): string | undefined {
|
||||
const value = request.headers['x-llm-gateway-subscription']
|
||||
?? request.headers['x-subscription-id'];
|
||||
return Array.isArray(value) ? value[0] : value;
|
||||
}
|
||||
|
||||
function subscriptionBridgeUrl(target: SubscriptionBridgeTarget): string {
|
||||
return (process.env[target.envKey] ?? target.defaultUrl).replace(/\/$/, '');
|
||||
}
|
||||
@ -857,12 +1042,18 @@ async function postSubscriptionBridgeCompletion(
|
||||
return { statusCode: chatResponse.status, ok: chatResponse.ok, data: chatData };
|
||||
}
|
||||
|
||||
const system = body.messages
|
||||
.filter((message) => message.role === 'system')
|
||||
.map((message) => contentToText(message.content))
|
||||
.filter(Boolean)
|
||||
.join('\n\n');
|
||||
const generateResponse = await fetch(`${bridgeUrl}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: body.model,
|
||||
prompt: inputText,
|
||||
system: system || undefined,
|
||||
stream: false,
|
||||
temperature: body.temperature ?? 0,
|
||||
max_tokens: body.max_tokens,
|
||||
@ -875,14 +1066,25 @@ async function postSubscriptionBridgeCompletion(
|
||||
async function callSubscriptionBridgeChatCompletion(
|
||||
body: OpenAIChatCompletionRequest,
|
||||
startMs: number,
|
||||
callId: string
|
||||
callId: string,
|
||||
requestedSubscriptionId: string | undefined,
|
||||
caller: string
|
||||
): Promise<{ ok: true; body: Record<string, unknown> } | { ok: false; statusCode: number; error: string } | null> {
|
||||
const target = subscriptionBridgeTargetForModel(body.model);
|
||||
const target = subscriptionBridgeTargetForSubscriptionId(requestedSubscriptionId)
|
||||
?? subscriptionBridgeTargetForModel(body.model);
|
||||
if (!target) return null;
|
||||
|
||||
const inputText = chatMessagesToPrompt(body.messages);
|
||||
const compressed = compressChatMessagesForBridge(body);
|
||||
const outboundRedactionMap = await buildOutboundPiiRedactionMap(
|
||||
chatMessagesToPrompt(compressed.body.messages, { includeSystem: true }),
|
||||
{ providerName: target.provider, caller, callId, language: 'en' }
|
||||
);
|
||||
const outboundBody = outboundRedactionMap
|
||||
? redactChatCompletionBody(compressed.body, outboundRedactionMap)
|
||||
: compressed.body;
|
||||
const inputText = chatMessagesToPrompt(outboundBody.messages, { includeSystem: false });
|
||||
try {
|
||||
const upstream = await postSubscriptionBridgeCompletion(target, body, inputText);
|
||||
const upstream = await postSubscriptionBridgeCompletion(target, outboundBody, inputText);
|
||||
if (!upstream.ok || upstream.data?.success === false) {
|
||||
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
|
||||
logger.warn(
|
||||
@ -892,9 +1094,39 @@ async function callSubscriptionBridgeChatCompletion(
|
||||
return { ok: false, statusCode: 424, error };
|
||||
}
|
||||
|
||||
const text = contentFromSubscriptionBridge(upstream.data);
|
||||
const modelText = contentFromSubscriptionBridge(upstream.data);
|
||||
const text = outboundRedactionMap ? restorePii(String(modelText), outboundRedactionMap) : String(modelText);
|
||||
const promptTokens = upstream.data?.usage?.prompt_tokens ?? Math.ceil(inputText.length / 4);
|
||||
const completionTokens = upstream.data?.usage?.completion_tokens ?? Math.ceil(String(text).length / 4);
|
||||
const completionTokens = upstream.data?.usage?.completion_tokens ?? Math.ceil(String(modelText).length / 4);
|
||||
const latencyMs = Date.now() - startMs;
|
||||
try {
|
||||
const db = getPool();
|
||||
void logCompressionMetric(db, {
|
||||
filePath: callId,
|
||||
mode: `${compressed.compression.method}:${compressed.compression.strategy}`,
|
||||
tokensBefore: compressed.compression.tokensBefore,
|
||||
tokensAfter: compressed.compression.tokensAfter,
|
||||
savingsPct: Math.round(compressed.compression.ratio * 10000) / 100,
|
||||
toolUsed: 'gateway',
|
||||
});
|
||||
void createRequestLogger(db).logRequest(
|
||||
callId,
|
||||
caller,
|
||||
'subscription_bridge',
|
||||
body.model,
|
||||
'approved',
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
0,
|
||||
latencyMs,
|
||||
0,
|
||||
false,
|
||||
undefined,
|
||||
);
|
||||
void recordSubscriptionUsage(db, target.subscriptionId, promptTokens + completionTokens);
|
||||
} catch (e) {
|
||||
logger.warn({ e, callId }, 'failed to log subscription bridge request');
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
body: {
|
||||
@ -918,8 +1150,10 @@ async function callSubscriptionBridgeChatCompletion(
|
||||
status: 'approved',
|
||||
provider: target.provider,
|
||||
subscription_id: target.subscriptionId,
|
||||
latency_ms: Date.now() - startMs,
|
||||
subscription_forced: Boolean(requestedSubscriptionId),
|
||||
latency_ms: latencyMs,
|
||||
passthrough: true,
|
||||
compression: buildCompressionResponse(compressed.compression),
|
||||
},
|
||||
},
|
||||
};
|
||||
@ -1303,7 +1537,18 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
}
|
||||
|
||||
const callId = `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
const subscriptionBridgeResult = await callSubscriptionBridgeChatCompletion(parsed.data, startMs, callId);
|
||||
const requestedSubscriptionId = requestedSubscriptionFromHeaders(request);
|
||||
const { caller: bridgeCaller } = detectCaller(request, 'openai-compatible', parsed.data.user);
|
||||
const subscriptionBridgeResult = await callSubscriptionBridgeChatCompletion(parsed.data, startMs, callId, requestedSubscriptionId, bridgeCaller);
|
||||
if (requestedSubscriptionId && !subscriptionBridgeResult) {
|
||||
return reply.status(404).send({
|
||||
error: {
|
||||
message: `Unknown subscription bridge: ${requestedSubscriptionId}`,
|
||||
type: 'not_found',
|
||||
code: 404,
|
||||
},
|
||||
});
|
||||
}
|
||||
if (subscriptionBridgeResult) {
|
||||
if (!subscriptionBridgeResult.ok) {
|
||||
return reply.status(subscriptionBridgeResult.statusCode).send({
|
||||
@ -1396,20 +1641,28 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
// Codex.app sends model=gpt-5.5 / gpt-5.1-codex-mini etc. These are
|
||||
// ChatGPT-subscription models the openai API itself rejects without
|
||||
// the right auth. Route them straight to the local codex-bridge
|
||||
// (PM2 process at 127.0.0.1:3253) which speaks codex-cli over OAuth.
|
||||
// (local bridge process) which speaks codex-cli over OAuth.
|
||||
if (isCodexBridgeModel(parsed.data.model)) {
|
||||
try {
|
||||
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://127.0.0.1:3253';
|
||||
const inputText = typeof parsed.data.input === 'string'
|
||||
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://localhost:3253';
|
||||
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
||||
const originalInputText = typeof parsed.data.input === 'string'
|
||||
? parsed.data.input
|
||||
: (Array.isArray(parsed.data.input)
|
||||
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
|
||||
const compression = compressContext(originalInputText, { enabled: true, mode: 'auto' });
|
||||
const inputText = compression.input;
|
||||
const outboundRedactionMap = await buildOutboundPiiRedactionMap(
|
||||
inputText,
|
||||
{ providerName: 'codex-bridge', caller: bridgeCaller, callId, language: 'en' }
|
||||
);
|
||||
const outboundInputText = outboundRedactionMap ? applyPiiTokens(inputText, outboundRedactionMap) : inputText;
|
||||
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: parsed.data.model,
|
||||
messages: [{ role: 'user', content: inputText }],
|
||||
messages: [{ role: 'user', content: outboundInputText }],
|
||||
}),
|
||||
});
|
||||
const upstreamText = await upstream.text();
|
||||
@ -1420,12 +1673,18 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
upstreamJson = { raw: upstreamText };
|
||||
}
|
||||
if (upstream.ok && upstreamJson?.success !== false) {
|
||||
const text = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
||||
const respBody = toOpenAIResponsesResponse({ output: text, model: parsed.data.model, status: 'approved' }, parsed.data.model);
|
||||
const modelText = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
||||
const text = outboundRedactionMap ? restorePii(String(modelText), outboundRedactionMap) : String(modelText);
|
||||
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
|
||||
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
||||
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(inputText.length / 4);
|
||||
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(text.length / 4);
|
||||
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(outboundInputText.length / 4);
|
||||
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(String(modelText).length / 4);
|
||||
const respBody = toOpenAIResponsesResponse({
|
||||
output: text,
|
||||
model: parsed.data.model,
|
||||
status: 'approved',
|
||||
tokens: { in: bridgeTokensIn, out: bridgeTokensOut },
|
||||
compression: buildCompressionResponse(compression),
|
||||
}, parsed.data.model);
|
||||
// Track against the merged OpenAI (ChatGPT+Codex) subscription pool.
|
||||
try {
|
||||
const subId = modelToSubscriptionId(parsed.data.model ?? '') ?? 'codex';
|
||||
@ -1435,7 +1694,16 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
}
|
||||
// Write to request_tracking so dashboard token totals include bridge calls.
|
||||
try {
|
||||
const bridgeLogger = createRequestLogger(getPool());
|
||||
const db = getPool();
|
||||
void logCompressionMetric(db, {
|
||||
filePath: callId,
|
||||
mode: `${compression.method}:${compression.strategy}`,
|
||||
tokensBefore: compression.tokensBefore,
|
||||
tokensAfter: compression.tokensAfter,
|
||||
savingsPct: Math.round(compression.ratio * 10000) / 100,
|
||||
toolUsed: 'gateway',
|
||||
});
|
||||
const bridgeLogger = createRequestLogger(db);
|
||||
void bridgeLogger.logRequest(
|
||||
callId,
|
||||
bridgeCaller,
|
||||
@ -1447,7 +1715,7 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
0,
|
||||
Date.now() - startMs,
|
||||
0,
|
||||
false,
|
||||
compression.applied,
|
||||
undefined,
|
||||
);
|
||||
} catch (e) {
|
||||
@ -1465,7 +1733,8 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
||||
latency_ms: Date.now() - startMs,
|
||||
confidence: 0,
|
||||
cost_usd: 0,
|
||||
compression_applied: false,
|
||||
compression_applied: compression.applied,
|
||||
metadata: { compression: buildCompressionResponse(compression) },
|
||||
model: parsed.data.model ?? 'gpt-5.5',
|
||||
} as any);
|
||||
} catch (e) {
|
||||
|
||||
@ -40,6 +40,7 @@ import { getSubscriptionWallet, recordSubscriptionUsage } from '../modules/subsc
|
||||
import { rememberFact, recallFacts, forgetCaller } from '../modules/knowledge-memory.js';
|
||||
import { getRaceStats } from '../modules/race-mode.js';
|
||||
import { dashboardAuthStatus, requireDashboardAuth } from '../modules/admin-auth.js';
|
||||
import { logCompressionMetric } from '../utils/tokenvault-hooks.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
@ -1760,6 +1761,14 @@ export async function dashboardRoute(fastify: FastifyInstance): Promise<void> {
|
||||
await requestLogger.logRequest(requestId, caller, 'usage_import', model, 'approved', tokensIn, tokensOut, costUsd, 0);
|
||||
await db.query(`UPDATE request_tracking SET created_at=$1 WHERE request_id=$2`, [usedAt, requestId]);
|
||||
}
|
||||
await logCompressionMetric(db, {
|
||||
filePath: requestId,
|
||||
mode: 'usage-import:none',
|
||||
tokensBefore: tokensIn,
|
||||
tokensAfter: tokensIn,
|
||||
savingsPct: 0,
|
||||
toolUsed: 'gateway',
|
||||
});
|
||||
return reply.status(200).send({ success: true, imported: { caller, model, day, tokensIn, tokensOut, costUsd, usedAt } });
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Failed to import usage report');
|
||||
|
||||
@ -30,6 +30,19 @@ interface SubscriptionTestBody extends SubscriptionBridgeBody {
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
interface SubscriptionChatParams {
|
||||
subscription_id: SubscriptionId;
|
||||
}
|
||||
|
||||
type SubscriptionChatBody = {
|
||||
model?: string;
|
||||
messages?: readonly unknown[];
|
||||
temperature?: number;
|
||||
max_tokens?: number;
|
||||
stream?: boolean;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type RunningBridge = ReturnType<typeof getRunningBridges>[number];
|
||||
|
||||
function configuredBridgeUrl(status: SubscriptionStatus, runningById: Map<string, RunningBridge>): string | null {
|
||||
@ -40,7 +53,7 @@ function configuredBridgeUrl(status: SubscriptionStatus, runningById: Map<string
|
||||
}
|
||||
|
||||
function defaultBridgeUrl(status: SubscriptionStatus): string {
|
||||
return `http://127.0.0.1:${status.descriptor.bridgePort}`;
|
||||
return `http://localhost:${status.descriptor.bridgePort}`;
|
||||
}
|
||||
|
||||
function bridgeBaseUrl(url: string): string {
|
||||
@ -67,23 +80,26 @@ function buildAccessCard(
|
||||
) {
|
||||
const defaultModel = status.descriptor.models[0]?.id ?? status.descriptor.id;
|
||||
const modelList = status.descriptor.models.map((model) => model.id);
|
||||
const subscriptionApiUrl = `${baseUrl}/api/subscriptions/${encodeURIComponent(status.descriptor.id)}/v1/chat/completions`;
|
||||
return {
|
||||
kind: 'gateway-subscription-access',
|
||||
subscriptionId: status.descriptor.id,
|
||||
providerName: status.descriptor.providerName,
|
||||
gatewayBaseUrl: baseUrl,
|
||||
chatCompletionsUrl: `${baseUrl}/v1/chat/completions`,
|
||||
subscriptionChatCompletionsUrl: subscriptionApiUrl,
|
||||
responsesUrl: `${baseUrl}/v1/responses`,
|
||||
modelsUrl: `${baseUrl}/v1/models`,
|
||||
subscriptionStatusUrl: `${baseUrl}/api/subscriptions?search=${encodeURIComponent(status.descriptor.id)}`,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer ${LLM_GATEWAY_API_KEY}',
|
||||
Authorization: 'Bearer <key>',
|
||||
'X-Caller-ID': 'your-service-name',
|
||||
'X-LLM-Gateway-Subscription': status.descriptor.id,
|
||||
},
|
||||
env: {
|
||||
LLM_GATEWAY_URL: baseUrl,
|
||||
LLM_GATEWAY_API_KEY: '${LLM_GATEWAY_API_KEY}',
|
||||
LLM_GATEWAY_API_KEY: '<key>',
|
||||
LLM_GATEWAY_MODEL: defaultModel,
|
||||
LLM_GATEWAY_SUBSCRIPTION: status.descriptor.id,
|
||||
},
|
||||
@ -100,6 +116,14 @@ function buildAccessCard(
|
||||
` -H "Content-Type: application/json"`,
|
||||
` -H "Authorization: Bearer $LLM_GATEWAY_API_KEY"`,
|
||||
` -H "X-Caller-ID: pilot-client"`,
|
||||
` -H "X-LLM-Gateway-Subscription: ${status.descriptor.id}"`,
|
||||
` -d '{"model":"${defaultModel}","messages":[{"role":"user","content":"gateway bridge smoke"}]}'`,
|
||||
].join(' \\\n'),
|
||||
subscriptionCurl: [
|
||||
`curl -X POST ${subscriptionApiUrl}`,
|
||||
` -H "Content-Type: application/json"`,
|
||||
` -H "Authorization: Bearer $LLM_GATEWAY_API_KEY"`,
|
||||
` -H "X-Caller-ID: pilot-client"`,
|
||||
` -d '{"model":"${defaultModel}","messages":[{"role":"user","content":"gateway bridge smoke"}]}'`,
|
||||
].join(' \\\n'),
|
||||
note: 'Use a gateway API key/admin token from your deployment. OAuth/subscription secrets stay inside the bridge process and are never exposed here.',
|
||||
@ -206,6 +230,27 @@ async function getSubscriptionSnapshot(request: FastifyRequest, search = '') {
|
||||
};
|
||||
}
|
||||
|
||||
async function getSubscriptionStatusOrReply(
|
||||
subscriptionId: SubscriptionId,
|
||||
reply: FastifyReply
|
||||
): Promise<SubscriptionStatus | null> {
|
||||
const statuses = await discoverSubscriptions();
|
||||
const status = statuses.find((candidate) => candidate.descriptor.id === subscriptionId);
|
||||
if (!status) {
|
||||
reply.status(404).send({ success: false, error: 'subscription not found' });
|
||||
return null;
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
async function ensureSubscriptionBridge(status: SubscriptionStatus): Promise<RunningBridge | null> {
|
||||
const running = getRunningBridges().find((bridge) => bridge.descriptor.id === status.descriptor.id);
|
||||
if (running) return running;
|
||||
if (status.bridgeRunning || status.bridgeUrl || process.env[status.descriptor.bridgeEnvKey]) return null;
|
||||
if (!status.installed || status.authenticated === false) return null;
|
||||
return await spawnBridge(status.descriptor);
|
||||
}
|
||||
|
||||
function contentFromBridgeResponse(data: any): string {
|
||||
return data?.choices?.[0]?.message?.content
|
||||
?? data?.content
|
||||
@ -475,4 +520,49 @@ export async function subscriptionsRoute(fastify: FastifyInstance): Promise<void
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post('/api/subscriptions/:subscription_id/v1/chat/completions', auth, async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const params = request.params as SubscriptionChatParams;
|
||||
const status = await getSubscriptionStatusOrReply(params.subscription_id, reply);
|
||||
if (!status) return reply;
|
||||
|
||||
try {
|
||||
await ensureSubscriptionBridge(status);
|
||||
const body = (request.body ?? {}) as SubscriptionChatBody;
|
||||
const model = typeof body.model === 'string'
|
||||
? body.model
|
||||
: status.descriptor.models[0]?.id ?? status.descriptor.id;
|
||||
const payload = {
|
||||
...body,
|
||||
model,
|
||||
};
|
||||
const caller = request.headers['x-caller-id'] ?? `subscription-${status.descriptor.id}`;
|
||||
const injected = await fastify.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/chat/completions',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-caller-id': Array.isArray(caller) ? caller[0] : String(caller),
|
||||
'x-llm-gateway-subscription': status.descriptor.id,
|
||||
},
|
||||
payload,
|
||||
});
|
||||
|
||||
let responseBody: unknown = injected.payload;
|
||||
try {
|
||||
responseBody = JSON.parse(injected.payload);
|
||||
} catch {
|
||||
// Keep raw payload for non-JSON bridge errors.
|
||||
}
|
||||
const contentType = injected.headers['content-type'];
|
||||
if (typeof contentType === 'string') reply.header('content-type', contentType);
|
||||
return reply.status(injected.statusCode).send(responseBody);
|
||||
} catch (error) {
|
||||
logger.error({ error, subscriptionId: status.descriptor.id }, 'Subscription gateway API call failed');
|
||||
return reply.status(424).send({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'subscription gateway API call failed',
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -203,7 +203,7 @@ LIGHTRAG_PORT=3140
|
||||
ENVIRONMENT=production
|
||||
|
||||
# LLM Backend
|
||||
OLLAMA_URL=http://192.168.178.213:11434
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_MODEL=qwen2.5:14b
|
||||
|
||||
# Vector Database
|
||||
@ -211,7 +211,7 @@ QDRANT_URL=http://localhost:6333
|
||||
EMBEDDING_MODEL=bge-m3
|
||||
|
||||
# PostgreSQL
|
||||
DATABASE_URL=postgresql://tip_kg:password@localhost:5432/tip_lightrag
|
||||
DATABASE_URL=postgresql://tip_kg@localhost:5432/tip_lightrag
|
||||
DB_POOL_SIZE=10
|
||||
|
||||
# Hybrid Retrieval
|
||||
@ -230,17 +230,17 @@ pip install -r requirements.txt
|
||||
python scripts/init_db.py
|
||||
|
||||
# Run sidecar
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 3140 --reload
|
||||
uvicorn app.main:app --host localhost --port 3140 --reload
|
||||
```
|
||||
|
||||
### Erik Deployment
|
||||
### Remote Deployment
|
||||
|
||||
```bash
|
||||
# Copy to Erik
|
||||
scp -r packages/lightrag-sidecar/ erik:/opt/llm-gateway/packages/
|
||||
scp -r packages/lightrag-sidecar/ "$DEPLOY_HOST:$APP_DIR/packages/"
|
||||
|
||||
# Install on Erik
|
||||
cd /opt/llm-gateway/packages/lightrag-sidecar
|
||||
cd "$APP_DIR/packages/lightrag-sidecar"
|
||||
python -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
@ -17,7 +17,7 @@ class Settings(BaseSettings):
|
||||
|
||||
# LLM Backend
|
||||
LLM_BACKEND: Literal["ollama", "claude"] = "ollama"
|
||||
OLLAMA_URL: str = "http://192.168.178.213:11434"
|
||||
OLLAMA_URL: str = "http://localhost:11434"
|
||||
OLLAMA_MODEL: str = "qwen2.5:14b" # For entity extraction
|
||||
|
||||
# Vector Search
|
||||
@ -27,7 +27,7 @@ class Settings(BaseSettings):
|
||||
VECTOR_SIMILARITY_THRESHOLD: float = 0.7
|
||||
|
||||
# Database
|
||||
DATABASE_URL: str = "postgresql://tip_kg:password@localhost/tip_lightrag"
|
||||
DATABASE_URL: str = "postgresql://tip_kg@localhost/tip_lightrag"
|
||||
DB_POOL_SIZE: int = 10
|
||||
DB_ECHO: bool = False # SQL logging
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/**
|
||||
* PM2 Ecosystem Config — LightRAG Sidecar on Erik (217.154.82.179)
|
||||
* PM2 Ecosystem Config - LightRAG Sidecar
|
||||
*
|
||||
* Deploy: pm2 start packages/lightrag-sidecar/ecosystem.config.cjs
|
||||
* Reload: pm2 reload lightrag-sidecar
|
||||
@ -12,10 +12,10 @@ module.exports = {
|
||||
{
|
||||
name: 'lightrag-sidecar',
|
||||
script: 'app/main.py',
|
||||
cwd: '/opt/llm-gateway/packages/lightrag-sidecar',
|
||||
cwd: process.env.LIGHTRAG_CWD || process.cwd(),
|
||||
interpreter: '/usr/bin/python3',
|
||||
interpreter_args: '-m uvicorn',
|
||||
args: 'app.main:app --host 0.0.0.0 --port 3140 --workers 2',
|
||||
args: `app.main:app --host ${process.env.HOST || 'localhost'} --port ${process.env.PORT || '3140'} --workers 2`,
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
env: {
|
||||
@ -24,11 +24,11 @@ module.exports = {
|
||||
ENVIRONMENT: 'production',
|
||||
LIGHTRAG_DOMAIN: 'transceiver',
|
||||
LLM_BACKEND: 'ollama',
|
||||
OLLAMA_URL: 'https://ollama.fichtmueller.org',
|
||||
OLLAMA_URL: process.env.OLLAMA_URL || 'http://localhost:11434',
|
||||
OLLAMA_MODEL: 'qwen2.5:14b',
|
||||
QDRANT_URL: 'http://localhost:6333',
|
||||
EMBEDDING_MODEL: 'bge-m3',
|
||||
DATABASE_URL: 'postgresql://tip_kg:tip_secure_2026@localhost:5432/tip_lightrag',
|
||||
DATABASE_URL: process.env.DATABASE_URL || 'postgresql://tip_kg@localhost:5432/tip_lightrag',
|
||||
DB_POOL_SIZE: '10',
|
||||
MAX_WORKERS: '4',
|
||||
LOG_LEVEL: 'info',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user