fix(gateway): redact outbound bridge prompts
This commit is contained in:
parent
d265320819
commit
6275c25e67
@ -0,0 +1,21 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { compressContext } from '../context-compressor.js';
|
||||||
|
|
||||||
|
describe('context compressor', () => {
|
||||||
|
it('checks short inputs without lossy compression', () => {
|
||||||
|
const result = compressContext('short operational prompt', { enabled: true, mode: 'auto' });
|
||||||
|
|
||||||
|
expect(result.applied).toBe(false);
|
||||||
|
expect(result.tokensSaved).toBe(0);
|
||||||
|
expect(result.tokensBefore).toBe(result.tokensAfter);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('compresses long contexts before routing', () => {
|
||||||
|
const result = compressContext('x'.repeat(80_000), { enabled: true, mode: 'auto' });
|
||||||
|
|
||||||
|
expect(result.applied).toBe(true);
|
||||||
|
expect(result.tokensSaved).toBeGreaterThan(0);
|
||||||
|
expect(result.tokensAfter).toBeLessThan(result.tokensBefore);
|
||||||
|
expect(result.input).toContain('[ctxlean omitted');
|
||||||
|
});
|
||||||
|
});
|
||||||
69
packages/gateway/src/modules/__tests__/pii-redaction.test.ts
Normal file
69
packages/gateway/src/modules/__tests__/pii-redaction.test.ts
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
import { afterEach, describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
getRedactMode,
|
||||||
|
redactPii,
|
||||||
|
restorePii,
|
||||||
|
shouldRedactFor,
|
||||||
|
} from '../pii-redaction.js';
|
||||||
|
|
||||||
|
const ORIGINAL_REDACT_PII_MODE = process.env['REDACT_PII_MODE'];
|
||||||
|
const ORIGINAL_PII_REDACTION_MODE = process.env['PII_REDACTION_MODE'];
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (ORIGINAL_REDACT_PII_MODE === undefined) {
|
||||||
|
delete process.env['REDACT_PII_MODE'];
|
||||||
|
} else {
|
||||||
|
process.env['REDACT_PII_MODE'] = ORIGINAL_REDACT_PII_MODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ORIGINAL_PII_REDACTION_MODE === undefined) {
|
||||||
|
delete process.env['PII_REDACTION_MODE'];
|
||||||
|
} else {
|
||||||
|
process.env['PII_REDACTION_MODE'] = ORIGINAL_PII_REDACTION_MODE;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('PII redaction', () => {
|
||||||
|
it('defaults to cloud_only redaction', () => {
|
||||||
|
delete process.env['REDACT_PII_MODE'];
|
||||||
|
delete process.env['PII_REDACTION_MODE'];
|
||||||
|
|
||||||
|
expect(getRedactMode()).toBe('cloud_only');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('supports explicit opt-out and the legacy alias', () => {
|
||||||
|
process.env['REDACT_PII_MODE'] = 'off';
|
||||||
|
process.env['PII_REDACTION_MODE'] = 'always';
|
||||||
|
expect(getRedactMode()).toBe('off');
|
||||||
|
|
||||||
|
delete process.env['REDACT_PII_MODE'];
|
||||||
|
process.env['PII_REDACTION_MODE'] = 'always';
|
||||||
|
expect(getRedactMode()).toBe('always');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('redacts and restores outward identifiers and API keys', () => {
|
||||||
|
const email = ['person', 'example.org'].join('@');
|
||||||
|
const ipAddress = [203, 0, 113, 42].join('.');
|
||||||
|
const apiKey = `sk-${'a'.repeat(24)}`;
|
||||||
|
const original = [
|
||||||
|
`Contact ${email} from ${ipAddress}.`,
|
||||||
|
`Use token ${apiKey}.`,
|
||||||
|
].join(' ');
|
||||||
|
|
||||||
|
const result = redactPii(original);
|
||||||
|
|
||||||
|
expect(result.redacted).toContain('<EMAIL_001>');
|
||||||
|
expect(result.redacted).toContain('<IP_ADDRESS_001>');
|
||||||
|
expect(result.redacted).toContain('<API_KEY_001>');
|
||||||
|
expect(result.redacted).not.toContain(email);
|
||||||
|
expect(result.redacted).not.toContain(ipAddress);
|
||||||
|
expect(result.redacted).not.toContain(apiKey);
|
||||||
|
expect(restorePii(result.redacted, result.restoreMap)).toBe(original);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('redacts cloud and bridge providers while leaving local providers raw in cloud_only mode', () => {
|
||||||
|
expect(shouldRedactFor('cloud_only', 'claude-bridge', 'gateway-test')).toBe(true);
|
||||||
|
expect(shouldRedactFor('cloud_only', 'api.openai.com', 'gateway-test')).toBe(true);
|
||||||
|
expect(shouldRedactFor('cloud_only', 'ollama', 'gateway-test')).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -10,9 +10,9 @@
|
|||||||
* trust boundary without redaction. The mapping lives only in process
|
* trust boundary without redaction. The mapping lives only in process
|
||||||
* memory for the duration of a single call — never persisted.
|
* memory for the duration of a single call — never persisted.
|
||||||
*
|
*
|
||||||
* Modes (env REDACT_PII_MODE):
|
* Modes (env REDACT_PII_MODE or PII_REDACTION_MODE):
|
||||||
* - off → no redaction
|
* - off → no redaction
|
||||||
* - cloud_only → redact when target provider is non-local
|
* - cloud_only → redact when target provider is non-local (default)
|
||||||
* (i.e. any *-bridge or hosted API; Ollama passes raw)
|
* (i.e. any *-bridge or hosted API; Ollama passes raw)
|
||||||
* - always → redact every call regardless of provider
|
* - always → redact every call regardless of provider
|
||||||
*
|
*
|
||||||
@ -23,7 +23,7 @@ import { logger } from '../observability/logger.js';
|
|||||||
export type RedactMode = 'off' | 'cloud_only' | 'always';
|
export type RedactMode = 'off' | 'cloud_only' | 'always';
|
||||||
export type PiiCategory =
|
export type PiiCategory =
|
||||||
| 'email' | 'phone' | 'credit_card' | 'iban' | 'ssn' | 'ip_address'
|
| 'email' | 'phone' | 'credit_card' | 'iban' | 'ssn' | 'ip_address'
|
||||||
| 'aws_key' | 'private_key' | 'jwt' | 'person_name';
|
| 'aws_key' | 'api_key' | 'private_key' | 'jwt' | 'person_name';
|
||||||
|
|
||||||
export interface PiiMatch {
|
export interface PiiMatch {
|
||||||
category: PiiCategory;
|
category: PiiCategory;
|
||||||
@ -113,6 +113,9 @@ const RULES: readonly DetectionRule[] = [
|
|||||||
// AWS access key IDs
|
// AWS access key IDs
|
||||||
{ category: 'aws_key', pattern: /\bAKIA[0-9A-Z]{16}\b/g },
|
{ category: 'aws_key', pattern: /\bAKIA[0-9A-Z]{16}\b/g },
|
||||||
|
|
||||||
|
// Common API access tokens (OpenAI-style, GitHub, GitLab, Slack)
|
||||||
|
{ category: 'api_key', pattern: /\b(?:sk-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{20,})\b/g },
|
||||||
|
|
||||||
// PEM-style private keys
|
// PEM-style private keys
|
||||||
{ category: 'private_key', pattern: /-----BEGIN (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----/g },
|
{ category: 'private_key', pattern: /-----BEGIN (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH |PGP |DSA )?PRIVATE KEY-----/g },
|
||||||
|
|
||||||
@ -127,7 +130,7 @@ const RULES: readonly DetectionRule[] = [
|
|||||||
|
|
||||||
const counters: Record<PiiCategory, number> = {
|
const counters: Record<PiiCategory, number> = {
|
||||||
email: 0, phone: 0, credit_card: 0, iban: 0, ssn: 0, ip_address: 0,
|
email: 0, phone: 0, credit_card: 0, iban: 0, ssn: 0, ip_address: 0,
|
||||||
aws_key: 0, private_key: 0, jwt: 0, person_name: 0,
|
aws_key: 0, api_key: 0, private_key: 0, jwt: 0, person_name: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
function tokenFor(category: PiiCategory, restoreMap: Map<string, string>): string {
|
function tokenFor(category: PiiCategory, restoreMap: Map<string, string>): string {
|
||||||
@ -190,7 +193,7 @@ export function restorePii(text: string, restoreMap: Map<string, string>): strin
|
|||||||
// ─── Mode helpers ────────────────────────────────────────────────────────────
|
// ─── Mode helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
export function getRedactMode(): RedactMode {
|
export function getRedactMode(): RedactMode {
|
||||||
const v = (process.env['REDACT_PII_MODE'] ?? 'off').toLowerCase();
|
const v = (process.env['REDACT_PII_MODE'] ?? process.env['PII_REDACTION_MODE'] ?? 'cloud_only').toLowerCase();
|
||||||
if (v === 'cloud_only' || v === 'always') return v;
|
if (v === 'cloud_only' || v === 'always') return v;
|
||||||
return 'off';
|
return 'off';
|
||||||
}
|
}
|
||||||
|
|||||||
@ -231,6 +231,111 @@ interface GatewayCompletionResult {
|
|||||||
body: Record<string, unknown>;
|
body: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function mergeRestoreMaps(target: Map<string, string>, source: Map<string, string>): void {
|
||||||
|
for (const [sourceToken, value] of source.entries()) {
|
||||||
|
const existing = Array.from(target.entries()).find(([, current]) => current === value);
|
||||||
|
if (existing) continue;
|
||||||
|
|
||||||
|
if (!target.has(sourceToken)) {
|
||||||
|
target.set(sourceToken, value);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const match = sourceToken.match(/^<(.+)_(\d+)>$/);
|
||||||
|
const prefix = match?.[1] ?? 'PII';
|
||||||
|
let next = 1;
|
||||||
|
let token = `<${prefix}_${String(next).padStart(3, '0')}>`;
|
||||||
|
while (target.has(token)) {
|
||||||
|
next += 1;
|
||||||
|
token = `<${prefix}_${String(next).padStart(3, '0')}>`;
|
||||||
|
}
|
||||||
|
target.set(token, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyPiiTokens(text: string, restoreMap: Map<string, string>): string {
|
||||||
|
let out = text;
|
||||||
|
const entries = Array.from(restoreMap.entries())
|
||||||
|
.filter(([, value]) => value.length > 0)
|
||||||
|
.sort((a, b) => b[1].length - a[1].length);
|
||||||
|
|
||||||
|
for (const [token, value] of entries) {
|
||||||
|
out = out.split(value).join(token);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildOutboundPiiRedactionMap(
|
||||||
|
text: string,
|
||||||
|
options: { providerName: string; caller: string; callId: string; language?: string }
|
||||||
|
): Promise<Map<string, string> | null> {
|
||||||
|
const redactMode = getRedactMode();
|
||||||
|
if (!text.trim() || !shouldRedactFor(redactMode, options.providerName, options.caller)) return null;
|
||||||
|
|
||||||
|
const restoreMap = new Map<string, string>();
|
||||||
|
const pii = redactPii(text);
|
||||||
|
mergeRestoreMaps(restoreMap, pii.restoreMap);
|
||||||
|
|
||||||
|
let nameCount = 0;
|
||||||
|
const presidioUrl = process.env['PRESIDIO_URL'];
|
||||||
|
if (presidioUrl) {
|
||||||
|
try {
|
||||||
|
const nameResult = await redactPersonNamesAsync(applyPiiTokens(text, restoreMap), presidioUrl, options.language ?? 'de');
|
||||||
|
nameCount = nameResult.count;
|
||||||
|
mergeRestoreMaps(restoreMap, nameResult.restoreMap);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err, callId: options.callId }, 'Presidio name redaction failed — continuing');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (restoreMap.size === 0) return null;
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
callId: options.callId,
|
||||||
|
caller: options.caller,
|
||||||
|
provider: options.providerName,
|
||||||
|
redactedCounts: pii.counts,
|
||||||
|
redactedNames: nameCount,
|
||||||
|
redactedTokens: restoreMap.size,
|
||||||
|
},
|
||||||
|
'PII redaction applied before outbound LLM call',
|
||||||
|
);
|
||||||
|
return restoreMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
function redactChatContent(
|
||||||
|
content: OpenAIChatCompletionRequest['messages'][number]['content'],
|
||||||
|
restoreMap: Map<string, string>
|
||||||
|
): OpenAIChatCompletionRequest['messages'][number]['content'] {
|
||||||
|
if (typeof content === 'string') return applyPiiTokens(content, restoreMap);
|
||||||
|
if (!Array.isArray(content)) return content;
|
||||||
|
|
||||||
|
return content.map((part) => {
|
||||||
|
if (typeof part === 'string') return applyPiiTokens(part, restoreMap);
|
||||||
|
if (!part || typeof part !== 'object') return part;
|
||||||
|
|
||||||
|
const copy: Record<string, unknown> = { ...(part as Record<string, unknown>) };
|
||||||
|
for (const key of ['text', 'input_text', 'output_text']) {
|
||||||
|
if (typeof copy[key] === 'string') copy[key] = applyPiiTokens(copy[key] as string, restoreMap);
|
||||||
|
}
|
||||||
|
return copy;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function redactChatCompletionBody(
|
||||||
|
body: OpenAIChatCompletionRequest,
|
||||||
|
restoreMap: Map<string, string>
|
||||||
|
): OpenAIChatCompletionRequest {
|
||||||
|
return {
|
||||||
|
...body,
|
||||||
|
messages: body.messages.map((message) => ({
|
||||||
|
...message,
|
||||||
|
content: redactChatContent(message.content, restoreMap),
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// input: string,
|
// input: string,
|
||||||
// caller: string,
|
// caller: string,
|
||||||
@ -397,7 +502,8 @@ function buildResponseBody(callId: string, decision: ReturnType<typeof route>, t
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function executeCompletion(body: CompletionRequest, startMs: number, callId: string): Promise<GatewayCompletionResult> {
|
async function executeCompletion(body: CompletionRequest, startMs: number, callId: string): Promise<GatewayCompletionResult> {
|
||||||
const { caller, language, context, options } = body;
|
const { caller, language, options } = body;
|
||||||
|
let context = body.context;
|
||||||
|
|
||||||
// ─── Plugin pre-hooks (PLUGINS_DIR) ────────────────────────────────────
|
// ─── Plugin pre-hooks (PLUGINS_DIR) ────────────────────────────────────
|
||||||
try {
|
try {
|
||||||
@ -413,33 +519,18 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ─── PII Redaction (REDACT_PII_MODE: off|cloud_only|always) ─────────────
|
// ─── PII Redaction (REDACT_PII_MODE: off|cloud_only|always) ─────────────
|
||||||
const redactMode = getRedactMode();
|
const systemContext = typeof context === 'object' && context && 'system' in context
|
||||||
let piiRestoreMap: Map<string, string> | null = null;
|
? String((context as any).system ?? '')
|
||||||
if (redactMode !== 'off' && shouldRedactFor(redactMode, 'unknown', caller)) {
|
: '';
|
||||||
const r = redactPii(body.input);
|
const piiRestoreMap = await buildOutboundPiiRedactionMap(
|
||||||
if (r.restoreMap.size > 0) {
|
[body.input, systemContext].filter(Boolean).join('\n\n'),
|
||||||
body = { ...body, input: r.redacted };
|
{ providerName: 'unknown', caller, callId, language }
|
||||||
piiRestoreMap = r.restoreMap;
|
|
||||||
logger.info(
|
|
||||||
{ callId, caller, redactedCounts: r.counts, redactedTokens: r.restoreMap.size },
|
|
||||||
'PII redaction applied',
|
|
||||||
);
|
);
|
||||||
}
|
if (piiRestoreMap) {
|
||||||
}
|
body = { ...body, input: applyPiiTokens(body.input, piiRestoreMap) };
|
||||||
|
if (systemContext && context) {
|
||||||
// ─── PII: person names via Presidio+GLiNER sidecar (async, non-blocking) ──
|
context = { ...context, system: applyPiiTokens(systemContext, piiRestoreMap) };
|
||||||
const presidioUrl = process.env['PRESIDIO_URL'];
|
body = { ...body, context };
|
||||||
if (presidioUrl && redactMode !== 'off' && shouldRedactFor(redactMode, 'unknown', caller)) {
|
|
||||||
try {
|
|
||||||
const nameResult = await redactPersonNamesAsync(body.input, presidioUrl);
|
|
||||||
if (nameResult.count > 0) {
|
|
||||||
body = { ...body, input: nameResult.redacted };
|
|
||||||
if (!piiRestoreMap) piiRestoreMap = new Map();
|
|
||||||
for (const [k, v] of nameResult.restoreMap) piiRestoreMap.set(k, v);
|
|
||||||
logger.info({ callId, caller, names: nameResult.count }, 'Person names redacted via Presidio');
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn({ err, callId }, 'Presidio name redaction failed — continuing');
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -517,7 +608,7 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
|||||||
|
|
||||||
// ─── Cache check (Tier 1: exact-match hash lookup) ─────────────────────
|
// ─── Cache check (Tier 1: exact-match hash lookup) ─────────────────────
|
||||||
const agenticNoCache = shouldBypassResponseCache(caller);
|
const agenticNoCache = shouldBypassResponseCache(caller);
|
||||||
const skipCache = agenticNoCache || (options as any)?.skip_cache === true;
|
const skipCache = agenticNoCache || (options as any)?.skip_cache === true || Boolean(piiRestoreMap);
|
||||||
const cacheableReq = {
|
const cacheableReq = {
|
||||||
caller,
|
caller,
|
||||||
task_type: body.task_type,
|
task_type: body.task_type,
|
||||||
@ -554,6 +645,14 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
|||||||
if (hit) {
|
if (hit) {
|
||||||
const latencyMs = Date.now() - startMs;
|
const latencyMs = Date.now() - startMs;
|
||||||
void recordCacheHit(dbForCache, hit.id);
|
void recordCacheHit(dbForCache, hit.id);
|
||||||
|
void logCompressionMetric(dbForCache, {
|
||||||
|
filePath: callId,
|
||||||
|
mode: 'cache:none',
|
||||||
|
tokensBefore: hit.tokensIn,
|
||||||
|
tokensAfter: hit.tokensIn,
|
||||||
|
savingsPct: 0,
|
||||||
|
toolUsed: 'gateway',
|
||||||
|
});
|
||||||
// Log cache hit as a successful request (status=approved, fallback=false)
|
// Log cache hit as a successful request (status=approved, fallback=false)
|
||||||
const requestLogger = createRequestLogger(dbForCache);
|
const requestLogger = createRequestLogger(dbForCache);
|
||||||
void requestLogger.logRequest(
|
void requestLogger.logRequest(
|
||||||
@ -655,12 +754,14 @@ async function executeCompletion(body: CompletionRequest, startMs: number, callI
|
|||||||
}
|
}
|
||||||
|
|
||||||
const latencyMs = Date.now() - startMs;
|
const latencyMs = Date.now() - startMs;
|
||||||
const outputText = ollamaResponse.response;
|
const modelOutputText = ollamaResponse.response;
|
||||||
|
const outputText = piiRestoreMap ? restorePii(modelOutputText, piiRestoreMap) : modelOutputText;
|
||||||
const validationOutput = await runPostValidation(outputText, { validators: decision.validators, language, output_format: decision.output_format, requires_fact_check: decision.requires_fact_check, schema: resolved.schema });
|
const validationOutput = await runPostValidation(outputText, { validators: decision.validators, language, output_format: decision.output_format, requires_fact_check: decision.requires_fact_check, schema: resolved.schema });
|
||||||
const confidenceResult = evaluateConfidence(validationOutput);
|
const confidenceResult = evaluateConfidence(validationOutput);
|
||||||
|
|
||||||
recordAllMetrics(caller, taskType, confidenceResult, ollamaResponse, decision, validationOutput);
|
recordAllMetrics(caller, taskType, confidenceResult, ollamaResponse, decision, validationOutput);
|
||||||
const { costUsd, costSavedUsd } = await auditAndTrackCosts(caller, taskType, compression.originalInput, outputText, latencyMs, ollamaResponse, resolved, decision, confidenceResult, validationOutput, classificationResult, callId, compression);
|
const auditOutputText = piiRestoreMap ? modelOutputText : outputText;
|
||||||
|
const { costUsd, costSavedUsd } = await auditAndTrackCosts(caller, taskType, compression.originalInput, auditOutputText, latencyMs, ollamaResponse, resolved, decision, confidenceResult, validationOutput, classificationResult, callId, compression);
|
||||||
|
|
||||||
latencySeconds.labels({ caller, task_type: taskType, model: ollamaResponse.model ?? decision.model }).observe(latencyMs / 1000);
|
latencySeconds.labels({ caller, task_type: taskType, model: ollamaResponse.model ?? decision.model }).observe(latencyMs / 1000);
|
||||||
|
|
||||||
@ -966,16 +1067,24 @@ async function callSubscriptionBridgeChatCompletion(
|
|||||||
body: OpenAIChatCompletionRequest,
|
body: OpenAIChatCompletionRequest,
|
||||||
startMs: number,
|
startMs: number,
|
||||||
callId: string,
|
callId: string,
|
||||||
requestedSubscriptionId?: string
|
requestedSubscriptionId: string | undefined,
|
||||||
|
caller: string
|
||||||
): Promise<{ ok: true; body: Record<string, unknown> } | { ok: false; statusCode: number; error: string } | null> {
|
): Promise<{ ok: true; body: Record<string, unknown> } | { ok: false; statusCode: number; error: string } | null> {
|
||||||
const target = subscriptionBridgeTargetForSubscriptionId(requestedSubscriptionId)
|
const target = subscriptionBridgeTargetForSubscriptionId(requestedSubscriptionId)
|
||||||
?? subscriptionBridgeTargetForModel(body.model);
|
?? subscriptionBridgeTargetForModel(body.model);
|
||||||
if (!target) return null;
|
if (!target) return null;
|
||||||
|
|
||||||
const compressed = compressChatMessagesForBridge(body);
|
const compressed = compressChatMessagesForBridge(body);
|
||||||
const inputText = compressed.inputText;
|
const outboundRedactionMap = await buildOutboundPiiRedactionMap(
|
||||||
|
chatMessagesToPrompt(compressed.body.messages, { includeSystem: true }),
|
||||||
|
{ providerName: target.provider, caller, callId, language: 'en' }
|
||||||
|
);
|
||||||
|
const outboundBody = outboundRedactionMap
|
||||||
|
? redactChatCompletionBody(compressed.body, outboundRedactionMap)
|
||||||
|
: compressed.body;
|
||||||
|
const inputText = chatMessagesToPrompt(outboundBody.messages, { includeSystem: false });
|
||||||
try {
|
try {
|
||||||
const upstream = await postSubscriptionBridgeCompletion(target, compressed.body, inputText);
|
const upstream = await postSubscriptionBridgeCompletion(target, outboundBody, inputText);
|
||||||
if (!upstream.ok || upstream.data?.success === false) {
|
if (!upstream.ok || upstream.data?.success === false) {
|
||||||
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
|
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@ -985,9 +1094,39 @@ async function callSubscriptionBridgeChatCompletion(
|
|||||||
return { ok: false, statusCode: 424, error };
|
return { ok: false, statusCode: 424, error };
|
||||||
}
|
}
|
||||||
|
|
||||||
const text = contentFromSubscriptionBridge(upstream.data);
|
const modelText = contentFromSubscriptionBridge(upstream.data);
|
||||||
|
const text = outboundRedactionMap ? restorePii(String(modelText), outboundRedactionMap) : String(modelText);
|
||||||
const promptTokens = upstream.data?.usage?.prompt_tokens ?? Math.ceil(inputText.length / 4);
|
const promptTokens = upstream.data?.usage?.prompt_tokens ?? Math.ceil(inputText.length / 4);
|
||||||
const completionTokens = upstream.data?.usage?.completion_tokens ?? Math.ceil(String(text).length / 4);
|
const completionTokens = upstream.data?.usage?.completion_tokens ?? Math.ceil(String(modelText).length / 4);
|
||||||
|
const latencyMs = Date.now() - startMs;
|
||||||
|
try {
|
||||||
|
const db = getPool();
|
||||||
|
void logCompressionMetric(db, {
|
||||||
|
filePath: callId,
|
||||||
|
mode: `${compressed.compression.method}:${compressed.compression.strategy}`,
|
||||||
|
tokensBefore: compressed.compression.tokensBefore,
|
||||||
|
tokensAfter: compressed.compression.tokensAfter,
|
||||||
|
savingsPct: Math.round(compressed.compression.ratio * 10000) / 100,
|
||||||
|
toolUsed: 'gateway',
|
||||||
|
});
|
||||||
|
void createRequestLogger(db).logRequest(
|
||||||
|
callId,
|
||||||
|
caller,
|
||||||
|
'subscription_bridge',
|
||||||
|
body.model,
|
||||||
|
'approved',
|
||||||
|
promptTokens,
|
||||||
|
completionTokens,
|
||||||
|
0,
|
||||||
|
latencyMs,
|
||||||
|
0,
|
||||||
|
false,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
void recordSubscriptionUsage(db, target.subscriptionId, promptTokens + completionTokens);
|
||||||
|
} catch (e) {
|
||||||
|
logger.warn({ e, callId }, 'failed to log subscription bridge request');
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
ok: true,
|
ok: true,
|
||||||
body: {
|
body: {
|
||||||
@ -1012,7 +1151,7 @@ async function callSubscriptionBridgeChatCompletion(
|
|||||||
provider: target.provider,
|
provider: target.provider,
|
||||||
subscription_id: target.subscriptionId,
|
subscription_id: target.subscriptionId,
|
||||||
subscription_forced: Boolean(requestedSubscriptionId),
|
subscription_forced: Boolean(requestedSubscriptionId),
|
||||||
latency_ms: Date.now() - startMs,
|
latency_ms: latencyMs,
|
||||||
passthrough: true,
|
passthrough: true,
|
||||||
compression: buildCompressionResponse(compressed.compression),
|
compression: buildCompressionResponse(compressed.compression),
|
||||||
},
|
},
|
||||||
@ -1399,7 +1538,8 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
|
|
||||||
const callId = `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
const callId = `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||||
const requestedSubscriptionId = requestedSubscriptionFromHeaders(request);
|
const requestedSubscriptionId = requestedSubscriptionFromHeaders(request);
|
||||||
const subscriptionBridgeResult = await callSubscriptionBridgeChatCompletion(parsed.data, startMs, callId, requestedSubscriptionId);
|
const { caller: bridgeCaller } = detectCaller(request, 'openai-compatible', parsed.data.user);
|
||||||
|
const subscriptionBridgeResult = await callSubscriptionBridgeChatCompletion(parsed.data, startMs, callId, requestedSubscriptionId, bridgeCaller);
|
||||||
if (requestedSubscriptionId && !subscriptionBridgeResult) {
|
if (requestedSubscriptionId && !subscriptionBridgeResult) {
|
||||||
return reply.status(404).send({
|
return reply.status(404).send({
|
||||||
error: {
|
error: {
|
||||||
@ -1505,18 +1645,24 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
if (isCodexBridgeModel(parsed.data.model)) {
|
if (isCodexBridgeModel(parsed.data.model)) {
|
||||||
try {
|
try {
|
||||||
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://localhost:3253';
|
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://localhost:3253';
|
||||||
|
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
||||||
const originalInputText = typeof parsed.data.input === 'string'
|
const originalInputText = typeof parsed.data.input === 'string'
|
||||||
? parsed.data.input
|
? parsed.data.input
|
||||||
: (Array.isArray(parsed.data.input)
|
: (Array.isArray(parsed.data.input)
|
||||||
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
|
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
|
||||||
const compression = compressContext(originalInputText, { enabled: true, mode: 'auto' });
|
const compression = compressContext(originalInputText, { enabled: true, mode: 'auto' });
|
||||||
const inputText = compression.input;
|
const inputText = compression.input;
|
||||||
|
const outboundRedactionMap = await buildOutboundPiiRedactionMap(
|
||||||
|
inputText,
|
||||||
|
{ providerName: 'codex-bridge', caller: bridgeCaller, callId, language: 'en' }
|
||||||
|
);
|
||||||
|
const outboundInputText = outboundRedactionMap ? applyPiiTokens(inputText, outboundRedactionMap) : inputText;
|
||||||
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
|
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: parsed.data.model,
|
model: parsed.data.model,
|
||||||
messages: [{ role: 'user', content: inputText }],
|
messages: [{ role: 'user', content: outboundInputText }],
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
const upstreamText = await upstream.text();
|
const upstreamText = await upstream.text();
|
||||||
@ -1527,11 +1673,11 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
upstreamJson = { raw: upstreamText };
|
upstreamJson = { raw: upstreamText };
|
||||||
}
|
}
|
||||||
if (upstream.ok && upstreamJson?.success !== false) {
|
if (upstream.ok && upstreamJson?.success !== false) {
|
||||||
const text = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
const modelText = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
||||||
|
const text = outboundRedactionMap ? restorePii(String(modelText), outboundRedactionMap) : String(modelText);
|
||||||
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
|
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
|
||||||
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(outboundInputText.length / 4);
|
||||||
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(inputText.length / 4);
|
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(String(modelText).length / 4);
|
||||||
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(text.length / 4);
|
|
||||||
const respBody = toOpenAIResponsesResponse({
|
const respBody = toOpenAIResponsesResponse({
|
||||||
output: text,
|
output: text,
|
||||||
model: parsed.data.model,
|
model: parsed.data.model,
|
||||||
@ -1548,7 +1694,16 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
}
|
}
|
||||||
// Write to request_tracking so dashboard token totals include bridge calls.
|
// Write to request_tracking so dashboard token totals include bridge calls.
|
||||||
try {
|
try {
|
||||||
const bridgeLogger = createRequestLogger(getPool());
|
const db = getPool();
|
||||||
|
void logCompressionMetric(db, {
|
||||||
|
filePath: callId,
|
||||||
|
mode: `${compression.method}:${compression.strategy}`,
|
||||||
|
tokensBefore: compression.tokensBefore,
|
||||||
|
tokensAfter: compression.tokensAfter,
|
||||||
|
savingsPct: Math.round(compression.ratio * 10000) / 100,
|
||||||
|
toolUsed: 'gateway',
|
||||||
|
});
|
||||||
|
const bridgeLogger = createRequestLogger(db);
|
||||||
void bridgeLogger.logRequest(
|
void bridgeLogger.logRequest(
|
||||||
callId,
|
callId,
|
||||||
bridgeCaller,
|
bridgeCaller,
|
||||||
|
|||||||
@ -40,6 +40,7 @@ import { getSubscriptionWallet, recordSubscriptionUsage } from '../modules/subsc
|
|||||||
import { rememberFact, recallFacts, forgetCaller } from '../modules/knowledge-memory.js';
|
import { rememberFact, recallFacts, forgetCaller } from '../modules/knowledge-memory.js';
|
||||||
import { getRaceStats } from '../modules/race-mode.js';
|
import { getRaceStats } from '../modules/race-mode.js';
|
||||||
import { dashboardAuthStatus, requireDashboardAuth } from '../modules/admin-auth.js';
|
import { dashboardAuthStatus, requireDashboardAuth } from '../modules/admin-auth.js';
|
||||||
|
import { logCompressionMetric } from '../utils/tokenvault-hooks.js';
|
||||||
|
|
||||||
const execFileAsync = promisify(execFile);
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
@ -1760,6 +1761,14 @@ export async function dashboardRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
await requestLogger.logRequest(requestId, caller, 'usage_import', model, 'approved', tokensIn, tokensOut, costUsd, 0);
|
await requestLogger.logRequest(requestId, caller, 'usage_import', model, 'approved', tokensIn, tokensOut, costUsd, 0);
|
||||||
await db.query(`UPDATE request_tracking SET created_at=$1 WHERE request_id=$2`, [usedAt, requestId]);
|
await db.query(`UPDATE request_tracking SET created_at=$1 WHERE request_id=$2`, [usedAt, requestId]);
|
||||||
}
|
}
|
||||||
|
await logCompressionMetric(db, {
|
||||||
|
filePath: requestId,
|
||||||
|
mode: 'usage-import:none',
|
||||||
|
tokensBefore: tokensIn,
|
||||||
|
tokensAfter: tokensIn,
|
||||||
|
savingsPct: 0,
|
||||||
|
toolUsed: 'gateway',
|
||||||
|
});
|
||||||
return reply.status(200).send({ success: true, imported: { caller, model, day, tokensIn, tokensOut, costUsd, usedAt } });
|
return reply.status(200).send({ success: true, imported: { caller, model, day, tokensIn, tokensOut, costUsd, usedAt } });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error({ error }, 'Failed to import usage report');
|
logger.error({ error }, 'Failed to import usage report');
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user