rene.fichtmueller/llm-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gateway): compress subscription bridge requests

Browse Source
This commit is contained in:
Rene Fichtmueller 2026-07-17 19:22:22 +02:00
parent a5f07a9ef8
commit 812b695e1a
1 changed files with 51 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
packages/gateway/src/routes/completion.ts
View File

@ -728,13 +728,40 @@ function isCodexBridgeModel(model: string | undefined): boolean {
return /^(codex-default|codex-|gpt-5(?:\.|-|$))/i.test(model ?? '');
}
function chatMessagesToPrompt(messages: OpenAIChatCompletionRequest['messages']): string {
function chatMessagesToPrompt(
messages: OpenAIChatCompletionRequest['messages'],
options: { includeSystem?: boolean } = {}
): string {
return messages
.filter((message) => options.includeSystem !== false || message.role !== 'system')
.map((message) => `${message.role}: ${contentToText(message.content)}`)
.join('\n\n')
.trim();
}
function compressChatMessagesForBridge(body: OpenAIChatCompletionRequest): {
body: OpenAIChatCompletionRequest;
inputText: string;
compression: CompressionResult;
} {
const inputText = chatMessagesToPrompt(body.messages, { includeSystem: false });
const compression = compressContext(inputText, { enabled: true, mode: 'auto' });
if (!compression.applied) return { body, inputText, compression };
const systemMessages = body.messages.filter((message) => message.role === 'system');
return {
body: {
...body,
messages: [
...systemMessages,
{ role: 'user', content: compression.input },
],
},
inputText: compression.input,
compression,
};
}
interface SubscriptionBridgeTarget {
subscriptionId: string;
provider: string;
@ -857,12 +884,18 @@ async function postSubscriptionBridgeCompletion(
return { statusCode: chatResponse.status, ok: chatResponse.ok, data: chatData };
}
const system = body.messages
.filter((message) => message.role === 'system')
.map((message) => contentToText(message.content))
.filter(Boolean)
.join('\n\n');
const generateResponse = await fetch(`${bridgeUrl}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: body.model,
prompt: inputText,
system: system || undefined,
stream: false,
temperature: body.temperature ?? 0,
max_tokens: body.max_tokens,
@ -880,9 +913,10 @@ async function callSubscriptionBridgeChatCompletion(
const target = subscriptionBridgeTargetForModel(body.model);
if (!target) return null;
const inputText = chatMessagesToPrompt(body.messages);
const compressed = compressChatMessagesForBridge(body);
const inputText = compressed.inputText;
try {
const upstream = await postSubscriptionBridgeCompletion(target, body, inputText);
const upstream = await postSubscriptionBridgeCompletion(target, compressed.body, inputText);
if (!upstream.ok || upstream.data?.success === false) {
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
logger.warn(
@ -920,6 +954,7 @@ async function callSubscriptionBridgeChatCompletion(
subscription_id: target.subscriptionId,
latency_ms: Date.now() - startMs,
passthrough: true,
compression: buildCompressionResponse(compressed.compression),
},
},
};
@ -1400,10 +1435,12 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
if (isCodexBridgeModel(parsed.data.model)) {
try {
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://127.0.0.1:3253';
const inputText = typeof parsed.data.input === 'string'
const originalInputText = typeof parsed.data.input === 'string'
? parsed.data.input
: (Array.isArray(parsed.data.input)
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
const compression = compressContext(originalInputText, { enabled: true, mode: 'auto' });
const inputText = compression.input;
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@ -1421,11 +1458,17 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
}
if (upstream.ok && upstreamJson?.success !== false) {
const text = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
const respBody = toOpenAIResponsesResponse({ output: text, model: parsed.data.model, status: 'approved' }, parsed.data.model);
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(inputText.length / 4);
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(text.length / 4);
const respBody = toOpenAIResponsesResponse({
output: text,
model: parsed.data.model,
status: 'approved',
tokens: { in: bridgeTokensIn, out: bridgeTokensOut },
compression: buildCompressionResponse(compression),
}, parsed.data.model);
// Track against the merged OpenAI (ChatGPT+Codex) subscription pool.
try {
const subId = modelToSubscriptionId(parsed.data.model ?? '') ?? 'codex';
@ -1447,7 +1490,7 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
0,
Date.now() - startMs,
0,
false,
compression.applied,
undefined,
);
} catch (e) {
@ -1465,7 +1508,8 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
latency_ms: Date.now() - startMs,
confidence: 0,
cost_usd: 0,
compression_applied: false,
compression_applied: compression.applied,
metadata: { compression: buildCompressionResponse(compression) },
model: parsed.data.model ?? 'gpt-5.5',
} as any);
} catch (e) {