fix(gateway): compress subscription bridge requests
This commit is contained in:
parent
a5f07a9ef8
commit
812b695e1a
@ -728,13 +728,40 @@ function isCodexBridgeModel(model: string | undefined): boolean {
|
|||||||
return /^(codex-default|codex-|gpt-5(?:\.|-|$))/i.test(model ?? '');
|
return /^(codex-default|codex-|gpt-5(?:\.|-|$))/i.test(model ?? '');
|
||||||
}
|
}
|
||||||
|
|
||||||
function chatMessagesToPrompt(messages: OpenAIChatCompletionRequest['messages']): string {
|
function chatMessagesToPrompt(
|
||||||
|
messages: OpenAIChatCompletionRequest['messages'],
|
||||||
|
options: { includeSystem?: boolean } = {}
|
||||||
|
): string {
|
||||||
return messages
|
return messages
|
||||||
|
.filter((message) => options.includeSystem !== false || message.role !== 'system')
|
||||||
.map((message) => `${message.role}: ${contentToText(message.content)}`)
|
.map((message) => `${message.role}: ${contentToText(message.content)}`)
|
||||||
.join('\n\n')
|
.join('\n\n')
|
||||||
.trim();
|
.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function compressChatMessagesForBridge(body: OpenAIChatCompletionRequest): {
|
||||||
|
body: OpenAIChatCompletionRequest;
|
||||||
|
inputText: string;
|
||||||
|
compression: CompressionResult;
|
||||||
|
} {
|
||||||
|
const inputText = chatMessagesToPrompt(body.messages, { includeSystem: false });
|
||||||
|
const compression = compressContext(inputText, { enabled: true, mode: 'auto' });
|
||||||
|
if (!compression.applied) return { body, inputText, compression };
|
||||||
|
|
||||||
|
const systemMessages = body.messages.filter((message) => message.role === 'system');
|
||||||
|
return {
|
||||||
|
body: {
|
||||||
|
...body,
|
||||||
|
messages: [
|
||||||
|
...systemMessages,
|
||||||
|
{ role: 'user', content: compression.input },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
inputText: compression.input,
|
||||||
|
compression,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
interface SubscriptionBridgeTarget {
|
interface SubscriptionBridgeTarget {
|
||||||
subscriptionId: string;
|
subscriptionId: string;
|
||||||
provider: string;
|
provider: string;
|
||||||
@ -857,12 +884,18 @@ async function postSubscriptionBridgeCompletion(
|
|||||||
return { statusCode: chatResponse.status, ok: chatResponse.ok, data: chatData };
|
return { statusCode: chatResponse.status, ok: chatResponse.ok, data: chatData };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const system = body.messages
|
||||||
|
.filter((message) => message.role === 'system')
|
||||||
|
.map((message) => contentToText(message.content))
|
||||||
|
.filter(Boolean)
|
||||||
|
.join('\n\n');
|
||||||
const generateResponse = await fetch(`${bridgeUrl}/api/generate`, {
|
const generateResponse = await fetch(`${bridgeUrl}/api/generate`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: body.model,
|
model: body.model,
|
||||||
prompt: inputText,
|
prompt: inputText,
|
||||||
|
system: system || undefined,
|
||||||
stream: false,
|
stream: false,
|
||||||
temperature: body.temperature ?? 0,
|
temperature: body.temperature ?? 0,
|
||||||
max_tokens: body.max_tokens,
|
max_tokens: body.max_tokens,
|
||||||
@ -880,9 +913,10 @@ async function callSubscriptionBridgeChatCompletion(
|
|||||||
const target = subscriptionBridgeTargetForModel(body.model);
|
const target = subscriptionBridgeTargetForModel(body.model);
|
||||||
if (!target) return null;
|
if (!target) return null;
|
||||||
|
|
||||||
const inputText = chatMessagesToPrompt(body.messages);
|
const compressed = compressChatMessagesForBridge(body);
|
||||||
|
const inputText = compressed.inputText;
|
||||||
try {
|
try {
|
||||||
const upstream = await postSubscriptionBridgeCompletion(target, body, inputText);
|
const upstream = await postSubscriptionBridgeCompletion(target, compressed.body, inputText);
|
||||||
if (!upstream.ok || upstream.data?.success === false) {
|
if (!upstream.ok || upstream.data?.success === false) {
|
||||||
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
|
const error = bridgeErrorMessage(upstream.data, upstream.statusCode);
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@ -920,6 +954,7 @@ async function callSubscriptionBridgeChatCompletion(
|
|||||||
subscription_id: target.subscriptionId,
|
subscription_id: target.subscriptionId,
|
||||||
latency_ms: Date.now() - startMs,
|
latency_ms: Date.now() - startMs,
|
||||||
passthrough: true,
|
passthrough: true,
|
||||||
|
compression: buildCompressionResponse(compressed.compression),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@ -1400,10 +1435,12 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
if (isCodexBridgeModel(parsed.data.model)) {
|
if (isCodexBridgeModel(parsed.data.model)) {
|
||||||
try {
|
try {
|
||||||
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://127.0.0.1:3253';
|
const bridgeUrl = process.env['CODEX_BRIDGE_URL'] ?? 'http://127.0.0.1:3253';
|
||||||
const inputText = typeof parsed.data.input === 'string'
|
const originalInputText = typeof parsed.data.input === 'string'
|
||||||
? parsed.data.input
|
? parsed.data.input
|
||||||
: (Array.isArray(parsed.data.input)
|
: (Array.isArray(parsed.data.input)
|
||||||
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
|
? parsed.data.input.map((p: any) => typeof p?.content === 'string' ? p.content : (Array.isArray(p?.content) ? p.content.map((c: any) => c?.text ?? '').join(' ') : '')).join(' ') : '');
|
||||||
|
const compression = compressContext(originalInputText, { enabled: true, mode: 'auto' });
|
||||||
|
const inputText = compression.input;
|
||||||
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
|
const upstream = await fetch(`${bridgeUrl}/v1/chat/completions`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
@ -1421,11 +1458,17 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
}
|
}
|
||||||
if (upstream.ok && upstreamJson?.success !== false) {
|
if (upstream.ok && upstreamJson?.success !== false) {
|
||||||
const text = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
const text = upstreamJson?.content ?? upstreamJson?.response ?? upstreamJson?.choices?.[0]?.message?.content ?? '';
|
||||||
const respBody = toOpenAIResponsesResponse({ output: text, model: parsed.data.model, status: 'approved' }, parsed.data.model);
|
|
||||||
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
|
logger.info({ callId, model: parsed.data.model, len: text.length }, 'codex-bridge passthrough OK');
|
||||||
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
const bridgeCaller = (request.headers['x-llm-interceptor-caller'] as string) || 'codex-app';
|
||||||
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(inputText.length / 4);
|
const bridgeTokensIn = (upstreamJson?.usage?.prompt_tokens as number | undefined) ?? Math.ceil(inputText.length / 4);
|
||||||
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(text.length / 4);
|
const bridgeTokensOut = (upstreamJson?.usage?.completion_tokens as number | undefined) ?? Math.ceil(text.length / 4);
|
||||||
|
const respBody = toOpenAIResponsesResponse({
|
||||||
|
output: text,
|
||||||
|
model: parsed.data.model,
|
||||||
|
status: 'approved',
|
||||||
|
tokens: { in: bridgeTokensIn, out: bridgeTokensOut },
|
||||||
|
compression: buildCompressionResponse(compression),
|
||||||
|
}, parsed.data.model);
|
||||||
// Track against the merged OpenAI (ChatGPT+Codex) subscription pool.
|
// Track against the merged OpenAI (ChatGPT+Codex) subscription pool.
|
||||||
try {
|
try {
|
||||||
const subId = modelToSubscriptionId(parsed.data.model ?? '') ?? 'codex';
|
const subId = modelToSubscriptionId(parsed.data.model ?? '') ?? 'codex';
|
||||||
@ -1447,7 +1490,7 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
0,
|
0,
|
||||||
Date.now() - startMs,
|
Date.now() - startMs,
|
||||||
0,
|
0,
|
||||||
false,
|
compression.applied,
|
||||||
undefined,
|
undefined,
|
||||||
);
|
);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@ -1465,7 +1508,8 @@ export async function completionRoute(fastify: FastifyInstance): Promise<void> {
|
|||||||
latency_ms: Date.now() - startMs,
|
latency_ms: Date.now() - startMs,
|
||||||
confidence: 0,
|
confidence: 0,
|
||||||
cost_usd: 0,
|
cost_usd: 0,
|
||||||
compression_applied: false,
|
compression_applied: compression.applied,
|
||||||
|
metadata: { compression: buildCompressionResponse(compression) },
|
||||||
model: parsed.data.model ?? 'gpt-5.5',
|
model: parsed.data.model ?? 'gpt-5.5',
|
||||||
} as any);
|
} as any);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user