From b6928265bf28541e3d35048bae9b1da8ae5adf60 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Mon, 6 Apr 2026 00:57:03 +0200 Subject: [PATCH] fix: serialize Claude API calls via queue to prevent 429 rate-limit spam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier-1 Anthropic API has 40K TPM — with ~20K tokens per pipeline step, concurrent calls immediately hit the limit. enqueueClaude() serializes all generateClaude() calls so only one runs at a time, eliminating the flood of 429-retry-429-retry loops. --- packages/api/src/llm/client.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/api/src/llm/client.ts b/packages/api/src/llm/client.ts index 794771d..5d97c7d 100644 --- a/packages/api/src/llm/client.ts +++ b/packages/api/src/llm/client.ts @@ -31,6 +31,16 @@ function sleep(ms: number): Promise { // ANTHROPIC CLAUDE PROVIDER // ═══════════════════════════════════════════════════════ +// Serialize Claude API calls to stay within TPM limits +// Tier-1 has 40,000 TPM — with ~20K tokens/step, only 1 concurrent call safe +let claudeQueue: Promise = Promise.resolve(); + +function enqueueClaude(fn: () => Promise): Promise { + const result = claudeQueue.then(() => fn()); + claudeQueue = result.catch(() => {}); + return result; +} + async function generateClaude( systemPrompt: string, userPrompt: string, @@ -40,6 +50,7 @@ async function generateClaude( throw new Error("ANTHROPIC_API_KEY not set — cannot use Claude provider"); } + return enqueueClaude(async () => { const startTime = Date.now(); const resp = await fetch("https://api.anthropic.com/v1/messages", { @@ -90,6 +101,7 @@ async function generateClaude( totalDuration: duration * 1_000_000, // ns for compat evalCount: data.usage.output_tokens, }; + }); // end enqueueClaude } // ═══════════════════════════════════════════════════════