/** * LLM client for blog generation — supports Ollama (local), Anthropic Claude (API), * and Claude-Code (flat-rate via claude-bridge on Erik). * * Provider selection: * BLOG_LLM_PROVIDER=claude-code → Claude via claude-bridge (flat-rate, recommended) * BLOG_LLM_PROVIDER=anthropic → Claude Sonnet/Haiku via Anthropic API * BLOG_LLM_PROVIDER=ollama → local adapter bridge / Ollama-compatible endpoint (default) * * Claude-code is preferred: uses Claude Code subscription (flat-rate), no API costs. * The default local blog model is the latest RunPod-trained FO_BlogLLM adapter. */ import { existsSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434"; const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || ""; const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-sonnet-4-20250514"; const CLAUDE_BRIDGE_URL = process.env.CLAUDE_BRIDGE_URL || "http://localhost:3250"; // ── Runtime-switchable provider state ────────────────────────────────────── // Reads from /opt/tip/blog-llm-settings.json if present (written by /api/blog/llm/switch). // Falls back to process.env, then to defaults. No restart required for switches. const SETTINGS_FILE = join(process.env.TIP_ROOT || "/opt/tip", "blog-llm-settings.json"); interface LlmSettings { provider: string; ollamaModel: string } function loadSettings(): LlmSettings { try { if (existsSync(SETTINGS_FILE)) { const raw = JSON.parse(readFileSync(SETTINGS_FILE, "utf8")) as LlmSettings; return { provider: raw.provider || "ollama", ollamaModel: raw.ollamaModel || "fo-blog-v7" }; } } catch { /* ignore corrupt file */ } return { provider: process.env.BLOG_LLM_PROVIDER || "ollama", ollamaModel: process.env.OLLAMA_LLM_MODEL || "fo-blog-v7", }; } let _settings = loadSettings(); /** Switch the active LLM provider at runtime. Persists to settings file. */ export function setLlmProvider(provider: string, ollamaModel?: string): void { _settings = { provider, ollamaModel: ollamaModel || _settings.ollamaModel }; try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}`); } /** Returns the currently active provider config. */ export function getLlmProvider(): LlmSettings { return { ..._settings }; } // Convenience getters used below (re-read on every call for zero-latency switch) function provider(): string { return _settings.provider; } function llmModel(): string { return _settings.ollamaModel; } interface LlmResponse { text: string; model: string; totalDuration: number; evalCount: number; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } // ═══════════════════════════════════════════════════════ // ANTHROPIC CLAUDE PROVIDER // ═══════════════════════════════════════════════════════ // Serialize Claude API calls to stay within TPM limits // Tier-1 has 40,000 TPM — with ~20K tokens/step, only 1 concurrent call safe let claudeQueue: Promise = Promise.resolve(); let claudeQueueEnqueueTime = 0; export function resetClaudeQueue(): void { claudeQueue = Promise.resolve(); claudeQueueEnqueueTime = 0; console.log("[LLM] Claude queue reset — previous stuck requests cleared"); } function enqueueClaude(fn: () => Promise): Promise { claudeQueueEnqueueTime = Date.now(); const result = claudeQueue.then(() => { // Auto-reset if queue has been stalled > 15 minutes (prevents deadlock on stuck requests) if (Date.now() - claudeQueueEnqueueTime > 900000) { console.warn("[LLM] Claude queue auto-reset after 15min stall"); return Promise.reject(new Error("Claude queue auto-reset: previous request timed out")); } return fn(); }); claudeQueue = result.catch(() => {}); return result; } // Direct API call without going through the serialization queue — used for 429 retries // to avoid the circular-promise deadlock that recursive enqueueClaude creates async function claudeApiCall( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, retryCount = 0, ): Promise { const startTime = Date.now(); const MAX_RETRIES = 3; const RETRY_DELAYS = [10000, 30000, 60000]; const resp = await fetch("https://api.anthropic.com/v1/messages", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": ANTHROPIC_API_KEY, "anthropic-version": "2023-06-01", }, body: JSON.stringify({ model: ANTHROPIC_MODEL, max_tokens: options?.maxTokens ?? 4096, temperature: options?.temperature ?? 0.7, system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (!resp.ok) { const errText = await resp.text(); if (resp.status === 429 && retryCount < MAX_RETRIES) { const delay = RETRY_DELAYS[retryCount] ?? 60000; console.log(`[LLM] Claude 429 — retrying in ${delay / 1000}s (attempt ${retryCount + 1}/${MAX_RETRIES})...`); await sleep(delay); return claudeApiCall(systemPrompt, userPrompt, options, retryCount + 1); } throw new Error(`Claude API failed: ${resp.status} ${errText.slice(0, 200)}`); } const data = await resp.json() as { content: Array<{ type: string; text: string }>; model: string; usage: { input_tokens: number; output_tokens: number }; }; const text = data.content .filter((c) => c.type === "text") .map((c) => c.text) .join(""); const duration = Date.now() - startTime; console.log(`[LLM] Claude ${data.model}: ${data.usage.input_tokens}+${data.usage.output_tokens} tokens, ${duration}ms`); return { text, model: data.model, totalDuration: duration * 1_000_000, // ns for compat evalCount: data.usage.output_tokens, }; } async function generateClaude( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { if (!ANTHROPIC_API_KEY) { throw new Error("ANTHROPIC_API_KEY not set — cannot use Claude provider"); } // Use enqueueClaude for serialization, but call claudeApiCall (not generateClaude) // for retries to avoid circular-promise deadlock return enqueueClaude(() => claudeApiCall(systemPrompt, userPrompt, options)); } // ═══════════════════════════════════════════════════════ // OLLAMA PROVIDER (existing) // ═══════════════════════════════════════════════════════ let ollamaQueue: Promise = Promise.resolve(); let queueDepth = 0; let lastQueueEnqueueTime = 0; export function resetOllamaQueue(): void { ollamaQueue = Promise.resolve(); queueDepth = 0; console.log("[LLM] Queue reset — previous stuck requests cleared"); } export function getQueueDepth(): number { return queueDepth; } function enqueueOllama(fn: () => Promise): Promise { queueDepth++; lastQueueEnqueueTime = Date.now(); const result = ollamaQueue.then(() => { if (Date.now() - lastQueueEnqueueTime > 900000) { console.warn("[LLM] Queue auto-reset after 15min stall"); queueDepth = Math.max(0, queueDepth - 1); return Promise.reject(new Error("Queue auto-reset: previous request timed out")); } return fn(); }); ollamaQueue = result.catch(() => {}).then(() => { queueDepth = Math.max(0, queueDepth - 1); }); return result; } async function generateOllama( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { return enqueueOllama(async () => { const RETRY_DELAYS = [15000, 30000, 60000]; for (let attempt = 0; attempt <= RETRY_DELAYS.length; attempt++) { if (attempt > 0) { const delay = RETRY_DELAYS[attempt - 1]; console.log(`Blog LLM: 429 rate-limit — retrying in ${delay / 1000}s (attempt ${attempt}/${RETRY_DELAYS.length})`); await sleep(delay); } const resp = await fetch(`${OLLAMA_URL}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: llmModel(), prompt: userPrompt, system: systemPrompt, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (resp.status === 429) { if (attempt < RETRY_DELAYS.length) continue; throw new Error(`Ollama generate failed: 429 Too Many Requests (all retries exhausted)`); } if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama generate failed: ${resp.status} ${errText}`); } const data = await resp.json() as { response: string; model: string; total_duration: number; eval_count: number; }; return { text: data.response, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; } throw new Error("Ollama generate: unreachable"); }); } // ═══════════════════════════════════════════════════════ // CLAUDE-CODE PROVIDER (claude-bridge — flat-rate via Claude Code subscription) // ═══════════════════════════════════════════════════════ async function generateClaudeBridge( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { const startTime = Date.now(); // claude-bridge expects combined prompt — system + user joined with double newline const fullPrompt = `${systemPrompt}\n\n${userPrompt}`; const resp = await fetch(`${CLAUDE_BRIDGE_URL}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: fullPrompt }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (!resp.ok) { const errText = await resp.text(); throw new Error(`Claude bridge failed: ${resp.status} ${errText.slice(0, 200)}`); } const data = await resp.json() as { success: boolean; content?: string; error?: string }; if (!data.success || !data.content) { throw new Error(`Claude bridge returned empty response: ${JSON.stringify(data)}`); } const duration = Date.now() - startTime; console.log(`[LLM] Claude-bridge: ${data.content.length} chars, ${duration}ms`); return { text: data.content, model: "claude-code", totalDuration: duration * 1_000_000, // ns for compat with Ollama callers evalCount: Math.ceil(data.content.length / 4), // approx tokens }; } // ═══════════════════════════════════════════════════════ // PUBLIC API — auto-routes to configured provider // ═══════════════════════════════════════════════════════ export async function generate( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { if (provider() === "claude-code") { return generateClaudeBridge(systemPrompt, userPrompt, options); } if (provider() === "anthropic" && ANTHROPIC_API_KEY) { return generateClaude(systemPrompt, userPrompt, options); } return generateOllama(systemPrompt, userPrompt, options); } /** Chat-style generation with message history (Ollama only for now) */ export async function chat( messages: ReadonlyArray<{ role: "system" | "user" | "assistant"; content: string }>, options?: { temperature?: number; maxTokens?: number }, ): Promise { return enqueueOllama(async () => { const resp = await fetch(`${OLLAMA_URL}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: llmModel(), messages, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(300000), }); if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama chat failed: ${resp.status} ${errText}`); } const data = await resp.json() as { message: { content: string }; model: string; total_duration: number; eval_count: number; }; return { text: data.message.content, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; }); } /** Check if configured LLM provider is available */ export async function checkHealth(): Promise<{ ok: boolean; model: string; provider: string; error?: string }> { const p = provider(); const m = llmModel(); if (p === "claude-code") { try { const resp = await fetch(`${CLAUDE_BRIDGE_URL}/health`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return { ok: false, model: "claude-code", provider: "claude-code", error: `HTTP ${resp.status}` }; return { ok: true, model: "claude-code", provider: "claude-code" }; } catch (err) { return { ok: false, model: "claude-code", provider: "claude-code", error: (err as Error).message }; } } if (p === "anthropic" && ANTHROPIC_API_KEY) { return { ok: true, model: ANTHROPIC_MODEL, provider: "anthropic" }; } try { const resp = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return { ok: false, model: m, provider: "ollama", error: `HTTP ${resp.status}` }; const data = await resp.json() as { models: Array<{ name: string }> }; const hasModel = data.models.some((tag) => tag.name.includes(m.split(":")[0])); return { ok: hasModel, model: m, provider: "ollama", error: hasModel ? undefined : `Model ${m} not found` }; } catch (err) { return { ok: false, model: m, provider: "ollama", error: (err as Error).message }; } }