/** * LLM client for blog generation — supports Ollama (local), Anthropic Claude (API), * and Claude-Code (flat-rate via claude-bridge on Erik). * * Provider selection: * BLOG_LLM_PROVIDER=claude-code → Claude via claude-bridge (flat-rate, recommended) * BLOG_LLM_PROVIDER=anthropic → Claude Sonnet/Haiku via Anthropic API * BLOG_LLM_PROVIDER=ollama → local adapter bridge / Ollama-compatible endpoint (default) * * Claude-code is preferred: uses Claude Code subscription (flat-rate), no API costs. * The default local blog model is the latest RunPod-trained FO_BlogLLM adapter. */ import { existsSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434"; const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || ""; const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-sonnet-4-20250514"; const CLAUDE_BRIDGE_URL = process.env.CLAUDE_BRIDGE_URL || "http://localhost:3250"; // ── Runtime-switchable provider state ────────────────────────────────────── // Reads from /opt/tip/blog-llm-settings.json if present (written by /api/blog/llm/switch). // Falls back to process.env, then to defaults. No restart required for switches. // // AUTO-DISCOVERY: At startup and on a periodic refresh, the active fo-blog-v* model // is validated against Ollama's actual model list. If the configured model no longer // exists (e.g. Magatama trained a new version and Ollama removed older tags), the // highest available fo-blog-v* version is picked automatically — no manual env or // settings-file update needed after each training cycle. const SETTINGS_FILE = join(process.env.TIP_ROOT || "/opt/tip", "blog-llm-settings.json"); const STATIC_FALLBACK_MODEL = "fo-blog-v10"; const DISCOVERY_REFRESH_MS = Number.parseInt(process.env.BLOG_LLM_DISCOVERY_REFRESH_MS || "", 10) || 10 * 60_000; interface LlmSettings { provider: string; ollamaModel: string } function loadSettingsRaw(): LlmSettings { try { if (existsSync(SETTINGS_FILE)) { const raw = JSON.parse(readFileSync(SETTINGS_FILE, "utf8")) as Partial; return { provider: raw.provider || process.env.BLOG_LLM_PROVIDER || "ollama", ollamaModel: raw.ollamaModel || process.env.OLLAMA_LLM_MODEL || STATIC_FALLBACK_MODEL, }; } } catch { /* ignore corrupt file */ } return { provider: process.env.BLOG_LLM_PROVIDER || "ollama", ollamaModel: process.env.OLLAMA_LLM_MODEL || STATIC_FALLBACK_MODEL, }; } /** * Sort fo-blog-v{N}[-r{M}] tags newest-first. * * Magatama convention (confirmed by /api/llm/status?lane=fo_blogllm): * - `fo-blog-vN` ← base tag, the active production model after adoption * - `fo-blog-vN-rM` ← revision metadata, intermediate adapter save * * So within the same major N, the BASE tag wins over -rM revisions. * * Order: higher N > lower N; within same N, base ("no -r") > any -rM revision. */ function compareFoBlogVersionsDesc(a: string, b: string): number { const re = /^fo-blog-v(\d+)(?:-r(\d+))?$/; const ma = re.exec(a); const mb = re.exec(b); if (!ma || !mb) return a.localeCompare(b); const va = Number.parseInt(ma[1], 10); const vb = Number.parseInt(mb[1], 10); if (va !== vb) return vb - va; // Same major version: base tag (no -r suffix) wins over any -rM revision const aIsBase = ma[2] === undefined; const bIsBase = mb[2] === undefined; if (aIsBase !== bIsBase) return aIsBase ? -1 : 1; // Both have -rM: higher M is newer const ra = ma[2] ? Number.parseInt(ma[2], 10) : 0; const rb = mb[2] ? Number.parseInt(mb[2], 10) : 0; return rb - ra; } interface OllamaTag { name: string } interface OllamaTagsResponse { models: OllamaTag[] } /** Probe Ollama for available fo-blog-v* models. Returns [] on any error (non-fatal). */ async function fetchOllamaFoBlogTags(): Promise { try { const resp = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return []; const data = await resp.json() as OllamaTagsResponse; return (data.models || []) .map(m => m.name.replace(/:latest$/, "")) .filter(n => /^fo-blog-v\d+(?:-r\d+)?$/.test(n)); } catch { return []; } } /** * Reconcile configured model against Ollama reality. * * Priority: * 1. Configured model (env or settings file) — if Ollama actually serves it * 2. Highest fo-blog-v* version Ollama actually serves — auto-discovered * 3. Static fallback STATIC_FALLBACK_MODEL — last resort * * Non-blocking: any Ollama failure leaves _settings untouched. */ async function reconcileWithOllama(): Promise { const configured = _settings.ollamaModel; if (!configured.startsWith("fo-blog-v")) return; // only manage fo-blog-* lane const available = await fetchOllamaFoBlogTags(); if (available.length === 0) return; if (available.includes(configured)) return; // configured model still exists const sorted = [...available].sort(compareFoBlogVersionsDesc); const winner = sorted[0]; if (!winner || winner === configured) return; console.log(`[LLM] auto-discovery: configured "${configured}" not in Ollama; switching to latest available "${winner}" (candidates: ${sorted.join(", ")})`); _settings = { ..._settings, ollamaModel: winner }; try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } } let _settings = loadSettingsRaw(); // Fire-and-forget initial reconciliation. Subsequent refresh runs every DISCOVERY_REFRESH_MS. void reconcileWithOllama(); setInterval(() => { void reconcileWithOllama(); }, DISCOVERY_REFRESH_MS).unref(); /** Switch the active LLM provider at runtime. Persists to settings file. */ export function setLlmProvider(provider: string, ollamaModel?: string): void { _settings = { provider, ollamaModel: ollamaModel || _settings.ollamaModel }; try { writeFileSync(SETTINGS_FILE, JSON.stringify(_settings, null, 2), "utf8"); } catch { /* non-fatal */ } console.log(`[LLM] Provider switched → ${provider}${ollamaModel ? ` (${ollamaModel})` : ""}`); } /** Returns the currently active provider config. */ export function getLlmProvider(): LlmSettings { return { ..._settings }; } /** * Force an immediate auto-discovery reconciliation against Ollama. * Returns the active settings after reconcile. */ export async function refreshLlmAutoDiscovery(): Promise { await reconcileWithOllama(); return { ..._settings }; } // Convenience getters used below (re-read on every call for zero-latency switch) function provider(): string { return _settings.provider; } function llmModel(): string { return _settings.ollamaModel; } interface LlmResponse { text: string; model: string; totalDuration: number; evalCount: number; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } // ═══════════════════════════════════════════════════════ // ANTHROPIC CLAUDE PROVIDER // ═══════════════════════════════════════════════════════ // Serialize Claude API calls to stay within TPM limits // Tier-1 has 40,000 TPM — with ~20K tokens/step, only 1 concurrent call safe let claudeQueue: Promise = Promise.resolve(); let claudeQueueEnqueueTime = 0; export function resetClaudeQueue(): void { claudeQueue = Promise.resolve(); claudeQueueEnqueueTime = 0; console.log("[LLM] Claude queue reset — previous stuck requests cleared"); } function enqueueClaude(fn: () => Promise): Promise { claudeQueueEnqueueTime = Date.now(); const result = claudeQueue.then(() => { // Auto-reset if queue has been stalled > 15 minutes (prevents deadlock on stuck requests) if (Date.now() - claudeQueueEnqueueTime > 900000) { console.warn("[LLM] Claude queue auto-reset after 15min stall"); return Promise.reject(new Error("Claude queue auto-reset: previous request timed out")); } return fn(); }); claudeQueue = result.catch(() => {}); return result; } // Direct API call without going through the serialization queue — used for 429 retries // to avoid the circular-promise deadlock that recursive enqueueClaude creates async function claudeApiCall( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, retryCount = 0, ): Promise { const startTime = Date.now(); const MAX_RETRIES = 3; const RETRY_DELAYS = [10000, 30000, 60000]; const resp = await fetch("https://api.anthropic.com/v1/messages", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": ANTHROPIC_API_KEY, "anthropic-version": "2023-06-01", }, body: JSON.stringify({ model: ANTHROPIC_MODEL, max_tokens: options?.maxTokens ?? 4096, temperature: options?.temperature ?? 0.7, system: systemPrompt, messages: [{ role: "user", content: userPrompt }], }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (!resp.ok) { const errText = await resp.text(); if (resp.status === 429 && retryCount < MAX_RETRIES) { const delay = RETRY_DELAYS[retryCount] ?? 60000; console.log(`[LLM] Claude 429 — retrying in ${delay / 1000}s (attempt ${retryCount + 1}/${MAX_RETRIES})...`); await sleep(delay); return claudeApiCall(systemPrompt, userPrompt, options, retryCount + 1); } throw new Error(`Claude API failed: ${resp.status} ${errText.slice(0, 200)}`); } const data = await resp.json() as { content: Array<{ type: string; text: string }>; model: string; usage: { input_tokens: number; output_tokens: number }; }; const text = data.content .filter((c) => c.type === "text") .map((c) => c.text) .join(""); const duration = Date.now() - startTime; console.log(`[LLM] Claude ${data.model}: ${data.usage.input_tokens}+${data.usage.output_tokens} tokens, ${duration}ms`); return { text, model: data.model, totalDuration: duration * 1_000_000, // ns for compat evalCount: data.usage.output_tokens, }; } async function generateClaude( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { if (!ANTHROPIC_API_KEY) { throw new Error("ANTHROPIC_API_KEY not set — cannot use Claude provider"); } // Use enqueueClaude for serialization, but call claudeApiCall (not generateClaude) // for retries to avoid circular-promise deadlock return enqueueClaude(() => claudeApiCall(systemPrompt, userPrompt, options)); } // ═══════════════════════════════════════════════════════ // OLLAMA PROVIDER (existing) // ═══════════════════════════════════════════════════════ let ollamaQueue: Promise = Promise.resolve(); let queueDepth = 0; let lastQueueEnqueueTime = 0; export function resetOllamaQueue(): void { ollamaQueue = Promise.resolve(); queueDepth = 0; console.log("[LLM] Queue reset — previous stuck requests cleared"); } export function getQueueDepth(): number { return queueDepth; } function enqueueOllama(fn: () => Promise): Promise { queueDepth++; lastQueueEnqueueTime = Date.now(); const result = ollamaQueue.then(() => { if (Date.now() - lastQueueEnqueueTime > 900000) { console.warn("[LLM] Queue auto-reset after 15min stall"); queueDepth = Math.max(0, queueDepth - 1); return Promise.reject(new Error("Queue auto-reset: previous request timed out")); } return fn(); }); ollamaQueue = result.catch(() => {}).then(() => { queueDepth = Math.max(0, queueDepth - 1); }); return result; } async function generateOllama( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { return enqueueOllama(async () => { const RETRY_DELAYS = [15000, 30000, 60000]; for (let attempt = 0; attempt <= RETRY_DELAYS.length; attempt++) { if (attempt > 0) { const delay = RETRY_DELAYS[attempt - 1]; console.log(`Blog LLM: 429 rate-limit — retrying in ${delay / 1000}s (attempt ${attempt}/${RETRY_DELAYS.length})`); await sleep(delay); } const resp = await fetch(`${OLLAMA_URL}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: llmModel(), prompt: userPrompt, system: systemPrompt, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (resp.status === 429) { if (attempt < RETRY_DELAYS.length) continue; throw new Error(`Ollama generate failed: 429 Too Many Requests (all retries exhausted)`); } if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama generate failed: ${resp.status} ${errText}`); } const data = await resp.json() as { response: string; model: string; total_duration: number; eval_count: number; }; return { text: data.response, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; } throw new Error("Ollama generate: unreachable"); }); } // ═══════════════════════════════════════════════════════ // CLAUDE-CODE PROVIDER (claude-bridge — flat-rate via Claude Code subscription) // ═══════════════════════════════════════════════════════ async function generateClaudeBridge( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { const startTime = Date.now(); // claude-bridge expects combined prompt — system + user joined with double newline const fullPrompt = `${systemPrompt}\n\n${userPrompt}`; const resp = await fetch(`${CLAUDE_BRIDGE_URL}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: fullPrompt }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (!resp.ok) { const errText = await resp.text(); throw new Error(`Claude bridge failed: ${resp.status} ${errText.slice(0, 200)}`); } const data = await resp.json() as { success: boolean; content?: string; error?: string }; if (!data.success || !data.content) { throw new Error(`Claude bridge returned empty response: ${JSON.stringify(data)}`); } const duration = Date.now() - startTime; console.log(`[LLM] Claude-bridge: ${data.content.length} chars, ${duration}ms`); return { text: data.content, model: "claude-code", totalDuration: duration * 1_000_000, // ns for compat with Ollama callers evalCount: Math.ceil(data.content.length / 4), // approx tokens }; } // ═══════════════════════════════════════════════════════ // PUBLIC API — auto-routes to configured provider // ═══════════════════════════════════════════════════════ export async function generate( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { if (provider() === "claude-code") { return generateClaudeBridge(systemPrompt, userPrompt, options); } if (provider() === "anthropic" && ANTHROPIC_API_KEY) { return generateClaude(systemPrompt, userPrompt, options); } return generateOllama(systemPrompt, userPrompt, options); } /** Chat-style generation with message history (Ollama only for now) */ export async function chat( messages: ReadonlyArray<{ role: "system" | "user" | "assistant"; content: string }>, options?: { temperature?: number; maxTokens?: number }, ): Promise { return enqueueOllama(async () => { const resp = await fetch(`${OLLAMA_URL}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: llmModel(), messages, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(300000), }); if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama chat failed: ${resp.status} ${errText}`); } const data = await resp.json() as { message: { content: string }; model: string; total_duration: number; eval_count: number; }; return { text: data.message.content, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; }); } /** Check if configured LLM provider is available */ export async function checkHealth(): Promise<{ ok: boolean; model: string; provider: string; error?: string }> { const p = provider(); const m = llmModel(); if (p === "claude-code") { try { const resp = await fetch(`${CLAUDE_BRIDGE_URL}/health`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return { ok: false, model: "claude-code", provider: "claude-code", error: `HTTP ${resp.status}` }; return { ok: true, model: "claude-code", provider: "claude-code" }; } catch (err) { return { ok: false, model: "claude-code", provider: "claude-code", error: (err as Error).message }; } } if (p === "anthropic" && ANTHROPIC_API_KEY) { return { ok: true, model: ANTHROPIC_MODEL, provider: "anthropic" }; } try { const resp = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return { ok: false, model: m, provider: "ollama", error: `HTTP ${resp.status}` }; const data = await resp.json() as { models: Array<{ name: string }> }; const hasModel = data.models.some((tag) => tag.name.includes(m.split(":")[0])); return { ok: hasModel, model: m, provider: "ollama", error: hasModel ? undefined : `Model ${m} not found` }; } catch (err) { return { ok: false, model: m, provider: "ollama", error: (err as Error).message }; } }