/** * Ollama LLM client for blog generation and content enhancement. * * Uses qwen2.5:14b on Mac Studio (.213) for text generation. * Supports streaming and non-streaming modes. * Includes 429 retry with exponential backoff + server-side concurrency guard. */ const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434"; const LLM_MODEL = process.env.OLLAMA_LLM_MODEL || "qwen2.5:14b"; interface LlmResponse { text: string; model: string; totalDuration: number; evalCount: number; } /** Sleep helper */ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } /** * Server-side concurrency guard — Ollama processes one generation at a time. * Queue ensures sequential execution even with multiple concurrent API requests. */ let ollamaQueue: Promise = Promise.resolve(); function enqueueOllama(fn: () => Promise): Promise { const result = ollamaQueue.then(fn); // Keep queue alive even if fn throws (attach no-op error handler on chain) ollamaQueue = result.catch(() => {}); return result; } /** Generate text from a system prompt + user prompt — with 429 retry/backoff + queue */ export async function generate( systemPrompt: string, userPrompt: string, options?: { temperature?: number; maxTokens?: number; timeoutMs?: number }, ): Promise { return enqueueOllama(async () => { const RETRY_DELAYS = [15000, 30000, 60000]; // 15s, 30s, 60s for (let attempt = 0; attempt <= RETRY_DELAYS.length; attempt++) { if (attempt > 0) { const delay = RETRY_DELAYS[attempt - 1]; console.log(`Blog LLM: 429 rate-limit — retrying in ${delay / 1000}s (attempt ${attempt}/${RETRY_DELAYS.length})`); await sleep(delay); } const resp = await fetch(`${OLLAMA_URL}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: LLM_MODEL, prompt: userPrompt, system: systemPrompt, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(options?.timeoutMs ?? 300000), }); if (resp.status === 429) { if (attempt < RETRY_DELAYS.length) continue; throw new Error(`Ollama generate failed: 429 Too Many Requests (all retries exhausted)`); } if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama generate failed: ${resp.status} ${errText}`); } const data = await resp.json() as { response: string; model: string; total_duration: number; eval_count: number; }; return { text: data.response, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; } throw new Error("Ollama generate: unreachable"); }); } /** Chat-style generation with message history */ export async function chat( messages: ReadonlyArray<{ role: "system" | "user" | "assistant"; content: string }>, options?: { temperature?: number; maxTokens?: number }, ): Promise { return enqueueOllama(async () => { const resp = await fetch(`${OLLAMA_URL}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: LLM_MODEL, messages, stream: false, options: { temperature: options?.temperature ?? 0.7, num_predict: options?.maxTokens ?? 4096, }, }), signal: AbortSignal.timeout(300000), }); if (!resp.ok) { const errText = await resp.text(); throw new Error(`Ollama chat failed: ${resp.status} ${errText}`); } const data = await resp.json() as { message: { content: string }; model: string; total_duration: number; eval_count: number; }; return { text: data.message.content, model: data.model, totalDuration: data.total_duration, evalCount: data.eval_count, }; }); } /** Check if Ollama is available and model is loaded */ export async function checkHealth(): Promise<{ ok: boolean; model: string; error?: string }> { try { const resp = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(5000) }); if (!resp.ok) return { ok: false, model: LLM_MODEL, error: `HTTP ${resp.status}` }; const data = await resp.json() as { models: Array<{ name: string }> }; const hasModel = data.models.some((m) => m.name.includes(LLM_MODEL.split(":")[0])); return { ok: hasModel, model: LLM_MODEL, error: hasModel ? undefined : `Model ${LLM_MODEL} not found` }; } catch (err) { return { ok: false, model: LLM_MODEL, error: (err as Error).message }; } }