/** * backendAdapter.ts — Unified LLM backend adapter interface. * * Normalizes differences between backends (Ollama, vLLM, OpenAI, etc.) into * a common LLMEvent stream type. Each backend implements the LLMBackend interface * and produces LLMEvent objects that the runner consumes uniformly. */ // ─── LLMEvent stream type ─────────────────────────────────────────────────── export interface LLMEvent { type: 'chunk' | 'tool-call' | 'tool-result' | 'finish' | 'error'; content?: string; toolCall?: { id: string; name: string; args: string }; toolResult?: { id: string; content: string; error?: string }; finishReason?: string | null; error?: string; } export type LLMEventStream = AsyncIterable; // ─── Backend adapter interface ────────────────────────────────────────────── export interface LLMBackend { /** Unique identifier for this backend */ readonly id: string; /** Human-readable name */ readonly name: string; /** * Send a prompt (with optional tool definitions) and receive a normalized * event stream. The caller iterates the stream to consume chunks, tool calls, * and the final finish event. */ complete( messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, tools?: Array<{ name: string; description?: string; parameters: Record }>, options?: BackendOptions, ): LLMEventStream; /** Check if the backend is healthy / reachable */ healthCheck(): Promise; } export interface BackendOptions { temperature?: number; maxTokens?: number; topP?: number; stopSequences?: string[]; [key: string]: unknown; } // ─── Backend registry ─────────────────────────────────────────────────────── const registeredBackends = new Map(); export function registerBackend(backend: LLMBackend): void { registeredBackends.set(backend.id, backend); } export function getBackend(id: string): LLMBackend | undefined { return registeredBackends.get(id); } export function listBackends(): LLMBackend[] { return Array.from(registeredBackends.values()); } // ─── Cascade backend (fallback chain) ─────────────────────────────────────── export class CascadeBackend implements LLMBackend { readonly id = 'cascade'; readonly name = 'Cascade (fallback chain)'; constructor(private readonly backends: LLMBackend[]) {} async* complete( messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, tools?: Array<{ name: string; description?: string; parameters: Record }>, options?: BackendOptions, ): AsyncIterable { let lastError: Error | undefined; for (const backend of this.backends) { try { const stream = await backend.complete(messages, tools, options); // Forward events from this backend for await (const event of stream) { yield event; } return; // Success — stop trying } catch (err) { lastError = err instanceof Error ? err : new Error(String(err)); continue; // Try next backend } } // All backends failed — yield an error event yield { type: 'error', error: `All ${this.backends.length} backends failed. Last error: ${lastError?.message}`, }; } async healthCheck(): Promise { return this.backends.some((b) => b.healthCheck()); } } // ─── Ollama backend adapter ───────────────────────────────────────────────── export class OllamaBackend implements LLMBackend { readonly id = 'ollama'; readonly name = 'Ollama'; constructor( private readonly model: string, private readonly baseUrl: string = 'http://localhost:11434', ) {} async *complete( messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, tools?: Array<{ name: string; description?: string; parameters: Record }>, options?: BackendOptions, ): LLMEventStream { const response = await fetch(`${this.baseUrl}/api/chat`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: this.model, messages, stream: true, tools: tools?.map((t) => ({ type: 'function', function: { name: t.name, description: t.description ?? '', parameters: t.parameters, }, })), ...options, }), }); if (!response.ok) { yield { type: 'error', error: `Ollama HTTP ${response.status}: ${response.statusText}` }; return; } const reader = response.body?.getReader(); if (!reader) { yield { type: 'error', error: 'Ollama response has no body' }; return; } let buffer = ''; let toolCallBuffer = ''; let toolCallName = ''; let toolCallId = ''; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += new TextDecoder().decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() ?? ''; for (const line of lines) { if (!line.trim()) continue; const json = JSON.parse(line); if (json.message?.role === 'assistant') { const content = json.message.content; if (content) { yield { type: 'chunk', content }; } // Detect tool calls in the message if (json.message.tool_calls) { for (const tc of json.message.tool_calls) { toolCallId = tc.id ?? `tool_${Date.now()}`; toolCallName = tc.function?.name ?? ''; toolCallBuffer += tc.function?.arguments ?? ''; } } } // Accumulate tool call arguments if (toolCallName && json.message?.tool_calls) { for (const tc of json.message.tool_calls) { if (tc.function?.name === toolCallName) { toolCallBuffer += tc.function?.arguments ?? ''; } } } } } } finally { reader.releaseLock(); } // Emit tool-call event if we accumulated arguments if (toolCallName && toolCallBuffer) { yield { type: 'tool-call', toolCall: { id: toolCallId, name: toolCallName, args: toolCallBuffer }, }; } yield { type: 'finish', finishReason: 'stop' }; } async healthCheck(): Promise { try { const res = await fetch(`${this.baseUrl}/api/tags`, { signal: AbortSignal.timeout(5000) }); return res.ok; } catch { return false; } } } // ─── OpenAI backend adapter ───────────────────────────────────────────────── export class OpenAIBackend implements LLMBackend { readonly id = 'openai'; readonly name = 'OpenAI'; constructor( private readonly model: string, private readonly apiKey: string, private readonly baseUrl: string = 'https://api.openai.com/v1', ) {} async *complete( messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, tools?: Array<{ name: string; description?: string; parameters: Record }>, options?: BackendOptions, ): LLMEventStream { const response = await fetch(`${this.baseUrl}/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${this.apiKey}`, }, body: JSON.stringify({ model: this.model, messages, stream: true, tools: tools?.map((t) => ({ type: 'function', function: { name: t.name, description: t.description ?? '', parameters: t.parameters, }, })), ...options, }), }); if (!response.ok) { const errBody = await response.text(); yield { type: 'error', error: `OpenAI HTTP ${response.status}: ${errBody}` }; return; } const reader = response.body?.getReader(); if (!reader) { yield { type: 'error', error: 'OpenAI response has no body' }; return; } let buffer = ''; let accumulatedArgs = ''; let currentToolId = ''; let currentToolName = ''; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += new TextDecoder().decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() ?? ''; for (const line of lines) { if (!line.trim() || !line.startsWith('data: ')) continue; const data = line.slice(6); if (data === '[DONE]') continue; const json = JSON.parse(data); const delta = json.choices?.[0]?.delta; if (delta?.content) { yield { type: 'chunk', content: delta.content }; } if (delta?.tool_calls?.[0]) { const tc = delta.tool_calls[0]; if (tc.id) currentToolId = tc.id; if (tc.function?.name) currentToolName = tc.function.name; accumulatedArgs += tc.function?.arguments ?? ''; } } } } finally { reader.releaseLock(); } if (currentToolName && accumulatedArgs) { yield { type: 'tool-call', toolCall: { id: currentToolId, name: currentToolName, args: accumulatedArgs }, }; } yield { type: 'finish', finishReason: 'stop' }; } async healthCheck(): Promise { try { const res = await fetch(`${this.baseUrl}/models`, { headers: { Authorization: `Bearer ${this.apiKey}` }, signal: AbortSignal.timeout(5000), }); return res.ok; } catch { return false; } } } // ─── vLLM backend adapter ─────────────────────────────────────────────────── export class vLLMBackend implements LLMBackend { readonly id = 'vllm'; readonly name = 'vLLM'; constructor( private readonly model: string, private readonly baseUrl: string = 'http://localhost:8000/v1', ) {} async *complete( messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, tools?: Array<{ name: string; description?: string; parameters: Record }>, options?: BackendOptions, ): LLMEventStream { const response = await fetch(`${this.baseUrl}/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: this.model, messages, stream: true, tools: tools?.map((t) => ({ type: 'function', function: { name: t.name, description: t.description ?? '', parameters: t.parameters, }, })), ...options, }), }); if (!response.ok) { yield { type: 'error', error: `vLLM HTTP ${response.status}: ${await response.text()}` }; return; } const reader = response.body?.getReader(); if (!reader) { yield { type: 'error', error: 'vLLM response has no body' }; return; } let buffer = ''; let accumulatedArgs = ''; let currentToolId = ''; let currentToolName = ''; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += new TextDecoder().decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() ?? ''; for (const line of lines) { if (!line.trim() || !line.startsWith('data: ')) continue; const data = line.slice(6); if (data === '[DONE]') continue; const json = JSON.parse(data); const delta = json.choices?.[0]?.delta; if (delta?.content) { yield { type: 'chunk', content: delta.content }; } if (delta?.tool_calls?.[0]) { const tc = delta.tool_calls[0]; if (tc.id) currentToolId = tc.id; if (tc.function?.name) currentToolName = tc.function.name; accumulatedArgs += tc.function?.arguments ?? ''; } } } } finally { reader.releaseLock(); } if (currentToolName && accumulatedArgs) { yield { type: 'tool-call', toolCall: { id: currentToolId, name: currentToolName, args: accumulatedArgs }, }; } yield { type: 'finish', finishReason: 'stop' }; } async healthCheck(): Promise { try { const res = await fetch(`${this.baseUrl}/models`, { signal: AbortSignal.timeout(5000) }); return res.ok; } catch { return false; } } }