/** * Provider streaming implementations for @geepers/chat server mode. * * 5 families covering 10 providers: * - OpenAI-compatible (openai SDK): OpenAI, xAI, Groq, Perplexity, Mistral, HuggingFace * - Anthropic (@anthropic-ai/sdk): Anthropic * - Gemini (fetch): Google Gemini * - Cohere (fetch): Cohere * - Ollama (fetch): Ollama local */ import { resolveApiKey } from './detect.js'; interface ChatMessage { role: string; content: string; } // ========== OpenAI-compatible family ========== const OPENAI_COMPAT: Record = { openai: { baseURL: 'https://api.openai.com/v1', envKeys: ['OPENAI_API_KEY'] }, xai: { baseURL: 'https://api.x.ai/v1', envKeys: ['XAI_API_KEY', 'GROK_API_KEY'] }, groq: { baseURL: 'https://api.groq.com/openai/v1', envKeys: ['GROQ_API_KEY'] }, perplexity: { baseURL: 'https://api.perplexity.ai', envKeys: ['PERPLEXITY_API_KEY'] }, mistral: { baseURL: 'https://api.mistral.ai/v1', envKeys: ['MISTRAL_API_KEY'] }, huggingface: { baseURL: 'https://router.huggingface.co/v1', envKeys: ['HF_API_KEY', 'HUGGINGFACE_API_KEY'] }, }; async function* streamOpenAICompat( provider: string, model: string, messages: ChatMessage[], apiKey: string, ): AsyncGenerator { const { default: OpenAI } = await import('openai'); const config = OPENAI_COMPAT[provider]; const client = new OpenAI({ apiKey, baseURL: config.baseURL }); const stream = await client.chat.completions.create({ model, messages: messages.map(m => ({ role: m.role as 'user' | 'assistant' | 'system', content: m.content })), stream: true, }); for await (const chunk of stream) { const content = chunk.choices[0]?.delta?.content; if (content) yield content; } } // ========== Anthropic ========== async function* streamAnthropic( model: string, messages: ChatMessage[], apiKey: string, ): AsyncGenerator { const { default: Anthropic } = await import('@anthropic-ai/sdk'); const client = new Anthropic({ apiKey }); const systemParts = messages.filter(m => m.role === 'system').map(m => m.content); const chatMessages = messages .filter(m => m.role !== 'system') .map(m => ({ role: m.role as 'user' | 'assistant', content: m.content })); const stream = client.messages.stream({ model, messages: chatMessages, max_tokens: 4096, ...(systemParts.length ? { system: systemParts.join('\n\n') } : {}), }); for await (const event of stream) { if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { yield event.delta.text; } } } // ========== Gemini (fetch) ========== async function* streamGemini( model: string, messages: ChatMessage[], apiKey: string, ): AsyncGenerator { // Convert to Gemini format const contents = messages .filter(m => m.role !== 'system') .map(m => ({ role: m.role === 'assistant' ? 'model' : 'user', parts: [{ text: m.content }], })); const systemParts = messages.filter(m => m.role === 'system').map(m => m.content); const body: Record = { contents }; if (systemParts.length) { body.systemInstruction = { parts: [{ text: systemParts.join('\n\n') }] }; } const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:streamGenerateContent?alt=sse&key=${apiKey}`; const res = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }); if (!res.ok) throw new Error(`Gemini API error: ${res.status} ${await res.text()}`); yield* parseSSEStream(res, (parsed) => { return parsed?.candidates?.[0]?.content?.parts?.[0]?.text || ''; }); } // ========== Cohere (fetch) ========== async function* streamCohere( model: string, messages: ChatMessage[], apiKey: string, ): AsyncGenerator { // Use Cohere v2 chat API (OpenAI-like) const res = await fetch('https://api.cohere.com/v2/chat', { method: 'POST', headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ model, messages: messages.map(m => ({ role: m.role, content: m.content })), stream: true, }), }); if (!res.ok) throw new Error(`Cohere API error: ${res.status} ${await res.text()}`); yield* parseSSEStream(res, (parsed) => { if (parsed?.type === 'content-delta') { return parsed?.delta?.message?.content?.text || ''; } return ''; }); } // ========== Ollama (fetch, NDJSON) ========== async function* streamOllama( model: string, messages: ChatMessage[], ): AsyncGenerator { const host = process.env.OLLAMA_HOST || 'http://localhost:11434'; const res = await fetch(`${host}/api/chat`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model, messages: messages.map(m => ({ role: m.role, content: m.content })), stream: true, }), }); if (!res.ok) throw new Error(`Ollama error: ${res.status} ${await res.text()}`); const reader = res.body?.getReader(); if (!reader) return; const decoder = new TextDecoder(); let buffer = ''; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); // Ollama streams NDJSON (one JSON object per line) const lines = buffer.split('\n'); buffer = lines.pop() || ''; for (const line of lines) { if (!line.trim()) continue; try { const parsed = JSON.parse(line); if (parsed.message?.content) yield parsed.message.content; } catch { /* skip malformed lines */ } } } } // ========== SSE stream parser helper ========== async function* parseSSEStream( res: globalThis.Response, extract: (parsed: any) => string, ): AsyncGenerator { const reader = res.body?.getReader(); if (!reader) return; const decoder = new TextDecoder(); let buffer = ''; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() || ''; for (const line of lines) { if (!line.startsWith('data: ')) continue; const data = line.slice(6).trim(); if (data === '[DONE]') return; try { const parsed = JSON.parse(data); const text = extract(parsed); if (text) yield text; } catch { /* skip */ } } } } // ========== Router ========== export async function* streamChat( provider: string, model: string, messages: ChatMessage[], apiKey: string, ): AsyncGenerator { if (provider in OPENAI_COMPAT) { yield* streamOpenAICompat(provider, model, messages, apiKey); } else if (provider === 'anthropic') { yield* streamAnthropic(model, messages, apiKey); } else if (provider === 'gemini') { yield* streamGemini(model, messages, apiKey); } else if (provider === 'cohere') { yield* streamCohere(model, messages, apiKey); } else if (provider === 'ollama') { yield* streamOllama(model, messages); } else { throw new Error(`Unknown provider: ${provider}`); } }