import type { ChatRole, ChatStreamEvent } from "@/types/llm-chat"; import type { ConnectivityResult, LLMProviderWithSecret, ModelListItem, } from "@/types/llm-provider"; // --------------------------------------------------------------------------- // Multimodal content types (OpenAI-compatible) // --------------------------------------------------------------------------- /** * OpenAI-compatible content part for multimodal messages. * Supports text and image_url parts. */ export type ContentPart = | { type: "text"; text: string } | { type: "image_url"; image_url: { url: string; detail?: "low" | "high" | "auto" }; }; /** * Message content can be either a plain string (text-only) or an array of * content parts (multimodal). Supports text and image_url parts. */ export type LLMMessageContent = string | ContentPart[]; // --------------------------------------------------------------------------- // OpenAI-compatible transport // --------------------------------------------------------------------------- // // We intentionally avoid pulling in the `openai` npm package: // - It ships its own fetch/stream polyfills that clash with Next.js Edge. // - We only need two endpoints (`/models` and `/chat/completions`). // - Hand-rolling the SSE parser lets us recognise the `reasoning_content` // delta emitted by Qwen/DeepSeek/QwQ-style reasoning models, which the // official SDK types do not cover yet. const MODELS_TIMEOUT_MS = 15_000; function authHeaders(provider: LLMProviderWithSecret): HeadersInit { return { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json", }; } export async function listModels( provider: LLMProviderWithSecret, ): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), MODELS_TIMEOUT_MS); try { const resp = await fetch(`${provider.baseUrl}/models`, { method: "GET", headers: authHeaders(provider), signal: controller.signal, }); if (!resp.ok) { const body = await resp.text().catch(() => ""); throw new Error( `/models returned ${resp.status}: ${body.slice(0, 200) || resp.statusText}`, ); } const json = (await resp.json()) as { data?: ModelListItem[]; models?: ModelListItem[]; }; // OpenAI + DashScope compatible mode both use `data`; some providers // emit `models` instead. Handle both. return json.data ?? json.models ?? []; } finally { clearTimeout(timer); } } export async function testConnectivity( provider: LLMProviderWithSecret, ): Promise { const start = Date.now(); try { const models = await listModels(provider); return { ok: true, latencyMs: Date.now() - start, modelCount: models.length, }; } catch (err) { return { ok: false, latencyMs: Date.now() - start, error: { code: "CONNECTIVITY_FAILED", message: err instanceof Error ? err.message : String(err), }, }; } } // --------------------------------------------------------------------------- // Streaming chat completion // --------------------------------------------------------------------------- export interface ChatCompletionStreamParams { model: string; messages: Array<{ role: ChatRole; content: LLMMessageContent }>; temperature?: number; signal?: AbortSignal; } /** * Call `/chat/completions` with `stream: true` and surface a normalised * event stream. The returned `ReadableStream` emits `ChatStreamEvent`s for: * * - `content.delta` → `choices[0].delta.content` * - `reasoning.delta` → `choices[0].delta.reasoning_content` (reasoning models) * - `done` → final terminator, optionally with token usage * - `error` → transport or parse failure * * The caller is responsible for translating the events into SSE frames for * the browser; see `app/api/v1/llm/chat/route.ts`. */ export async function chatCompletionStream( provider: LLMProviderWithSecret, params: ChatCompletionStreamParams, ): Promise> { const resp = await fetch(`${provider.baseUrl}/chat/completions`, { method: "POST", headers: authHeaders(provider), body: JSON.stringify({ model: params.model, messages: params.messages, stream: true, ...(params.temperature !== undefined ? { temperature: params.temperature } : {}), }), signal: params.signal, }); if (!resp.ok || !resp.body) { const body = await resp.text().catch(() => ""); throw new Error( `chat/completions returned ${resp.status}: ${body.slice(0, 400) || resp.statusText}`, ); } return parseOpenAISseStream(resp.body); } /** * Parse the upstream SSE stream into `ChatStreamEvent`s. The protocol: * * data: {"choices":[{"delta":{"content":"..."}}]}\n\n * data: {"choices":[{"delta":{"reasoning_content":"..."}}]}\n\n * data: [DONE]\n\n * * Handling rules: * - Skip empty lines and comments (`:`). * - A single logical event can span multiple `data:` lines; concatenate. * - `[DONE]` terminates with a `done` event. * - JSON parse errors emit an `error` event but do not tear the stream. */ function parseOpenAISseStream( source: ReadableStream, ): ReadableStream { const decoder = new TextDecoder("utf-8"); let buffer = ""; return new ReadableStream({ async start(controller) { const reader = source.getReader(); let finished = false; const finish = () => { if (!finished) { finished = true; controller.close(); } }; try { while (true) { const { value, done } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); // Split on SSE frame terminator (\n\n). let terminatorIdx = buffer.indexOf("\n\n"); while (terminatorIdx !== -1) { const rawFrame = buffer.slice(0, terminatorIdx); buffer = buffer.slice(terminatorIdx + 2); const event = decodeFrame(rawFrame); if (event) { controller.enqueue(event); if (event.type === "done") { finish(); return; } } terminatorIdx = buffer.indexOf("\n\n"); } } // Flush any trailing frame. if (buffer.trim()) { const event = decodeFrame(buffer); if (event) controller.enqueue(event); } controller.enqueue({ type: "done" }); finish(); } catch (err) { controller.enqueue({ type: "error", code: "STREAM_ERROR", message: err instanceof Error ? err.message : String(err), }); finish(); } }, }); } function decodeFrame(frame: string): ChatStreamEvent | null { const lines = frame.split("\n"); const dataParts: string[] = []; for (const line of lines) { if (!line || line.startsWith(":")) continue; if (line.startsWith("data:")) { dataParts.push(line.slice(5).trimStart()); } } if (dataParts.length === 0) return null; const payload = dataParts.join("\n"); if (payload === "[DONE]") { return { type: "done" }; } try { const json = JSON.parse(payload) as { choices?: Array<{ delta?: { content?: string; reasoning_content?: string; }; finish_reason?: string | null; }>; usage?: { prompt_tokens?: number; completion_tokens?: number }; }; const delta = json.choices?.[0]?.delta; if (delta?.reasoning_content) { return { type: "reasoning.delta", delta: delta.reasoning_content }; } if (delta?.content) { return { type: "content.delta", delta: delta.content }; } if (json.choices?.[0]?.finish_reason) { return { type: "done", usage: json.usage ? { promptTokens: json.usage.prompt_tokens, completionTokens: json.usage.completion_tokens, } : undefined, }; } // Non-delta keep-alive frames (role announcements, empty deltas). return null; } catch (err) { return { type: "error", code: "PARSE_ERROR", message: err instanceof Error ? err.message : String(err), }; } }