// Native Ollama /api/chat streaming provider. // // Talks directly to Ollama's NDJSON endpoint and owns the whole request - // num_ctx, think, keep_alive, truncate:false - which the OpenAI-compatible // endpoint pi would otherwise use can't carry. This is the core of the // extension; the other modules exist to configure and feed this function. // // Reliability defences: // Ghost-token retry — Ollama occasionally generates output tokens but streams // nothing visible (done:true, eval_count > 0, empty message). The provider // detects this on the first NDJSON line and retries up to OLLAMA_NATIVE_GHOST_RETRIES // times (default 2). At 20% per-attempt failure rate, 2 retries → 99.2% success. // // Truncation detection — if the connection closes before a done:true chunk // arrives, partial events have already been pushed to the consumer so we can't // retry safely. We surface a clear error instead of silently accepting partial // output. import { dbg, dumpRequest, dumpResponseLine } from "./debug.js"; import { convertContext, type PiContext } from "./convert.js"; import { describeEmptyGeneration, describeOllamaError } from "./errors.js"; import { openChat, targetFor } from "./ollama-client.js"; import type { OllamaChunk, OllamaRequest } from "./wire.js"; import type { OllamaExtensionSettings } from "./settings.js"; import { resolveModelSettings } from "./model-config.js"; import type { TelemetrySink } from "./telemetry.js"; import { parseGenerationMetrics } from "./throughput.js"; import { resolveThink, type ThinkingLevelMap } from "./thinking.js"; // ============================================================================ // Types — minimal structural interfaces that match pi-ai's shapes. // We declare them locally to avoid a hard import-time dependency on the exact // installed version of @earendil-works/pi-ai. // ============================================================================ export interface PiModel { id: string; api: string; provider?: string; baseUrl?: string; headers?: Record; contextWindow?: number; maxTokens?: number; input?: ("text" | "image")[]; reasoning?: boolean; // Carried through from our registration by pi's provider-composer; older // pi versions drop it, which falls back to the legacy on/off mapping. thinkingLevelMap?: ThinkingLevelMap; } interface PiSimpleStreamOptions { signal?: AbortSignal; headers?: Record; temperature?: number; maxTokens?: number; // pi's SimpleStreamOptions.reasoning: ThinkingLevel ("minimal"|"low"|"medium"| // "high"|"xhigh") — "off" is not a member; thinking-off arrives as the field // being absent. Typed as plain string here: this crosses a runtime boundary // from whatever pi version hosts us, and resolveThink treats a literal "off" // defensively. reasoning?: string; onPayload?: (body: unknown, model: PiModel) => Promise | unknown; onResponse?: ( info: { status: number; headers: Record }, model: PiModel, ) => Promise | void; } // Matches AssistantMessageEventStream's push/end interface. interface EventStream { push(event: unknown): void; end(): void; } // ============================================================================ // Constants // ============================================================================ const DEFAULT_NUM_CTX = 32768; // ============================================================================ // Ghost-token detection // ============================================================================ function isGhostChunk(chunk: OllamaChunk): boolean { if (!chunk.done) return false; if ((chunk.eval_count ?? 0) === 0) return false; const m = chunk.message; if (!m) return true; const hasContent = (m.content && m.content.length > 0) || (m.thinking && m.thinking.length > 0) || (m.tool_calls && m.tool_calls.length > 0); return !hasContent; } // ============================================================================ // Tool-call ID generation (fallback when Ollama omits one) // ============================================================================ let toolCallCounter = 0; function generateToolCallId(): string { return `ollama_${Date.now()}_${++toolCallCounter}`; } // ============================================================================ // Stop-reason mapping // ============================================================================ function mapDoneReason(reason: string | undefined): string { switch (reason) { case "stop": case "end": case undefined: return "stop"; case "length": return "length"; default: return "stop"; } } /** Signals the swallowed-tool-call guard decides on (see below). */ export interface SwallowSignals { sawToolCalls: boolean; sawDoneChunk: boolean; outputTokens: number; chunksReceived: number; } /** * Swallowed-tool-call detection (issue #3): the stream completed normally with * visible content but no tool call, yet the model generated far more tokens * than it streamed — the signature of a tool call Ollama buffered and dropped * before sending (gemma4:e4b after a long think). * * chunksReceived ≈ streamed tokens only on LOCAL streams (~1 token per NDJSON * chunk). Batched streams — Ollama Cloud emits ~30 tokens/chunk — deflate the * streamed ratio and false-positived this guard on healthy turns (issue #4), * so the guard now stands down when tokens-per-chunk indicates batching. The * real issue-#3 swallow ran ~2.8 tokens/chunk, well under the gate. Tradeoff * accepted: a swallow on a batched stream is currently undetectable — its * signature is uncharacterized (the n=1 calibration discipline: thresholds * come from observed logs, not guesses). * * Threshold calibration: issue-#3 log ratio 0.36 on the swallow vs 0.81–0.92 * on healthy local tool turns; issue-#4 log ~30 tokens/chunk on Ollama Cloud. */ export function shouldFlagSwallowedToolCall(s: SwallowSignals): boolean { const SWALLOW_MIN_OUTPUT_TOKENS = 200; // ignore short turns (noise floor) const SWALLOW_MAX_STREAMED_RATIO = 0.6; // streamed/generated below this is suspect const SWALLOW_MIN_TOKEN_GAP = 200; // and the absolute shortfall must be large const SWALLOW_MAX_TOKENS_PER_CHUNK = 6; // above this the stream is batched — stand down (issue #4) if (s.sawToolCalls || !s.sawDoneChunk) return false; if (s.outputTokens <= SWALLOW_MIN_OUTPUT_TOKENS) return false; if ( s.chunksReceived > 0 && s.outputTokens / s.chunksReceived > SWALLOW_MAX_TOKENS_PER_CHUNK ) { return false; } return ( s.chunksReceived / s.outputTokens < SWALLOW_MAX_STREAMED_RATIO && s.outputTokens - s.chunksReceived > SWALLOW_MIN_TOKEN_GAP ); } // ============================================================================ // Main streaming function // ============================================================================ export interface ChatBodyInputs { modelId: string; messages: OllamaRequest["messages"]; options: OllamaRequest["options"]; /** Resolved keep_alive override, or undefined = omit the field (server decides). */ keepAlive: string | number | undefined; /** Whether the model supports thinking — Ollama rejects `think` otherwise. */ reasoningCapable: boolean; /** pi's reasoning level for this turn (absent = thinking off). */ reasoningLevel: string | undefined; /** The model's per-level think values (gh#13), or undefined = legacy on/off. */ thinkingLevelMap: ThinkingLevelMap | undefined; /** Already-converted tool definitions, or undefined to omit. */ tools: OllamaRequest["tools"] | undefined; } /** * Build the exact /api/chat request body. Pure — extracted from streamOllama * so the wire contract is unit-testable (adopted from PR #6's test approach): * the conditional fields must be KEY-ABSENT when off, not undefined-valued. * In particular keep_alive: a per-request value OVERRIDES the server's * OLLAMA_KEEP_ALIVE, so the honest default is to omit it entirely (gh#5). */ export function buildChatRequestBody(inputs: ChatBodyInputs): OllamaRequest { const body: OllamaRequest = { model: inputs.modelId, messages: inputs.messages, stream: true, options: inputs.options, // Never let Ollama silently drop the oldest messages to fit num_ctx: // it would answer without them and report a shrunken prompt_eval_count, // so pi would never compact. Refused instead, the overflow reaches pi as // an error it recognizes (describeOllamaError) and pi compacts and retries. truncate: false, }; if (inputs.keepAlive !== undefined) { body.keep_alive = inputs.keepAlive; } // Only sent for thinking-capable models; Ollama rejects `think` on models // without thinking support. Also omitted when off was asked for on a model // that can't switch thinking off (resolveThink returns undefined). if (inputs.reasoningCapable) { const think = resolveThink(inputs.reasoningLevel, inputs.thinkingLevelMap); if (think !== undefined) body.think = think; } if (inputs.tools !== undefined) { body.tools = inputs.tools; } return body; } /** * The model options every request for this model carries. num_ctx decides how * Ollama loads the model, so the chat turn and the warm-up (warm.ts) both use * this: a warm-up with any other num_ctx would just make the first turn reload. */ export function requestOptions( model: { contextWindow?: number }, settings: { numCtx: number }, turn?: { temperature?: number; maxTokens?: number }, ): NonNullable { const options: NonNullable = { num_ctx: model.contextWindow ?? settings.numCtx ?? DEFAULT_NUM_CTX, }; if (turn?.temperature !== undefined) options.temperature = turn.temperature; if (turn?.maxTokens !== undefined) options.num_predict = turn.maxTokens; return options; } /** * The /api/chat request for a turn: pi's context converted, the model's * options, keep_alive and think. The one place this happens - the prompt * warm-up (warm.ts) builds its request here too, so Ollama sees the exact * prefix the first real turn will send. */ export function buildTurnRequest( model: PiModel, context: PiContext, options: Pick | undefined, settings: OllamaExtensionSettings, ): OllamaRequest { const supportsVision = model.input?.includes("image") ?? false; const { messages, tools } = convertContext(context, supportsVision); // model.contextWindow already incorporates the context-length override and // the capped default (index.ts toProviderModel), so this wire value and pi's // UI counter agree. return buildChatRequestBody({ modelId: model.id, messages, options: requestOptions(model, settings, options), keepAlive: resolveModelSettings(settings.modelConfig, settings, model.id).keepAlive, reasoningCapable: Boolean(model.reasoning), reasoningLevel: options?.reasoning, thinkingLevelMap: model.thinkingLevelMap, tools, }); } export function streamOllama( model: PiModel, context: PiContext, options: PiSimpleStreamOptions | undefined, settings: OllamaExtensionSettings, StreamClass: new () => EventStream, telemetry?: TelemetrySink, ): EventStream { const stream = new StreamClass(); // Telemetry is strictly best-effort: a throwing sink must never fail or // alter the model turn. const report = (fn: (sink: TelemetrySink) => void) => { if (!telemetry) return; try { fn(telemetry); } catch (e) { dbg("telemetry-error", { error: String(e) }); } }; (async () => { // Build the output message shell — populated incrementally as chunks arrive. const output: Record = { role: "assistant", content: [], api: model.api, provider: model.provider ?? "ollama", model: model.id, usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, stopReason: "stop", timestamp: Date.now(), }; try { const target = targetFor(settings, model.baseUrl); let body = buildTurnRequest(model, context, options, settings); const numCtx = body.options?.num_ctx; // Allow callers to inspect or replace the request body. if (options?.onPayload) { const next = await options.onPayload(body, model); if (next !== undefined) body = next as OllamaRequest; } dbg("request", { baseUrl: target.baseUrl, model: body.model, messages: body.messages.length, tools: body.tools?.length ?? 0, num_ctx: body.options?.num_ctx, think: body.think, lastMessageRole: body.messages[body.messages.length - 1]?.role, }); // ── Ghost-token retry loop ────────────────────────────────────────── // Read the first NDJSON line of each attempt. If it's a ghost chunk // (done:true, eval_count > 0, empty message) cancel and retry. // On a healthy response, prepend the first line to initialBuffer so // the main streaming loop processes it normally. const maxRetries = settings.ghostRetries; let dumpId: string | null = null; let reader!: ReadableStreamDefaultReader; let initialBuffer = ""; let ghostAttempts = 0; const decoder = new TextDecoder(); while (true) { dumpId = dumpRequest(body); const response = await openChat(target, body, { signal: options?.signal, headers: [model.headers, options?.headers], }); if (options?.onResponse) { const hdrs: Record = {}; response.headers.forEach((v, k) => { hdrs[k] = v; }); await options.onResponse({ status: response.status, headers: hdrs }, model); } dbg("response-status", { status: response.status, ok: response.ok, attempt: ghostAttempts, }); if (!response.ok) { const text = await response.text().catch(() => ""); throw new Error( describeOllamaError(text, { endpoint: "/api/chat", status: response.status, numCtx, apiKeySent: Boolean(target.apiKey), }), ); } if (!response.body) { throw new Error("Ollama /api/chat returned no response body"); } reader = response.body.getReader(); // Read until we have at least one complete NDJSON line. let buf = ""; let firstLine: string | null = null; let streamEndedEarly = false; while (firstLine === null) { const { value, done: streamDone } = await reader.read(); if (streamDone) { streamEndedEarly = true; break; } buf += decoder.decode(value, { stream: true }); const nl = buf.indexOf("\n"); if (nl !== -1) { firstLine = buf.slice(0, nl).trim(); buf = buf.slice(nl + 1); if (!firstLine) firstLine = null; // empty line — keep looking } } // Defensive structure: only short-circuit when there's nothing to // inspect. If we have a firstLine — even one that arrived alongside // a stream-end signal — we still want to check it for ghost-chunk // patterns and preserve it in initialBuffer for the main loop. // This decouples correctness from the inner loop's exit invariants. if (streamEndedEarly && firstLine === null) { initialBuffer = buf; break; } let firstChunk: OllamaChunk | null = null; if (firstLine !== null) { try { firstChunk = JSON.parse(firstLine) as OllamaChunk; } catch { firstChunk = null; } } // firstLine !== null is implied by firstChunk being non-null (firstChunk // is only assigned inside the `if (firstLine !== null)` parse block), but // the explicit check is needed for TS narrowing of dumpResponseLine. if (firstLine !== null && firstChunk && isGhostChunk(firstChunk)) { dbg("ghost-detected", { attempt: ghostAttempts, evalCount: firstChunk.eval_count, maxRetries, }); dbg("chunk", firstLine); dumpResponseLine(dumpId, firstLine); await reader.cancel().catch(() => undefined); ghostAttempts++; if (ghostAttempts > maxRetries) { throw new Error( describeEmptyGeneration({ evalCount: firstChunk.eval_count ?? 0, attempts: ghostAttempts, toolsOffered: (body.tools?.length ?? 0) > 0, }), ); } continue; } // Healthy or unparseable — preserve firstLine if we have one, so the // main loop can attempt parsing too (and surface a clear error if it // was malformed). This handles the case where the stream delivered // content alongside its done signal in a single read. initialBuffer = (firstLine !== null ? `${firstLine}\n` : "") + buf; break; } // Past the ghost-retry loop: this attempt is the one that streams. report((t) => t.started(model.id)); stream.push({ type: "start", partial: output }); // ── Content block helpers ─────────────────────────────────────────── type Block = | { type: "text"; text: string } | { type: "thinking"; thinking: string } | { type: "toolCall"; id: string; name: string; arguments: Record }; const blocks = output.content as Block[]; let currentBlock: Block | null = null; const blockIndex = () => blocks.length - 1; const finishBlock = (block: typeof currentBlock) => { if (!block) return; if (block.type === "text") { stream.push({ type: "text_end", contentIndex: blockIndex(), content: block.text, partial: output, }); } else if (block.type === "thinking") { stream.push({ type: "thinking_end", contentIndex: blockIndex(), content: block.thinking, partial: output, }); } else if (block.type === "toolCall") { stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall: block, partial: output, }); } }; // ── Main streaming loop ───────────────────────────────────────────── // Drain-first: process any complete lines already in the buffer before // issuing another reader.read(). This is required because the ghost // retry loop may have seeded initialBuffer with real content. let buffer = initialBuffer; let sawToolCalls = false; let sawDoneChunk = false; let chunksReceived = 0; let streamDoneFlag = false; let finalChunk: OllamaChunk | undefined; outer: while (true) { while (true) { const nl = buffer.indexOf("\n"); if (nl === -1) break; const line = buffer.slice(0, nl).trim(); buffer = buffer.slice(nl + 1); if (!line) continue; chunksReceived++; dbg("chunk", line); dumpResponseLine(dumpId, line); let chunk: OllamaChunk; try { chunk = JSON.parse(line) as OllamaChunk; } catch (e) { dbg("parse-error", { line, error: String(e) }); continue; } if (chunk.error) { throw new Error( describeOllamaError(chunk.error, { endpoint: "/api/chat", numCtx }), ); } if (!output.responseId && chunk.created_at) { output.responseId = `ollama-${chunk.created_at}-${chunk.model}`; } const m = chunk.message; if (m) { // Thinking deltas if (m.thinking !== undefined && m.thinking.length > 0) { const thinkingDelta = m.thinking; if (!currentBlock || currentBlock.type !== "thinking") { finishBlock(currentBlock); currentBlock = { type: "thinking", thinking: "" }; blocks.push(currentBlock); stream.push({ type: "thinking_start", contentIndex: blockIndex(), partial: output, }); } currentBlock.thinking += m.thinking; report((t) => t.progress(thinkingDelta.length)); stream.push({ type: "thinking_delta", contentIndex: blockIndex(), delta: m.thinking, partial: output, }); } // Text deltas if (m.content !== undefined && m.content.length > 0) { const textDelta = m.content; if (!currentBlock || currentBlock.type !== "text") { finishBlock(currentBlock); currentBlock = { type: "text", text: "" }; blocks.push(currentBlock); stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output, }); } currentBlock.text += m.content; report((t) => t.progress(textDelta.length)); stream.push({ type: "text_delta", contentIndex: blockIndex(), delta: m.content, partial: output, }); } // Tool calls — Ollama delivers each as a complete parsed object. // Emit start+delta+end as a synchronous burst (mirrors google.ts). if (m.tool_calls && m.tool_calls.length > 0) { if (currentBlock) { finishBlock(currentBlock); currentBlock = null; } sawToolCalls = true; for (const wireTc of m.tool_calls) { const args = wireTc.function.arguments ?? {}; const argsString = JSON.stringify(args); const providedId = wireTc.id; const isDuplicate = providedId !== undefined && blocks.some( (b) => b.type === "toolCall" && b.id === providedId, ); const id = !providedId || isDuplicate ? generateToolCallId() : providedId; const toolCall: Block & { type: "toolCall" } = { type: "toolCall", id, name: wireTc.function.name, arguments: args, }; blocks.push(toolCall); stream.push({ type: "toolcall_start", contentIndex: blockIndex(), partial: output, }); stream.push({ type: "toolcall_delta", contentIndex: blockIndex(), delta: argsString, partial: output, }); stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall, partial: output, }); } } } if (chunk.done) { sawDoneChunk = true; finalChunk = chunk; finishBlock(currentBlock); currentBlock = null; const inputTokens = chunk.prompt_eval_count ?? 0; const outputTokens = chunk.eval_count ?? 0; output.usage = { input: inputTokens, output: outputTokens, cacheRead: 0, cacheWrite: 0, totalTokens: inputTokens + outputTokens, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; // Ollama returns done_reason:"stop" even on tool-call turns. output.stopReason = sawToolCalls ? "toolUse" : mapDoneReason(chunk.done_reason); break outer; } } if (streamDoneFlag) break; const { value, done: streamDone } = await reader.read(); if (streamDone) { dbg("stream-end", { reason: "reader-done", buffered: buffer.length }); streamDoneFlag = true; continue; } buffer += decoder.decode(value, { stream: true }); } finishBlock(currentBlock); dbg("done", { stopReason: output.stopReason, sawToolCalls, chunksReceived, contentBlocks: blocks.length, blockTypes: blocks.map((b) => b.type), inputTokens: (output.usage as Record).input, outputTokens: (output.usage as Record).output, }); if (options?.signal?.aborted) { throw new Error("Request was aborted"); } // Empty response: the connection closed before any chunk arrived. // Indicates Ollama crashed, the model failed to load, or an upstream // network issue. Distinct from truncation (which has partial chunks). if (!sawDoneChunk && chunksReceived === 0) { dbg("empty-response", { chunksReceived, bufferLength: buffer.length }); throw new Error( `Ollama stream ended without sending any chunks. Check Ollama logs — ` + `the model may have failed to load, the server may have closed the ` + `connection, or an upstream network issue may have occurred. Retry the request.`, ); } // Truncation: real content arrived but the connection closed without // a done:true terminator. Can't auto-retry — events already emitted. if (!sawDoneChunk && chunksReceived > 0) { dbg("truncated", { chunksReceived, blockTypes: blocks.map((b) => b.type), }); throw new Error( `Ollama stream truncated mid-response: received ${chunksReceived} chunk(s) of ` + `${blocks.map((b) => b.type).join("+") || "no"} content, but the connection ` + `closed before any chunk with done:true was emitted. This is an Ollama-side ` + `reliability issue, often triggered when the model attempts to generate a tool ` + `call. Retry the turn.`, ); } // Post-stream ghost check: eval_count > 0 but nothing visible arrived. const hasMeaningfulContent = blocks.some( (b) => (b.type === "text" && (b as { type: "text"; text: string }).text.trim().length > 0) || (b.type === "thinking" && (b as { type: "thinking"; thinking: string }).thinking.trim().length > 0) || b.type === "toolCall", ); const outputTokens = (output.usage as Record).output ?? 0; if (outputTokens > 0 && !hasMeaningfulContent) { dbg("ghost-tokens", { evalCount: outputTokens, chunksReceived }); throw new Error( describeEmptyGeneration({ evalCount: outputTokens, // Any earlier attempts were empty too (the ghost retries). attempts: ghostAttempts + 1, toolsOffered: (body.tools?.length ?? 0) > 0, }), ); } // Swallowed-tool-call detection — decision extracted to the pure // shouldFlagSwallowedToolCall (top of file) so the thresholds and the // issue-#4 batched-stream stand-down are unit-testable. The post-stream // ghost check above misses this case because thinking + text DID stream // first; without the guard the turn ends as a silent no-op ("I'll // proceed with the edits" → nothing happens). if ( shouldFlagSwallowedToolCall({ sawToolCalls, sawDoneChunk, outputTokens, chunksReceived, }) ) { dbg("swallowed-toolcall", { outputTokens, chunksReceived, streamedRatio: chunksReceived / outputTokens, tokenGap: outputTokens - chunksReceived, blockTypes: blocks.map((b) => b.type), }); throw new Error( `Ollama generated ${outputTokens} tokens but streamed only ${chunksReceived}, ` + `then ended the turn with no tool call. The ~${outputTokens - chunksReceived} ` + `unstreamed tokens were almost certainly a tool call that Ollama's parser ` + `buffered and dropped before sending — a known Ollama-side issue with smaller ` + `models after extended reasoning. The turn was not silently completed and no ` + `tool call was lost on our side. Retry the turn, or use a more capable ` + `tool-calling model.`, ); } // Only an accepted response counts as a completed measurement - every // reliability guard above has passed by this point. report((t) => t.completed( finalChunk ? parseGenerationMetrics(finalChunk) : undefined, { sawToolCalls }, ), ); stream.push({ type: "done", reason: output.stopReason, message: output }); stream.end(); } catch (error) { output.stopReason = options?.signal?.aborted ? "aborted" : "error"; output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); dbg("error", { stopReason: output.stopReason, message: output.errorMessage, }); report((t) => t.failed()); stream.push({ type: "error", reason: output.stopReason, error: output }); stream.end(); } })(); return stream; }