/** * Local LLM Adapter * * Connects to any local OpenAI-compatible inference server: * llama.cpp, Ollama, LM Studio, or similar. * No API key required — the server runs locally. * * Supported runtimes: * llama.cpp: llama-server -m model.gguf --port 8080 --ctx-size 4096 * Ollama: ollama serve (default port 11434) * LM Studio: Start server in UI (default port 1234) * * The server must expose: POST http://localhost:PORT/v1/chat/completions * * @version 1.0.0 */ import { BaseLLMAdapter } from '../base-adapter'; import type { Capabilities, LLMProviderConfig, LLMCompletionRequest, LLMCompletionResponse, LLMStreamChunk } from '../types'; type LocalLLMAdapterConfig = Omit & { apiKey?: string; model?: string; /** * Use Ollama's native /api/chat endpoint instead of /v1/chat/completions. * Auto-detected when the base URL contains ':11434' (Ollama default port). * Required for thinking models (qwen3, deepseek-r1, etc.) because the OpenAI * compat layer drops tool_calls in responses that include thinking tokens. * Verified 2026-06-16: /v1/chat/completions returns toolCalls=[] for qwen3:4b * with tools; /api/chat returns tool_calls correctly. */ nativeOllamaApi?: boolean; }; export declare const LOCAL_LLM_MODELS: readonly ["qwen3-4b-instruct-2507", "mistral-small-4", "mistral-7b-instruct", "llama-3.1-8b-instruct", "llama-3.2-3b-instruct", "granite-4.0-1b", "phi-4-mini-instruct"]; export type LocalLLMModel = (typeof LOCAL_LLM_MODELS)[number]; /** * Capability manifest — Ollama / llama.cpp / LM Studio / vLLM via * OpenAI-compatible interface. Capabilities are PER-MODEL not * per-provider; this is the conservative manifest for the * runtime itself. Brains needing specific local-model superpowers * (e.g. tool-use on deepseek-v3.1) should override via per-deployment * capability declarations. * * Per /research task_1778109552044_xhmm — populate per-model * capability sheets for the models actually run (deepseek-v3.1:671b, * gpt-oss:120b, kimi-1T cloud, etc.). Until then: conservative defaults. * * Exported as a constant so the capability-aware router can read it * without instantiating the adapter — single source of truth per W.GOLD.006. */ export declare const LOCAL_LLM_CAPABILITIES: Capabilities; export declare class LocalLLMAdapter extends BaseLLMAdapter { readonly name: "local-llm"; readonly models: readonly ["qwen3-4b-instruct-2507", "mistral-small-4", "mistral-7b-instruct", "llama-3.1-8b-instruct", "llama-3.2-3b-instruct", "granite-4.0-1b", "phi-4-mini-instruct"]; readonly defaultHoloScriptModel: string; readonly capabilities: Capabilities; private readonly localBaseURL; /** True → complete() uses /api/chat (native Ollama); false → /v1/chat/completions. */ private readonly useNativeOllamaApi; constructor(config?: LocalLLMAdapterConfig); protected getDefaultModel(): string; /** * Send a chat completion request to the local LLM server. * * Two paths depending on `useNativeOllamaApi` (auto-detected from port 11434): * * Ollama native (/api/chat, stream:false) — used when useNativeOllamaApi=true. * Ollama's /v1/chat/completions OpenAI-compat shim silently drops tool_calls * for thinking models (qwen3, deepseek-r1) because thinking tokens precede * tool calls and the compat layer misroutes them. The native endpoint does not * have this bug. Verified 2026-06-16: /v1 → toolCalls=0, /api/chat → toolCalls=1. * * OpenAI-compat (/v1/chat/completions) — used for llama.cpp / LM Studio / vLLM. */ complete(request: LLMCompletionRequest, model?: string): Promise; /** * Injects `/no_think` into the system prompt for qwen3-family models when * thinking mode is off. Ollama ≤0.30.x silently ignores `think: false` and * routes thinking tokens into the `content` field, bloating outputs and * corrupting tool-call parsing. The `/no_think` directive works at the model * tokenizer level, independent of Ollama version. * Verified: Ollama 0.30.8 + qwen3:4b — `think:false` ignored, `/no_think` works. */ private _withNoThinkMessages; /** * Returns `{}` — we never send `think:false` in the Ollama payload. * * Confirmed 2026-06-16: `think:false` disables the decode-time grammar mask * that enables structured JSON tool calls for BOTH qwen3 AND Gemma 4 families * (same root cause as Ollama #15260 / vLLM #39130 — mask deferred until the * end-of-thinking token which never fires when thinking is closed, so the model * emits prose instead of tool_calls JSON). With thinking ON, Ollama 0.30.8 * correctly routes thinking to `message.thinking` (separate field) and leaves * `message.content` clean — _stripThinkBlock() handles any edge-case bleed. * Thinking is soft-suppressed via `/no_think` in the system prompt * (_withNoThinkMessages), which reduces thinking tokens without breaking * tool-call structured output. */ private _thinkParam; /** * Ollama 0.30.x (qwen3): strips the opener but leaves the thinking * body + closing tag inside message.content. Strip everything up to * and including so the returned content is the model's actual reply. * When future Ollama separates thinking into message.thinking, content will * arrive clean and this is a no-op. */ private _stripThinkBlock; private completeNativeOllama; private completeOpenAICompat; /** Shared fetch+error handling for both complete() paths. */ private fetchJson; /** Build a unified LLMCompletionResponse from either response format. */ private buildResponse; /** * Map Ollama's tool definition shape (function.parameters) from our * ToolSpec shape (input_schema). Ollama's /api/chat uses `parameters` * where our ToolSpec uses `input_schema` — same schema, different key. */ private mapToolsToOllama; /** * Map Ollama's `done_reason` to our unified `finishReason`. */ private mapDoneReason; /** * Stream a completion as provider-agnostic chunks via Ollama's native * `/api/chat` endpoint with `stream: true`. * * Ollama returns NDJSON — one JSON object per line. Each line carries an * incremental `message.content` text delta and/or a `message.tool_calls` * array. The final line has `done: true` with usage statistics. * * Translation rules: * message.content (non-empty) → text_delta * message.tool_calls → tool_use_start + tool_use_end per tool * (Ollama sends complete tool calls in one * shot, no streamed JSON fragments, so no * tool_use_input_delta chunks) * done: true → message_stop (with finishReason + usage) * * No `withRetry` — partial-text retries would re-emit prefix tokens and * corrupt downstream state (the same contract as AnthropicAdapter). * Pre-flight failures (429, 5xx, network) throw before the first chunk; * mid-stream failures yield a `message_stop` with `finishReason: 'error'` * and the partial state observed so far. */ streamCompletion(request: LLMCompletionRequest, model?: string): AsyncIterable; /** Pre-flight for both streaming paths: POST, status-check, throw before the first chunk. */ private preflightStream; private streamNativeOllama; /** * Stream a completion via the OpenAI-compatible SSE surface * (`POST /v1/chat/completions`, `stream: true`) — llama.cpp llama-server, * HoloServe (pytorch-holo), LM Studio, vLLM. * * SSE framing: `data: {json}` lines, terminated by `data: [DONE]`. Each JSON * chunk carries `choices[0].delta.content` text deltas and/or `delta.tool_calls` * argument FRAGMENTS (accumulated per tool-call index, emitted as * tool_use_start + tool_use_end once the stream finishes — OpenAI semantics: * arguments are only complete at finish). Usage rides the final data chunk * when the server sends it (llama-server and HoloServe both do). * * Same error contract as the Ollama path: pre-flight failures throw before the * first chunk; mid-stream failures yield message_stop with finishReason 'error' * then throw. `request.grammar` passes through for valid-by-construction output. */ private streamOpenAICompat; /** * Returns the HoloScript-tuned system prompt for local models. */ protected getHoloScriptSystemPrompt(): string; /** * Check if the local LLM server is reachable. * Delegates to BaseLLMAdapter.healthCheckLocalServer — same /health → * /v1/models fallback, branded error message for this adapter. */ healthCheck(): Promise<{ ok: boolean; latencyMs: number; error?: string; }>; } export {};