/** * LlamaCppBackend — `LlmBackend` implementation that runs a GGUF model * in-process via `node-llama-cpp` v3. * * Removes the Ollama daemon dependency entirely (see * `docs/scope-memos/v0.4.0-llama-cpp-backend-2026-05-04.md` and * `docs/prior-art/llama-cpp-backend-2026-05-04.md`). * * Lifecycle: model + context are loaded lazily on first `chat`/`countTokens` * call (the `getLlama()` boot is non-trivial — Metal init, weights mmap). * `backend-factory.ts` memoizes instances per modelPath so subsequent calls * reuse the loaded model. The bridge process holds the loaded weights for * its lifetime; OS reaps everything on shutdown. * * Concurrency: a single `LlamaContext` runs one prompt at a time (it * serializes internally on a `LlamaContextSequence`). This matches Ollama's * Metal-serialized behavior on the same hardware — concurrent MCP calls * already queue, so in-process queueing here is functionally equivalent. * * Token counting: exact, via `model.tokenize(text).length`. No proxy drift. */ import type { LlmBackend, ChatOptions, ChatResult } from './backend.js'; export interface LlamaCppBackendOptions { /** Absolute path to the GGUF model file. */ modelPath: string; /** * Context window size in tokens. Fixed at context-creation time — unlike * Ollama's per-call `num_ctx`, llama.cpp pre-allocates KV cache for the * declared size. Pick the largest size the tier ever needs. */ contextSize: number; } export declare class LlamaCppBackend implements LlmBackend { private readonly opts; private llama; private model; private context; private chatChain; private loadingPromise; private disposed; constructor(opts: LlamaCppBackendOptions); get modelId(): string; /** * Lazy boot. Idempotent — safe to call repeatedly. Concurrent callers * share a single boot via `loadingPromise` (double-checked lock pattern). * On boot failure, partially-allocated resources are cleaned up before * the error rethrows so the next call can retry from a clean state. */ private ensureLoaded; chat(opts: ChatOptions, signal?: AbortSignal): Promise; private _chatSerialized; /** * Exact token count using the loaded model's tokenizer. No proxy drift. * Yields to the event loop on large inputs to keep MCP keep-alives flowing. */ countTokens(text: string): Promise; ping(): Promise; /** * Free model + context. After dispose(), this backend is unusable — * subsequent chat() / countTokens() / ping() calls throw. * * Awaits any in-flight chat queue tail so we don't free resources while * a native generation is still running on the context. */ dispose(): Promise; } //# sourceMappingURL=llama-cpp-backend.d.ts.map