/** * MlxHttpBackend — `LlmBackend` implementation that delegates inference to a * local oMLX inference server over HTTP. * * oMLX exposes an OpenAI-compatible `/v1/chat/completions` endpoint backed * by MLX weights running on Apple Silicon. See * `docs/scope-memos/v0.5.0-tier-d-eval-2026-05-06.md` §D3 for benchmark * data and integration rationale. * * Lifecycle: oMLX is managed independently of the Node MCP host. Start it * before launching the bridge: * brew services start jundot/omlx/omlx * * Concurrency: each `chat()` call is a separate HTTP request; oMLX * serializes them (one generation at a time on Metal). * * Token counting: approximate — the exact tokenizer is not exposed over * HTTP. Returns `Math.ceil(text.length / 3.5)` (±15 % vs exact). The * chunker applies a safety margin on top of proxy counts, so this is safe. * * Grammar-constrained output: when `opts.format` (JSON Schema) is set, * the backend sends `response_format: { type: "json_schema", strict: true }`. * oMLX enforces the schema at decode time (enum constraints binding, required * fields produced), making it the unified backend for classify and extract tools. * * See: docs/scope-memos/v0.5.0-tier-d-eval-2026-05-06.md */ import type { LlmBackend, ChatOptions, ChatResult } from './backend.js'; export interface MlxHttpBackendOptions { /** * Base URL of the MLX bridge server, e.g. `"http://127.0.0.1:8080"`. * No trailing slash. */ baseUrl: string; /** * Context window size passed to the server in the request. Informational * only — the server-side model's context is fixed at load time. Used for * the `modelId` label and telemetry. */ numCtx?: number; /** * Model name to pass in the `model` field of each OpenAI request. * * - Required by oMLX for model routing by name. * - When omitted, auto-detected on first request via `GET /v1/models` * and cached for the backend lifetime. */ modelName?: string; /** * Thinking-trace suppression mechanism for this backend's model. * * - `'no_think'` (default): append `/no_think` to the user prompt (Qwen3 * thinking models honor it; inert on non-thinking models). * - `'chat_template'`: send `chat_template_kwargs: { enable_thinking: false }` * and leave the prompt untouched. Required for Qwen3-VL / Qwen3.5. * * Default preserves the exact B/C/D request shape (migration-snapshot contract). */ thinkingMode?: 'no_think' | 'chat_template'; /** * @internal Override the circuit-breaker total wait budget (default 5000 ms). * Used by tests to keep the "never recovers" path under vitest's 5s timeout. */ _restartPollBudgetMs?: number; /** * @internal Override the circuit-breaker poll interval (default 200 ms). */ _restartPollIntervalMs?: number; } /** * Normalize a JSON Schema for OpenAI Structured Outputs strict mode. * * oMLX strict mode requires every object node to satisfy: * 1. `additionalProperties: false` * 2. `required` lists every key in `properties` * * Without these, oMLX silently falls back to non-strict mode and the model * output is unconstrained — exactly the bug the strict mode was meant to * prevent. This helper walks the schema and patches every object node — * including nullable union types (`type: ["object", "null"]`) — in-place on a * deep clone, so the caller's schema is unchanged. * * Recurses into every applicator position (`properties`, `items`, * `prefixItems`, `anyOf`/`oneOf`/`allOf`, `$defs`, …) so nested AND sibling * subschemas are all tightened; only applicator keywords are walked, so * data-bearing `enum`/`const`/`default` values are never mutated. `$ref` is * left alone (oMLX resolves refs; the extract path rejects `$ref` upstream in * `sanitizeSchemaForStrictMode`); pattern / format are not strict-mode concerns. */ export declare function normalizeForStrictMode(schema: Record): Record; export declare class MlxHttpBackend implements LlmBackend { /** * Circuit-breaker constants for the oMLX-aborted-mid-request recovery * path. See `chat()` doc-comment and * `docs/notes/v0.5.x-omlx-stability-2026-05-11.md` for the failure * mode this protects against. */ private static readonly RESTART_POLL_INTERVAL_MS; private static readonly RESTART_POLL_BUDGET_MS; /** * Substrings on `Error.message` / `Error.code` / `Error.cause.*` that we * treat as "server died mid-request" — i.e. eligible for one circuit- * breaker retry after the launchd-managed oMLX has restarted. Anything * else propagates unchanged (HTTP 4xx/5xx, authentic JSON parse errors * on a complete response body, etc.). * * Two failure modes observed in production (oMLX SIGABRT mid-request): * 1. `fetch()` itself rejects before any response — undici raises * `TypeError: fetch failed` with cause `{ code: 'ECONNRESET' }` * or `{ code: 'UND_ERR_SOCKET' }`. * 2. Response headers arrive, then `response.json()` aborts mid-stream * with `TypeError: terminated` (undici 6+ wording for a connection * that was severed mid-body). Our `_chatOnce` wraps this as * "MlxHttpBackend: failed to parse JSON response — terminated"; * `"terminated"` covers both the unwrapped and wrapped forms. * * Deliberately NOT included: our own "MlxHttpBackend: network error — …" * wrapper text. If we matched that we'd retry on every fetch failure * (TLS handshake, DNS, etc.), defeating the narrow purpose of the breaker. */ private static readonly CONNECTION_RESET_MARKERS; private readonly baseUrl; private readonly configuredModelName?; /** Cached after first auto-detect or set from configuredModelName. */ private resolvedModelName?; private readonly restartPollBudgetMs; private readonly restartPollIntervalMs; private readonly thinkingMode; constructor(opts: MlxHttpBackendOptions); get modelId(): string; /** * Return the model name to pass in OpenAI requests. * * If `modelName` was supplied at construction time, use it directly. * Otherwise, query `GET /v1/models`, pick the first entry, and cache it * so subsequent calls skip the round-trip. */ private resolveModelName; /** * Run a chat-style completion against the MLX bridge server. * * Honors the AbortSignal by passing it to `fetch`. If the server returns * a non-2xx status, throws with the status + body for debuggability. * * **Circuit breaker**: when fetch fails with a connection-reset class * error (ECONNRESET / socket hang up / "fetch failed") — the symptom of * oMLX aborting mid-request while launchd auto-restarts it (see * `docs/notes/v0.5.x-omlx-stability-2026-05-11.md`) — this poll-pings * `/health` for up to 5s and, on recovery, retries the request exactly * once. Other errors (HTTP 4xx/5xx, JSON parse, user-aborted signal, * authentic refusals) propagate unchanged. * * This makes oMLX crashes recoverable from the caller's perspective at * a ~5s worst-case latency tax, instead of a connection-reset error. */ chat(opts: ChatOptions, signal?: AbortSignal): Promise; /** * The actual single HTTP attempt — used by `chat()` and its retry path. * Same observable behavior as the pre-circuit-breaker implementation. */ private _chatOnce; /** * Approximate token count via {@link estimateTokens}: Latin/code ≈ `ceil(chars/3.5)`, * CJK glyphs ≈ 1:1 (a flat `/3.5` proxy under-counts CJK ~3×). The exact tokenizer is * not exposed over HTTP; the chunker's 0.85 safety margin absorbs the residual drift. */ countTokens(text: string): Promise; /** * Liveness check: GET /health. Throws if the server is unreachable or * returns a non-2xx status. */ ping(): Promise; /** * Returns true if the error looks like the connection was severed by * the *server* mid-request (as opposed to a 4xx/5xx response or a * client-side abort). Walks the `Error.cause` chain (Node 22's fetch * sometimes nests the underlying socket error one level deep). */ private static isConnectionResetError; /** * Returns the first CONNECTION_RESET_MARKERS substring that appears in * the error's message or code chain (walking `cause` up to 4 levels), * or `null` if none match. Used both for the boolean check and for * telemetry attribution (which marker triggered the breaker). */ private static firstMatchingMarker; /** * Poll `/health` until oMLX answers or the budget runs out. Sleeps * `RESTART_POLL_INTERVAL_MS` between attempts. Respects the caller's * AbortSignal so a chunked-summarize job's overall cancel can still * unblock from inside the wait. */ private waitForRestart; } //# sourceMappingURL=mlx-http-backend.d.ts.map