/** * Tier-based model routing. * * The bridge routes each MCP tool invocation to a model tier. Tiers are * size/latency buckets, not capability buckets — the tool decides which * tier it wants based on expected workload (short classify vs long summary). * * All tiers route through `MlxHttpBackend` to a local oMLX * (https://github.com/jundot/omlx) inference server. Single backend, single * inference engine, KV-cache persistence across requests, schema strictness * via OpenAI Structured Outputs (`response_format: { type: "json_schema", * strict: true }`). * * Default models (mlx-community on Hugging Face): * - Tier B: `Qwen3-4B-Instruct-2507-4bit` (~2.5 GB) — non-thinking variant. * The "Instruct-2507" suffix matters: bare `Qwen3-4B`/`Qwen3-8B` are * thinking models that emit `...` traces by default and * burn the per-tool MAX_OUTPUT_TOKENS cap before producing real output. * Instruct-2507 is the explicitly-non-thinking line. * - Tier C: `Qwen3-8B-4bit` (~5 GB) — thinking variant; bridge injects * `/no_think` into user messages to disable reasoning at call time. * Long-form summarize at numCtx=32 768. * - Tier D: `Qwen3-14B-4bit` (~8 GB) — also thinking; same `/no_think` * injection. Hardest tasks (classify subtle, transform on dense input). * * All Apache-2.0. Download into `~/.omlx/models/` via: * npm run download-models * Start oMLX: * brew services start jundot/omlx/omlx */ export type Tier = 'B' | 'C' | 'D' | 'V'; export interface TierConfig { /** * Base URL of the oMLX (or any OpenAI-compatible) inference server. * Default: `"http://127.0.0.1:8000"` (oMLX's default port). */ mlxUrl?: string; /** * Model name sent in the OpenAI `model` field — must match a directory name * under `~/.omlx/models/`. oMLX serves multiple models from one process and * routes requests by this name. * * Example: `"Qwen3-4B-Instruct-2507-4bit"` */ mlxModelName?: string; /** * Context window size in tokens. * * Informational for `MlxHttpBackend` — the server-side model's max context * is fixed at load time. Used for the chunker's safety margin and tool * `maxInputTokens`. * * Tier B → 8192 (fast 4B model) * Tier C → 32768 (8B model with longer context) * Tier D → 16384 (14B model; ~9-10 GB at this context size) * Tier V → 32768 (Qwen3-VL-4B vision model; image-bearing calls) */ numCtx?: number; /** * How `MlxHttpBackend` suppresses the model's reasoning trace for this tier. * * - `'no_think'` (default): append `/no_think` to the user prompt. Works on * Qwen3 thinking models (8B/14B); inert on the non-thinking Instruct-2507. * - `'chat_template'`: send `chat_template_kwargs: { enable_thinking: false }` * in the request, leaving the prompt untouched. Required for Qwen3-VL / * Qwen3.5 — `/no_think` does NOT disable thinking on those chat templates. * * Absent → `'no_think'` (preserves the exact B/C/D request contract captured * by `migration-snapshot.test.ts`). */ thinkingMode?: 'no_think' | 'chat_template'; } export interface BridgeConfig { tiers: Record; /** Fallback tier when a tool has no explicit mapping. */ defaultTier: Tier; /** Per-tool tier assignment. Absent keys fall back to defaultTier. */ toolTierMap?: Record; } export declare const DEFAULT_CONFIG: BridgeConfig; export interface ResolveOptions { /** Override the model for a specific tier (e.g. via CLI flag). */ tierOverrides?: Partial>>; } /** * Apply overrides on top of a base config. Leaves unspecified fields alone. */ export declare function withOverrides(base: BridgeConfig, opts?: ResolveOptions): BridgeConfig; export declare function tierForTool(config: BridgeConfig, toolName: string): Tier; export declare function modelForTool(config: BridgeConfig, toolName: string): TierConfig; /** * Display/telemetry identifier for a tier. Prefers the explicit model name * when set; falls back to the URL. Always returns a non-empty string so * footers/_meta never carry undefined. */ export declare function tierModelLabel(tcfg: TierConfig): string; //# sourceMappingURL=tiers.d.ts.map