/** * Verifies that `config.overrideProfile` on `SendMessageOptions` plumbs an * ad-hoc profile through both the `RetryProvider` normalization step (which * resolves model/maxTokens/effort/etc.) and the `CallSiteRoutingProvider` * provider-selection step. * * The end-to-end contract: a caller setting * `config.overrideProfile = "fast"` on a single send must see the request * land on the profile's provider with the profile's model — without * modifying the workspace's `activeProfile` or any call-site entry. This * makes per-conversation pinned profiles (PR 6+) work. */ import { beforeEach, describe, expect, test } from "bun:test"; // These suites exercise override-profile PLUMBING through legacy-shaped // fixtures (llm.default-centric, no defaultProvider). Pinned to the // flag-off cascade; override-or-default resolution semantics are pinned by // llm-resolver-override-or-default.test.ts and the inference-profile loop // suite. import { CallSiteRoutingProvider } from "../providers/call-site-routing.js"; import { CallSiteConfiguredProvider } from "../providers/provider-send-message.js"; import { RetryProvider } from "../providers/retry.js"; import type { Message, Provider, ProviderResponse, SendMessageOptions, } from "../providers/types.js"; import { ProviderError } from "../util/errors.js"; import { setConfig } from "./helpers/set-config.js"; const DUMMY_MESSAGES: Message[] = [ { role: "user", content: [{ type: "text", text: "hi" }] }, ]; function makeResponse(model: string): ProviderResponse { return { content: [{ type: "text", text: "ok" }], model, usage: { inputTokens: 1, outputTokens: 1 }, stopReason: "end_turn", }; } // Seed `llm` into the real workspace config; the loader schema-merges the // raw partial over defaults exactly as `LLMSchema.parse` did for the mock. function setLlmConfig(raw: unknown): void { setConfig("llm", raw); } beforeEach(() => { setLlmConfig({}); }); describe("SendMessageOptions.config.overrideProfile", () => { test("CallSiteConfiguredProvider injects the resolving call site when callers omit it", async () => { let captured: SendMessageOptions | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options; return makeResponse("anthropic"); }, }; const provider = new CallSiteConfiguredProvider(inner, "mainAgent"); await provider.sendMessage(DUMMY_MESSAGES, { config: { model: "claude-opus-4-7" }, }); expect(captured?.config).toMatchObject({ callSite: "mainAgent", model: "claude-opus-4-7", }); }); test("CallSiteConfiguredProvider preserves explicit per-call call sites", async () => { let captured: SendMessageOptions | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options; return makeResponse("anthropic"); }, }; const provider = new CallSiteConfiguredProvider(inner, "mainAgent"); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "conversationTitle" }, }); expect(captured?.config?.callSite).toBe("conversationTitle"); }); test("CallSiteConfiguredProvider forwards its stored overrideProfile when the caller sets callSite", async () => { let captured: SendMessageOptions | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options; return makeResponse("anthropic"); }, }; const provider = new CallSiteConfiguredProvider( inner, "inference", "byok-sonnet", ); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "inference" }, }); // Naming a call site must not opt the caller out of profile propagation: // the two fields are independent, and downstream resolution reads the // override off the config it is handed. expect(captured?.config?.overrideProfile).toBe("byok-sonnet"); }); test("a profile-pinned send resolves the profile's model, not the call-site default", async () => { setLlmConfig({ profiles: { "balanced-together": { provider: "together", model: "moonshotai/Kimi-K2-Instruct", }, }, }); let captured: Record | undefined; const inner: Provider = { name: "together", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("together"); }, }; const provider = new CallSiteConfiguredProvider( new RetryProvider(inner), "inference", "balanced-together", ); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "inference" }, }); // Losing the override strands a Together connection on the `inference` // call-site default (`cost-optimized`), whose model is a Fireworks id — // the provider half stays correct, so the upstream 400 names a model the // caller never chose. expect(captured?.model).toBe("moonshotai/Kimi-K2-Instruct"); }); test("RetryProvider resolves model from named profile when overrideProfile is set", async () => { setLlmConfig({ profiles: { fast: { provider: "anthropic", model: "claude-haiku-4-5-20251001" }, }, }); let captured: Record | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("anthropic"); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent", overrideProfile: "fast" }, }); // The override profile wins resolution, so its model is what lands on // the wire config. expect(captured?.model).toBe("claude-haiku-4-5-20251001"); // `overrideProfile` is a routing key — it must not leak to the provider. expect(captured?.overrideProfile).toBeUndefined(); // `callSite` is also stripped post-resolve. expect(captured?.callSite).toBeUndefined(); }); test("CallSiteRoutingProvider switches transport when overrideProfile changes the provider (via provider_connection)", async () => { setLlmConfig({ profiles: { fast: { provider: "openai", provider_connection: "openai-conn", model: "gpt-5.4", }, }, }); const calls = { default: 0, alt: 0 }; const defaultProvider: Provider = { name: "anthropic", async sendMessage() { calls.default++; return makeResponse("anthropic"); }, }; const altProvider: Provider = { name: "openai", async sendMessage() { calls.alt++; return makeResponse("openai"); }, }; const wrapped = new CallSiteRoutingProvider( defaultProvider, async (connectionName) => connectionName === "openai-conn" ? altProvider : null, ); const response = await wrapped.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent", overrideProfile: "fast" }, }); expect(calls.default).toBe(0); expect(calls.alt).toBe(1); expect(response.model).toBe("openai"); }); test("CallSiteRoutingProvider attributes provider errors to the actual resolved connection", async () => { setLlmConfig({ profiles: { fast: { provider: "openai", provider_connection: "openai-conn", model: "gpt-5.4", }, }, }); const providerError = new ProviderError( "OpenAI API error (429): Too many requests", "openai", 429, { reason: "rate_limited" }, ); const defaultProvider = makeThrowingProvider( "anthropic", new Error("default provider should not be called"), ); const altProvider = makeThrowingProvider("openai", providerError); altProvider.routeAttribution = { connectionName: "openai-recovered", isManagedRoute: false, }; const wrapped = new CallSiteRoutingProvider( defaultProvider, async (connectionName) => connectionName === "openai-conn" ? altProvider : null, { connectionName: "vellum", isManagedRoute: true }, ); await expect( wrapped.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent", overrideProfile: "fast" }, }), ).rejects.toBe(providerError); expect(providerError.routeAttribution).toEqual({ connectionName: "openai-recovered", profileName: "fast", isManagedRoute: false, }); }); test("CallSiteRoutingProvider attributes soft fallback errors to the actual default connection", async () => { setLlmConfig({ profiles: { fast: { provider: "openai", provider_connection: "openai-conn", model: "gpt-5.4", }, }, }); const providerError = new ProviderError( "Anthropic API error (429): Too many requests", "anthropic", 429, { reason: "rate_limited" }, ); const defaultProvider = makeThrowingProvider("anthropic", providerError); defaultProvider.routeAttribution = { connectionName: "anthropic-default", isManagedRoute: false, }; const wrapped = new CallSiteRoutingProvider( defaultProvider, async () => null, defaultProvider.routeAttribution, ); await expect( wrapped.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent", overrideProfile: "fast" }, }), ).rejects.toBe(providerError); expect(providerError.routeAttribution).toEqual({ connectionName: "anthropic-default", profileName: "fast", isManagedRoute: false, }); }); test("missing overrideProfile name silently falls through to base resolution", async () => { setLlmConfig({ // The call-site tweak applies last in resolution, so it pins the model // base resolution lands on when the override name doesn't resolve. callSites: { mainAgent: { provider: "anthropic", model: "claude-opus-4-7" }, }, profiles: { fast: { provider: "anthropic", model: "claude-haiku-4-5-20251001" }, }, }); let captured: Record | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("anthropic"); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent", overrideProfile: "does-not-exist" }, }); // Falls through to base resolution (call-site tweak applied over the // default winner) since the named profile isn't found. expect(captured?.model).toBe("claude-opus-4-7"); }); test("absent overrideProfile leaves prior resolution behavior intact", async () => { setLlmConfig({ callSites: { mainAgent: { provider: "anthropic", model: "claude-opus-4-7" }, }, profiles: { fast: { provider: "anthropic", model: "claude-haiku-4-5-20251001" }, }, }); let captured: Record | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("anthropic"); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "mainAgent" }, }); expect(captured?.model).toBe("claude-opus-4-7"); }); test("overrideProfile is stripped even when callSite is absent", async () => { let captured: Record | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("anthropic"); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(DUMMY_MESSAGES, { config: { overrideProfile: "fast" }, }); // `overrideProfile` must never leak as a wire-format field, even when no // callSite is set (the resolver never runs, but the leak guard still // applies). expect(captured?.overrideProfile).toBeUndefined(); }); }); function makeThrowingProvider(name: string, error: Error): Provider { return { name, async sendMessage(): Promise { throw error; }, }; } describe("SendMessageOptions.config.forceOverrideProfile", () => { test("CallSiteConfiguredProvider forwards forceOverrideProfile into the send config", async () => { let captured: SendMessageOptions | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options; return makeResponse("anthropic"); }, }; const provider = new CallSiteConfiguredProvider( inner, "inference", "strong", true, ); await provider.sendMessage(DUMMY_MESSAGES, {}); expect(captured?.config).toMatchObject({ callSite: "inference", overrideProfile: "strong", forceOverrideProfile: true, }); }); test("the override profile outranks a call-site profile pin, forced or not", async () => { // The advisor scenario in miniature: the `inference` call site is pinned // to a cheap profile, but a caller supplies a stronger profile for its // own send. Under single-winner resolution the override sits at the top // of the selection chain for every call site, so it wins with or without // `forceOverrideProfile` (the flag is a no-op kept for API compat). setLlmConfig({ profiles: { cheap: { provider: "anthropic", model: "claude-haiku-4-5-20251001" }, strong: { provider: "anthropic", model: "claude-opus-4-8" }, }, callSites: { inference: { profile: "cheap" } }, }); const send = async (force: boolean) => { let captured: Record | undefined; const inner: Provider = { name: "anthropic", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return makeResponse("anthropic"); }, }; const provider = new RetryProvider(inner); await provider.sendMessage(DUMMY_MESSAGES, { config: { callSite: "inference", overrideProfile: "strong", ...(force ? { forceOverrideProfile: true } : {}), }, }); return captured; }; // Without the flag, the override (`strong`) already wins over the // call-site pin (`cheap`). expect((await send(false))?.model).toBe("claude-opus-4-8"); // With the flag, the result is identical — forcing changes nothing. const forced = await send(true); expect(forced?.model).toBe("claude-opus-4-8"); // The routing keys are stripped before the provider wire request. expect(forced?.overrideProfile).toBeUndefined(); expect(forced?.forceOverrideProfile).toBeUndefined(); }); });