/** * Verifies the field-precedence contract for `AgentLoop.run(..., callSite)`. * * When `callSite` is set, the loop must NOT pre-set the * `max_tokens`/`thinking`/`effort`/`speed` fields from `this.config` * (sourced from `llm.default`) because the downstream * `RetryProvider.normalizeSendMessageOptions` only fills these fields when * they're undefined. If the loop pre-sets them, every per-call-site override * for these knobs is silently ignored. * * Precedence (highest wins): * 1. Per-run explicit (from `run()`'s `model` param) * 2. Call-site resolved values (from `resolveCallSiteConfig` via the * normalizer) * 3. Conversation defaults (`this.config.*`, from `llm.default`) * * The tests pipe the loop's per-call options through `RetryProvider` so we * can observe the final, post-resolution config that downstream provider * clients consume. */ import { beforeEach, describe, expect, test } from "bun:test"; import { AgentLoop } from "../agent/loop.js"; import { RetryProvider } from "../providers/retry.js"; import type { Message, Provider, ProviderResponse, SendMessageOptions, } from "../providers/types.js"; import { setConfig } from "./helpers/set-config.js"; const userMessage: Message = { role: "user", content: [{ type: "text", text: "hi" }], }; /** Seed the workspace `llm` config block for real; the loader schema-merges it. */ function setLlmConfig(raw: unknown): void { setConfig("llm", raw); } beforeEach(() => { setLlmConfig({}); }); /** * Build a provider that captures the final `config` it receives. Wrap it in * `RetryProvider` so the call-site resolver runs over whatever the agent loop * emits — exactly mirroring production wiring. */ function makePipeline(providerName: string): { provider: Provider; lastConfig: () => Record | undefined; } { let captured: Record | undefined; const inner: Provider = { name: providerName, async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { captured = options?.config as Record | undefined; return { content: [{ type: "text", text: "ok" }], model: "mock-model", usage: { inputTokens: 1, outputTokens: 1 }, stopReason: "end_turn", }; }, }; return { provider: new RetryProvider(inner), lastConfig: () => captured, }; } describe("AgentLoop — call-site precedence", () => { test("call-site maxTokens wins over conversation default when callSite is set", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", maxTokens: 64000, }, callSites: { mainAgent: { maxTokens: 4096 } }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000 }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", }); expect(lastConfig()!.max_tokens).toBe(4096); }); test("call-site effort wins over conversation default when callSite is set", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", effort: "high", }, callSites: { mainAgent: { effort: "low" } }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000, effort: "high", }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", }); expect(lastConfig()!.effort).toBe("low"); }); test("call-site speed wins over conversation default when callSite is set", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", speed: "standard", }, callSites: { mainAgent: { speed: "fast" } }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000, effort: "high", // Conversation default is "fast" (which would normally be applied) — // ensure the call-site value is the one that ends up on the wire. speed: "fast", }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", }); expect(lastConfig()!.speed).toBe("fast"); }); test("call-site thinking wins over conversation default when callSite is set", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", // Default thinking enabled. thinking: { enabled: true, streamThinking: true }, }, callSites: { // Call site disables thinking — must be honoured even though the // conversation default has it on. mainAgent: { thinking: { enabled: false } }, }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000, // Conversation default also has thinking on — without the fix, this // would pre-set `thinking: { type: "adaptive" }` and mask the // call-site override. thinking: { enabled: true }, }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", }); // Call-site override resolves `thinking.enabled: false`, so the // RetryProvider normalizer must send Anthropic's explicit disabled shape. expect(lastConfig()!.thinking).toEqual({ type: "disabled" }); }); test("call-site thinking is converted to Anthropic wire-format when enabled", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", thinking: { enabled: true, streamThinking: true }, }, callSites: { mainAgent: {} }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000 }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", }); // Must be wire-format `{ type: "adaptive" }` so the Anthropic SDK's // `ThinkingConfigParam` accepts it. The schema-shape `{ enabled, // streamThinking }` would be a runtime API error. expect(lastConfig()!.thinking).toEqual({ type: "adaptive" }); }); test("conversation defaults still apply when callSite is absent", async () => { setLlmConfig({ default: { provider: "anthropic", model: "claude-default", // Resolver values that would *normally* be filled — but we don't // pass a callSite, so they must not surface. maxTokens: 999, effort: "low", }, }); const { provider, lastConfig } = makePipeline("anthropic"); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 64000, effort: "high", speed: "fast", thinking: { enabled: true }, }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, }); const config = lastConfig()!; expect(config.max_tokens).toBe(64000); expect(config.effort).toBe("high"); expect(config.speed).toBe("fast"); // No callSite → loop sets the wire-format thinking directly. expect(config.thinking).toEqual({ type: "adaptive" }); }); });