/** * Verifies that the per-turn `overrideProfile` plumbed into `AgentLoop.run()` * surfaces on every `SendMessageOptions.config` the loop emits, and that * `SubagentManager.spawn()` propagates an `overrideProfile` set on its * `SubagentConfig` into the subagent's `runAgentLoop()` call. * * Together these establish where a pinned profile does and does not travel: it * applies to every LLM call within the pinning conversation's own turn, and it * stops at the spawn boundary. `executeSubagentSpawn` sets `overrideProfile` * on the `SubagentConfig` only for a profile named explicitly on the spawn, so * the propagation above carries a caller's choice rather than a parent's pin. * * With no `overrideProfile` set, the field is omitted from `providerConfig` * rather than carrying `undefined`. */ import { describe, expect, mock, test } from "bun:test"; // These suites exercise override-profile PLUMBING through legacy-shaped // fixtures (llm.default-centric, no defaultProvider). Pinned to the // flag-off cascade; override-or-default resolution semantics are pinned by // llm-resolver-override-or-default.test.ts and the inference-profile loop // suite. import { AgentLoop } from "../agent/loop.js"; import type { Message, Provider, ProviderResponse, SendMessageOptions, ToolDefinition, } from "../providers/types.js"; import { setConfig } from "./helpers/set-config.js"; const userMessage: Message = { role: "user", content: [{ type: "text", text: "hi" }], }; function textResponse(text: string): ProviderResponse { return { content: [{ type: "text", text }], model: "mock-model", usage: { inputTokens: 1, outputTokens: 1 }, stopReason: "end_turn", }; } function toolUseResponse( id: string, name: string, input: Record, ): ProviderResponse { return { content: [{ type: "tool_use", id, name, input }], model: "mock-model", usage: { inputTokens: 1, outputTokens: 1 }, stopReason: "tool_use", }; } /** * Build a provider that records every `SendMessageOptions.config` it sees so * the test can assert how the agent loop populated `overrideProfile` on each * iteration of the multi-turn tool loop. */ function makeRecordingProvider(responses: ProviderResponse[]): { provider: Provider; configs: () => Array | undefined>; } { const configs: Array | undefined> = []; let i = 0; const provider: Provider = { name: "mock", async sendMessage( _messages: Message[], options?: SendMessageOptions, ): Promise { configs.push(options?.config as Record | undefined); const response = responses[i] ?? responses[responses.length - 1]; i++; return response; }, }; return { provider, configs: () => configs }; } describe("AgentLoop.run — overrideProfile plumbing", () => { test("forwards overrideProfile to providerConfig on every LLM call (multi-turn)", async () => { // Two tool-use turns followed by a final text response so the loop // performs three provider sends. Every send must carry the same // overrideProfile that was passed into `run()`. const { provider, configs } = makeRecordingProvider([ toolUseResponse("t1", "echo", { value: "first" }), toolUseResponse("t2", "echo", { value: "second" }), textResponse("done"), ]); const dummyTools: ToolDefinition[] = [ { name: "echo", description: "Echo back the input", input_schema: { type: "object", properties: { value: { type: "string" } }, }, }, ]; const toolExecutor = async ( _name: string, _input: Record, ) => ({ content: "ok", isError: false }); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 1024 }, tools: dummyTools, toolExecutor: toolExecutor, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", overrideProfile: "fast", }); // Three sends — initial + two tool round-trips. expect(configs()).toHaveLength(3); for (const cfg of configs()) { expect(cfg?.overrideProfile).toBe("fast"); } }); test("omits overrideProfile from providerConfig when unset (default behavior unchanged)", async () => { const { provider, configs } = makeRecordingProvider([textResponse("hi")]); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 1024 }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, }); // Single send, no overrideProfile field at all. expect(configs()).toHaveLength(1); expect(configs()[0]).toBeDefined(); expect("overrideProfile" in (configs()[0] ?? {})).toBe(false); }); test("missing overrideProfile name still flows through (silent fall-through is the resolver's job)", async () => { // The agent loop must NOT validate the profile name — that's the // resolver's responsibility. The loop forwards whatever string it // receives so a non-existent profile silently falls back at the // provider layer (covered by provider-send-message-override-profile.test.ts). const { provider, configs } = makeRecordingProvider([textResponse("hi")]); const loop = new AgentLoop({ provider: provider, systemPrompt: "system", conversationId: "test-conversation", config: { maxTokens: 1024 }, }); await loop.run({ requestId: "test-request", messages: [userMessage], onEvent: () => {}, trust: { sourceChannel: "vellum", trustClass: "unknown" }, callSite: "mainAgent", overrideProfile: "does-not-exist", }); expect(configs()[0]?.overrideProfile).toBe("does-not-exist"); }); }); // ── Subagent profile forwarding ────────────────────────────────────────── // Capture the SubagentManager → Conversation handshake so we can verify the // `overrideProfile` from `SubagentConfig` is forwarded into the spawned // subagent's `runAgentLoop()` invocation. Same pattern as // `subagent-call-site-routing.test.ts`. interface CapturedRunAgentLoopOptions { isInteractive?: boolean; isUserMessage?: boolean; titleText?: string; callSite?: string; overrideProfile?: string; forceOverrideProfile?: boolean; } const capturedRunAgentLoopOptions: CapturedRunAgentLoopOptions[] = []; class FakeConversation { constructor() {} updateClient() {} setTrustContext() {} setAuthContext() {} getAuthContext() { return undefined; } setAssistantId() {} hasSystemPromptOverride = false; setSubagentAllowedTools() {} setPreactivatedSkillIds() {} preactivateSkills() {} preactivateSkillsAsync() {} setSpawnHints() {} injectInheritedContext() {} setActiveBranchId() {} setBranchTag() {} setForkPolicy() {} setForkParentMessageCount() {} setForkParentSystemPrompt() {} enqueueMessage() { return { rejected: false, queued: false }; } abort() {} dispose() {} messages = []; usageStats = { inputTokens: 0, outputTokens: 0, estimatedCost: 0 }; sendToClient() {} loadFromDb() { return Promise.resolve(); } persistUserMessage() { return Promise.resolve({ id: "msg-id", deduplicated: false }); } runAgentLoop( _content: string, _userMessageId: string, options?: CapturedRunAgentLoopOptions, ) { capturedRunAgentLoopOptions.push({ ...(options ?? {}) }); return Promise.resolve(); } getCurrentSystemPrompt() { return "system"; } } mock.module("../daemon/conversation.js", () => ({ Conversation: FakeConversation, })); mock.module("../persistence/conversation-bootstrap.js", () => ({ bootstrapConversation: () => ({ id: "conv-id" }), })); mock.module("../prompts/system-prompt.js", () => ({ buildSystemPrompt: () => "system prompt", buildSubagentSystemPrompt: () => "subagent system", })); const anthropicStub = { name: "anthropic" }; mock.module("../providers/registry.js", () => ({ getProvider: () => anthropicStub, listProviders: () => ["anthropic"], initializeProviders: async () => {}, resolveProviderFromConnection: async () => anthropicStub, })); import { VELLUM_MANAGED_CONNECTION_NAME } from "../providers/vellum-model-routing.js"; // Connection-aware resolver path: satisfy // `tryResolveProviderForConnectionName` lookups so resolveDefaultProvider // returns a usable provider for any connection name the winning profile // references. The managed connection must be the platform-auth sentinel row: // a managed profile routing through anything else resolves to the platform // instead. Other names behave as personal anthropic connections. mock.module("../providers/inference/connections.js", () => ({ getConnection: (_db: unknown, name: string) => name === VELLUM_MANAGED_CONNECTION_NAME ? { id: 1, name, provider: "vellum", auth: { type: "platform" }, } : { id: 1, name, provider: "anthropic", auth_strategy: "user_managed_credential", credential_alias: null, metadata_json: null, created_at: new Date().toISOString(), updated_at: new Date().toISOString(), }, })); /** * Seed the workspace `llm` config block for real. `activeProfile` and * `callSites` vary per test. */ function seedLlmConfig(options?: { activeProfile?: string; callSites?: Record; }): void { setConfig("llm", { profiles: { // Complete (provider + model) so the profile is a usable winner at // every rung of the single-winner selection chain. fast: { source: "user", provider: "anthropic", model: "claude-haiku-4-5-20251001", }, }, ...(options?.activeProfile === undefined ? {} : { activeProfile: options.activeProfile }), callSites: options?.callSites ?? {}, }); } seedLlmConfig(); import { SubagentManager } from "../subagent/manager.js"; describe("SubagentManager.spawn: forwards a configured overrideProfile", () => { test("forwards overrideProfile from SubagentConfig into runAgentLoop", async () => { capturedRunAgentLoopOptions.length = 0; const manager = new SubagentManager(); await manager.spawn( { parentConversationId: "parent-1", label: "child", objective: "do the thing", overrideProfile: "fast", }, () => {}, ); // The spawned subagent's runAgentLoop receives the `subagentSpawn` // callSite and whatever `overrideProfile` the config carries, which // `executeSubagentSpawn` sets only for an explicitly named profile. expect(capturedRunAgentLoopOptions).toHaveLength(1); const captured = capturedRunAgentLoopOptions[0]; expect(captured.callSite).toBe("subagentSpawn"); expect(captured.overrideProfile).toBe("fast"); expect("forceOverrideProfile" in captured).toBe(false); }); test("forwards forced overrideProfile from SubagentConfig into runAgentLoop", async () => { capturedRunAgentLoopOptions.length = 0; const manager = new SubagentManager(); await manager.spawn( { parentConversationId: "parent-forced", label: "child", objective: "do the thing", overrideProfile: "fast", forceOverrideProfile: true, }, () => {}, ); expect(capturedRunAgentLoopOptions).toHaveLength(1); const captured = capturedRunAgentLoopOptions[0]; expect(captured.callSite).toBe("subagentSpawn"); expect(captured.overrideProfile).toBe("fast"); expect(captured.forceOverrideProfile).toBe(true); }); test("omits overrideProfile when SubagentConfig does not set it", async () => { capturedRunAgentLoopOptions.length = 0; const manager = new SubagentManager(); await manager.spawn( { parentConversationId: "parent-2", label: "child", objective: "do the thing", }, () => {}, ); expect(capturedRunAgentLoopOptions).toHaveLength(1); const captured = capturedRunAgentLoopOptions[0]; expect(captured.callSite).toBe("subagentSpawn"); // Field must be absent rather than carrying `undefined`, mirroring the // agent loop's "field omitted when unset" contract. expect("overrideProfile" in captured).toBe(false); expect("forceOverrideProfile" in captured).toBe(false); }); }); // ── Nested subagent spawn: no profile crosses the boundary ─────────────── // When a subagent's agent loop is running under a turn profile, the executor // closure plumbs that value into `ToolContext.overrideProfile`. That value // describes the spawning turn, so `executeSubagentSpawn` leaves it behind: a // child resolves the `subagentSpawn` call site on its own, and a forwarded // override would both re-price the child and file its spend as a pin nobody // set. These tests cover the nesting levels where a profile could otherwise // compound down a chain of children. mock.module("../persistence/conversation-crud.js", () => ({ setConversationProcessingStartedAt: () => {}, getConversationOverrideProfile: () => undefined, reserveMessage: mock(async () => ({ id: "msg-reserve" })), })); import { getSubagentManager } from "../subagent/index.js"; import { executeSubagentSpawn } from "../tools/subagent/spawn.js"; describe("executeSubagentSpawn: the spawn boundary carries no profile", () => { test("a subagent spawning a subagent does not pass its own turn profile down", async () => { const manager = getSubagentManager(); const originalSpawn = manager.spawn.bind(manager); let capturedConfig: Record | undefined; manager.spawn = async (config: Record) => { capturedConfig = config; return "nested-subagent-id"; }; try { // The second-level spawn: this tool call happens inside the first // subagent's tool context, where `overrideProfile` was populated by // `runAgentLoopImpl` from its own turn snapshot. That snapshot describes // the parent's turn, so it stops here rather than compounding down a // chain of nested children. const result = await executeSubagentSpawn( { label: "nested", objective: "do nested work" }, { workingDir: "/tmp", conversationId: "subagent-conv-id", trustClass: "guardian", sendToClient: () => {}, overrideProfile: "fast", } as import("../tools/types.js").ToolContext, ); expect(result.isError).toBe(false); expect(capturedConfig).toBeDefined(); expect("overrideProfile" in capturedConfig!).toBe(false); } finally { manager.spawn = originalSpawn; } }); test("no override anywhere leaves the child on the subagentSpawn default", async () => { const manager = getSubagentManager(); const originalSpawn = manager.spawn.bind(manager); let capturedConfig: Record | undefined; manager.spawn = async (config: Record) => { capturedConfig = config; return "nested-subagent-id-2"; }; try { await executeSubagentSpawn( { label: "nested", objective: "do nested work" }, { workingDir: "/tmp", conversationId: "subagent-conv-id-2", trustClass: "guardian", sendToClient: () => {}, // no overrideProfile } as import("../tools/types.js").ToolContext, ); expect(capturedConfig).toBeDefined(); expect("overrideProfile" in capturedConfig!).toBe(false); } finally { manager.spawn = originalSpawn; } }); test("a workspace active profile does not reach the child, pinned or not", async () => { // The child resolves `llm.callSites.subagentSpawn` on its own, so the // spawn must forward nothing either way: a forwarded override would // outrank an explicit call-site pin under single-winner resolution and // overturn the operator's choice. const manager = getSubagentManager(); const originalSpawn = manager.spawn.bind(manager); let capturedConfig: Record | undefined; manager.spawn = async (config: never) => { capturedConfig = config; return "nested-subagent-id-3"; }; seedLlmConfig({ activeProfile: "fast" }); try { const baseContext = { workingDir: "/tmp", conversationId: "subagent-conv-id-3", trustClass: "guardian", sendToClient: () => {}, } as import("../tools/types.js").ToolContext; await executeSubagentSpawn( { label: "nested", objective: "do nested work" }, baseContext, ); expect("overrideProfile" in capturedConfig!).toBe(false); seedLlmConfig({ activeProfile: "fast", callSites: { subagentSpawn: { profile: "fast" } }, }); capturedConfig = undefined; await executeSubagentSpawn( { label: "nested", objective: "do nested work" }, baseContext, ); expect("overrideProfile" in capturedConfig!).toBe(false); } finally { manager.spawn = originalSpawn; seedLlmConfig(); } }); });