/** * T38 — matched requested-baseline outcome fixtures for the opt-in Sai subagent router. * * The router has no `enabled` flag. The public launch gate is `readSubagentRoutingSettings()` in * `runs/foreground/subagent-executor.ts`: it returns `undefined` while * `saiAdvisory.routes.subagent.enabled` is not `true`, and the executor then never calls * `routeSubagentLaunch` at all. So the matched baseline for every case below is the *requested * input* — asserting what survives — and never a synthetic "route off" router call, which the * function cannot express. Every expectation is written out independently of the router's own * filters so a changed filter fails a fixture instead of silently redefining the contract. * * These fixtures exercise the injectable fake `ask` seam (T30 C7). They establish selection, gating, * budget, and fallback plumbing only. They are not evidence about model quality, downstream task * success, latency, or cost savings. */ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { describe, it } from "node:test"; import type { AgentConfig } from "../../src/agents/agents.ts"; import { createChildSafeState } from "../../src/extension/fanout-child.ts"; import { createSubagentExecutor } from "../../src/runs/foreground/subagent-executor.ts"; import type { ResolvedSubagentCapabilityCeiling } from "../../src/runs/shared/capability-ceiling.ts"; import { emitSubagentRoutingTelemetry, readSubagentRoutingSettings, routeSubagentLaunch, type SubagentRoutingInput, type SubagentRoutingResult, type SubagentRoutingSettings, } from "../../src/runs/shared/sai-subagent-routing.ts"; import { makeMinimalCtx } from "../support/helpers.ts"; /** Independently restated, so dropping one from the runtime set fails these fixtures. */ const PROTECTED = ["read", "contact_supervisor", "intercom", "structured_output", "subagent", "subagent_supervisor"]; function agent(name: string, extra: Partial = {}): AgentConfig { return { name, description: `${name} agent`, systemPromptMode: "append", inheritProjectContext: true, inheritGlobalContext: true, inheritSkills: true, systemPrompt: "", source: "builtin", filePath: `${name}.md`, ...extra } as AgentConfig; } const agents: AgentConfig[] = [ agent("delegate"), agent("reviewer", { tools: ["read", "grep"] }), agent("worker", { tools: ["read", "bash", "edit", "write"], model: "anthropic/opus", modelProvider: "anthropic", modelSource: "frontmatter" }), agent("auditor", { tools: ["read", "contact_supervisor", "intercom", "subagent_supervisor"] }), agent("surgical", { tools: [...PROTECTED, "grep", "bash", "src/tool.ts", "notes.js"], excludeTools: ["bash"] }), agent("wide", { tools: ["read", ...Array.from({ length: 15 }, (_value, index) => `t${String(index + 1).padStart(2, "0")}`)] }), agent("secret", { disabled: true }), agent("external", { runner: { type: "external-cli" } as AgentConfig["runner"] }), agent("runtime", { source: "runtime" }), agent("outsider", { tools: ["read", "grep"] }), ]; function ceilingFor(allowedAgents: string[]): ResolvedSubagentCapabilityCeiling { return { version: 1, allowedAgents, denyExtensions: false, sources: ["t38"] }; } /** Excludes `runtime` (runtime source) and `outsider` (not listed) from any offer. */ const ceiling = ceilingFor(["delegate", "reviewer", "worker", "auditor", "surgical", "wide", "secret", "external"]); /** Independent expectation: native, enabled, in-ceiling agent names (`secret` disabled, `external` runner, `runtime` source, `outsider` outside the ceiling). */ const ROUTABLE = ["auditor", "delegate", "reviewer", "surgical", "wide", "worker"]; const baseSettings = (overrides: Partial = {}): SubagentRoutingSettings => ({ connection: { provider: "tokenin", model: "jev-1.14", timeoutMs: 1_000, minConfidence: 0.6 }, maxBytes: 16_384, contextChars: 4_000, genericAgents: ["delegate"], tiers: { simple: "p/small", reasoning: "p/big" }, ...overrides, }); type AskRecord = { allowed: Record; maxBytes: number; payloadBytes: number; connection: SubagentRoutingSettings["connection"]; payload: unknown; }; type Reply = { choices?: Record; rejected?: Record; failure?: string; elapsedMs?: number; }; /** * A fake Sai. It mirrors the facade's validation: with `failure`, every requested question is * rejected with that reason; otherwise only offered questions can carry a choice. `answerUnoffered` * deliberately violates that validation, to check the router's own boundary. */ function fakeAsk(replyFor: Reply | ((call: number, record: AskRecord) => Reply), asked: AskRecord[], options: { answerUnoffered?: boolean } = {}) { return async (_ctx: unknown, connection: AskRecord["connection"], request: { payload: unknown; maxBytes: number; allowed: Record }) => { const record: AskRecord = { allowed: request.allowed, maxBytes: request.maxBytes, payloadBytes: Buffer.byteLength(JSON.stringify(request.payload), "utf8"), connection, payload: request.payload, }; asked.push(record); const reply = typeof replyFor === "function" ? replyFor(asked.length - 1, record) : replyFor; const rejected = reply.failure ? { ...Object.fromEntries(Object.keys(request.allowed).map((question) => [question, reply.failure])), ...reply.rejected } : { ...reply.rejected }; const choices = Object.fromEntries( Object.entries(reply.choices ?? {}).filter(([question]) => options.answerUnoffered === true || Object.hasOwn(request.allowed, question)), ); return { choices, rejected, ...(reply.failure ? { failure: reply.failure } : {}), elapsedMs: reply.elapsedMs ?? 5 }; }; } const ctx = {} as never; /** Route once and enforce the router's existing two-ask ceiling on every fixture. */ async function route( input: Omit & { agents?: AgentConfig[] }, reply: Reply | ((call: number, record: AskRecord) => Reply), options: { answerUnoffered?: boolean; settings?: Partial } = {}, ): Promise<{ result: SubagentRoutingResult; asked: AskRecord[]; input: SubagentRoutingInput }> { const asked: AskRecord[] = []; const settings = baseSettings(options.settings); const full: SubagentRoutingInput = { agents, ceiling, ...input }; const result = await routeSubagentLaunch(full, ctx, settings, fakeAsk(reply, asked, options) as never); assert.ok(asked.length <= 2, `routing must stay within two asks, saw ${asked.length}`); for (const ask of asked) assert.ok(ask.payloadBytes <= ask.maxBytes, "payload stays inside the route's byte budget"); return { result, asked, input: full }; } /** The requested launch, unchanged: same identity, no routed agent, no tier, no dropped tool. */ function assertAsRequested(input: SubagentRoutingInput, result: SubagentRoutingResult): void { assert.equal(result.agent, input.agent, "agent stays as requested"); assert.equal(result.agents, input.agents, "agent list is the requested list, by identity"); assert.equal(result.routedAgent, undefined, "nothing was routed"); assert.equal(result.tier, undefined, "no tier was applied"); assert.deepEqual(result.droppedTools, [], "no tool was dropped"); } function questionsOf(record: AskRecord): Record; instructions: { question: string; focus?: string } }> { return (record.payload as { questions: Record; instructions: { question: string; focus?: string } }> }).questions; } const sorted = (values: readonly string[]) => [...values].sort(); describe("Sai subagent routing outcomes", () => { it("asks nothing when the request leaves nothing to decide", async () => { const { result, asked, input } = await route({ task: "t", agent: "auditor", model: "x/y" }, {}); assert.equal(asked.length, 0); assert.equal(result.questions, 0); assert.equal(result.elapsedMs, 0); assertAsRequested(input, result); assert.equal(result.agent, "auditor"); }); it("keeps an explicit model authoritative: no tier question, no agent model rewrite", async () => { const { result, asked } = await route( { task: "t", agent: "worker", model: "explicit/model" }, { choices: { tool_0: { choice: "unneeded", confidence: 0.9 } } }, ); assert.equal(asked.length, 1); assert.deepEqual(Object.keys(asked[0]!.allowed), ["tool_0", "tool_1", "tool_2"]); assert.equal(Object.hasOwn(asked[0]!.allowed, "tier"), false); assert.equal(asked[0]!.maxBytes, baseSettings().maxBytes); assert.deepEqual(asked[0]!.connection, baseSettings().connection); // The router never rewrites the launched agent's model; only a tier answer may. assert.equal(result.agents.find((entry) => entry.name === "worker")!.model, "anthropic/opus"); assert.equal(result.tier, undefined); assert.deepEqual(result.droppedTools, ["bash"]); }); it("never re-offers an explicit specialist, even one the ceiling excludes", async () => { const restricted = ceilingFor(["delegate", "reviewer"]); const asked: AskRecord[] = []; const input: SubagentRoutingInput = { task: "t", agent: "outsider", agents, ceiling: restricted }; const result = await routeSubagentLaunch(input, ctx, baseSettings(), fakeAsk({}, asked) as never); // `outsider` declares `grep`, so tool narrowing still asks - but never about the agent. assert.equal(asked.length, 1); assert.equal(Object.hasOwn(asked[0]!.allowed, "agent"), false); assert.equal(result.agent, "outsider"); assert.equal(result.agents, agents); // Preflight, not the router, is what rejects an out-of-ceiling agent. }); it("offers generic and omitted agents only native, enabled, in-ceiling candidates", async () => { const agentless = await route({ task: "t" }, { choices: { agent: { choice: "reviewer", confidence: 0.9 } } }); // One ask for the agent, then the routed agent's own bounded question set (`grep`, tier). assert.equal(agentless.asked.length, 2); assert.deepEqual(Object.keys(agentless.asked[0]!.allowed), ["agent"]); assert.deepEqual(sorted(agentless.asked[0]!.allowed.agent!), ROUTABLE); assert.deepEqual(sorted(Object.keys(agentless.asked[1]!.allowed)), ["tier", "tool_0"]); assert.equal(agentless.result.agent, "reviewer"); assert.equal(agentless.result.routedAgent, "reviewer"); // Choosing an agent does not copy or narrow the discovered list by itself. assert.equal(agentless.result.agents, agents); const generic = await route({ task: "t", agent: "delegate" }, { choices: { agent: { choice: "keep", confidence: 0.9 } } }); // The requested generic agent is not re-offered; `keep` is. assert.deepEqual(sorted(generic.asked[0]!.allowed.agent!), sorted(["keep", ...ROUTABLE.filter((name) => name !== "delegate")])); // `keep` leaves the generic agent, whose only open question is the tier. assert.deepEqual(Object.keys(generic.asked[1]!.allowed), ["tier"]); assert.equal(generic.result.agent, "delegate"); assert.equal(generic.result.routedAgent, undefined); assertAsRequested(generic.input, generic.result); const routed = await route({ task: "t", agent: "delegate" }, { choices: { agent: { choice: "worker", confidence: 0.9 } } }); assert.equal(routed.result.agent, "worker"); assert.equal(routed.result.routedAgent, "worker"); assert.equal(routed.result.agents, agents); assert.deepEqual(sorted(Object.keys(routed.asked[1]!.allowed)), ["tier", "tool_0", "tool_1", "tool_2"]); }); it("never offers, drops, or adds a protected tool", async () => { const { result, asked } = await route( { task: "t", agent: "surgical", model: "x/y" }, { choices: { tool_0: { choice: "unneeded", confidence: 0.9 } } }, ); // `bash` is excluded by the agent itself and the path-like entries never qualify, so `grep` is the only question. assert.equal(asked.length, 1); assert.deepEqual(Object.keys(asked[0]!.allowed), ["tool_0"]); const questions = questionsOf(asked[0]!); assert.deepEqual(Object.keys(questions), ["tool_0"]); assert.match(questions.tool_0!.instructions.question, /grep/); for (const protectedTool of PROTECTED) assert.doesNotMatch(JSON.stringify(asked[0]!.payload), new RegExp(`"${protectedTool}"`)); assert.deepEqual(result.droppedTools, ["grep"]); const surgical = result.agents.find((entry) => entry.name === "surgical")!; assert.deepEqual(surgical.excludeTools, ["bash", "grep"]); assert.deepEqual(surgical.tools, agents.find((entry) => entry.name === "surgical")!.tools); // Nothing was added and no protected tool can appear in the narrowed set. for (const protectedTool of PROTECTED) assert.ok(!(surgical.excludeTools ?? []).includes(protectedTool)); }); it("caps tool questions at ten and keeps every tool past the cap", async () => { const { result, asked } = await route( { task: "t", agent: "wide", model: "x/y" }, { choices: Object.fromEntries(Array.from({ length: 10 }, (_value, index) => [`tool_${index}`, { choice: "unneeded", confidence: 0.9 }])) }, ); const offered = Object.keys(asked[0]!.allowed); assert.equal(offered.length, 10); assert.deepEqual(offered, Array.from({ length: 10 }, (_value, index) => `tool_${index}`)); assert.deepEqual(result.droppedTools, Array.from({ length: 10 }, (_value, index) => `t${String(index + 1).padStart(2, "0")}`)); assert.ok(!result.droppedTools.includes("t11")); assert.equal(result.agents.find((entry) => entry.name === "wide")!.tools!.length, agents.find((entry) => entry.name === "wide")!.tools!.length); }); it("leaves the launch as requested on missing, low-confidence, unknown-choice, and timeout answers", async () => { const shapes: Array<{ reply: Reply; abstained: string }> = [ { reply: { failure: "timeout" }, abstained: "timeout" }, { reply: { failure: "no-credential" }, abstained: "no-credential" }, { reply: { failure: "malformed" }, abstained: "malformed" }, { reply: { rejected: { agent: "low-confidence" } }, abstained: "low-confidence" }, { reply: { rejected: { agent: "unknown-choice" } }, abstained: "unknown-choice" }, { reply: { rejected: { agent: "missing" } }, abstained: "missing" }, ]; for (const shape of shapes) { const { result, asked, input } = await route({ task: "t" }, shape.reply); assert.equal(asked.length, 1, shape.abstained); assertAsRequested(input, result); assert.equal(result.agent, undefined); assert.equal(result.abstained, shape.abstained); assert.equal(result.questions, 1, "the attempt is counted"); } }); it("applies partial answers only to their own question", async () => { const { result, asked } = await route({ task: "t", agent: "worker" }, { choices: { tier: { choice: "simple" }, tool_0: { choice: "unneeded" }, tool_1: { choice: "unneeded", confidence: 0.9 } }, rejected: { tool_2: "low-confidence" }, }); assert.equal(asked.length, 1); assert.deepEqual(sorted(Object.keys(asked[0]!.allowed)), ["tier", "tool_0", "tool_1", "tool_2"]); assert.deepEqual(asked[0]!.allowed.tier, ["simple", "reasoning"]); // A tier answer needs no confidence; dropping a tool does. assert.equal(result.tier, "simple"); assert.deepEqual(result.droppedTools, ["edit"]); const worker = result.agents.find((entry) => entry.name === "worker")!; assert.equal(worker.model, "p/small"); assert.equal(worker.modelProvider, undefined); assert.deepEqual(worker.excludeTools, ["edit"]); assert.equal(result.abstained, undefined); assert.equal(result.questions, 4); const secondAskFails = await route({ task: "t", agent: "delegate" }, (call) => call === 0 ? { choices: { agent: { choice: "worker", confidence: 0.9 } } } : { failure: "transport" }); assert.equal(secondAskFails.asked.length, 2); assert.equal(secondAskFails.result.agent, "worker"); assert.equal(secondAskFails.result.agents, agents); assert.equal(secondAskFails.result.tier, undefined); assert.deepEqual(secondAskFails.result.droppedTools, []); assert.equal(secondAskFails.result.abstained, "transport"); assert.equal(secondAskFails.result.questions, 1 + 4); }); it("ignores an agent name that was not offered", async () => { const generic = await route({ task: "t", agent: "delegate" }, { choices: { agent: { choice: "outsider", confidence: 0.9 } } }, { answerUnoffered: true }); assert.equal(generic.result.agent, "delegate"); assert.equal(generic.result.routedAgent, undefined); assert.equal(generic.result.agents, agents); // The unoffered agent's tools are never asked about; the kept generic agent has none. assert.deepEqual(Object.keys(generic.asked[1]!.allowed), ["tier"]); const agentless = await route({ task: "t" }, { choices: { agent: { choice: "external", confidence: 0.9 } } }, { answerUnoffered: true }); assert.equal(agentless.result.agent, undefined); assertAsRequested(agentless.input, agentless.result); assert.equal(agentless.asked.length, 1); }); it("emits content-free, selection-only telemetry and nothing when nothing was asked", async () => { const { result } = await route({ task: "TASK-CANARY t", agent: "worker" }, { choices: { tier: { choice: "simple" } } }); const seen: Array<{ channel: string; data: Record }> = []; emitSubagentRoutingTelemetry({ emit: (channel, data) => seen.push({ channel, data: data as Record }) }, result); assert.equal(seen.length, 1); assert.equal(seen[0]!.channel, "sai-routing"); // Selected tier is a selection, never a success/latency/cost/savings claim. assert.deepEqual(Object.keys(seen[0]!.data).sort(), ["candidates", "dropped", "elapsedMs", "event", "outcome", "route", "tier"]); assert.equal(seen[0]!.data.outcome, "sai"); assert.equal(seen[0]!.data.tier, "simple"); assert.equal(seen[0]!.data.dropped, 0); assert.doesNotMatch(JSON.stringify(seen), /TASK-CANARY|worker agent|p\/small/); const silent: Array<{ channel: string }> = []; const zeroAsk = await route({ task: "t", agent: "auditor", model: "x/y" }, {}); emitSubagentRoutingTelemetry({ emit: (channel) => silent.push({ channel }) }, zeroAsk.result); assert.deepEqual(silent, []); }); it("keeps the public launch gate in settings, outside the router", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "sai-routing-outcomes-")); const file = path.join(dir, "settings.json"); const write = (value: unknown) => fs.writeFileSync(file, JSON.stringify(value)); try { write({ saiAdvisory: { routes: {} } }); assert.equal(readSubagentRoutingSettings(file), undefined); write({ saiAdvisory: { routes: { subagent: { enabled: false } } } }); assert.equal(readSubagentRoutingSettings(file), undefined); // Only an explicit boolean enables the route; a truthy string does not. write({ saiAdvisory: { routes: { subagent: { enabled: "true" } } } }); assert.equal(readSubagentRoutingSettings(file), undefined); write({ saiAdvisory: { routes: { subagent: { enabled: true, tiers: { simple: "p/small" }, genericAgents: ["delegate"] } } } }); const enabled = readSubagentRoutingSettings(file)!; assert.deepEqual(enabled.tiers, { simple: "p/small" }); // No enable flag reaches the router: route-off is not expressible as a router call. assert.equal(Object.hasOwn(enabled, "enabled"), false); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); it("dispatches no routing work from a public launch while the gate is closed", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "sai-routing-executor-")); const previous = process.env.SELESAI_CODING_AGENT_DIR; process.env.SELESAI_CODING_AGENT_DIR = dir; const events: Array<{ channel: string; data: unknown }> = []; try { const run = async (settings: unknown) => { fs.writeFileSync(path.join(dir, "settings.json"), JSON.stringify(settings)); events.length = 0; const executor = createSubagentExecutor({ pi: { events: { emit: (channel: string, data: unknown) => events.push({ channel, data }), on() { return () => {}; } }, getSessionName() { return "parent"; } } as never, state: createChildSafeState(), config: { maxSubagentDepth: 2, control: {}, intercomBridge: {} } as never, asyncByDefault: false, tempArtifactsDir: os.tmpdir(), getSubagentSessionRoot: () => os.tmpdir(), expandTilde: (value) => value, discoverAgents: () => ({ agents: [] as never[] }), }); const result = await executor.executePublic("agentless", { task: "work" }, new AbortController().signal, undefined, makeMinimalCtx(os.tmpdir()) as never); return { result, text: result.content.map((part) => part.text ?? "").join("\n"), routing: events.filter((event) => event.channel === "sai-routing") }; }; const closed = await run({ saiAdvisory: { routes: { subagent: { enabled: false } } } }); assert.equal(closed.result.isError, true); assert.match(closed.text, /Sai subagent routing \(saiAdvisory\.routes\.subagent\.enabled\) is off/); assert.deepEqual(closed.routing, [], "a closed gate emits no routing decision"); // Gate open, but nothing routable to ask about: still no dispatch, still no decision event. const open = await run({ saiAdvisory: { routes: { subagent: { enabled: true } } } }); assert.equal(open.result.isError, true); assert.match(open.text, /Sai could not choose an agent/); assert.deepEqual(open.routing, []); } finally { if (previous === undefined) delete process.env.SELESAI_CODING_AGENT_DIR; else process.env.SELESAI_CODING_AGENT_DIR = previous; fs.rmSync(dir, { recursive: true, force: true }); } }); });