{"version":3,"file":"agent-selection-eval.d.ts","sourceRoot":"","sources":["../../src/core/agent-selection-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,EAEN,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,MAAM,sCAAsC,CAAC;AAE9C,iEAAiE;AACjE,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,iFAAiF;IACjF,QAAQ,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrF;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,GAAG,GAAE,MAAsB,GAAG,gBAAgB,EAAE,CAU/E;AAED,yFAAyF;AACzF,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,GAAG,SAAS,CAiBtE;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,SAAS,gBAAgB,EAAE,EAAE,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,MAAM,EAAE,CASnH;AASD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,OAAO,CAAC,kBAAkB,EAAE;IAAE,MAAM,EAAE,KAAK,CAAA;CAAE,CAAC,EACvD,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACrC,oBAAoB,CAiDtB;AAED,MAAM,MAAM,qBAAqB,GAC9B;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC,kBAAkB,EAAE;QAAE,MAAM,EAAE,KAAK,CAAA;KAAE,CAAC,CAAA;CAAE,CAAC;AAE5G,uFAAuF;AACvF,wBAAsB,qBAAqB,CAC1C,UAAU,EAAE,SAAS,gBAAgB,EAAE,EACvC,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,SAAS,EACzC,KAAK,EAAE,YAAY,GAAG,SAAS,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CAmBhC;AAED,yCAAyC;AACzC,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,oBAAoB,GAAG,MAAM,CAkB/E","sourcesContent":["/**\n * Does the agent roster steer a dispatch to the right agent?\n *\n * The `Task` tool asks the model to pick a `subagent_type` from\n * `<available_agents>`, and that block contains nothing but each agent's\n * summarized description. So agent selection is the same question G4 asks about\n * skills — \"given these descriptions and this situation, which one fires?\" —\n * and it reuses the same harness rather than growing a second one.\n *\n * What this exists to decide: `plan` and `explore` ship with the same tools, the\n * same isolation and the same `background` flag, differing only in model tier\n * and output contract — and `complexity` on the Task tool already expresses the\n * tier. Whether they are two agents or one is a question about whether the model\n * can actually tell them apart from their descriptions, which is measurable and\n * was previously being argued from intuition.\n *\n * The candidates use `summarizeAgentDescription`, not the raw frontmatter: the\n * summary is what the system prompt actually emits, and evaluating the full\n * description would score text the model never sees.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { loadAgentRegistry, summarizeAgentDescription } from \"./agent-registry.js\";\nimport {\n\trunTriggerEval,\n\ttype TriggerCandidate,\n\ttype TriggerCase,\n\ttype TriggerEvalOutcome,\n\ttype TriggerJudge,\n} from \"./extensions/plugins/trigger-eval.js\";\n\n/** How often one agent was chosen where another was expected. */\nexport interface ConfusionEntry {\n\texpected: string;\n\tactual: string;\n\tcount: number;\n}\n\nexport interface AgentSelectionReport {\n\tcorpusHash: string;\n\tagents: string[];\n\tcaseCount: number;\n\t/** Cases where the expected agent was chosen, over cases expecting any agent. */\n\taccuracy?: number;\n\t/** Cases correctly left to the parent, over cases expecting no delegation. */\n\tinlineAccuracy?: number;\n\t/** Every wrong pick, most frequent first. The pairs here are the finding. */\n\tconfusion: ConfusionEntry[];\n\t/** Per-agent recall: chosen / expected. An agent nobody picks is dead weight. */\n\tperAgent: Array<{ agent: string; expected: number; chosen: number; recall: number }>;\n}\n\n/**\n * The built-in agents as judge candidates, described exactly as the system\n * prompt describes them.\n *\n * `own: true` for all of them: unlike a plugin eval there is no foreign roster\n * to discriminate against, so every case is scored against the same closed set.\n * The `expect: null` cases carry the discriminative half instead — they ask\n * whether the model declines to delegate work it should keep.\n */\nexport function agentCandidates(cwd: string = process.cwd()): TriggerCandidate[] {\n\tconst registry = loadAgentRegistry({ cwd, includeBuiltins: true, includeClaude: false });\n\treturn registry\n\t\t.list()\n\t\t.map((agent) => ({\n\t\t\tname: agent.name,\n\t\t\tdescription: summarizeAgentDescription(agent.description ?? \"\"),\n\t\t\town: true,\n\t\t}))\n\t\t.sort((a, b) => a.name.localeCompare(b.name));\n}\n\n/** Load a gold set: `{ \"cases\": [{ \"prompt\": \"...\", \"expect\": \"explore\" | null }] }`. */\nexport function loadAgentCases(file: string): TriggerCase[] | undefined {\n\tif (!existsSync(file)) return undefined;\n\ttry {\n\t\tconst raw = JSON.parse(readFileSync(file, \"utf-8\")) as { cases?: unknown };\n\t\tif (!Array.isArray(raw.cases)) return undefined;\n\t\tconst cases = raw.cases.filter(\n\t\t\t(c): c is TriggerCase =>\n\t\t\t\t!!c &&\n\t\t\t\ttypeof c === \"object\" &&\n\t\t\t\ttypeof (c as TriggerCase).prompt === \"string\" &&\n\t\t\t\t(c as TriggerCase).prompt.trim().length > 0 &&\n\t\t\t\t((c as TriggerCase).expect === null || typeof (c as TriggerCase).expect === \"string\"),\n\t\t);\n\t\treturn cases.length > 0 ? cases : undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Every expected agent in the gold set must exist in the roster.\n *\n * A typo'd or removed agent name would otherwise score as a permanent miss and\n * read as a description problem, which is the most expensive way to be wrong\n * about an eval.\n */\nexport function validateAgentCases(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string[] {\n\tconst known = new Set(candidates.map((c) => c.name));\n\tconst problems: string[] = [];\n\tfor (const [i, testCase] of cases.entries()) {\n\t\tif (testCase.expect !== null && !known.has(testCase.expect)) {\n\t\t\tproblems.push(`case ${i} expects \"${testCase.expect}\", which is not in the roster`);\n\t\t}\n\t}\n\treturn problems;\n}\n\nfunction hashCorpus(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string {\n\tconst h = createHash(\"sha256\");\n\tfor (const c of candidates) h.update(`${c.name} ${c.description} `);\n\tfor (const c of cases) h.update(`${c.prompt} ${c.expect ?? \"\"} `);\n\treturn h.digest(\"hex\").slice(0, 16);\n}\n\n/**\n * Turn a scored run into the report that answers the design question.\n *\n * `runTriggerEval`'s recall/specificity are the right numbers for a plugin\n * defending itself against a foreign roster. Here the roster is closed, so the\n * useful shape is a confusion matrix: which agent loses to which, and how often.\n * \"explore and plan are interchangeable\" is a claim about one cell.\n */\nexport function summarizeAgentSelection(\n\toutcome: Extract<TriggerEvalOutcome, { status: \"ran\" }>,\n\tcandidates: readonly TriggerCandidate[],\n): AgentSelectionReport {\n\tconst confusion = new Map<string, ConfusionEntry>();\n\tconst expectedCounts = new Map<string, number>();\n\tconst chosenCounts = new Map<string, number>();\n\tlet delegated = 0;\n\tlet delegatedCorrect = 0;\n\tlet inline = 0;\n\tlet inlineCorrect = 0;\n\n\tfor (const result of outcome.record.results) {\n\t\tif (result.expected === null) {\n\t\t\tinline++;\n\t\t\tif (result.actual === null) inlineCorrect++;\n\t\t\tcontinue;\n\t\t}\n\t\tdelegated++;\n\t\texpectedCounts.set(result.expected, (expectedCounts.get(result.expected) ?? 0) + 1);\n\t\tif (result.correct) {\n\t\t\tdelegatedCorrect++;\n\t\t\tchosenCounts.set(result.expected, (chosenCounts.get(result.expected) ?? 0) + 1);\n\t\t\tcontinue;\n\t\t}\n\t\t// \"(none)\" is a real outcome, not a missing value: declining to delegate\n\t\t// work that should have been delegated is a different failure from picking\n\t\t// the wrong agent, and collapsing them would hide which one is happening.\n\t\tconst actual = result.actual ?? \"(none)\";\n\t\tconst key = `${result.expected} ${actual}`;\n\t\tconst entry = confusion.get(key) ?? { expected: result.expected, actual, count: 0 };\n\t\tentry.count++;\n\t\tconfusion.set(key, entry);\n\t}\n\n\tconst perAgent = candidates\n\t\t.map((candidate) => {\n\t\t\tconst expected = expectedCounts.get(candidate.name) ?? 0;\n\t\t\tconst chosen = chosenCounts.get(candidate.name) ?? 0;\n\t\t\treturn { agent: candidate.name, expected, chosen, recall: expected > 0 ? chosen / expected : 0 };\n\t\t})\n\t\t.sort((a, b) => a.recall - b.recall);\n\n\treturn {\n\t\tcorpusHash: outcome.record.corpusHash,\n\t\tagents: candidates.map((c) => c.name),\n\t\tcaseCount: outcome.record.caseCount,\n\t\taccuracy: delegated > 0 ? delegatedCorrect / delegated : undefined,\n\t\tinlineAccuracy: inline > 0 ? inlineCorrect / inline : undefined,\n\t\tconfusion: [...confusion.values()].sort((a, b) => b.count - a.count),\n\t\tperAgent,\n\t};\n}\n\nexport type AgentSelectionOutcome =\n\t| { status: \"not-run\"; reason: string }\n\t| { status: \"ran\"; report: AgentSelectionReport; outcome: Extract<TriggerEvalOutcome, { status: \"ran\" }> };\n\n/** Score the roster against a gold set. Never throws; a missing model is `not-run`. */\nexport async function runAgentSelectionEval(\n\tcandidates: readonly TriggerCandidate[],\n\tcases: readonly TriggerCase[] | undefined,\n\tjudge: TriggerJudge | undefined,\n): Promise<AgentSelectionOutcome> {\n\tif (cases && cases.length > 0) {\n\t\tconst problems = validateAgentCases(candidates, cases);\n\t\tif (problems.length > 0) return { status: \"not-run\", reason: `invalid gold set: ${problems.join(\"; \")}` };\n\t}\n\n\tconst outcome = await runTriggerEval(\"agent-selection\", candidates, cases, judge);\n\tif (outcome.status === \"not-run\") return outcome;\n\n\treturn {\n\t\tstatus: \"ran\",\n\t\treport: {\n\t\t\t...summarizeAgentSelection(outcome, candidates),\n\t\t\t// The shared harness hashes its own way; restate it over exactly what\n\t\t\t// this eval judged so two reports are comparable on their own terms.\n\t\t\tcorpusHash: hashCorpus(candidates, cases ?? []),\n\t\t},\n\t\toutcome,\n\t};\n}\n\n/** Human-readable report for the CLI. */\nexport function formatAgentSelectionReport(report: AgentSelectionReport): string {\n\tconst pct = (v: number | undefined) => (v === undefined ? \"n/a\" : `${Math.round(v * 100)}%`);\n\tconst lines = [\n\t\t`corpus ${report.corpusHash} - ${report.caseCount} case(s) over ${report.agents.length} agent(s): ${report.agents.join(\", \")}`,\n\t\t`delegation accuracy ${pct(report.accuracy)}   inline accuracy ${pct(report.inlineAccuracy)}`,\n\t\t\"\",\n\t\t\"per agent (lowest recall first):\",\n\t];\n\tfor (const row of report.perAgent) {\n\t\tlines.push(`  ${row.agent.padEnd(18)} ${row.chosen}/${row.expected} chosen  (${pct(row.recall)})`);\n\t}\n\tif (report.confusion.length > 0) {\n\t\tlines.push(\"\", \"confusions (expected then actual):\");\n\t\tfor (const entry of report.confusion) {\n\t\t\tlines.push(`  ${entry.expected} => ${entry.actual}   x${entry.count}`);\n\t\t}\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}