/** * Does the agent roster steer a dispatch to the right agent? * * The `Task` tool asks the model to pick a `subagent_type` from * ``, and that block contains nothing but each agent's * summarized description. So agent selection is the same question G4 asks about * skills — "given these descriptions and this situation, which one fires?" — * and it reuses the same harness rather than growing a second one. * * What this exists to decide: `plan` and `explore` ship with the same tools, the * same isolation and the same `background` flag, differing only in model tier * and output contract — and `complexity` on the Task tool already expresses the * tier. Whether they are two agents or one is a question about whether the model * can actually tell them apart from their descriptions, which is measurable and * was previously being argued from intuition. * * The candidates use `summarizeAgentDescription`, not the raw frontmatter: the * summary is what the system prompt actually emits, and evaluating the full * description would score text the model never sees. */ import { type TriggerCandidate, type TriggerCase, type TriggerEvalOutcome, type TriggerJudge } from "./extensions/plugins/trigger-eval.js"; /** How often one agent was chosen where another was expected. */ export interface ConfusionEntry { expected: string; actual: string; count: number; } export interface AgentSelectionReport { corpusHash: string; agents: string[]; caseCount: number; /** Cases where the expected agent was chosen, over cases expecting any agent. */ accuracy?: number; /** Cases correctly left to the parent, over cases expecting no delegation. */ inlineAccuracy?: number; /** Every wrong pick, most frequent first. The pairs here are the finding. */ confusion: ConfusionEntry[]; /** Per-agent recall: chosen / expected. An agent nobody picks is dead weight. */ perAgent: Array<{ agent: string; expected: number; chosen: number; recall: number; }>; } /** * The built-in agents as judge candidates, described exactly as the system * prompt describes them. * * `own: true` for all of them: unlike a plugin eval there is no foreign roster * to discriminate against, so every case is scored against the same closed set. * The `expect: null` cases carry the discriminative half instead — they ask * whether the model declines to delegate work it should keep. */ export declare function agentCandidates(cwd?: string): TriggerCandidate[]; /** Load a gold set: `{ "cases": [{ "prompt": "...", "expect": "explore" | null }] }`. */ export declare function loadAgentCases(file: string): TriggerCase[] | undefined; /** * Every expected agent in the gold set must exist in the roster. * * A typo'd or removed agent name would otherwise score as a permanent miss and * read as a description problem, which is the most expensive way to be wrong * about an eval. */ export declare function validateAgentCases(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[]): string[]; /** * Turn a scored run into the report that answers the design question. * * `runTriggerEval`'s recall/specificity are the right numbers for a plugin * defending itself against a foreign roster. Here the roster is closed, so the * useful shape is a confusion matrix: which agent loses to which, and how often. * "explore and plan are interchangeable" is a claim about one cell. */ export declare function summarizeAgentSelection(outcome: Extract, candidates: readonly TriggerCandidate[]): AgentSelectionReport; export type AgentSelectionOutcome = { status: "not-run"; reason: string; } | { status: "ran"; report: AgentSelectionReport; outcome: Extract; }; /** Score the roster against a gold set. Never throws; a missing model is `not-run`. */ export declare function runAgentSelectionEval(candidates: readonly TriggerCandidate[], cases: readonly TriggerCase[] | undefined, judge: TriggerJudge | undefined): Promise; /** Human-readable report for the CLI. */ export declare function formatAgentSelectionReport(report: AgentSelectionReport): string; //# sourceMappingURL=agent-selection-eval.d.ts.map