/** * Darwin — Multi-Critic Evaluator * * Runs 3 specialized critics in parallel and takes the median score. * More robust than a single critic — reduces bias and random variance. * * Agent-aware: Different agents get different evaluation criteria. * * Investigator Critics: * A: Facts & Sources — accuracy, citations, primary documents * B: Honesty & Courage — intellectual bravery, clear positions, uncomfortable truths * C: Completeness & Structure — full investigation, proper format, both sides covered * * Writer Critics: * A: Task Compliance & Accuracy — did the writer follow the brief? Correct claims? * B: Persuasion & Voice — tone, engagement, conviction, audience awareness * C: Substance & Originality — depth, concrete value, fresh angles */ /** Critic prompt definition */ export interface CriticPromptDef { name: string; prompt: string; } /** Score result from a single critic */ export interface CriticScore { critic: string; score: number; report: string; } /** Combined result from multi-critic evaluation */ export interface MultiCriticResult { /** Median score across all critics */ medianScore: number; /** Individual critic results */ critics: CriticScore[]; /** Combined report text */ combinedReport: string; } /** Function that runs a critic and returns its output */ export type RunCriticFn = (systemPrompt: string, task: string, criticName: string) => Promise; /** Options for {@link runMultiCritic} (v0.7.0). */ export interface RunMultiCriticOptions { /** * v0.7.0 — Strip markdown from the agent output BEFORE handing it to the * critics, so they score CONTENT, not FORMAT. * * Why: LLM-as-judge research consistently documents a FORMAT/STYLE bias — * judges tend to prefer well-formatted (markdown) answers over identical * plain prose. When evolving prompts produce outputs in different formats, * an un-normalised judge measures formatting, not quality, and the optimizer * drifts toward "add more bold/headers" instead of "be more correct". * Normalising both candidates' outputs the same way removes that confound. * (Position bias, by contrast, is reported to be negligible on current * frontier judge models, so this normalises format only — not order.) * * Off by default (byte-for-byte v0.6.0 behaviour). Turn ON for agents whose * deliverable is prose; leave OFF when the format itself is the deliverable * (e.g. an agent that must "produce a markdown table"). */ normalizeForJudging?: boolean; /** * v0.12.0 — Bring your own judges: an explicit critic-prompt set for THIS * call, bypassing the built-in {@link getCriticPrompts} name lookup. * * Why: the built-in `AGENT_CRITIC_MAP` covers a handful of generic agent * archetypes (investigator / writer / research / critic / analyst / …). * Any fleet with domain agents beyond those archetypes previously had to * FORK this file to register its own judges — unknown names silently fall * back to `INVESTIGATOR_PROMPTS`, which mis-scores domain output (e.g. a * game-simulation turn judged as an investigative report). With this * option the caller keeps its critic sets in its own codebase and passes * the right set per call. * * Semantics: any count ≥ 1 works (the median handles even counts). Entries * that are not a `{ name, prompt }` pair of non-empty strings are dropped — * a config-loaded judge list with holes degrades instead of crashing. An * empty array, a non-array, or an array with no usable entry falls back to * the built-in lookup, so misconfigured callers get v0.11 behaviour instead * of judging with zero critics. * * Judge contract: each prompt must instruct the critic to emit * `===SCORE=== N` (or an `X/10` figure) — outputs without either count as * a failed critic, and if ALL critics fail the result is `medianScore: 0`. */ criticPrompts?: CriticPromptDef[]; /** * v0.12.0 — Override the output label used in the evaluation preamble * ("Evaluate the following {label} for the task …"). Pairs with * {@link RunMultiCriticOptions.criticPrompts} for agents outside the * built-in `AGENT_OUTPUT_LABELS` map, whose label would otherwise be the * generic "output". Ignored when empty/whitespace-only. */ outputLabel?: string; } /** * v0.7.0 — Strip markdown formatting to plain prose for style-bias-free * judging. Preserves the words and line structure; removes only formatting * tokens (headers, bold/italic, code fences + inline code, links→text, * images→alt, list bullets, blockquotes, table pipes, horizontal rules, * strikethrough). Pure, deterministic. */ export declare function stripMarkdownForJudging(text: string): string; /** * Get the right critic prompts for an agent. * Falls back to investigator prompts for unknown agents (backward-compatible). */ export declare function getCriticPrompts(agentName: string): CriticPromptDef[]; /** @deprecated Use getCriticPrompts(agentName) instead. Kept for backward compatibility. */ export declare const CRITIC_PROMPTS: CriticPromptDef[]; /** * Run multiple specialized critics and return the median score. * Critics are selected based on the agent being evaluated. * * @param agentOutput - The agent's output to evaluate * @param task - The original task description * @param runCritic - Function to run a critic (injected, uses Claude CLI) * @param agentName - Name of the agent being evaluated (determines which critic set to use) */ export declare function runMultiCritic(agentOutput: string, task: string, runCritic: RunCriticFn, agentName?: string, options?: RunMultiCriticOptions): Promise; //# sourceMappingURL=multi-critic.d.ts.map