/**
 * Darwin — Offline Eval (v0.14.0)
 *
 * The benchmark harness as a first-class, injectable API: run N prompt
 * variants ("arms") over a frozen task set, score every output, and report
 * per-task means + arm deltas against the first arm (the baseline).
 *
 * This is Darwin's answer to the dataset+metric eval loop that offline
 * optimizers (DSPy's `Evaluate`, gepa-ts/dsts trainsets) are built around —
 * adapted to Darwin's shape: the same arms/tasks/judge wiring the shipped
 * `benchmark/` harness uses, generalised to any agent, any prompt set, and
 * any metric. It complements (not replaces) the online loop: seed and vet
 * prompts offline, then let the live A/B gate decide under real traffic.
 *
 * Zero hard deps, nothing here talks to an LLM directly — the caller injects
 * `run` (execute a prompt on a task) and `score` (judge one output). The CLI
 * (`darwin eval`) wires those to `runAgent` + the built-in critic; tests wire
 * them to deterministic fakes.
 */
export interface EvalTask {
    /** Stable identifier — shown in reports, used to align arms. */
    id: string;
    /** Optional task-type tag (mirrors `darwin run --task-type`). */
    type?: string;
    /** The task text handed to the agent. */
    task: string;
}
/**
 * Parse an eval task set from JSON text. Accepts either a bare array of
 * tasks or `{ "tasks": [...] }`, and validates the fields that the eval
 * loop depends on — a malformed set fails loudly HERE, not as a `undefined`
 * task string inside an LLM call N minutes into a sweep.
 */
export declare function parseEvalTasks(jsonText: string): EvalTask[];
/** One prompt variant under evaluation. */
export interface EvalArm {
    /** Canonical label — e.g. a stored version ("v3") or "candidate". Must be unique. */
    label: string;
    /** The full system-prompt text this arm runs with. */
    promptText: string;
    /**
     * Marks the agent's currently-active version. Display-only: the renderer
     * appends `*` to the label; the label itself stays canonical so it never
     * collides with `--versions` matching, stored labels, or JSON consumers.
     */
    active?: boolean;
}
/** Execute one (promptText, task) cell and return the raw agent output. */
export type EvalRunFn = (promptText: string, task: EvalTask) => Promise<string>;
/**
 * Score one output for one task. Return `null` for "no score" (the sample is
 * excluded from the mean instead of polluting it). The CLI wires this to the
 * built-in critic (1–10); programmatic callers can plug any metric —
 * exact-match, rubric judge, regex, latency — anything reducible to a number.
 */
export type EvalScoreFn = (task: EvalTask, output: string) => Promise<number | null>;
export interface EvalCellResult {
    taskId: string;
    /** Mean of the scored samples, or null when every sample failed/abstained. */
    mean: number | null;
    /** How many samples produced a score. */
    samples: number;
    /** How many run/score attempts threw (dropped, not counted as samples). */
    failures: number;
}
export interface EvalArmResult {
    label: string;
    /** Mirrors {@link EvalArm.active} — display metadata, not part of the label. */
    active: boolean;
    perTask: EvalCellResult[];
    /** Mean over the per-task means that scored (macro average). */
    mean: number | null;
    /** Number of tasks with at least one scored sample. */
    scoredTasks: number;
}
export interface EvalReport {
    agentName: string;
    taskCount: number;
    runsPerCell: number;
    arms: EvalArmResult[];
    /**
     * PAIRED arm deltas against the FIRST arm (the baseline): each delta is the
     * mean of per-task differences over the tasks where BOTH arms scored, so
     * asymmetric failures cannot skew the comparison (an arm that only scored
     * the easy tasks would otherwise look better than it is). `pairedTasks`
     * says how many tasks the delta is based on; `null` when no task scored in
     * both arms. The baseline's own entry aligns the array with `arms` (delta 0
     * when it scored at all, `null` otherwise).
     */
    deltas: {
        label: string;
        delta: number | null;
        pairedTasks: number;
    }[];
    startedAt: string;
    completedAt: string;
}
export interface RunEvalOptions {
    agentName: string;
    arms: EvalArm[];
    tasks: EvalTask[];
    run: EvalRunFn;
    score: EvalScoreFn;
    /** Samples per (arm × task) cell — >1 averages out judge variance. Default 1. */
    runsPerCell?: number;
    /** Progress callback — the CLI prints dots/scores, tests ignore it. */
    onCell?: (arm: EvalArm, task: EvalTask, cell: EvalCellResult) => void;
}
/**
 * Run the full arms × tasks × runsPerCell grid sequentially and aggregate.
 *
 * Sequential on purpose: eval sweeps run against rate-limited providers and
 * subscription CLIs; a transient failure drops ONE sample (recorded in
 * `failures`) instead of aborting the sweep — the shipped benchmark's
 * behaviour, kept here.
 */
export declare function runEval(opts: RunEvalOptions): Promise<EvalReport>;
/**
 * Render an {@link EvalReport} as the fixed-width table the CLI prints and
 * writes to `.darwin/reports/`. Pure string building — no I/O.
 */
export declare function renderEvalReport(report: EvalReport): string;
//# sourceMappingURL=eval-runner.d.ts.map