import type { CaseResult, EvalCase, EvalTurn, ExperimentReport, Scorer } from './types.js';
export interface ExperimentConfig<TInput = unknown> {
    name: string;
    cases: readonly EvalCase<TInput>[];
    /** Applied to every case unless the case overrides them. */
    scorers: readonly Scorer[];
    /**
     * Execute one case. Returning an `EvalTurn` rather than driving `query()`
     * here keeps the harness independent of how you construct a turn —
     * scripted mock, real provider, or a whole agent behind a facade.
     */
    run: (input: TInput, evalCase: EvalCase<TInput>, signal: AbortSignal) => Promise<EvalTurn>;
    /**
     * Wall-clock deadline for one case, including execution and every
     * scorer. Unset means no deadline.
     *
     * A non-cooperative `run` closure or scorer is detached by an independent
     * race, so it cannot hold the suite forever. The same signal is passed to
     * both phases; I/O-owning scorers should forward it to their transport,
     * as `judgeScorer` does.
     *
     * A turn that exhausts the budget is reported as failed. A scorer that
     * exhausts the remainder is unavailable, making an otherwise unjudged
     * case inconclusive. Either way the suite continues: forty cases should
     * not be lost to one operation that hung.
     */
    timeoutMs?: number;
    /** Mean score a case must reach to count as passed. Default 1. */
    passThreshold?: number;
    /** Cases to run at once. Default 1 — deterministic ordering by default. */
    concurrency?: number;
    onCaseFinish?: (result: CaseResult) => void;
}
/**
 * Run a dataset and score it.
 *
 * There was no evaluation harness of any kind: no dataset, no scorer, no
 * judge, no trajectory assertion. So every behavior change in this SDK —
 * a tool description, the `search_tools` top-k, the compaction threshold —
 * shipped with no regression signal behind it, and the only way to notice
 * a degradation was for a user to hit it.
 *
 * The scorer contract requires a `reason` on every score, so a report says
 * what got worse rather than only that something did.
 */
export declare function runExperiment<TInput>(config: ExperimentConfig<TInput>): Promise<ExperimentReport>;
export declare function formatReport(report: ExperimentReport): string;
//# sourceMappingURL=experiment.d.ts.map