import type { BudgetLedger } from '../types/lifecycle.js';
import { type EvaluationCache } from './cache.js';
import type { Dataset, DatasetExample, Evaluator, ExampleResult, Experiment, ExperimentStore, MetricSummary, SummaryEvaluator } from '../types/evaluate.js';
/** Turns one example into an output. A completion, an agent run, a graph, or plain code. */
export type EvaluationTarget<I = unknown> = (inputs: I, context: {
    example: DatasetExample<I>;
    run: number;
    signal?: AbortSignal;
}) => Promise<unknown> | unknown;
/** Options for `evaluate()`. */
export interface EvaluateOptions {
    /** Names the experiment. Defaults to the dataset name plus a timestamp. */
    name?: string;
    /** Examples evaluated at once. Defaults to 4; `1` runs them in order. */
    concurrency?: number;
    /**
     * Times each example is run. Above 1 for a target that is not deterministic; a function gives
     * each example its own count, for a dataset where some cases are noisier than others.
     */
    repetitions?: number | ((example: DatasetExample) => number);
    /** Evaluators over the whole experiment, such as a pass rate. */
    summary?: SummaryEvaluator[];
    /** Where the experiment is saved. */
    store?: ExperimentStore;
    /** Application data stored with the experiment. */
    metadata?: Record<string, unknown>;
    /** Stops an example that hangs. */
    timeoutMs?: number;
    /**
     * Cancels the evaluation. No further examples start, the examples in flight receive the signal,
     * and `evaluate()` rejects with the abort reason instead of storing a partial experiment that
     * could later be mistaken for a complete baseline.
     */
    signal?: AbortSignal;
    /**
     * Reads the cost of one output. Defaults to `meta.cost.amount` on a completion, a numeric
     * `meta.cost` on an image result, or a numeric `cost` field on anything else.
     */
    cost?: (output: unknown, example: DatasetExample) => number | undefined;
    /** Called after each example finishes, for progress reporting. */
    onResult?: (result: ExampleResult) => void;
    /**
     * Reuses target outputs across experiments: an example whose inputs, repetition, and target
     * `fingerprint` are unchanged is scored again from the stored output instead of being run, so a
     * comparison re-runs only what changed. Evaluators always run. Failures are never stored.
     */
    cache?: EvaluationCache;
    /**
     * Identifies what decides the target's output — a model, a prompt version, settings — so a changed
     * target never reuses another's outputs. Required with `cache`; `fingerprintOf()` builds one from
     * any settings object.
     */
    fingerprint?: string;
    /**
     * A spend cap. Each example run reserves `estimate` US dollars from `ledger`, such as a
     * `budgetLedger()`, before it starts, and is reconciled with what it cost; a cached output costs
     * nothing. Once the budget runs out no further example starts, the ones in flight finish, and
     * `evaluate()` rejects with an `EvaluationBudgetError` that carries the results it paid for. The
     * incomplete experiment is neither returned nor stored.
     */
    budget?: {
        ledger: BudgetLedger;
        estimate: number;
        tenantId?: string;
    };
    /** Replaces the system clock, for tests. */
    now?: () => Date;
}
/**
 * Thrown by `evaluate()` when its budget runs out before every example has run. `results` holds the
 * examples that did run, scored; `cause` is the ledger's `BudgetExceededError`.
 */
export declare class EvaluationBudgetError extends Error {
    /** The ledger's refusal. */
    readonly cause: Error;
    /** The examples that ran before the budget ran out. */
    readonly results: ExampleResult[];
    constructor(
    /** The ledger's refusal. */
    cause: Error, 
    /** The examples that ran before the budget ran out. */
    results: ExampleResult[]);
}
/**
 * Runs a target over a dataset and scores it.
 *
 * One entry point for every kind of target, because the question is always the same: for each
 * example, produce an output, score it, and keep the result somewhere a later run can be compared
 * against. Everything that differs — how an output is produced, what counts as good — is a function
 * the caller supplies.
 */
export declare function evaluate<I = unknown, O = unknown>(target: EvaluationTarget<I>, dataset: Dataset<I, O>, evaluators: Array<Evaluator<I, O>>, options?: EvaluateOptions): Promise<Experiment>;
/** Mean, spread, and an interval per metric, so one lucky run is not mistaken for an improvement. */
export declare function summarize(results: ExampleResult[]): MetricSummary[];
/** Count, mean, sample standard deviation, minimum, maximum, and 95% interval of the mean for a set of scores. */
export declare function stats(values: number[]): Omit<MetricSummary, 'key' | 'passRate'>;