/**
 * Paired comparison between two configs in one run record.
 *
 * Aggregate means are the wrong instrument for "should we change this
 * default". Both configs score the *same* queries, so the informative
 * question is per-query: how many did B improve, how many did it hurt, and
 * could that split have come from chance? A 3pp mean gap can be two queries
 * out of sixty moving, which is not a reason to change anything.
 *
 * Reports a two-sided sign test over the queries where the two configs
 * differ. Ties carry no information about direction and are excluded, which
 * is what makes the test paired.
 */
import type { EvalQueryResult } from "./eval.js";
/** Per-query metrics that can be compared. */
export type ComparableMetric = "recallAt1" | "recallAt5" | "recallAt10" | "recallAt50" | "mrr";
export interface PairedQueryDelta {
    id: string;
    class: string;
    before: number;
    after: number;
}
export interface PairedComparison {
    metric: ComparableMetric;
    before: string;
    after: string;
    better: number;
    worse: number;
    tied: number;
    meanBefore: number;
    meanAfter: number;
    /** Two-sided sign-test p-value over non-tied queries. 1 when all tie. */
    pValue: number;
    deltas: PairedQueryDelta[];
}
export interface ComparableRun {
    perQuery: Array<{
        id: string;
        class: string;
        results: EvalQueryResult[];
    }>;
}
/** Two-sided sign test: probability of a split at least this lopsided under
 *  a fair coin. */
export declare function signTestPValue(better: number, worse: number): number;
/**
 * Pair the *same* config across two run records, so "did my change help?" is
 * answerable at all. `comparePaired` compares two configs inside one run;
 * this compares one config before and after a retrieval change.
 *
 * Only sound when both runs scored the same corpus and the same gold set —
 * otherwise the delta includes corpus drift. The caller checks provenance.
 */
export declare function mergeRunsForComparison(before: ComparableRun, after: ComparableRun, label: string): ComparableRun;
export declare function comparePaired(run: ComparableRun, before: string, after: string, metric: ComparableMetric): PairedComparison;
export declare function formatComparison(comparison: PairedComparison): string;
//# sourceMappingURL=eval-compare.d.ts.map