/**
 * Two arms on real traffic, and the three things that stop it being theatre.
 *
 * `eval` compares two prompts on cases somebody wrote; `route` compares two
 * models on the same. Both measure agreement in a laboratory. The traffic is
 * the only place the real question gets answered — and the moment a comparison
 * runs on real traffic, three failures become available that a laboratory does
 * not have.
 *
 * ## 1. A winner where there is none
 *
 * Two arms always produce two numbers, and one of them is always larger. An
 * A/B report that names a winner from that is a coin flip with a dashboard.
 * The verdict here is **three-valued**, the way `verify`'s has been since
 * 1.39: A wins, B wins, or **not separable on this traffic** — and the third
 * one comes with the number of outcomes per arm that *would* separate them, so
 * "run it longer" is a quantified instruction rather than a shrug.
 *
 * ## 2. Peeking
 *
 * A test stopped on the first afternoon it looked good is not a test. The
 * stopping rule is declared **before** the experiment starts, and the report
 * says whether it was honoured. It cannot enforce that — nobody can stop
 * somebody reading a number early — but it can make an early stop *visible*
 * to whoever reads the result later, which is the part that matters.
 *
 * ## 3. Quality and cost judged apart, then together
 *
 * The interesting arm is almost never better *and* cheaper. It is better and
 * dearer, and the decision needs one figure nobody computes: **what an extra
 * success costs**. That is the marginal figure — the difference in spend over
 * the difference in successes — and it is the number a product decision
 * actually turns on, printed rather than left as an exercise.
 *
 * ## The statistics are shown, not asserted
 *
 * Wilson score intervals per arm and Newcombe's interval on the difference,
 * because both behave at the small samples this will actually see. The
 * intervals are returned, not just the verdict: a reader who disagrees with
 * the threshold can see the numbers it was applied to, which is the same
 * discipline `eval` established by running the original twice before judging
 * anything against it.
 */
import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
/**
 * The declaration, made before the experiment runs.
 *
 * Its whole purpose is to exist *earlier* than the result. A stopping rule
 * invented after looking at the numbers is not a stopping rule.
 */
export interface ExperimentDeclaration {
    /** Two arms. More would need a multiple-comparison correction nobody asked for. */
    arms: [string, string];
    /**
     * Outcomes each arm must record before the result may be read.
     *
     * Declared as a count rather than a duration, because a duration is a proxy
     * for a count and the proxy breaks the week traffic doubles.
     */
    minOutcomesPerArm: number;
}
/** What one arm actually did. */
export interface ExperimentArm {
    name: string;
    tally: OutcomeTally;
    /** Everything this arm spent, recorded outcome or not. */
    totalUsd: number;
}
export interface ArmResult {
    name: string;
    successes: number;
    /** Calls carrying a *declared* outcome — the denominator. */
    recorded: number;
    /** Successes over recorded, or null when nothing was recorded. */
    rate: number | null;
    /** Wilson score interval on that rate, or null. */
    interval: {
        low: number;
        high: number;
    } | null;
    /** Spend on calls carrying a declared outcome. */
    recordedUsd: number;
}
export type Separation = 'a-wins' | 'b-wins' | 'not-separable';
export type NotSeparableReason = 'interval-includes-zero' | 'no-difference-observed' | 'nothing-recorded';
export interface Marginal {
    /**
     * What one extra success costs, going from the worse arm to the better one.
     *
     * `(dearer spend − cheaper spend) / (more successes − fewer successes)`,
     * both per call so arms of different sizes compare. Null when the better arm
     * is also the cheaper one, because then nothing is being bought and a
     * "cost per extra success" would be a negative number people would quote.
     */
    usdPerExtraSuccess: number | null;
    /** The arm that resolved more, by rate. */
    better: string;
    /** Whether that arm also costs more per call. */
    dearer: boolean;
}
export interface ExperimentResult {
    a: ArmResult;
    b: ArmResult;
    separation: Separation;
    /** Why, when not separable. A refusal never arrives bare. */
    notSeparable: NotSeparableReason | null;
    /** Newcombe's interval on (rate A − rate B), or null. */
    difference: {
        point: number;
        low: number;
        high: number;
    } | null;
    /**
     * Outcomes **per arm** that would separate the observed difference, or null.
     *
     * Null when the arms recorded the same rate: no sample size separates a
     * difference of zero, and returning a very large number would read as "keep
     * going" when the honest answer is "there is nothing here to find".
     */
    outcomesNeededPerArm: number | null;
    stopping: {
        declared: number;
        /** Whether both arms cleared the declared minimum. */
        honoured: boolean;
        /** The arm that has not, when one has not. */
        short: string | null;
    };
    marginal: Marginal | null;
}
export declare function runExperiment(declaration: ExperimentDeclaration, arms: {
    a: ExperimentArm;
    b: ExperimentArm;
}, vocabulary: OutcomeVocabulary | null): ExperimentResult;
//# sourceMappingURL=experiment.d.ts.map