import type { LlmProvider } from './types.js';
/**
 * Golden-set evaluation.
 *
 * Everything else Trazum reports is arithmetic: tokens, prices, multiplication.
 * This is the one question arithmetic cannot answer — does the shorter prompt
 * still do the job? — and the README has been answering it with a caveat
 * ("the aggressive level can change nuance; read the diff") because a rules
 * engine genuinely cannot know.
 *
 * The trap here is comparing the two prompts' outputs and calling the
 * difference a regression. A model asked the same question twice does not
 * answer identically, so "the optimised prompt diverged on 3 of 10 cases" is
 * meaningless on its own — it might be better than the original manages
 * against itself.
 *
 * So the original is run twice per case first, and that self-agreement is the
 * yardstick. The optimised prompt is judged against the model's own variance,
 * not against an imaginary determinism it never had. It costs a third call per
 * case and it is the only reason the number means anything.
 */
export interface EvalCase {
    /** The input this case feeds the prompt. */
    input: string;
    /** The original prompt's two answers, used to measure its own variance. */
    baseline: [string, string];
    /** The optimised prompt's answer. */
    optimized: string;
    /** How closely the original agreed with itself (0-1). */
    selfSimilarity: number;
    /** How closely the optimised answer matched the original's first (0-1). */
    crossSimilarity: number;
}
export type EvalVerdict = 'indistinguishable' | 'within-noise' | 'diverges' | 'inconclusive';
export interface EvalReport {
    provider: string;
    model: string;
    /**
     * The model the candidate answer came from.
     *
     * Equal to `model` on the ordinary comparison — same model, two prompts. It
     * differs when the question is the other one: **same prompt, two models**, which
     * is what a routing decision is. `profile` prices that route exactly and can say
     * nothing at all about whether the cheaper model still does the job; this is the
     * measurement that can.
     */
    candidateModel: string;
    cases: EvalCase[];
    /** Mean agreement of the original prompt with itself. The yardstick. */
    selfAgreement: number;
    /** Mean agreement between the original and the optimised prompt. */
    crossAgreement: number;
    verdict: EvalVerdict;
    /** Total provider calls made, so the cost is never a surprise. */
    callsMade: number;
}
export interface EvaluateOptions {
    /**
     * How many cases to run at once. Kept low by default: this hammers someone
     * else's endpoint, and a rate limit tripped halfway through wastes every
     * call already paid for.
     */
    concurrency?: number;
    /**
     * Where the candidate answer comes from. Defaults to `provider`.
     *
     * This is the whole routing axis, and it needed no new yardstick. The baseline
     * prompt is still run **twice on the original model** to measure that model's own
     * variance, and the candidate is still judged against it — so the question
     * becomes "does the cheaper model agree with the expensive one more closely than
     * the expensive one agrees with itself?", which is the honest form of "is this
     * route safe".
     *
     * A verdict built any other way would be a threshold somebody picked. This one is
     * the model's own noise floor, measured on the same cases in the same run.
     */
    candidateProvider?: LlmProvider;
}
/**
 * Builds the prompt for one case.
 *
 * A template gets its first placeholder filled; anything else gets the input
 * appended. Substituting is the honest reading of a prompt written with
 * `{{query}}` — appending would test a prompt nobody runs.
 */
export declare function fillPrompt(prompt: string, input: string): string;
/** Agreement between two answers, 0-1. */
/**
 * How closely two answers agree, 0 to 1.
 *
 * Exported because `prune.ts` measures the same thing and must measure it the
 * same way. It was a private function here first, and the copy that appeared in
 * `prune.ts` was a bag-of-words F1 while this is Jaccard over normalised text —
 * two different numbers under one name, with a comment in the copy claiming they
 * were the same measure. Sharing the function is what makes that comment true.
 */
export declare function agreement(a: string, b: string): number;
/**
 * Turns the two agreement figures into a verdict.
 *
 * The comparison is always relative. An optimised prompt agreeing with the
 * original 0.85 of the time looks alarming until you see the original agrees
 * with itself 0.86 — at which point the optimisation changed nothing the model
 * was not already doing on its own.
 *
 * `inconclusive` exists because a model that is wildly inconsistent with itself
 * cannot be used to judge anything. Reporting a confident verdict off that
 * would be worse than admitting the test does not work here.
 */
export declare function verdictFor(selfAgreement: number, crossAgreement: number): EvalVerdict;
/** Runs `tasks` with a bounded number in flight, preserving order. */
export declare function pooled<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]>;
/**
 * Runs both prompt versions over a set of inputs and reports whether the
 * optimisation changed the answers.
 *
 * Costs **three provider calls per case**: the original twice, the optimised
 * once. The doubled original is what makes the result interpretable, and
 * `callsMade` reports the total so the bill is never a surprise.
 */
export declare function evaluate(originalPrompt: string, optimizedPrompt: string, inputs: readonly string[], provider: LlmProvider, options?: EvaluateOptions): Promise<EvalReport>;
//# sourceMappingURL=evaluate.d.ts.map