/**
 * Human gold dataset agreement metrics.
 *
 * Why this module exists
 * ----------------------
 * Bootstrap CI gives us *precision* — how stable the judge is across
 * resampled evaluations. It does not give us *validity* — whether the judge is
 * scoring the right thing at all. A judge can be extremely consistent (CI very
 * narrow) and yet systematically biased; that's an undetected failure mode.
 *
 * Human gold (or stronger-model gold as a proxy) provides an external anchor.
 * We compute agreement between the LLM judge and the gold annotator, and report
 * the result alongside the Bootstrap CI. Both numbers must be acceptable for a
 * conclusion to be trustworthy.
 *
 * Three metrics are exported:
 *
 *  - **Krippendorff's α (interval weights)** — primary. Distribution-free,
 *    doesn't assume coders are exchangeable (good fit when one "coder" is a model
 *    and the other a human annotator). Uses interval distance δ²=(c−k)² — a
 *    defensible choice for 1-5 Likert; an ordinal-distance variant would change α
 *    (BREAKING-COMPARABILITY) and is not implemented here.
 *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
 *    useful as a sanity check. Reports lower than α when marginals diverge.
 *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't
 *    penalize systematic offset (a judge that always scores 1 lower than gold
 *    has Pearson 1 but α < 1). Listed for completeness.
 *
 * All three are wrapped with a bootstrap CI on α (the primary) so the user can
 * see uncertainty on the agreement number itself when N is small.
 */
import { type BootstrapCI } from '../eval-core/bootstrap.js';
export interface RatingPair {
    /** Per-sample identifier; used only for diagnostics. */
    unitId: string;
    /** Score from coder A — convention: gold annotator goes here. */
    coderA: number;
    /** Score from coder B — convention: LLM judge goes here. */
    coderB: number;
}
export interface AgreementResult {
    /** Krippendorff α (interval weights). 1 = perfect, 0 = chance, < 0 = worse than chance. */
    alpha: number;
    /** 95% bootstrap CI on α — width signals uncertainty due to sample size. */
    alphaCI: BootstrapCI;
    /** Quadratic-weighted κ. */
    weightedKappa: number;
    /** Pearson product-moment correlation. NaN if either coder has zero variance. */
    pearson: number;
    /** How many rating pairs went into the calculation. */
    sampleCount: number;
}
/**
 * Krippendorff's α with interval weights for two coders.
 *
 * Implementation follows Krippendorff (2011), "Computing Krippendorff's
 * Alpha-Reliability". For interval scales the metric is δ²(c, k) = (c − k)².
 *
 * For two coders with one rating per unit:
 *   - Coincidence matrix entries o_{c,k} count both (a_u, b_u) and (b_u, a_u),
 *     so total mass n_·· = 2N.
 *   - D_o = Σ o_{c,k} · δ²(c,k) / n_··
 *   - D_e = Σ n_c · n_k · δ²(c,k) / (n_·· (n_·· − 1))
 *   - α   = 1 − D_o / D_e
 *
 * Returns NaN when D_e = 0 (e.g. all ratings identical across both coders) —
 * agreement is undefined when there is no variance to disagree about.
 */
export declare function computeKrippendorffAlpha(pairs: RatingPair[]): number;
/**
 * Cohen's quadratic-weighted κ for two coders on a numeric scale.
 *
 *   w(i,j)   = (i − j)² / (K − 1)²    (disagreement weight, scaled to [0,1])
 *   po_w     = mean over units of (1 − w(a_u, b_u))
 *   pe_w     = Σ_{i,j} marg_a(i) · marg_b(j) · (1 − w(i,j)) / N²
 *   κ_w      = (po_w − pe_w) / (1 − pe_w)
 *
 * For continuous (non-integer) scores we still need a scale range; the caller
 * passes (min, max). Default 1..5 fits omk's standard rubric.
 */
export declare function computeWeightedKappa(pairs: RatingPair[], scale?: {
    min: number;
    max: number;
}): number;
/**
 * Pearson product-moment correlation. NaN when either coder has zero variance.
 */
export declare function computePearson(pairs: RatingPair[]): number;
/**
 * Compute α + κ + Pearson, with bootstrap CI on α.
 *
 * Bootstrap resamples pairs (units), not individual ratings — pairs are the
 * unit of replication; resampling individual ratings would break the (a,b) tie.
 */
export declare function computeAgreementWithCI(pairs: RatingPair[], options?: {
    samples?: number;
    seed?: number;
    alpha?: number;
    scale?: {
        min: number;
        max: number;
    };
}): AgreementResult;