/** * Trust gate — decides whether an ensemble's scores are allowed to be BELIEVED, * one level up from {@link aggregateJudgeVerdicts} (which only reduces ONE * artifact's raters to a composite). A composite is a number; this is the check * that the number means anything. It is the code "Enforced by" for the * measurement-validation skill's after-gate ("is this result allowed to be * believed"). * * Three checks, each fail-loud and named in `trustReasons`: * (1) inter-rater reliability over the corpus ≥ `irrFloor` — raters that * disagree no better than chance carry no signal to optimize against. * (2) per-item rater spread ≤ `spreadCeiling` — for EACH item, raters must * converge on THAT item. * (3) surviving raters per item ≥ `minSurvivors` — a mean over one or two * raters is an anecdote, not an ensemble. * * CRITICAL metric semantics — per-item spread is rater disagreement about the * SAME item: `max(score) − min(score)` across the raters that scored THAT item * (max over its dimensions), never pooled across different items or across the * baseline/candidate sides. Pooling reads a genuine quality gap BETWEEN items as * "the raters split" and so trips the gate exactly when the finding is largest — * the failure mode the after-gate exists to prevent. The corpus IRR (check 1) * leans on the substrate's `interRaterReliability`, whose expected-disagreement * denominator already pools across items, so genuine item-to-item variation * RAISES reliability rather than lowering it. */ import { type JudgeVerdict } from '@tangle-network/agent-eval'; /** One item's raters: the per-judge verdicts {@link aggregateJudgeVerdicts} * reduces, tagged with the item they scored so spread stays within-item. */ export interface TrustItem { /** Stable item identifier — surfaces in `perItemSpread` and `trustReasons`. */ itemId: string; /** The raters' verdicts for THIS item (one per judge call). A failed judge * (`perDimension: null`) is dropped before spread/IRR, never folded as 0. */ verdicts: readonly JudgeVerdict[]; } /** Thresholds for {@link trustVerdicts}. All overridable; defaults are the * conservative after-gate bar. */ export interface TrustThresholds { /** Minimum corpus inter-rater reliability (Krippendorff-style α). Below this * the raters agree no better than chance. Default 0.2. */ irrFloor?: number; /** Maximum per-item rater spread (`max − min` over a single item's surviving * raters, across its dimensions). Above this the raters split ON THAT ITEM. * Default 0.5. */ spreadCeiling?: number; /** Minimum surviving (non-failed) raters required per item. Default 3. */ minSurvivors?: number; } /** Result of the trust gate. `trustworthy` iff every check passed; `trustReasons` * is empty iff `trustworthy`. */ export interface TrustVerdict { /** True iff IRR ≥ floor AND every item's spread ≤ ceiling AND every item has * ≥ `minSurvivors` surviving raters. */ trustworthy: boolean; /** One entry per FAILED check, each naming its number + the offending value. * Empty iff `trustworthy`. */ trustReasons: string[]; /** Corpus inter-rater reliability actually measured (the check-1 value). */ interRaterReliability: number; /** Per-item spread (`max − min` over surviving raters, max over dimensions), * keyed by `itemId`. The check-2 input, surfaced for drill-down. */ perItemSpread: Record; } /** * Decide whether an ensemble's per-item verdicts are trustworthy enough to * believe a lift computed from them. Pure: no LLM, no I/O, no clock, no random — * the same `items` + `thresholds` always yield the same verdict. * * Sibling to {@link aggregateJudgeVerdicts}: that reduces ONE item's raters to a * composite; this audits the raters ACROSS items and reports whether the * composites are believable. Run it on the corpus of held-out items before * reporting any lift over their scores. * * @throws if `items` is empty — an empty corpus has no measurable trust, and a * silent `trustworthy: true` over zero evidence is the exact lie the gate * exists to refuse. */ export declare function trustVerdicts(items: readonly TrustItem[], thresholds?: TrustThresholds): TrustVerdict;