import { type EvidenceDigestMode } from '../../gates/stats/index.js'; import type { QualificationSplit } from '../../gates/stats/index.js'; export { PairedDifferenceError, clopperPearsonInterval, freezeQualificationSplit, normalQuantile, pairedBinaryDifferenceInterval, pairedMeanDifferenceBootstrap, pairedNonInferiority, verifyQualificationSplits, wilsonScoreInterval, } from '../../gates/stats/index.js'; export type { PairedBinaryCounts, PairedDifferenceInterval, QualificationSplit, } from '../../gates/stats/index.js'; /** Digest modes accepted when verifying a frozen benchmark plan. Canonical-only by default; pass an explicit legacy allowlist to verify pre-migration evidence. */ export type BenchmarkDigestModes = readonly EvidenceDigestMode[]; export type BinaryBenchmarkPlanVersion = 'decision-binary-benchmark-plan/v1' | 'decision-binary-benchmark-plan/v2'; export interface FrozenBinaryBenchmarkPlan { schemaVersion: BinaryBenchmarkPlanVersion; splits: readonly QualificationSplit[]; /** Hash of sorted (id, label, slice) triples across all splits, before any held-out predictions. */ datasetDigest: `sha256:${string}`; minimumOverallN: number; minimumSliceN: number; maximumSelectiveRisk: number; maximumReviewRate: number; maximumBrier: number; digest: `sha256:${string}`; } export interface BinaryBenchmarkLabel { id: string; label: 0 | 1; slice: string; } /** Freeze thresholds and all three label sets before acquiring test predictions. */ export declare function freezeBinaryBenchmarkPlan(splits: readonly QualificationSplit[], labels: readonly BinaryBenchmarkLabel[], limits: Pick): FrozenBinaryBenchmarkPlan; /** * Verifies a frozen plan digest without scoring. Canonical-only by default; pass * `{ digestModes: ['canonical', 'legacy'] }` to allowlist the versioned legacy * (`JSON.stringify`) digest for pre-migration v1 evidence. Fresh v2 records are * canonical-only: a legacy digest on v2 fields never verifies, even under an * explicit allowlist. */ export declare function verifyBinaryBenchmarkPlanDigest(plan: FrozenBinaryBenchmarkPlan, labels: readonly BinaryBenchmarkLabel[], options?: { digestModes?: BenchmarkDigestModes; }): EvidenceDigestMode | null; /** A separately anchored digest is required: a caller-created plan cannot attest its own preregistration. */ export declare function evaluatePreregisteredBinaryBenchmark(plan: FrozenBinaryBenchmarkPlan, trustedPlanDigest: `sha256:${string}`, labels: readonly BinaryBenchmarkLabel[], samples: readonly BinaryQualificationSample[], options?: { digestModes?: BenchmarkDigestModes; }): { decision: 'pass' | 'fail' | 'insufficient-evidence'; reasons: string[]; metrics: ReturnType; }; export interface BinaryQualificationSample { id: string; slice: string; label: 0 | 1; probability: number; accepted: boolean; latencyMs: number; inputTokens: number | null; outputTokens: number | null; costUsd: number | null; calls: number; retries: number; fallbacks: number; } export interface BinarySliceMetrics { sampleN: number; errorRate: number; brier: number; logLoss: number; expectedCalibrationError: number; coverage: number; selectiveRisk: number | null; reviewRate: number; errorWilson95: readonly [number, number]; latencyMs: { p50: number; p95: number; p99: number; }; inputTokens: number | null; outputTokens: number | null; costUsd: number | null; calls: number; retries: number; fallbacks: number; } /** Refuses to score unregistered or non-held-out samples. */ export declare function evaluateBinaryHeldout(splits: readonly QualificationSplit[], samples: readonly BinaryQualificationSample[]): { overall: BinarySliceMetrics; slices: Record; }; /** Descriptive fit diagnostics; calibration observations are never held-out test evidence. */ export declare function evaluateBinaryCalibration(splits: readonly QualificationSplit[], samples: readonly BinaryQualificationSample[]): { overall: BinarySliceMetrics; slices: { [k: string]: BinarySliceMetrics; }; }; export interface OrdinalQualificationSample { id: string; trueLevel: number; predictedLevel: number; levels: number; } /** Mean absolute level error and exact match, scored on a frozen held-out split. */ export declare function evaluateOrdinalHeldout(splits: readonly QualificationSplit[], samples: readonly OrdinalQualificationSample[]): { sampleN: number; exactRate: number; meanAbsoluteError: number; normalizedAbsoluteError: number; }; export interface RankingQualificationSample { id: string; /** Higher score means preferred; scores must refer to the same stable option IDs. */ gold: Readonly>; predicted: Readonly>; } /** Counts gold-comparable option pairs; ties in the prediction are errors, not wins. */ export declare function evaluateRankingHeldout(splits: readonly QualificationSplit[], samples: readonly RankingQualificationSample[]): { sampleN: number; comparablePairs: number; concordance: number | null; }; export interface PairedQualificationSample { id: string; control: string; /** Same task after hostile-state perturbation or repeated invocation. */ observed: string; } /** Measures categorical movement without claiming that either output is correct. */ export declare function measurePairedMovement(pairs: readonly PairedQualificationSample[]): { sampleN: number; changedN: number; changedRate: number; changedWilson95: readonly [number, number]; }; //# sourceMappingURL=quality.d.ts.map