import { type DecisionAskInput, type DecisionAskResult } from './ask.js'; import { type TokenPrice } from './pricing.js'; import type { DecisionConfig } from '../config/decision-config.js'; export type BenchmarkRates = NonNullable['cost']>; export interface BenchmarkCase { id: string; split: 'tuning' | 'holdout'; input: DecisionAskInput; expected: string | boolean | number; } export interface BenchmarkPrice extends TokenPrice { model: string; source: string; checkedAt: string; } export interface BenchmarkObservation { answer: string | boolean | number | null; confidence: number | null; probability?: number; status: 'answered' | 'fallback'; model: string | null; usage: (NonNullable & { cachedInputTokens?: number | null; }) | null; latencyMs: number; /** Captured sourced rate or provider-reported USD; null is never a free call. */ cost?: { usd: number | null; price: BenchmarkPrice | null; source?: 'provider-reported'; }; } export interface BenchmarkRow { id: string; baseline: BenchmarkObservation; jev: BenchmarkObservation[]; } export interface BenchmarkPolicy { threshold: number; passes: number; agreement: number; } export interface BenchmarkCapture { schema: 'aiwg-decision-benchmark-capture/v1'; datasetDigest: string; capturedAt: string; baselineLabel: string; rows: BenchmarkRow[]; /** Interrupted captures retain real observations only and can be extended. */ collection?: { stopReason: 'complete' | 'incomplete' | 'usd-limit' | 'unknown-cost'; calls: number; knownUsd: number; }; } export declare const BENCHMARK_PASS_COUNTS: readonly [1, 3, 5, 10, 15, 20]; export declare const MAX_BENCHMARK_PASSES = 20; /** Descriptive collection totals, not policy-selection evidence. Missing usage stays unknown. */ export declare function benchmarkComparison(capture: BenchmarkCapture, pricing?: BenchmarkRates): string; export declare function benchmarkDigest(cases: BenchmarkCase[]): string; export declare function validateBenchmarkCases(value: unknown): asserts value is BenchmarkCase[]; export interface CollectBenchmarkOptions { live: boolean; maxCalls: number; maxUsd: number; passes: number; baselineLabel: string; baseline: (input: DecisionAskInput) => Promise; jev?: (input: DecisionAskInput) => Promise; baselinePrices?: BenchmarkPrice[]; jevPrices?: BenchmarkPrice[]; /** Reuse a complete Jev response set while collecting a new baseline. */ jevCapture?: BenchmarkCapture; extendCapture?: BenchmarkCapture; holdout?: number; seed?: string; env?: NodeJS.ProcessEnv; now?: () => number; onCheckpoint?: (capture: BenchmarkCapture) => Promise; } /** Opt-in only. One invocation is one provider call; production ask never calls this collector. */ export declare function collectBenchmark(dataset: BenchmarkCase[], options: CollectBenchmarkOptions): Promise; export declare function benchmarkScoringDeviation(cases: BenchmarkCase[], capture: BenchmarkCapture, passes: number): { passes: number; observations: number; answered: number; rawAccuracy: number; answerDisagreementRate: number | null; meanConfidence: number | null; meanWithinCaseConfidenceStdDev: number | null; meanWithinCaseScaleScoreStdDev: number | null; meanScaleMaeGold: number | null; meanScaleMaeBaseline: number | null; meanYesNoBrier: number | null; cases: { id: string; split: "tuning" | "holdout"; kind: "scale" | "yes-no" | "choices"; observations: number; answered: number; rawCorrect: number; nonModalAnswers: number; confidenceMean: number | null; confidenceStdDev: number | null; scaleScoreMean: number | null; scaleScoreStdDev: number | null; scaleScoreMin: number | null; scaleScoreMax: number | null; scaleMaeGold: number | null; scaleMaeBaseline: number | null; yesNoBrier: number | null; }[]; }; /** Threshold/pass/consensus sweeps reuse captures. Holdout NEVER selects the policy. */ export declare function analyzeBenchmark(cases: BenchmarkCase[], capture: BenchmarkCapture, policies: BenchmarkPolicy[], pricing?: BenchmarkRates, options?: { minimumAccuracy?: number; maximumFallbackRate?: number; }): { schema: string; datasetDigest: string; baselineLabel: string; pricing: { llm_per_mtok?: number; jev_per_mtok?: number; jev_input_per_mtok?: number; jev_output_per_mtok?: number; source?: "typesafe-docs" | "manual" | "typesafe-snapshot"; checked_at?: string; jev_model?: string; } | null; minimumAccuracy: number; maximumFallbackRate: number; recommended: BenchmarkPolicy; measurements: { policy: BenchmarkPolicy; tuning: { split: "tuning" | "holdout"; cases: number; accepted: number; fallbackRate: number; selectiveAccuracy: number | null; baselineAccuracy: number; routedAccuracy: number; agreementWithBaseline: number | null; estimatedJevUsd: number | null; baselineUsd: number | null; routedUsd: number | null; savingsUsd: number | null; unpricedCases: number; meanJevMs: number; meanBaselineMs: number; meanRoutedMs: number; }; holdout: { split: "tuning" | "holdout"; cases: number; accepted: number; fallbackRate: number; selectiveAccuracy: number | null; baselineAccuracy: number; routedAccuracy: number; agreementWithBaseline: number | null; estimatedJevUsd: number | null; baselineUsd: number | null; routedUsd: number | null; savingsUsd: number | null; unpricedCases: number; meanJevMs: number; meanBaselineMs: number; meanRoutedMs: number; }; }[]; scoringDeviation: { passes: number; observations: number; answered: number; rawAccuracy: number; answerDisagreementRate: number | null; meanConfidence: number | null; meanWithinCaseConfidenceStdDev: number | null; meanWithinCaseScaleScoreStdDev: number | null; meanScaleMaeGold: number | null; meanScaleMaeBaseline: number | null; meanYesNoBrier: number | null; cases: { id: string; split: "tuning" | "holdout"; kind: "scale" | "yes-no" | "choices"; observations: number; answered: number; rawCorrect: number; nonModalAnswers: number; confidenceMean: number | null; confidenceStdDev: number | null; scaleScoreMean: number | null; scaleScoreStdDev: number | null; scaleScoreMin: number | null; scaleScoreMax: number | null; scaleMaeGold: number | null; scaleMaeBaseline: number | null; yesNoBrier: number | null; }[]; }[]; deviationDefinitions: { scaleScoreStdDev: string; confidenceStdDev: string; answerDisagreementRate: string; scaleMae: string; yesNoBrier: string; }; limitations: string[]; }; export declare function benchmarkDeviationComparison(report: ReturnType): string; export declare const MAX_BENCHMARK_BYTES: number; /** Captures contain values only: no paths, executable hooks, or provider settings. */ export declare function validateBenchmarkCapture(cases: BenchmarkCase[], value: unknown): asserts value is BenchmarkCapture; /** Stable by ID and seed, independent of file order and observed answers. */ export declare function splitBenchmarkHoldout(cases: BenchmarkCase[], fraction: number, seed?: string): BenchmarkCase[]; //# sourceMappingURL=benchmark.d.ts.map