/** * Calibration harness + judge economics (OK-9 W6/W7; evidence base * research/2026-08-18-switchyard-routing-fusion-deep-dive.md [DD]). * * QUADRANT METHOD (Switchyard, DD §2): pair pure-capable and pure-efficient * counterfactual runs by task id, then sweep the corroborative escalation * threshold. Quadrants are named from the router's efficient-first operating * point so Switchyard's selection rule reads literally: * RESCUE — efficient-fail ∩ capable-pass (escalation rescues the task) * LOSS — efficient-pass ∩ capable-fail (escalation loses quality) * SAFE — both pass · HARD — both fail * (The deep-dive's parentheticals record the same four sets from the capable * arm's perspective; the rule — "the lowest threshold that rescues RESCUE * without over-escalating LOSS" — only operates from the router's side.) * DD's caveat applies: in-router efficient outcomes inherit capable-arm * context, so counterfactual pure-efficient runs are the honest arm, and any * measured RESCUE rate is a bound, not a point estimate. * * CPT/APGR REPORT (RouteLLM's evaluation frame, DD §5): per threshold, the * strong-call fraction (x) against the quality gap closed between the * efficient-only and capable-only baselines (y). * * JUDGE BREAK-EVEN (OK-9.4; LangChain's formula, DD §1): a routing judge * pays for itself when offloaded fraction × (dear − cheap) ≥ judge cost, * i.e. breakEven = judgeCost / (dearCost − cheapCost), computed from live * catalogue rates ($/1M tokens). A judge whose break-even ≥ 1 can never pay * for itself — the posture default serves instead. */ import type { RoutingEvent } from "../shift/activity.js"; import type { Tier } from "../shift/tier.js"; /** One recorded calibration run: one task, one tier, one outcome. */ export interface CalibrationRun { taskId: string; /** The tier arm this run served. */ tier: Tier; /** * The corroborative scorer value recorded for the task at routing time * (0..1) — the quantity the threshold sweeps against. */ score: number; outcome: "pass" | "fail"; } export type CalibrationQuadrant = "rescue" | "loss" | "safe" | "hard"; /** Per-threshold row of the sweep (the CPT/APGR report line). */ export interface CalibrationRow { threshold: number; /** Paired tasks escalated to the capable tier at this threshold. */ escalated: number; /** RESCUE-quadrant tasks escalated (rescued) at this threshold. */ rescued: number; rescueTotal: number; /** LOSS-quadrant tasks escalated (over-escalated) at this threshold. */ overEscalatedLoss: number; lossTotal: number; /** SAFE/HARD escalations — pure cost, no quality effect either way. */ safeEscalated: number; hardEscalated: number; /** CPT x-axis: fraction of paired tasks served by the strong tier. */ strongCallFraction: number; /** Pass rate of the routed mix at this threshold. */ routedQuality: number; /** * APGR y-axis: fraction of the efficient→capable quality gap the routed * mix closes. Undefined when the arms tie (no gap to close). */ qualityGapClosed: number | undefined; } export interface CalibrationReport { /** Tasks present in BOTH arms — the only tasks that enter a quadrant. */ paired: number; capableRuns: number; efficientRuns: number; /** Task ids seen in only one arm (reported, excluded from quadrants). */ unpairedCapable: string[]; unpairedEfficient: string[]; quadrants: Record; /** Pass rates over the paired set (the APGR baselines). */ capableQuality: number; efficientQuality: number; rows: CalibrationRow[]; recommendedThreshold: number | undefined; recommendationReason: string; } /** * The swept candidates (OK-9 W6): 0.3..0.8 in 0.05 steps. The band brackets * Switchyard's shipped 0.5 (one maxed signal ≈ 0.4621) on both sides. */ export declare const CALIBRATION_THRESHOLDS: readonly number[]; /** Minimal shape check — corrupt or schema-drifting lines are skipped. */ export declare function isCalibrationRun(value: unknown): value is CalibrationRun; /** * Read a JSONL file of {@link CalibrationRun} records. One corrupt line is * skipped, never fatal — same posture as the fusion runs log. A missing file * reads as empty. */ export declare function readCalibrationRuns(logPath: string): Promise; /** * Same read, with the skip count surfaced: a default runs file full of * off-shape lines looks identical to an empty one without it (E017 review). */ export declare function readCalibrationRunsDetailed(logPath: string): Promise<{ runs: CalibrationRun[]; skipped: number; }>; /** Split pooled records into the two arms by their recorded tier. */ export declare function splitCalibrationArms(runs: readonly CalibrationRun[]): { capableRuns: CalibrationRun[]; efficientRuns: CalibrationRun[]; }; /** * The quadrant table + threshold sweep + recommendation. Pure over the run * records: same input → same report. * * Pairing: the last record per task id within an arm wins (re-runs supersede). * The task's routing score comes from the capable arm when present — that is * the in-router arm — else from the efficient counterfactual. */ export declare function runCalibration(input: { capableRuns: readonly CalibrationRun[]; efficientRuns: readonly CalibrationRun[]; thresholds?: readonly number[]; }): CalibrationReport; /** Catalogue rates in USD per million tokens (the pi-ai ModelCostRates shape). */ export interface ModelRates { input: number; output: number; } /** Token estimate for one call, used to turn catalogue rates into per-call cost. */ export interface CallTokenEstimate { inputTokens: number; outputTokens: number; } /** A short classification verdict: prompt + bounded reply. */ export declare const DEFAULT_JUDGE_TOKENS: CallTokenEstimate; /** A mid-size agentic turn (tool transcript in, patch out). */ export declare const DEFAULT_CALL_TOKENS: CallTokenEstimate; /** Cost in USD of one call at catalogue rates ($/1M tokens). */ export declare function callCostUsd(rates: ModelRates, estimate: CallTokenEstimate): number; /** Any catalogue model with pricing satisfies this (pi-ai Model included). */ export interface PricedModel { id: string; cost: ModelRates; } export interface JudgeBreakEvenInput { judge: PricedModel; cheap: PricedModel; dear: PricedModel; judgeTokens?: CallTokenEstimate; callTokens?: CallTokenEstimate; } export interface JudgeBreakEven { judgeCostUsd: number; cheapCostUsd: number; dearCostUsd: number; /** * LangChain's break-even: judgeCost / (dearCost − cheapCost) — the fraction * of calls that must offload to the cheap tier for the judge to pay for * itself. Undefined when the tier gap is not positive (economics * unmeasurable — a dear tier priced at or below cheap). */ breakEven: number | undefined; /** True when the judge can ever pay for itself (breakEven defined and < 1). */ viable: boolean; } /** * The judge break-even meter (OK-9.4): computed from live catalogue pricing, * so a provider repricing flips the verdict without a code change. */ export declare function judgeBreakEven(input: JudgeBreakEvenInput): JudgeBreakEven; /** * The one-line judge-economics reading (report line and event reason share * this wording so the feed and the artefact never disagree). */ export declare function judgeBreakEvenLine(be: JudgeBreakEven, pair: { cheap: string; dear: string; }): string; /** * The activity-feed form of the break-even meter (OK-9 W7): emitted through * the same redacting sink as every routing event when a fusion run resolves * its judge. The judge arbitrates merges, so it rides the review stage. */ export declare function judgeBreakEvenEvent(be: JudgeBreakEven, judgeModelId: string, pair: { cheap: string; dear: string; }): RoutingEvent; /** * Render the quadrant table + threshold sweep + recommendation as plain * lines (the CLI prints them; the dated record file embeds them). */ export declare function renderCalibrationReport(report: CalibrationReport, options?: { breakEven?: JudgeBreakEven; pair?: { cheap: string; dear: string; }; }): string[]; //# sourceMappingURL=calibrate.d.ts.map