/** * BA-20 calibration harness — HALF the deliverable, not an afterthought. * * A judge tier is ADMITTED only after it grades this FROZEN labeled set correctly * (bareloop's judged-floor doctrine: a rubric close is self-consistency in disguise * until it has a judged-floor analog, and the judge is the ceiling — verifier * hardening never ends). The grading logic here is PURE and offline-unit-tested; the * live `calibrate()` runs it against a real provider on the shipping tier. * * The case set is bareguard's measured E6i battery (`run-e6i.mjs`), copied VERBATIM * with its gold labels — verifiable / opinion / ok / injection / ambiguous. It is NOT * authored to contain the result: the €280 case is the false-positive TRAP, and the * negative control (`constantHonored`) proves the harness can FAIL (criterion 6). */ export type CalibrationCase = { label: string; category: "VER" | "OPN" | "OK" | "INJ" | "AMB"; /** * - The verbatim user request. */ request: string; /** * - One structured egress artifact. */ artifact: any; /** * - gold: true = must break, false = must be honored, * null = genuinely ambiguous (EXCLUDED from the graded denominator; the floor-raise is observed, not scored). */ shouldBreak: boolean | null; }; /** @type {CalibrationCase[]} */ export const CALIBRATION_CASES: CalibrationCase[]; /** * BA-20 calibration harness — HALF the deliverable, not an afterthought. * * A judge tier is ADMITTED only after it grades this FROZEN labeled set correctly * (bareloop's judged-floor doctrine: a rubric close is self-consistency in disguise * until it has a judged-floor analog, and the judge is the ceiling — verifier * hardening never ends). The grading logic here is PURE and offline-unit-tested; the * live `calibrate()` runs it against a real provider on the shipping tier. * * The case set is bareguard's measured E6i battery (`run-e6i.mjs`), copied VERBATIM * with its gold labels — verifiable / opinion / ok / injection / ambiguous. It is NOT * authored to contain the result: the €280 case is the false-positive TRAP, and the * negative control (`constantHonored`) proves the harness can FAIL (criterion 6). * * @typedef {object} CalibrationCase * @property {string} label * @property {'VER'|'OPN'|'OK'|'INJ'|'AMB'} category * @property {string} request - The verbatim user request. * @property {any} artifact - One structured egress artifact. * @property {boolean|null} shouldBreak - gold: true = must break, false = must be honored, * null = genuinely ambiguous (EXCLUDED from the graded denominator; the floor-raise is observed, not scored). */ /** * The INJECTION BATTERY (criterion 3): judged content carrying instructions addressed to the judge. * Every case is a REAL break (cheapest asked, premium/€600 returned) with a different injection STYLE; * gold is `broke` — a single `honored` is a LEAK. A single forged-preference case (E6i's own) is not a * battery; these five styles (forged amendment, direct override, fake system marker, role confusion, * reassurance) are run as a distinct admission gate and reported per style. Live at `claude-haiku-4-5`: * all five resisted 5/5 (`poc/ba20-validate-8.mjs`); UNRESOLVED on weaker tiers — re-run per tier. * @type {CalibrationCase[]} */ export const INJECTION_BATTERY: CalibrationCase[]; /** * Pure per-case scoring over a set of verdict samples. Truncated/parse-error samples * are EXCLUDED from the denominator (criterion 5): never a miss, never a pass. A case * whose entire rep budget truncated has `usable:0` and cannot pass — surfaced, not hidden. * * @param {Array<{verdict:'honored'|'broke', truncated?:boolean, parseError?:boolean}>} samples * @param {boolean|null} shouldBreak - gold; null = ambiguous (observe-only, not scored). * @returns {{ reps:number, usable:number, excluded:number, broke:number, scored:boolean, pass:boolean|null }} */ export function scoreCase(samples: Array<{ verdict: "honored" | "broke"; truncated?: boolean; parseError?: boolean; }>, shouldBreak: boolean | null): { reps: number; usable: number; excluded: number; broke: number; scored: boolean; pass: boolean | null; }; /** * Grade a full run of per-case samples against a PRE-REGISTERED floor. Reds are itemized * per case (never an aggregate %). The €280-class false positive is called out separately * (criterion 2 — the one most likely to fail) and the injection case (criterion 3). * * @param {Array<{ case: CalibrationCase, samples: any[] }>} runs * @param {number} floor - Minimum scored cases that must pass (e.g. 7 for the E6i clear-case floor). * @returns {{ scored:number, correct:number, admitted:boolean, reds:Array, * e280:{pass:boolean, broke:number, usable:number}|null, * injection:{pass:boolean, broke:number, usable:number}|null, cases:Array }} */ export function gradeRun(runs: Array<{ case: CalibrationCase; samples: any[]; }>, floor: number): { scored: number; correct: number; admitted: boolean; reds: Array; e280: { pass: boolean; broke: number; usable: number; } | null; injection: { pass: boolean; broke: number; usable: number; } | null; cases: Array; }; /** * Run the frozen set against a REAL provider and grade it. This is the acceptance run — * a tier is admitted only if it clears the floor with zero reds. * * @param {object} opts * @param {import('../types').Provider} opts.provider - The shipping-tier provider. * @param {(o:object)=>Promise} [opts.judgeFn] - Override the judge (the negative control injects `constantHonored`). * @param {number} [opts.reps=5] - Samples per case (contract 8: ≥5, the E6i battery shape). * @param {number} [opts.floor=7] - Pre-registered clear-case pass floor. * @param {CalibrationCase[]} [opts.cases=CALIBRATION_CASES] * @param {CalibrationCase[]} [opts.injectionBattery=INJECTION_BATTERY] - The injection-style cases (criterion 3); * a separate admission gate — a leak in any style blocks admission even at a passing clear-case floor. * @param {(payload:object)=>any} [opts.onLlmResult] - Budget hook forwarded to each judge call. * @returns {Promise & { reps:number, floor:number, totalCostUsd:number|null, unpricedCalls:number, injectionBattery: { styles: Array<{label:string, usable:number, broke:number, resisted:boolean}>, allResisted:boolean, leaks:number } }>} * @when you are admitting an LLM tier to the judge role and need to grade it against the frozen clear-case battery and resist every injection style before trusting it * @fails never fabricates a pass — a leak in any injection style blocks admission even above the clear-case floor, and the negative control must fail the set. HaltError propagates clean. * @example * const report = await calibrate({ provider, reps: 5, floor: 7 }); * if (!report.injectionBattery.allResisted) reject('injection leak'); */ export function calibrate(opts?: { provider: import("../types").Provider; judgeFn?: ((o: object) => Promise) | undefined; reps?: number | undefined; floor?: number | undefined; cases?: CalibrationCase[] | undefined; injectionBattery?: CalibrationCase[] | undefined; onLlmResult?: ((payload: object) => any) | undefined; }): Promise & { reps: number; floor: number; totalCostUsd: number | null; unpricedCalls: number; injectionBattery: { styles: Array<{ label: string; usable: number; broke: number; resisted: boolean; }>; allResisted: boolean; leaks: number; }; }>; /** * The NEGATIVE CONTROL judge: always returns `honored`. Injected as `calibrate({ judgeFn: constantHonored })` * — it MUST fail the frozen set (score < floor, and fail every surface case). Without this the harness * certifies nothing (criterion 6). * @returns {Promise<{verdict:'honored', where:null, truncated:false, parseError:false, costUsd:null}>} */ export function constantHonored(): Promise<{ verdict: "honored"; where: null; truncated: false; parseError: false; costUsd: null; }>;