/** * Turning recorded rows into the three output-savings tiers. * * WHERE THIS SITS. `src/proxy/output-savings.ts` is the arithmetic -- pure * accumulators, strata, two estimators and an interval -- and knows nothing * about a ledger row. This file is the reading: it decides which rows belong in * which arm of which comparison, and it is the one place that decision is made, * so the live ledger and a day already folded into totals cannot disagree about * it. * * TWO COMPARISONS, NOT ONE, because there are two interventions and conflating * them would publish a figure about neither: * * - COMPRESSION, which every request gets. Rows we declined to compress are * an observational baseline against rows we did. Nobody randomized this, so * the two groups differ in ways besides the treatment -- the declined rows * are declined BECAUSE they were small or unusual -- and the figure is * labelled estimated for exactly that reason. * * - THE OUTPUT SHAPER, which withholds itself from a random fraction of * conversations. That fraction is a real control arm, so the difference * between the arms is a measurement. It exists only when an operator turned * the holdout on, and the tier says nothing at all when they did not. * * STRATIFIED ON PRE-TREATMENT FEATURES ONLY. The size bucket is taken from the * bytes the request arrived with, never the bytes we forwarded: a stratum that * depended on the compressed size would put every effective compression in a * smaller bucket than the baseline it is compared against, and the comparison * would measure its own bucketing. That error produces a large, stable, * entirely fictional saving, and nothing in the output looks wrong. * * BOUNDED BY CONSTRUCTION, which is what makes it safe to carry across a * retention fold. Every component of a stratum key comes from a fixed * vocabulary -- ten model families, four turn kinds, five size buckets, tools * or not -- so there are at most 400 strata per arm no matter how long the * proxy runs. Three numbers each; a day's folded output ledger cannot grow * without limit the way a per-request store could. * * COUNTS ONLY, NEVER CONTENT. A stratum key is four words from fixed lists and * an accumulator is three numbers. No prompt, no reply, no conversation key, * no path. */ import { type Accum, type OutputSavingsEstimate, type OutputSavingsLedger, type OutputWaste } from '../proxy/output-savings.js'; import type { AccountingRecord } from '../proxy/accounting.js'; /** * The two comparisons, kept apart. * * `compression` USES baseline-vs-treatment; `shaper` USES control-vs-treatment. * The unused map in each stays empty, and that is deliberate: one structure * with a spare slot is easier to fold, serialize and validate than two shapes, * and an estimator pointed at the wrong map returns nothing rather than a * plausible number from the wrong rows. */ export interface OutputLedgers { readonly compression: OutputSavingsLedger; readonly shaper: OutputSavingsLedger; /** * The deferral holdout, which is on the OTHER SIDE OF THE BILL. * * ITS UNIT IS PROMPT TOKENS, NOT OUTPUT TOKENS, and that is the one thing a * reader of this structure has to keep straight: the figure it produces must * never be added to either of the two above. It lives here because it is the * same shape, folds the same way and is pruned by the same policy -- a * control-vs-treatment comparison over pre-treatment strata -- and because a * second parallel set of files to serialize, validate and retain would be a * second thing to get wrong for no gain. * * WHY IT IS MEASURED AND NOT COMPUTED. Deferral's saving lands in the prompt * the provider assembles, which we never see. The computed figure counts the * schemas we marked and is right about them, but it cannot count the search * schema the provider expands server-side, so it is an upper bound. This * comparison takes both arms' billed prompt tokens from the provider's own * usage and is therefore net of everything, including that. */ readonly deferral: OutputSavingsLedger; /** * Tier 3, which needs no strata at all. * * ONE ACCUMULATOR, NOT A MAP, BECAUSE THERE IS NOTHING TO COMPARE AGAINST. * The other two tiers stratify so that like is compared with like across two * arms. An echo ratio has no second arm -- it is a property of one response -- * so stratifying it would only split a mean into means nobody asked for. */ readonly echo: Accum; } export declare function emptyOutputLedgers(): OutputLedgers; export declare function mergeOutputLedgers(into: OutputLedgers, from: OutputLedgers): void; /** * Folds one recorded row into both comparisons. * * A ROW WITH NO REPORTED OUTPUT IS NOT A ROW WITH ZERO OUTPUT. A refused * request, a transport failure, a response whose usage never arrived: none of * them tell us how much the model wrote, and entering them as zero would pull * both arms' means toward nothing in proportion to how often each arm failed. * They are skipped, which is the only honest treatment of a missing * measurement. * * THE SHAPER COMPARISON ONLY SEES ROWS THAT CARRY AN ARM, and rows carry one * only while a holdout is configured. So a ledger written with no experiment * running contributes nothing to the measured tier, and the measured tier * reports nothing -- rather than reporting a null result for a trial that was * never run. */ export declare function recordOutputRow(ledgers: OutputLedgers, record: AccountingRecord): void; /** * The two estimated tiers, each from the comparison that can support it. * * `measured` IS NULL WHENEVER NO HOLDOUT RAN, and the renderer prints nothing * for it. `estimated` always has a value once any row carried an output count, * because there is always a declined-compression group to compare against -- * though it may be a thin one, which is what the interval and * `pooledRequests` are for. */ export interface OutputTiers { readonly estimated: OutputSavingsEstimate; readonly measured: OutputSavingsEstimate | null; /** * Tier 3. Null unless the opt-in scanner ran, and NEVER a token figure -- it * is a share of a reply, carried in its own units so nothing can add it to a * saving. */ readonly waste: OutputWaste | null; } export declare function outputTiers(ledgers: OutputLedgers): OutputTiers; /** * The deferral holdout's figure, in PROMPT tokens -- deliberately not a tier. * * IT IS NOT A MEMBER OF `OutputTiers` AND MUST NEVER BECOME ONE. Every number * in that interface is output tokens; this one is input tokens, and the two sit * on opposite sides of the bill at different prices. Returning it separately is * the type system saying what a comment would only ask: nothing can add this to * an output saving by accident, because there is no field to add it into. * * WHY IT EXISTS BESIDE THE COMPUTED PROMPT DELTA. We count the deferred schemas * exactly -- we hold them -- so the computed prompt-level delta is arithmetic, * not an estimate. What arithmetic cannot establish is whether the provider * honoured the beta and actually left those schemas out of the context it * billed. This figure is the difference between the billed `input_tokens` of * conversations whose tools were deferred and those randomly withheld from the * feature, so it comes from the provider's own meter and answers exactly that. * * NULL UNLESS A HOLDOUT RAN, which is the normal case: a holdout makes the * control arm pay for schemas the treatment arm does not, so it is off by * default and the computed figure has to stand without it. */ export declare function deferralTier(ledgers: OutputLedgers): OutputSavingsEstimate | null; /** * One arm's strata as a plain object, for a rollup file. * * THREE NUMBERS PER STRATUM, IN AN ARRAY, because a rollup line is written once * per day per ledger and read on every report; `[n, sum, sumsq]` is a third the * bytes of three named fields and carries exactly the same information. The * order is fixed by `TRIPLE` below and validated on the way back in. */ export type SerializedArm = Record; export interface SerializedOutputLedgers { readonly compression: { readonly baseline: SerializedArm; readonly treatment: SerializedArm; readonly control: SerializedArm; }; readonly shaper: { readonly baseline: SerializedArm; readonly treatment: SerializedArm; readonly control: SerializedArm; }; readonly deferral: { readonly baseline: SerializedArm; readonly treatment: SerializedArm; readonly control: SerializedArm; }; /** * The echo accumulator as one `[n, sum, sumsq]`, omitted when nothing was * scanned -- so a day from a build or an operator without the scanner carries * no field at all rather than a zero that would read as "no waste found". */ readonly echo?: readonly number[]; } export declare function serializeOutputLedgers(ledgers: OutputLedgers): SerializedOutputLedgers; /** * Both ledgers back from a rollup line, or null when the line is not one. * * AN ABSENT FIELD IS AN EMPTY LEDGER, NOT A REFUSAL, so a rollup written before * this existed still folds -- it simply contributes no output rows, which is * the truth about it. A field that is PRESENT and malformed is refused, because * then something did write output figures and this build cannot read them. */ export declare function parseOutputLedgers(value: unknown): OutputLedgers | null; //# sourceMappingURL=output.d.ts.map