/** * Per-request token accounting, so the cost of compressing can be attributed * rather than argued about. * * WHY THIS EXISTS. The first end-to-end measurement of the proxy showed four * THOL tasks taking 36 turns against control's 35 -- effectively no turn cost -- * while costing $0.58 against $0.43. Turns barely moved and money moved a lot, * which means the difference is per-turn token spend rather than extra work. * Nothing in the rig could say which kind of token: a cache read bills at 0.1x, * a cache write at 1.25x, and a plain input token at 1.0x, so a strategy can cut * the token COUNT and raise the BILL by moving tokens between those classes. * * The benchmark models that split from recorded payloads. It cannot observe it, * because only the provider says what it actually charged. This reads the * `usage` the provider reports and writes it beside what compression did to the * same request, so the two can be joined per turn. * * OFF UNLESS ASKED FOR. It writes a file, and a proxy that writes files nobody * requested is a proxy nobody should run. `TOKEN_OPTIMIZER_PROXY_ACCOUNTING` * names the path; absent, nothing here does anything. * * FAILS OPEN, ALWAYS. This is instrumentation. An optimizer that wedges the * agent is worse than one that saves nothing, and that goes double for a * measurement it was not asked to take. */ import type { OutputArm } from './output-savings.js'; import type { Readable } from 'node:stream'; /** The token classes a provider bills separately. */ export interface RequestUsage { input_tokens?: number; output_tokens?: number; cache_creation_input_tokens?: number; cache_read_input_tokens?: number; /** Responses input_tokens includes this subset; do not add it a second time. */ cached_input_tokens?: number; } /** * Pulls usage numbers out of a response fragment, keeping the LAST value seen * for each key. * * Last-wins is the correct rule rather than a convenience. A streaming response * reports input and cache tokens once in `message_start` and then reports * `output_tokens` again in every `message_delta`, cumulatively -- so the final * occurrence is the total and an earlier one is a partial count. */ export declare function scanUsage(text: string, into: RequestUsage): void; /** What compression did to one request, as the summary already reports it. */ export interface CompressionFacts { readonly compressed: boolean; /** * Which arm of the output-shaper holdout the request was in, when one ran. * * THE ONLY FIELD HERE THAT IS NOT A FACT ABOUT THE INPUT, and it is here * because the question it answers cannot be answered anywhere else. The * shaper withholds itself from a fraction of conversations so its effect on * what the model WRITES can be measured rather than estimated, and that * measurement is a difference between the two arms' output token counts -- * which only this row holds. A label recorded without the count, or a count * recorded without the label, each make the experiment unreadable. * * ABSENT WHEN NO EXPERIMENT RAN, never defaulted to an arm. Every request * being labelled "treatment" would read as a trial whose control arm came * back empty, rather than as a trial that was never started. * * NOT REQUEST CONTENT. It is one of two fixed words, derived from a hash of * a conversation key that is itself never recorded. */ readonly outputArm?: OutputArm; /** * Which arm of the TOOL-DEFERRAL holdout the request was in, when one ran. * * THE OTHER SIDE OF THE BILL FROM `outputArm`, and here for the same reason. * Deferral's saving lands in the prompt the PROVIDER assembles, which we * never see; all we see is what it billed for one. So the only proof the * saving is real is the difference between the billed prompt tokens of * conversations whose tools were deferred and those whose were not -- and * that difference needs the label and the usage on one row, which is this one. * * ABSENT WHEN NO EXPERIMENT RAN, never defaulted to an arm, and NOT request * content: one of two fixed words from a hash of a key that is never recorded. */ readonly deferralArm?: OutputArm; readonly reason?: string; readonly anchorReason?: string; readonly elisions?: number; /** References actually forwarded by the Responses deduplicator. */ readonly dedupReferences?: number; /** * Whole units this request moved out of the body. * * HERE, AND NOT ON AccountingRecord, which is the lesson of three wrong * turns. The record extends this interface, so a field declared here arrives * there; declared there it is populated by nothing, because the proxy builds * a CompressionFacts and the record is assembled from it. * * The value is the proxy's per-request spill delta, taken from the wrapper * built beside the sink at startup. It must NOT be obtained by wrapping the * sink again: compress/types.ts:229 memoises what each sink has spilled in a * WeakMap keyed on the sink function, so a per-call wrapper presents a fresh * key, empties the memo and re-spills the same blocks every turn. Measured, * that read 845 against a true 75 -- a plausible figure rather than an error, * which is the dangerous kind. * * It exists so an operator running the withholding arm can see how much it * held back. Without it `src/savings/` reports nothing about the arm that * measures 2.81x against their 1.69x on what a subscription cap buys. */ readonly withheldUnits?: number; readonly deferredTools?: number; readonly deferredToolChars?: number; /** * NOT HERE. Withheld and reinstated counts live on the telemetry rollup, not * on this record, and this is where I put the declaration by mistake. * * `reinstatedUnits` was declared on this type and populated nowhere: the * value is passed to `noteRequest` in server.ts, which builds a * `RequestFacts` for the rollup, and `spilledBlocks` goes to the same place. * So the field here read as recorded and never was -- the exact failure mode * reproducibility.mjs refuses a record for, since a field that is present and * unusable is worse than an absent one. * * It matters because the two sinks are not interchangeable. The rollup is * where `spilled` and `reinstated` are counted; `src/savings/` reads THIS * record from the accounting ledger and can see neither. So the savings * report cannot show an operator how much the withholding arm held back or * how often it came back -- the two numbers that decide whether the arm is * paying for them. * * AND THEY DO NOT BELONG ON THIS TYPE EITHER. This record is assembled by * spreading a `CompressionFacts` into it (server.ts:1553), so a count has to * be on THAT type to arrive here -- which is also why the compiler rejected * the field when I first tried to populate it: the object I was writing to is * an `Omit`, not this. The next attempt starts on * CompressionFacts, carrying the per-request spill delta, and arrives here for * free. * * AND IT MUST NOT BE DONE BY SPREADING THE RESULT. I tried exactly that -- * wrap compressBody's sink to tally the calls, then return * `{ ...result, summary: { ...result.summary, withheldUnits } }` -- and the * proxy's own `spilledBlocks` figure went from 75 to 845 over the same 1,323 * requests, close to the 854 elisions. Wrapping a sink only delegates, so * that change should have been count-neutral and was not. It is reverted, and * the cause is not established: either `spill` is invoked per elided block * rather than per whole-unit move, or copying the result object changes * something downstream that depends on its identity. * * BISECTED SINCE, and it is the wrapper alone. Applying only the sink wrapper * -- counting into a variable that is then discarded, no result copied, no * field added -- still moves the figure from 75 to 845. The result spread is * innocent. And `spillTo` returns a plain arrow with no attached properties, * so the engine cannot be reading anything off the sink that a wrapper drops. * * What a known-answer test DID settle (tests/unit/proxy/spill-counts-what.test.ts): * one sink call is one whole-unit move, not one elided fragment, and a move is * also counted as an elision. Four incompressible messages give four calls and * four elisions. So the replay's 854 elisions against 75 spills means most of * its elisions are in-place -- and a delegating wrapper cannot turn an * in-place elision into a move. The mechanism is still unexplained. * * EXPLAINED. compress/types.ts:229 holds `const SPILLED = new WeakMap()` -- a memo of what each sink has already moved out, keyed on the SINK * FUNCTION ITSELF. The proxy builds its sink once per proxy, so that memo * survives every request and a block already spilled is not spilled again: * 75. My wrapper was built fresh inside each `compressBody` call, so every * request presented a new key, the memo was empty each time, and the same * blocks moved out over and over: 845. The 11x was the deduplication being * defeated, exactly once per request. * * So a sink's IDENTITY is load-bearing, and anything that wraps one -- to * count, to log, to test -- must be hoisted to the sink's own lifetime or it * silently turns spill dedup off. That is also why the count belongs where the * proxy already puts it: `spilledBlocks`, incremented in the wrapper created * beside the sink at startup, IS the per-request withheld count and is * correct. The only thing missing was ever carrying it onto CompressionFacts * so the savings ledger could see it. An 11x move in a measurement from a * refactor that cannot affect it means the measurement was not understood, * and shipping a report built on it would publish that misunderstanding. */ /** * Characters of cached knowledge added to the request. * * RECORDED BECAUSE IT IS THE ONE THING HERE THAT MAKES A REQUEST BIGGER, * and it was the one fact the ledger did not carry. The proxy printed * `+1927 injected` to its log while the ledger line beside it said nothing, * so an A/B of the knowledge block read its own effect as zero and would * have reported the feature inert. A cost that only appears in a log a * measurement does not parse is a cost nobody attributes. */ readonly injectedChars?: number; readonly systemChars?: number; readonly toolsChars?: number; readonly toolCount?: number; readonly coreToolChars?: number; readonly mcpToolChars?: number; readonly topTools?: string; readonly messagesChars?: number; readonly messageCount?: number; /** * The model the request names, when it names one. * * THE ONE FIELD THAT MAKES A ROW PRICEABLE. Every other number here is a * size, and a size cannot be turned into money: the provider's catalog is * keyed by model, so a ledger row without one can report tokens avoided and * never a dollar figure. The MCP path already carries a model for exactly * this reason and the proxy path -- the component responsible for the larger * saving -- carried none, so its rows could only ever have been reported * unpriced. * * AN IDENTIFIER, NOT CONTENT. It is a value chosen from the provider's own * published catalog, in the same class as the tool names already recorded * here, and it is read from the PARSED request rather than matched out of * the raw bytes. No part of the conversation, system prompt or tool schema * is retained with it -- which is what keeps the always-on ring and the * loopback `/__token-optimizer/transformations` endpoint as safe as they * were before this field existed. */ readonly model?: string; readonly beforeBytes: number; readonly afterBytes: number; } /** * What the request cost in tokens on each side, or why that is not known. * * BYTES ARE NOT THE UNIT A BILL IS DENOMINATED IN. `beforeBytes` and * `afterBytes` are exact and free, and they are the wrong unit: a provider * charges per token, and the ratio between the two is not constant across * JSON structure, prose and code. A savings figure computed from bytes is a * proxy for the thing the user actually pays, and this field is the thing * itself. * * COUNTS ONLY. No body, no fragment of one, and no text of any kind appears * here -- that constraint is what lets the transformations ring and the ledger * stay always-on rather than opt-in like capture. * * THE PROVIDER'S OWN COUNT STAYS WHERE IT IS, in `usage`. Keeping our estimate * of the after-body separate from the provider's billed count for that same * body is what makes every single request a free calibration of this * instrument; merging them would throw that away. */ export type TokenAccountingFacts = { readonly measured: true; /** Our estimate for the body we would have sent. Nobody billed for it. */ readonly beforeTokens: number; /** Our estimate for the body we did send; compare with `usage`. */ readonly afterTokens: number; /** The encoder, so this figure can be compared with any other. */ readonly method: string; } | { readonly measured: false; /** * Why there is no figure. A NAMED REASON, NEVER A ZERO: a zero in a * savings column reads as a request the proxy did not improve, which * would be a false measurement rather than a missing one. */ readonly reason: string; }; /** One line of the ledger: what we sent, and what it was billed as. */ export interface AccountingRecord extends CompressionFacts { /** Monotonic durations. Upstream includes transport and provider processing. */ readonly timing?: { readonly transformMs: number; readonly upstreamHeadersMs?: number; readonly upstreamMs: number; }; readonly ts: string; readonly path: string; readonly status: number; /** No HTTP response was received; usage remains unknown, not zero. */ readonly transportError?: string; readonly usage: RequestUsage; /** * Tokens before and after, under our own encoder -- absent entirely on a * proxy built without token accounting, so an old ledger line stays valid. */ readonly tokens?: TokenAccountingFacts; /** * The share of this reply's word n-grams that were already in the context. * * OUTPUT WASTE, WHICH IS NOT A SAVING. The other output figures in this * feature are differences between two populations. This one has no * counterfactual at all -- it is a property of a single response, and it says * how much of the reply the model had already been shown. A high ratio is an * opportunity to attack, never a token anyone saved, and nothing may add it to * one. * * ABSENT WHEN NOT SCANNED, which is a different fact from zero. The scan is * opt-in, refuses a context too short or too large to stand behind, and * refuses a body it could not finish reading -- so a missing field means "no * figure", while a `0` means "measured, and nothing was echoed". * * A RATIO, NEVER THE TEXT IT CAME FROM. See `./echo.ts`. */ readonly echoRatio?: number; } /** The ledger path, or null when accounting was not asked for. */ export declare function accountingPath(env?: NodeJS.ProcessEnv): string | null; /** * Appends one record. Swallows every failure by design -- see the file header. */ export declare function appendRecord(path: string, record: AccountingRecord): void; /** * Watches a response stream for usage numbers without touching what it carries. * * OBSERVES, NEVER INTERPOSES. Attaching a `data` listener does not consume the * stream in flowing mode, so the existing `pipe` still delivers every byte * unchanged -- which matters more here than anywhere, because an SSE stream has * to arrive as it is produced and this is a byte-faithful proxy. * * BOUNDED. The parser retains only bounded usage objects and structural state, * never the assistant's response content. */ export declare function tapUsage(stream: Readable, done: (usage: RequestUsage) => void, contentEncoding?: string, /** * An extra reader of the decoded response text, for an instrument that needs * the words rather than the usage numbers. * * SHARES THE ONE DECODE. The tap already decompresses the response to find * the usage object, and a second consumer attaching its own decoder would * double that cost on the one stream the proxy must not slow down. It is * handed the same decoded text, in order, and whatever it throws is swallowed * for the same reason everything else here is -- the response has already * been delivered. */ watch?: (text: string) => void): void; //# sourceMappingURL=accounting.d.ts.map