/** * WHAT THE PROXY SAVED, IN TOKENS, measured rather than inferred. * * THE GAP THIS CLOSES. `grep -rn analytics src/proxy/*.ts` returned nothing: * the component responsible for the largest savings in the product contributed * no row to any savings surface, so `token-optimizer-savings` had to print a * line disclaiming its own scope. The proxy knew exactly what it had done -- * `beforeBytes`, `afterBytes` and the provider's own billed token count for * the rewritten body -- and none of it was ever expressed in the unit a bill is * denominated in. * * WHY BOTH SIDES ARE COUNTED LOCALLY. The provider bills for what we sent, so * the after-body has an exact, free, authoritative token count. The before-body * does not and never can: nobody billed for a request we did not make. The * honest measurement is therefore both sides under ONE instrument, which gives * a difference whose error largely cancels, plus the provider's count of the * after-body as an independent check on that instrument -- a calibration point * every single request supplies for free. That is strictly stronger provenance * than the MCP tool path, which has no external check at all. * * WHY A WORKER THREAD, which is not a shape used anywhere else in this package. * Tokenizing is linear and not cheap: measured at 0.33ms/KB, so 42ms for a * 128KB body and 165ms for a 512KB one. A proxy that blocked its event loop * for 165ms per request would add that to every concurrent request and stall * the streaming of any other response in flight -- it would make the product * slower to measure how much it had saved. The alternatives were all worse: an * inline count blocks, a chunked count miscounts at every seam it introduces, * and skipping the large bodies would silently drop exactly the requests that * matter most. * * AND THE WALL-CLOCK COST IS ZERO, because the count is started before the * request is forwarded and awaited in the usage tap -- it runs inside the * upstream round trip, which is always longer. * * A REFUSAL IS NAMED, NEVER A ZERO. Every path that cannot produce a count * says which one it was, and the record carries that word instead of a token * figure. A zero here would read as a request the proxy did not improve. */ /** * Why a request has no token figures. Each of these is a different operational * story and collapsing them into one "unavailable" would make the first * question about a gap in the ledger unanswerable. */ export declare const TOKEN_REFUSALS: Readonly<{ /** More counts were outstanding than the bound allows; this one was not run. */ readonly QueueFull: "queue-full"; /** The worker could not be started at all -- the count was never attempted. */ readonly WorkerUnavailable: "worker-unavailable"; /** The worker was started and then died or reported an error for this job. */ readonly WorkerFailed: "worker-failed"; /** Accounting was shut down while this count was outstanding. */ readonly ShutDown: "shut-down"; }>; export type TokenRefusal = (typeof TOKEN_REFUSALS)[keyof typeof TOKEN_REFUSALS]; /** * THE INSTRUMENT IS NAMED IN THE RESULT, matching what the MCP path records * under `tokenCountMethod`. A token figure whose encoder is unstated cannot be * compared with one from anywhere else, including a later version of this file. */ export declare const TOKEN_METHOD = "tiktoken-gpt-4-compatible-local-estimate"; export type TokenCountResult = { readonly measured: true; readonly beforeTokens: number; readonly afterTokens: number; readonly method: string; } | { readonly measured: false; readonly reason: TokenRefusal; }; /** A tokenizer that can be swapped for a synchronous one in tests. */ export interface TokenizerBackend { count(before: string, after: string): Promise<{ beforeTokens: number; afterTokens: number; }>; shutdown(): Promise; } /** * A count in flight, with its result readable synchronously once it arrives. * * WHY `settled` EXISTS. The ledger row was written synchronously the moment the * provider's usage settled, and attaching a token count must not quietly turn * that into a deferred write -- a row that lands a tick later is a row that is * not there yet for anything reading the file, and it reorders against rows * from paths that never await anything. In the normal case the count finishes * well inside the upstream round trip, so the caller can read it here and stay * synchronous; `done` is the fallback for the case where the round trip beat * the count, which a local upstream can do. */ export interface PendingCount { /** The result, or null while the count is still running. */ settled(): TokenCountResult | null; readonly done: Promise; } /** Wraps a count so its result can be read without awaiting. */ export declare function trackCount(count: Promise): PendingCount; export interface TokenAccounting { /** * Count both bodies under one instrument. Never throws, never blocks the * caller's loop for the duration of the count. */ countPair(before: Buffer | string, after: Buffer | string): Promise; /** * Starts the thread and the encoder before any request needs them. * * WITHOUT THIS THE FIRST PROXIED REQUEST PAYS FOR THE INSTRUMENT: measured at * ~190ms for thread start plus tiktoken's one-time table load, against ~32ms * to count a 216KB body afterwards. That is the one request guaranteed to * have its count arrive after its response, and on a short-lived proxy it * could be the only request there is. */ warmUp(): void; /** Counts refused so far, by reason, for whoever asks why the ledger has gaps. */ refusals(): Readonly>; shutdown(): Promise; } export interface TokenAccountingOptions { /** Swapped in tests; the default runs the count on a worker thread. */ readonly backend?: TokenizerBackend; readonly maxPending?: number; } export declare function createTokenAccounting(options?: TokenAccountingOptions): TokenAccounting; //# sourceMappingURL=token-accounting.d.ts.map