/** * What keeps the savings record from growing without a bound, and without lying. * * THE PROBLEM IT SOLVES. The proxy appends one JSONL line per request for as * long as a ledger is configured, and nothing ever removed one. A daemon left * running writes until the disk says no, and the component that noticed first * was whatever else needed that disk. * * WHY NOT SIMPLY DELETE THE OLD LINES. Every figure the savings report prints * is a fold over these rows, so dropping a row silently reduces a number an * operator reads as their saving -- the report would get quieter the longer it * ran and say nothing about why. So a line is never dropped before what it * contributes has been added to a durable per-day total, and the report folds * those totals in alongside the live rows. * * WHY A DAY IS THE GRAIN, and why that is exact rather than approximate: every * window the report prints opens at the start of a local day (`windows.ts`), so * a whole local day is either entirely inside a window or entirely outside it. * There is no day to split and no boundary to round, which is the property that * makes a pre-folded day indistinguishable from the rows it came from. * * THE WINDOW IS DERIVED, NOT CHOSEN. It is the longest window the report shows * plus a margin, read off `windowBoundaries` rather than written down here -- * so a fifth window added to the report cannot quietly start reading from a * retention policy that already threw its rows away. */ import { type SerializedOutputLedgers } from './output.js'; import type { AccountingRecord } from '../proxy/accounting.js'; /** * Days kept past the longest window the report shows. * * The grain makes the boundary exact, so this is not correcting for rounding. * It buys the things that only raw rows can answer: a per-request breakdown at * the edge of the longest window, a clock that moved, and an operator looking * at yesterday's ledger by hand after reading today's report. */ export declare const RETENTION_MARGIN_DAYS = 5; /** * How many days of raw rows the report itself needs. * * READ OFF THE WINDOWS, by measuring the oldest dated boundary they open at. * `All time` has no boundary and asks for nothing: it is the window the folded * totals serve. */ export declare function longestReportWindowDays(now?: Date): number; /** The oldest local day whose raw rows are kept. Everything older is folded. */ export declare function retentionDays(now?: Date): number; /** * The byte ceiling the age window is backed by. * * AN AGE WINDOW ALONE IS NOT A BOUND. It bounds how long a row survives, not * how many arrive: a single heavy day of proxied traffic can outgrow a month of * a quiet one, and the file only shrinks when that day ages out. 16 MiB is * around forty thousand rows at the length these records run to, which is more * traffic than a day of agent work produces and small enough that nobody * notices it on a disk. Past it, the oldest days are folded early -- the same * operation the window performs, applied for a different reason, so there is * only one path that ever removes a row. */ export declare const MAX_LEDGER_BYTES: number; /** * The millisecond upper bounds of the transform-latency histogram. * * A HISTOGRAM, BECAUSE A SUM CANNOT ANSWER THE QUESTION. The figure an operator * needs beside a saving is not the average transform cost but the one a slow * request pays, and a quantile is not recoverable from a sum and a count. A * fixed set of bounds is, because adding two days' bucket counts element-wise * gives exactly the counts the two days' rows would have produced -- so a * quantile read off a folded day is the same quantile, to the width of its * bucket, that the rows themselves would have given. * * SPACED FOR OUR OWN HALF OF THE TRADE, not for the provider's. Transforming a * request is sub-millisecond to tens of milliseconds work, so the resolution * sits there; the long tail is kept coarse because past a few hundred * milliseconds the only fact that matters is that it happened. * * THE LAST BUCKET IS UNBOUNDED, and is reported as a bound that was exceeded * rather than as a number, so a pathological request can never be published as * though it had been measured at 5000 ms. */ export declare const TRANSFORM_MS_BOUNDS: readonly number[]; /** Which bucket a duration falls in. Never out of range: the last is open. */ export declare function transformBucket(ms: number): number; /** * A quantile of a bucketed set of transform durations. * * REPORTED AS AN UPPER BOUND, NOT AS A VALUE. The histogram knows that a * request fell between two bounds, not where between them, so the honest figure * is the bound: "95% of transforms finished within 20 ms". Interpolating inside * the bucket would invent a precision the fold threw away, and the invented * digits would be the ones an operator quoted. * * `exceeded` MARKS THE OPEN BUCKET, so the slowest requests are never published * as having taken the last bound. The caller renders that as a floor. * * NULL WHEN NOTHING WAS TIMED, so a build that wrote no timings reads as * unknown rather than as instant. */ export declare function transformQuantile(buckets: readonly number[], fraction: number): { readonly ms: number; readonly exceeded: boolean; } | null; /** * One set of running totals: the only shape a folded day is ever stored in. * * DEFINED HERE, AND USED BY THE READER, so the two cannot drift. A field the * report starts accumulating but the rollup does not store would make the * folded half of the record quietly shorter than the live half, and the seam * would sit exactly where nobody looks -- at the far edge of the longest * window, in the figure labelled "all time". * * MUTABLE BY DESIGN: this is a fold. */ export interface RollupTotals { requests: number; /** Whole units the withholding arm moved out of these requests. */ withheldUnits: number; billedRequests: number; countedRequests: number; pricedRequests: number; calibratedRequests: number; tokensSaved: number; tokensBefore: number; cost: number; oursTokens: number; billedTokens: number; /** * Rows that carried a timing block, which is not the same as `requests`. * * COUNTED SEPARATELY SO AN UNTIMED ROW IS NEVER A ZERO. A proxy built before * the timings existed writes rows with no `timing` at all, and averaging * those in as zero milliseconds would publish the one latency claim nobody * can defend: that the transform was free. */ timedRequests: number; /** Milliseconds we spent rewriting, summed over `timedRequests`. */ transformMs: number; /** Milliseconds the provider took, summed over the same rows. */ upstreamMs: number; /** The slowest single transform in the day. Exact across a fold, via MAX. */ transformMsMax: number; /** Transform durations bucketed by `TRANSFORM_MS_BOUNDS`, so a quantile folds. */ transformBuckets: number[]; } export declare function emptyTotals(): RollupTotals; /** Adds one set of totals into another. The folded half's only arithmetic. */ export declare function addTotals(into: RollupTotals, from: RollupTotals): void; /** * A day of rows, pre-folded, as it is stored and as the report reads it back. * * `records` IS NOT DERIVABLE FROM `totals`. The report distinguishes a row that * was never charged for from one that was charged but carried no count, and a * row that contributed a signed delta from one that contributed zero -- and the * last of those cannot be recovered from any sum. They are stored. * * `byModel` HOLDS ONLY THE ROWS THE BREAKDOWN HOLDS, which is the rows with a * non-zero delta, exactly as the live reader buckets them. Storing every row * here would give the folded days a longer model table than the live ones. */ export interface RollupRow { readonly kind: 'proxy-day-rollup'; readonly version: 1; /** The local calendar day, `YYYY-MM-DD`. */ readonly day: string; readonly records: { readonly total: number; readonly measured: number; readonly unbilled: number; readonly uncounted: number; readonly skippedLines: number; }; readonly totals: RollupTotals; readonly byModel: Readonly>; /** * The day's output-savings strata, so a folded day keeps its full weight. * * NOT DERIVABLE FROM `totals`, which is why it is here. The output tiers are * differences of means with an interval, and an interval needs the spread * the rows had -- `n`, `sum` and `sumsq` per stratum per arm. A day reduced * to its means would still produce a point estimate and would produce it * with no band, so the first prune would quietly turn every past day's * output figure into a number with no stated uncertainty. * * BOUNDED, so this cannot grow the file without limit: every part of a * stratum key comes from a fixed vocabulary, which caps a day at 400 strata * per arm however long the proxy ran -- see `src/savings/output.ts`. * * ABSENT ON A ROLLUP WRITTEN BEFORE THIS EXISTED, which folds as an empty * ledger rather than being refused: that day contributed no output rows, * which is the truth about it. */ readonly output?: SerializedOutputLedgers; } /** * Whether a parsed line is a rollup this build can read back. * * EVERY FIELD IS CHECKED, not just the tag. A rollup is a number the report * prints without any row behind it to sanity-check against, so a half-written * or future-shaped one has to be refused and counted rather than folded in as * a partial day -- which would read as a quiet week. */ export declare function looksLikeRollup(value: unknown): value is RollupRow; /** * Folds one record into one set of totals. * * SHARED WITH THE READER rather than copied into it. This is the arithmetic * that decides what a saving is, and the folded days and the live rows have to * agree on it exactly or the seam between them becomes a step in the figure. * * `tokensBefore` COMES FROM THE SAME ROWS AS `tokensSaved`, which is the rule * `windows.ts` already follows: a percentage whose numerator is the measured * rows and whose denominator is every row would fall as more unmeasurable * traffic arrived, reading as the proxy getting worse at the moment it was * being told less. */ export declare function foldRecord(totals: RollupTotals, record: AccountingRecord): void; /** Where a ledger's folded days live: beside it, never inside it. */ export declare function rollupPath(ledger: string): string; export interface PruneLimits { /** Days of raw rows to keep. Defaults to `retentionDays(now)`. */ readonly retentionDays?: number; /** Ceiling the kept rows are brought under. Defaults to `MAX_LEDGER_BYTES`. */ readonly maxBytes?: number; } export interface PruneOutcome { /** Whether anything was folded. A no-op prune rewrites nothing. */ readonly changed: boolean; readonly bytesBefore: number; readonly bytesAfter: number; readonly linesKept: number; readonly linesFolded: number; readonly daysFolded: number; /** Lines that were not a record, folded as skipped so the count survives. */ readonly linesSkipped: number; /** Why the fold happened, for the caller that reports it. */ readonly reason: 'age' | 'bytes' | 'none'; } /** * Folds the ledger's old days into its rollup file and rewrites what is left. * * THE ROLLUP IS WRITTEN BEFORE THE LEDGER IS REWRITTEN, and the order is what * makes a crash survivable. Stopping between the two leaves a day present in * both files, and the reader resolves that by preferring the live rows and * ignoring the rollup for any day it has rows for -- so the worst a crash * costs is a rollup row nobody reads. Writing the ledger first would instead * lose the day outright, which is the failure that shows up as a quiet week. * * BOTH WRITES GO THROUGH A TEMPORARY FILE AND A RENAME, so a reader never sees * half of either file, and a prune that dies mid-write leaves both as they * were. * * CALLED FROM THE WRITER, never beside it. See `forEachLine`. */ export declare function pruneProxyLedger(ledger: string, now?: Date, limits?: PruneLimits): PruneOutcome; //# sourceMappingURL=retention.d.ts.map