/** * Cross-block dedup: the same bytes, sent twice, charged twice. * * WHY THIS IS THE BIGGEST REMAINING WIN, and why the per-block engines cannot * find it. Every engine here is a pure function of ONE block, so it cannot see * that the file it is compressing is the same file the agent read nine turns * ago. An agentic coding session repeats itself constantly and for good * reasons: read a file, edit it, read it back; run the tests, fix, run them * again; grep, follow a hit, grep the same pattern from a different directory. * The second copy is pure cost -- it teaches the model nothing the first copy * did not already say. * * THE REFERENT IS INSIDE THE REQUEST, WHICH IS THE WHOLE POINT. * * HeadRoom dedups too -- a content hash naturally collapses repeats -- but * their marker points OUT of the payload, at a cache entry keyed by that hash. * When the entry is missing the model receives `<> [unresolved: * entry not found]` (their #2509), a dead token mid-context. A back-reference * here points at bytes that are still in the very request being sent. It cannot * miss, because there is no lookup: the content the model needs is above. * * That is why these elisions are `lossless: true` with no `recoverAt`. The rule * is not "we hope it is retrievable" -- it is that the output alone fully * determines what was removed, which is exactly what `lossless` means. * * TWO KINDS OF REPEAT, IN ORDER OF VALUE: * * verbatim the referent was never rewritten -- it sits before the cache * frontier, or nothing claimed it -- so the later copy can be * dropped whole and never compressed at all. This is the case * that pays best: a file re-read after an edit whose earlier copy * is already in the cached prefix costs nothing on the wire. * * compressed the referent was rewritten, so the later copy is compared * AFTER compression. Identical compressed forms mean the later * block would have contributed nothing new, and the reference is * still exact -- it names bytes the model can read above. * * WHAT IS DELIBERATELY NOT DONE: near-duplicate matching. A file read before * and after an edit is NOT the same file, and the difference is precisely what * the agent is looking at. Collapsing "almost the same" would hide the edit -- * a silent, confident wrong answer, which is worse than the tokens it saves. * Only exact equality dedups here. */ import type { Elision, Stamp } from './types.js'; /** * Below this a reference cannot pay for itself -- derived, not chosen. * * THE OLD NUMBER WAS A GUESS, AND IT WAS WRONG BY A FACTOR OF FOUR. It read 600 * on the strength of a comment estimating the marker at "about 130 characters". * Measured across the twelve benchmark workloads, the markers actually emitted * run 31 to 83 characters, median 59. So the floor is computed instead: the * widest marker this module can render, quoting a full `QUOTE_CHARS`, doubled. * A block is pointed at only when the pointer costs at most half of it. * Marginal savings are still refused -- now at the size where they are in fact * marginal, rather than four times above it. * * A PRE-FILTER, NOT THE GUARANTEE. It bounds the marker by its widest possible * form and by a byte count of zero, both of which flatter a real block. * `worthPointingAt` weighs the marker that will actually be emitted. */ export declare const MIN_DEDUP_BYTES: number; /** One block on its way through a strategy. */ export interface DedupBlock { /** The text as it stands now -- already compressed, if it was going to be. */ readonly text: string; /** The text before any engine touched it. */ readonly original: string; /** * May this block be rewritten at all? False for a signed message or content * behind the cache frontier, both of which must stay byte-identical. */ readonly touchable: boolean; /** * The key `text` already decodes under -- whatever engine compressed it. * * CARRIED THROUGH RATHER THAN RECOMPUTED. A literal block leaves this pass * untouched, and the caller still has to decode the markers inside it, so the * stamp it arrived with is the one that comes back out on `stamps`. Omitted * means the text carries no marker of ours -- an untouchable block, which is * the original bytes. */ readonly stamp?: Stamp; } export interface DedupResult { /** One text per input block, in order. */ readonly texts: readonly string[]; readonly elisions: readonly Elision[]; /** * One key per output text, aligned index for index with `texts`. * * ALIGNED, NOT A LIST OF THE ONES THAT EXIST, because the decoder consumes it * positionally: `rehydrateSequence` is handed one block at a time with the * stamp for that block, and a reference resolves by rebuilding a block from * further up the array with ITS stamp. A compacted list would silently shift * every key one position left of the text it belongs to. * * At a literal this is the stamp the block arrived with; at a reference it is * the one this pass minted for the marker it wrote. */ readonly stamps: readonly Stamp[]; } /** * The referent a back-reference names, or null when the line is not one. * * THE INVERSE LIVES BESIDE THE ENCODER, for the reason `rehydrate` gives for * centralising the envelope: a hand-written inverse in another file drifts from * the grammar it is supposed to invert, and one that has drifted into being too * forgiving passes everything. The three markers above are the whole grammar, * and this is where they are read back. * * NOT A LOSSY MARKER, WHICH IS THE WHOLE POINT OF READING IT BACK. The content * is in the same request, above -- that is exactly the claim made below when * these elisions are recorded `lossless: true` with `recoverAt: null`. A * decoder shown one block in isolation cannot check that claim, so it refused, * and three by-design references sat on a list of suspected data loss. */ export interface BackReference { /** * The quoted opening with its ellipsis stripped, so it is a prefix of the * referent's reader head. Null on the cheap repeat form, which carries a * label and nothing else. */ readonly needle: string | null; /** The ordinal in `(#n)` or `as #n above`, or null where there is none. */ readonly label: number | null; /** * True on the run form, which names its referent by ORDER rather than by * quote: the block after the one the preceding reference resolved to. It * carries neither a needle nor a label, so a reader with no preceding * reference cannot resolve it -- and must refuse rather than guess. */ readonly follows: boolean; } export declare function readBackReference(line: string, stamp?: Stamp): BackReference | null; /** * The one block above whose head this quote names, or null when it names no * single one. * * FAILING CLOSED IS THE POINT. `quoteFor` widens a quote until exactly one * block above answers to it, so zero matches or two mean the output and this * reader disagree about what is above -- and a decoder that picked one anyway * would be vouching for a reconstruction it did not make. * * Duplicates collapse first because the encoder compared against a Set: two * byte-identical untouchable blocks are one rival to it, and counting them as * two here would refuse a reference that is perfectly well defined. */ export declare function findReferent(needle: string, above: readonly string[]): string | null; /** * The key a reference marker written by this pass will carry. * * MINTED HERE ONLY WHEN THE CALLER DID NOT CHOOSE. `withStamp` promises that * every engine entry point honours a stamp the caller pinned, and that `null` * means `do not stamp`; this pass is reached from the strategies rather than * from the router, so it has to keep that promise itself. It minted * unconditionally until the comparator needed a reproducible run and found that * the stamp it had pinned was being ignored on exactly this path. * * `undefined` means the caller did not think about it, and gets a key over the * bytes the reference replaced -- stable turn over turn for the same block in * the same conversation, which is what keeps a cached prefix matching. */ export declare function dedupBlocks(blocks: readonly DedupBlock[], stamp?: Stamp): DedupResult; //# sourceMappingURL=dedup.d.ts.map