// canonical.ts — THE canonical-segmentation convention, defined once. // // The store's canonical reading of a byte stream — "the id of each single-byte // leaf, chained into flat branches" — is a WRITE/READ CONTRACT split across // four consumers that previously each re-implemented the chaining loop: // // WRITE side (training): // • learning.indexSubSpans — interns sliding windows of the two // {@link canonicalWindows} lengths over the stream's leaf ids; // • learning.deposit — interns the WHOLE stream as one flat branch // (via {@link leafIdPrefix}) so every deposit has a byte-level identity. // READ side (inference): // • recognition's canonical pass — chains leaf ids from every position and // probes each growing prefix as a flat branch, up to {@link chainReach}; // • attention.canonicalChunkId — reads a chunk's window of leaf ids // (via {@link leafIdRun}) and probes its prefixes for the best anchor. // (geometry.knownPrefixLength is the same convention below the store layer, // expressed through injected `leafAt`/`lookup` capabilities — it cannot // import this module without inverting the dependency spine, so its header // cross-references this file instead.) // // If the write side's windows or the read side's reach ever change, they must // change HERE — a drift between them silently makes recognition stop finding // what training indexed, which no type checker catches. import type { MindContext } from "./types.js"; /** The two sliding-window lengths the WRITE side interns over a stream's leaf * ids: W−1 and W (the river's grouping quantum and its off-by-one neighbour, * so a form straddling a group boundary is reachable from either cut). * * Widening this to cover the reader's full segment scale (W−1..2W) was * measured and REFUTED: it tripled the store (3,160 → 8,980 nodes on a * 200-pair corpus), slowed ingest 80%, and fixed not one test. Unknown * regions are not caused by the index's scale. */ export function canonicalWindows(W: number): [number, number] { return [W - 1, W]; } /** The READ side's chain reach: how many leaf ids a canonical chain may grow * to from one position — W², the deepest two-level composite the write side's * windows can spell. */ export function chainReach(W: number): number { return W * W; } /** The id of the single-byte leaf at position `p`, or null when that byte was * never interned. */ export function leafIdAt( ctx: MindContext, bytes: Uint8Array, p: number, ): number | null { return ctx.store.findLeaf(bytes.subarray(p, p + 1)); } /** The leaf ids of the window `[from, to)`, or null the moment ANY byte in it * is unknown — the all-or-nothing read {@link canonicalChunkId} anchors on. */ export function leafIdRun( ctx: MindContext, bytes: Uint8Array, from: number, to: number, ): number[] | null { const ids: number[] = []; for (let i = from; i < to; i++) { const lid = leafIdAt(ctx, bytes, i); if (lid === null) return null; ids.push(lid); } return ids; } /** The leaf ids of the LONGEST KNOWN PREFIX of `bytes` — stops at the first * unknown byte and returns what it has (possibly empty). The caller decides * what a partial prefix means (deposit only interns the whole-stream flat * branch when the prefix covers everything). */ export function leafIdPrefix( ctx: MindContext, bytes: Uint8Array, ): number[] { const ids: number[] = []; for (let i = 0; i < bytes.length; i++) { const lid = leafIdAt(ctx, bytes, i); if (lid === null) break; ids.push(lid); } return ids; } /** Which prefixes of `prefix ‖ tail` are STORED NODES — as lengths in the * tail's own coordinates, ascending, excluding the empty one. This is the * candidate set a rule needs to join an already-stored prefix to a suffix it * has not stored: a key names a relation exactly when `prefix ‖ tail[0..p]` IS * a node, and a key can end strictly inside the tail without sitting on any * fold boundary (a stored member's end is the end of ITS OWN stream, and the * fold never emits a cut at a stream's end). Measured: "stockholm mayor" * exists, leads on, and its boundary 6 is in neither the tail's cuts nor the * concatenation's. * * ONE cheap content-addressed probe per offset — `leafIdPrefix` walks the bytes * once (a point probe each), `findBranch` hashes the growing kid run — and NO * `resolve`, which is what keeps this off the O(suffix) vector folds the * recognition path pays. It stops at the first byte that was never interned, * which costs nothing real: a stored key's bytes are interned by construction. */ export function keyEnds( ctx: MindContext, prefix: Uint8Array, tail: Uint8Array, ): number[] { if (prefix.length === 0 || tail.length === 0) return []; const joined = new Uint8Array(prefix.length + tail.length); joined.set(prefix, 0); joined.set(tail, prefix.length); const ids = leafIdPrefix(ctx, joined); if (ids.length < prefix.length) return []; const ends: number[] = []; // The kid run GROWS by one id per offset; `findBranch` wants an array, so the // run is built once and pushed into, never re-sliced. Re-slicing // `ids.slice(0, prefix.length + p)` per offset made this O(|tail| · // (|prefix| + |tail|)) — quadratic in the tail, where the learning path this // follows slices a run that SHRINKS. Same ends, linear copying. const run = ids.slice(0, prefix.length); // The loop ENDS at the first byte that was never interned (`ids.length`): // every later prefix contains it, so none of them can be a node either — this // is where the scan stops, not a silent truncation of the answer. for (let p = 1; prefix.length + p <= ids.length; p++) { run.push(ids[prefix.length + p - 1]); if (ctx.store.findBranch(run) !== null) ends.push(p); } return ends; } /** The canonical W-window node ids of a byte stream, offset → id — the * CONTENT-ADDRESSED IDENTITY of every W-sized slice, under which any content * two deposits share IS the same node (hash-consing paid the comparison at * write time). The read side of the {@link canonicalWindows} contract asked * at every offset: a window over unknown bytes, or one never interned as a * flat branch, has no identity and is skipped. Confluence's meet and CAST's * frame detection both read shared/common content through this — never * through a byte scan. */ export function windowIds( ctx: MindContext, bytes: Uint8Array, ): Map { const W = ctx.space.maxGroup; const out = new Map(); for (let off = 0; off + W <= bytes.length; off++) { const ids = leafIdRun(ctx, bytes, off, off + W); if (ids === null) continue; const wid = ctx.store.findBranch(ids); if (wid !== null) out.set(off, wid); } return out; }