/** * Tuple fingerprints — Engine 5's data carrier. * * Engines 3 and 4 hunt duplicates by sorting the tuples THEMSELVES: text * records of eighty-odd characters, millions of JavaScript strings, each an * object the garbage collector has to track. That text is why the middle of a * big run is heavy — in scratch disk (5.8 GB of temp files for a 10 GB run), * in sort CPU (string comparisons), and in worker memory (chunks of live * strings plus the headroom V8 keeps for collecting them). * * None of it is needed to DETECT a duplicate. Detection only asks "are these * two the same?", and a hash answers that in 13 bytes: * * [hi 4B][lo 4B][row index 5B] big-endian, fixed width * * A 64-bit hash is not proof — two different tuples can collide — so a group * of records sharing a hash is a CANDIDATE, not a verdict. Candidates are few * (real duplicates plus a vanishing number of collisions: at ten million rows * the chance of even one false candidate is about 5e-6), and each is verified * by recomputing the actual tuples by row number, which the engine can do for * any row at any time. Verification makes the found duplicates EXACT — the * same rows engine 3's text sort names. * * Fixed width is its own reward, twice over. Sorting works on packed integer * arrays instead of strings — no allocation per record, no GC pressure, and a * pile of two million records is 26 MB instead of hundreds. And a record's * position in a file is `13 × ordinal`, so a sorted pile can be binary-searched * on disk: "is this tuple taken?" costs ~25 tiny reads and NO resident set at * all, which is what lets the repair run in bounded memory without the Bloom * filter's approximations. * * The 5-byte index covers 2^40 rows — a terabyte-scale run is ~10^10, three * orders of magnitude inside the limit. */ /** * How many piles for a run of `count` rows. * * Enough that a pile is a small fraction of the whole, capped so the write * buffers stay modest and the file count stays sane. A short run gets one pile * — the signal to stay on the exact text path, where hashing has nothing to * pay for itself with. */ export declare function bucketCountFor(count: number, cores: number): number; /** Bytes per record: 4 (hash hi) + 4 (hash lo) + 5 (row index). */ export declare const RECORD_BYTES = 13; /** Rows a 5-byte index can name. Checked at the door rather than wrapped silently. */ export declare const MAX_INDEX: number; /** The 64-bit fingerprint of a tuple key, as two 32-bit halves. */ export declare function hash64(key: string): { readonly hi: number; readonly lo: number; }; /** * Which pile a fingerprint belongs to. * * The same `h1 % buckets` the text piles use (`bucketOf` hashes the key with * cyrb128 and takes the first word), so the two routings agree — a tuple lands * in the same pile whichever carrier the engine runs on. */ export declare function fingerprintBucket(hi: number, buckets: number): number; /** Writes fingerprint records to a file, buffered. */ export declare class FingerprintWriter { private readonly fd; private buffer; private used; private written; constructor(path: string); write(hi: number, lo: number, index: number): void; /** Records written so far. */ get count(): number; private flush; close(): void; } /** * Reads fingerprint records back, buffered, without allocating per record. * After a successful `next()` the current record sits in `hi`/`lo`/`index`. */ export declare class FingerprintReader { private readonly fd; private readonly buffer; private have; private at; hi: number; lo: number; index: number; constructor(path: string); next(): boolean; close(): void; } /** * Sort any number of fingerprint files into ONE sorted file at `outPath`. * * External merge sort over fixed-width records: batches sorted in typed * arrays, then a k-way merge holding one record per run. Bounded memory at * any input size — a pile from a terabyte run sorts the same way a small one * does, just through more runs. * * Returns the record count, which the binary search needs to know the file's * extent without a stat-and-divide at every query. */ export declare function sortFingerprintFiles(inputs: readonly string[], outPath: string, tmpRoot?: string, batchSize?: number): number; /** * Row-index groups that share a fingerprint, from a SORTED file. * * These are candidates, not verdicts: a 64-bit collision between different * tuples would land here too, so the caller recomputes the true tuples for * every group and keeps only the rows whose tuples genuinely repeat. */ export declare function candidateGroups(sortedPath: string): Generator; /** * "Is this tuple already taken?" — answered by binary search on the sorted * pile files, in no memory at all. * * This is the disk-ledger idea in its final form: the sorted fingerprints ARE * the ledger, and a lookup is ~25 record-sized reads. The repair asks this a * few thousand times, so the read count is irrelevant next to what building an * in-RAM structure over every row would cost. * * Rows being reassigned by the repair have their old tuples freed, so a match * counts only if some matching record's row is NOT in `moving`. And a 64-bit * collision can only make the answer "taken" for a free tuple — the repair * then picks another combination; it can never hide a taken one. */ export declare class FingerprintLedger { private readonly sortedPaths; private readonly moving; private readonly fds; private readonly counts; private readonly probe; /** `sortedPaths[b]` must be pile `b`'s sorted file — lookups route by the pile hash. */ constructor(sortedPaths: readonly string[], moving: ReadonlySet); has(key: string): boolean; close(): void; } //# sourceMappingURL=fingerprint.d.ts.map