/** * Segment writer + reader — single binary file per segment. * * File layout (all offsets written in the footer, so the postings stream * can be written sequentially without seeking back): * * [postings region] — concatenated VByte posting lists * [doc-length sidecar] — uint32 docCount, then [uint32 docId, uint32 length] * docCount * [tombstones region] — uint32 count, then uint32[count] sorted docIds * [term dictionary] — TermDict.serialize() output * [footer] — magic, version, offsets, CRC32s * * Footer layout (all little-endian uint32 unless noted): * 4 bytes magic = SEGMENT_MAGIC * 4 bytes version = SEGMENT_VERSION (2) * 4 bytes postingsOffset (always 0; retained for forward-compat) * 4 bytes postingsLength * 4 bytes sidecarOffset * 4 bytes sidecarLength * 4 bytes tombstonesOffset * 4 bytes tombstonesLength * 4 bytes dictOffset * 4 bytes dictLength * 4 bytes postingsCrc32 * 4 bytes sidecarCrc32 * 4 bytes tombstonesCrc32 * 4 bytes dictCrc32 * 4 bytes docCount * 4 bytes termCount * 64 bytes total footer size */ import type { StorageBackend, BlobWriteStream } from "./storage.js"; import type { Posting } from "./codec.js"; import type { DictEntry } from "./term-dict.js"; export declare class SegmentCorruptionError extends Error { readonly region: "postings" | "sidecar" | "tombstones" | "dict" | "footer"; constructor(region: "postings" | "sidecar" | "tombstones" | "dict" | "footer", detail: string); } /** * Streaming segment writer. Postings are encoded and written to the stream * immediately as each term arrives — no in-memory accumulation of the full * postings region. Only the term dictionary entries (one small record per distinct * term) and doc-length sidecar are held in memory. * * Usage: * const stream = await backend.createWriteStream(`${id}.seg`); * const writer = new SegmentWriter(stream); * // For each term in lex-sorted order: * await writer.writeTerm(term, sortedDocIds, sortedTfs); * // Set doc lengths (can be called before writeTerm calls): * writer.setDocLength(docId, length); * writer.setTombstones(sortedDocIds); * // Commit: * await writer.finish(); */ export declare class SegmentWriter { private readonly stream; /** Running CRC32 over the postings region — updated as each term is streamed. */ private readonly postingsCrc; /** Dictionary entries accumulated during writeTerm calls. */ private readonly dictEntries; /** Running byte offset within the postings region. */ private postingsOffset; /** * Packed sidecar: interleaved [docId, len, docId, len, ...] in uint32 pairs. * Grows with doubling-style reallocation. 8 bytes/doc vs ~64-80 for Map. */ private sidecarArr; private sidecarCount; private tombstonesArr; private lastTerm; constructor(stream: BlobWriteStream); /** * Write one term's postings. docIds MUST be in strictly ascending order (no duplicates). * Terms MUST be called in lex-sorted order (strictly ascending — no duplicates). */ writeTerm(term: string, sortedDocIds: number[], sortedTfs: number[]): Promise; setDocLength(docId: number, length: number): void; setTombstones(docIds: number[]): void; get termCount(): number; get docCount(): number; /** * Write sidecar, tombstones, term dictionary, footer, then commit the stream. * After finish() the stream is closed; do not call writeTerm after finish(). */ finish(): Promise; } /** * Reads an immutable segment file. Verifies CRC32 on open. * All region data is loaded eagerly (v0.1; lazy mmap is v0.2+). */ export declare class SegmentReader { private readonly postingsRegion; private readonly dict; /** * Interleaved sorted array: [docId0, len0, docId1, len1, ...]. * Sorted by docId ascending. Binary search via docLen(). * Replaces Map to avoid 1M+ slot heap allocation for large merged segments. */ private readonly docLenArr; /** Sorted tombstone docIds — docs that have been removed and belong to prior segments. */ readonly tombstones: Uint32Array; readonly docCount: number; readonly termCount: number; private constructor(); static open(path: string, backend: StorageBackend): Promise; /** Byte length of the postings region buffer held in memory. */ get postingsBytes(): number; /** Byte length of the doc-length sidecar Uint32Array held in memory. */ get docLenBytes(): number; /** Byte length of the tombstones Uint32Array held in memory. */ get tombstoneBytes(): number; /** Returns true if docId is tombstoned in this segment (binary search). */ isTombstoned(docId: number): boolean; /** Look up a term's metadata. Returns undefined if not present. */ lookupTerm(term: string): DictEntry | undefined; /** Lazy posting iterator for a term. Returns an empty iterator if term not found. */ postings(term: string): Iterator; /** Fully decode postings for a term — convenience for tests and scoring. */ decodePostings(term: string): { docIds: number[]; tfs: number[]; }; /** Return the stored document length for BM25 normalization (binary search). */ docLen(docId: number): number; /** Iterate all (docId, length) pairs in this segment's sidecar — for compaction. */ docLenEntries(): Generator<[number, number]>; /** All terms in sorted order (for compaction merging). */ terms(): Generator; } //# sourceMappingURL=segment.d.ts.map