/** * TermLog — high-level facade over SegmentManager. * * Adds string↔number docId mapping, automatic tokenization, and BM25 search. * The SegmentManager, BM25Ranker, etc. are still exported for advanced callers. * * The docId mapping is persisted as an append-only journal (`docids.log`) with * periodic snapshots (`docids.snap`). The log is appended before each manifest * commit so the two are always consistent. On close the log is collapsed into * the snapshot. */ import type { StorageBackend } from "./storage.js"; import type { Tokenizer } from "./tokenizer.js"; export declare class MappingCorruptionError extends Error { readonly detail: string; constructor(detail: string); } export declare class TokenizerMismatchError extends Error { readonly persisted: string; readonly runtime: string; constructor(persisted: string, runtime: string); } /** * Thrown when opening with `agentId` against a directory that still contains a * legacy single-writer `manifest.json` (no per-agent manifests yet). The caller * must explicitly migrate via `TermLog.migrateLegacy(dir, agentId)` before * retrying the open. Migration is one-time per index. */ export declare class LegacyIndexError extends Error { readonly dir: string; constructor(dir: string); } /** Thrown when an agent's local numId counter would overflow its assigned slot range. */ export declare class SlotExhaustedError extends Error { readonly agentId: string; readonly slot: number; constructor(agentId: string, slot: number); } export interface TermLogOptions { dir: string; backend?: StorageBackend; /** * Enable multi-writer mode. When set: * - Files are namespaced per agent (`manifest-.json`, * `seg--NNNNNN.seg`, `docids-.snap/.log`, `.lock-`). * - The agent's numId range is partitioned via the `slots.json` registry so * two writers never allocate colliding numIds. * - The reader view is the union of this agent's commits + every other agent's * committed manifests in the same directory. * * Without an agentId, the manager runs in legacy single-writer mode using * `manifest.json` and a single `.lock`. The two modes are exclusive within * one directory: opening with `agentId` against a legacy index throws * `LegacyIndexError` until `TermLog.migrateLegacy()` is called. */ agentId?: string; tokenizer?: Tokenizer; flushThreshold?: number; /** Size-tiered compaction fanout — how many same-tier segments trigger a merge. Default 4. */ fanout?: number; k1?: number; b?: number; } export declare class TermLog { private readonly mgr; private readonly tokenizer; private readonly backend; private readonly k1; private readonly b; private readonly dir; private readonly agentId; /** numId slot assigned by `slots.json`; undefined in legacy single-writer mode. */ private readonly slot; /** Highest valid numId for this agent's slot (exclusive upper bound). */ private readonly slotMaxExclusive; /** Own (writable) string docId → numeric docId. */ private readonly strToNum; /** Own (writable) numeric docId → string docId. */ private readonly numToStr; /** * External agents' string → numId mappings, keyed by agentId. Read-only * snapshot loaded on open(); refresh() rebuilds it. Used to (a) tombstone * external docs when the same string is re-added by this agent, and (b) * resolve search hits whose numId lives in an external slot. */ private externalStrToNum; /** External numId → string docId (mirror of `externalStrToNum`). */ private externalNumToStr; /** Own monotonically increasing numId allocator within the assigned slot. */ private nextNumId; /** Pending log lines (adds/removes) not yet flushed to docids-*.log. */ private readonly pendingLog; /** Serializes add/remove — guards strToNum/numToStr/pendingLog and coordinates with mgr. */ private _lock; private serialize; private constructor(); static open(opts: TermLogOptions): Promise; /** * One-time migration from legacy single-writer layout to multi-writer. * Renames `manifest.json`, every `seg-NNNNNN.seg`, `docids.snap`, and * `docids.log` to the agent-suffixed equivalents and rewrites the manifest's * segment ids in place. Assigns the migrating agent slot 0. * * Idempotent: if `manifest-.json` already exists, returns silently. * Throws if any non-own per-agent manifest exists (multiple agents have * already been writing — caller must not call migrateLegacy in that case). */ static migrateLegacy(dir: string, agentId: string): Promise; /** Re-scan external manifests + docids files; pick up commits from other agents. */ refresh(): Promise; private loadDocIds; /** * Load all other agents' docids snapshots + logs into the external mapping * tables. Called at open() in multi-writer mode and again by refresh(). Quiet * on per-agent failures so a corrupt sibling doesn't block this writer. */ private loadExternalDocIds; /** Resolve a string docId to all known numeric IDs (own + external). */ private resolveAllNumIds; /** Resolve a numeric docId to its string form, checking own + external maps. */ private resolveNumIdToStr; /** Append pending deltas to docids-*.log (called by onBeforeManifest — O(delta) not O(N)). */ private saveDocIds; /** Write a full snapshot and delete the log (called on close/compaction). */ private snapshotDocIds; /** Add or update a document. Tokenizes text and indexes it. */ add(docId: string, text: string): Promise; /** Remove a document by its string docId. Idempotent. */ remove(docId: string): Promise; /** * Search for documents matching a query string. * Returns results sorted by BM25 score descending. */ search(query: string, opts?: { limit?: number; mode?: "and" | "or"; }): Promise>; /** Explicitly flush the write buffer. Auto-flush still triggers on threshold. */ flush(): Promise; /** Merge segments. Snapshots docIds after compact to bound log growth. */ compact(): Promise; /** Close: flush pending writes and release the advisory lock. */ close(): Promise; /** Number of indexed documents (flushed to segments). */ docCount(): number; /** Number of active segments. */ segmentCount(): number; /** * Approximate in-memory footprint in bytes. * * Sums: * - SegmentManager: postingsRegion buffers + docLen/tombstone Uint32Arrays + * TermDict entry objects + write buffer terms + pendingTombstones Set. * - TermLog facade: strToNum and numToStr Maps — estimated at 80 bytes per * entry (V8 Map node overhead + two small integers or a short string). * * The result is a lower-bound approximation suitable for memory-budget * decisions; it deliberately undercounts V8 object header overhead. */ estimatedBytes(): number; } //# sourceMappingURL=termlog.d.ts.map