/** * TextLineIndex * * A literal-text line index kept **separate** from the symbol (call-graph / * signature) index in `vector-index.ts`. It stores raw lines of walked files — * markup, stylesheets, templates, plain text, and the non-symbol remainder of * code files — so that literal strings the user can see on screen (UI copy, * error messages, hard-coded labels) are findable even when they live in static * markup that extracts no symbols (e.g. a "Message completed" banner in * index.html). * * Design (decision fd256fde): * - **Separate LanceDB table** (`text_lines`), never the call graph. Text lines * are never nodes and never contribute to fanIn/fanOut, hubs, entrypoints, * communities or PageRank — graph purity by construction, not by per-call-site * filtering. * - **BM25-only, no embeddings.** Literal lookup wants exact lexical match, not * vector similarity; this keeps build cost and index size bounded and results * deterministic. * - Reuses the BM25 machinery already in `vector-index.ts` * (`buildBm25Corpus` / `tokenize` / `bm25Score`) rather than reimplementing it. * * Storage: /text-line-index/ (LanceDB database folder) * Table name: "text_lines" */ /** One indexed line of a text file. */ export interface TextLineRecord { /** `${filePath}:${lineNumber}` — unique per line. */ id: string; filePath: string; /** 1-based line number. */ lineNumber: number; /** The raw line text (truncated if very long). */ text: string; } export interface TextSearchResult { filePath: string; lineNumber: number; text: string; /** BM25 relevance score, higher = more relevant. */ score: number; } /** A file to index: its repo-relative path and full content. */ export interface TextFileInput { filePath: string; content: string; } /** * Prefix of a staging directory. Exported so a leaked one can be recognised: the build stages the * new index here and renames it into place, so a process killed mid-build (an OOM-kill, a Ctrl-C) * leaves a directory the size of a full index behind. */ export declare const STAGING_PREFIX = "text-line-index.building-"; /** * Test-only: lower the flush threshold so a small fixture exercises the MULTI-BATCH path. * * Without this a test would need ~500k lines to cross the production threshold twice, because a * file's lines are appended whole before the threshold is checked. A fixture that never crosses * it twice silently tests only the single-flush path — which is exactly how a mutation that * re-created the table on every flush (clobbering earlier batches) passed a first draft of these * tests. Returns the previous value so callers can restore it. */ export declare function _setBuildFlushLinesForTesting(n: number): number; /** Test-only: clear the in-memory BM25 cache to force the cold path. */ export declare function _resetTextLineIndexCachesForTesting(): void; /** * Split a file into indexable line records. Blank / whitespace-only lines are * skipped; over-long lines are truncated, never dropped. */ export declare function extractLines(filePath: string, content: string): TextLineRecord[]; export declare class TextLineIndex { /** * Remove staging directories left by builds that died before they could rename or clean up. * * Only directories whose owning process is gone are removed — a staging directory belongs to a * live build until it is renamed, and a build on a large repository legitimately runs for a long * time, so age alone is not a safe signal. Best effort: a sweep that cannot run must never stop * an analysis. */ static sweepLeakedStaging(outputDir: string): Promise; /** Returns true if a text-line index has been built for this output dir. */ static exists(outputDir: string): boolean; /** * Build (or rebuild) the text-line index from a set of files. Overwrites any * existing table. Files that yield no indexable lines contribute nothing. * Returns the number of lines indexed. * * `files` may be an array OR an async iterable, and the async form is the one that matters at * scale: this used to materialize ONE RECORD OBJECT PER SOURCE LINE for the entire repository * before handing the whole array to LanceDB. On a large repository that is millions of live * objects on top of every file's text, and it was the point at which `openlore install` ran out * of heap — measured on microsoft/TypeScript (80,113 files), which died here after the call * graph and the keyword index had both completed successfully. * * Records are flushed every {@link BUILD_FLUSH_LINES} lines, so peak residency is one batch * rather than the whole corpus. Passing an async iterable additionally lets the CALLER avoid * holding every file's content at once; an array argument keeps working unchanged. * * The build is ATOMIC, and that is not incidental. Flushing incrementally into the live table * would mean the first flush destroys the previous index and every later failure leaves a * TRUNCATED one — verified by killing a build mid-flush: the old rows were gone, some new ones * were present, and `exists()` still reported a healthy index, so the watcher would have gone * on patching a permanently partial corpus forever. It also meant a concurrent reader saw a * half-built index for the whole build, and two concurrent builds failed outright on a LanceDB * commit conflict. * * So the batches go into a private per-process directory and the finished index is moved into * place with a single rename. A build that throws, is killed, or races another build leaves the * previous index untouched; the only observable states are "the old index" and "the new one". */ static build(outputDir: string, files: Iterable | AsyncIterable): Promise<{ lines: number; files: number; }>; /** * Incrementally update the index for changed and deleted files. Changed files * have their old lines replaced; deleted files have their lines removed. The * cached BM25 corpus is patched in place. No-op if the index does not exist. */ static updateFiles(outputDir: string, changed: TextFileInput[], deletedPaths?: string[]): Promise<{ lines: number; }>; /** * BM25-only search over the text lines. Returns up to `limit` `file:line` * matches ordered by relevance. Optionally restrict to a single file. */ static searchText(outputDir: string, query: string, opts?: { limit?: number; filePath?: string; }): Promise; /** * Patch the cached BM25 corpus: drop rows for `affectedPaths`, splice in * `newRecords`, rebuild the corpus. No-op when nothing is cached (the next * search rebuilds from the table). */ private static _patchCache; } //# sourceMappingURL=text-line-index.d.ts.map