/** * The document index — the append-only metadata log that records which * documents are indexed, at which version, and whether that is still true. * * The log (`.monomind/knowledge/doc-metadata.jsonl`) is the pipeline's source * of truth. Everything here reads or writes it: * * - WRITE: append a version record, tombstone a removed one. * - READ: the last-wins live view, the full history, and lookups by path, * canonical URL or `:` docId. * - JUDGE: which content hashes are current, which chunk keys are therefore * superseded, and which indexed files no longer exist on disk. * * Nothing here touches the vector store. Removal tombstones metadata and lets * the superseded filter hide the chunks — mark, don't destroy. * * Split out of document-pipeline.ts. * * @module v1/cli/knowledge/document-index */ import type { DocumentMeta, ReconcileReport } from './document-types.js'; export declare const METADATA_FILE = "doc-metadata.jsonl"; export declare function metadataPath(rootDir: string): string; /** Every record ever appended, in order, including superseded versions and * removal tombstones. Reads the path directly instead of via `metadataPath`, * which mkdir's. */ export declare function readMetadataLog(rootDir: string): DocumentMeta[]; export declare function readMetadata(rootDir: string): DocumentMeta[]; export declare function appendMetadata(rootDir: string, meta: DocumentMeta): void; /** * One ingest's live view of the metadata log — the answer to "what is already * indexed here", kept current as the ingest itself indexes things. * * WHY IT IS NOT AN ARRAY. `ingestDirectory` used to read the log once and hand * the same frozen snapshot to every file, which made a batch blind to its own * writes: two members of one capture envelope both looked new, both got * indexed, and a first-ever ingest of one page reported two documents and * `versions: 2`. Re-reading the log per file would have fixed that by throwing * away the very thing the cache exists for — the log is read once and parsed * once, however large it has grown. * * So the cache stays, and the WRITES go through it: `record` after a version * commits, `forget` after a tombstone, mirroring `readMetadata`'s last-wins * key. It is keyed by root as well, because a sweep can route captures to * several stores (`global`, `profile:`) and each has its own log. * * Deliberately not shared between ingests: it is a within-one-call view, not a * process-wide cache that could go stale behind a concurrent writer. */ export interface MetadataCache { /** The live record this ingest would supersede, by path or by capture * identity — the rule `ingestDocument` keys dedupe on. */ find(rootDir: string, filePath: string, scope: string, canonicalUrl?: string): DocumentMeta | undefined; /** A version just committed under `rootDir`. */ record(rootDir: string, meta: DocumentMeta): void; /** A record just tombstoned under `rootDir`. */ forget(rootDir: string, filePath: string, scope: string): void; } export declare function createMetadataCache(): MetadataCache; export declare function removeMetadataEntry(rootDir: string, filePath: string, scope: string): void; /** Content hashes of the documents currently indexed under `rootDir`. */ export declare function liveContentHashes(rootDir: string): Set; /** True when a metadata log exists under `rootDir`. * * An empty live-hash set has two very different causes: the log is missing (we * cannot judge what is current) or the log exists and every document has been * removed (nothing is current). Collapsing them made `doc remove` of the LAST * document a no-op — the tombstoned chunks came straight back in search. * * Reads the path directly instead of via `metadataPath`, which mkdir's. */ export declare function hasKnowledgeMetadata(rootDir: string): boolean; /** * True when `key` is a document chunk whose version is no longer current. * Non-`doc:` keys are never superseded. When no metadata is available nothing * is filtered, because "no metadata" must not read as "everything is stale". * * `metadataPresent` defaults to the old `live.size > 0` heuristic so existing * two-argument callers keep their exact behaviour; pass `hasKnowledgeMetadata` * to also filter correctly once the last document has been removed. */ export declare function isSupersededKey(key: string, live: Set, metadataPresent?: boolean): boolean; export declare function supersededOverfetchLimit(limit: number, live: Set): number; export declare function listDocuments(rootDir?: string, scope?: string): DocumentMeta[]; /** * Every recorded version of one document, oldest first (RCL-06). * * `target` is either an indexed file path or a capture's `canonicalUrl`. Read * from the append-only log rather than the last-wins view, which is how a * superseded version stays addressable after its record has been replaced or * tombstoned. * * Best-effort by design: `readMetadata` compacts the log once it passes 1MB * and keeps only live records, so history older than a compaction is gone. * The `supersedes` pointer on each record is the durable part. */ export declare function listDocumentVersions(rootDir?: string, target?: string, scope?: string): DocumentMeta[]; /** * The indexed record for whatever a caller wrote down: a file path, a * capture's `canonicalUrl` (fragment ignored), or the `:` docId. * * Live records win. A superseded version is only reached through the * append-only log, and only when nothing live matches — otherwise citing a * re-captured page would resolve against the version it replaced. */ export declare function findDocumentRecord(rootDir?: string, target?: string, scope?: string): DocumentMeta | undefined; export declare function removeDocument(filePath: string, scope?: string, rootDir?: string): Promise; /** * True for macOS AppleDouble sidecars (`._name`). * * Matches on the BASENAME PREFIX only. A legitimate document may contain `._` * elsewhere in its name (`v1._2-release.md`), or live under a dot-directory * that is deliberately indexed (`.monodesign/` critique snapshots), and * neither may be rejected. */ export declare function isResourceFork(filePath: string): boolean; /** * Reconcile the document index against the filesystem: find index entries whose * source file no longer exists and, only when explicitly asked, tombstone them. * * WHY — `removeDocument` only ever tombstoned metadata, and nothing has ever * compared the index against the disk, so a deleted file stayed searchable * forever. Measured 2026-07-28: 109 of 257 live entries (42.4%) had no file * behind them, including `docs/concepts/memory.md`. The Second Brain was * answering questions from documents the user had deleted. * * WHY IT IS THIS CAUTIOUS — "drop the index entry when the file is missing" is * a rule with a known catastrophic reading. A missing file is also an unmounted * volume, a checked-out branch, a partial clone, or a permissions failure. Two * guards were tried against real data and REJECTED; they are recorded here so * they are not re-proposed: * * - "abort if >50% of entries are missing" — the real, legitimate missing * fraction was 42.4%, so the threshold never fires in the one case we have. * Any threshold that would have blocked this reconcile is fitted to nothing. * - "only reconcile when the parent directory still exists" — 26 of the 109 * missing files had no parent directory, because `docs/concepts`, * `docs/adrs` and `docs/commands` were legitimately deleted wholesale. A * deleted directory and an unmounted volume are indistinguishable there. * * What does discriminate is the ROOT. An intact, readable root carrying a * metadata log means the tree is genuinely present, so a missing file is * genuinely gone. A missing root means nothing beneath it is knowable and * nothing may be removed — hence throw rather than reconcile. * * Removal tombstones metadata; it does not delete store rows. Chunks stay on * disk and fall out of search through the existing superseded filter, which * keeps this consistent with the mark-don't-destroy rule and leaves the whole * operation reversible from the archive. */ export declare function reconcileIndex(rootDir?: string, opts?: { scope?: string; apply?: boolean; }): Promise; //# sourceMappingURL=document-index.d.ts.map