/** * Document Pipeline — wires text extraction, chunking, embedding, and SQLite storage * into an end-to-end ingest/search/export pipeline for the Second Brain. * * @module v1/cli/knowledge/document-pipeline */ export interface IngestResult { filePath: string; chunksIndexed: number; scope: string; skipped: boolean; error?: string; } export interface BatchIngestResult { filesProcessed: number; filesSkipped: number; totalChunks: number; errors: string[]; results: IngestResult[]; } export interface KnowledgeExcerpt { /** Memory entry id — pass back to memory_feedback/bridgeApplyFeedback to rate usefulness. */ id: string; filePath: string; text: string; similarity: number; chunkIndex: number; scope: string; /** True when this chunk belongs to a document version that has since been * re-ingested (its contentHash is no longer the file's current one). Only * ever set when the caller opted into `includeSuperseded`. */ superseded?: boolean; } export interface DocumentMeta { filePath: string; contentHash: string; chunkCount: number; indexedAt: string; scope: string; size: number; } export declare function ingestDocument(filePath: string, scope?: string, rootDir?: string, _metadataCache?: DocumentMeta[]): Promise; export declare function ingestDirectory(dirPath: string, scope?: string, opts?: { rootDir?: string; onProgress?: (file: string, done: number, total: number) => void; }): Promise; /** Content hashes of the documents currently indexed under `rootDir`. */ export declare function liveContentHashes(rootDir: string): Set; /** True when a metadata log exists under `rootDir`. * * An empty live-hash set has two very different causes: the log is missing (we * cannot judge what is current) or the log exists and every document has been * removed (nothing is current). Collapsing them made `doc remove` of the LAST * document a no-op — the tombstoned chunks came straight back in search. * * Reads the path directly instead of via `metadataPath`, which mkdir's. */ export declare function hasKnowledgeMetadata(rootDir: string): boolean; /** * True when `key` is a document chunk whose version is no longer current. * Non-`doc:` keys are never superseded. When no metadata is available nothing * is filtered, because "no metadata" must not read as "everything is stale". * * `metadataPresent` defaults to the old `live.size > 0` heuristic so existing * two-argument callers keep their exact behaviour; pass `hasKnowledgeMetadata` * to also filter correctly once the last document has been removed. */ export declare function isSupersededKey(key: string, live: Set, metadataPresent?: boolean): boolean; export declare function supersededOverfetchLimit(limit: number, live: Set): number; export declare function searchKnowledge(query: string, opts?: { scope?: string; limit?: number; minScore?: number; rootDir?: string; /** which store(s): project-only, global-only, or both (default). */ store?: 'project' | 'global' | 'all'; /** Return chunks from superseded document versions too, flagged * `superseded: true`. Default false — see the note above `liveContentHashes`. */ includeSuperseded?: boolean; /** Skip cross-encoder reranking. Default false. */ skipRerank?: boolean; }): Promise; export declare function listDocuments(rootDir?: string, scope?: string): DocumentMeta[]; export declare function removeDocument(filePath: string, scope?: string, rootDir?: string): Promise; /** * True for macOS AppleDouble sidecars (`._name`). * * Matches on the BASENAME PREFIX only. A legitimate document may contain `._` * elsewhere in its name (`v1._2-release.md`), or live under a dot-directory * that is deliberately indexed (`.monodesign/` critique snapshots), and * neither may be rejected. */ export declare function isResourceFork(filePath: string): boolean; export interface ReconcileReport { /** Indexed documents whose source file is no longer on disk. */ missing: DocumentMeta[]; /** Total index entries examined. */ scanned: number; /** False for a dry run — the default. */ applied: boolean; /** Entries actually tombstoned. Always 0 when `applied` is false. */ removed: number; /** Where removed records were archived, when anything was removed. */ archivePath?: string; } /** * Reconcile the document index against the filesystem: find index entries whose * source file no longer exists and, only when explicitly asked, tombstone them. * * WHY — `removeDocument` only ever tombstoned metadata, and nothing has ever * compared the index against the disk, so a deleted file stayed searchable * forever. Measured 2026-07-28: 109 of 257 live entries (42.4%) had no file * behind them, including `docs/concepts/memory.md`. The Second Brain was * answering questions from documents the user had deleted. * * WHY IT IS THIS CAUTIOUS — "drop the index entry when the file is missing" is * a rule with a known catastrophic reading. A missing file is also an unmounted * volume, a checked-out branch, a partial clone, or a permissions failure. Two * guards were tried against real data and REJECTED; they are recorded here so * they are not re-proposed: * * - "abort if >50% of entries are missing" — the real, legitimate missing * fraction was 42.4%, so the threshold never fires in the one case we have. * Any threshold that would have blocked this reconcile is fitted to nothing. * - "only reconcile when the parent directory still exists" — 26 of the 109 * missing files had no parent directory, because `docs/concepts`, * `docs/adrs` and `docs/commands` were legitimately deleted wholesale. A * deleted directory and an unmounted volume are indistinguishable there. * * What does discriminate is the ROOT. An intact, readable root carrying a * metadata log means the tree is genuinely present, so a missing file is * genuinely gone. A missing root means nothing beneath it is knowable and * nothing may be removed — hence throw rather than reconcile. * * Removal tombstones metadata; it does not delete store rows. Chunks stay on * disk and fall out of search through the existing superseded filter, which * keeps this consistent with the mark-don't-destroy rule and leaves the whole * operation reversible from the archive. */ export declare function reconcileIndex(rootDir?: string, opts?: { scope?: string; apply?: boolean; }): Promise; export declare function exportToOKF(outputDir: string, rootDir?: string, scope?: string): Promise<{ exported: number; outputDir: string; }>; export declare function importFromOKF(bundleDir: string, scope?: string, rootDir?: string): Promise; //# sourceMappingURL=document-pipeline.d.ts.map