/** * Content types for stored content. */ export type ContentType = 'code' | 'logs' | 'prose' | 'json' | 'mixed' | 'binary_desc'; /** * Source types indicating where content originated. */ export type ContentSourceType = 'file_read' | 'pdf_read' | 'bash_output' | 'web_fetch' | 'user_input' | 'tool_result' | 'image_analysis' | 'transcript' | 'file_write' | 'reference' /** Written by the model itself via store_content — a finding, decision or idea, not tool output (task 185). */ | 'agent_note'; /** * A chunk of stored content for retrieval. */ export interface ContentChunk { /** Chunk index (0-based) */ index: number; /** The chunk content */ content: string; /** Token estimate for this chunk */ tokenEstimate: number; } /** * Metadata for stored content (persisted to index). */ export interface StoredContentMeta { /** Unique content ID (cnt_xxx) */ id: string; /** When content was stored */ timestamp: number; /** Where the content came from */ sourceType: ContentSourceType; /** Original tool name if from tool result */ sourceTool?: string; /** Original content size in characters */ originalSize: number; /** Estimated token count */ tokenEstimate: number; /** Detected content type */ contentType: ContentType; /** Brief auto-generated summary */ summary: string; /** Number of chunks */ chunkCount: number; /** SHA-256 hash for deduplication */ contentHash: string; /** Additional context (file path, command, etc.) */ metadata: Record; } /** * Full stored content including chunks. */ export interface StoredContent extends StoredContentMeta { /** Content chunks for retrieval */ chunks: ContentChunk[]; } /** * Options for storing content. */ export interface StoreOptions { /** Source type */ sourceType: ContentSourceType; /** Original tool name */ sourceTool?: string; /** Content type (auto-detected if not provided) */ contentType?: ContentType; /** Summary (auto-generated if not provided) */ summary?: string; /** Additional metadata */ metadata?: Record; } /** * Options for searching content. */ export interface SearchOptions { /** Filter by source types */ sourceTypes?: ContentSourceType[]; /** Filter by content types */ contentTypes?: ContentType[]; /** Maximum results */ limit?: number; /** Search mode */ mode?: 'keyword' | 'semantic' | 'hybrid'; /** Pre-expanded query terms (from context-aware query expansion) */ expandedTerms?: string[]; /** * Restrict results to content stored by these threads (task 143). Only * meaningful in a shared store. Entries stored before attribution existed * carry no thread list and are therefore excluded by any filter — an explicit * filter is a request for a known subset, so silently including unattributed * content would make the filter a lie. */ threads?: string[]; } /** * Search result with relevance score. */ export interface ContentSearchResult { /** Content metadata */ meta: StoredContentMeta; /** Relevance score (0-1) */ score: number; /** Matching snippet */ snippet: string; } /** * Snap a window's START forward to the NEAREST whitespace boundary (task 107). * * Windows were cut at a raw character offset, so a snippet routinely opened * mid-word (`...FT, exit code, duration)`). Moves forward only, and at most * `BOUNDARY_SNAP_CHARS`, so a snippet can never exceed its budget. `\s` already * covers newlines, so the nearest boundary is a line break whenever one is * closer — deliberately NOT a line-break preference, which could discard up to * `BOUNDARY_SNAP_CHARS` of good content to reach a distant break. */ export declare function alignSnippetStart(content: string, start: number): number; /** * Snap a window's END back to the NEAREST whitespace boundary (task 107). Moves * backward only, and at most `BOUNDARY_SNAP_CHARS`, so the snippet stays within * budget and never closes mid-word. */ export declare function alignSnippetEnd(content: string, end: number): number; /** * Max chunk embeddings to backfill synchronously on the search path (task 080). * The rest are deferred to a background runner so a large embedding backlog can * never dominate a search. Mirrors history's `MAX_INLINE_BACKFILL` (076). */ export declare const MAX_INLINE_CHUNK_BACKFILL = 16; /** * Decide how much missing-chunk backfill to do inline vs. in the background. * Pure (no I/O) so the bounding policy is trivially testable — mirror of * `retriever.planEmbeddingBackfill` (task 080). */ export declare function planChunkBackfill(missingCount: number, inFlight: boolean): { inlineCount: number; scheduleBackground: boolean; }; /** * Deferred snippet descriptor (task 082 Phase 2). The keyword/semantic passes * SCORE every candidate but record only how to build its snippet — the actual * `retrieve()` is paid only for the final top-N results, after ranking. This * collapses the per-match content/chunk read fan-out (hundreds of awaited * `loadIndex` fs.stat + disk reads per search on a topical corpus) to ~`limit`. */ export type SnippetSpec = { kind: 'summary'; } | { kind: 'content'; firstSig: string | null; } | { kind: 'chunk'; chunkIndex: number; }; /** * A ranked content candidate whose snippet is NOT yet materialized (task 083 * Lever 2). `searchScored()` returns these; a consumer materializes a snippet * (via `materializeCandidateSnippet`) only for the candidates that survive its * own budget merge — collapsing the snippet disk-read fan-out from ~`limit` * (≈150) to the handful actually used. `snip` is an opaque deferred descriptor: * pass the candidate back to `materializeCandidateSnippet` to build its snippet. */ export interface ContentSearchCandidate { /** Content metadata */ meta: StoredContentMeta; /** Relevance score (0-1) */ score: number; /** Opaque deferred-snippet descriptor — pass back to materializeCandidateSnippet(). */ snip: SnippetSpec; } /** One unparseable line of `index.jsonl`, reported rather than thrown (task 148 D2). */ export interface DamagedIndexLine { /** 1-based line number in the file. */ line: number; /** Byte length of the damaged line. */ length: number; /** Leading characters, for identifying the tear in a log. */ head: string; } /** * Parse `index.jsonl` content into entries, skipping damaged lines. * * Pure so the tolerance rule is testable without a store on disk. A damaged * line is reported, never thrown: `index.jsonl` is appended to concurrently by * co-threads sharing one store (task 143), so a torn record is a thing that * happens — losing the other 5,000 because of it is not. */ export declare function parseIndexLines(content: string): { entries: StoredContentMeta[]; damaged: DamagedIndexLine[]; }; /** * Append one line to `index.jsonl`, holding the same lock a rewrite holds. * * Exported so the one rule has one owner — the store uses it, and a concurrency * harness can drive the real append path from a separate process without * standing up an embedding provider. * * Throws if the lock cannot be taken: a silently dropped append is exactly the * failure being removed, so failing loudly is the lesser harm. */ export declare function appendIndexLineLocked(indexPath: string, line: string): Promise; /** * ContentStore manages externalized content storage and retrieval. * * Content is stored in a thread-specific directory: * - index.jsonl: Metadata for all stored content * - chunks/: Directory containing actual content chunks */ export declare class ContentStore { readonly basePath: string; private indexPath; private chunksDir; /** In-memory cache for the content index. null = not yet loaded. */ private cachedIndex; /** * Position of each id within `cachedIndex` (task 143). `index.jsonl` is * append-only, so an entry is UPDATED by appending a fresh line with the same * id and letting the later line win — the same rule the embeddings file now * follows. This map is what makes that replacement O(1) on the read path. */ private cachedIndexPos; /** File size at the time the index cache was last populated. -1 = unknown. */ private cachedIndexSize; /** Guard against concurrent prune sweeps (fire-and-forget from store()). */ private pruning; /** Line numbers already reported as damaged — the index is read every search. */ private reportedDamagedLines; /** * Report damaged index lines once each. Loud enough to be actionable (the * ordimor tear went unnoticed for 29h), quiet enough not to print on every * search — `loadIndex` runs on the hot path. */ private warnDamagedIndex; /** * In-process cache of full per-entry content (task 079), keyed by content id * → joined chunk text. The keyword search loop reads every non-summary-match * entry's full content on every query; without this it re-reads most of the * store from disk each search (the measured RAG bottleneck). Chunks are * append-only immutable (`store()` only writes new ids; `findByHash` dedups), * so entries stay valid across searches and are evicted only on delete/prune. */ private fullContentCache; /** Running total of bytes held in `fullContentCache` for FIFO bounding. */ private fullContentCacheBytes; /** * Inverted keyword index (task 082 Phase 2): lowercased whitespace-delimited * token → the content ids whose full text contains that token. null = not yet * built. Replaces the keyword loop's per-search `retrieve()` of EVERY non- * summary-match entry (~960 awaited `loadIndex` fs.stat + scans per query, the * durable `keyword=12s` pole) with an O(vocab + matches) lookup that touches * only the entries that actually match. Every query term is provably space- * free (built from camelCase/snake_case/word/proper-noun extraction or a * `\s+` split), so a space-free term matches an entry's content iff it is a * substring of one of that entry's tokens — making this index exactly * equivalent to the original full-content `.includes()` (RLM-neutral). * Chunks are append-only immutable, so postings stay valid across searches; * `store()` extends the index, delete/prune invalidate it. */ private keywordIndex; /** * Ids already folded into `keywordIndex` (task 143). Lets the index catch up * on entries written by another instance or another process instead of being * built once and then only ever extended by this instance's own `store()`. */ private keywordIndexedIds; /** * In-process cache of individual chunk text (task 082 Phase 2), keyed by * `"{id}:{chunkIndex}"`. The semantic snippet pass and the embed backfill both * call `retrieve(id, chunkIndex)`, which otherwise hits disk (and an awaited * `loadIndex` fs.stat) per call — a per-match fan-out that balloons under * contention exactly like the keyword pole. Same immutability/eviction * contract as `fullContentCache`; FIFO-bounded by the shared cap. */ private chunkContentCache; /** Running total of bytes held in `chunkContentCache` for FIFO bounding. */ private chunkContentCacheBytes; /** * The thread writing through this instance (task 143). Stamped onto every * entry it stores so a shared store can say who put a thing there. */ readonly threadName?: string; /** * Whether this store is shared with co-threads. Attribution is only RENDERED * when it is: in a single-thread store every line would carry the same name, * which is pure per-turn dilution for zero information. */ readonly shared: boolean; constructor(basePath: string, options?: { threadName?: string; shared?: boolean; }); /** * Ensure storage directories exist. */ private ensureDirectories; /** * Get path to the images directory for cached image files. */ getImagesDir(): string; /** * Ensure the images directory exists. */ ensureImagesDir(): Promise; /** * Compute SHA-256 hash of content for deduplication. */ private computeHash; /** * Split content into chunks of approximately chunkSize tokens. */ private createChunks; /** * Detect structural boundaries in non-code content and split into structural units. * Each unit is a coherent block: a headed section, table, code fence, paragraph, etc. */ private detectStructuralUnits; /** * Split non-code content into structure-aware chunks. * Respects headers, tables, code fences, and paragraph boundaries. * Merges small units and splits oversized ones. */ private createStructuralChunks; /** * Regex patterns that identify top-level code block boundaries. */ private static readonly CODE_BLOCK_PATTERNS; /** * Pattern matching import/use/include lines. */ private static readonly IMPORT_PATTERN; /** * Check if a line is a code block boundary (top-level declaration). */ private isBlockBoundary; /** * Split code content into chunks at function/class/type boundaries. * Falls back to line-based splitting for oversized single blocks. */ private createCodeChunks; /** * Generate a brief summary of content. * Delegates to the shared heuristic summary generator in content-detector. */ private generateSimpleSummary; /** * ONE thread-filter predicate, applied at every filter site in `searchCore` * (keyword and semantic). Absent filter = everything, exactly as before. */ private matchesThreadFilter; /** * Read the thread attribution list off an entry. Always an array; entries * stored before task 143 have none and read as empty. */ static threadsOf(meta: StoredContentMeta): string[]; /** * Split an entry's attribution into "other threads" and "me" (task 185). * * The reader's own name is never rendered as an origin: `[via ordimor]` shown * to ordimor told it nothing, and shown to a sibling it was indistinguishable * from a tag naming the sibling itself. Renderers show `others` and use * `self` only to say "and you". */ static attributionFor(meta: StoredContentMeta, self: string | undefined): { others: string[]; self: boolean; }; /** * The bracketed origin tag for retrieved context, or '' when there is nothing * to say: unshared store, unattributed entry, or stored only by the reader. */ static attributionTag(meta: StoredContentMeta, self: string | undefined, shared: boolean): string; /** * Record that this thread also stored an existing entry's content. * * The update is an APPENDED index line with the same id, not a rewrite: the * index is append-only and a whole-file rewrite is exactly the read-modify- * write that loses concurrent work (the Phase 1 defect). `loadIndex` lets the * later line win. */ private addThreadAttribution; /** * Follow a thread rename through the attribution tags (task 186). * * `mapping` is old name → new name; a master rename passes itself and every * cascaded co-thread in one map so the store is walked once. Every entry whose * `threads` names a renamed thread gets ONE appended index line with the * substitution made — appended, never rewritten, for the same reason as * `addThreadAttribution`: a whole-file rewrite is the read-modify-write that * lost 8% of ordimor's store (task 148 D1). Returns how many entries changed. */ renameThreads(mapping: Record): Promise; /** * Store content externally and return metadata. * Returns existing content ID if content hash matches (deduplication). */ store(content: string, options: StoreOptions): Promise; /** * Simple content type detection based on heuristics. */ private detectContentType; /** * Find content by hash (for deduplication). */ findByHash(hash: string): Promise; /** * Load the content index with in-memory caching. * Uses incremental read when the file has grown since last cache. */ /** * Fold appended index lines into the live array, letting a later line for an * id replace an earlier one IN PLACE (task 143). Position is held at the id's * first appearance so the array stays in store order — `prune()` evicts from * the front, and reordering on an attribution update would evict the wrong * entries. */ private mergeIndexEntries; private setIndexCache; loadIndex(): Promise; /** * Retrieve content by ID. * @param id - Content ID * @param chunkIndex - Optional specific chunk index * @returns Full content or specific chunk */ retrieve(id: string, chunkIndex?: number): Promise; /** * Insert a full-content entry into the in-process cache, evicting oldest * entries (FIFO — Map preserves insertion order) once the byte cap is hit. */ private cacheFullContent; /** Drop a single id from the full-content cache, keeping the byte total exact. */ private evictFullContent; /** * Insert a single chunk's text into the chunk cache (task 082 Phase 2), * FIFO-evicting once the shared byte cap is hit. Mirrors `cacheFullContent`. */ private cacheChunkContent; /** Drop a single chunk key from the chunk cache, keeping the byte total exact. */ private evictChunkContent; /** Evict every cached chunk belonging to a content id (on delete/prune). */ private evictChunksForId; /** * Lazily build (or return) the inverted keyword index (task 082 Phase 2). * Reads each entry's full content once via `retrieve()` (warming * `fullContentCache` as a side effect) and maps every unique lowercased * whitespace token to the ids whose content contains it. Built once per * process and kept warm: `store()` extends it, delete/prune invalidate it. */ private loadKeywordIndex; /** * Add one entry's unique tokens to a keyword index map. Tokenization mirrors * the original keyword scan's matching domain exactly: a space-free query term * matches `content.toLowerCase().includes(term)` iff it is a substring of one * of these `content.toLowerCase().split(/\s+/)` tokens — so the index yields * identical matches (RLM-neutral). */ private indexEntryTokens; /** * Ids whose content contains `term` as a substring of some token — the * inverted-index equivalent of the original per-entry `contentLower.includes`, * scanning the (small) vocabulary instead of the whole corpus (task 082). */ private keywordMatchIds; /** * Build a result's snippet from its deferred spec (task 082 Phase 2). The * exact byte-for-byte output the keyword/semantic passes used to build inline * — just paid only for the top-N results, after ranking. */ private materializeSnippet; /** * Rank scored candidates by score (stable, descending) and take the top-`limit` * (task 082 Phase 2 / 083). Sort is stable and the candidate set/insertion * order is identical to the pre-deferral path, so the ranked output — and any * snippet materialized from it — is RLM-neutral. Snippets are NOT materialized * here; callers decide (eagerly via `materializeRanked`, or lazily via * `materializeCandidateSnippet` once survivors are known — task 083 Lever 2). */ private rankCandidates; /** * Materialize snippets for an already-ranked entry list (task 082 Phase 3 — * the eager path used by `search()`, e.g. the MCP `search_content` tool and * tests). Each snippet is an independent disk read; the prior sequential await * chain made snippet the residual search pole, so materialization runs in a * bounded, order-preserving worker pool. Results are written by index so the * output is byte-identical to the sequential path (RLM-neutral); concurrency * is bounded so a batch of large full-content retrieves can't spike memory. */ private materializeRanked; /** * Materialize one candidate's snippet (task 083 Lever 2). A consumer that runs * its own budget merge (the RAG retriever) calls this lazily — only for the * candidates that survive — so the snippet disk-read fan-out collapses from * ~`limit` (≈150) to the handful actually injected. The snippet text is * identical to what `materializeRanked`/`search()` would have produced for the * same candidate (same `meta` + same `snip` spec), so retrieval is RLM-neutral. * * `maxTokens` is the caller's snippet budget (task 107). The RAG retriever * passes its budget-scaled cap; omitting it keeps the 300-char default size. */ materializeCandidateSnippet(candidate: ContentSearchCandidate, maxTokens?: number): Promise; /** * Get metadata for stored content. */ getMeta(id: string): Promise; /** * Get previews of all chunks for a stored content item. * Returns a brief preview of each chunk for navigation (table of contents). */ getChunkPreviews(id: string): Promise<{ index: number; tokens: number; preview: string; }[] | null>; /** * List all stored content. */ list(options?: { limit?: number; sourceTypes?: ContentSourceType[]; }): Promise; /** * Delete stored content by ID. * Removes from index, chunks, and embeddings. * @returns true if content was found and deleted, false if not found */ delete(id: string): Promise; /** * Evict old/excess content to keep the store bounded. * * Policy (newest `timestamp` = most recent, always retained): * 1. By age — evict entries older than `maxAgeMs`. * 2. By count — if still over `maxItems`, evict the oldest until at the cap. * * Batched: a single index rewrite and a single embeddings rebuild, regardless * of how many entries are evicted (avoids the O(n²) cost of per-id `delete`). * * @returns the number of content items evicted */ prune(options?: { maxItems?: number; maxAgeMs?: number; }): Promise<{ evicted: number; }>; /** * Delete embeddings for a batch of content IDs in one rebuild. * Mirrors `deleteEmbeddings` but removes every `"{id}:*"` key for all ids at * once, so a prune of N items rewrites the embeddings file only once. */ private deleteEmbeddingsBatch; /** * Delete embeddings for a content ID. * Rebuilds the binary embeddings file without the deleted entries. */ private deleteEmbeddings; /** * Append one line to `index.jsonl`, holding the same lock a rewrite holds. * * Task 148 D1. `O_APPEND` alone is not enough here, and the concurrency test * proved it: a rewrite is `read` → write `tmp` → `rename`, and an append that * lands on the OLD inode between the read and the rename is discarded with * the inode. Atomicity of the append never enters into it. So on this file * appenders take the lock too — the cost is a create+unlink per stored item, * against the alternative of losing records silently, which is what ordimor * did for a week. * * Deliberately different from `saveContentEmbeddingsCached`, where appenders * skip the lock (task 143 P1). That decision was made against a concurrency * proof covering appends-vs-appends only; appends-vs-rewrite has the same * rename window there and is recorded as its own follow-up rather than * changed here on a hot path without its own measurement. * * Throws if the lock cannot be taken. A silently dropped append is precisely * the failure mode being removed — failing loudly is the lesser harm. */ private appendIndexLine; /** * Remove `ids` from the index, as a locked, atomic, read-modify-write. * * Task 148 D1. The previous form took the caller's in-memory array and did a * bare `fs.writeFile` — truncate, then write — with no lock. Two failures * followed from that, and both were observed on ordimor's shared store: * * 1. A concurrent `O_APPEND` landing inside the truncate window tears a * record in half (the 505-byte fragment on line 5,001). * 2. Worse and quieter: the caller's array is a SNAPSHOT taken before the * chunk unlinks, so every entry another process appended in between was * silently deleted. 435 orphaned chunk sets in 47 clusters, ~8% of the * store, gone with no error. * * So the fix is not merely `tmp`+`rename`: the surviving set must be computed * from a FRESH read taken under the lock. Failing to take the lock means * leaving the file alone — a rewrite that cannot exclude appenders must not * run, which is the same posture `rewriteContentEmbeddings` takes (task 143 * P1). Rule #8: one rule for every whole-file rewrite in the store. * * @returns true if the rewrite ran; false if the lock could not be taken. */ private removeFromIndex; /** * Get embeddings for all chunks of a specific content item. * Generates missing embeddings first if needed. * Returns a Map keyed by chunk index. */ getEmbeddingsForContent(contentId: string): Promise | null>; /** * Load content embeddings using the cached binary-format loader. */ private loadContentEmbeddings; /** * Save content embeddings using the cached binary-format saver. */ private saveContentEmbeddings; /** * Collect every (contentId, chunkIndex) pair that lacks an embedding. Cheap: * both the index and the embeddings map come from in-process caches. */ private collectMissingChunks; /** * Embed a specific set of chunks and persist them. Returns the number embedded. * Shared by embed-on-write, the bounded search-path backfill, and the full * background runner (task 080). */ private embedChunks; /** * Generate embeddings for all content that doesn't have them yet. Unbounded — * used by the background backfill runner and `getEmbeddingsForContent`. The * search path uses the bounded `backfillForSearch` instead (task 080). */ generateMissingEmbeddings(): Promise; /** * Fire-and-forget embed of a freshly-stored entry's chunks (task 080 — * embed-on-write, mirroring history's 028). Keeps the search path's missing- * embedding count at zero in steady state so semantic search never embeds on * the critical path. Errors are swallowed; `backfillForSearch` is the safety net. */ private embedNewChunks; /** * Backfill missing chunk embeddings off the search critical path (task 080). * Embeds at most `MAX_INLINE_CHUNK_BACKFILL` inline and defers the rest to a * deduped background runner, so a large embedding backlog (cold thread, or an * embed-on-write miss) can never dominate a search. In steady state embed-on- * write keeps the missing count at zero and this no-ops. */ private backfillForSearch; /** * Run a full embedding backfill in the background, deduped per base path so * successive searches don't stack full re-embeds or race the embeddings file * (task 080). Never throws into the caller. */ private scheduleBackgroundBackfill; /** * Search stored content with keyword, semantic, or hybrid search. Returns * ranked results with materialized snippets — the eager path used by the MCP * `search_content` tool and tests. The RAG retriever uses `searchScored()` * instead, deferring snippet materialization past its budget merge (task 083). */ search(query: string, options?: SearchOptions): Promise; /** * Like `search()` but returns ranked candidates WITHOUT materialized snippets * (task 083 Lever 2). The caller materializes a snippet (via * `materializeCandidateSnippet`) only for the candidates that survive its own * budget merge, collapsing the snippet disk-read fan-out from ~`limit` (≈150) * to the handful actually used. Same scoring, ranking, and slice as `search()`, * so any snippet built from these candidates is RLM-neutral. */ searchScored(query: string, options?: SearchOptions): Promise; /** * Score, rank, and slice candidates for a query — the shared scoring path for * `search()` (eager snippets) and `searchScored()` (deferred snippets, task * 083). Returns the ranked top-`limit` `ScoredEntry[]` WITHOUT materializing * snippets; the caller chooses how and when to materialize them. */ private searchCore; /** * Get statistics about stored content. */ getStats(): Promise<{ totalItems: number; totalSize: number; totalTokens: number; bySourceType: Record; byContentType: Record; }>; } //# sourceMappingURL=content-store.d.ts.map