/** * Optional embedding-based semantic search for Hippo. * Uses @huggingface/transformers (local, zero API keys, ~22MB model). * Falls back silently if the library is not installed. */ import { MemoryEntry } from './memory.js'; export declare const DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2"; export declare const EMBEDDING_MODEL_META_KEY = "embedding_model"; /** * Bump whenever `embeddingInputText`'s composition changes in a way that * changes the resulting vectors for existing entries. Folded into the stored * index identity (see `embeddingIndexIdentity`) so a text-format change is * treated exactly like an embedding-model change: the next embed-touching * operation detects the mismatch and reindexes automatically. Format 2 = * `path:*` tags excluded (see `embeddingInputText`); format 1 (implicit, no * suffix) = `${content} ${tags.join(' ')}` including path tags. */ export declare const EMBED_TEXT_FORMAT = 2; /** * The stored-index identity for a given embedding provider id: folds * `EMBED_TEXT_FORMAT` into the provider id so index-identity comparisons * automatically invalidate on either a model change OR a text-format change. * This is the ONE choke point both `embeddingModelRequiresReindex` (compare * side) and `saveStoredEmbeddingModel` (save side) go through — they MUST * version identically, or every call reindexes in a loop (or none ever do). */ export declare function embeddingIndexIdentity(providerId: string): string; /** * Build the text embedded for a memory entry: content plus its tags, joined * by a space and trimmed — the same shape as the legacy * `` `${e.content} ${e.tags.join(' ')}`.trim() `` composition, minus `path:*` * tags. * * `path:*` tags are excluded because they are auto-derived from * `process.cwd()` (see `extractPathTags` in cli.ts) and carry every path * component of the store's location, INCLUDING the store directory name * itself. That means identical content embeds to a DIFFERENT vector * depending on WHERE the store happens to live — e.g. a fresh benchmark run * under `tempfile.mkdtemp()` gets a new directory name (hence new path * tokens, hence a new vector) every single run, even with byte-identical * content ingested in byte-identical order. This was diagnosed as the * DOMINANT root cause of cross-fresh-ingest recall-rank variance measured on * LoCoMo (mean evidence-recall@5 stdev 0.0175 across 4 fresh re-ingests of * identical data; see `benchmarks/LOCOMO_INVESTIGATION.md`, "Determinism * characterization"). It is a real product defect beyond benchmarks too: * retrieval semantics should not depend on a project directory's name. * * Only `path:*` is excluded. Other tags (`conv:`, `session:`, `speaker:`, * `dia:`, `error`, `scope:`, etc.) remain embedded — they carry semantic * meaning. Path relevance at recall time is already handled explicitly by * the v39 scope-isolation layer (`origin_project`, `pathOverlapScore`), so * embedding-level path tokens are redundant with a dedicated mechanism * rather than a feature. */ export declare function embeddingInputText(entry: { content: string; tags: string[]; }): string; /** * Per-model pooling dispatch for Transformers.js's feature-extraction * pipeline. BGE family models were trained with CLS pooling (per BAAI's * official inference code in `FlagEmbedding`); MiniLM and most sentence- * transformers models use mean pooling. Unknown model ids default to mean * — that is the safe choice because most third-party models adopt the * sentence-transformers convention, and the alternative ('cls') silently * degrades vector quality for mean-pooling models. */ export declare function poolingFor(model: string): 'cls' | 'mean'; /** * Per-model input-prefix dispatch. The intfloat/e5 family was trained with * asymmetric "query: " / "passage: " prefixes — the model only matches the * two halves correctly when each side carries its prefix at inference. BGE * also has prefix conventions for some downstream tasks, but symmetric use * without prefixes is the documented default for `bge-*-en-v1.5`, so we leave * BGE alone here. Symmetric models (MiniLM, BGE) and unknown models return * an empty prefix. * * `role` semantics: * - 'query' — the text is the user's question / search input. * - 'passage' — the text is a document being indexed. * - undefined or absent — symmetric path; no prefix is applied even for * asymmetric models (preserves backwards compatibility with the legacy * two-argument `getEmbedding(text, model)` API). */ export type EmbeddingRole = 'query' | 'passage'; export declare function prefixFor(model: string, role?: EmbeddingRole): string; /** * Check (synchronously) if @xenova/transformers or @huggingface/transformers is installed. */ export declare function isEmbeddingAvailable(): boolean; export declare function resolveEmbeddingModel(hippoRoot: string, explicitModel?: string): string; /** * Persist the stored-index identity for `model` (see `embeddingIndexIdentity`). * Versioning happens INSIDE this function, not at call sites, so every caller * — current and future — gets the identity format for free. Must stay * consistent with the compare side in `embeddingModelRequiresReindex`. */ export declare function saveStoredEmbeddingModel(hippoRoot: string, model: string): void; export declare function resolveIndexedEmbeddingModel(hippoRoot: string, index?: Record): string | null; export declare function embeddingModelRequiresReindex(hippoRoot: string, model: string, index?: Record): boolean; /** * Get an embedding vector for a piece of text. * Returns an empty array if transformers is not available or fails. * * Pass `role: 'query'` / `'passage'` to engage asymmetric prefixing for * model families that require it (currently intfloat/e5-*). Omitting `role` * keeps the legacy symmetric behavior (no prefix), so BGE / MiniLM callers * don't need to change. */ export declare function getEmbedding(text: string, model?: string, role?: EmbeddingRole): Promise; /** * Cosine similarity between two vectors. Handles unnormalized vectors. * Returns 0 for empty or mismatched vectors. */ export declare function cosineSimilarity(a: number[], b: number[]): number; /** * Load the cached embedding index from disk. * Returns an empty object if the file doesn't exist or is corrupt. */ export declare function loadEmbeddingIndex(hippoRoot: string): Record; /** * Save the embedding index to disk. */ export declare function saveEmbeddingIndex(hippoRoot: string, index: Record): void; /** * Embed a single memory entry and cache the result in the embedding index. */ export declare function embedMemory(hippoRoot: string, entry: MemoryEntry, model?: string): Promise; /** * Embed all entries in hippoRoot that don't already have cached vectors. * Prunes orphaned embeddings for memories that no longer exist. * Returns the count of newly embedded entries. */ export declare function embedAll(hippoRoot: string, model?: string): Promise; //# sourceMappingURL=embeddings.d.ts.map