import type { DatabaseHandle } from '../store/db.js'; import type { Embedder } from './types.js'; /** * Describes the token-budget and method used to discover it for a particular * embedder model. All callers should use `chunkBudgetChars` to size chunks. */ export interface EmbedderCapacity { advertisedMaxTokens: number; discoveredMaxTokens: number; /** Floor(0.9 × min(advertised, discovered)) — safe operating budget. */ chunkBudgetTokens: number; /** chunkBudgetTokens × charsPerTokenEstimate — char limit for chunker. */ chunkBudgetChars: number; method: 'tokenizer_config' | 'api_show' | 'probe' | 'manual' | 'fallback'; } /** * Hard floor on `discovered_max_tokens`. v1.7.2 shipped an unfloored ratchet * (one freak chunk on a long note could halve the budget down to ~115), * cascading more chunks into "too long" → more ratcheting → runaway. Below * 256 tokens (~640 chars) embeddings produce single-sentence chunks that * destroy retrieval quality, so we never let the ratchet go lower. Each * full reindex pass also calls `resetDiscoveredCapacity` to wipe drift. */ export declare const MIN_DISCOVERED_TOKENS = 256; /** * Get (or probe + cache) the effective token capacity for the given embedder. * * - TransformersEmbedder: reads `model_max_length` from the loaded pipeline * tokenizer, cross-checks against the KNOWN_MAX_TOKENS validation table. * - OllamaEmbedder: calls `/api/show` and reads `model_info[*.context_length]`. * - All results are stored in `embedder_capability` for fast subsequent reads. * - A positive `OBSIDIAN_BRAIN_MAX_CHUNK_TOKENS` env var overrides the * discovered capacity with method='manual'. */ export declare function getCapacity(db: DatabaseHandle, embedder: Embedder): Promise; /** * Reset the cached `discovered_max_tokens` back to `advertised_max_tokens` so * a fresh reindex pass starts from the model's full advertised limit and * doesn't inherit drift from a previous run. No-op if no row exists yet — * the next `getCapacity()` call will probe and insert. * * Called once at the top of `IndexPipeline.index()` (full-vault reindex). * Not called on `indexSingleNote()` — single-file indexing is too frequent * to take this hit on every keystroke. */ export declare function resetDiscoveredCapacity(db: DatabaseHandle, embedder: Embedder): void; /** * Record a chunk that failed to embed. Called by the fault-tolerant indexer * (V1.7.0-Faulttol) — wired here so the schema and helper live in one place. */ export declare function recordFailedChunk(db: DatabaseHandle, chunkId: string, noteId: string, reason: string, errorMessage: string | null): void; /** * Reduce the cached discovered_max_tokens for the embedder by recording * a lower bound inferred from a failed chunk's token count. The new * discovered value is halved relative to the failing token count so the * next index pass tries meaningfully smaller chunks. * * This is a one-way ratchet — we never increase the discovered bound via * this function. */ export declare function reduceDiscoveredMaxTokens(db: DatabaseHandle, embedder: Embedder, failedChunkTokenCount: number): void;