/** * LanceDB Store Module * * Vector database wrapper using LanceDB for storing and searching code chunk embeddings. * This is the core storage component that enables semantic search capabilities. * * Features: * - Vector similarity search with configurable top-k results * - Batch insert for efficient indexing * - Delete by file path for incremental updates * - Path pattern matching for file-based queries * - Stale lockfile detection and cleanup */ /** * Record structure for chunks stored in LanceDB * * This matches the RFC specification for the database schema. * The index signature is required for LanceDB compatibility. * * SMCP-086: Added optional metadata fields for AST-based chunking: * - chunk_type: Type of code construct (function, class, method, etc.) * - chunk_name: Name of the function/class/method * - chunk_signature: Full signature (for functions/methods) * - chunk_docstring: Extracted docstring/comment * - chunk_parent: Parent name (e.g., class name for methods) * - chunk_tags: Comma-separated semantic tags * * SMCP-098: Added chunk_hash for incremental reindexing: * - chunk_hash: Position-independent hash of chunk text for change detection */ export interface ChunkRecord { /** UUIDv4 unique identifier for the chunk */ id: string; /** Relative file path (forward-slash separated) */ path: string; /** Chunk content text */ text: string; /** Float32[384] embedding vector */ vector: number[]; /** Start line in source file (1-indexed) */ start_line: number; /** End line in source file (1-indexed) */ end_line: number; /** SHA256 hash of the source file content */ content_hash: string; /** Position-independent SHA256 hash of chunk text (for detecting unchanged content) */ chunk_hash?: string; /** Chunk type (function, class, method, etc.) */ chunk_type?: string; /** Name of the function/class/method */ chunk_name?: string; /** Full function/method signature */ chunk_signature?: string; /** Extracted docstring or comment */ chunk_docstring?: string; /** Parent name (e.g., class name for methods) */ chunk_parent?: string; /** Comma-separated semantic tags */ chunk_tags?: string; /** Programming language */ chunk_language?: string; /** Index signature for LanceDB compatibility */ [key: string]: string | number | number[] | undefined; } /** * Search result structure returned from vector search */ export interface SearchResult { /** Relative file path */ path: string; /** Chunk content text */ text: string; /** Similarity score (0.0 - 1.0, higher is more similar) */ score: number; /** Start line in source file */ startLine: number; /** End line in source file */ endLine: number; /** Chunk type (function, class, method, etc.) */ chunkType?: string; /** Name of the function/class/method */ chunkName?: string; /** Full function/method signature */ chunkSignature?: string; /** Extracted docstring or comment */ chunkDocstring?: string; /** Parent name (e.g., class name for methods) */ chunkParent?: string; /** Semantic tags */ chunkTags?: string[]; /** Programming language */ chunkLanguage?: string; } /** * SMCP-098: Structure for existing chunks retrieved for incremental reindexing */ export interface ExistingChunk { /** Chunk ID */ id: string; /** Chunk text content */ text: string; /** Start line in source file */ startLine: number; /** End line in source file */ endLine: number; /** Position-independent hash of chunk text */ chunkHash: string; /** Full embedding vector (needed for reuse) */ vector: number[]; } /** * Vector index type for LanceDB * * - 'ivf_pq': IVF with Product Quantization - good balance of speed and accuracy * - 'none': No vector index (brute force search) - best for small datasets */ export type VectorIndexType = 'ivf_pq' | 'none'; /** * Distance metric for vector similarity search */ export type DistanceMetric = 'l2' | 'cosine' | 'dot'; /** * Configuration options for vector index creation * * SMCP-091: Enable proper IVF-PQ vector index creation for faster search. * Note: GPU acceleration (CUDA/MPS) is NOT available in the Node.js SDK * as of LanceDB v0.23.0. Index building runs on CPU only. */ export interface VectorIndexConfig { /** * The type of vector index to create. * Default is 'ivf_pq' for datasets >= 10K chunks, 'none' otherwise. */ indexType?: VectorIndexType; /** * The number of IVF partitions to create. * This value should scale with the number of rows in the dataset. * Default: sqrt(numRows), min 1, max 256 */ numPartitions?: number; /** * Number of sub-vectors for PQ compression. * Controls how much the vector is compressed. * Default: dimension / 16 (or dimension / 8 if not divisible by 16) */ numSubVectors?: number; /** * Distance metric to use for the index. * Must match the metric used during search. * Default: 'l2' (Euclidean distance) */ distanceType?: DistanceMetric; /** * Max iterations for IVF kmeans training. * Default: 50 */ maxIterations?: number; /** * Sample rate for kmeans training. * Total training vectors = sampleRate * numPartitions * Default: 256 */ sampleRate?: number; } /** * Information about a created vector index */ export interface VectorIndexInfo { /** Whether a vector index exists */ hasIndex: boolean; /** The type of index if one exists */ indexType?: VectorIndexType; /** Number of partitions (for IVF index) */ numPartitions?: number; /** Number of sub-vectors (for PQ index) */ numSubVectors?: number; /** Distance metric used */ distanceType?: DistanceMetric; /** Time taken to create the index in milliseconds */ indexCreationTimeMs?: number; /** Total chunks at time of index creation */ chunkCount?: number; } /** * Minimum chunk count to create a vector index. * Below this threshold, brute-force search is fast enough. */ export declare const MIN_CHUNKS_FOR_INDEX = 10000; /** * Maximum number of partitions for IVF index. * More partitions slow down the partition selection phase. */ export declare const MAX_IVF_PARTITIONS = 256; /** * Default sample rate for kmeans training. */ export declare const DEFAULT_SAMPLE_RATE = 256; /** * Default max iterations for kmeans training. */ export declare const DEFAULT_MAX_ITERATIONS = 50; /** Name of the table storing project chunks */ declare const TABLE_NAME = "project_docs"; /** * @deprecated Use CODE_VECTOR_DIMENSION or DOCS_VECTOR_DIMENSION instead. * Default embedding vector dimension (kept for backward compatibility) */ declare const VECTOR_DIMENSION = 384; /** Vector dimension for code embeddings (384 dims from BGE-small model) */ declare const CODE_VECTOR_DIMENSION = 384; /** Vector dimension for docs embeddings (768 dims from BGE-base model) */ declare const DOCS_VECTOR_DIMENSION = 768; /** Stale lockfile threshold in milliseconds (5 minutes) */ declare const STALE_LOCKFILE_AGE_MS: number; /** * Detect and remove stale lockfiles in the database directory * * Uses async file operations with partial TOCTOU mitigation: * 1. First checks if directory exists using async access * 2. Opens lockfile with 'r+' mode to verify no one else is using it * 3. Only deletes after successfully acquiring the file handle * * BUG #8 LIMITATION DOCUMENTATION: * There is an inherent TOCTOU race between closing the file handle (fd.close()) * and unlinking the lockfile (fs.unlink()). Another process could acquire the * lock in this ~1ms window. This is acceptable because: * * 1. MCP servers are designed as one-per-project - multiple concurrent indexing * processes on the same project is not a supported use case * 2. The race window is very small (~1ms between close and unlink) * 3. Platform-specific atomic locks (flock on Unix, LockFileEx on Windows) * would add significant complexity for minimal benefit in our use case * 4. The stale lockfile cleanup is a recovery mechanism for crashed processes, * not a primary locking mechanism - the main lock is held by LanceDB itself * * If multi-process safety becomes critical, consider: * - flock() on Unix via node-flock or similar * - LockFileEx() on Windows via native bindings * - External lock manager process * - Advisory file locking with proper OS support * * @param dbPath - Path to the LanceDB directory */ declare function cleanupStaleLockfiles(dbPath: string): Promise; /** * Normalize distance to similarity score (0.0 - 1.0) * * LanceDB returns L2 distance where smaller is better. * We convert to similarity score where larger is better. * * @param distance - L2 distance from LanceDB * @returns Similarity score (0.0 - 1.0) */ declare function distanceToScore(distance: number): number; /** * Convert a glob pattern to SQL LIKE pattern * * @deprecated Use globToSafeLikePattern from utils/sql.ts for better SQL injection protection. * This function is kept for backward compatibility but does not properly escape * LIKE wildcards (%, _, [) in literal parts of the pattern. * * @param globPattern - Glob pattern (e.g., "*.ts", "src/*.ts") * @returns SQL LIKE pattern */ declare function globToLikePattern(globPattern: string): string; /** * LanceDB Store wrapper for vector search operations * * Provides a high-level interface for storing and searching code chunk embeddings. * Handles database lifecycle, CRUD operations, and vector similarity search. * * @example * ```typescript * // Use default dimension (384 for code) * const store = new LanceDBStore('/path/to/index'); * await store.open(); * * // Or specify a custom dimension * const docsStore = new LanceDBStore('/path/to/index', DOCS_VECTOR_DIMENSION); * await docsStore.open(); * * // Insert chunks * await store.insertChunks(chunks); * * // Search * const results = await store.search(queryVector, 10); * * await store.close(); * ``` */ export declare class LanceDBStore { private indexPath; private dbPath; private db; private table; private isOpen; /** The vector dimension for this store */ private readonly vectorDimension; /** Mutex for protecting concurrent database operations */ private readonly mutex; /** Reference to cleanup handler for unregistration */ private cleanupHandler; /** * Create a new LanceDBStore instance * * @param indexPath - Path to the index directory (e.g., ~/.mcp/search/indexes/) * @param vectorDimension - Dimension of embedding vectors (defaults to CODE_VECTOR_DIMENSION = 384) */ constructor(indexPath: string, vectorDimension?: number); /** * Get the vector dimension for this store * @returns The dimension of embedding vectors used by this store */ getVectorDimension(): number; /** * Open the database connection * * Creates the database directory if it doesn't exist. * Cleans up any stale lockfiles before connecting. * Creates the table with correct schema if it doesn't exist. */ open(): Promise; /** * Close the database connection * * Safe to call multiple times - will be a no-op if already closed. */ close(): Promise; /** * Delete the entire database * * Removes all data and the database directory. * The store will be closed after this operation. */ delete(): Promise; /** * Ensure the database is open */ private ensureOpen; /** * Ensure the table exists, creating it if necessary with initial data */ private ensureTable; /** * Get the table, throwing if it doesn't exist */ private getTable; /** * Insert chunks into the database * * Handles large batches efficiently by splitting into smaller batches. * Creates the table on first insert if it doesn't exist. * Protected by mutex to prevent concurrent write operations. * * @param chunks - Array of chunk records to insert */ insertChunks(chunks: ChunkRecord[]): Promise; /** * Internal method to insert chunks in batches */ private insertChunksInternal; /** * Delete all chunks for a given file path * Protected by mutex to prevent concurrent write operations. * * @param relativePath - Relative path of the file (forward-slash separated) * @returns Number of chunks deleted */ deleteByPath(relativePath: string): Promise; /** * Get list of all indexed file paths * Protected by mutex to ensure consistent reads during concurrent operations. * Uses limit to avoid unbounded memory usage (Bug #11). * * @param limit - Maximum number of unique paths to return (default: 10000) * @returns Array of unique file paths */ getIndexedFiles(limit?: number): Promise; /** * Count total number of chunks in the database * * @returns Total chunk count */ countChunks(): Promise; /** * Count number of unique indexed files * * @returns Number of unique files */ countFiles(): Promise; /** * Perform vector similarity search * Protected by mutex to prevent reads during concurrent write operations. * * BUG #23 FIX: Added top_k upper bound validation. While Zod validates at * the tool level (1-50), direct calls to this method could pass arbitrary * values. We enforce a reasonable maximum to prevent resource exhaustion. * * @param queryVector - Query embedding vector (dimension must match store's vectorDimension) * @param topK - Maximum number of results to return (default: 10, max: 100) * @returns Search results sorted by similarity score (descending) */ search(queryVector: number[], topK?: number): Promise; /** * Search for files matching a glob pattern * Protected by mutex to ensure consistent reads during concurrent operations. * * @param pattern - Glob pattern (e.g., "*.ts", "src/*.ts") * @param limit - Maximum number of results (default: 20) * @returns Array of matching file paths */ searchByPath(pattern: string, limit?: number): Promise; /** * Retrieve chunks by their IDs * Used for hybrid search result fusion. * * BUG #13 FIX: Added UUID format validation for defense in depth. * IDs are expected to be UUIDv4 format (generated by system), but * validating format provides an extra layer of security. * * @param ids - Array of chunk IDs to retrieve (expected UUID format) * @returns Map of ID to SearchResult (for found chunks) */ getChunksById(ids: string[]): Promise>; /** * Get all chunk IDs from the store * Used for FTS index rebuilding. * * @returns Array of all chunk IDs */ getAllChunkIds(): Promise; /** * Get all chunks with their text content * Used for FTS index rebuilding. * * @returns Array of {id, path, text} objects */ getAllChunksForFTS(): Promise>; /** * Calculate optimal index parameters based on dataset size * * Uses adaptive parameters to balance index build time and search quality: * - numPartitions: sqrt(numRows), clamped to [1, MAX_IVF_PARTITIONS] * - numSubVectors: dimension / 16 (or / 8 if not divisible) * * @param numRows - Number of rows in the dataset * @returns Optimized IVF-PQ parameters */ private calculateIndexParams; /** * Create a vector index on the table for faster similarity search * * SMCP-091: Creates an IVF-PQ index for efficient approximate nearest neighbor search. * This significantly improves search performance for large datasets (>10K chunks). * * Note: GPU acceleration (CUDA/MPS) is NOT available in the LanceDB Node.js SDK * as of v0.23.0. Index building runs on CPU only. When LanceDB adds GPU support * to the Node.js SDK, we can enable it here. * * @param config - Optional configuration for the index * @returns Information about the created index * * @example * ```typescript * const store = new LanceDBStore('/path/to/index'); * await store.open(); * await store.insertChunks(chunks); * * // Create index with default adaptive parameters * const indexInfo = await store.createVectorIndex(); * * // Or with custom parameters * const indexInfo = await store.createVectorIndex({ * numPartitions: 128, * numSubVectors: 24, * distanceType: 'l2' * }); * ``` */ createVectorIndex(config?: VectorIndexConfig): Promise; /** * Get information about the existing vector index * * @returns Index information or null if no index exists */ getVectorIndexInfo(): Promise; /** * Get the storage size of the database in bytes * * Uses async file operations to avoid blocking the event loop. * * @returns Size in bytes */ getStorageSize(): Promise; /** * Check if the store has been opened */ get opened(): boolean; /** * Check if the table exists and has data */ hasData(): Promise; /** * Get all chunks for a specific file path * * Used for incremental reindexing to retrieve existing chunks * so we can compare them with new chunks and avoid re-embedding unchanged content. * * @param relativePath - Relative path of the file (forward-slash separated) * @returns Array of existing chunks with their embeddings and hashes */ getChunksForFile(relativePath: string): Promise; /** * Delete chunks by their IDs * * Used for surgical removal of specific chunks during incremental reindexing. * * @param ids - Array of chunk IDs to delete * @returns Number of chunks deleted */ deleteChunksByIds(ids: string[]): Promise; /** * Update chunk metadata (line numbers) without re-embedding * * Used for moved chunks that have the same content but different positions. * This avoids expensive re-embedding for content that hasn't changed. * * @param chunkId - ID of the chunk to update * @param metadata - New metadata to apply * @returns true if update was successful */ updateChunkMetadata(chunkId: string, metadata: { startLine: number; endLine: number; }): Promise; } export { TABLE_NAME, VECTOR_DIMENSION, CODE_VECTOR_DIMENSION, DOCS_VECTOR_DIMENSION, STALE_LOCKFILE_AGE_MS, distanceToScore, globToLikePattern, cleanupStaleLockfiles, }; //# sourceMappingURL=lancedb.d.ts.map