/** * Index Manager Module * * Central orchestrator for all indexing operations. Coordinates file discovery, * policy filtering, chunking, embedding, and storage. Handles both full indexing * and incremental updates. * * Features: * - Full project indexing with progress reporting * - Incremental updates (single file and batch delta) * - Atomic operations (rollback on failure) * - Memory management with file batching */ import { DeltaResult } from '../storage/fingerprints.js'; import { Config } from '../storage/config.js'; import { IndexingPolicy } from './indexPolicy.js'; /** * Batch size for processing files * 50 files per batch balances memory usage and progress granularity */ export declare const FILE_BATCH_SIZE = 50; /** * Streaming batch size for high memory situations * When memory is above 80%, we use this smaller batch size to avoid * accumulating too much data in memory before writing to DB. * 3 files at a time provides a good balance between safety and speed. */ export declare const STREAMING_BATCH_SIZE = 3; /** * Progress phases during indexing */ export type IndexPhase = 'scanning' | 'chunking' | 'embedding' | 'storing'; /** * Progress information during indexing operations */ export interface IndexProgress { /** Current phase of the indexing operation */ phase: IndexPhase; /** Current item number being processed */ current: number; /** Total items to process in this phase */ total: number; /** Current file being processed (optional) */ currentFile?: string; } /** * Callback function for progress updates */ export type ProgressCallback = (progress: IndexProgress) => void; /** * Result of a file scan operation */ export interface ScanResult { /** Files that passed filtering and will be indexed */ files: string[]; /** Total files found in project (before filtering) */ totalFilesInProject: number; /** Number of files excluded by policy */ excludedFiles: number; } /** * Result of an indexing operation */ export interface IndexResult { /** Whether the operation completed successfully */ success: boolean; /** Number of files indexed */ filesIndexed: number; /** Number of chunks created */ chunksCreated: number; /** Total duration in milliseconds */ durationMs: number; /** Errors encountered (if any) */ errors?: string[]; /** Total files found in project (before filtering) */ totalFilesInProject?: number; /** Number of files excluded by policy */ excludedFiles?: number; /** Number of code files indexed (when docs indexing is enabled) */ codeFilesIndexed?: number; /** Number of doc files indexed (when docs indexing is enabled) */ docsFilesIndexed?: number; /** Number of doc chunks created (when docs indexing is enabled) */ docsChunksCreated?: number; } /** * Statistics for an index */ export interface IndexStats { /** Total number of indexed files */ totalFiles: number; /** Total number of chunks */ totalChunks: number; /** Storage size in bytes */ storageSizeBytes: number; /** ISO timestamp of last full index */ lastFullIndex: string; /** ISO timestamp of last incremental update (optional) */ lastIncrementalUpdate?: string; } /** * Scan a project directory for indexable files * * Recursively finds all files that should be indexed based on: * - Hardcoded deny patterns (always excluded) * - User exclude patterns from config * - Gitignore rules (if respectGitignore is enabled) * - Binary file detection * - File size limits * * @param projectPath - Absolute path to the project root * @param policy - Initialized IndexingPolicy instance * @param config - Project configuration * @param onProgress - Optional callback for progress updates * @returns ScanResult with files to index and stats about total/excluded files */ export declare function scanFiles(projectPath: string, policy: IndexingPolicy, config: Config, onProgress?: ProgressCallback): Promise; /** * Create a full index for a project * * Pipeline stages: * 1. Initialize components (policy, store, fingerprints) * 2. Scan files (apply policy) * 3. Process files in batches (chunk, embed) * 4. Store in LanceDB * 5. Update fingerprints and metadata * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param onProgress - Optional progress callback * @returns IndexResult with operation details */ export declare function createFullIndex(projectPath: string, indexPath: string, onProgress?: ProgressCallback): Promise; /** * Update a single file in the index * * SMCP-098: Uses incremental reindexing to avoid re-embedding unchanged chunks. * This dramatically improves performance for large files with small edits. * * Handles three cases: * - File exists and changed: Use incremental diff to minimize re-embedding * - File exists and new: Add all chunks (full embedding) * - File deleted: Remove chunks * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param relativePath - Relative path of the file to update */ export declare function updateFile(projectPath: string, indexPath: string, relativePath: string): Promise; /** * Remove a file from the index * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param relativePath - Relative path of the file to remove */ export declare function removeFile(projectPath: string, indexPath: string, relativePath: string): Promise; /** * Apply a delta (batch of changes) to the index * * Processes: * - Added files: Chunk, embed, and insert * - Modified files: Delete old chunks, then add new ones * - Removed files: Delete chunks and fingerprints * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param delta - DeltaResult with added, modified, and removed files * @param onProgress - Optional progress callback * @returns IndexResult with operation details */ export declare function applyDelta(projectPath: string, indexPath: string, delta: DeltaResult, onProgress?: ProgressCallback): Promise; /** * Index Manager class for managing project indexes * * Provides high-level operations for: * - Creating and rebuilding indexes * - Incremental file updates * - Index deletion * - Status and statistics * * @example * ```typescript * const manager = new IndexManager('/path/to/project', '/path/to/index'); * * // Create a new index * const result = await manager.createIndex((progress) => { * console.log(`${progress.phase}: ${progress.current}/${progress.total}`); * }); * * // Update a single file * await manager.updateFile('src/utils/helper.ts'); * * // Get statistics * const stats = await manager.getStats(); * console.log(`Indexed ${stats.totalFiles} files`); * ``` */ export declare class IndexManager { private readonly projectPath; private readonly indexPath; /** * Create a new IndexManager instance * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory (optional - derived from projectPath if not provided) */ constructor(projectPath: string, indexPath?: string); /** * Get the project path */ getProjectPath(): string; /** * Get the index path */ getIndexPath(): string; /** * Create a new index for the project * * Creates a complete index from scratch, deleting any existing index data. * * @param onProgress - Optional callback for progress updates * @returns IndexResult with operation details */ createIndex(onProgress?: ProgressCallback): Promise; /** * Rebuild the index from scratch * * Alias for createIndex - deletes existing index and recreates it. * * @param onProgress - Optional callback for progress updates * @returns IndexResult with operation details */ rebuildIndex(onProgress?: ProgressCallback): Promise; /** * Delete the index completely * * Removes all index data including: * - LanceDB database * - Fingerprints * - Metadata * - Config (preserves if you want to keep settings) */ deleteIndex(): Promise; /** * Update a single file in the index * * @param relativePath - Relative path of the file (forward-slash separated) */ updateFile(relativePath: string): Promise; /** * Remove a file from the index * * @param relativePath - Relative path of the file (forward-slash separated) */ removeFile(relativePath: string): Promise; /** * Apply a batch of changes to the index * * @param delta - DeltaResult with added, modified, and removed files * @param onProgress - Optional callback for progress updates * @returns IndexResult with operation details */ applyDelta(delta: DeltaResult, onProgress?: ProgressCallback): Promise; /** * Check if an index exists for this project * * @returns true if the index exists and has metadata */ isIndexed(): Promise; /** * Get index statistics * * @returns IndexStats with file count, chunk count, etc. * @throws MCPError if index doesn't exist */ getStats(): Promise; } export type { DeltaResult } from '../storage/fingerprints.js'; //# sourceMappingURL=indexManager.d.ts.map