import type { EmbeddingBackend } from '../embeddings/index.js'; import { type ConceptCluster, type ClusteringResult, type ClusteringOptions } from './clustering.js'; /** * Represents a chunk of code that has been indexed. * Each chunk contains a portion of a source file with its location metadata. */ export interface CodeChunk { /** Unique identifier for this chunk (format: filepath:startLine-endLine) */ id: string; /** Relative path to the source file from the project root */ filepath: string; /** The actual source code content of this chunk */ content: string; /** Starting line number in the source file (1-indexed) */ startLine: number; /** Ending line number in the source file (1-indexed) */ endLine: number; /** Programming language of the source code */ language: string; /** Vector embedding for semantic search (populated during indexing) */ embedding?: number[]; /** Type of code symbol (function, class, method, etc.) - from AST chunking */ symbolType?: 'function' | 'class' | 'method' | 'interface' | 'type' | 'variable' | 'import' | 'other'; /** Name of the code symbol (e.g., 'UserService', 'MyClass.constructor') - from AST chunking */ symbolName?: string; } /** * Statistics about chunking method usage during indexing. * Tracks how many files used AST-aware vs line-based chunking. */ export interface ChunkingStats { /** Number of files chunked with AST parsing (TypeScript/JavaScript) */ astChunked: number; /** Number of files chunked with tree-sitter parsing (Python, Go, etc.) */ treeSitterChunked: number; /** Number of files that fell back to line-based chunking */ lineBasedChunked: number; /** Files where AST parsing failed and fell back to line-based */ astFallbacks: string[]; /** Files where tree-sitter parsing failed and fell back to line-based */ treeSitterFallbacks: string[]; } /** * Status information about the code index. */ export interface IndexStatus { /** Whether the codebase has been indexed */ indexed: boolean; /** Number of files that have been indexed */ fileCount: number; /** Total number of code chunks in the index */ chunkCount: number; /** ISO timestamp of the last index update, or null if never indexed */ lastUpdated: string | null; /** Path to the LanceDB index directory */ indexPath: string; /** Name of the embedding backend used */ embeddingBackend?: string; /** Model identifier used for embeddings */ embeddingModel?: string; /** Whether index corruption was detected */ corrupted?: boolean; /** Description of detected corruption */ corruptionReason?: string; /** Whether the current embedding backend differs from the indexed backend */ backendMismatch?: boolean; /** Description of the backend mismatch */ backendMismatchReason?: string; /** Whether the index is currently being rebuilt due to backend change */ isIndexing?: boolean; /** Statistics about chunking methods used during indexing */ chunkingStats?: ChunkingStats; } /** * Options for searching similar code. */ export interface SearchSimilarOptions { /** File path to find similar code for (relative to project root) */ filepath?: string; /** Starting line number (1-indexed, requires filepath) */ startLine?: number; /** Ending line number (1-indexed, requires filepath) */ endLine?: number; /** Code snippet to find similar code for (alternative to filepath) */ code?: string; /** Maximum number of results to return (default: 10) */ limit?: number; /** Minimum similarity score threshold 0-1 (default: 0) */ threshold?: number; /** Exclude the source chunk from results (default: true) */ excludeSelf?: boolean; } /** * Result from similar code search, includes similarity score. */ export interface SimilarCodeResult extends CodeChunk { /** Similarity score from 0 to 1 (1 = identical) */ similarity: number; } /** * Options for searching code. */ export interface SearchOptions { /** Natural language query to search for */ query: string; /** Maximum number of results to return (default: 10) */ limit?: number; /** Glob pattern to filter results by file path (e.g., "src/\**", "!test/\**") */ pathPattern?: string; /** Filter results to specific languages (e.g., ["typescript", "javascript"]) */ languages?: string[]; } /** * Progress information during indexing operations. * Used to report status to callers via the progress callback. */ export interface IndexProgress { /** Current phase of the indexing process */ phase: 'scanning' | 'chunking' | 'embedding' | 'storing' | 'complete'; /** Current progress count within the phase */ current: number; /** Total items to process in the current phase */ total: number; /** Human-readable status message */ message: string; /** Estimated time remaining in seconds */ etaSeconds?: number; } /** * Callback function for receiving indexing progress updates. */ export type ProgressCallback = (progress: IndexProgress) => void; /** * Summary of the codebase structure and concept areas */ export interface CodebaseSummary { /** Total number of files indexed */ totalFiles: number; /** Total number of code chunks */ totalChunks: number; /** Languages detected in the codebase */ languages: { language: string; fileCount: number; chunkCount: number; }[]; /** Discovered concept clusters */ concepts: ConceptCluster[]; /** Quality score for the clustering (silhouette score, -1 to 1) */ clusteringQuality: number; /** Timestamp when summary was generated */ generatedAt: string; } /** * Code indexer that uses LanceDB for vector storage and semantic search. * * Provides functionality to: * - Index a codebase by chunking files and generating embeddings * - Perform hybrid semantic + keyword search * - Support incremental indexing (only re-index changed files) * * @example * ```typescript * const backend = await createEmbeddingBackend(); * const indexer = new CodeIndexer('/path/to/project', backend); * await indexer.initialize(); * * // Index the codebase * await indexer.indexCodebase(); * * // Search for code * const results = await indexer.search('authentication middleware'); * ``` */ export declare class CodeIndexer { private db; private table; private metadataTable; private embeddingBackend; private indexPath; private projectPath; private config; /** LRU cache for query embeddings with TTL to avoid recomputing identical queries */ private queryEmbeddingCache; /** Cache for query results to deduplicate semantically similar queries */ private queryResultCache; /** Tracks chunking method usage during current indexing operation */ private currentChunkingStats; /** Create empty chunking stats */ private createEmptyChunkingStats; constructor(projectPath: string, embeddingBackend: EmbeddingBackend); initialize(): Promise; private get metadataPath(); private get checkpointPath(); /** * Save indexing checkpoint to disk for crash recovery. * Strips chunk content to reduce checkpoint size - content is re-read on resume. */ private saveCheckpoint; /** * Re-read chunk content from source files. * Used when resuming from a checkpoint that has stripped content. */ private rehydrateChunkContent; /** * Load indexing checkpoint from disk. * Returns null if no checkpoint exists or if it's invalid. */ private loadCheckpoint; /** * Clear the indexing checkpoint file. */ private clearCheckpoint; /** * Save index metadata to disk */ private saveIndexMetadata; /** * Load index metadata from disk */ private loadIndexMetadata; /** * Get the modification time of a file */ private getFileMtime; /** * Collect modification times for multiple files. * Used for checkpoint freshness validation. */ private collectFileMtimes; /** * Validate that checkpoint files haven't been modified since checkpoint creation. * Returns true if checkpoint is fresh, false if files have changed. */ private validateCheckpointFreshness; /** * Get stored metadata for all indexed files */ private getStoredMetadata; /** * Detect which files have been added, modified, or deleted */ private detectFileChanges; /** * Get all project files matching the configured patterns. * Used for change detection and staleness checking. */ private getProjectFiles; /** * Check if the index is stale (files have been modified since last index). * Returns true if any files have been added, modified, or deleted. * This is a lightweight check that only compares file modification times. */ checkIfStale(): Promise<{ stale: boolean; reason?: string; }>; /** * Save metadata for indexed files */ private saveFileMetadata; getStatus(): Promise; /** * Check if the current embedding backend differs from the one used to create the index. * Returns mismatch status and reason if mismatched. */ private checkBackendMismatch; /** * Validate index integrity by checking metadata consistency. * Returns corruption status and reason if corrupted. */ private validateIndexIntegrity; indexCodebase(patterns?: string[], excludePatterns?: string[], forceReindex?: boolean, onProgress?: ProgressCallback, autoRepair?: boolean): Promise<{ filesIndexed: number; chunksCreated: number; incremental: boolean; repaired?: boolean; }>; /** * Perform a full reindex of all files */ private indexFull; /** * Log chunking statistics summary, with warnings for fallbacks */ private logChunkingStats; /** * Perform incremental indexing - only process changed files */ private indexIncremental; /** * Resume indexing from a saved checkpoint. * Handles each checkpoint phase appropriately. */ private resumeFromCheckpoint; /** * Generate embeddings for chunks in batches. * Supports configurable batch size and delay between batches for rate limiting. */ private embedChunks; private chunkFile; /** * Chunk a file using AST-aware parsing */ private chunkFileWithAST; /** * Chunk a file using tree-sitter AST parsing (Python, Go, Rust, Java, Kotlin) */ private chunkFileWithTreeSitter; /** * Chunk a file using line-based splitting (fallback) */ private chunkFileByLines; private getLanguage; /** * Get query embedding from cache or compute it. * Uses TTLCache for LRU eviction and TTL-based expiration. */ private getQueryEmbedding; /** * Check if search options match (for cache key comparison). */ private searchOptionsMatch; /** * Find a cached result for a semantically similar query. * Returns the cached results if found, or null if no similar query is cached. */ private findCachedQueryResult; /** * Cache a query result for future deduplication. */ private cacheQueryResult; /** * Clear the query result cache. * Called when the index is updated to ensure fresh results. */ clearQueryResultCache(): void; /** * Check if a filepath matches a glob pattern. * Supports negation patterns starting with '!'. */ private matchesPathPattern; /** * Search with options object */ search(options: SearchOptions): Promise; /** * Search with query string and optional limit (backward compatible) */ search(query: string, limit?: number): Promise; /** * Calculate keyword match score for hybrid search */ private calculateKeywordScore; /** * Find code chunks semantically similar to a given code snippet or file location. * This is useful for finding duplicate logic, similar implementations, or related code. */ searchSimilar(options: SearchSimilarOptions): Promise; clearIndex(): Promise; private get clusteringMetadataPath(); /** * Clear clustering metadata file */ private clearClusteringMetadata; /** * Save clustering result to metadata file. * Includes index checksum for cache invalidation. */ private saveClusteringMetadata; /** * Load clustering result from metadata file. * Validates that the cached clustering matches the current index checksum. * Returns null if cache is stale or missing. */ private loadClusteringMetadata; /** * Cluster the indexed codebase into semantic concept areas. * Uses k-means clustering on embeddings to discover related code groups. */ clusterConcepts(options?: ClusteringOptions): Promise; /** * List all discovered concept clusters. * Returns cached clustering result if available, otherwise clusters first. */ listConcepts(forceRecluster?: boolean): Promise; /** * Search for code within a specific concept cluster. * Returns chunks that belong to the specified cluster, optionally filtered by query. */ searchByConcept(conceptId: number, query?: string, limit?: number): Promise; /** * Generate a comprehensive summary of the codebase including concept areas. */ summarizeCodebase(clusteringOptions?: ClusteringOptions): Promise; } //# sourceMappingURL=indexer.d.ts.map