/** * Docs Index Manager Module * * Orchestrator for documentation indexing operations. Coordinates scanning doc files, * chunking with prose-optimized parameters, generating embeddings, and storing in the * docs LanceDB table. * * Features: * - Full documentation indexing with progress reporting * - Incremental updates (single file and batch delta) * - Prose-optimized chunking with larger chunks and more overlap * - Separate storage from code index */ import { DocsDeltaResult } from '../storage/docsFingerprints.js'; import { Config } from '../storage/config.js'; import { IndexingPolicy } from './indexPolicy.js'; import { type ExtractedComment } from './commentExtractor.js'; /** * Batch size for processing doc files * 50 files per batch balances memory usage and progress granularity */ export declare const DOC_FILE_BATCH_SIZE = 50; /** * Streaming batch size for high memory situations * When memory is above 80%, we use this smaller batch size */ export declare const DOC_STREAMING_BATCH_SIZE = 3; /** * Progress phases during docs indexing */ export type DocsIndexPhase = 'scanning' | 'chunking' | 'embedding' | 'storing'; /** * Progress information during docs indexing operations */ export interface DocsIndexProgress { /** Current phase of the indexing operation */ phase: DocsIndexPhase; /** Current item number being processed */ current: number; /** Total items to process in this phase */ total: number; /** Current file being processed (optional) */ currentFile?: string; } /** * Callback function for progress updates */ export type DocsProgressCallback = (progress: DocsIndexProgress) => void; /** * Result of a docs indexing operation */ export interface DocsIndexResult { /** Whether the operation completed successfully */ success: boolean; /** Number of doc files indexed */ filesIndexed: number; /** Number of chunks created */ chunksCreated: number; /** Total duration in milliseconds */ durationMs: number; /** Errors encountered (if any) */ errors?: string[]; /** Warning message (e.g., when 0 docs indexed despite files existing) */ warning?: string; /** Number of doc files found by glob before filtering (for diagnostics) */ globFilesFound?: number; /** Number of code files scanned for comments (SMCP-100) */ codeFilesScanned?: number; /** Number of comments extracted from code (SMCP-100) */ commentsExtracted?: number; } /** * Statistics for a docs index */ export interface DocsStats { /** Total number of indexed doc files */ totalDocs: number; /** Total number of doc chunks */ totalDocChunks: number; /** Storage size in bytes */ storageSizeBytes: number; } /** * Scan a project directory for documentation files * * Finds all files matching doc patterns (*.md, *.txt) that should be indexed based on: * - Hardcoded deny patterns (always excluded) * - User exclude patterns from config * - Gitignore rules (if respectGitignore is enabled) * * @param projectPath - Absolute path to the project root * @param policy - Initialized IndexingPolicy instance * @param config - Project configuration * @param onProgress - Optional callback for progress updates * @returns Scan result with files array and diagnostic info */ export interface ScanDocFilesResult { /** Files that passed filtering and will be indexed */ files: string[]; /** Total files found by glob before filtering */ globFilesFound: number; } export declare function scanDocFiles(projectPath: string, policy: IndexingPolicy, config: Config, onProgress?: DocsProgressCallback): Promise; /** * Scan code files and extract documentation comments * * Extracts JSDoc, docstrings, and other documentation comments from code files * to make them searchable via the docs index. * * @param projectPath - Absolute path to the project root * @param policy - Initialized IndexingPolicy instance * @param config - Project configuration * @param onProgress - Optional callback for progress updates * @returns Array of extracted comments with metadata */ export declare function extractCodeComments(projectPath: string, policy: IndexingPolicy, config: Config, onProgress?: DocsProgressCallback): Promise<{ comments: ExtractedComment[]; codeFilesScanned: number; errors: string[]; }>; /** * Create a full docs index for a project * * Pipeline stages: * 1. Initialize components (policy, store, fingerprints) * 2. Scan doc files (apply policy) * 3. Process files in batches (chunk, embed) * 4. Store in DocsLanceDB * 5. Update docs-fingerprints.json * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param onProgress - Optional progress callback * @returns DocsIndexResult with operation details */ export declare function createDocsIndex(projectPath: string, indexPath: string, onProgress?: DocsProgressCallback): Promise; /** * Update a single doc file in the index * * Handles three cases: * - File exists and changed: Delete old chunks, add new ones * - File exists and new: Add chunks * - File deleted: Remove chunks * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param relativePath - Relative path of the file to update */ export declare function updateDocFile(projectPath: string, indexPath: string, relativePath: string): Promise; /** * Remove a doc file from the index * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param relativePath - Relative path of the file to remove */ export declare function removeDocFile(projectPath: string, indexPath: string, relativePath: string): Promise; /** * Apply a delta (batch of changes) to the docs index * * Processes: * - Added files: Chunk, embed, and insert * - Modified files: Delete old chunks, then add new ones * - Removed files: Delete chunks and fingerprints * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory * @param delta - DocsDeltaResult with added, modified, and removed files * @param onProgress - Optional progress callback * @returns DocsIndexResult with operation details */ export declare function applyDocsDelta(projectPath: string, indexPath: string, delta: DocsDeltaResult, onProgress?: DocsProgressCallback): Promise; /** * Docs Index Manager class for managing documentation indexes * * Provides high-level operations for: * - Creating and rebuilding docs indexes * - Incremental doc file updates * - Index deletion * - Status and statistics * * @example * ```typescript * const manager = new DocsIndexManager('/path/to/project', '/path/to/index'); * await manager.initialize(); * * // Create a new docs index * const result = await manager.createDocsIndex((progress) => { * console.log(`${progress.phase}: ${progress.current}/${progress.total}`); * }); * * // Update a single doc file * await manager.updateDocFile('docs/README.md'); * * // Get statistics * const stats = await manager.getDocsStats(); * console.log(`Indexed ${stats.totalDocs} doc files`); * * await manager.close(); * ``` */ export declare class DocsIndexManager { private readonly projectPath; private readonly indexPath; private store; private fingerprints; private isInitialized; /** * Create a new DocsIndexManager instance * * @param projectPath - Absolute path to the project root * @param indexPath - Absolute path to the index directory (optional - derived from projectPath if not provided) */ constructor(projectPath: string, indexPath?: string); /** * Initialize the manager (open store and load fingerprints) */ initialize(): Promise; /** * Close the manager (close store) */ close(): Promise; /** * Get the project path */ getProjectPath(): string; /** * Get the index path */ getIndexPath(): string; /** * Create a new docs index for the project * * Creates a complete docs index from scratch, deleting any existing docs index data. * * @param onProgress - Optional callback for progress updates * @returns DocsIndexResult with operation details */ createDocsIndex(onProgress?: DocsProgressCallback): Promise; /** * Rebuild the docs index from scratch * * Alias for createDocsIndex - deletes existing docs index and recreates it. * * @param onProgress - Optional callback for progress updates * @returns DocsIndexResult with operation details */ rebuildDocsIndex(onProgress?: DocsProgressCallback): Promise; /** * Delete the docs index completely * * Removes all docs index data including: * - DocsLanceDB database * - Docs fingerprints */ deleteDocsIndex(): Promise; /** * Update a single doc file in the index * * @param relativePath - Relative path of the file (forward-slash separated) */ updateDocFile(relativePath: string): Promise; /** * Remove a doc file from the index * * @param relativePath - Relative path of the file (forward-slash separated) */ removeDocFile(relativePath: string): Promise; /** * Apply a batch of changes to the docs index * * @param delta - DocsDeltaResult with added, modified, and removed files * @param onProgress - Optional callback for progress updates * @returns DocsIndexResult with operation details */ applyDelta(delta: DocsDeltaResult, onProgress?: DocsProgressCallback): Promise; /** * Check if a docs index exists for this project * * @returns true if the docs index exists and has data */ isDocsIndexed(): Promise; /** * Get docs index statistics * * @returns DocsStats with file count, chunk count, etc. */ getDocsStats(): Promise; /** * Scan for doc files in the project * * @param config - Optional config override * @param onProgress - Optional progress callback * @returns Scan result with files array and diagnostic info */ scanDocFiles(config?: Config, onProgress?: DocsProgressCallback): Promise; } export type { DocsDeltaResult } from '../storage/docsFingerprints.js'; //# sourceMappingURL=docsIndexManager.d.ts.map