/** * Chunking Engine * * Implements text chunking for splitting source files into indexable segments. * Uses recursive character text splitting with configurable chunk size and overlap. * Tracks line numbers for each chunk to enable navigation. * * Based on RFC Section 5.3: Chunking Engine */ import { type ChunkMetadata } from './astChunking.js'; /** * A chunk of text from a source file */ export interface Chunk { /** UUIDv4 unique identifier for the chunk */ id: string; /** Chunk text content */ text: string; /** Source file path (relative to project root) */ path: string; /** Starting line number (1-based) */ startLine: number; /** Ending line number (1-based) */ endLine: number; /** SHA256 hash of the source file content */ contentHash: string; /** Rich metadata from AST parsing (optional) */ metadata?: ChunkMetadata; } /** * Configuration options for text splitting */ export interface SplitOptions { /** Target chunk size in characters (~4000 for ~1000 tokens) */ chunkSize: number; /** Overlap size in characters (~800 for ~200 tokens) */ chunkOverlap: number; /** Separators to use for splitting in priority order */ separators: string[]; } /** * Internal chunk structure with line number tracking */ export interface ChunkWithLines { /** Chunk text content */ text: string; /** Starting line number (1-based) */ startLine: number; /** Ending line number (1-based) */ endLine: number; } /** * Default split options as specified in the RFC * * - chunkSize: ~1000 tokens = ~4000 characters (1 token ~ 4 chars) * - chunkOverlap: ~200 tokens = ~800 characters * - separators: paragraph breaks, line breaks, spaces, then character-level */ export declare const DEFAULT_SPLIT_OPTIONS: SplitOptions; /** * Maximum file size to read entirely into memory (MCP-26) * Files larger than this will be processed with streaming */ export declare const MAX_IN_MEMORY_SIZE: number; /** * Split text recursively using separators in priority order * * Tries each separator in order until chunks are small enough. * Falls back to character-level splitting if no separator works. * * @param text - The text to split * @param options - Split configuration options * @param maxChunks - Maximum number of chunks allowed (default: MAX_CHUNKS_PER_FILE) * @returns Array of text chunks * @throws ResourceLimitError if chunk count exceeds maxChunks * * @example * ```typescript * const chunks = splitText(fileContent); * // Each chunk is ~4000 characters with ~800 character overlap * ``` */ export declare function splitText(text: string, options?: Partial, maxChunks?: number): string[]; /** * Split text with line number tracking for each chunk * * Each chunk includes start and end line numbers (1-based) to enable * navigation back to the source file. * * @param text - The text to split * @param options - Split configuration options * @param maxChunks - Maximum number of chunks allowed (default: MAX_CHUNKS_PER_FILE) * @returns Array of chunks with line number information * @throws ResourceLimitError if chunk count exceeds maxChunks * * @example * ```typescript * const chunks = splitWithLineNumbers(fileContent); * console.log(chunks[0].startLine); // 1 * console.log(chunks[0].endLine); // 42 * ``` */ export declare function splitWithLineNumbers(text: string, options?: Partial, maxChunks?: number): ChunkWithLines[]; /** * Chunk a file into indexable segments * * Reads the file, splits it into chunks, and assigns UUIDs and metadata * to each chunk. The content hash is computed for the entire file content. * * For large files (>10MB), uses streaming to avoid memory explosion (MCP-26). * * @param absolutePath - Absolute path to the file on disk * @param relativePath - Relative path from project root (stored in chunk) * @param options - Optional split configuration * @returns Promise resolving to array of chunks with IDs and metadata * @throws MCPError with FILE_NOT_FOUND if file doesn't exist * @throws MCPError with PERMISSION_DENIED if file can't be read * * @example * ```typescript * const chunks = await chunkFile( * '/Users/dev/project/src/index.ts', * 'src/index.ts' * ); * console.log(chunks[0].id); // 'a1b2c3d4-...' * console.log(chunks[0].path); // 'src/index.ts' * console.log(chunks[0].startLine); // 1 * ``` */ export declare function chunkFile(absolutePath: string, relativePath: string, options?: Partial): Promise; /** * Chunk a file synchronously * * Use this only when async is not possible. * Prefer chunkFile for normal operations. * * @param absolutePath - Absolute path to the file on disk * @param relativePath - Relative path from project root (stored in chunk) * @param options - Optional split configuration * @returns Array of chunks with IDs and metadata * @throws MCPError with FILE_NOT_FOUND if file doesn't exist */ export declare function chunkFileSync(absolutePath: string, relativePath: string, options?: Partial): Chunk[]; /** * Chunking strategy type * * - 'character': Traditional fixed-size chunking with overlap * - 'code-aware': Heuristic-based code-aware chunking (regex patterns) * - 'ast': AST-based chunking using Tree-sitter (richest metadata) */ export type ChunkingStrategy = 'character' | 'code-aware' | 'ast'; /** * Options for smart chunking */ export interface SmartChunkOptions extends Partial { /** Chunking strategy to use */ strategy?: ChunkingStrategy; } /** * Chunk a file using the specified strategy * * When strategy is 'code-aware': * - For supported languages (TS, JS, Python), splits at semantic boundaries * - Falls back to character-based chunking for unsupported languages * - Falls back on any errors * * When strategy is 'character' (default): * - Uses traditional fixed-size chunking with overlap * * @param absolutePath - Absolute path to the file on disk * @param relativePath - Relative path from project root (stored in chunk) * @param options - Chunking options including strategy * @returns Promise resolving to array of chunks with IDs and metadata * * @example * ```typescript * // Use code-aware chunking for TypeScript file * const chunks = await chunkFileWithStrategy( * '/path/to/file.ts', * 'src/file.ts', * { strategy: 'code-aware' } * ); * ``` */ export declare function chunkFileWithStrategy(absolutePath: string, relativePath: string, options?: SmartChunkOptions): Promise; //# sourceMappingURL=chunking.d.ts.map