/** * Markdown Header Chunking Engine (SMCP-099) * * Splits markdown documents by headers for better context preservation. * Instead of arbitrary character-based splits, chunks at semantic boundaries * (h1-h6) so each chunk represents a complete section. * * Features: * - Parse ATX headers (# heading) and setext headers (=== and ---) * - Track header hierarchy for breadcrumb context * - Handle frontmatter (YAML between ---) * - Treat code blocks as atomic units (don't split inside them) * - Sub-chunk large sections while preserving header context */ import type { Chunk } from './chunking.js'; /** * A parsed markdown section */ export interface MarkdownSection { /** Header level (1-6 for h1-h6, 0 for content before any header) */ level: number; /** Header text (empty for level 0) */ title: string; /** Section content (excluding the header line itself) */ content: string; /** Header hierarchy path: ["Installation", "Prerequisites", "Node.js"] */ path: string[]; /** Starting line number (1-based) */ startLine: number; /** Ending line number (1-based) */ endLine: number; } /** * Metadata for markdown chunks */ export interface MarkdownChunkMetadata { /** Header hierarchy path */ headerPath: string[]; /** Header level (1-6, or 0 for pre-header content) */ headerLevel: number; /** Section title (last element of headerPath) */ sectionTitle: string; /** Part number for sub-chunked large sections */ part?: number; /** Total parts for sub-chunked large sections */ totalParts?: number; } /** * Options for markdown chunking */ export interface MarkdownChunkOptions { /** Maximum chunk size in characters (default: 8000) */ maxChunkSize: number; /** Minimum chunk size in characters (default: 500) */ minChunkSize: number; /** Include header path as breadcrumb in chunk text (default: true) */ includeHeaderPath: boolean; /** Chunk overlap in characters for sub-chunked sections (default: 500) */ chunkOverlap: number; } /** * Default markdown chunk options (prose-optimized) */ export declare const DEFAULT_MARKDOWN_CHUNK_OPTIONS: MarkdownChunkOptions; /** * A chunk with markdown-specific metadata */ export interface MarkdownChunk { /** Chunk text content */ text: string; /** Starting line number (1-based) */ startLine: number; /** Ending line number (1-based) */ endLine: number; /** Markdown-specific metadata */ metadata: MarkdownChunkMetadata; } /** * Detect and skip YAML frontmatter at the start of content * * Frontmatter is YAML between --- delimiters at the very start of the file. * * @param content - The markdown content * @returns Object with content without frontmatter and line offset */ export declare function stripFrontmatter(content: string): { content: string; lineOffset: number; }; /** * Find all code block ranges in the content * * Code blocks are fenced with ``` or ~~~ and should be treated as atomic units. * * @param lines - Array of content lines * @returns Array of [startLine, endLine] tuples (0-based indices) */ export declare function findCodeBlockRanges(lines: string[]): Array<[number, number]>; /** * Check if a line index is inside a code block * * @param lineIndex - 0-based line index * @param codeBlockRanges - Array of code block ranges * @returns true if the line is inside a code block */ export declare function isInsideCodeBlock(lineIndex: number, codeBlockRanges: Array<[number, number]>): boolean; /** * Parse an ATX header line * * @param line - The line to parse * @returns Object with level and title, or null if not a header */ export declare function parseATXHeader(line: string): { level: number; title: string; } | null; /** * Check if the next line makes current line a setext header * * Setext headers: * - === (any number) for h1 * - --- (any number, but 3+) for h2 * * @param nextLine - The line following the potential header text * @returns Header level (1 or 2) or null if not a setext underline */ export declare function parseSetextUnderline(nextLine: string): number | null; /** * Parse markdown content into sections * * Handles both ATX headers (# style) and setext headers (=== and --- style). * Respects code blocks (doesn't parse headers inside them). * * @param content - The markdown content to parse * @param lineOffset - Line offset for frontmatter handling (default: 0) * @returns Array of parsed sections */ export declare function parseMarkdownSections(content: string, lineOffset?: number): MarkdownSection[]; /** * Format a section for embedding, optionally including header path breadcrumb * * @param section - The section to format * @param options - Chunk options * @returns Formatted section text */ export declare function formatSection(section: MarkdownSection, options: MarkdownChunkOptions): string; /** * Sub-chunk a large section while preserving header context * * Splits by paragraphs first, then by sentences if needed. * Each sub-chunk includes the section header for context. * * @param section - The large section to sub-chunk * @param options - Chunk options * @returns Array of markdown chunks */ export declare function subChunkSection(section: MarkdownSection, options: MarkdownChunkOptions): MarkdownChunk[]; /** * Chunk markdown content by headers * * Main entry point for markdown-aware chunking. Parses headers, * creates chunks at section boundaries, and sub-chunks large sections. * * @param content - The markdown content to chunk * @param options - Chunk options (optional, defaults to DEFAULT_MARKDOWN_CHUNK_OPTIONS) * @returns Array of markdown chunks with metadata */ export declare function chunkMarkdownContent(content: string, options?: Partial): MarkdownChunk[]; /** * Check if markdown header chunking should be used for a file * * Currently only for .md files (not .txt as they lack structure). * * @param relativePath - The file path to check * @returns true if markdown header chunking should be used */ export declare function shouldUseMarkdownChunking(relativePath: string): boolean; /** * Chunk a markdown file by headers * * Reads the file and chunks it using header-aware splitting. * * @param absolutePath - Absolute path to the file on disk * @param relativePath - Relative path from project root (stored in chunk) * @param fileContent - The file content (passed from caller to avoid re-reading) * @param contentHash - SHA256 hash of the content * @param options - Chunk options * @returns Array of full Chunk objects with IDs and metadata */ export declare function chunkMarkdownFile(absolutePath: string, relativePath: string, fileContent: string, contentHash: string, options?: Partial): Chunk[]; //# sourceMappingURL=markdownChunking.d.ts.map