/** * Deterministic prose chunker for oversize document classification. * * Splits a document into overlapping fixed-size chunks so each chunk fits * inside Haiku's input context window. The chunker is purely mechanical — * it makes no semantic claim about where chunk boundaries should fall. * Ontological boundaries remain Haiku's job (the document * chunker that *did* try to be semantic was deleted because it leaked * sections at the boundaries it picked). * * Overlap exists so a section straddling a chunk boundary appears in BOTH * surrounding chunks; the merge step then unions the same-kind ranges so * the boundary section isn't double-counted in the writer. * * Char counts are estimated from token counts via a fixed 3.5 chars/token * ratio (English prose average). The estimate is conservative — Haiku * tokenises slightly differently per script, but 3.5 leaves ~10% headroom * for non-English content before bumping into the model's hard ceiling. */ export interface RangedSection { /** Section kind from the classifier's closed enumeration. */ kind: string; /** Inclusive whole-document start offset. */ sourceStart: number; /** Exclusive whole-document end offset. */ sourceEnd: number; /** Per-section summary; longer wins on merge tie-break. */ summary: string; } export interface DocumentChunk { /** Substring of the source document covered by this chunk. */ chunkText: string; /** Whole-document offset where this chunk's text begins. */ baseOffset: number; } export interface ChunkOptions { /** Maximum chunk length in characters (already token→char converted). */ chunkSize: number; /** Overlap in characters between consecutive chunks. */ overlap: number; } export declare function chunkDocument(text: string, opts: ChunkOptions): DocumentChunk[]; export declare const MERGE_OVERLAP_THRESHOLD = 0.5; export declare function mergeOverlappingSections(input: T[]): T[]; //# sourceMappingURL=document-chunker.d.ts.map