import type { AtomicTextRange, TextChunk } from './index.js'; import { type ContainmentBudget } from './token-containment.js'; /** * Semantic Chunker configuration * Based on paper recommendations: hardThreshold=0.6, initConst=1.5, c=0.9 */ export interface SemanticChunkerConfig { /** Hard threshold for minimum similarity (default: 0.6) */ hardThreshold: number; /** Initial constant for first sentence pair (default: 1.5) */ initConst: number; /** Scaling constant for threshold calculation (default: 0.9) */ c: number; /** Minimum chunk length in characters (default: 50) */ minChunkLength: number; } /** * Embedder interface for generating embeddings. * * `getTokenLimit` and `countTokens` are optional; without them the chunker * skips token containment. */ export interface EmbedderInterface { /** `'document'` embeds behind the model's document prompt; omitted, its default prompt. */ embedBatch(texts: string[], role?: 'document'): Promise; /** Resolved token cap, or `null` when no limit could be resolved. */ getTokenLimit?(): Promise; /** True, unclamped token lengths of each text, prompt included. */ countTokens?(texts: string[], role?: 'document'): Promise; /** The model's document prompt, or `''`. */ getDocumentPrompt?(): Promise; /** Whether ingestion embeds chunks behind their document title. */ readonly titlePrefix?: boolean; /** Whether ingestion adds section paths to the final embedding input. */ readonly headingPrefix?: boolean; } /** * Garbage chunks, language-agnostically: empty after trimming, only decoration * characters (`----`, `====`), or one character repeated over 80% of the text. * Anything alphanumeric is kept. Applied after the minChunkLength filter. */ export declare function isGarbageChunk(text: string): boolean; /** Default minimum chunk length in characters */ export declare const DEFAULT_MIN_CHUNK_LENGTH = 50; /** * The embedder's token budget, or `null` when the optional members are absent * or no limit could be resolved. Containment is then skipped entirely. * * Every measurement is taken with `textPrefix` prepended and the document * prompt applied, since the caller embeds each chunk behind both. */ export declare function resolveContainmentBudget(embedder: EmbedderInterface, textPrefix: string): Promise; /** * Semantic chunker using the Max-Min algorithm: a sentence joins the current * chunk when its maximum similarity to any member exceeds the minimum * similarity between existing members, adjusted by the threshold. */ export declare class SemanticChunker { private readonly config; constructor(config?: Partial); /** * Split text into semantically coherent chunks. `textPrefix` is the text the * caller will prepend to each chunk before embedding; it is not added to the * returned chunks, but token containment budgets for it. */ chunkText(text: string, embedder: EmbedderInterface, atomicRanges?: readonly AtomicTextRange[], textPrefix?: string): Promise; /** * Whether a sentence group becomes stored chunks. A group holding a * containment piece is already admitted, judged before the split, so neither * filter re-runs on a fragment of it. Otherwise garbage is rejected and the * minimum length is waived for an atomic unit, which must stay whole. */ private admitsGroup; /** * Group sentences into chunks using Max-Min algorithm */ /** * Whether a sentence continues the group being built: `initConst`-scaled * similarity while the group holds one sentence, Max-Min beyond that, and * never past `MAX_SENTENCES`. */ private continuesGroup; private groupSentences; /** * Decide if a sentence should be added to the current chunk * Based on Max-Min algorithm from the paper */ private shouldAddToChunk; /** * Minimum pairwise similarity within a chunk, over the last WINDOW_SIZE * sentences only. The approximation follows the Max-Min paper: recent * sentences are what determine coherence. */ private getMinSimilarity; /** * Get maximum similarity between a sentence and any sentence in the chunk */ private getMaxSimilarity; /** * Calculate dynamic threshold based on chunk size * threshold = max(c * minSim * sigmoid(|C|), hardThreshold) */ private calculateThreshold; /** * Sigmoid function */ private sigmoid; /** * Calculate cosine similarity between two vectors * Public for testing */ cosineSimilarity(vec1: number[], vec2: number[]): number; } //# sourceMappingURL=semantic-chunker.d.ts.map