import { type Tokenizer } from "@llamaindex/env";
declare class TextSplit {
    textChunk: string;
    numCharOverlap: number | undefined;
    constructor(textChunk: string, numCharOverlap?: number | undefined);
}
type SplitRep = {
    text: string;
    numTokens: number;
};
export declare const defaultSentenceTokenizer: (text: string) => string[];
/**
 * Tokenizes sentences. Suitable for Chinese, Japanese, and Korean. Use instead of `defaultSentenceTokenizer`.
 * @param text
 * @returns string[]
 */
export declare function cjkSentenceTokenizer(sentence: string): string[];
export declare const defaultParagraphSeparator: string;
/**
 * SentenceSplitter is our default text splitter that supports splitting into sentences, paragraphs, or fixed length chunks with overlap.
 *
 * One of the advantages of SentenceSplitter is that even in the fixed length chunks it will try to keep sentences together.
 */
export declare class SentenceSplitter {
    chunkSize: number;
    chunkOverlap: number;
    private tokenizer;
    private paragraphSeparator;
    private chunkingTokenizerFn;
    private splitLongSentences;
    constructor(options?: {
        chunkSize?: number;
        chunkOverlap?: number;
        tokenizer?: Tokenizer;
        paragraphSeparator?: string;
        chunkingTokenizerFn?: (text: string) => string[];
        splitLongSentences?: boolean;
    });
    private getEffectiveChunkSize;
    getParagraphSplits(text: string, effectiveChunkSize?: number): string[];
    getSentenceSplits(text: string, effectiveChunkSize?: number): string[];
    /**
     * Splits sentences into chunks if necessary.
     *
     * This isn't great behavior because it can split down the middle of a
     * word or in non-English split down the middle of a Unicode codepoint
     * so the splitting is turned off by default. If you need it, please
     * set the splitLongSentences option to true.
     * @param sentenceSplits
     * @param effectiveChunkSize
     * @returns
     */
    private processSentenceSplits;
    combineTextSplits(newSentenceSplits: SplitRep[], effectiveChunkSize: number): TextSplit[];
    splitTextWithOverlaps(text: string, extraInfoStr?: string): TextSplit[];
    splitText(text: string, extraInfoStr?: string): string[];
}
export {};