/**
 * Tokenizer — splits input text into tokens with position tracking,
 * and generates n-gram phrase windows for phrase detection.
 */
export interface Token {
    /** The raw text of the token */
    value: string;
    /** Start index in the original string */
    start: number;
    /** End index (exclusive) in the original string */
    end: number;
}
/**
 * Tokenize input text into individual words, preserving positions.
 * Splits on whitespace and common punctuation boundaries.
 */
export declare function tokenize(input: string): Token[];
/**
 * Generate n-gram phrase windows from tokens.
 * For phrase detection, we need to check sequences of 2-4 words.
 */
export declare function phraseWindows(tokens: Token[], maxSize?: number): PhraseWindow[];
export interface PhraseWindow {
    tokens: Token[];
    phrase: string;
    start: number;
    end: number;
}