export type TokenType = "word" | "number" | "punct" | "symbol" | "whitespace" | "unknown";
export interface Token {
    value: string;
    type: TokenType;
    start: number;
    end: number;
}
export interface TokenizerOptions {
    lowercase?: boolean;
    mergeSymbols?: boolean;
}
export declare class Tokenizer {
    private options;
    constructor(options?: TokenizerOptions);
    tokenize(text: string): Token[];
    private classifyOptimized;
    private isBoundaryOptimized;
}