/**
 * Tokenizer abstraction — pluggable text analysis.
 * Default: UnicodeTokenizer matching agentdb's regex (lowercase, min 1 char).
 */
export interface Tokenizer {
    /** Stable identifier persisted in the manifest. */
    readonly kind: string;
    /** Minimum token length, persisted in the manifest. Defaults to 1 if not provided. */
    readonly minLen?: number;
    tokenize(text: string): string[];
}
/**
 * Default Unicode tokenizer — extracts sequences of Unicode word characters,
 * lowercases them, and filters by minimum length. Matches agentdb's TextIndex
 * tokenization so BM25 scores are comparable across both implementations.
 */
export declare class UnicodeTokenizer implements Tokenizer {
    readonly kind: "unicode";
    readonly minLen: number;
    constructor(minLen?: number);
    tokenize(text: string): string[];
}
export declare const DEFAULT_TOKENIZER: UnicodeTokenizer;
//# sourceMappingURL=tokenizer.d.ts.map