/**
 * Search tokenizer.
 *
 * Handles mixed Latin + CJK content — the single biggest quality lever
 * over plain substring search. Latin is split on non-word boundaries;
 * CJK (no spaces) is decomposed into overlapping bigrams so a query like
 * "身份验证" still scores against doc text "身份验证流程".
 */
export interface TokenizeOptions {
    stem?: boolean;
}
export declare function tokenize(text: string, opts?: TokenizeOptions): string[];
/** Character bigrams over arbitrary text — used by fuzzy matching. */
export declare function charBigrams(text: string): string[];
/** Term-frequency map. */
export declare function tfMap(text: string, stem: boolean): Map<string, number>;
//# sourceMappingURL=tokenizer.d.ts.map