/** The lexical units shared by evidence discovery and bounded relevance scoring. */
export declare function evidenceTokens(text: string): string[];
/** Iterate large stored text without materializing a token array. */
export declare function evidenceTokenEntries(text: string): IterableIterator<RegExpExecArray>;
/** No stemming, locale-sensitive folding or accent normalization. */
export declare function evidenceTokenKey(token: string, caseSensitive?: boolean): string;
export declare function isEvidenceToken(text: string): boolean;
/** `from` may lie inside a token; such a suffix must not become a new word. */
export declare function evidenceTokenMatcher(terms: readonly string[], caseSensitive: boolean): (text: string, from: number) => {
    index: number;
    length: number;
} | undefined;
//# sourceMappingURL=evidence-tokens.d.ts.map