export declare const STOPWORDS: Set<string>;
/**
 * Lowercase, strip non-alphanumeric, drop single-char tokens and stopwords.
 *
 * This is the tokeniser whose output the 0.697 BM25 baseline was measured on.
 * Any change here changes both the eval scorer AND the production scorer in
 * lockstep — which is the invariant that motivated extracting it.
 */
export declare function contentTokens(text: string): string[];
//# sourceMappingURL=text-tokens.d.ts.map