/**
 * Standard English stop words set for information retrieval.
 * Preserves common API identifiers like 'id', 'get', 'set', etc.
 */
export declare const STOP_WORDS: Set<string>;
/**
 * Splits text into normalized alphanumeric tokens, handling camelCase,
 * snake_case, kebab-case, punctuation, and stop-word filtration.
 */
export declare function tokenize(text: string): string[];
//# sourceMappingURL=tokenizer.d.ts.map