/**
 * An identifier-aware tokenizer for scoring, not for search.
 *
 * A coding conversation's vocabulary is paths, identifiers and error
 * names. A word tokenizer sees `src/store.mjs` as one token that matches
 * nothing and `removeTodo` as another; the model, asked about `store`,
 * is looking at both. So a path splits on its separators, a `camelCase`
 * or `snake_case` identifier splits into its words AND keeps its whole
 * form, and a dotted attribute key (`namzu.turn.id`) yields both the key
 * and its parts. Everything is lower-cased; nothing is stemmed, because
 * a stemmer that turns `tests` into `test` also turns `testing` into it,
 * and the false matches cost more than the missed ones in a corpus this
 * small. Stop words are the handful that carry no relevance in any
 * sentence and would otherwise dominate term frequency.
 */
/** Tokens of `text`, lower-cased, with identifiers and paths opened up. */
export declare function tokenize(text: string): string[];
/** Term frequencies of `tokens`. */
export declare function termFrequencies(tokens: readonly string[]): Map<string, number>;
//# sourceMappingURL=tokenize.d.ts.map