/** Unicode and identifier tokenization without aliases or translation. @module dsh-tool-retriever/retrieval/tokenizer */
/**
 * Normalize a complete identifier for strict name matching.
 * @param value - Raw query or tool name.
 * @returns Lowercase identifier components separated by one space.
 */
export declare function normalizeIdentifier(value: string): string;
/**
 * Tokenize natural language and programming identifiers with deterministic CJK fragments.
 * @param value - Text from a query or tool schema.
 * @returns Tokens in source order, including repeated terms for BM25 term frequency.
 */
export declare function tokenize(value: string): string[];
//# sourceMappingURL=tokenizer.d.ts.map