/**
 * Identifier-aware tokenization for code.
 *
 * Splits identifiers on camelCase, snake_case, kebab-case, and separators while
 * preserving the exact original token. Produces both the original token and its
 * normalized search tokens.
 */
export interface TokenizedIdentifier {
    original: string;
    lower: string;
    searchTokens: string[];
}
/** Tokenize a full query into normalized search tokens. */
export declare function tokenizeQuery(query: string): string[];
/** Tokenize a single code identifier (preserving the original). */
export declare function tokenizeIdentifier(identifier: string): TokenizedIdentifier;
/** Produce the document text used for lexical indexing of a chunk. */
export declare function lexicalChunkText(options: {
    path: string;
    text: string;
    symbolName?: string;
    qualifiedName?: string;
}): string;
//# sourceMappingURL=tokenize.d.ts.map