/**
 * Tokenizer for TONL query path expressions
 *
 * Converts a path string into a stream of tokens for parsing.
 */
import { Token, TokenType } from './types.js';
/**
 * Tokenize a path expression into tokens
 *
 * @param input - The path expression string
 * @returns Array of tokens
 * @throws ParseError if the input contains invalid syntax
 *
 * @example
 * ```typescript
 * const tokens = tokenize('user.name');
 * // Returns: [
 * //   { type: 'IDENTIFIER', value: 'user', position: 0, length: 4 },
 * //   { type: 'DOT', value: '.', position: 4, length: 1 },
 * //   { type: 'IDENTIFIER', value: 'name', position: 5, length: 4 },
 * //   { type: 'EOF', value: null, position: 9, length: 0 }
 * // ]
 * ```
 */
export declare function tokenize(input: string): Token[];
/**
 * Check if a token is of a specific type
 */
export declare function isTokenType(token: Token, type: TokenType): boolean;
/**
 * Check if a token is an operator
 */
export declare function isOperator(token: Token): boolean;
/**
 * Get the precedence of an operator token
 * Higher number = higher precedence
 */
export declare function getOperatorPrecedence(token: Token): number;
//# sourceMappingURL=tokenizer.d.ts.map