import type { BasePrefixedNumberToken, DocStringToken, ErrorToken, LBraceToken, LBracketToken, LParenToken, MultiLineCommentToken, NumberToken, OperatorToken, RBraceToken, RBracketToken, RParenToken, RegexpShorthandToken, ReservedSymbolToken, SingleLineCommentToken, StringToken, SymbolToken, Token, TokenDescriptor, WhitespaceToken } from './token';
type Tokenizer<T extends Token> = (input: string, position: number) => TokenDescriptor<T | ErrorToken>;
export declare const NO_MATCH: TokenDescriptor<never>;
export declare const tokenizeDocString: Tokenizer<DocStringToken>;
export declare const tokenizeNumber: Tokenizer<NumberToken>;
export declare const tokenizeBasePrefixedNumber: Tokenizer<BasePrefixedNumberToken>;
export declare const tokenizeSymbol: Tokenizer<SymbolToken>;
export declare const tokenizeReservedSymbolToken: Tokenizer<ReservedSymbolToken>;
export declare const tokenizeOperator: Tokenizer<OperatorToken>;
export declare const tokenizeMultiLineComment: Tokenizer<MultiLineCommentToken>;
export declare const tokenizeShebang: Tokenizer<SingleLineCommentToken>;
export declare const tokenizeSingleLineComment: Tokenizer<SingleLineCommentToken>;
export declare const tokenizers: [Tokenizer<WhitespaceToken>, Tokenizer<MultiLineCommentToken>, Tokenizer<SingleLineCommentToken>, Tokenizer<ReservedSymbolToken>, Tokenizer<LParenToken>, Tokenizer<RParenToken>, Tokenizer<LBracketToken>, Tokenizer<RBracketToken>, Tokenizer<LBraceToken>, Tokenizer<RBraceToken>, Tokenizer<DocStringToken>, Tokenizer<StringToken>, Tokenizer<RegexpShorthandToken>, Tokenizer<BasePrefixedNumberToken>, Tokenizer<NumberToken>, Tokenizer<OperatorToken>, Tokenizer<SymbolToken>];
export {};