import { Tokenizer } from "../common";
export declare const RULES: string[][];
export declare const RE: RegExp;
export declare function makeTokenizer(re: RegExp, keywords: Set<string>): Tokenizer;
export declare const tokenize: Tokenizer;