import { Tokenizer } from "../common";
export declare function makeTokenizer(re: RegExp, keywords: Set<string>): Tokenizer;
export declare const tokenize: Tokenizer;