import { Token, TokenizerOptions } from './tokenizer';
export default function tokenize(input: any, options?: TokenizerOptions): Token[];