import { IToken } from "tokenizr";
/**
 * Tokenizes a given text.
 *
 * @param text The text.
 *
 * @returns An array of tokens.
 */
export default function tokenize(text: string): IToken[];