/**
 * A tokenizer callback that can modify part of the tokenizer output.
 */
export declare type ModifyCallback = (token: Token) => Token;
/**
 * A tokenizer that can search for tokens using a regular expression.
 *
 * For example, with the input "&lt;p&gt;test&lt;/p&gt;" and a regular expression "/<.*?>/", then the output will be ["&lt;p&gt;", "test", "&lt;/p&gt;"].
 *
 * If defined, modify will be called on every new element added to the output.
 * @param input {string} - is the input that will be tokenized.
 * @param regex {RegExp} - is the regular expression that will be used to search for tokens.
 * @param modify {ModifyCallback} - is used to more efficiently modify the output of the tokenizer.
 */
export declare const RegexTokenizer: (input: string, regex: RegExp, modify?: ModifyCallback) => Token[];
/**
 * An interface for the output of a tokenizer.
 */
export interface Token {
    value: string;
    isToken: boolean;
}