import { Token, TokenCallback, Tokenizer } from "../types/tokenizer";
/**
 * A simple class to make creating simple tokenizers simpler.
 */
export declare class CustomTokenizer implements Tokenizer {
    private readonly callback;
    /**
     * Create a new CustomTokenizer.
     * @param callback {CustomTokenizerCallback} - the callback that will be ran on the input to the tokenizer.
     */
    constructor(callback: CustomTokenizerCallback);
    /**
     * @inheritDoc
     */
    tokenize(input: string, modify?: TokenCallback): Token[];
}
/**
 * The callback that will be ran on the input to the tokenizer
 */
export declare type CustomTokenizerCallback = (input: string) => Token[];