export interface ITokens {
    type: string;
    value: string;
    position: number;
}
export declare function tokenizer(input: string): ITokens[];