export declare class Tokenizer {
    pat: RegExp;
    mergeable_ranks: Record<string, number>;
    special_tokens: Record<string, number>;
    decode_map: Record<number, string>;
    static llama: (path: string) => Promise<Tokenizer>;
    constructor(pat: RegExp, mergeable_ranks: Record<string, number>, special_tokens: Record<string, number>);
    get bos_id(): number;
    get stop_tokens(): number[];
    decode(toks: number[]): string;
    encode(text: string, allowSpecial?: boolean): number[];
    private bpe_encode;
    encode_role: (role: string) => number[];
    encode_message: (role: string, content: string) => number[];
}