import { ILRUCache } from './lru';
/**
 * This is a Typescript version of OpenAI's tiktoken implementation of
 * Byte pair encoding(BPE): https://en.wikipedia.org/wiki/Byte_pair_encoding,
 * the goal is to support context tokenization for OpenAI large language models in .NET runtime.
 * Reference: https://github.com/openai/tiktoken/blob/main/src/lib.rs
 */
export declare class TikTokenizer {
    private regex?;
    private encoder?;
    private decoder?;
    private specialTokensRegex?;
    private specialTokensEncoder?;
    private specialTokensDecoder?;
    private textEncoder;
    private textDecoder;
    readonly cache: ILRUCache;
    /**
     * Take the encoder tokens mapping from OpenAI tiktoken dump to build the encoder
     * For gpt-3.5-turbo/gpt4, you can download the BPE tokens mapping from:
     * https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken
     * @param tikTokenBpeFileOrDict BPE rank file path or parsed dictionary
     * @param specialTokensEncoder special tokens encoder
     * @param regexPattern regex pattern to split the input text
     * @param cacheSize cache size
     */
    constructor(tikTokenBpeFileOrDict: string | Map<Uint8Array, number>, specialTokensEncoder: ReadonlyMap<string, number>, regexPattern: string, cacheSize?: number);
    protected init(bpeDict: ReadonlyMap<Uint8Array, number>, specialTokensEncoder: ReadonlyMap<string, number>, regexPattern: string): void;
    private findNextSpecialToken;
    /**
     * Encode a string with a set of allowed special tokens that are not broken apart.
     * @param text text to encode
     * @param allowedSpecial allowed special tokens
     * @returns number[] encoded token ids
     */
    encode(text: string, allowedSpecial?: ReadonlyArray<string>): number[];
    private encodeSpecialToken;
    private encodeByIndex;
    private encodeTrimSuffixByIndex;
    /**
     * Encode a piece of text limited by max token count through trimming suffix
     * @param text text to encode
     * @param maxTokenCount max token count
     * @param allowedSpecial allowed special tokens
     * @returns { tokenIds: number[], text: string } encoded token ids and trimmed text
     */
    encodeTrimSuffix(text: string, maxTokenCount: number, allowedSpecial: ReadonlyArray<string>): {
        tokenIds: number[];
        text: string;
    };
    /**
     * Encode a piece of text limited by max token count through trimming prefix
     * @param text text to encode
     * @param maxTokenCount max token count
     * @param allowedSpecial allowed special tokens
     * @returns { tokenIds: number[], text: string } encoded token ids and trimmed text
     */
    encodeTrimPrefix(text: string, maxTokenCount: number, allowedSpecial?: ReadonlyArray<string>): {
        tokenIds: number[];
        text: string;
    };
    /**
     * Decode an array of integer token ids
     * @param tokens array of integer token ids
     * @returns string decoded text
     */
    decode(tokens: number[]): string;
}