import { TikTokenizer } from "./tikTokenizer";
export declare const MODEL_TO_ENCODING: ReadonlyMap<string, string>;
/**
 * Get the special tokens from the encoder name
 * @param encoder encoder name
 * @returns Map<string, number> special tokens mapping
 */
export declare function getSpecialTokensByEncoder(encoder: string): Map<string, number>;
/**
 * Get the special tokens from the model name
 * @param modelName model name
 * @returns Map<string, number> special tokens mapping
 */
export declare function getSpecialTokensByModel(modelName: string): Map<string, number>;
/**
 * Get the regex pattern from the encoder name
 * @param encoder encoder name
 * @returns string regex pattern
 */
export declare function getRegexByEncoder(encoder: string): string;
/**
 * Get the regex pattern from the model name
 * @param modelName model name
 * @returns string regex pattern
 */
export declare function getRegexByModel(modelName: string): string;
/**
 * Create a tokenizer from a model name
 * @param modelName model name
 * @param extraSpecialTokens extra special tokens
 */
export declare function createByModelName(modelName: string, extraSpecialTokens?: ReadonlyMap<string, number> | null): Promise<TikTokenizer>;
/**
 * Create a tokenizer from an encoder name
 * @param encoderName encoder name
 * @param extraSpecialTokens extra special tokens
 * @returns TikTokenizer tokenizer
 */
export declare function createByEncoderName(encoderName: string, extraSpecialTokens?: ReadonlyMap<string, number> | null): Promise<TikTokenizer>;
/**
 * Create a tokenizer from a file
 * @param tikTokenBpeFileOrDict BPE rank file in tiktoken format or parsed dictionary
 * @param specialTokensEncoder special tokens mapping
 * @param regexPattern regex pattern
 * @param cacheSize cache size
 * @returns TikTokenizer tokenizer
 */
export declare function createTokenizer(tikTokenBpeFileOrDict: string | Map<Uint8Array, number>, specialTokensEncoder: ReadonlyMap<string, number>, regexPattern: string, cacheSize?: number): TikTokenizer;