declare const usableTokenizers: readonly ["cl100k_base", "cl100k_base_special_tokens", "r50k_base", "p50k_base", "gpt2", "llama3", "o200k_base", "o200k_base_special_tokens", "codestral"];
export type UsableTokenizer = typeof usableTokenizers[number];
export type OpenAIMessageRole = 'system' | 'user' | 'assistant' | 'tool';
export type PriomptTokenizer = {
    name: string;
    encodeTokens: (text: string) => Promise<number[]>;
    decodeTokens: (tokens: number[]) => Promise<string>;
    numTokens: (text: string) => Promise<number>;
    estimateNumTokensFast_SYNCHRONOUS_BE_CAREFUL: (text: string) => number;
    estimateNumTokensFast: (text: string) => Promise<number>;
    estimateTokensUsingCharCount: (text: string) => [number, number];
    getHeaderStringForMessage: (message: {
        role: OpenAIMessageRole;
        name?: string;
        to?: string;
    }) => string;
    getHeaderTokensForMessage: (message: {
        role: OpenAIMessageRole;
        name?: string;
        to?: string;
    }) => Promise<number[]>;
    getEosTokenId: () => number;
    getEosToken: () => string;
    applyChatTemplate: (messages: {
        role: OpenAIMessageRole;
        name?: string;
        to?: string;
        content: string | string[];
    }[], options?: {
        doNotAddGenerationPrompt?: boolean;
    }) => string;
    applyChatTemplateTokens: (messages: {
        role: OpenAIMessageRole;
        name?: string;
        to?: string;
        content: string | string[];
    }[], options?: {
        doNotAddGenerationPrompt?: boolean;
    }) => Promise<number[]>;
    shouldAddEosTokenToEachMessage: boolean;
};
export declare const CL100K: PriomptTokenizer;
export declare const CL100K_SPECIAL_TOKENS: PriomptTokenizer;
export declare const O200K: PriomptTokenizer;
export declare const O200K_SPECIAL_TOKENS: PriomptTokenizer;
export declare const CODESTRAL_ONLY_USE_ESTIMATE_NUM_TOKENS_FAST: PriomptTokenizer;
export declare const getTokenizerByName_ONLY_FOR_OPENAI_TOKENIZERS: (name: UsableTokenizer) => PriomptTokenizer;
export declare function numTokens(text: string, opts: {
    tokenizer: UsableTokenizer;
}): Promise<number>;
export declare function estimateNumTokensFast_SYNCHRONOUS_BE_CAREFUL(text: string, opts: {
    tokenizer: UsableTokenizer;
}): number;
export declare function estimateNumTokensFast(text: string, opts: {
    tokenizer: UsableTokenizer;
}): Promise<number>;
export declare function encodeTokens(text: string, opts: {
    tokenizer: UsableTokenizer;
}): Promise<number[]>;
export declare function decodeTokens(tokens: number[], opts: {
    tokenizer: UsableTokenizer;
}): Promise<string>;
export declare function estimateTokensUsingBytecount(text: string, tokenizer: UsableTokenizer): [number, number];
export declare function estimateTokensUsingCharcount(text: string, tokenizer: UsableTokenizer): [number, number];
export declare function numTokensForImage(dimensions: {
    width: number;
    height: number;
}, detail: 'low' | 'high' | 'auto'): number;
export declare const openaiGetHeaderTokensForMessage: (message: {
    role: 'system' | 'user' | 'assistant' | 'tool';
    name?: string;
    to?: string;
}, tokenizer: UsableTokenizer) => Promise<number[]>;
export declare const openaiGetHeaderStringForMessage: (message: {
    role: 'system' | 'user' | 'assistant' | 'tool';
    name?: string;
    to?: string;
}, tokenizer: UsableTokenizer) => string;
export declare const applyCodestralChatTemplate: (messages: {
    role: OpenAIMessageRole;
    name?: string;
    to?: string;
    content: string | string[];
}[], options?: {
    doNotAddGenerationPrompt?: boolean;
}) => string;
export {};
//# sourceMappingURL=tokenizer.d.ts.map