import type { ITokenizer as ITsxTokenizer, OutputMode, Raw } from '@vscode/prompt-tsx';
import type { LanguageModelChatTool } from 'vscode';
export declare enum TokenizerType {
    CL100K = "cl100k_base",
    O200K = "o200k_base",
    Llama3 = "llama3"
}
export interface ITokenizer extends ITsxTokenizer<OutputMode.Raw> {
    /**
     * Return the length of `text` in number of tokens.
     *
     * @param text The input text
     */
    tokenLength(text: string | Raw.ChatCompletionContentPart): Promise<number>;
    countMessageTokens(message: Raw.ChatMessage): Promise<number>;
    countMessagesTokens(messages: Raw.ChatMessage[]): Promise<number>;
    countToolTokens(tools: readonly LanguageModelChatTool[]): Promise<number>;
}
//# sourceMappingURL=tokenizer.d.ts.map