import type { CancellationToken, LanguageModelChatMessage } from 'vscode';
import { ModeToChatMessageType, OutputMode, Raw } from '../output/mode';
/**
 * Represents a tokenizer that can be used to tokenize text in chat messages.
 */
export interface ITokenizer<M extends OutputMode = OutputMode> {
    /**
     * This mode this tokenizer operates on.
     */
    readonly mode: M;
    /**
     * Return the length of `part` in number of tokens. If the model does not
     * support the given kind of part, it may return 0.
     *
     * @param {str} text - The input text
     * @returns {number}
     */
    tokenLength(part: Raw.ChatCompletionContentPart, token?: CancellationToken): Promise<number> | number;
    /**
     * Returns the token length of the given message.
     */
    countMessageTokens(message: ModeToChatMessageType[M]): Promise<number> | number;
}
export declare class VSCodeTokenizer implements ITokenizer<OutputMode.VSCode> {
    private countTokens;
    readonly mode = OutputMode.VSCode;
    constructor(countTokens: (text: string | LanguageModelChatMessage, token?: CancellationToken) => Thenable<number>, mode: OutputMode);
    tokenLength(part: Raw.ChatCompletionContentPart, token?: CancellationToken): Promise<number>;
    countMessageTokens(message: LanguageModelChatMessage): Promise<number>;
}