import { ITokenizer, TokenizerType } from '../../../util/common/tokenizer';
import { ITelemetryService } from '../../telemetry/common/telemetry';
export declare const ITokenizerProvider: import("../../../util/common/services").ServiceIdentifier<ITokenizerProvider>;
export interface TokenizationEndpoint {
    readonly tokenizer: TokenizerType;
}
export interface ITokenizerProvider {
    readonly _serviceBrand: undefined;
    acquireTokenizer(endpoint: TokenizationEndpoint): ITokenizer;
}
/**
 * BaseTokensPerCompletion is the minimum tokens for a completion request.
 * Replies are primed with <|im_start|>assistant<|message|>, so these tokens represent the
 * special token and the role name.
 */
export declare const BaseTokensPerCompletion = 3;
export declare const BaseTokensPerMessage = 3;
export declare const BaseTokensPerName = 1;
export declare class TokenizerProvider implements ITokenizerProvider {
    readonly _serviceBrand: undefined;
    private readonly _cl100kTokenizer;
    private readonly _o200kTokenizer;
    constructor(useWorker: boolean, telmetryService: ITelemetryService);
    dispose(): void;
    /**
     * Gets a tokenizer for a given model family
     * @param endpoint The endpoint you want to acquire a tokenizer for
     */
    acquireTokenizer(endpoint: TokenizationEndpoint): ITokenizer;
}
export declare function calculateImageTokenCost(imageUrl: string, detail: 'low' | 'high' | 'auto' | undefined): number;
/**
 * Estimates the token cost of a base64-encoded document (e.g. PDF) without BPE tokenization.
 * Uses a size-based heuristic to avoid tokenizing large binary payloads and polluting
 * the LRU cache. Intentionally conservative (overestimates) to avoid exceeding context limits.
 */
export declare function estimateDocumentTokenCost(base64Data: string | undefined): number;
//# sourceMappingURL=tokenizer.d.ts.map