/**
 * Shared tokenizer utility for all Voyage models (local and API).
 * Loads tokenizers from HuggingFace Hub via @huggingface/transformers.
 */
export declare function getTransformers(): Promise<any>;
/** Load tokenizer for any Voyage model (cached). */
export declare function getTokenizerForModel(model: string): Promise<any>;
export interface TokenizeResult {
    tokens: string[];
    ids: number[];
}
/** Tokenize texts using the model's tokenizer. */
export declare function tokenizeTexts(model: string, texts: string[]): Promise<TokenizeResult[]>;