import { ITokenizer } from './i-tokenizer.js'; /** * Pluggable tokenizer factory — addresses issues #123 / #124. * * Resolution order: * 1. Google AI models (`gemini-*`) — GoogleAITokenizer when * GOOGLE_AI_API_KEY is set, else HeuristicTokenizer. * 2. Tiktoken-compatible families (GPT, Claude) — TiktokenTokenizer. * 3. HeuristicTokenizer fallback for everything else. * * Instances are cached per model name so callers don't pay for repeated * allocation of the native tiktoken encoder, and so their per-tokenizer * LRU caches can be shared across call sites. */ export declare class TokenizerFactory { private static readonly instances; static create(modelName: string): ITokenizer; static createFromEnv(): ITokenizer; /** * Release every cached tokenizer. Call this on server shutdown so * native tiktoken encoders are freed. */ static disposeAll(): void; private static build; } //# sourceMappingURL=tokenizer-factory.d.ts.map