type EncodeOptions = { addSpecialTokens?: boolean; maxLength?: number; padToMaxLength?: boolean; returnAttentionMask?: boolean; }; type BatchEncodeReturn = { inputIds: number[][]; attentionMask?: number[][]; }; export declare class Tokenizer { private vocab; private invVocab; private modelMaxLength; private padToken; private unkToken; private specialTokenId; private normalizerRegex; constructor(); /** Normalize input using the JSON "Replace" normalizer (remove disallowed chars). */ private normalize; /** Pre-tokenizer: split to individual Unicode characters. */ private preTokenize; /** Apply post-processor template: single sequence -> $ $ (as per JSON). */ private applyPostProcessor; /** Remove a *single* leading & trailing special token ($) if present. */ private stripPostProcessor; /** Map chars -> ids (unknown map to "$" id). */ private toIds; /** Map ids -> tokens (unknown id -> "$"). */ private toTokens; /** Truncate to maxLength (if provided), otherwise config length. */ private truncate; /** Pad sequence with "$" to maxLen. */ private padTo; /** Build attention mask: 1 for real tokens, 0 for pad. */ private attentionMask; /** Encode a single string -> token IDs. */ encode(text: string, opts?: EncodeOptions): number[]; /** Decode token IDs -> string (drops a single leading/trailing $ added by post-processor). */ decode(ids: number[], stripSpecial?: boolean): string; /** Pad a batch to the same length; returns padded ids (no masks). */ pad(batch: number[][], toLength?: number): number[][]; /** Batch encode with optional padding & attention masks (HuggingFace-like). */ batchEncode(texts: string[], { addSpecialTokens, maxLength, padToMaxLength, returnAttentionMask, }?: EncodeOptions): BatchEncodeReturn; } export declare const tokenizer: Tokenizer; export {}; //# sourceMappingURL=tokenizer.d.ts.map