/**
 * Tokenizer for GGUF models
 * Implements BPE tokenization compatible with Llama/Qwen models
 */
import type { GGUFFile } from '../types/model.js';
export interface TokenizerConfig {
    bosTokenId?: number;
    eosTokenId?: number;
    padTokenId?: number;
    unkTokenId?: number;
}
/**
 * BPE Tokenizer for GGUF models
 */
export declare class Tokenizer {
    private vocab;
    private reverseVocab;
    private merges;
    readonly bosTokenId: number;
    readonly eosTokenId: number;
    readonly padTokenId: number;
    readonly unkTokenId: number;
    readonly vocabSize: number;
    constructor(vocab: string[], merges: string[] | null, config?: TokenizerConfig);
    private findSpecialToken;
    /**
     * Load tokenizer from GGUF file metadata
     */
    static fromGGUF(gguf: GGUFFile): Tokenizer;
    /**
     * Encode text to token IDs
     */
    encode(text: string): number[];
    /**
     * Pre-tokenize text into pieces
     * This handles special tokens and splits on whitespace/punctuation
     */
    private preTokenize;
    private buildSpecialTokenPattern;
    private escapeRegex;
    /**
     * Split text into pieces based on whitespace and unicode categories
     * Similar to GPT-2/Qwen tokenization
     */
    private splitText;
    /**
     * Apply BPE encoding to a single piece
     */
    private bpeEncode;
    private static gpt2ByteToChar;
    private static gpt2CharToByte;
    /**
     * Initialize GPT-2 byte encoding tables (lazy initialization)
     */
    private static initGPT2Tables;
    /**
     * GPT-2 style byte-to-unicode mapping
     * Maps bytes to unicode characters to avoid whitespace/control char issues
     */
    private byteToUnicode;
    /**
     * Decode GPT-2 style unicode characters back to bytes
     * Converts Ġ -> space, Ċ -> newline, etc.
     */
    private decodeGPT2Bytes;
    /**
     * Encode individual characters/symbols to token IDs
     */
    private encodeCharacters;
    /**
     * Decode token IDs to text
     */
    decode(tokens: number[]): string;
    /**
     * Decode a single token ID to text
     */
    decodeToken(token: number): string;
    /**
     * Get token string for a given ID (raw, no decoding)
     */
    getToken(id: number): string | undefined;
    /**
     * Get ID for a given token string
     */
    getTokenId(token: string): number | undefined;
}
/**
 * Parse a GGUF file with full tokenizer data (no array size limit)
 */
export declare function parseGGUFWithTokenizer(filePath: string): Promise<GGUFFile>;
//# sourceMappingURL=index.d.ts.map