/**
 * bpe.ts – Browser-side Byte Pair Encoding (BPE) tokenizer.
 */
export interface BPEEncodeOptions {
    addBos?: boolean;
    addEos?: boolean;
}
export type PadSide = 'right' | 'left';
/** A Hugging Face `tokenizer.json` (or a bare `{ vocab, merges }`) for import. */
export interface HuggingFaceTokenizerSpec {
    model?: {
        vocab?: Record<string, number>;
        merges?: Array<string | [string, string]>;
    };
    vocab?: Record<string, number>;
    merges?: Array<string | [string, string]>;
}
/** Override the special-token strings to match an imported tokenizer. */
export interface SpecialTokenOverrides {
    bos?: string;
    eos?: string;
    pad?: string;
    unk?: string;
}
export declare class BPETokenizer {
    vocab: Map<string, number>;
    idToToken: Map<number, string>;
    merges: Map<string, number>;
    bosToken: string;
    eosToken: string;
    padToken: string;
    unkToken: string;
    bosId: number | null;
    eosId: number | null;
    padId: number | null;
    constructor();
    load(vocab: string | Record<string, number>, merges: string | string[]): Promise<void>;
    /**
     * Learn a byte-level BPE vocabulary + merges from a text corpus, then load
     * them into this tokenizer. Makes the tokenizer self-contained (no external
     * vocab file): train on your data, then `encode`/`decode` round-trips real
     * text and frequent sequences compress to single tokens. The base vocabulary
     * always covers all 256 byte symbols, so any input is representable.
     */
    train(corpus: string | string[], opts?: {
        numMerges?: number;
        minPairFreq?: number;
    }): void;
    loadFromObjects(vocabObj: Record<string, number>, mergeArr: string[]): void;
    /**
     * Seed this tokenizer from an existing Hugging Face `tokenizer.json` — the
     * *import-merges* path (vs `train`, which learns a fresh vocab from a corpus).
     *
     * BPE merges + vocab are portable DATA, not architecture: a proven code
     * tokenizer (GPT-2 / Llama / StarCoder family) gives Evermind a battle-tested
     * code vocabulary on day one. Those tokenizers use the same GPT-2 byte→unicode
     * mapping this class already implements ({@link BYTE_ENCODER}), so the imported
     * vocab is directly compatible with `encode`/`decode`/`_bpe`.
     *
     * Accepts the full `tokenizer.json` object (reads `.model.vocab` / `.model.merges`)
     * or a bare `{ vocab, merges }`. `merges` may be the classic `"a b"` strings or
     * the newer `["a", "b"]` pair arrays. Special-token strings can be overridden to
     * match the source tokenizer (e.g. `<|endoftext|>`, `<s>`, `</s>`).
     */
    loadHuggingFace(spec: HuggingFaceTokenizerSpec, specials?: SpecialTokenOverrides): void;
    /**
     * Fetch and import a Hugging Face `tokenizer.json` by URL — the IDE's
     * "import merges from a repo" path. Resolves once the tokenizer is loaded.
     */
    loadHuggingFaceUrl(url: string, specials?: SpecialTokenOverrides): Promise<void>;
    encode(text: string, opts?: BPEEncodeOptions): number[];
    decode(ids: number[]): string;
    _bpe(word: string): string[];
    padOrTruncate(ids: number[], maxLen: number, side?: PadSide): number[];
    get vocabSize(): number;
}
//# sourceMappingURL=bpe.d.ts.map