import type { ModelId } from '../models.js';
import { BaseTokenizer } from './base/BaseTokenizer.js';
type HfEncodeResult = {
    ids: Array<number>;
    tokens: Array<string>;
};
type HfEncodeOptions = {
    add_special_tokens?: boolean;
};
type HfDecodeOptions = {
    clean_up_tokenization_spaces?: boolean;
    skip_special_tokens?: boolean;
};
type HfTokenizer = {
    decode: (tokenIds: Array<number>, options?: HfDecodeOptions) => string;
    encode: (text: string, options?: HfEncodeOptions) => HfEncodeResult;
};
type TokenizerState = {
    tokenizer: HfTokenizer;
};
export declare class HuggingFaceTokenizer extends BaseTokenizer<TokenizerState> {
    readonly modelId: ModelId;
    constructor(modelId: ModelId);
    protected createState(): {
        tokenizer: HfTokenizer;
    };
    protected encodeWithState(text: string, state: TokenizerState): number[];
    protected tokenizeWithState(text: string, state: TokenizerState): import("../tokenization.js").RawTokenizeResult<string>;
}
export {};