import type { RawTextTokenizeResult, RawTokenizeResult, TokenizeInput } from '../../tokenization.js';
export declare abstract class BaseTokenizer<TState> {
    #private;
    protected abstract createState(): TState;
    protected disposeState(_state: TState): void;
    encode(input: TokenizeInput): number[];
    protected abstract encodeWithState(text: string, state: TState): Array<number>;
    free(): void;
    protected getState(): TState;
    getTokenCount(input: TokenizeInput): number;
    tokenize<InputGeneric extends TokenizeInput>(input: InputGeneric): RawTokenizeResult<InputGeneric>;
    protected abstract tokenizeWithState(text: string, state: TState): RawTextTokenizeResult;
}