export interface Tokenizer {
    encode(content: string): number[];
}