export declare const tokenizer: (nBuckets: number, size: number, offset: number) => (text: string) => string[];
export declare const tokenize: (text: string) => string[];
export { tokenize as segment };
//# sourceMappingURL=tokenize.d.ts.map