import { type SegmentStream } from '../types/SegmentStream.js';
import { type Token } from '../types/Token.js';
export declare function tokenizer<T>(segments: SegmentStream<T>): Generator<Token<T>>;