export interface ITokenRange {
    start: number;
    end: number;
}
export interface IToken {
    type: TokenType;
    value: string;
    position: ITokenRange;
}
export declare enum TokenType {
    Keyword = 0,
    Identifier = 1,
    SingleLineComment = 2,
    MultiLineComment = 3,
    LiteralString = 4,
    LiteralNumber = 5,
    Punctuator = 6
}
export interface ITokenizerOptions {
    /**
     * File name
     */
    file: string;
    contents: Uint8Array;
    textEncoder: ITextEncoder;
    textDecoder: ITextDecoder;
}
export interface ITextEncoder {
    encode(value: string): Uint8Array;
}
export interface ITextDecoder {
    decode(value: Uint8Array): string;
}
export default class Tokenizer {
    #private;
    constructor({ contents, file, textDecoder, textEncoder }: ITokenizerOptions);
    tokens(): ReadonlyArray<IToken>;
    comments(): ReadonlyArray<IToken>;
    tokenize(): this;
}
//# sourceMappingURL=Tokenizer.d.ts.map