import { File, TokenizedFile, Region } from "@dodona/dolos-core";
import { Language } from "../language.js";
export interface Token {
    token: string;
    location: Region;
}
export type TokenizerOptions = Partial<{
    includeComments: boolean;
}>;
export declare abstract class Tokenizer {
    readonly language: Language;
    protected readonly options: TokenizerOptions;
    constructor(language: Language, options?: TokenizerOptions);
    /**
     * Runs the parser on a given string. Returns a list of Tokens
     * containing the stringified version of the token and the
     * corresponding position.
     *
     * @param text The text string to parse
     */
    abstract generateTokens(text: string): Token[];
    /**
     * Returns a tokenized version of the given file.
     *
     * @param file The file to parse
     */
    tokenizeFile(file: File): TokenizedFile;
    /**
     * Returns a stringified version of the tokens in the buffer
     *
     * @param text The buffer to stringify
     */
    tokenize(text: string): string;
    /**
     * Runs the stringifier on a given buffer. Returns a tuple containing the
     * stringified version and an array containing a mapping from each token to
     * the corresponding token in the original buffer.
     *
     * @param text The text buffer to stringify
     */
    tokenizeWithMapping(text: string): [string[], Region[]];
    /**
     * Returns a new token-object.
     * Just a shorthand for {token: ..., location: ...}.
     *
     * @param token the text of the token
     * @param location the location of the token
     */
    protected newToken(token: string, location: Region): Token;
}
//# sourceMappingURL=tokenizer.d.ts.map