export type CodeTokenType = 'comment' | 'string' | 'number' | 'keyword' | 'function' | 'type' | 'property' | 'tag' | 'attribute' | 'operator' | 'punctuation' | 'inserted' | 'deleted' | 'plain';
export type CodeToken = {
    type: CodeTokenType;
    value: string;
};
export declare function hasGrammar(language: string | undefined): boolean;
export declare function tokenizeCode(code: string, language: string | undefined): CodeToken[];