import { TextRange } from '../common/textRange';
import { TextRangeCollection } from '../common/textRangeCollection';
import { OperatorFlags, OperatorType, Token } from './tokenizerTypes';
export interface TokenizerOutput {
    tokens: TextRangeCollection<Token>;
    lines: TextRangeCollection<TextRange>;
    typeIgnoreLines: Map<number, IgnoreComment>;
    pyrightIgnoreLines: Map<number, IgnoreComment>;
    typeIgnoreAll: IgnoreComment | undefined;
    predominantEndOfLineSequence: string;
    hasPredominantTabSequence: boolean;
    predominantTabSequence: string;
    predominantSingleQuoteCharacter: string;
}
export interface IgnoreCommentRule {
    text: string;
    range: TextRange;
}
export interface IgnoreComment {
    range: TextRange;
    rulesList: IgnoreCommentRule[] | undefined;
}
export declare class Tokenizer {
    private _cs;
    private _tokens;
    private _prevLineStart;
    private _parenDepth;
    private _lineRanges;
    private _indentAmounts;
    private _typeIgnoreAll;
    private _typeIgnoreLines;
    private _pyrightIgnoreLines;
    private _comments;
    private _fStringStack;
    private _activeFString;
    private _crCount;
    private _crLfCount;
    private _lfCount;
    private _indentCount;
    private _indentTabCount;
    private _indentSpacesTotal;
    private _singleQuoteCount;
    private _doubleQuoteCount;
    private _useNotebookMode;
    tokenize(text: string, start?: number, length?: number, initialParenDepth?: number, useNotebookMode?: boolean): TokenizerOutput;
    static getOperatorInfo(operatorType: OperatorType): OperatorFlags;
    static isWhitespace(token: Token): boolean;
    static isPythonKeyword(name: string, includeSoftKeywords?: boolean): boolean;
    static isPythonIdentifier(value: string): boolean;
    static isOperatorAssignment(operatorType?: OperatorType): boolean;
    static isOperatorComparison(operatorType?: OperatorType): boolean;
    private _addNextToken;
    private _handleCharacter;
    private _addLineRange;
    private _handleNewLine;
    private _readIndentationAfterNewLine;
    private _setIndent;
    private _tryIdentifier;
    private _isPossibleNumber;
    private _tryNumber;
    private _tryOperator;
    private _handleInvalid;
    private _getComments;
    private _getIPythonMagicsKind;
    private _handleIPythonMagics;
    private _handleComment;
    private _getIgnoreCommentRulesList;
    private _addComments;
    private _getStringPrefixLength;
    private _getQuoteTypeFlags;
    private _handleString;
    private _handleFStringMiddle;
    private _skipToEndOfStringLiteral;
    private _skipFloatingPointCandidate;
    private _skipFractionalNumber;
    private _skipDecimalNumber;
}