# vim:fileencoding=utf-8
# License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
# globals: readfile, writefile, stat_file, sha1sum, ast_from_json,
# globals: ast_to_json, make_lazy_ast_module, decode_cache, encode_cache
from __python__ import hash_literals

from ast import (
    AST_ArgsDef,
    AST_Array,
    AST_Assign,
    AST_Binary,
    AST_BlockStatement,
    AST_Break,
    AST_Call,
    AST_CallArg,
    AST_CallArgs,
    AST_Catch,
    AST_Class,
    AST_ClassCall,
    AST_Conditional,
    AST_Constant,
    AST_Continue,
    AST_DWLoop,
    AST_Debugger,
    AST_Decorator,
    AST_Definitions,
    AST_DictComprehension,
    AST_Directive,
    AST_Do,
    AST_Dot,
    AST_Else,
    AST_EmptyStatement,
    AST_Except,
    AST_ExpressiveObject,
    AST_False,
    AST_Finally,
    AST_ForIn,
    AST_ForJS,
    AST_Function,
    AST_GeneratorComprehension,
    AST_Hole,
    AST_If,
    AST_Import,
    AST_ImportedVar,
    AST_Imports,
    AST_ListComprehension,
    AST_Method,
    AST_New,
    AST_Null,
    AST_Number,
    AST_Object,
    AST_ObjectKeyVal,
    AST_PropAccess,
    AST_RegExp,
    AST_Return,
    AST_Scope,
    AST_Set,
    AST_SetComprehension,
    AST_SetItem,
    AST_Seq,
    AST_SimpleStatement,
    AST_Splice,
    AST_String,
    AST_Sub,
    AST_ItemAccess,
    AST_SymbolAlias,
    AST_SymbolCatch,
    AST_SymbolDefun,
    AST_SymbolFunarg,
    AST_SymbolLambda,
    AST_SymbolNonlocal,
    AST_SymbolRef,
    AST_SymbolVar,
    AST_This,
    AST_Throw,
    AST_Toplevel,
    AST_True,
    AST_Try,
    AST_UnaryPrefix,
    AST_Undefined,
    AST_Var,
    AST_VarDef,
    AST_Verbatim,
    AST_While,
    AST_With,
    AST_WithClause,
    AST_Yield,
    AST_Await,
    AST_Assert,
    AST_Existential,
    is_node_type,
    TreeWalker
)
from errors import SyntaxError, ImportError
from tokenizer import tokenizer, is_token, RESERVED_WORDS
from utils import (
    make_predicate, array_to_hash, defaults, has_prop, cache_file_name
)


COMPILER_VERSION = '__COMPILER_VERSION__'
PYTHON_FLAGS = {
    'dict_literals': True,
    'overload_getitem': True,
    'bound_methods': True,
    'hash_literals': True,
}
CACHE_VERSION = 1


def get_compiler_version():
    return COMPILER_VERSION


def static_predicate(names):
    return {k: True for k in names.split(' ')}

NATIVE_CLASSES = {
    'Image': {},
    'FileReader': {},
    'RegExp': {},
    'Error': {},
    'EvalError': {},
    'InternalError': {},
    'RangeError': {},
    'ReferenceError': {},
    'SyntaxError': {},
    'TypeError': {},
    'URIError': {},
    'Object': {'static': static_predicate(
        'getOwnPropertyNames getOwnPropertyDescriptor'
        ' getOwnPropertyDescriptors'
        ' getOwnPropertySymbols keys entries values create defineProperty'
        ' defineProperties getPrototypeOf setPrototypeOf assign'
        ' seal isSealed is preventExtensions isExtensible'
        ' freeze isFrozen'
    )},
    'String': {'static': static_predicate('fromCharCode')},
    'Array': {'static': static_predicate('isArray from of')},
    'Function': {},
    'Date': {'static': static_predicate('UTC now parse')},
    'ArrayBuffer': {'static': static_predicate('isView transfer')},
    'DataView': {},
    'Float32Array': {},
    'Float64Array': {},
    'Int16Array': {},
    'Int32Array': {},
    'Int8Array': {},
    'Uint16Array': {},
    'Uint32Array': {},
    'Uint8Array': {},
    'Uint8ClampedArray': {},
    'Map': {},
    'WeakMap': {},
    'Proxy': {},
    'Set': {},
    'WeakSet': {},
    'Promise': {'static': static_predicate('all race reject resolve')},
    'WebSocket': {},
    'XMLHttpRequest': {},
    'TextEncoder': {},
    'TextDecoder': {},
    'MouseEvent': {},
    'Event': {},
    'CustomEvent': {},
    'Blob': {},
}
ERROR_CLASSES = {
    'Exception': {},
    'AttributeError': {},
    'IndexError': {},
    'KeyError': {},
    'ValueError': {},
    'UnicodeDecodeError': {},
    'AssertionError': {},
    'ZeroDivisionError': {},
}
COMMON_STATIC = static_predicate('call apply bind toString')
FORBIDDEN_CLASS_VARS = 'prototype constructor'.split(' ')

# -----[ Parser (constants) ]-----
UNARY_PREFIX = make_predicate('typeof void delete ~ - + ! @')

ASSIGNMENT = make_predicate('= += -= /= //= *= %= >>= <<= >>>= |= ^= &=')

PRECEDENCE = (def (a, ret):
    for i in range(a.length):
        b = a[i]
        for j in range(b.length):
            ret[b[j]] = i + 1
    return ret
)([
    # lowest precedence
    ['||'],
    ['&&'],
    ['|'],
    ['^'],
    ['&'],
    ['==', '===', '!=', '!=='],
    ['<', '>', '<=', '>=', 'in', 'nin', 'instanceof'],
    ['>>', '<<', '>>>'],
    ['+', '-'],
    ['*', '/', '//', '%'],
    ['**']
    # highest precedence
], {})

STATEMENTS_WITH_LABELS = array_to_hash(['for', 'do', 'while', 'switch'])

ATOMIC_START_TOKEN = array_to_hash([
    'atom', 'num', 'string', 'regexp', 'name', 'js'
])

compile_time_decorators = ['staticmethod', 'external', 'property']


def has_simple_decorator(decorators, name):
    remove = v'[]'
    for v'var i = 0; i < decorators.length; i++':
        s = decorators[i]
        if is_node_type(s, AST_SymbolRef) and not s.parens and s.name is name:
            remove.push(i)
    if remove.length:
        remove.reverse()
        for v'var i = 0; i < remove.length; i++':
            decorators.splice(remove[i], 1)
        return True
    return False


def has_setter_decorator(decorators, name):
    remove = v'[]'
    for v'var i = 0; i < decorators.length; i++':
        s = decorators[i]
        if is_node_type(s, AST_Dot) and is_node_type(
            s.expression,
            AST_SymbolRef
        ) and s.expression.name is name and s.property is 'setter':
            remove.push(i)
    if remove.length:
        remove.reverse()
        for v'var i = 0; i < remove.length; i++':
            decorators.splice(remove[i], 1)
        return True
    return False


# -----[ Parser ]-----
def create_parser_ctx(
    S, import_dirs, module_id, baselib_items,
    imported_module_ids, imported_modules,
    importing_modules, options
):
    def next():
        S.prev = S.token
        if S.peeked.length:
            S.token = S.peeked.shift()
        else:
            S.token = S.input()

        return S.token

    def is_(type, value):
        return is_token(S.token, type, value)

    def peek():
        if not S.peeked.length:
            S.peeked.push(S.input())
        return S.peeked[0]

    def prev():
        return S.prev

    def croak(msg, line, col, pos, is_eof):
        # note: undefined means nothing was passed in,
        # None/null means a null value was passed in
        ctx = S.input.context()
        raise new SyntaxError(
            msg,
            ctx.filename,
            (line if line is not undefined else ctx.tokline),
            (col if col is not undefined else ctx.tokcol),
            (pos if pos is not undefined else ctx.tokpos),
            is_eof
        )

    def token_error(token, msg):
        is_eof = token.type is 'eof'
        croak(msg, token.line, token.col, undefined, is_eof)

    def unexpected(token):
        if token is undefined:
            token = S.token
        token_error(
            token,
            'Unexpected token: ' + token.type + ' «' + token.value + '»'
        )

    # --- Error recovery (only active when
    # options.recover_errors is True) --------
    # These helpers let the LSP obtain a usable AST from a file that contains
    # syntax errors. They are never invoked on the normal compile path, so they
    # cannot introduce a performance or behavior regression there.

    def is_recoverable_error(e):
        return v'e instanceof SyntaxError'

    def record_recovery(e):
        S.recovered_errors.push({
            'message': e.message, 'line': e.line, 'col': e.col, 'pos': e.pos,
            'endpos': e.endpos, 'is_eof': e.is_eof,
        })

    async def skip_to_recovery_point():
        # Resynchronize after a syntax error: always consume at least one token
        # (guaranteeing forward progress) then stop at the start of the next
        # logical line, a block dedent, or eof. next() may itself raise a
        # tokenizer level SyntaxError (e.g. an unterminated string on the
        # skipped
        # line); such errors are recorded and skipping continues. The tokenizer
        # advances its position for every such error (and never raises for an
        # unexpected character in recover mode), so this loop always terminates.
        while not is_('eof'):
            try:
                next()
            except as e:
                if is_recoverable_error(e):
                    record_recovery(e)
                    if is_('eof'):
                        return
                    continue
                raise
            if is_('eof') or is_('punc', '}') or S.token.nlb:
                return

    def expect_token(type, val):
        if is_(type, val):
            return next()
        token_error(
            S.token,
            'Unexpected token ' + S.token.type +
            ' «' + S.token.value + '»' +
            ', expected ' + type + ' «' + val + '»'
        )

    def expect(punc):
        return expect_token('punc', punc)

    def semicolon():
        if is_('punc', ';'):
            next()
            S.token.nlb = True

    def embed_tokens(parser):
        async def with_embedded_tokens():
            start = S.token
            expr = await parser()
            if expr is undefined:
                unexpected()
            end = prev()
            expr.start = start
            expr.end = end
            return expr
        return with_embedded_tokens

    def scan_for_top_level_callables(body):
        ans = v'[]'
        # Get the named functions and classes
        if Array.isArray(body):
            for obj in body:
                if is_node_type(
                    obj,
                    AST_Function
                ) or is_node_type(obj, AST_Class):
                    if obj.name:
                        ans.push(obj.name.name)
                    else:
                        token_error(
                            obj.start,
                            'Top-level functions must have names'
                        )
                else:
                    # skip inner scopes
                    if is_node_type(obj, AST_Scope):
                        continue
                    for x in ['body', 'alternative']:
                        opt = obj[x]
                        if opt:
                            ans = ans.concat(scan_for_top_level_callables(opt))

                        if is_node_type(
                            opt,
                            AST_Assign
                        ) and not (is_node_type(opt.right, AST_Scope)):
                            ans = ans.concat(
                                scan_for_top_level_callables(opt.right)
                            )

        elif body.body:
            # recursive descent into wrapper statements that contain body blocks
            ans = ans.concat(scan_for_top_level_callables(body.body))
            if body.alternative:
                ans = ans.concat(scan_for_top_level_callables(body.alternative))

        return ans

    def scan_for_classes(body):
        ans = {}
        for obj in body:
            if is_node_type(obj, AST_Class):
                ans[obj.name.name] = obj
        return ans

    def scan_for_local_vars(body):
        """
        Pick out all variables being assigned to from within this scope,
        we'll mark them as local

        body        body to be scanned
        """
        localvars = v'[]'
        seen = {}

        def push(x):
            if has_prop(seen, x):
                return
            seen[x] = True
            localvars.push(x)

        def extend(arr):
            for x in arr:
                push(x)

        def scan_in_array(arr):
            for x in arr:
                if is_node_type(x, AST_Seq):
                    x = x.to_array()
                elif is_node_type(x, AST_Array):
                    x = x.elements
                if Array.isArray(x):
                    scan_in_array(x)
                else:
                    if not is_node_type(x, AST_PropAccess):
                        push(x.name)

        def add_assign_lhs(lhs):
            if is_node_type(lhs, AST_Seq):
                lhs = new AST_Array({'elements': lhs.to_array()})
            if is_node_type(lhs, AST_Array):
                # assignment to an implicit tuple
                push('ρσ_unpack')
                scan_in_array(lhs.elements)
            elif lhs.name:
                # assignment to a single variable
                push(lhs.name)

        def add_for_in(stmt):
            if is_node_type(stmt.init, AST_Array):
                # iteration via implicit tuple
                push('ρσ_unpack')
                scan_in_array(stmt.init.elements)
            else:
                # iteration via a single variable
                push(stmt.init.name)

        if Array.isArray(body):
            # this is a body of statements
            for stmt in body:
                # skip inner scopes
                if is_node_type(stmt, AST_Scope):
                    continue

                # recursive descent into conditional, loop and exception bodies
                for option in ('body', 'alternative', 'bcatch', 'condition'):
                    opt = stmt[option]
                    if opt:
                        extend(scan_for_local_vars(opt))

                    if is_node_type(
                        opt,
                        AST_Assign
                    ) and not (is_node_type(opt.right, AST_Scope)):
                        extend(scan_for_local_vars(opt.right))

                # pick up iterators from loops
                if is_node_type(stmt, AST_ForIn):
                    add_for_in(stmt)
                elif is_node_type(stmt, AST_DWLoop):
                    extend(scan_for_local_vars(stmt))
                elif is_node_type(stmt, AST_With):
                    push('ρσ_with_exception'), push('ρσ_with_suppress')
                    for clause in stmt.clauses:
                        if clause.alias:
                            push(clause.alias.name)

        elif body.body:
            # recursive descent into wrapper statements that contain body blocks
            extend(scan_for_local_vars(body.body))
            if body.alternative:
                extend(scan_for_local_vars(body.alternative))

        elif is_node_type(body, AST_Assign):
            # this is a single assignment operator
            if body.is_chained():
                is_compound_assign = False
                for lhs in body.traverse_chain()[0]:
                    add_assign_lhs(lhs)
                    if is_node_type(
                        lhs,
                        AST_Seq
                    ) or is_node_type(lhs, AST_Array):
                        is_compound_assign = True
                        break
                if is_compound_assign:
                    push('ρσ_chain_assign_temp')
            else:
                add_assign_lhs(body.left)
            if not is_node_type(body.right, AST_Scope):
                extend(scan_for_local_vars(body.right))

        elif is_node_type(body, AST_ForIn):
            add_for_in(body)

        return localvars

    def scan_for_nonlocal_defs(body):
        vars = v'[]'
        if Array.isArray(body):
            for stmt in body:
                if is_node_type(stmt, AST_Scope):
                    continue

                # don't invade nested scopes
                if is_node_type(stmt, AST_Definitions):
                    for vardef in stmt.definitions:
                        vars.push(vardef.name.name)

                for option in ('body', 'alternative'):
                    nonlocal vars
                    opt = stmt[option]
                    if opt:
                        vars = vars.concat(scan_for_nonlocal_defs(opt))

        elif body.body:
            vars = vars.concat(scan_for_nonlocal_defs(body.body))
            if body.alternative:
                vars = vars.concat(scan_for_nonlocal_defs(body.alternative))

        return vars

    async def return_():
        if is_('punc', ';'):
            semicolon()
            value = None
        else:
            is_end_of_statement = S.token.nlb or is_('eof') or is_('punc', '}')
            if is_end_of_statement:
                value = None
            else:
                value = await expression(True)
                semicolon()
        return value

    @embed_tokens
    async def statement():
        # From Kovid: The next three lines were a hack to try to support
        # statements
        # starting with a regexp literal. However, it did not work, for example:
        # echo 'f=1\n/asd/.test()' | rs -> parse error
        # So we just accept that this cannot be supported in RS, and avoid
        # hacks that mess
        # with the internal state of S. In any case,
        # statements starting with a literal are very rare.
        if S.token.type is 'operator' and S.token.value.substr(0, 1) is '/':
            token_error(
                S.token,
                'RapydScript does not support statements starting with'
                ' regexp literals'
            )

        S.statement_starting_token = S.token
        tmp_ = S.token.type
        p = prev()
        if p and not S.token.nlb and ATOMIC_START_TOKEN[p.type] and not is_(
            'punc',
            ':'
        ):
            unexpected()
        if tmp_ is 'string':
            return await simple_statement()
        elif tmp_ is 'shebang':
            tmp_ = S.token.value
            next()
            return new AST_Directive({
                'value': tmp_
            })
        elif (tmp_ is 'num' or tmp_ is 'regexp' or tmp_ is 'operator'
              or tmp_ is 'atom' or tmp_ is 'js'):
            return await simple_statement()
        elif tmp_ is 'punc':
            tmp_ = S.token.value
            if tmp_ is ':':
                bs_body = await block_()
                return new AST_BlockStatement({
                    'start': S.token,
                    'body': bs_body,
                    'end': prev()
                })
            elif tmp_ is '{' or tmp_ is '[' or tmp_ is '(':
                return await simple_statement()
            elif tmp_ is ';':
                next()
                return new AST_EmptyStatement({
                    'stype': ';', 'start': prev(), 'end': prev()
                })
            else:
                unexpected()
        elif tmp_ is 'name':
            if (is_token(peek(), 'punc', ':')) token_error(
                peek(),
                'invalid syntax, colon not allowed here'
            )
            return await simple_statement()
        elif tmp_ is 'keyword':
            tmp_ = S.token.value
            next()
            if tmp_ is 'break':
                return break_cont(AST_Break)
            elif tmp_ is 'continue':
                return break_cont(AST_Continue)
            elif tmp_ is 'debugger':
                semicolon()
                return new AST_Debugger()
            elif tmp_ is 'do':
                do_body = await in_loop(statement)
                do_cond = await (async def ():
                    expect('.')
                    expect_token('keyword', 'while')
                    tmp = await expression(True)
                    if is_node_type(tmp, AST_Assign):
                        croak(
                            'Assignments in do loop conditions are not allowed'
                        )
                    semicolon()
                    return tmp
                )()
                return new AST_Do({
                    'body': do_body,
                    'condition': do_cond,
                })
            elif tmp_ is 'while':
                while_cond = await expression(True)
                if is_node_type(while_cond, AST_Assign):
                    croak(
                        'Assignments in while loop conditions are not allowed'
                    )
                if not is_('punc', ':'):
                    croak('Expected a colon after the while statement')
                while_body = await in_loop(statement)
                return new AST_While({
                    'condition': while_cond,
                    'body': while_body
                })
            elif tmp_ is 'for':
                if is_('js'):
                    return await for_js()
                return await for_()
            elif tmp_ is 'from':
                return await import_(True)
            elif tmp_ is 'import':
                return await import_(False)
            elif tmp_ is 'class':
                return await class_()
            elif tmp_ is 'async':
                if not is_('keyword', 'def'):
                    croak("Expected 'def' after 'async'")
                next()
                start = prev()
                func = await function_(S.in_class[-1], False, True)
                func.start = start
                func.end = prev()
                chain = await subscripts(func, True)
                if chain is func:
                    return func
                else:
                    return new AST_SimpleStatement({
                        'start': start,
                        'body': chain,
                        'end': prev()
                    })
            elif tmp_ is 'def':
                start = prev()
                func = await function_(S.in_class[-1], False)
                func.start = start
                func.end = prev()
                chain = await subscripts(func, True)
                if chain is func:
                    return func
                else:
                    return new AST_SimpleStatement({
                        'start': start,
                        'body': chain,
                        'end': prev()
                    })
            elif tmp_ is 'assert':
                start = prev()
                cond = await expression(False)
                msg = None
                if is_('punc', ','):
                    next()
                    msg = await expression(False)
                return new AST_Assert({
                    'start': start, 'condition': cond,
                    'message': msg, 'end': prev()
                })
            elif tmp_ is 'if':
                return await if_()
            elif tmp_ is 'pass':
                semicolon()
                return new AST_EmptyStatement({
                    'stype': 'pass', 'start': prev(), 'end': prev()
                })
            elif tmp_ is 'return':
                if S.in_function is 0:
                    croak("'return' outside of function")
                if S.functions[-1].is_generator:
                    croak("'return' not allowed in a function with yield")
                S.functions[-1].is_generator = False

                ret_val = await return_()
                return new AST_Return({'value': ret_val})
            elif tmp_ is 'await':
                if S.in_function is 0 or not S.functions[-1].is_async:
                    croak("'await' outside of async function")
                value = await expr_atom(True)
                node = new AST_Await({'value': value})
                semicolon()
                return new AST_SimpleStatement({'body': node})
            elif tmp_ is 'yield':
                return await yield_()
            elif tmp_ is 'raise':
                if S.token.nlb:
                    return new AST_Throw({
                        'value': new AST_SymbolCatch({
                            'name': 'ρσ_Exception'
                        })
                    })

                tmp = await expression(True)
                semicolon()
                return new AST_Throw({
                    'value': tmp
                })
            elif tmp_ is 'try':
                return await try_()
            elif tmp_ is 'nonlocal':
                tmp = await nonlocal_()
                semicolon()
                return tmp
            elif tmp_ is 'global':
                tmp = await nonlocal_(True)
                semicolon()
                return tmp
            elif tmp_ is 'with':
                return await with_()
            else:
                unexpected()

    async def with_():
        clauses = v'[]'
        start = S.token
        while True:
            if is_('eof'):
                unexpected()
            expr = await expression()
            alias = None
            if is_('keyword', 'as'):
                next()
                alias = as_symbol(AST_SymbolAlias)
            clauses.push(new AST_WithClause({
                'expression': expr, 'alias': alias
            }))
            if is_('punc', ','):
                next()
                continue
            if not is_('punc', ':'):
                unexpected()
            break

        if not clauses.length:
            token_error(start, 'with statement must have at least one clause')
        body = await statement()

        return new AST_With({
            'clauses': clauses,
            'body': body
        })

    async def simple_statement(tmp):
        tmp = await expression(True)
        semicolon()
        return new AST_SimpleStatement({
            'body': tmp
        })

    def break_cont(t):
        if S.in_loop is 0:
            croak(t.name.slice(4) + ' not inside a loop or switch')
        semicolon()
        return new t()

    async def yield_():
        if S.in_function is 0:
            croak("'yield' outside of function")
        if S.functions[-1].is_async:
            croak("'yield' not allowed in an async function")
        if S.functions[-1].is_generator is False:
            croak("'yield' not allowed in a function with return")
        S.functions[-1].is_generator = True
        is_yield_from = is_('keyword', 'from')
        if is_yield_from:
            next()
        yld_val = await return_()
        return new AST_Yield({'is_yield_from': is_yield_from, 'value': yld_val})

    async def for_(list_comp):
        #        expect("(")
        init = None
        if not is_('punc', ';'):
            init = await expression(True, True)
            # standardize AST_Seq into array now for consistency
            if is_node_type(init, AST_Seq):
                if is_node_type(
                    init.car,
                    AST_SymbolRef
                ) and is_node_type(init.cdr, AST_SymbolRef):
                    # Optimization to prevent runtime call to ρσ_flatten
                    # when init is simply (a, b)
                    tmp = init.to_array()
                else:
                    tmp = [init]
                init = new AST_Array({
                    'start': init.start,
                    'elements': tmp,
                    'end': init.end
                })

            if is_('operator', 'in'):
                if is_node_type(init, AST_Var) and init.definitions.length > 1:
                    croak(
                        'Only one variable declaration allowed in for..in loop'
                    )
                next()
                return await for_in(init, list_comp)

        unexpected()

    async def for_in(init, list_comp):
        lhs = init.definitions[0].name if is_node_type(init, AST_Var) else None
        obj = await expression(True)
        #        expect(")")
        if list_comp:
            return {
                'init': init,
                'name': lhs,
                'object': obj
            }

        fi_body = await in_loop(statement)
        return new AST_ForIn({
            'init': init,
            'name': lhs,
            'object': obj,
            'body': fi_body
        })

    # A native JavaScript for loop - for v"var i=0; i<5000; i++":
    async def for_js():
        condition = as_atom_node()
        fj_body = await in_loop(statement)
        return new AST_ForJS({
            'condition': condition,
            'body': fj_body
        })

    # scan function/class body for nested class declarations
    def get_class_in_scope(expr):
        # TODO: Currently if a local variable shadows a class name defined in
        # an outerscope, the logic below will identify that variable as a
        # class. This bug was always present. Fixing it will require the parser
        # to maintain a list of local variables for every AST_Scope and provide
        # an easy way to walk the ast tree upwards.
        if is_node_type(expr, AST_SymbolRef):
            # check Native JS classes
            if has_prop(NATIVE_CLASSES, expr.name):
                return NATIVE_CLASSES[expr.name]
            if has_prop(ERROR_CLASSES, expr.name):
                return ERROR_CLASSES[expr.name]

            # traverse in reverse to check local variables first
            for s in range(S.classes.length - 1, -1, -1):
                if has_prop(S.classes[s], expr.name):
                    return S.classes[s][expr.name]

        elif is_node_type(expr, AST_Dot):
            referenced_path = []
            # this one is for detecting classes inside modules and
            # eventually nested classes
            while is_node_type(expr, AST_Dot):
                referenced_path.unshift(expr.property)
                expr = expr.expression
            if is_node_type(expr, AST_SymbolRef):
                referenced_path.unshift(expr.name)
                # now 'referenced_path' should contain the full path
                # of potential class
                if len(referenced_path) > 1:
                    class_name = referenced_path.join('.')
                    for s in range(S.classes.length - 1, -1, -1):
                        if has_prop(S.classes[s], class_name):
                            return S.classes[s][class_name]
        return False

    def import_error(message):
        ctx = S.input.context()
        raise new ImportError(
            message,
            ctx.filename,
            ctx.tokline,
            ctx.tokcol,
            ctx.tokpos
        )

    async def do_import(key):
        if has_prop(imported_modules, key):
            return
        # Circular import check must come BEFORE loading_promises so we
        # detect cycles
        # even when the same module is being loaded by a parallel chain.
        if has_prop(importing_modules, key) and importing_modules[key]:
            import_error(
                'Detected a recursive import of: ' + key +
                ' while importing: ' + module_id
            )
        # If a parallel chain is already loading this module, wait for it
        # instead of
        # starting a duplicate load.  loading_promises is shared across
        # the entire
        # compilation via options.loading_promises.
        loading_promises = options.loading_promises
        if has_prop(loading_promises, key):
            await loading_promises[key]
            return

        # Register this load BEFORE the first await so that any parallel caller
        # that reaches this point sees the in-progress entry and waits.
        # Use a single-element list to capture the resolve function from
        # the Promise
        # executor without needing nonlocal (mutating a container,
        # not rebinding).
        _rfn = v'[null]'

        def _capture_resolve(resolve):
            _rfn[0] = resolve
        loading_promises[key] = new Promise(_capture_resolve)

        try:
            # Ensure that the package containing this module is also imported
            package_module_id = key.split('.')[:-1].join('.')
            if len(package_module_id) > 0:
                await do_import(package_module_id)

            if options.for_linting:
                imported_modules[key] = {
                    'is_cached': True,
                    'classes': {},
                    'module_id': key,
                    'exports': [],
                    'nonlocalvars': [],
                    'baselib': {},
                    'outputs': {},
                    'discard_asserts': options.discard_asserts,
                }
                return

            # stat_file returns {mtimeMs, content?}.  For real filesystems
            # mtimeMs is a
            # number and content is absent; for virtual filesystems
            # content holds the
            # already-read source to avoid a double read.
            async def safe_stat(base_path):  # noqa
                for i, path in enumerate([
                    base_path + '.pyj', base_path + '/__init__.pyj'
                ]):
                    try:
                        st = await stat_file(path)
                        return [
                            st.mtimeMs,
                            path,
                            st.content if has_prop(st, 'content') else None,
                        ]
                    except as e:
                        if (e.code is 'ENOENT' or e.code is 'EPERM'
                                or e.code is 'EACCESS'):
                            if i is 1:
                                return None, None, None
                        if i is 1:
                            raise

            file_mtime = filename = prefetched_src = None
            modpath = key.replace(/\./g, '/')

            for location in import_dirs:
                if location:
                    file_mtime, filename, prefetched_src = await safe_stat(
                        location + '/' + modpath
                    )
                    if filename is not None:
                        break
            if filename is None:
                import_error(
                    "Failed Import: '" + key +
                    "' module doesn't exist in any of the import"
                    ' directories: ' + import_dirs.join(':')
                )

            # Read the cache file.  I/O starts here; any CPU work done
            # before the
            # await can overlap with the disk read.
            cache_read = readfile(
                cache_file_name(filename, options.module_cache_dir)
            )

            cached = None
            ast_json_str = None
            try:
                _split = decode_cache(await cache_read)
                if _split is not None:
                    cached = JSON.parse(_split.meta)
                    ast_json_str = _split.ast
            except:
                cached = None

            srchash = src_code = None
            cache_ok = (
                cached and cached.v is CACHE_VERSION
                and cached.compiler_version is COMPILER_VERSION
            )
            if (cache_ok and file_mtime is not None
                    and cached.mtime is not None
                    and cached.mtime is file_mtime):
                # Fast path: mtime matches — source file is unchanged,
                # skip reading and hashing it
                srchash = cached.signature
            else:
                # Need source content to validate with sha1sum (or to
                # re-parse on miss)
                src_code = prefetched_src or await readfile(filename, 'utf-8')
                srchash = sha1sum(src_code)
                if not cache_ok or cached.signature is not srchash:
                    cached = None

            if cached is not None:
                # Load all deps in parallel — cached module deps have been
                # pre-validated as
                # non-circular so Promise.all is safe.  The loading_promises
                # dict ensures
                # that if two parallel chains try to load the same dep,
                # only one actually
                # does the work; the other waits.
                pending = v'[]'
                for ikey in cached.imported_module_ids:
                    if not has_prop(imported_modules, ikey):
                        pending.push(do_import(ikey))
                if pending.length > 0:
                    await Promise.all(pending)
                # Reconstruct classes dict from the simplified cache
                # (no full AST nodes).
                classes = {}
                for cname in Object.keys(cached.classes or {}):
                    ci = cached.classes[cname]
                    classes[cname] = {
                        'name': {'name': cname},
                        'static': ci.static or {},
                        'bound': ci.bound or [],
                        'classvars': ci.classvars or {},
                    }
                meta = {
                    'module_id': key,
                    'filename': filename,
                    'import_order': Object.keys(imported_modules).length,
                    'imports': imported_modules,
                    'exports': [{'name': n} for n in cached.exported_names],
                    'baselib': cached.baselib or {},
                    'classes': classes,
                }
                if cached.has_side_effects is not undefined:
                    meta['_has_side_effects'] = cached.has_side_effects
                if cached.index is not undefined:
                    meta['_cache_index'] = cached.index
                module = make_lazy_ast_module(ast_json_str, meta)
                imported_modules[key] = module
            else:
                await parse(
                    src_code,
                    {
                        'filename': filename,
                        'toplevel': None,
                        'basedir': options.basedir,
                        'libdir': options.libdir,
                        'import_dirs': options.import_dirs,
                        'module_id': key,
                        'imported_modules': imported_modules,
                        'importing_modules': importing_modules,
                        'discard_asserts': options.discard_asserts,
                        'module_cache_dir': options.module_cache_dir,
                        '_src_hash': srchash,
                        '_file_mtime': file_mtime,
                        'loading_promises': loading_promises,
                    }
                )
        # This function will add the module to imported_modules itself
        finally:
            del loading_promises[key]
            if _rfn[0]:
                _rfn[0](None)

        imported_modules[key].srchash = srchash

        for bitem in Object.keys(imported_modules[key].baselib):
            baselib_items[bitem] = True

    def read_python_flags():
        expect_token('keyword', 'import')
        bracketed = is_('punc', '(')
        if bracketed:
            next()
        while True:
            if not is_('name'):
                croak('Name expected')
            name = S.token.value
            val = False if name.startsWith('no_') else True
            if not val:
                name = name.slice(3)
            if not PYTHON_FLAGS:
                croak('Unknown __python__ flag: ' + name)
            S.scoped_flags.set(name, val)
            next()
            if is_('punc', ','):
                next()
            else:
                if bracketed:
                    if is_('punc', ')'):
                        next()
                    else:
                        continue
                break
        return new AST_EmptyStatement({
            'stype': 'scoped_flags', 'start': prev(), 'end': prev()
        })

    async def import_(from_import):
        ans = new AST_Imports({'imports': []})
        while True:
            tok = tmp = name = last_tok = await expression(False)
            key = ''
            while is_node_type(tmp, AST_Dot):
                key = '.' + tmp.property + key
                tmp = last_tok = tmp.expression
            key = tmp.name + key
            if from_import and key is '__python__':
                return read_python_flags()
            alias = None
            if not from_import and is_('keyword', 'as'):
                next()
                alias = as_symbol(AST_SymbolAlias)
            aimp = new AST_Import({
                'module': name,
                'key': key,
                'alias': alias,
                'argnames': None,
                'body': def ():
                    return imported_modules[key]
            })
            aimp.start, aimp.end = tok.start, last_tok.end
            ans.imports.push(aimp)
            if from_import:
                break
            if is_('punc', ','):
                next()
            else:
                break

        for imp in ans['imports']:
            await do_import(imp.key)
            if imported_module_ids.indexOf(imp.key) is -1:
                imported_module_ids.push(imp.key)
            classes = imported_modules[key].classes
            if from_import:
                expect_token('keyword', 'import')
                imp.argnames = argnames = []
                bracketed = is_('punc', '(')
                if bracketed:
                    next()
                exports = {}
                for symdef in imported_modules[key].exports:
                    exports[symdef.name] = True
                while True:
                    aname = as_symbol(AST_ImportedVar)
                    if not options.for_linting and not has_prop(
                        exports,
                        aname.name
                    ):
                        import_error(
                            'The symbol "' + aname.name +
                            '" is not exported from the module: ' + key
                        )
                    if is_('keyword', 'as'):
                        next()
                        aname.alias = as_symbol(AST_SymbolAlias)
                    argnames.push(aname)
                    if is_('punc', ','):
                        next()
                    else:
                        if bracketed:
                            if is_('punc', ')'):
                                next()
                            else:
                                continue
                        break

                # Put imported class names in the outermost scope
                for argvar in argnames:
                    obj = classes[argvar.name]
                    if obj:
                        key = argvar.alias.name if argvar.alias else argvar.name
                        S.classes[-1][key] = {
                            'static': obj.static,
                            'bound': obj.bound,
                            'classvars': obj.classvars,
                        }
            else:
                for cname in Object.keys(classes):
                    obj = classes[cname]
                    key = imp.alias.name if imp.alias else imp.key
                    S.classes[-1][key + '.' + obj.name.name] = {
                        'static': obj.static,
                        'bound': obj.bound,
                        'classvars': obj.classvars,
                    }

        return ans

    async def class_():
        name = as_symbol(AST_SymbolDefun)
        if not name:
            unexpected()

        # detect external classes
        externaldecorator = has_simple_decorator(S.decorators, 'external')

        class_details = {
            'static': {},
            'bound': v'[]',
            'classvars': {},
            'processing': name.name,
            'provisional_classvars': {},
        }
        bases = v'[]'
        class_parent = None

        # read the bases of the class, if any
        if is_('punc', '('):
            S.in_parenthesized_expr = True
            next()
            while True:
                if is_('punc', ')'):
                    S.in_parenthesized_expr = False
                    next()
                    break
                a = await expr_atom(False)
                if class_parent is None:
                    class_parent = a
                bases.push(a)
                if is_('punc', ','):
                    next()
                    continue

        docstrings = v'[]'
        cls_decorators = (def ():
            d = []
            for decorator in S.decorators:
                d.push(new AST_Decorator({
                    'expression': decorator
                }))
            S.decorators = v'[]'
            return d
        )()
        cls_body = await (async def (loop, labels):
            # navigate to correct location in the module tree and
            # append the class
            S.in_class.push(name.name)
            S.classes[S.classes.length - 1][name.name] = class_details
            S.classes.push({})
            S.scoped_flags.push()
            S.in_function += 1
            S.in_loop = 0
            S.labels = []
            a = await block_(docstrings)
            S.in_function -= 1
            S.scoped_flags.pop()
            S.classes.pop()
            S.in_class.pop()
            S.in_loop = loop
            S.labels = labels
            return a
        )(S.in_loop, S.labels)
        definition = new AST_Class({
            'name': name,
            'docstrings': docstrings,
            'module_id': module_id,
            'dynamic_properties': Object.create(None),
            'parent': class_parent,
            'bases': bases,
            'localvars': [],
            'classvars': class_details.classvars,
            'static': class_details.static,
            'external': externaldecorator,
            'bound': class_details.bound,
            'statements': [],
            'decorators': cls_decorators,
            'body': cls_body,
        })
        class_details.processing = False
        # find the constructor
        for stmt in definition.body:
            if is_node_type(stmt, AST_Method):
                if stmt.is_getter or stmt.is_setter:
                    descriptor = definition.dynamic_properties[stmt.name.name]
                    if not descriptor:
                        descriptor = definition.dynamic_properties[
                            stmt.name.name
                        ] = {}
                    descriptor['getter' if stmt.is_getter else 'setter'] = stmt
                elif stmt.name.name is '__init__':
                    definition.init = stmt
        # find the class variables
        class_var_names = {}

        # Ensure that if a class variable refers to another class variable in
        # its initialization, the referenced variables' names is correctly
        # mangled.
        def walker():
            def visit_node(node, descend):
                if is_node_type(node, AST_Method):
                    class_var_names[node.name.name] = True
                    return
                if is_node_type(node, AST_Function):
                    return
                if is_node_type(node, AST_Assign) and is_node_type(
                    node.left,
                    AST_SymbolRef
                ):
                    varname = node.left.name
                    if FORBIDDEN_CLASS_VARS.indexOf(varname) is not -1:
                        token_error(
                            node.left.start,
                            varname + ' is not allowed as a class variable name'
                        )
                    class_var_names[varname] = True
                    definition.classvars[varname] = True
                elif is_node_type(node, AST_SymbolRef) and has_prop(
                    class_var_names,
                    node.name
                ):
                    node.thedef = new AST_SymbolDefun({
                        'name': name.name + '.prototype.' + node.name
                    })
                if descend:
                    descend.call(node)
            this._visit = visit_node
        visitor = new walker()

        for stmt in definition.body:
            if not is_node_type(stmt, AST_Class):
                stmt.walk(visitor)
                definition.statements.push(stmt)
        return definition

    async def function_(in_class, is_expression, is_async=False):
        name = as_symbol(
            AST_SymbolDefun if in_class else AST_SymbolLambda
        ) if is_('name') else None
        if in_class and not name:
            croak('Cannot use anonymous function as class methods')
        is_anonymous = not name

        staticmethod = property_getter = property_setter = False
        if in_class:
            staticloc = has_simple_decorator(S.decorators, 'staticmethod')
            property_getter = has_simple_decorator(S.decorators, 'property')
            property_setter = has_setter_decorator(S.decorators, name.name)
            if staticloc:
                if property_getter or property_setter:
                    croak(
                        'A method cannot be both static and a'
                        ' property getter/setter'
                    )
                S.classes[S.classes.length - 2][in_class].static[
                    name.name
                ] = True
                staticmethod = True
            elif (name.name is not '__init__'
                    and S.scoped_flags.get('bound_methods')):
                S.classes[S.classes.length - 2][in_class].bound.push(name.name)

        expect('(')
        S.in_parenthesized_expr = True
        ctor = AST_Method if in_class else AST_Function
        return_annotation = None
        is_generator = v'[]'
        docstrings = v'[]'

        fn_argnames = await (async def ():
            a = v'[]'
            starargs_sym = undefined
            kwargs_sym = undefined
            defaults = {}
            has_defaults = False
            first = True
            seen_names = {}
            def_line = S.input.context().tokline
            current_arg_name = None
            name_token = None
            async def get_arg():
                nonlocal current_arg_name, name_token
                current_arg_name = S.token.value
                if has_prop(seen_names, current_arg_name):
                    token_error(prev(), "Can't repeat parameter names")
                if current_arg_name is 'arguments':
                    token_error(
                        prev(),
                        "Can't use the name arguments as a parameter"
                        ' name, it is reserved by JavaScript'
                    )
                seen_names[current_arg_name] = True
                # save these in order to move back if we have an annotation
                name_token = S.token
                name_ctx = S.input.context()
                # check if we have an argument annotation
                ntok = peek()
                if ntok.type is 'punc' and ntok.value is ':':
                    next()
                    expect(':')
                    annotation = await maybe_conditional()
                    # and now, do as_symbol without the next() at the end
                    # since we are already at the next comma (or end bracket)
                    if not is_token(name_token, 'name'):
                        # assuming the previous context in case
                        # the annotation was over the line
                        if S.input.context().tokline is not def_line:
                            croak('Name expected', name_ctx.tokline)
                        else:
                            croak('Name expected')
                        return None
                    sym = new AST_SymbolFunarg({
                        'name': name_token.value,
                        'start': S.token,
                        'end': S.token,
                        'annotation': annotation
                    })
                    return sym
                else:
                    if not is_('name'):
                        # there is no name, which is an error we should
                        # report on the same line as the definition, so
                        # move to that is we're not already there.
                        if S.input.context().tokline is not def_line:
                            croak('Name expected', def_line)
                        else:
                            croak('Name expected')
                        return None
                    sym = new AST_SymbolFunarg({
                        'name': current_arg_name,
                        'start': S.token,
                        'end': S.token,
                        'annotation': None
                    })
                    next()
                    return sym
            while not is_('punc', ')'):
                if first:
                    first = False
                else:
                    expect(',')
                    if is_('punc', ')'):
                        break
                if is_('operator', '**'):
                    # **kwargs
                    next()
                    if kwargs_sym:
                        token_error(
                            name_token,
                            "Can't define multiple **kwargs"
                            ' in function definition'
                        )
                    kwargs_sym = await get_arg()
                elif is_('operator', '*'):
                    # *args
                    next()
                    if starargs_sym:
                        token_error(
                            name_token,
                            "Can't define multiple *args in function definition"
                        )
                    if kwargs_sym:
                        token_error(
                            name_token,
                            "Can't define *args after **kwargs"
                            ' in function definition'
                        )
                    starargs_sym = await get_arg()
                else:
                    if starargs_sym or kwargs_sym:
                        token_error(
                            name_token,
                            "Can't define a formal parameter after"
                            ' *args or **kwargs'
                        )
                    a.push(await get_arg())
                    if is_('operator', '='):
                        if kwargs_sym:
                            token_error(
                                name_token,
                                "Can't define an optional formal parameter"
                                ' after **kwargs'
                            )
                        next()
                        defaults[current_arg_name] = await expression(False)
                        has_defaults = True
                    else:
                        if has_defaults:
                            token_error(
                                name_token,
                                "Can't define required formal parameters"
                                ' after optional formal parameters'
                            )
            next()
            # check if we have a return type annotation
            if is_('punc', '->'):
                next()
                nonlocal return_annotation
                return_annotation = await maybe_conditional()
            S.in_parenthesized_expr = False
            return new AST_ArgsDef({
                'args': a,
                'starargs': starargs_sym,
                'kwargs': kwargs_sym,
                'defaults': defaults,
                'has_defaults': has_defaults,
                'is_simple_func': (
                    not starargs_sym and not kwargs_sym and not has_defaults
                ),
            })
        )()

        fn_decorators = (def ():
            d = v'[]'
            for decorator in S.decorators:
                d.push(new AST_Decorator({
                    'expression': decorator
                }))
            S.decorators = v'[]'
            return d
        )()

        fn_body = await (async def (loop, labels):
            S.in_class.push(False)
            S.classes.push({})
            S.scoped_flags.push()
            S.in_function += 1
            S.functions.push({'is_async': is_async})
            S.in_loop = 0
            S.labels = []
            a = await block_(docstrings)
            S.in_function -= 1
            S.scoped_flags.pop()
            is_generator.push(bool(S.functions.pop().is_generator))
            S.classes.pop()
            S.in_class.pop()
            S.in_loop = loop
            S.labels = labels
            return a
        )(S.in_loop, S.labels)

        definition = new ctor({
            'name': name,
            'is_expression': is_expression,
            'is_anonymous': is_anonymous,
            'argnames': fn_argnames,
            'localvars': [],
            'decorators': fn_decorators,
            'docstrings': docstrings,
            'body': fn_body,
        })
        definition.return_annotation = return_annotation
        definition.is_generator = is_generator[0]
        definition.is_async = is_async
        if is_node_type(definition, AST_Method):
            definition.static = staticmethod
            definition.is_getter = property_getter
            definition.is_setter = property_setter
            if definition.argnames.args.length < 1 and not definition.static:
                croak(
                    'Methods of a class must have at least one argument,'
                    ' traditionally named self'
                )
            if definition.name and definition.name.name is '__init__':
                if definition.is_generator:
                    croak(
                        'The __init__ method of a class cannot be a'
                        ' generator (yield not allowed)'
                    )
                if definition.is_async:
                    croak('The __init__ method of a class cannot be async')
                if property_getter or property_setter:
                    croak(
                        'The __init__ method of a class cannot be a'
                        ' property getter/setter'
                    )
        if definition.is_async and definition.is_generator:
            croak('A function cannot be both async and a generator')
        if definition.is_generator:
            baselib_items['yield'] = True

        # detect local variables, strip function arguments
        assignments = scan_for_local_vars(definition.body)
        for i in range(assignments.length):
            for j in range(definition.argnames.args.length + 1):
                if j is definition.argnames.args.length:
                    definition.localvars.push(
                        new_symbol(AST_SymbolVar, assignments[i])
                    )
                elif (j < definition.argnames.args.length
                        and assignments[i] is definition.argnames.args[j].name):
                    break

        nonlocals = scan_for_nonlocal_defs(definition.body)
        nonlocals = {name for name in nonlocals}
        definition.localvars = definition.localvars.filter(
            def (v):
                return not nonlocals.has(v.name)
        )
        return definition

    async def if_():
        cond = await expression(True)
        body = await statement()
        belse = None
        if is_('keyword', 'elif') or is_('keyword', 'else'):
            if is_('keyword', 'else'):
                next()
            else:
                S.token.value = 'if'
            # effectively converts 'elif' to 'else if'
            belse = await statement()

        return new AST_If({
            'condition': cond,
            'body': body,
            'alternative': belse
        })

    def is_docstring(stmt):
        if is_node_type(stmt, AST_SimpleStatement):
            if is_node_type(stmt.body, AST_String):
                return stmt.body
        return False

    async def block_(docstrings):
        prev_whitespace = S.token.leading_whitespace
        expect(':')
        a = v'[]'
        if not S.token.nlb:
            while not S.token.nlb:
                if is_('eof'):
                    if options.recover_errors:
                        break
                    unexpected()
                if options.recover_errors:
                    try:
                        stmt = await statement()
                    except as berr:
                        if is_recoverable_error(berr):
                            record_recovery(berr)
                            await skip_to_recovery_point()
                            continue
                        raise
                else:
                    stmt = await statement()
                if docstrings:
                    ds = is_docstring(stmt)
                    if ds:
                        docstrings.push(ds)
                        continue
                a.push(stmt)
        else:
            current_whitespace = S.token.leading_whitespace
            if (current_whitespace.length is 0
                    or prev_whitespace is current_whitespace):
                croak('Expected an indented block')
            while not is_('punc', '}'):
                if is_('eof'):
                    # end of file, terminate block automatically
                    return a
                if options.recover_errors:
                    try:
                        stmt = await statement()
                    except as berr:
                        if is_recoverable_error(berr):
                            record_recovery(berr)
                            await skip_to_recovery_point()
                            continue
                        raise
                else:
                    stmt = await statement()
                if docstrings:
                    ds = is_docstring(stmt)
                    if ds:
                        docstrings.push(ds)
                        continue
                a.push(stmt)
            next()
        return a

    async def try_():
        body = await block_()
        bcatch = v'[]'
        bfinally = None
        belse = None
        while is_('keyword', 'except'):
            start = S.token
            next()
            exceptions = []
            if not is_('punc', ':') and not is_('keyword', 'as'):
                exceptions.push(as_symbol(AST_SymbolVar))
                while is_('punc', ','):
                    next()
                    exceptions.push(as_symbol(AST_SymbolVar))

            name = None
            if is_('keyword', 'as'):
                next()
                name = as_symbol(AST_SymbolCatch)

            except_body = await block_()
            bcatch.push(new AST_Except({
                'start': start,
                'argname': name,
                'errors': exceptions,
                'body': except_body,
                'end': prev()
            }))

        if is_('keyword', 'else'):
            start = S.token
            next()
            belse_body = await block_()
            belse = new AST_Else({
                'start': start,
                'body': belse_body,
                'end': prev()
            })

        if is_('keyword', 'finally'):
            start = S.token
            next()
            bfinally_body = await block_()
            bfinally = new AST_Finally({
                'start': start,
                'body': bfinally_body,
                'end': prev()
            })

        if not bcatch.length and not bfinally:
            croak('Missing except/finally blocks')

        return new AST_Try({
            'body': body,
            'bcatch': (
                new AST_Catch({'body': bcatch}) if bcatch.length else None
            ),
            'bfinally': bfinally,
            'belse': belse
        })

    async def vardefs(symbol_class):
        a = []
        while True:
            vd_start = S.token
            vd_name = as_symbol(symbol_class)
            if is_('operator', '='):
                next()
                varval = await expression(False)
            else:
                varval = None
            a.push(new AST_VarDef({
                'start': vd_start,
                'name': vd_name,
                'value': varval,
                'end': prev()
            }))
            if not is_('punc', ','):
                break
            next()

        return a

    async def nonlocal_(is_global):
        defs = await vardefs(AST_SymbolNonlocal)
        if is_global:
            for vardef in defs:
                S.globals.push(vardef.name.name)
        return new AST_Var({
            'start': prev(),
            'definitions': defs,
            'end': prev()
        })

    async def new_():
        start = S.token
        expect_token('operator', 'new')
        newexp = await expr_atom(False)

        if is_('punc', '('):
            S.in_parenthesized_expr = True
            next()
            args = await func_call_list()
            S.in_parenthesized_expr = False
        else:
            args = await func_call_list(True)
        return await subscripts(new AST_New({
            'start': start,
            'expression': newexp,
            'args': args,
            'end': prev()
        }), True)

    def string_():
        strings = v'[]'
        start = S.token
        while True:
            strings.push(S.token.value)
            if peek().type is not 'string':
                break
            next()
        return new AST_String({
            'start': start,
            'end': S.token,
            'value': strings.join('')
        })

    def token_as_atom_node():
        tok = S.token
        tmp_ = tok.type
        if tmp_ is 'name':
            return token_as_symbol(tok, AST_SymbolRef)
        elif tmp_ is 'num':
            return new AST_Number({
                'start': tok,
                'end': tok,
                'value': tok.value
            })
        elif tmp_ is 'string':
            return string_()
        elif tmp_ is 'regexp':
            return new AST_RegExp({
                'start': tok,
                'end': tok,
                'value': tok.value
            })
        elif tmp_ is 'atom':
            tmp__ = tok.value
            if tmp__ is 'False':
                return new AST_False({
                    'start': tok,
                    'end': tok
                })
            elif tmp__ is 'True':
                return new AST_True({
                    'start': tok,
                    'end': tok
                })
            elif tmp__ is 'None':
                return new AST_Null({
                    'start': tok,
                    'end': tok
                })
        elif tmp_ is 'js':
            return new AST_Verbatim({
                'start': tok,
                'end': tok,
                'value': tok.value,
            })
        token_error(
            tok,
            'Expecting an atomic token (number/string/bool/regexp/js/None)'
        )

    def as_atom_node():
        ret = token_as_atom_node()
        next()
        return ret

    async def expr_atom(allow_calls):
        if is_('operator', 'new'):
            return await new_()

        start = S.token
        if is_('punc'):
            tmp_ = start.value
            if tmp_ is '(':
                prev_parens = S.in_parenthesized_expr
                S.in_parenthesized_expr = True
                next()
                if is_('punc', ')'):
                    next()
                    S.in_parenthesized_expr = prev_parens
                    return new AST_Array({'elements': []})
                ex = await expression(True)
                if is_('keyword', 'for'):
                    ret = await read_comprehension(
                        new AST_GeneratorComprehension({'statement': ex}),
                        ')'
                    )
                    S.in_parenthesized_expr = prev_parens
                    return ret
                ex.start = start
                ex.end = S.token
                if is_node_type(ex, AST_SymbolRef):
                    ex.parens = True
                if not is_node_type(ex, AST_GeneratorComprehension):
                    expect(')')
                if is_node_type(ex, AST_UnaryPrefix):
                    ex.parenthesized = True
                S.in_parenthesized_expr = prev_parens
                return await subscripts(ex, allow_calls)
            elif tmp_ is '[':
                return await subscripts(await array_(), allow_calls)
            elif tmp_ is '{':
                return await subscripts(await object_(), allow_calls)

            unexpected()

        if is_('keyword', 'class'):
            next()
            cls = await class_()
            cls.start = start
            cls.end = prev()
            return await subscripts(cls, allow_calls)

        if is_('keyword', 'async'):
            next()
            if not is_('keyword', 'def'):
                croak("Expected 'def' after 'async'")
            next()
            func = await function_(False, True, True)
            func.start = start
            func.end = prev()
            return await subscripts(func, allow_calls)

        if is_('keyword', 'def'):
            next()
            func = await function_(False, True)
            func.start = start
            func.end = prev()
            return await subscripts(func, allow_calls)

        if is_('keyword', 'await'):
            next()
            if S.in_function is 0 or not S.functions[-1].is_async:
                croak("'await' outside of async function")
            value = await expr_atom(True)
            return new AST_Await({'value': value})

        if is_('keyword', 'yield'):
            next()
            return await yield_()

        if ATOMIC_START_TOKEN[S.token.type]:
            return await subscripts(as_atom_node(), allow_calls)

        unexpected()

    async def expr_list(closing, allow_trailing_comma, allow_empty, func_call):
        first = True
        a = []
        saw_starargs = False
        while not is_('punc', closing):
            if saw_starargs:
                token_error(
                    prev(),
                    '*args must be the last argument in a function call'
                )

            if first:
                first = False
            else:
                expect(',')
            if allow_trailing_comma and is_('punc', closing):
                break

            if is_('operator', '*') and func_call:
                saw_starargs = True
                next()

            if is_('punc', ',') and allow_empty:
                a.push(new AST_Hole({
                    'start': S.token,
                    'end': S.token
                }))
            else:
                a.push(await expression(False))

        next()
        if func_call:
            pargs = []
            kwargs = []
            for arg in a:
                if is_node_type(arg, AST_Assign):
                    kwargs.push([arg.left, arg.right])
                else:
                    pargs.push(arg)
            return new AST_CallArgs({
                'args': pargs, 'kwargs': kwargs,
                'kwarg_items': v'[]', 'starargs': saw_starargs
            })
        return a

    async def func_call_list(empty):
        a = v'[]'
        first = True
        kwargs = v'[]'
        kwarg_items = v'[]'
        starargs = False
        if empty:
            return new AST_CallArgs({
                'args': a, 'kwargs': kwargs,
                'kwarg_items': kwarg_items, 'starargs': starargs
            })
        single_comprehension = False
        while not is_('punc', ')') and not is_('eof'):
            if not first:
                if S.token.nlb and not is_('punc', ','):
                    pass
                else:
                    expect(',')
                    if is_('punc', ')'):
                        break
            if is_('operator', '*'):
                next()
                arg = await expression(False)
                a.push(new AST_CallArg({'value': arg, 'is_array': True}))
                starargs = True
            elif is_('operator', '**'):
                next()
                kwarg_items.push(as_symbol(AST_SymbolRef, False))
                starargs = True
            else:
                arg = await expression(False)
                if is_node_type(arg, AST_Assign):
                    kwargs.push([arg.left, arg.right])
                else:
                    if is_('keyword', 'for'):
                        if not first:
                            croak(
                                'Generator expression must be'
                                ' parenthesized if not sole argument'
                            )
                        comp = await read_comprehension(
                            new AST_GeneratorComprehension({'statement': arg}),
                            ')'
                        )
                        a.push(new AST_CallArg({
                            'value': comp, 'is_array': False
                        }))
                        single_comprehension = True
                        break
                    a.push(new AST_CallArg({'value': arg, 'is_array': False}))
            first = False
        if not single_comprehension:
            next()
        return new AST_CallArgs({
            'args': a, 'kwargs': kwargs,
            'kwarg_items': kwarg_items, 'starargs': starargs
        })

    @embed_tokens
    async def array_():
        expect('[')
        expr = []
        if not is_('punc', ']'):
            expr.push(await expression(False))
            if is_('keyword', 'for'):
                # list comprehension
                return await read_comprehension(
                    new AST_ListComprehension({'statement': expr[0]}),
                    ']'
                )

            if not is_('punc', ']'):
                expect(',')

        arr_rest = await expr_list(']', True, True)
        return new AST_Array({
            'elements': expr.concat(arr_rest)
        })

    @embed_tokens
    async def object_():
        expect('{')
        first = True
        has_non_const_keys = False
        is_pydict = S.scoped_flags.get('dict_literals', False)
        is_jshash = S.scoped_flags.get('hash_literals', False)
        a = []
        while not is_('punc', '}'):
            if not first:
                expect(',')
            if is_('punc', '}'):
                # allow trailing comma
                break
            first = False

            start = S.token
            ctx = S.input.context()
            orig = ctx.expecting_object_literal_key
            ctx.expecting_object_literal_key = True
            try:
                left = await expression(False)
            finally:
                ctx.expecting_object_literal_key = orig
            if is_('keyword', 'for'):
                # is_pydict is irrelevant here
                return await read_comprehension(
                    new AST_SetComprehension({'statement': left}),
                    '}'
                )
            if a.length is 0 and (is_('punc', ',') or is_('punc', '}')):
                end = prev()
                return await set_(start, end, left)
            if not is_node_type(left, AST_Constant):
                has_non_const_keys = True
            expect(':')
            obj_val = await expression(False)
            a.push(new AST_ObjectKeyVal({
                'start': start,
                'key': left,
                'value': obj_val,
                'end': prev()
            }))
            if a.length is 1 and is_('keyword', 'for'):
                return await dict_comprehension(a, is_pydict, is_jshash)

        next()
        return new (
            AST_ExpressiveObject if has_non_const_keys else AST_Object
        )({'properties': a, 'is_pydict': is_pydict, 'is_jshash': is_jshash,})

    async def set_(start, end, expr):
        ostart = start
        a = [new AST_SetItem({'start': start, 'end': end, 'value': expr})]
        while not is_('punc', '}'):
            expect(',')
            start = S.token
            if is_('punc', '}'):
                # allow trailing comma
                break
            set_val = await expression(False)
            a.push(new AST_SetItem({
                'start': start, 'value': set_val, 'end': prev()
            }))
        next()
        return new AST_Set({'items': a, 'start': ostart, 'end': prev()})

    async def read_comprehension(obj, terminator):
        if is_node_type(obj, AST_GeneratorComprehension):
            baselib_items['yield'] = True
        S.in_comprehension = True
        # in case we are already in a parenthesized expression
        S.in_parenthesized_expr = False
        expect_token('keyword', 'for')
        forloop = await for_(True)
        obj.init = forloop.init
        obj.name = forloop.name
        obj.object = forloop.object
        if is_('punc', terminator):
            obj.condition = None
        else:
            expect_token('keyword', 'if')
            obj.condition = await expression(True)
        expect(terminator)
        S.in_comprehension = False
        return obj

    async def dict_comprehension(a, is_pydict, is_jshash):
        if a.length:
            left, right = a[0].key, a[0].value
        else:
            left = await expression(False)
            if not is_('punc', ':'):
                return await read_comprehension(
                    new AST_SetComprehension({'statement': left}),
                    '}'
                )
            expect(':')
            right = await expression(False)
        return await read_comprehension(
            new AST_DictComprehension({
                'statement': left, 'value_statement': right,
                'is_pydict': is_pydict, 'is_jshash': is_jshash
            }),
            '}'
        )

    def as_name():
        tmp = S.token
        next()
        tmp_ = tmp.type
        if (tmp_ is 'name' or tmp_ is 'operator'
                or tmp_ is 'keyword' or tmp_ is 'atom'):
            return tmp.value
        else:
            unexpected()

    def token_as_symbol(tok, ttype):
        name = tok.value
        if RESERVED_WORDS[name] and name is not 'this':
            croak(name + ' is a reserved word')
        return new (AST_This if name is 'this' else ttype)({
            'name': v"String(tok.value)",
            'start': tok,
            'end': tok
        })

    def as_symbol(ttype, noerror):
        if not is_('name'):
            if not noerror:
                croak('Name expected')
            return None

        sym = token_as_symbol(S.token, ttype)
        next()
        return sym

    # for generating/inserting a new symbol
    def new_symbol(type, name):
        sym = new (AST_This if name is 'this' else type)({
            'name': v"String(name)",
            'start': None,
            'end': None
        })
        return sym

    def is_static_method(cls, method):
        if has_prop(
            COMMON_STATIC,
            method
        ) or (cls.static and has_prop(cls.static, method)):
            return True
        else:
            return False

    async def getitem(expr, allow_calls):
        start = expr.start
        next()
        is_py_sub = S.scoped_flags.get('overload_getitem', False)
        slice_bounds = v'[]'
        is_slice = False
        if is_('punc', ':'):
            # slice [:n]
            slice_bounds.push(None)
        else:
            slice_bounds.push(await expression(False))

        if is_('punc', ':'):
            # slice [n:m?]
            is_slice = True
            next()
            if is_('punc', ':'):
                slice_bounds.push(None)
            elif not is_('punc', ']'):
                slice_bounds.push(await expression(False))

        if is_('punc', ':'):
            # slice [n:m:o?]
            next()
            if is_('punc', ']'):
                unexpected()
            else:
                slice_bounds.push(await expression(False))

        expect(']')

        if is_slice:
            if is_('operator', '='):
                # splice-assignment (arr[start:end] = ...)
                next()  # swallow the assignment
                splice_assign = await expression(True)
                return await subscripts(new AST_Splice({
                    'start': start,
                    'expression': expr,
                    'property': slice_bounds[0] or new AST_Number({
                        'value': 0
                    }),
                    'property2': slice_bounds[1],
                    'assignment': splice_assign,
                    'end': prev()
                }), allow_calls)
            elif slice_bounds.length is 3:
                # extended slice (arr[start:end:step])
                slice_bounds.unshift(slice_bounds.pop())
                if not slice_bounds[-1]:
                    slice_bounds.pop()
                    if not slice_bounds[-1]:
                        slice_bounds.pop()
                elif not slice_bounds[-2]:
                    slice_bounds[-2] = new AST_Undefined()
                _ca = v'[]'
                _ca.push(new AST_CallArg({'value': expr, 'is_array': False}))
                for _sb in slice_bounds:
                    _ca.push(new AST_CallArg({'value': _sb, 'is_array': False}))
                return await subscripts(
                    new AST_Call({
                        'start': start,
                        'expression': new AST_SymbolRef({
                            'name': (
                                'ρσ_delslice'
                                if S.in_delete else 'ρσ_eslice'
                            )
                        }),
                        'args': new AST_CallArgs({
                            'args': _ca, 'kwargs': v'[]',
                            'kwarg_items': v'[]', 'starargs': False
                        }),
                        'end': prev()
                    }),
                    allow_calls
                )
            else:
                # regular slice (arr[start:end])
                slice_bounds = [
                    new AST_Number({'value': 0}) if i is None else i
                    for i in slice_bounds
                ]
                if S.in_delete:
                    _ca2 = v'[]'
                    _ca2.push(new AST_CallArg({
                        'value': expr, 'is_array': False
                    }))
                    _ca2.push(new AST_CallArg({
                        'value': new AST_Number({'value': 1}),
                        'is_array': False
                    }))
                    for _sb in slice_bounds:
                        _ca2.push(new AST_CallArg({
                            'value': _sb, 'is_array': False
                        }))
                    return await subscripts(
                        new AST_Call({
                            'start': start,
                            'expression': new AST_SymbolRef({
                                'name': 'ρσ_delslice'
                            }),
                            'args': new AST_CallArgs({
                                'args': _ca2, 'kwargs': v'[]',
                                'kwarg_items': v'[]', 'starargs': False
                            }),
                            'end': prev()
                        }),
                        allow_calls
                    )

                _ca3 = v'[]'
                for _sb in slice_bounds:
                    _ca3.push(new AST_CallArg({
                        'value': _sb, 'is_array': False
                    }))
                return await subscripts(
                    new AST_Call({
                        'start': start,
                        'expression': new AST_Dot({
                            'start': start,
                            'expression': expr,
                            'property': 'slice',
                            'end': prev()
                        }),
                        'args': new AST_CallArgs({
                            'args': _ca3, 'kwargs': v'[]',
                            'kwarg_items': v'[]', 'starargs': False
                        }),
                        'end': prev()
                    }),
                    allow_calls
                )
        else:
            # regular index (arr[index])
            if is_py_sub:
                assignment = None
                assign_operator = ''
                if is_('operator') and ASSIGNMENT[S.token.value]:
                    assign_operator = S.token.value[:-1]
                    next()
                    assignment = await expression(True)
                return await subscripts(new AST_ItemAccess({
                    'start': start,
                    'expression': expr,
                    'property': slice_bounds[0] or new AST_Number({
                        'value': 0
                    }),
                    'assignment': assignment,
                    'assign_operator': assign_operator,
                    'end': prev()
                }), allow_calls)

            return await subscripts(new AST_Sub({
                'start': start,
                'expression': expr,
                'property': slice_bounds[0] or new AST_Number({
                    'value': 0
                }),
                'end': prev()
            }), allow_calls)

    async def call_(expr):
        start = expr.start
        prev_parens = S.in_parenthesized_expr
        S.in_parenthesized_expr = True
        next()
        if not expr.parens and get_class_in_scope(expr):
            # this is an object being created using a class
            call_args = await func_call_list()
            ret = await subscripts(new AST_New({
                'start': start,
                'expression': expr,
                'args': call_args,
                'end': prev()
            }), True)
            S.in_parenthesized_expr = prev_parens
            return ret
        else:
            if is_node_type(expr, AST_Dot):
                c = get_class_in_scope(expr.expression)

            if c:
                # generate class call
                funcname = expr

                classcall_args = await func_call_list()
                ret = await subscripts(new AST_ClassCall({
                    'start': start,
                    'class': expr.expression,
                    'method': funcname.property,
                    'static': is_static_method(c, funcname.property),
                    'args': classcall_args,
                    'end': prev()
                }), True)
                S.in_parenthesized_expr = prev_parens
                return ret
            elif is_node_type(expr, AST_SymbolRef):
                tmp_ = expr.name
                if tmp_ is 'jstype':
                    jstype_args = await func_call_list()
                    ret = new AST_UnaryPrefix({
                        'start': start,
                        'operator': 'typeof',
                        'expression': jstype_args.args[0].value,
                        'end': prev()
                    })
                    S.in_parenthesized_expr = prev_parens
                    return ret
                elif tmp_ is 'isinstance':
                    args = await func_call_list()
                    if args.args.length is not 2:
                        croak(
                            'isinstance() must be called with exactly'
                            ' two arguments'
                        )
                    ret = new AST_Binary({
                        'start': start,
                        'left': args.args[0].value,
                        'operator': 'instanceof',
                        'right': args.args[1].value,
                        'end': prev()
                    })
                    S.in_parenthesized_expr = prev_parens
                    return ret

            # fall-through to basic function call
            basic_args = await func_call_list()
            ret = await subscripts(new AST_Call({
                'start': start,
                'expression': expr,
                'args': basic_args,
                'end': prev()
            }), True)
            S.in_parenthesized_expr = prev_parens
            return ret

    async def get_attr(expr, allow_calls):
        next()
        prop = as_name()
        c = get_class_in_scope(expr)
        if c:
            classvars = c.provisional_classvars if c.processing else c.classvars
            if classvars and v'classvars[prop]':
                prop = 'prototype.' + prop

        return await subscripts(new AST_Dot({
            'start': expr.start,
            'expression': expr,
            'property': prop,
            'end': prev()
        }), allow_calls)

    async def existential(expr, allow_calls):
        ans = new AST_Existential({
            'start': expr.start, 'end': S.token, 'expression': expr
        })
        next()
        ttype = S.token.type
        val = S.token.value
        if (S.token.nlb or ttype is 'keyword'
                or ttype is 'operator' or ttype is 'eof'):
            ans.after = None
            return ans
        if ttype is 'punc':
            if val is '.':
                ans.after = '.'
            elif val is '[':
                is_py_sub = S.scoped_flags.get('overload_getitem', False)
                ans.after = 'g' if is_py_sub else '['
            elif val is '(':
                if not allow_calls:
                    unexpected()
                ans.after = '('
            else:
                ans.after = None
                return ans
            return await subscripts(ans, allow_calls)

        ans.after = await expression()
        return ans

    async def subscripts(expr, allow_calls):
        if is_('punc', '.'):
            return await get_attr(expr, allow_calls)

        if is_('punc', '[') and not S.token.nlb:
            return await getitem(expr, allow_calls)

        if allow_calls and is_('punc', '(') and not S.token.nlb:
            return await call_(expr)

        if is_('punc', '?'):
            return await existential(expr, allow_calls)

        return expr

    async def maybe_unary(allow_calls):
        start = S.token
        if is_('operator', '@'):
            if S.parsing_decorator:
                croak('Nested decorators are not allowed')
            next()
            S.parsing_decorator = True
            expr = await expression()
            S.parsing_decorator = False
            S.decorators.push(expr)
            return new AST_EmptyStatement({
                'stype': '@', 'start': prev(), 'end': prev()
            })
        if is_('operator') and UNARY_PREFIX[start.value]:
            next()
            is_parenthesized = is_('punc', '(')
            S.in_delete = start.value is 'delete'
            expr = await maybe_unary(allow_calls)
            S.in_delete = False
            ex = make_unary(
                AST_UnaryPrefix,
                start.value,
                expr,
                is_parenthesized
            )
            ex.start = start
            ex.end = prev()
            return ex

        val = await expr_atom(allow_calls)
        return val

    def make_unary(ctor, op, expr, is_parenthesized):
        return new ctor({
            'operator': op,
            'expression': expr,
            'parenthesized': is_parenthesized
        })

    async def expr_op(left, min_prec, no_in):
        op = S.token.value if is_('operator') else None
        if op is '!' and peek().type is 'operator' and peek().value is 'in':
            next()
            S.token.value = op = 'nin'

        if no_in and (op is 'in' or op is 'nin'):
            op = None

        prec = PRECEDENCE[op] if op is not None else None
        if prec is not None and prec > min_prec:
            next()
            right = await expr_op(await maybe_unary(True), prec, no_in)
            ret = new AST_Binary({
                'start': left.start,
                'left': left,
                'operator': op,
                'right': right,
                'end': right.end
            })
            return await expr_op(ret, min_prec, no_in)
        return left

    async def expr_ops(no_in):
        return await expr_op(await maybe_unary(True), 0, no_in)

    async def maybe_conditional(no_in):
        start = S.token
        expr = await expr_ops(no_in)
        if (is_('keyword', 'if') and (
                S.in_parenthesized_expr or (
                    S.statement_starting_token is not S.token
                    and not S.in_comprehension
                    and not S.token.nlb
                ))):
            next()
            ne = await expression(False)
            expect_token('keyword', 'else')
            cond_alt = await expression(False, no_in)
            conditional = new AST_Conditional({
                'start': start,
                'condition': ne,
                'consequent': expr,
                'alternative': cond_alt,
                'end': peek()
            })
            return conditional
        return expr

    def create_assign(data):
        if data.right and is_node_type(
            data.right,
            AST_Seq
        ) and (
            is_node_type(data.right.car, AST_Assign)
            or is_node_type(data.right.cdr, AST_Assign)
        ) and data.operator is not '=':
            token_error(
                data.start,
                'Invalid assignment operator for chained'
                ' assignment: ' + data.operator
            )
        ans = AST_Assign(data)
        if S.in_class.length and S.in_class[-1]:
            class_name = S.in_class[-1]
            if is_node_type(ans.left, AST_SymbolRef) and S.classes.length > 1:
                c = S.classes[-2][class_name]
                if c:
                    if ans.is_chained():
                        for lhs in ans.traverse_chain()[0]:
                            c.provisional_classvars[lhs.name] = True
                    else:
                        c.provisional_classvars[ans.left.name] = True
        return ans

    async def maybe_assign(no_in, only_plain_assignment):
        start = S.token
        left = await maybe_conditional(no_in)
        val = S.token.value
        if is_('operator') and ASSIGNMENT[val]:
            if only_plain_assignment and val is not '=':
                croak(
                    'Invalid assignment operator for chained assignment: '
                    + val
                )
            next()
            assign_right = await maybe_assign(no_in, True)
            return create_assign({
                'start': start,
                'left': left,
                'operator': val,
                'right': assign_right,
                'end': prev()
            })
        return left

    async def expression(commas, no_in):
        # if there is an assignment, we want the sequences to pivot
        # around it to allow for tuple packing/unpacking
        start = S.token
        expr = await maybe_assign(no_in)

        def build_seq(a):
            if a.length is 1:
                return a[0]

            return new AST_Seq({
                'start': start,
                'car': a.shift(),
                'cdr': build_seq(a),
                'end': peek()
            })
        if commas:
            left = v'[ expr ]'
            while is_(
                'punc', ','
            ) and (not peek().nlb or S.in_parenthesized_expr) or (
                S.in_parenthesized_expr and S.token.nlb
                and not is_('punc', ')')
            ):
                if is_('punc', ','):
                    next()
                if is_node_type(expr, AST_Assign):
                    left[-1] = left[-1].left
                    seq_cdr = await expression(True, no_in)
                    return create_assign({
                        'start': start,
                        'left': (
                            left[0] if left.length is 1
                            else new AST_Array({'elements': left})
                        ),
                        'operator': expr.operator,
                        'right': new AST_Seq({
                            'car': expr.right, 'cdr': seq_cdr
                        }),
                        'end': peek()
                    })

                expr = await maybe_assign(no_in)
                left.push(expr)

            # if last one was an assignment, fix it
            if left.length > 1 and is_node_type(left[-1], AST_Assign):
                left[-1] = left[-1].left
                return create_assign({
                    'start': start,
                    'left': new AST_Array({
                        'elements': left
                    }),
                    'operator': expr.operator,
                    'right': expr.right,
                    'end': peek()
                })

            return build_seq(left)
        return expr

    async def in_loop(cont):
        S.in_loop += 1
        ret = await cont()
        S.in_loop -= 1
        return ret

    async def run_parser():
        start = S.token = next()
        body = v'[]'
        docstrings = v'[]'
        first_token = True
        toplevel = options.toplevel
        while not is_('eof'):
            if options.recover_errors:
                try:
                    element = await statement()
                except as rerr:
                    if is_recoverable_error(rerr):
                        record_recovery(rerr)
                        await skip_to_recovery_point()
                        continue
                    raise
            else:
                element = await statement()
            if first_token and is_node_type(
                element,
                AST_Directive
            ) and element.value.indexOf('#!') is 0:
                shebang = element.value
            else:
                # do not process strings as docstrings if we are
                # concatenating toplevels
                ds = not toplevel and is_docstring(element)
                if ds:
                    docstrings.push(ds)
                else:
                    body.push(element)
            first_token = False

        end = prev()
        if toplevel:
            toplevel.body = toplevel.body.concat(body)
            toplevel.end = end
            toplevel.docstrings
        else:
            toplevel = new AST_Toplevel({
                'start': start,
                'body': body,
                'shebang': shebang,
                'end': end,
                'docstrings': docstrings,
            })

        if options.recover_errors:
            # Combine parser-level recovered errors with any collected by the
            # tokenizer (e.g. skipped unexpected characters) for the
            # LSP to report.
            recovered = S.recovered_errors
            try:
                tok_ctx = S.input.context()
                if (tok_ctx and tok_ctx.recovered_errors
                        and tok_ctx.recovered_errors.length):
                    recovered = recovered.concat(tok_ctx.recovered_errors)
            except:
                pass
            toplevel.recovered_errors = recovered

        toplevel.nonlocalvars = scan_for_nonlocal_defs(
            toplevel.body
        ).concat(S.globals)
        toplevel.localvars = []
        toplevel.exports = []
        seen_exports = {}

        def add_item(item, isvar):
            if (toplevel.nonlocalvars.indexOf(item) < 0):
                symbol = new_symbol(AST_SymbolVar, item)
                if isvar:
                    toplevel.localvars.push(symbol)
                if not has_prop(seen_exports, item):
                    toplevel.exports.push(symbol)
                    seen_exports[item] = True

        for item in scan_for_local_vars(toplevel.body):
            add_item(item, True)
        for item in scan_for_top_level_callables(toplevel.body):
            add_item(item, False)

        toplevel.filename = options.filename
        toplevel.imported_module_ids = imported_module_ids
        toplevel.classes = scan_for_classes(toplevel.body)
        toplevel.import_order = Object.keys(imported_modules).length
        toplevel.module_id = module_id
        imported_modules[module_id] = toplevel
        toplevel.imports = imported_modules
        toplevel.baselib = baselib_items
        toplevel.scoped_flags = S.scoped_flags.stack[0]
        importing_modules[module_id] = False
        toplevel.comments_after = S.token.comments_before or v'[]'

        if (options._src_hash and options.filename
                and module_id is not '__main__'):
            try:
                # Simplified class info (no full AST nodes) for
                # lazy-shell reconstruction.
                classes_cache = {}
                for cname in Object.keys(toplevel.classes or {}):
                    cls = toplevel.classes[cname]
                    classes_cache[cname] = {
                        'static': cls.static or {},
                        'bound': cls.bound or [],
                        'classvars': cls.classvars or {},
                    }
                # Single pass: compute has_side_effects + tree-shaking index.
                # The index lets the tree-shaker resolve the fixpoint iteration
                # (steps 4-5) without accessing mod.body — no AST
                # deserialization
                # until step 6 (pruning), which needs actual nodes anyway.
                has_se = False
                idx_exec_refs_set = {}
                idx_import_bindings = {}
                idx_top_defs = {}

                def _collect_refs_into(node, refs_set):  # noqa
                    def _rv(n):  # noqa
                        if is_node_type(n, AST_SymbolRef):
                            refs_set[n.name] = True
                    node.walk(new TreeWalker(_rv))

                for idx_stmt in toplevel.body:
                    if is_node_type(
                        idx_stmt,
                        AST_Function
                    ) or is_node_type(idx_stmt, AST_Class):
                        if idx_stmt.name:
                            idx_pinned = False
                            if idx_stmt.decorators:
                                for idx_dec in idx_stmt.decorators:
                                    if idx_dec.expression and is_node_type(
                                        idx_dec.expression,
                                        AST_SymbolRef
                                    ) and idx_dec.expression.name == 'no_prune':
                                        idx_pinned = True
                                        break
                            idx_def_refs = {}
                            _collect_refs_into(idx_stmt, idx_def_refs)
                            idx_top_defs[idx_stmt.name.name] = {
                                'pinned': idx_pinned,
                                'refs': Object.keys(idx_def_refs),
                            }
                    elif is_node_type(idx_stmt, AST_Imports):
                        for idx_imp in idx_stmt.imports:
                            if idx_imp.argnames and idx_imp.argnames.length:
                                for idx_arg in idx_imp.argnames:
                                    idx_local = (
                                        idx_arg.alias.name
                                        if idx_arg.alias else idx_arg.name
                                    )
                                    idx_import_bindings[idx_local] = {
                                        'source': idx_imp.key,
                                        'imported_name': idx_arg.name,
                                        'is_namespace': False,
                                    }
                            else:
                                idx_local = (
                                    idx_imp.alias.name
                                    if idx_imp.alias
                                    else idx_imp.key.split('.')[0]
                                )
                                idx_import_bindings[idx_local] = {
                                    'source': idx_imp.key,
                                    'imported_name': None,
                                    'is_namespace': True,
                                }
                    else:
                        if not has_se:
                            se_found = v'[false]'

                            def se_check(se_node, se_descend):  # noqa
                                if is_node_type(se_node, AST_Call):
                                    se_found[0] = True
                                    return True
                            idx_stmt.walk(new TreeWalker(se_check))
                            if se_found[0]:
                                has_se = True
                        _collect_refs_into(idx_stmt, idx_exec_refs_set)

                cache_meta = {
                    'v': CACHE_VERSION,
                    'signature': options._src_hash,
                    'compiler_version': COMPILER_VERSION,
                    'imported_module_ids': toplevel.imported_module_ids,
                    'exported_names': [sym.name for sym in toplevel.exports],
                    'baselib': toplevel.baselib or {},
                    'classes': classes_cache,
                    'has_side_effects': has_se,
                    'index': {
                        'exec_refs': Object.keys(idx_exec_refs_set),
                        'import_bindings': idx_import_bindings,
                        'top_defs': idx_top_defs,
                    },
                }
                if (options._file_mtime is not None
                        and options._file_mtime is not undefined):
                    cache_meta['mtime'] = options._file_mtime
                # Binary format: metadata JSON followed by newline
                # and binary AST pool.
                await writefile(
                    cache_file_name(options.filename, options.module_cache_dir),
                    encode_cache(JSON.stringify(cache_meta), toplevel)
                )
            except as e:
                console.error(
                    'Failed to write AST cache file:',
                    options.filename,
                    'with error:',
                    e
                )

        return toplevel

    return run_parser


async def parse(text, options):
    options = defaults(options, {
        'filename': None,  # name of the file being parsed
        'module_id': '__main__',  # The id of the module being parsed
        'toplevel': None,
        # If True certain actions are not performed, such as importing modules
        'for_linting': False,
        'import_dirs': v'[]',
        # Map of class names to AST_Class that are available in the
        # global namespace (used by the REPL)
        'classes': undefined,
        'scoped_flags': {},  # Global scoped flags (used by the REPL)
        'discard_asserts': False,
        'module_cache_dir': '',
        # When True, continue parsing past syntax errors (used by the
        # LSP). See run_parser/block_.
        'recover_errors': False,
    })
    import_dirs = [x for x in options.import_dirs]
    for location in v'[options.libdir, options.basedir]':
        if location:
            import_dirs.push(location)
    module_id = options.module_id
    baselib_items = {}
    imported_module_ids = []
    imported_modules = options.imported_modules or {}
    importing_modules = options.importing_modules or {}
    # loading_promises tracks modules currently being loaded across the
    # entire compilation.
    # It is shared by passing it through options so parallel sub-parse()
    # calls all see it.
    if not options.loading_promises:
        options.loading_promises = {}
    importing_modules[module_id] = True

    # The internal state of the parser
    S = {
        'input': tokenizer(
            text, options.filename, options.recover_errors
        ) if jstype(text) is 'string' else text,
        # syntax errors swallowed during recovery (recover_errors mode only)
        'recovered_errors': v'[]',
        'token': None,
        'prev': None,
        'peeked': [],
        'in_function': 0,
        'statement_starting_token': None,
        'in_comprehension': False,
        'in_parenthesized_expr': False,
        'in_delete': False,
        'in_loop': 0,
        'in_class': [False],
        'classes': [{}],
        'functions': [{}],
        'labels': [],
        'decorators': v'[]',
        'parsing_decorator': False,
        'globals': v'[]',
        'scoped_flags': {
            'stack': v'[options.scoped_flags || Object.create(null)]',
            'push': def (): this.stack.push(Object.create(None));,
            'pop': def (): this.stack.pop();,
            'get': def (name, defval):
                for v'var i = this.stack.length - 1; i >= 0; i--':
                    d = this.stack[i]
                    q = d[name]
                    if q:
                        return q
                return defval
            ,
            'set': def (name, val): this.stack[-1][name] = val;,
        },
    }

    if options.classes:
        for cname in options.classes:
            obj = options.classes[cname]
            S.classes[0][cname] = {
                'static': obj.static,
                'bound': obj.bound,
                'classvars': obj.classvars,
            }

    if jstype(text) is 'string' and not options._src_hash:
        options._src_hash = sha1sum(text)

    return await create_parser_ctx(
        S,
        import_dirs,
        module_id,
        baselib_items,
        imported_module_ids,
        imported_modules,
        importing_modules,
        options
    )()