""" ********************************************************************** A RapydScript to JavaScript compiler. https://github.com/atsepkov/RapydScript -------------------------------- (C) --------------------------------- Author: Alexander Tsepkov http://www.pyjeon.com Distributed under BSD license: Copyright 2013 (c) Alexander Tsepkov ********************************************************************** """ from utils import makePredicate, defaults, ImportError, js_error, RAPYD_PREFIX, find_if import ast import tokenizer NATIVE_CLASSES = None COMMON_STATIC = None CLASS_MAP = None BASELIB = None STDLIB = None def array_to_hash(a): ret = {} for i in range(len(a)): ret[a[i]] = True return ret # it needs for reinitialization between in-browser compilations def init_mod(): nonlocal NATIVE_CLASSES, COMMON_STATIC, CLASS_MAP, BASELIB, STDLIB NATIVE_CLASSES = { # javascript 'Image': {}, 'RegExp': {}, 'Error': {}, 'Object': { static: [ "assign", # ES6 "getOwnPropertyNames", "keys", "create", "defineProperty", "defineProperties", "getPrototypeOf", # ES6 "setPrototypeOf", # ES6 "getOwnPropertyDescriptor", # ES6 "getOwnPropertyDescriptors", # ES6 # experimental "values", "entries" ] }, 'String': { static: [ "fromCharCode" ] }, 'Array': { static: [ "isArray", "from", "of" ] }, 'Number': { static: [ "isFinite", "isNaN" ] }, 'Function': {}, 'Date': { static: [ "UTC", "now", "parse" ] }, 'Boolean': {}, 'ArrayBuffer': { static: [ "isView", "transfer" ] }, 'DataView': {}, 'Float32Array': {}, 'Float64Array': {}, 'Int16Array': {}, 'Int32Array': {}, 'Int8Array': {}, 'Uint16Array': {}, 'Uint32Array': {}, 'Uint8Array': {}, 'Uint8ClampedArray': {}, 'Map': {}, # ES6 'WeakMap': {}, # ES6 'Set': {}, # ES6 'WeakSet': {}, # ES6 'Promise': { # ES6 static: [ "all", "race", "reject", "resolve" ] }, # baselib "AssertionError": {}, "IndexError": {}, "KeyError": {}, "TypeError": {}, "ValueError": {}, } COMMON_STATIC = [ "call", "apply", "bind", "toString" ] CLASS_MAP = {} # top-level classes will be populated into here # detect common python stdlib methods - these will be auto-injected into the code when called BASELIB = { key: 0 for key in [ "abs", "all", "any", "bin", "bind", "rebind_all", "with__name__", "cmp", "chr", "dir", "enumerate", "eslice", "extends", "filter", "hex", "in", "iterable", "len", "map", "max", "min", "merge", "mixin", "print", "range", "reduce", "reversed", "sorted", "sum", "type", "zip", "getattr", "setattr", "hasattr", "eq", "kwargs", "AssertionError", "IndexError", "KeyError", "TypeError", "ValueError" ]} STDLIB = [ "abs", "bin", "cmp", "chr", "dir", "hex", "max", "min", "merge", "mixin", "print", "range", "reduce", "getattr", "setattr", "hasattr", # unique to RapydScript "eq", "bind", "rebind_all", "type", # list operations "all", "any", "enumerate", "filter", "len", "map", "reversed", "sum", "zip", # errors "AssertionError", "IndexError", "KeyError", "TypeError", "ValueError", ] def has_simple_decorator(decorators, name): remove = [] for JS('var i = 0; i < decorators.length; i++'): s = decorators[i] if isinstance(s, ast.SymbolRef) and not s.parens and s.name is name: remove.push(i) if remove.length: remove.reverse() for JS('var i = 0; i < remove.length; i++'): decorators.splice(remove[i], 1) return True return False # -----[ Parser (constants) ]----- UNARY_PREFIX = makePredicate([ "typeof", "void", "delete", "--", "++", "!", "~", "-", "+", "*", # eventually I'll rename this to ... "@" ]) ASSIGNMENT = makePredicate([ "=", "+=", "-=", "/=", "//=", "*=", "%=", ">>=", "<<=", ">>>=", "|=", "^=", "&=" ]) STATEMENTS_WITH_LABELS = array_to_hash([ "for", "do", "while", "switch" ]) ATOMIC_START_TOKEN = array_to_hash([ "atom", "num", "string", "regexp", "name" ]) # -----[ Parser ]----- def parse($TEXT, options): if not STDLIB: init_mod() # needs to have per module class_map = {} options = defaults(options, { strict: False, # whether to use strict JavaScript mode strict_names: False, # if false RS will fix names (e.g. var -> var_ϟ) that are not RS- but JS-keywords filename: None, # name of the file being parsed auto_bind: False, # whether auto-binding of methods to classes is enabled module_id: '__main__', # The id of the module being parsed es6: False, # whether we're using EcmaScript 6 mode toplevel: None, # existing tree to append to, if None, new tree will be started import_dirs: [], # List of directories to scan for imports dropDecorators: [], # Decorators to omit from compilation dropImports: [], # Imports to omit from compilation dropDocstrings: False, # If true, omit docstrings from compilation classes: None, # Map of class names to ast.Class that are available in the global namespace (used by the REPL) readfile: JS('typeof require != "undefined"') ? require('fs').readFileSync : None # File reader }) module_id = options.module_id # fallthrough order: --> relative basedir, user-specified import dirs, standard libdir import_dirs = options.import_dirs[:] if options.libdir: import_dirs.push(options.libdir) if options.basedir: import_dirs.unshift(options.basedir) depends_on = {} # for each module imports tracking PRE_IMPORTED = options.PRE_IMPORTED or {} IMPORTED = options.IMPORTED or {} IMPORTING = options.IMPORTING or {} IMPORTING[module_id] = True S = { input: (JS('typeof $TEXT') is "string" ? tokenizer.tokenizer($TEXT, options.filename) : $TEXT), token: None, # current token being scanned prev: None, # previous token peeked: None, # cached version of next token in_directives: True, # per-scope directive, i.e. "use strict" in_loop: 0, # number of loops we're currently nested inside of in_scope: [ { # relevant context for given scope type: None, # scope type vars: {}, # variables in this scope nonlocal: {}, # variables shared with outer scope functions: {}, # list of functions declared in this scope and their signatures classes: {} # classes visible from this scope } ], labels: [], decorators: [], # decorators we've most-recently seen, these get reset as soon as we see function body in_seq: False, # whether we're currently parsing a sequence in_decorator: False # whether we're currently parsing a decorator } if options.classes: for cname in options.classes: obj = options.classes[cname] S.in_scope[0].classes[cname] = { 'static': obj.static, 'bound': obj.bound } S.token = next() def maybe_keyword(name): if not tokenizer.IS_ANY_KEYWORD(name): return name # fix only some keywords but not all if not options.strict_names and tokenizer.JS_KEYWORDS_AUTOFIX(name): #console.log(name, options.filename, prev()) return name + '_ϟ' token_error(prev(), "Can't use the name '"+ name +"' as a parameter/variable name, it is reserved by JavaScript") def is_(type, value): return tokenizer.is_token(S.token, type, value) def peek(): return S.peeked or (S.peeked = S.input()) def next(): S.prev = S.token if S.peeked: # next token is already cached, reuse it S.token = S.peeked S.peeked = None else: # process next token S.token = S.input() S.in_directives = S.in_directives and (S.token.type is "string" or is_("punc", ";")) return S.token def prev(): return S.prev def croak(msg, line, col, pos, is_eof): # note: undefined means nothing was passed in, None/null means a null value was passed in ctx = S.input.context() tokenizer.js_error( msg, ctx.filename, line is not undefined ? line : ctx.tokline, col is not undefined ? col : ctx.tokcol, pos is not undefined ? pos : ctx.tokpos, is_eof ) def token_error(token, msg): is_eof = (token.type is 'eof') ? True : False croak(msg, token.line, token.col, undefined, is_eof) def unexpected(token): if token is undefined: token = S.token token_error(token, "Unexpected token: " + token.type + " «" + token.value + "»") def expect_token(type, val): if is_(type, val): return next() token_error(S.token, "Unexpected token " + S.token.type + " «" + S.token.value + "»" + ", expected " + type + " «" + val + "»") def expect(punc): if punc is "," and S.token.comma_expected: if is_("punc", ","): next() return S.token return expect_token("punc", punc) def can_insert_semicolon(): return not options.strict and (S.token.newline_before or is_("eof") or is_("punc", "}")) def semicolon(): if is_("punc", ";"): next() S.token.newline_before = True def parenthesised(): expect("(") exp = expression(True) expect(")") return exp def embed_tokens(parser): return def(): start = S.token expr = parser.apply(this, arguments) if expr is undefined: unexpected() end = prev() expr.start = start expr.end = end return expr def is_nested_comparison(stmt): """ Check if the statement is a nested comparison """ comparators = { "<": True, ">": True, "<=": True, ">=": True, "==": True, "!=": True, "===": True, "!==": True } if isinstance(stmt, ast.Binary) and stmt.operator in comparators \ and isinstance(stmt.left, ast.Binary) and stmt.left.operator in comparators: return True else: return False def scan_for_top_level_callables(body): ans = [] # Get the named functions and classes if Array.isArray(body): for name in dir(body): obj = body[name] if isinstance(obj, ast.Function) or isinstance(obj, ast.Class): if obj.name: ans.push(obj.name) else: token_error(obj.start, "Top-level functions must have names") else: # skip inner scopes if isinstance(obj, ast.Scope): continue for x in ['body', 'alternative']: opt = obj[x] if opt: ans = ans.concat(scan_for_top_level_callables(opt)) if isinstance(opt, ast.Assign) and not (isinstance(opt.right, ast.Scope)): ans = ans.concat(scan_for_top_level_callables(opt.right)) elif body.body: # recursive descent into wrapper statements that contain body blocks ans = ans.concat(scan_for_top_level_callables(body.body)) if body.alternative: ans = ans.concat(scan_for_top_level_callables(body.alternative)) return ans def block_statement(): return statement(True) @embed_tokens def statement(expect_block): if is_("operator", "/") or is_("operator", "/="): S.peeked = None S.token = S.input(S.token.value[1:]) if expect_block: if is_("punc", ":"): return ast.BlockStatement({ start: S.token, body: block_(), end: prev() }) else: unexpected() tmp_ = S.token.type if tmp_ is "string": dir = S.in_directives stat = simple_statement() # XXXv2: decide how to fix directives if dir and isinstance(stat.body, ast.String) and not is_("punc", ","): return ast.Directive({ value: stat.body.value }) return stat elif tmp_ is "shebang": tmp_ = S.token.value next() return ast.Directive({ value: tmp_ }) elif tmp_ is "num" or tmp_ is "regexp" or tmp_ is "operator" or tmp_ is "atom": return simple_statement() elif tmp_ is "punc": tmp_ = S.token.value if tmp_ is "{" or tmp_ is "[" or tmp_ is "(": return simple_statement() elif tmp_ is ";": next() return ast.EmptyStatement() else: unexpected() elif tmp_ is "name": if S.token.value in ['set', 'get']: if not options.es6: croak("Class getters/setters require ES6 compilation mode") type = S.token.value start = S.token.start next() if not (S.in_scope[-1].type is "class"): croak("Getter/setter outside of class") return accessor_(type, start, S.in_scope[-1].name or True) return (tokenizer.is_token(peek(), "punc", ":") ? labeled_statement() : simple_statement()) elif tmp_ is "keyword": tmp_ = S.token.value if tokenizer.ES6_KEYWORDS(tmp_) and not options.es6: token_error(prev() or S.token, "«" + tmp_ + "» keyword not supported with ES5 output, use --ecmascript6 compilation flag") next() if tmp_ is "break": return break_cont(ast.Break) elif tmp_ is "continue": return break_cont(ast.Continue) elif tmp_ is "debugger": semicolon() return ast.Debugger() elif tmp_ is "do": return ast.Do({ body: in_loop(block_statement), condition: def(): expect(".") expect_token("keyword", "while") tmp = expression(True) semicolon() return tmp .call(this) }) elif tmp_ is "while": return ast.While({ condition: expression(True), body: in_loop(block_statement) }) elif tmp_ is "for": if is_("name", "JS") or is_("string:v"): return for_js() return for_() elif tmp_ is "from": return import_(True) elif tmp_ is "import": return import_(False) elif tmp_ is "class": BASELIB["extends"] += 1 if options.auto_bind: BASELIB["rebind_all"] += 1 if not options.es6: BASELIB["with__name__"] += 1 return class_() elif tmp_ is "def": start = prev() func = function_(S.in_scope[-1].type is "class" ? S.in_scope[-1].name : False) func.start = start func.end = prev() chain = subscripts(func, True) if chain is func: return func else: return ast.SimpleStatement({ start: start, body: chain, end: prev() }) elif tmp_ is "if": return if_() elif tmp_ is "pass": semicolon() return ast.EmptyStatement() elif tmp_ is "return" or tmp_ is "yield": if S.in_scope[-1].type is not "function": croak("'return' outside of function") if tmp_ is "yield": result = yield_() else: ctor = ast.Return result = new ctor({ value: (is_("punc", ";") ? (def(): semicolon() return None )() : (can_insert_semicolon() ? None : (def(): tmp = expression(True) semicolon() return tmp )() ) ) }) # check if user violated a contract they agreed to follow if (S.in_scope[-1].return_annotation): expectedType = S.in_scope[-1].return_annotation.resolveType(S.in_scope) actualType = result.resolveType(S.in_scope) if actualType not in [expectedType, '?']: croak("Type annotation states that function returns " + expectedType + ", actual returned type is " + actualType + "") return result elif tmp_ is "switch": return ast.Switch({ expression: parenthesised(), body: in_loop(switch_body_) }) elif tmp_ is "raise": if S.token.newline_before: return ast.Throw({ value: ast.SymbolCatch({ name: "ՐՏ_Exception" }) }) tmp = expression(True) semicolon() return ast.Throw({ value: tmp }) elif tmp_ is "try": return try_() elif tmp_ is "nonlocal": tmp = nonlocal_() semicolon() return tmp elif tmp_ is "const": tmp = const_() semicolon() return tmp elif tmp_ is "with": return ast.With({ expression: parenthesised(), body: block_statement() }) else: unexpected() def labeled_statement(): label = as_symbol(ast.Label) if find_if(def(l): return l.name is label.name , S.labels): # ECMA-262, 12.12: An ECMAScript program is considered # syntactically incorrect if it contains a # LabelledStatement that is enclosed by a # LabelledStatement with the same Identifier as label. croak("Label " + label.name + " defined twice") expect(":") S.labels.push(label) stat = statement() S.labels.pop() return ast.LabeledStatement({ body: stat, label: label }) def simple_statement(tmp): tmp = expression(True) semicolon() return ast.SimpleStatement({ body: tmp }) def break_cont(type_): label = None if not can_insert_semicolon(): label = as_symbol(ast.LabelRef, True) if label is not None: if not find_if(def(l): return l.name is label.name , S.labels): croak("Undefined label " + label.name) elif S.in_loop is 0: croak(type(type_) + " not inside a loop or switch") semicolon() return new type_({ label: label }) def seq_to_array(seq): # convert ast.Seq into ast.Array return ast.Array({ start: seq.start, elements: seq.to_array(), end: seq.end }) def for_(list_comp): # expect("("); init = None if not is_("punc", ";"): init = expression(True, True) # standardize ast.Seq into array now for consistency if isinstance(init, ast.Seq): init = seq_to_array(init) if is_("operator", "in"): if isinstance(init, ast.Var) and init.definitions.length > 1: croak("Only one variable declaration allowed in for..in loop") next() return for_in(init, list_comp) unexpected() def for_in(init, list_comp): lhs = isinstance(init, ast.Var) ? init.definitions[0].name : None obj = expression(True) # mark iterator variable as local if isinstance(init, ast.Array): for i, element in enumerate(init.elements): value = None if isinstance(obj, ast.Call) and isinstance(obj.expression, ast.SymbolRef) and obj.expression.name is 'enumerate': # FIXME: instead of doing this hack, annotate all of baselib and have compiler rely on it # a naive assumption that user won't overwrite inbuilt functions if i == 0: value = "Number" mark_local_assignment(element, value) else: # infer object type for the iteration sequence value = None if isinstance(obj, ast.Call) and isinstance(obj.expression, ast.SymbolRef) and obj.expression.name is 'range': # FIXME: instead of doing this hack, annotate all of baselib and have compiler rely on it # a naive assumption that user won't overwrite inbuilt functions value = "Number" mark_local_assignment(init, value) BASELIB["iterable"] += 1 if list_comp: return { init: init, name: lhs, object: obj } return ast.ForIn({ init: init, name: lhs, object: obj, body: in_loop(block_statement) }) # A native JavaScript for loop - for JS("var i=0; i<5000; i++"): def for_js(): condition = expression(True, True) return ast.ForJS({ condition: condition, body: in_loop(block_statement) }) # scan function/class body for nested class declarations def get_class_in_scope(expr): # TODO: Currently if a local variable shadows a class name defined in # an outerscope, the logic below will identify that variable as a # class. This bug was always present. Fixing it will require the parser # to maintain a list of local variables for every ast.Scope and provide # an easy way to walk the ast tree upwards. if isinstance(expr, ast.SymbolRef): # check Native JS classes if expr.name in NATIVE_CLASSES: return NATIVE_CLASSES[expr.name] # traverse in reverse to check local variables first for s in range(S.in_scope.length-1, -1, -1): if expr.name in S.in_scope[s].classes: return S.in_scope[s].classes[expr.name] elif isinstance(expr, ast.Dot): referenced_path = [] # this one is for detecting classes inside modules and eventually nested classes while isinstance(expr, ast.Dot): referenced_path.unshift(expr.property) expr = expr.expression if isinstance(expr, ast.SymbolRef): referenced_path.unshift(expr.name) # now 'referenced_path' should contain the full path of potential class if len(referenced_path) > 1: class_name = referenced_path.join('.') for s in range(S.in_scope.length-1, -1, -1): if class_name in S.in_scope[s].classes: return S.in_scope[s].classes[class_name] return False def do_import(key): if key in IMPORTED: return # we'll reuse the original imported code if IMPORTING[key]: # cyclical import raise ImportError('Detected a recursive import of: ' + key + ' while importing: ' + module_id, options.filename) # Ensure that the package containing this module is also imported package_module_id = key.split('.')[:-1].join('.') if len(package_module_id) > 0 and not IMPORTING[package_module_id]: do_import(package_module_id) if key in IMPORTED: return # maybe already imported while package is imported def safe_read(base_path): for i, path in enumerate([base_path + '.pyj', base_path + '/__init__.pyj']): try: return [options.readfile(path, "utf-8"), path] except as e: if e.code is 'ENOENT' or e.code is 'EPERM' or e.code is 'EACCESS': if i is 1: return None, None else: raise src_code = filename = None modpath = key.replace(/\./g, '/') for location in import_dirs: if location: try: data, filename = safe_read(location + '/' + modpath) except as e: if isinstance(e, ImportError): raise e msg = "Failed Import: '" + key + "', due to an error: " + e.message raise ImportError(msg, options.filename, e) if data is not None: src_code = data break if src_code is None: raise ImportError( "Failed Import: '" + key + "' module doesn't exist in any of the import directories: " + import_dirs.join(', '), options.filename ) contents = parse(src_code, { filename: filename, toplevel: None, readfile: options.readfile, basedir: options.basedir, libdir: options.libdir, module_id: key, PRE_IMPORTED: PRE_IMPORTED, IMPORTED: IMPORTED, IMPORTING: IMPORTING, auto_bind: options.auto_bind, es6: options.es6, import_dirs: options.import_dirs, dropDecorators: options.dropDecorators, dropImports: options.dropImports, dropDocstrings: options.dropDocstrings, }) if len(package_module_id) > 0: subs = not IMPORTING[package_module_id] \ ? IMPORTED[package_module_id].submodules \ : PRE_IMPORTED[package_module_id].submodules if not (key in subs): subs.push(key) def import_(from_import): ans = ast.Imports({'imports':[]}) imp_with = None while True: package_pref = '' # allow import submodules into __init__.pyj if (options.filename or '').endsWith('/__init__.pyj'): package_pref = options.module_id + '.' if not PRE_IMPORTED[options.module_id]: PRE_IMPORTED[options.module_id] = {submodules : []} # dot expander: from .module -> from package.module if is_('punc', '.'): if not package_pref: package_pref = (/^(.+?\.)[^\.]+$/.exec(options.module_id or '') or ['', ''])[1] # FIX: if not package_pref: raise Error next() else: package_pref = '' tmp = name = expression(False) key = '' while isinstance(tmp, ast.Dot): key = "." + tmp.property + key tmp = tmp.expression key = package_pref + tmp.name + key if not keepDecoratorOrImport(key, True): return ast.EmptyStatement({ start: prev(), end: prev() }) alias = None if not from_import: if imp_with is None: if is_('keyword', 'with'): imp_with = key next() continue else: imp_with = '' if imp_with: key = imp_with + '.' + key elif is_('keyword', 'as'): next() alias = as_symbol(ast.SymbolAlias) # assume `import .module` is shortcut for `import package.module as module` elif package_pref: alias = new_symbol(ast.SymbolAlias, name.name) imp = ast.Import({ 'module': name, 'key': key, 'alias': alias, 'argnames':None, 'body':def(): return IMPORTED[key] }) ans.imports.push(imp) if from_import: break if is_('punc', ','): next() else: break from_pack_imp = [] for imp in ans['imports']: do_import(imp.key) depends_on[imp.key] = True cur_imported = IMPORTED[imp.key] argnames = [] if from_import: expect_token("keyword", "import") imp.argnames = argnames = [] from_pack_module_names = [] while True: if not is_('name'): unexpected() name = S.token.value # maybe `from package import module` if cur_imported.is_package \ and not cur_imported.exports.find(def(it): return it.name is name;): key = imp.key + '.' + name aname = ast.Import({ 'module': expression(False), 'key': key, 'alias': new_symbol(ast.SymbolAlias, name), 'argnames':None, 'body':def(): return IMPORTED[key] }) from_pack_imp.push(aname) from_pack_module_names.push(aname) do_import(key) else: aname = as_symbol(ast.ImportedVar) argnames.push(aname) if is_('keyword', 'as'): next() aname.alias = as_symbol(ast.SymbolAlias) if is_('punc', ','): next() else: break # Put imported class names in the outermost scope classes = cur_imported.classes for argvar in argnames: obj = classes[argvar.name] if obj: key = (argvar.alias) ? argvar.alias.name : argvar.name S.in_scope[-1].classes[key] = { "static": obj.static, "bound": obj.bound } for mod_name in from_pack_module_names: Object.assign(S.in_scope[S.in_scope.length-1].classes, IMPORTED[mod_name.key].top_classes(mod_name.alias.name)) else: key = imp.alias ? imp.alias.name : imp.key; Object.assign(S.in_scope[S.in_scope.length-1].classes, cur_imported.top_classes(key)) [].push.apply(ans.imports, from_pack_imp) return ans def class_(): start = prev() name = as_symbol(ast.SymbolClass) if not name: unexpected() # detect external classes externaldecorator = has_simple_decorator(S.decorators, 'external') class_details = { "static": [], "bound": {}, "subscribers": [], "on_static_change": def(cb): this.subscribers.push(cb); ,"push_static" : def(meth): this.static.push(meth) this.subscribers.forEach(def(cb): cb(meth, True);); ,"pop_static" : def(meth): idx = this.static.indexOf(meth) if idx >=0: this.static.splice(idx, 1) this.subscribers.forEach(def(cb): cb(meth, False);); } # class setup parent = None docstring = None definition = ast.Class({ start: start, name: name, module_id:module_id, parent: (def(): if is_("punc", "("): next() if is_('punc', ')'): S.in_parenthesized_expr = False next() return None atom = expr_atom(False) expect(")") nonlocal parent parent = stringifyName(atom) if parent and (parent_details = S.in_scope[-1].classes[parent]): [].push.apply(class_details.static, parent_details.static) return atom else: return None )(), localvars: [], static: class_details.static, external: externaldecorator, bound: class_details.bound, statements: [], decorators: (def(): d = [] for decorator in S.decorators: if decorator is 'kwargs': BASELIB['kwargs'] += 1 d.push(ast.Decorator({ expression: decorator })) S.decorators = [] return d )(), body: (def(loop, labels): # navigate to correct location in the module tree and append the class # state "push" S.in_scope[-1].classes[name.name] = class_details S.in_scope.push({ type: "class", name: name.name, parent: parent, nonlocal: {}, functions: {}, vars: {}, classes: {} }) S.in_directives = True S.in_loop = 0 S.labels = [] a = block_() nonlocal docstring docstring = S.in_scope[-1].docstring # state "pop" S.in_scope.pop() S.in_loop = loop S.labels = labels return a )(S.in_loop, S.labels), docstring: docstring, end: prev() }) # find the constructor for i in dir(definition.body): stmt = definition.body[i] if isinstance(stmt, ast.Method) and stmt.name.name is "__init__": definition.init = stmt break # find class variables class_vars_names = {} # Ensure that if a class variable refers to another class variable in # its initialization, the referenced variables' names are correctly # mangled. def walker(): this._visit = def(node, descend): if isinstance(node, ast.Method): class_vars_names[node.name.name] = True return elif isinstance(node, ast.Assign) and isinstance(node.left, ast.SymbolRef): class_vars_names[node.left.name] = True for child in node: if isinstance(node[child], ast.SymbolRef) and Object.prototype.hasOwnProperty.call(class_vars_names, node[child].name): node[child] = ast.SymbolClassRef({ 'class': name, 'name': node[child].name }) if descend: descend.call(node) visitor = new walker() for stmt in definition.body: if not isinstance(stmt, ast.Class) and not isinstance(stmt, ast.Method): stmt.walk(visitor) definition.statements.push(stmt) if S.in_scope.length is 1: # we're in top level class_map[definition.name.name] = definition CLASS_MAP[definition.name.name] = definition return definition def function_(in_class_or_xobject, ctor): in_class = in_xobject = False if in_class_or_xobject: in_xobject = in_class_or_xobject is True ? True : False in_class = in_xobject ? False : in_class_or_xobject start = prev() is_accessor = ctor is ast.ObjectGetter or ctor is ast.ObjectSetter name = as_symbol(in_class_or_xobject \ ? ( is_accessor ? ast.SymbolAccessor : ast.SymbolDefun) \ : ast.SymbolLambda, True) if in_class_or_xobject and not name: unexpected() generator = False # whether this function is a generator localvars = None # hash of local vars assigned to within the function staticmethod = False # whether this function declares itseld as a static method function_args = {} # hash of arguments this function sees return_annotation = None # return annotation for the function, if any if not S.in_decorator: # prevents this logic from firing in a function that exists within the decorator call # special decorators are decorators that have special meaning to the compiler has_special_decorator = def(name): return has_simple_decorator(S.decorators, name) if in_class: cls_details = S.in_scope[-2].classes[in_class] if has_special_decorator("staticmethod"): staticmethod = True # may be inherited from parent static if not (name.name in cls_details.static): cls_details.push_static(name.name) # may overwrites parent static elif name.name in cls_details.static: cls_details.pop_static(name.name) if has_special_decorator("bind") or name.name is not "__init__" and options.auto_bind: BASELIB["bind"] += 1 S.in_scope[-2].classes[in_class].bound[name.name] = True expect("(") if not ctor: if in_class_or_xobject: ctor = in_class and name.name is '__init__' \ ? ast.Constructor \ : ast.Method else: ctor = ast.Function docstring = None callsSuper = None superCall_expr = None definition = new ctor({ start: start, name: name, argnames: (def(a): defaults = {} first = True seen_names = {} def get_arg(): if Object.prototype.hasOwnProperty.call(seen_names, S.token.value): token_error(prev(), "Can't repeat parameter names") if S.token.value is 'arguments': token_error(prev(), "Can't use the name '"+ S.token.value +"' as a parameter name, it is reserved by JavaScript") if not is_("name"): croak("Name expected") return None name_token = S.token seen_names[name_token.value] = True name = maybe_keyword(name_token.value) # check if we have an argument annotation ntok = peek() if ntok.type is 'punc' and ntok.value is ':': # arg:Annotation next() expect(':') annotation = maybe_conditional() # and now, do as_symbol without the next() at the end # since we are already at the next comma (or end bracket) sym = ast.SymbolFunarg({ 'name': name, 'start': name_token, 'end': name_token, 'annotation': annotation ? ast.Annotation({ 'start': annotation.start, 'expression': annotation, 'end': annotation.end }) : None }) else: # arg (no annotation) sym = ast.SymbolFunarg({ 'name': name, 'start': name_token, 'end': name_token, 'annotation': None }) next() function_args[sym.name] = sym.annotation ? sym.annotation.resolveType(S.in_scope) : '?' return sym while not is_("punc", ")"): if first: first = False else: expect(",") if is_('operator', '**'): # FIXME: temporary assertion while I get the logic tested and out the door token_error(prev(), "**kwargs in function definition is not implemented yet, work in progress") # **kwargs next() if a.kwargs: token_error(prev(), "Can't define multiple **kwargs in function definition") a.kwargs = get_arg() elif is_('operator', '*'): # *args next() if a.starargs: token_error(prev(), "Can't define multiple *args in function definition") if a.kwargs: token_error(prev(), "Can't define *args after **kwargs in function definition") a.starargs = get_arg() else: if a.starargs or a.kwargs: token_error(prev(), "Can't define a formal parameter after *args or **kwargs") a.push(get_arg()) if is_("operator", "="): if a.kwargs: token_error(prev(), "Can't define an optional formal parameter after **kwargs") val = prev().value next() defaults[val] = expression(False) a.has_defaults = True else: if a.has_defaults: token_error(prev(), "Can't define required formal parameters after optional formal parameters") next() # check if we have a return type annotation if is_("punc", "->"): next() nonlocal return_annotation # return_annotation = expression(True) expr = expression(True) return_annotation = ast.Annotation({ start: expr.start, expression: expr, end: expr.end }) a.defaults = defaults return a )([]), decorators: S.in_decorator ? [] : (def(): d = [] for decorator in S.decorators: d.push(ast.Decorator({ expression: decorator })) S.decorators = [] return d )(), return_annotation: return_annotation, body: (def(loop, labels): # parse function content # state "push" S.in_scope.push({ type: "function", name: name ? name.name : None, return_annotation: return_annotation, nonlocal: {}, vars: {}, args: function_args, functions: {}, classes: {} }) S.in_directives = True S.in_loop = 0 S.labels = [] a = block_() # these transform variables that are used by other keys, hence the somewhat ugly use of nonlocal nonlocal generator, localvars, docstring, callsSuper, superCall_expr generator = S.in_scope[-1].generator docstring = S.in_scope[-1].docstring callsSuper = S.in_scope[-1].callsSuper superCall_expr = S.in_scope[-1].superCall_expr localvars = [new_symbol(ast.SymbolVar, variable) for variable in Object.keys(S.in_scope[-1].vars) if variable not in S.in_scope[-1].nonlocal] # state "pop" S.in_scope.pop() S.in_loop = loop S.labels = labels return a )(S.in_loop, S.labels), docstring: docstring, generator: generator, localvars: localvars, end: prev(), static: in_class and staticmethod }) if name: # track this function in local scope S.in_scope[-1].functions[name.name] = definition.resolveType(S.in_scope) # remove arguments from localvars args = [arg.name for arg in definition.argnames] definition.localvars = definition.localvars.filter(def(var_): return var_.name not in args;) # do a few extra safety checks on arguments if in_class_or_xobject and not staticmethod: if in_class: if ctor is ast.Constructor: definition.parent = S.in_scope[-1].parent definition.callsSuper = callsSuper if superCall_expr: superCall_expr.selfArg = definition.argnames[0] if definition.argnames.length < 1: croak("Class/object methods require at least one argument (self)", start.line, start.col, start.pos) if ctor is ast.ObjectGetter and definition.argnames.length is not 1: croak("Class/object getters don't take any arguments aside from one referencing the instance (self)", start.line, start.col, start.pos) elif ctor is ast.ObjectSetter and definition.argnames.length is not 2: croak("Class/object setters take exactly 2 arguments (self, value)", start.line, start.col, start.pos) return definition def accessor_(type, start, in_class): if type is "get": func = function_(in_class, ast.ObjectGetter) elif type is "set": func = function_(in_class, ast.ObjectSetter) else: croak("Expecting setter/getter, got '" + type + "' instead.") func.start = start func.end = prev() return func def if_(): cond = expression(True) body = block_statement() belse = None if is_("keyword", "elif") or is_("keyword", "else"): if is_("keyword", "else"): next() belse = block_statement() else: S.token.value = "if" # effectively converts 'elif' to 'else if' belse = statement() return ast.If({ condition: cond, body: body, alternative: belse }) def is_docstring(stmt): if isinstance(stmt, ast.Directive) and not S.in_scope[-1].docstring: return True return False def format_docstring(string): lines = string.split(/\n/g) # determine minimum indentation (first line doesn't count): indent = 1e6 for line in lines[1:]: if line.trim().length: pad = line.match(/^\s*/)[0] indent = Math.min(indent, pad.length) # remove indentation (first line is special): trimmed = [lines[0].trim()] if indent < 1e6: for line in lines[1:]: trimmed.push(line[indent:].replace(/\s+$/)) # strip off trailing and leading blank lines: while trimmed and not trimmed[-1]: trimmed.pop() while trimmed and not trimmed[0]: trimmed.shift() return trimmed.join('\n') def block_(): expect(":") a = [] if not S.token.newline_before: while not S.token.newline_before: if is_("eof"): unexpected() stmt = statement() if not a.length and is_docstring(stmt): # first statement is a docstring if not options.dropDocstrings: S.in_scope[-1].docstring = format_docstring(stmt.value) else: a.push(stmt) else: while not is_("punc", "}"): if is_("eof"): # end of file, terminate block automatically return a stmt = statement() if not a.length and is_docstring(stmt): # first statement is a docstring if not options.dropDocstrings: S.in_scope[-1].docstring = format_docstring(stmt.value) else: a.push(stmt) next() return a def switch_body_(): expect("{") a = [] cur = None branch = None while not is_("punc", "}"): if is_("eof"): unexpected() if is_("keyword", "case"): if branch: branch.end = prev() cur = [] branch = ast.Case({ start: (def(): tmp = S.token next() return tmp )(), expression: expression(True), body: cur }) a.push(branch) expect(":") elif is_("keyword", "default"): if branch: branch.end = prev() cur = [] branch = ast.Default({ start: (def(): tmp = S.token next() expect(":") return tmp )(), body: cur }) a.push(branch) else: if not cur: unexpected() cur.push(statement()) if branch: branch.end = prev() next() return a def try_(): body = block_() bcatch = [] bfinally = None while is_("keyword", "except"): start = S.token next() exceptions = [] if not is_("punc", ":") and not is_("keyword", "as"): exceptions.push(as_symbol(ast.SymbolVar)) while is_("punc", ","): next() exceptions.push(as_symbol(ast.SymbolVar)) name = None if is_("keyword", "as"): next() name = as_symbol(ast.SymbolCatch) bcatch.push(ast.Except({ start: start, argname: name, errors: exceptions, body: block_(), end: prev() })) if is_("keyword", "finally"): start = S.token next() bfinally = ast.Finally({ start: start, body: block_(), end: prev() }) if not bcatch.length and not bfinally: croak("Missing except/finally blocks") return ast.Try({ body: body, bcatch: (bcatch.length ? ast.Catch({ body: bcatch }) : None), bfinally: bfinally }) def vardefs(no_in, type): a = [] while True: symbol = ast.VarDef({ start: S.token, name: as_symbol(type is "const" ? ast.SymbolConst : type is "nonlocal" ? ast.SymbolNonlocal : ast.SymbolVar), end: prev() }) if type is "nonlocal": S.in_scope[-1].nonlocal[symbol.name.name] = True a.push(symbol) if not is_("punc", ","): break next() return a def nonlocal_(no_in): return ast.Var({ start: prev(), definitions: vardefs(no_in, "nonlocal"), end: prev() }) def const_(): return ast.Const({ start: prev(), definitions: vardefs(False, "const"), end: prev() }) def yield_(): if S.in_scope[-1].type is not "function": croak("'yield' outside of function") S.in_scope[-1].generator = True value = None yield_from = False if is_("punc", ";"): semicolon() elif not can_insert_semicolon(): if S.token.type is 'keyword' and S.token.value is 'from': yield_from = True next() value = expression(True) semicolon() ret = new ast.Yield({ value: value, yield_from: yield_from }) return ret def yield_request(): start = S.token expect_token("keyword", "yield") ret = yield_() ret.start = start ret.end = prev() ret.computedType = '?' ret.yield_request = True return ret def new_(): start = S.token expect_token("operator", "new") newexp = expr_atom(False) if is_("punc", "("): next() args = expr_list(")") else: args = [] return subscripts(ast.New({ start: start, expression: newexp, args: args, end: prev() }), True) def as_atom_node(token): tok = token or S.token tmp_ = tok.type if tmp_ is "name": if tok.value is 'NaN': ret = as_symbol(ast.NotANumber, token=tok) elif tok.value is 'undefined': ret = as_symbol(ast.Undefined, token=tok) else: ret = as_symbol(ast.SymbolRef, token=tok) elif tmp_ is "num": ret = ast.Number({ start: tok, end: tok, value: tok.value }) elif tmp_ is "string": if tok.subtype is "v": ret = new ast.Verbatim({ start: tok, value: tok.value, end: tok, }) else: ret = ast.String({ start: tok, end: tok, value: tok.value, modifier: tok.subtype }) elif tmp_ is "regexp": ret = ast.RegExp({ start: tok, end: tok, value: tok.value }) elif tmp_ is "atom": tmp__ = tok.value.toLowerCase() if /^(none)|(null)$/.test(tmp__): ret = ast.Null({ start: tok, end: tok }) elif tmp__ is "false": ret = ast.Boolean({ start: tok, value: False, end: tok }) elif tmp__ is "true": ret = ast.Boolean({ start: tok, value: True, end: tok }) else: raise Error('Parser seems corrupted') if not token: next() ret.resolveType(S.in_scope) # provides more context for output later return ret def expr_atom(allow_calls): if is_("operator", "new"): return new_() start = S.token if is_("punc"): tmp_ = start.value if tmp_ is "(": next() ex = expression(True) ex.start = start ex.end = S.token if isinstance(ex, ast.SymbolRef) or isinstance(ex, ast.Yield): ex.parens = True expect(")") return subscripts(ex, allow_calls) elif tmp_ is "[": return subscripts(array_(), allow_calls) elif tmp_ is "{": return subscripts(object_(), allow_calls) unexpected() if is_("keyword", "class"): next() cls = class_() cls.start = start cls.end = prev() return subscripts(cls, allow_calls) if is_("keyword", "def"): next() func = function_(False) func.start = start func.end = prev() return subscripts(func, allow_calls) if is_("keyword", "yield"): return yield_request() if ATOMIC_START_TOKEN[S.token.type]: return subscripts(as_atom_node(), allow_calls) unexpected() def expr_list(closing, allow_trailing_comma, allow_empty, func_call): first = True a = [] saw_starargs = False while not is_("punc", closing): if saw_starargs: token_error(prev(), "*args must be the last argument in a function call") if first: first = False else: expect(",") if allow_trailing_comma and is_("punc", closing): break if is_("operator", "*") and func_call: saw_starargs = True next() if is_("punc", ",") and allow_empty: a.push(ast.Hole({ start: S.token, end: S.token })) else: a.push(expression(False)) if func_call: tmp = [] tmp.kwargs = [] for i, arg in enumerate(a): if isinstance(arg, ast.Assign): # arg=val BASELIB['kwargs'] += 1 tmp.kwargs.push([arg.left, arg.right]) else: # regular argument tmp.push(arg) a = tmp next() if saw_starargs: a.starargs = True return a def func_call_list(): a = [] first = True a.kwargs = [] a.kwarg_items = kwargs = [] a.starargs = False while not is_("punc", ')'): if first: first = False else: expect(",") if is_('operator', '*'): # starargs argument next() arg = expression(False) arg.is_array = True a.push(arg) a.starargs = True elif is_('operator', '**'): # kwargs argument BASELIB['kwargs'] += 1 next() kwargs.push(as_symbol(ast.SymbolVar, False)) else: # arg=val assignment arg = expression(False) if isinstance(arg, ast.Assign): BASELIB['kwargs'] += 1 a.kwargs.push([arg.left, arg.right]) else: a.push(arg) next() return a def read_comprehension(object): # shared by list and dict comprehensions terminator = isinstance(object, ast.DictComprehension) ? '}' : ']' expect_token('keyword', 'for') forloop = for_(True) BASELIB["iterable"] += 1 object.init = forloop.init object.name = forloop.name object.object = forloop.object object.condition = is_("punc", terminator) ? None : (expect_token("keyword", "if"), expression(True)) expect(terminator) return object @embed_tokens def array_(): expect("[") expr = [] if not is_("punc", "]"): expr.push(expression(False)) if is_("keyword", "for"): # list comprehension return read_comprehension(ast.ListComprehension({ statement: expr[0] })) if is_("operator", "til"): # up to but not including upper limit BASELIB['range'] += 1 next() expr.push(expression(False)) ret = ast.Range({ start: S.token, left: expr[0], operator: "til", right: expr[1], end: prev() }) expect("]") return ret elif is_("operator", "to"): # now add a tiny number to make sure we include the upper limit BASELIB['range'] += 1 next() expr.push(expression(False)) ret = ast.Range({ start: S.token, left: expr[0], operator: "to", right: expr[1], end: prev() }) expect("]") return ret elif not is_("punc", "]"): expect(",") return ast.Array({ elements: expr.concat(expr_list("]", not options.strict, True)) }) @embed_tokens def object_(): maybe_dict_comprehension = False must_be_comprehension = False expect("{") first = True a = [] while not is_("punc", "}"): if not first: expect(",") if not options.strict and is_("punc", "}"): # allow trailing comma break start = S.token type = start.type if is_("operator", "*"): # spread operator if not options.es6: croak("Spread operator in object literals is only allowed in ES6 mode") a.push(maybe_unary(True)) first = False continue elif is_("operator", "@") or \ (is_("keyword", "def") or is_("name", ["get", "set"])) \ and peek().value is not ":": #value = {body:[]} while is_("operator", "@"): #value.body.push(maybe_unary(True)) maybe_unary(True) pass start = S.token fun_type = is_("keyword", "def") \ ? "def" \ : is_("name", ["get", "set"]) \ ? start.value \ : None if fun_type and peek().value is not ":": #value.body.push(expression(False)) next() if fun_type is "def": value = function_(True) else: value = accessor_(fun_type, start, True) #key = value.body[-1].name key = value.name else: unexpected() a.push(ast.ObjectKeyVal({ start: start, key: key, value: value, end: prev() })) first = False continue else: if first and not (peek().value in [':', ',', '}']): # possibly dict comprehension maybe_dict_comprehension = True must_be_comprehension = not is_("punc", "(") key = expression(False) name = None else: if is_("punc", "("): # computed key if not options.es6: croak("Computed properties are only allowed in ES6 mode") expect('(') key = expression(False) expect(')') else: # regular key key_ = as_property_name() name = key_.value if key_.type is "num": key = ast.Number({ start: start, value: name, end: prev() }) elif key_.type is "name" or key_.type is "keyword": if name in ["True", "False"]: key = ast.Boolean({ start: start, value: name, end: prev() }) else: key = ast.Identifier({ start: start, value: name, end: prev() }) else: key = ast.String({ start: start, value: name, end: prev() }) if is_('punc') and S.token.value in [',', '}'] and isinstance(key, ast.Identifier): # maybe es6 short notation value = key else: expect(":") value = expression(False) a.push(ast.ObjectKeyVal({ start: start, key: key, value: value, end: prev() })) if first and is_('keyword', 'for'): return read_comprehension(ast.DictComprehension({ statement: maybe_dict_comprehension ? key : as_atom_node(a[0].start), value_statement: a[0].value })) elif must_be_comprehension: unexpected() first = False next() return ast.ObjectLiteral({ properties: a }) def as_property_name(): tmp = S.token next() tmp_ = tmp.type if tmp_ is "num" or tmp_ is "string" or tmp_ is "name" or tmp_ is "operator" or tmp_ is "keyword" or tmp_ is "atom": return tmp else: unexpected() def as_name(): tmp = S.token next() tmp_ = tmp.type if tmp_ is "name" or tmp_ is "operator" or tmp_ is "keyword" or tmp_ is "atom": return tmp.value else: unexpected() def as_symbol(type, noerror, token): token_ = token or S.token js_reserved = False if type in [ast.SymbolDefun, ast.SymbolAccessor]: name = token_._value or token_.value tmp_ = token_.type if tmp_ is "name" or tmp_ is "num" or tmp_ is "string" or tmp_ is "keyword" or tmp_ is "atom" \ or tokenizer.is_token(token_, "operator:keyword"): js_reserved = tokenizer.ALL_JS_KEYWORDS(name) else: return None else: if not tokenizer.is_token(token_, "name"): if not noerror: croak("Name expected") return None name = token_.value if name is 'this': if type is ast.SymbolLambda: return None type = ast.This else: name = maybe_keyword(name) sym = new (True and type)({ name: (String)(name), js_reserved: js_reserved, start: token_, end: token_ }) if not token: next() return sym # for generating/inserting a new symbol def new_symbol(type, name): sym = new (name is "this" ? ast.This : type)({ name: (String)(name), start: None, end: None }) return sym def is_static_method(cls, method): if method in COMMON_STATIC or cls.static and method in cls.static: return True else: return False def mark_local_assignment(element, value): if value: computedType = JS('typeof value') is "string" ? value : value.resolveType(S.in_scope) else: computedType = "?" name = JS('typeof element') is "string" ? element : element.name if name: # not enough information to infer type yet # so all types are ? for now, in the future this function will infer type from context # that may need some refactoring of car/cdr logic if name in S.in_scope[-1].vars: S.in_scope[-1].vars[name].push(computedType) else: S.in_scope[-1].vars[name] = [computedType] def subscripts(expr, allow_calls): start = expr.start if is_("punc", "."): next() return subscripts(ast.Dot({ start: start, expression: expr, property: as_name(), end: prev() }), allow_calls) if is_("punc", "[") and not S.token.newline_before: next() slice_bounds = [] is_slice = False if is_("punc", ":"): # slice [:n] slice_bounds.push(None) else: # slice [n?] slice_bounds.push(expression(False)) if is_("punc", ":"): # slice [n:m?] is_slice = True next() if is_("punc", ":"): slice_bounds.push(None) elif not is_("punc", "]"): slice_bounds.push(expression(False)) if is_("punc", ":"): # slice [n:m:o?] BASELIB["eslice"] += 1 next() if is_("punc", "]"): unexpected() else: slice_bounds.push(expression(False)) expect("]") if is_slice: if is_("operator") and S.token.value is "=": # splice-assignment (arr[start:end] = ...) next() # swallow the assignment return subscripts(ast.Slice({ start: start, expression: expr, property: slice_bounds[0] or ast.Number({ value: 0 }), property2: slice_bounds[1], assignment: expression(True), end: prev() }), allow_calls) elif slice_bounds.length is 3: # extended slice (arr[start:end:step]) slice_bounds.unshift(slice_bounds.pop()) if not slice_bounds[-1]: slice_bounds.pop() if not slice_bounds[-1]: slice_bounds.pop() elif not slice_bounds[-2]: slice_bounds[-2] = ast.Undefined() return subscripts(ast.Call({ start: start, expression: ast.SymbolRef({ name: "eslice" }), args: [expr].concat(slice_bounds), end: prev() }), allow_calls) else: # regular slice (arr[start:end]) slice_bounds = [i is None ? ast.Number({ value: 0 }) : i for i in slice_bounds] return subscripts(ast.Call({ start: start, expression: ast.Dot({ start: start, expression: expr, property: "slice", end: prev() }), args: slice_bounds, end: prev() }), allow_calls) else: # regular index (arr[index]) return subscripts(ast.Sub({ start: start, expression: expr, property: slice_bounds[0] or ast.Number({ value: 0 }), end: prev() }), allow_calls) if allow_calls and is_("punc", "(") and not S.token.newline_before: next() if isinstance(expr, ast.SymbolRef) and expr.name is "JS": # raw JavaScript chunk of code str_ = expression(False) if not (isinstance(str_, ast.String)): token_error(prev(), "Compile-time function JS() can't evaluate variables or expressions") ret = ast.Verbatim({ start: start, value: str_.value, end: prev() }) expect(")") return subscripts(ret, True) elif not expr.parens and get_class_in_scope(expr): # this is an object being created using a class # check if this class is part of our standard library if expr.name in STDLIB: BASELIB[expr.name] += 1 if /Error$/.test(expr.name): # errors are classes BASELIB["extends"] += 1 return subscripts(ast.New({ start: start, expression: expr, args: func_call_list(), end: prev() }), True) else: if isinstance(expr, ast.Dot): className = get_class_in_scope(expr.expression) if className: # generate class call funcname = expr if funcname.property is "__init__": funcname.property = "constructor" # there are maybe non-super class calls, so # we have to prevent S.in_scope[-1].callsSuper from being changed if it is already set to True is_super = S.in_scope.length > 1 and S.in_scope[-2].type is 'class' and stringifyName(expr.expression) is S.in_scope[-2].parent S.in_scope[-1].callsSuper = S.in_scope[-1].callsSuper or is_super ast_ClassCall = ast.ClassCall({ start: start, class: expr.expression, method: funcname.property, super: is_super, static: is_static_method(className, funcname.property), args: func_call_list(), end: prev() }) # fix issue #128 # subscribe on staticmethod definitions if className.on_static_change: className.on_static_change(def(static_name, on_off): if static_name is funcname.property: ast_ClassCall.static = on_off ) return validateCallArgs(subscripts(ast_ClassCall, True)) elif isinstance(expr, ast.SymbolRef): tmp_ = expr.name # special functions that trigger addition of extra logic to generated JavaScript if tmp_ in STDLIB: BASELIB[tmp_] += 1 # NOTE: there is intentionally no return here, we want these functions to go through regular logic # after they trigger the appropriate baselib flag elif tmp_ is "isinstance": args = func_call_list() if args.length is not 2: croak("'isinstance' takes exactly 2 arguments") return ast.Binary({ start: start, operator: "instanceof", left: args[0], right: args[1], end: prev() }) elif tmp_ is "super": S.in_scope[-1].callsSuper = True S.in_scope[-1].superCall_expr = expr else: # avoid collision with reserved javascript keywords expr.name = maybe_keyword(expr.name) # fall-through to basic function call return validateCallArgs(subscripts(ast.Call({ start: start, expression: expr, args: func_call_list(), end: prev() }), True)) return expr def stringifyName(expr): if isinstance(expr, ast.Dot): return stringifyName(expr.expression) + "." + expr.property return expr.name def keepDecoratorOrImport(expr, imp=False): # helper function for figuring out whether this decorator should be dropped if imp: if not options.dropImports.length: return True else: if not options.dropDecorators.length: return True if JS('typeof expr') is 'string': # passed in a string name = expr elif isinstance(expr, ast.SymbolRef): # regular decorator name = expr.name elif isinstance(expr, ast.Dot): # dot decorator name = stringifyName(expr) elif isinstance(expr, ast.Call): # decorator with args name = stringifyName(expr.expression) else: croak("Unsupported decorator") if imp: return name not in options.dropImports else: return name not in options.dropDecorators def maybe_unary(allow_calls): start = S.token if is_('operator', '@'): if S.in_decorator: croak('Nested decorators are not allowed') next() S.in_decorator = True expr = expression(True) S.in_decorator = False if keepDecoratorOrImport(expr): S.decorators.push(expr) return ast.EmptyStatement({ stype: '@', start: prev(), end: prev() }) if is_("operator") and UNARY_PREFIX(start.value): next() ex = make_unary(ast.UnaryPrefix, start.value, maybe_unary(allow_calls)) ex.start = start ex.end = prev() return ex val = expr_atom(allow_calls) while is_("operator") and tokenizer.UNARY_POSTFIX(S.token.value) and not S.token.newline_before: val = make_unary(ast.UnaryPostfix, S.token.value, val) val.start = start val.end = S.token next() return val def make_unary(ctor, op, expr): return validateUnary(new ctor({ operator: op, expression: expr })) def validateBinary(astElement): # check for invalid operations using our knowledge of types # allow int + str for now left = astElement.left.resolveType(S.in_scope) right = astElement.right.resolveType(S.in_scope) op = astElement.operator # for in, instanceof, ==, !=, ===, !==, 'or', 'and', '=' everything is allowed # for 2 numbers any operator is valid # for 2 strings '+' and comparison operators are valid # for number and string only the + operator is valid # for ?, we don't have enough information # for everything else, no operator is valid if op not in ['in', 'instanceof', '==', '!=', '===', '!==', '||', '&&', '='] and ( left not in ['Number', 'String', 'Boolean', '?'] or right not in ['Number', 'String', 'Boolean', '?'] or (left is 'String' or right is 'String') and ( left is right \ ? op not in ['+', '+=', '<', '<=', '>', '>='] \ : op not in ['+', '+='] ) ): # make the representations a bit more human-friendly if left: if left[0] is '[': left = 'Array' elif left[0] is '{': left = 'Object' if right: if right[0] is '[': right = 'Array' elif right[0] is '{': right = 'Object' raise croak("cannot perform binary '" + op + "' operation on incompatible elements of type " + left + " and " + right + "") return astElement def validateUnary(astElement): # check that the prefix is compatible with the element element = astElement.expression.resolveType(S.in_scope) op = astElement.operator if not element: if op is not '!': raise croak("cannot perform unary '" + op + "' operation on incompatible element of type " + element) elif element not in ['Number', '?'] and op in ['+', '-'] or element[0] not in ['[', '{', '?'] and op is '*': # make the representations a bit more human-friendly if element[0] is '[': element = 'Array' elif element[0] is '{': element = 'Object' raise croak("cannot perform unary '" + op + "' operation on incompatible element of type " + element) return astElement def validateCallArgs(astElement): # check that passed in arguments match function signature # also, let's check that we're not passing too many arguments if isinstance(astElement.expression, ast.SymbolRef): name = astElement.expression.name found = False for scope in reversed(S.in_scope): for func in scope.functions: if func is name: signature = scope.functions[func] found = True break for variable in scope.vars: if variable is name: signature = scope.vars[func] found = True break if found: break # TODO: there is a case of having found == True and signature = undefined, it shouldn't happen if signature and signature[:9] is 'Function(': # TODO: fix this, current logic naively assumes there are no nested commas # validate the number of args first args = /\((.*)\)/.exec(signature)[1].split(',') if args.length is 1 and not args[0].length: args.pop() if args.length < astElement.args.length: croak("Function '" + name + "' takes " + args.length + " arguments, yet your call contains " + astElement.args.length + "") # now validate argument types for i, arg in enumerate(astElement.args): expected = args[i].trim() actual = arg.resolveType(S.in_scope) if expected is not '?' and actual not in [expected, '?']: croak("Function '" + name + "' expects argument " + i + " type of " + expected + ", but you're passing " + actual + "") return astElement def expr_op(left, min_prec, no_in): op = (is_("operator") ? S.token.value : None) not_in = False if op is "!" and peek().type is "operator" and peek().value is "in": next() op = "in" not_in = True if op is "in": if no_in: op = None else: BASELIB[op] += 1 prec = (op is not None ? tokenizer.PRECEDENCE[op] : None) if prec is not None and prec > min_prec: next() right = expr_op(maybe_unary(True), prec, no_in) if op in ['==', '!=']: # RapydScript's deep equality is unique to other compilers in that it has the same # performance (no overhead) as the naive identity operator native to JS, for that # reason it is now the default equality. If you wish to avoid it, use 'is' operator # instead BASELIB["eq"] += 1 ret = ast.DeepEquality({ start: left.start, left: left, operator: op, right: right, end: right.end }) else: ret = ast.Binary({ start: left.start, left: left, operator: op, right: right, end: right.end }) validateBinary(ret) if not_in: ret = ast.UnaryPrefix({ start: left.start, operator: "!", expression: ret, end: right.end }) return expr_op(ret, min_prec, no_in) return left def expr_ops(no_in): return expr_op(maybe_unary(True), 0, no_in) def maybe_conditional(no_in): start = S.token expr = expr_ops(no_in) if is_("operator", "?"): next() yes = expression(False) expect(":") return ast.Conditional({ start: start, condition: expr, consequent: yes, alternative: expression(False, no_in), end: peek() }) return expr def isAssignable(expr): # recursively checks if every element being assigned to is in fact assignable if isinstance(expr, ast.PropAccess): return True if isinstance(expr, ast.SymbolRef): if options.strict_names and tokenizer.IS_ANY_KEYWORD(expr.name): return False return True if isinstance(expr, ast.Array): for element in expr.elements: if not isAssignable(element): return False return True if isinstance(expr, ast.Seq): # remind me to get rid of these ast.Seq nodes, they do more harm than good if isAssignable(expr.car) and isAssignable(expr.cdr): return True return False def maybe_assign(no_in): start = S.token left = maybe_conditional(no_in) val = S.token.value if is_("operator") and ASSIGNMENT(val): if isAssignable(left): if isinstance(left, ast.SymbolRef) and val is not "=" and left.name not in S.in_scope[-1].vars and (not S.in_scope[-1].args or left.name not in S.in_scope[-1].args) and left.name not in S.in_scope[-1].nonlocal: croak("Attempting to increment/modify uninitialized variable '" + left.name + "', this can also occur if you're trying to shadow without initializing the variable in local scope.") next() right = maybe_assign(no_in) if not S.in_seq: # regular assignment of type a = ... will be picked up here # other assignments such as a, b = ..., [a, b] = ... will be picked up in sequence formation logic mark_local_assignment(left, right) return validateBinary(ast.Assign({ start: start, left: left, operator: val, right: right, end: prev() })) croak("Invalid assignment") return left def expression(commas, no_in): # if there is an assignment, we want the sequences to pivot # around it to allow for tuple packing/unpacking start = S.token expr = maybe_assign(no_in) if commas: left = [ expr ] while is_("punc", ",") and not peek().newline_before: S.in_seq = True next() if isinstance(expr, ast.Assign): # ast.Seq representation is ugly to decode for # assignments, let's convert data to array now # to avoid dealing with it left[-1] = left[-1].left if left.length is 1: if isinstance(left[0], ast.Seq): # convert sequence to array and add every variable to local scope leftAst = seq_to_array(left[0]) else: # left[0] is already an array leftAst = left[0] else: # left itself is an array leftAst = ast.Array({ elements: left }) # this is assignment of type [a, b] = ... and (a, b) = ..., scan for local vars right = seq_to_array(ast.Seq({ car: expr.right, cdr: expression(True, no_in) })) for index, element in enumerate(leftAst.elements): mark_local_assignment(element, right.elements[index]) return ast.Assign({ start: start, left: leftAst, operator: expr.operator, right: right, end: peek() }) expr = maybe_assign(no_in) left.push(expr) S.in_seq = False # transform assignments to (a, b, ...) into [a, b, ...] if isinstance(expr, ast.Assign) and isinstance(expr.left, ast.Seq): expr.left = seq_to_array(expr.left) # if last one was an assignment, fix it if left.length > 1 and isinstance(left[-1], ast.Assign): left[-1] = left[-1].left for index, element in enumerate(left): mark_local_assignment(element, isinstance(expr.right, ast.Array) ? expr.right.elements[index] : None) return ast.Assign({ start: start, left: ast.Array({ elements: left }), operator: expr.operator, right: expr.right, end: peek() }) #recursive sequence formation seq = (def build_seq(a): first = a.shift() # this will pick up array to array assignment if isinstance(first, ast.Assign): if isinstance(first.left, ast.Array): for index, element in enumerate(first.left.elements): mark_local_assignment(element, isinstance(first.right, ast.Array) ? first.right.elements[index] : None) if not a.length: return first return ast.Seq({ start: start, car: first, cdr: build_seq(a), end: peek() }) )(left) # seq.dump(3) # print('') return seq return expr def in_loop(cont): S.in_loop += 1 ret = cont() S.in_loop -= 1 return ret return (def(): start = S.token body = [] docstring = None first_token = True while not is_("eof"): element = statement() if first_token and isinstance(element, ast.Directive) and element.value.indexOf('#!') is 0: shebang = element.value elif not body.length and is_docstring(element): # first statement is a docstring if not options.dropDocstrings: docstring = format_docstring(element.value) else: body.push(element) first_token = False end = prev() toplevel = options.toplevel if toplevel: toplevel.body = toplevel.body.concat(body) toplevel.end = end else: toplevel = ast.TopLevel({ start: start, body: body, strict: True, shebang: shebang, docstring: docstring, end: end }) def uniq(element, index, arr): return arr.lastIndexOf(element) is index toplevel.filename = options.filename toplevel.nonlocalvars = Object.keys(S.in_scope[-1].nonlocal) assignments = Object.keys(S.in_scope[-1].vars) callables = scan_for_top_level_callables(toplevel.body).filter(uniq) toplevel.localvars = [] for item in assignments: if item not in toplevel.nonlocalvars: toplevel.localvars.push(new_symbol(ast.SymbolVar, item)) toplevel.exports = toplevel.localvars.concat(callables).filter(uniq) toplevel.depends_on = depends_on toplevel.submodules = [] toplevel.classes = class_map toplevel.top_classes = def(key): ret = {} key = key or '' cn = '' for JS('cn in this.classes'): obj = this.classes[cn] ret[key ? key + '.'+ cn : cn] = { "static": obj.static, "bound": obj.bound } for sub_id in this.submodules: sub_key = sub_id.replace(RegExp('^' + this.module_id.replace(/\./g, '\\.')), key) sub = IMPORTED[sub_id] Object.assign(ret, sub.top_classes(sub_key)) return ret toplevel.import_order = Object.keys(IMPORTED).length toplevel.module_id = module_id toplevel.is_package = (toplevel.filename or '').endsWith('/__init__.pyj') IMPORTED[module_id] = toplevel toplevel.imports = IMPORTED toplevel.baselib = BASELIB IMPORTING[module_id] = False if PRE_IMPORTED[module_id]: IMPORTED[module_id].submodules = PRE_IMPORTED[module_id].submodules del PRE_IMPORTED[module_id] return toplevel )()