#! /usr/bin/env python3 # -*- coding: utf-8 -*- ## tinpy.compiler is DEPRECATED, everything is moved into this file for easy embedding ## #from tinypy.compiler.__main__ import main #main() import os, sys, random DEBUG_PARSER = True DEBUG_ENCODER = True NEXT = 169 ## from encode.py ## try: ## scrambled interp codes ## from tpython_interpreter_codes_gen import * print('using external interp codes imported from tpython_interpreter_codes_gen.py') except ImportError: ## default interp code order ## print('using default interp codes') EOF,REGS,NAME,INTEGER,NUMBER,STRING,MOVE,IF,EQ,LE,LT,GGET,GSET,ADD,SUB,MUL,DIV,CMP,MGET,GET,SET,NE,NOT,IFN,ITER,HAS,IGET,DEL,IFACE,DICT,LIST,PARAMS,LEN,JUMP,SETJMP,CALL,DEF,RETURN,RAISE,NONE,MOD,LSH,RSH,POW,BITAND,BITOR,BITNOT,BITXOR,PASS,FILE,DEBUG,LINE = range(52) class DState: def __init__(self,code,fname): self.imports = [] self.function_stack = [] self.class_stack = [] self.classes = {} self.fast_meth_defs = {} self.in_return = False self.strings256 = [ '__name__', '__file__', '__dict__', '__call__', '__new__', '__init__', 'object', 'update', 'keys', 'values', 'version', 'modules', 'argv', 'sys', ] print( blue('number of internal tiny atomic strings: '), len(self.strings256) ) self.globals256_funcs = {} ## func name : args names self.globals256 = [] ## max size is 256 self.const_numbers = [] ## max size is 256 self.header = [] if '--debug' in sys.argv: self.nopos = False else: self.nopos = True self.code = code self.fname = fname self.lines = self.code.split('\n') self.stack = [] self.out = [('tag','EOF')] self._scopei = 0 self.tstack = [] self._tagi = 0 self.data = {} self.error = False def begin(self,gbl=False): if len(self.stack): self.stack.append((self.vars,self.r2n,self.n2r,self._tmpi,self.mreg,self.snum,self._globals,self.lineno,self.globals,self.rglobals,self.cregs,self.tmpc)) else: self.stack.append(None) self.vars = [] self.r2n = {} self.n2r = {} self._tmpi = 0 self.mreg = 0 self.snum = str(self._scopei) self._globals = gbl self.lineno = -1 self.globals = [] self.rglobals = [] self.cregs = ['regs'] self.tmpc = 0 self._scopei += 1 insert(self.cregs) def end(self): self.cregs.append(self.mreg) code(EOF) # This next line forces the encoder to # throw an exception if any tmp regs # were leaked within the frame # assert(self.tmpc == 0) #REG if self.tmpc != 0: print("Warning:\nencode.py contains a register leak\n") if len(self.stack) > 1: self.vars,self.r2n,self.n2r,self._tmpi,self.mreg,self.snum,self._globals,self.lineno,self.globals,self.rglobals,self.cregs,self.tmpc = self.stack.pop() else: self.stack.pop() def const_number(self, n): ## returns the address of the constant number if len(self.const_numbers) == 256: return -1 elif n in self.const_numbers: return self.const_numbers.index(n) else: self.const_numbers.append(n) const_addr = len(self.const_numbers)-1 code(NUMBER,a=0,b=2,c=const_addr, head=True) ## b=2 means do not set a register write(fpack(number(n)), head=True) return const_addr def insert(v): D.out.append(v) def write(v, head=False): if istype(v,'list'): if head: D.header.append( v ) else: insert(v) return for n in range(0,len(v),4): if head: D.header.append(('data',v[n:n+4])) else: insert(('data',v[n:n+4])) def setpos(v): if D.nopos: return line,x = v if line == D.lineno: return text = D.lines[line-1] D.lineno = line val = text + "\0"*(4-len(text)%4) code_16(LINE,int(len(val)/4),line) write(val) def code(i,a=0,b=0,c=0, head=False): if not istype(i,'number'): raise RuntimeError('code.i is not a number') if not istype(a,'number'): if a is None: ## probably should print a warning here a = 0 else: raise RuntimeError('code.a is not a number: %s' %a) if not istype(b,'number'): raise RuntimeError('code.b is not a number: %s' %b) if not istype(c,'number'): raise RuntimeError('code.c is not a number: %s' %c) write(('code',i,a,b,c), head=head) def code_16(i,a,b): if b < 0: b += 0x8000 code(i,a,(b&0xff00)>>8,(b&0xff)>>0) def get_code16(i,a,b): return ('code',i,a,(b&0xff00)>>8,(b&0xff)>>0) STRINGS_EX = ('print', 'compile') def _do_string(v,r=None, no_intern=False): r = get_tmp(r) val = v + "\0"*(4-len(v)%4) ## for the string to use the precomputed 8bit hash is must be less than 10 bytes, ## because the 8bit hash is shoved into the last byte, and second to last is the null terminator. if len(v) <= 10 and len(D.strings256) < 256 and not no_intern and v not in STRINGS_EX: if v not in D.strings256: D.strings256.append(v) code(179, r, len(v), D.strings256.index(v) ) else: code_16(STRING,r,len(v)) write(val) return r def do_string(t,r=None): if len(t.val)==1: r = get_tmp(r) code(178, a=r, b=ord(t.val)) return r elif hasattr(t, 'no_intern'): return _do_string(t.val,r, no_intern=True) else: return _do_string(t.val,r) def _do_number(v,r=None): r = get_tmp(r) const_addr = D.const_number(v) if const_addr != -1: ## TODO do not write number twice, this is needed until the rest of the code is refactored code(NUMBER,a=r,b=1,c=const_addr) else: code(NUMBER,a=r,b=0,c=0) write(fpack(number(v))) return r def _do_integer(v,r=None): r = get_tmp(r) #code(INTEGER,r,0,0) #write(fpack(number(v))) ## not packed or unpacked properly? TODO FIXME code(INTEGER, a=r, b=int(v), c=0) ## just small integers for now return r def do_number(t,r=None): if '--int-type' in sys.argv and '.' not in t.val and int(t.val) >= 0 and int(t.val) <= 255: return _do_integer(t.val,r) else: return _do_number(t.val,r) def get_tag(): k = str(D._tagi) D._tagi += 1 return k def stack_tag(): k = get_tag() D.tstack.append(k) return k def pop_tag(): D.tstack.pop() def tag(*t): t = D.snum+':'+':'.join([str(v) for v in t]) insert(('tag',t)) def jump(*t): t = D.snum+':'+':'.join([str(v) for v in t]) insert(('jump',t)) def setjmp(*t): t = D.snum+':'+':'.join([str(v) for v in t]) insert(('setjmp',t)) def fnc(*t): t = D.snum+':'+':'.join([str(v) for v in t]) r = get_reg(t) insert(('fnc',r,t)) return r def map_tags(): tags = {} out = [] n = 0 for item in D.header + D.out: if item[0] == 'tag': tags[item[1]] = n continue if item[0] == 'regs': out.append(get_code16(REGS,item[1],0)) n += 1 continue out.append(item) n += 1 for n in range(0,len(out)): item = out[n] if item[0] == 'jump': out[n] = get_code16(JUMP,0,tags[item[1]]-n) elif item[0] == 'setjmp': out[n] = get_code16(SETJMP,0,tags[item[1]]-n) elif item[0] == 'fnc': out[n] = get_code16(DEF,item[1],tags[item[2]]-n) for n in range(0,len(out)): item = out[n] if item[0] == 'data': out[n] = item[1] elif item[0] == 'code': i,a,b,c = item[1:] out[n] = chr(i)+chr(a)+chr(b)+chr(c) else: raise str(('huh?',item)) if len(out[n]) != 4: raise ('code '+str(n)+' is wrong length '+str(len(out[n]))) D.out = out def get_tmp(r=None): if r != None: return r return get_tmps(1)[0] def get_tmps(t): rs = alloc(t) regs = range(rs,rs+t) for r in regs: set_reg(r,"$"+str(D._tmpi)) D._tmpi += 1 D.tmpc += t #REG return regs def alloc(t): s = ''.join(["01"[r in D.r2n] for r in range(0,min(256,D.mreg+t))]) return s.index('0'*t) def is_tmp(r): if r is None: return False return (D.r2n[r][0] == '$') def un_tmp(r): n = D.r2n[r] free_reg(r) set_reg(r,'*'+n) def free_tmp(r): if is_tmp(r): free_reg(r) return r def free_tmps(r): for k in r: free_tmp(k) def get_reg(n): if n not in D.n2r: set_reg(alloc(1),n) return D.n2r[n] #def get_clean_reg(n): #if n in D.n2r: raise #set_reg(D.mreg,n) #return D.n2r[n] def set_reg(r,n): D.n2r[n] = r; D.r2n[r] = n D.mreg = max(D.mreg,r+1) def free_reg(r): if is_tmp(r): D.tmpc -= 1 n = D.r2n[r]; del D.r2n[r]; del D.n2r[n] def imanage(orig,fnc): if orig.val in ('+=', '-=', '*=', '/=') and '--beta' in sys.argv: print(orig) dest = orig.items[0] expr = orig.items[1] ## may contain sub expressions assert len(orig.items)==2 assert isinstance( dest, Token ) if dest.type == 'name' and len(dest.val)==1: if expr.type == 'number' and expr.val.isdigit(): b = int(expr.val) if b > 0 and b < 256: if orig.val == '+=': if dest.val in D.globals: code(92, a=ord(dest.val), b=b) else: if b == 1: code(90, a=get_reg(dest.val)) else: code(91, a=get_reg(dest.val), b=b) return None if dest.val in D.globals: a = ord(dest.val) if expr.type == 'name' and len(expr.val)==1 and expr.val in D.globals: b = ord(expr.val) if orig.val == '+=': code(101, a=a, b=b) elif orig.val == '-=': code(102, a=a, b=b) elif orig.val == '*=': code(103, a=a, b=b) elif orig.val == '/=': code(104, a=a, b=b) return None elif expr.type == 'symbol': opb, opc = expr.items if opb.type=='name' and len(opb.val)==1 and opb.val in D.globals: if opc.type=='name' and len(opc.val)==1 and opc.val in D.globals: b = ord(opb.val) c = ord(opc.val) if orig.val == '+=': if expr.val == '+': code(105, a=a, b=b, c=c) elif expr.val == '-': code(106, a=a, b=b, c=c) elif expr.val == '*': code(107, a=a, b=b, c=c) elif expr.val == '/': code(108, a=a, b=b, c=c) return None items = orig.items orig.val = orig.val[:-1] t = Token(orig.pos,'symbol','=',[items[0],orig]) return fnc(t) def unary(i,tb,r=None): r = get_tmp(r) b = do(tb) code(i,r,b) if r != b: free_tmp(b) return r def infix(i,tb,tc,r=None): r = get_tmp(r) b = do(tb,r) c = do(tc) code(i,r,b,c) if r != b: free_tmp(b) free_tmp(c) return r def logic_infix(op, tb, tc, _r=None): t = get_tag() r = do(tb, _r) if _r != r: free_tmp(_r) #REG if op == 'and': code(IF, r) elif op == 'or': code(IFN, r) jump(t, 'end') _r = r r = do(tc, _r) if _r != r: free_tmp(_r) #REG tag(t, 'end') return r def _do_none(r=None): r = get_tmp(r) code(NONE,r) return r def do_symbol(t,r=None): sets = ['='] isets = ['+=','-=','*=','/=', '|=', '&=', '^='] cmps = ['<','>','<=','>=','==','!='] metas = { '+':ADD,'*':MUL,'/':DIV,'**':POW, '-':SUB, '%':MOD,'>>':RSH,'<<':LSH, '&':BITAND,'|':BITOR,'^':BITXOR, } if t.val == 'None': return _do_none(r) if t.val == 'True': return _do_number('1',r) if t.val == 'False': return _do_number('0',r) items = t.items if t.val in ['and','or']: return logic_infix(t.val, items[0], items[1], r) if t.val in isets: return imanage(t,do_symbol) if t.val == 'is': if items[1].type == 'symbol' and items[1].val == 'None': r = get_tmp(r) b = do(items[0], r) code(159, r, b) if r != b: free_tmp(b) return r else: return infix(EQ,items[0],items[1],r) if t.val == 'isnot': if items[1].type == 'symbol' and items[1].val == 'None': r = get_tmp(r) b = do(items[0], r) code(158, r, b) if r != b: free_tmp(b) return r else: return infix(CMP,items[0],items[1],r) if t.val == 'not': return unary(NOT, items[0], r) if t.val == 'in': return infix(HAS,items[1],items[0],r) if t.val == 'notin': r = infix(HAS,items[1],items[0],r) zero = _do_number('0') code(EQ,r,r,free_tmp(zero)) return r if t.val in sets: if items[0].type == 'name' and items[0].val in D.globals and len(items[0].val)==1: a = items[0] expr = items[1] if expr.type == 'symbol' and expr.val in '+-/*': b = expr.items[0] c = expr.items[1] if b.type == 'name' and c.type == 'name' and len(b.val)==1 and len(c.val)==1: if b.val in D.globals and c.val in D.globals: if expr.val == '+': code(68, a=ord(a.val), b=ord(b.val), c=ord(c.val)) elif expr.val == '-': code(69, a=ord(a.val), b=ord(b.val), c=ord(c.val)) elif expr.val == '*': code(70, a=ord(a.val), b=ord(b.val), c=ord(c.val)) elif expr.val == '/': code(71, a=ord(a.val), b=ord(b.val), c=ord(c.val)) return None if D.function_stack and t.val == '=' and r is None: if items[0].type == 'name' and items[0].val not in D.globals: a = items[0] expr = items[1] if expr.type == 'symbol' and expr.val == '+': b = expr.items[0] c = expr.items[1] ## is this really safe? not really any speed up if b.type=='name' and b.val not in D.globals and c.type=='number' and c.val.isdigit(): num = int(c.val) if num > 0 and num < 256: #r = get_reg(a) ## crashes r = do_local(a,r) if D.function_stack and b.val in D.function_stack[-1]['args']: code(190, a=r, b=D.function_stack[-1]['args'].index(b.val), c=num) else: code(190, a=r, b=get_reg(b.val), c=num) return r return do_set_ctx(items[0],items[1]); elif t.val in cmps: b,c = items[0],items[1] v = t.val if v[0] in ('>','>='): b,c,v = c,b,'<'+v[1:] cd = EQ if v == '<': cd = LT if v == '<=': cd = LE if v == '!=': cd = NE return infix(cd,b,c,r) else: # if t.val in D.vars: #return do_local(t,r) a = items[0] b = items[1] if a.type == 'name' and a.val in D.globals and len(a.val)==1: if b.type == 'name' and b.val in D.globals and len(b.val)==1: print('global fast op return reg=', r) if r is None: r = 0 if t.val == '+': code(64, a=r, b=ord(a.val), c=ord(b.val)) return r elif t.val == '-': code(65, a=r, b=ord(a.val), c=ord(b.val)) return r elif t.val == '*': code(66, a=r, b=ord(a.val), c=ord(b.val)) return r elif t.val == '/': code(67, a=r, b=ord(a.val), c=ord(b.val)) return r if a.type=='name' and a.val not in D.globals and r: if b.type=='number' and str(b.val).isdigit(): num = int(b.val) if num > 0 and num < 256: if t.val == '+': ## is this really safe? not really any speed up if D.function_stack and a.val in D.function_stack[-1]['args']: code(190, a=r, b=D.function_stack[-1]['args'].index(a.val), c=num) else: code(190, a=r, b=get_reg(a.val), c=num) return r return infix(metas[t.val],items[0],items[1],r) def do_set_ctx(k,v): if k.type == 'name': if (D._globals and k.val not in D.vars) or (k.val in D.globals) or (k.val in D.globals256): if k.val in D.globals256: #print("Global256: ", k.val, " index=", D.globals256.index(k.val), ' stack-level=', len(D.stack)) b = do(v) code(156, D.globals256.index(k.val), b ) free_tmp(b) elif len(D.stack)==1 and len(D.globals256)<256: D.globals256.append(k.val) print("New Global256: ", k.val, " index=", D.globals256.index(k.val), ' stack-level=', len(D.stack)) print(v) print(v.type) print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') if v.type=='dict': ## global dicts are not garbage collected b = do_dict(v, is_global=True) elif v.type=='list': ## global lists are not garbage collected b = do_list(v, is_global=True) elif v.type=='comp': ## global lists are not garbage collected b = do_comp(v, is_global=True) else: b = do(v) code(156, D.globals256.index(k.val), b ) free_tmp(b) else: #print("Global: ", k.val, ' stack-level=', len(D.stack)) c = do_string(k) b = do(v) code(GSET,c,b) free_tmp(c) free_tmp(b) return #print("Local: ", k.val, ' stack-level=', len(D.stack)) a = do_local(k) b = do(v) code(MOVE,a,b) free_tmp(b) return a elif k.type in ('tuple','list'): if v.type in ('tuple','list'): n,tmps = 0,[] for kk in k.items: vv = v.items[n] tmp = get_tmp(); tmps.append(tmp) r = do(vv) code(MOVE,tmp,r) free_tmp(r) #REG n+=1 n = 0 for kk in k.items: vv = v.items[n] tmp = tmps[n] free_tmp(do_set_ctx(kk,Token(vv.pos,'reg',tmp))) #REG n += 1 return r = do(v); un_tmp(r) n, tmp = 0, Token(v.pos,'reg',r) for tt in k.items: free_tmp(do_set_ctx(tt,Token(tmp.pos,'get',None,[tmp,Token(tmp.pos,'number',str(n))]))) #REG n += 1 free_reg(r) return r = do(k.items[0]) rr = do(v) tmp = do(k.items[1]) code(SET,r,tmp,rr) free_tmp(r) #REG free_tmp(tmp) #REG return rr def manage_seq(i,a,items,sav=0): l = max(sav,len(items)) n = 0 tmps = get_tmps(l) for tt in items: r = tmps[n] b = do(tt,r) if r != b: code(MOVE,r,b) free_tmp(b) n +=1 if not len(tmps): code(i,a,0,0) return 0 code(i,a,tmps[0],len(items)) free_tmps(tmps[sav:]) return tmps[0] def p_filter(items): a,b,c,d = [],[],None,None for t in items: if t.type == 'symbol' and t.val == '=': b.append(t) elif t.type == 'args': c = t elif t.type == 'nargs': d = t else: a.append(t) return a,b,c,d def do_import(t): if len(t.items) == 1: mod = t.items[0] name = mod else: mod, name = t.items if mod.val not in D.imports: D.imports.append(mod.val) mod.type = 'string' mod.no_intern = True v = do_call(Token(t.pos,'call',None,[ Token(t.pos,'name','__import__'), mod, Token(t.pos, 'symbol', 'None')] )) name.type = 'name' do_set_ctx(name,Token(t.pos,'reg',v)) def do_from(t): mod = t.items[0] items = t.items[1] mod.type = 'string' if items.type == 'args': # it really shouldn't be args -- need to fix the parser. if items.val == '*': items.type = 'string' elif items.type == 'name': items.type = 'string' items = Token(items.pos, 'tuple', None, [items]) elif items.type == 'tuple': for item in items.items: item.type = 'string' else: u_error('SyntaxError', D.code, t.pos) v = do(Token(t.pos,'call',None,[ Token(t.pos,'name','__import__'), mod, items, ])) un_tmp(v) free_tmp(do(Token(t.pos, 'call', None, [ Token(t.pos, 'name', '__merge__'), Token(t.pos, 'name', '__dict__'), Token(t.pos, 'reg', v) #REG ] ))) free_reg(v) def do_globals(t): for t in t.items: if t.val not in D.globals: D.globals.append(t.val) if len(t.val)>1 and t.val not in D.globals256 and len(D.globals256) < 256: D.globals256.append(t.val) def do_del(tt): for t in tt.items: if t.type=='name': r = do(t) code(DEL, r) free_tmp(r) else: ## assume dictionary `del foo['bar']` r = do(t.items[0]) r2 = do(t.items[1]) code(DEL,r,r2) free_tmp(r) free_tmp(r2) def do_call(t,r=None): assert t.type == 'call' if DEBUG_ENCODER: print(t) ## fast printing optimization if t.items[0].type == 'name' and t.items[0].val == 'print': if len(t.items[1:]) >= 2 and t.items[-1].type == 'symbol' and t.items[-1].val == '=': kwargs = t.items[-1] if kwargs.items[0].type=='name' and kwargs.items[0].val == 'end': if kwargs.items[1].type=='string' and kwargs.items[1].val == '': if t.items[1].type == 'string': if len(t.items[1].val)==1: code(60, a=ord(t.items[1].val)) return None ## TODO fix unicode bugs, and test compiler with Python3 elif len(t.items[1].val)==2 and t.items[1].val == '·': ## middle dot with bad encoding #code(60, a=ord(u'·')) ## code 183, is printed as `�` code(60, a=ord('.')) ## workaround change to a regular dot return None r = get_tmp(r) items = t.items a,b,c,d = p_filter(t.items[1:]) ## special case for directly calling `MyClass.__init__(self, ...)` from a subclass if t.items[0].type == 'mget' and t.items[0].items[0].type=='name' and t.items[0].items[0].val in D.classes: if t.items[0].items[1].type=='string' and t.items[0].items[1].val=='__init__': #func_name = t.items[0].items[1].val ## this is __init__ but actually we are cheating and calling the class like a regular function call func_name = t.items[0].items[0].val if not b and c is None and d is None: if func_name in D.globals256_funcs: ## global fast call - bypasses paramlist ## if DEBUG_ENCODER: print(orange('doing fast call MyClass.__init__(self,...)')) args = D.globals256_funcs[func_name] assert len(args)==len(a) assert len(args) >= 1 ## because `self` must be passed as the first arg in this case tmpargs = get_tmps( len(a) ) for i, tmpreg in enumerate(tmpargs): bb = do(a[i], tmpreg) if bb != tmpreg: code(MOVE,tmpreg, bb) free_tmp(bb) ## this trick shoves the arg count into the byte code type header itself (200+numargs) ## the last register (RC) is set to the first reg addr, because get_tmps is a range, this hack works. code( 200+len(args), r, D.globals256.index(func_name), tmpargs[0] ) for tmp in tmpargs: assert tmp != 0 free_tmp(tmp) return r elif t.items[0].type == 'mget' and t.items[0].items[0].type=='name' and not b and not c and not d: #ALWAYS SAFE#if t.items[0].items[1].type=='string' and t.items[0].items[0].val !='self': if t.items[0].items[1].type=='string' and t.items[0].items[0].val: ## some variable.foo(...), check if this is a fast callable method methname = t.items[0].items[1].val classinfo = None methinfo = None hits = 0 can_fast_call = 0 for cname in D.classes: cinfo = D.classes[cname] if methname in cinfo['methods'] and cinfo['methods'][methname]: classinfo = cinfo methinfo = cinfo['methods'][methname] hits += 1 if methinfo['can_fast_call']: can_fast_call += 1 if hits == 1 and methinfo['can_fast_call'] and not D.in_return: fast_meth_name = '%s::%s' % (classinfo['name'], methinfo['name']) #fast_meth_name = '__%s__' % methinfo['name'] if len(D.globals256) < 256 and len(D.globals256_funcs) < 256 and methinfo['global256'] is None: assert fast_meth_name not in D.globals256 assert fast_meth_name not in D.globals256_funcs D.globals256.append(fast_meth_name) D.globals256_funcs[fast_meth_name] = methinfo['args'] methinfo['global256'] = D.globals256.index(fast_meth_name) D.fast_meth_defs[ fast_meth_name ] = methinfo['token'] if methinfo['global256']: if DEBUG_ENCODER: print(orange('doing fast method call of type %s and %s(...)' %(classinfo['name'], methinfo['name']))) func_name = fast_meth_name args = D.globals256_funcs[func_name] assert len(args)-1 == len(a) ## need to insert the mget named var here, because it is `self` a.insert(0, t.items[0].items[0]) tmpargs = get_tmps( len(a) ) for i, tmpreg in enumerate(tmpargs): bb = do(a[i], tmpreg) if bb != tmpreg: code(MOVE,tmpreg, bb) free_tmp(bb) code( 200+len(args), r, D.globals256.index(func_name), tmpargs[0] ) for tmp in tmpargs: assert tmp != 0 free_tmp(tmp) return r elif hits and hits==can_fast_call and False: ## not safe yet if DEBUG_ENCODER: print(yellow('doing method call of unknown class type %s(...)' %methinfo['name'])) fnc = do(items[0]) ## need to insert the mget named var here, because it is `self` a.insert(0, t.items[0].items[0]) tmpargs = get_tmps( len(a) ) for i, tmpreg in enumerate(tmpargs): bb = do(a[i], tmpreg) if bb != tmpreg: code(MOVE,tmpreg, bb) free_tmp(bb) code( 180+len(a), r, fnc, tmpargs[0] ) for tmp in tmpargs: assert tmp != 0 free_tmp(tmp) free_tmp(fnc) #REG return r elif t.items[0].type == 'name' and t.items[0].val in D.globals256: if t.items[0].val in D.classes: class_name = t.items[0].val if DEBUG_ENCODER: print(orange('class `%s` in globals256 and in D.classes' %class_name)) if not b and c is None and d is None: if class_name in D.globals256_funcs: ## global fast call - bypasses paramlist ## args = D.globals256_funcs[class_name] if DEBUG_ENCODER: print(orange('class name in globals256_funcs')) print(a) print(args) assert len(args)-1==len(a) ## because the first item of `args` is `self` if not len(a): code( 232, r, D.globals256.index(class_name) ) else: tmpargs = get_tmps( len(a) ) if DEBUG_ENCODER: print(orange('temp regs:'), tmpargs) for i, tmpreg in enumerate(tmpargs): bb = do(a[i], tmpreg) if bb != tmpreg: code(MOVE,tmpreg, bb) free_tmp(bb) ## this trick shoves the arg count into the byte code type header itself (200+numargs) ## the last register (RC) is set to the first reg addr, because get_tmps is a range, this hack works. code( 232+len(args), r, D.globals256.index(class_name), tmpargs[0] ) for tmp in tmpargs: assert tmp != 0 free_tmp(tmp) return r else: func_name = t.items[0].val if DEBUG_ENCODER: print(orange('calling function `%s` in globals256' %func_name)) if not b and c is None and d is None: if func_name in D.globals256_funcs: ## global fast call - bypasses paramlist ## if DEBUG_ENCODER: print(orange('doing fast call')) print(red(func_name)) args = D.globals256_funcs[func_name] assert len(args)==len(a) if len(args) <= 1: if not args: code(200, r, D.globals256.index(t.items[0].val) ) else: t1 = do(a[0]) code(201, r, D.globals256.index(t.items[0].val), t1 ) free_tmp(t1) return r else: tmpargs = get_tmps( len(a) ) for i, tmpreg in enumerate(tmpargs): bb = do(a[i], tmpreg) if bb != tmpreg: code(MOVE,tmpreg, bb) free_tmp(bb) ## this trick shoves the arg count into the byte code type header itself (200+numargs) ## the last register (RC) is set to the first reg addr, because get_tmps is a range, this hack works. code( 200+len(args), r, D.globals256.index(t.items[0].val), tmpargs[0] ) for tmp in tmpargs: assert tmp != 0 free_tmp(tmp) return r ### this is not safe and a very rare case like `foo=getattr(ob,'bar')` #else: # ## global fast call ## # if len(a)==1: # t1 = do(a[0]) # code(200, r, D.globals256.index(t.items[0].val), t1 ) # free_tmp(t1) # return r if DEBUG_ENCODER: print(yellow('---doing regular call---')) print( items[0] ) print(yellow('------------------------')) fnc = do(items[0]) e = None if len(b) != 0 or d != None: e = do(Token(t.pos,'dict',None,[])); un_tmp(e); for p in b: p.items[0].type = 'string' t1 = do(p.items[0]) t2 = do(p.items[1]) code(SET,e,t1,t2) free_tmp(t1) free_tmp(t2) if d: free_tmp(do( Token(t.pos,'call',None, [ Token(t.pos,'name', '__merge__'), Token(t.pos,'reg',e), d.items[0] ]))) #REG manage_seq(PARAMS,r,a) if c != None: t1 = _do_string('*') t2 = do(c.items[0]) code(SET,r,t1,t2) free_tmp(t1); free_tmp(t2) #REG if e != None: t1 = _do_none() code(SET,r,t1,e) free_tmp(t1) #REG code(CALL,r,fnc,r) free_tmp(fnc) #REG return r def do_name(t,r=None): if t.val in D.vars: return do_local(t,r) if t.val not in D.rglobals: D.rglobals.append(t.val) r = get_tmp(r) if t.val in D.globals256: code(157, r, D.globals256.index(t.val) ) else: c = do_string(t) code(GGET,r,c) free_tmp(c) return r def do_local(t,r=None): if t.val in D.rglobals: D.error = True u_error('UnboundLocalError',D.code,t.pos) if t.val not in D.vars: D.vars.append(t.val) return get_reg(t.val) def do_fast_meth_def(tok, meth_name): items = tok.items t = get_tag() rf = fnc(t,'end') D.begin() setpos(tok.pos) ## note: this is optimized away when not in debug mode a,b,c,d = p_filter(items[1].items) for p in a: v = do_local(p) ## do function body ## free_tmp(do(items[2])) #REG D.end() tag(t,'end') #if D._globals: do_globals(Token(tok.pos,0,0,[items[0]])) #r = do_set_ctx(items[0],Token(tok.pos,'reg',rf)) code(156, D.globals256.index(meth_name), rf ) free_tmp(rf) def do_def(tok,kls=None): items = tok.items is_global256 = False func_name = items[0].val if len(D.stack)==1 and len(D.globals256)<256 and func_name not in D.globals256 and not D.class_stack: ## allows recursive functions ## D.globals256.append(func_name) is_global256 = True D.globals256_funcs[func_name] = [] t = get_tag() rf = fnc(t,'end') D.begin() setpos(tok.pos) ## note: this is optimized away when not in debug mode a,b,c,d = p_filter(items[1].items) methinfo = None if D.class_stack: classinfo = D.classes[ D.class_stack[-1] ] assert func_name not in classinfo['methods'] methinfo = { 'name' : func_name, 'args' : a, 'returns' : None, 'token' : tok, 'is_abstract' : None, 'is_polymorphic' : False, ## is overridden by a subclass 'can_fast_call' : not b and not c and not d, 'global256' : None, ## optional secondary generated fast call function } classinfo['methods'][ func_name ] = methinfo D.function_stack.append({'name':func_name, 'args':[]}) for arg in a: D.function_stack[-1]['args'].append(arg.val) if is_global256 and not b and not c and not d and len(a) <= 29: assert not kls ## super fast global function call, bypassing the params list for i,p in enumerate(a): D.globals256_funcs[func_name].append(p.val) v = do_local(p) set_reg(i, p.val) ## this should have fixed the get reg problem for function args ??? elif func_name=="__init__" and len(D.class_stack)==1 and D.class_stack[-1] in D.globals256 and not b and not c and not d and len(a) <= 29: class_name = D.class_stack[-1] assert class_name not in D.globals256_funcs D.globals256_funcs[class_name] = [] for p in a: D.globals256_funcs[class_name].append(p.val) v = do_local(p) elif methinfo and methinfo['can_fast_call'] and False: ## not safe yet for p in a: v = do_local(p) else: ## this is a clever trick by Hassey, because __params is the first local variable, ## it is known to be at register zero, at runtime in tp_ops.pyc++ - the tp_call ## function shoves the param list into register zero just after calling tp_enter_frame, ## and just before tp_run_frame - which calls the bytecode compiled function. r = do_local(Token(tok.pos,'name','__params')) ## using the same technique as above and a special bytecode, the param list can ## be bypassed entirely, and instead the function args can be directly set in the ## function registers. do_info(items[0].val) ## note: this is optimized away when not in debug mode for p in a: v = do_local(p) ## old style params are slightly slower, but has better calling support, unlimited arg count, keyword args, etc. if '--old-style-params' in sys.argv: tmp = _do_none() code(GET,v,r,tmp) free_tmp(tmp) #REG else: code(NEXT,v,r) for p in b: v = do_local(p.items[0]) do(p.items[1],v) tmp = _do_none() code(IGET,v,r,tmp) free_tmp(tmp) #REG if c != None: v = do_local(c.items[0]) tmp = _do_string('*') code(GET,v,r,tmp) free_tmp(tmp) #REG if d != None: e = do_local(d.items[0]) code(DICT,e,0,0) tmp = _do_none() code(IGET,e,r,tmp) free_tmp(tmp) #REG ## do function body ## free_tmp(do(items[2])) #REG D.end() tag(t,'end') D.function_stack.pop() rn = do_string(items[0]) code(NAME,rf, rn) if kls == None: if D._globals: do_globals(Token(tok.pos,0,0,[items[0]])) r = do_set_ctx(items[0],Token(tok.pos,'reg',rf)) else: code(SET,kls,rn,rf) free_tmp(rn) free_tmp(rf) def do_class(t): tok = t items = t.items parent = None set_base = False if items[0].type == 'name': name = items[0].val parent = Token(tok.pos,'name','object') else: name = items[0].items[0].val parent = items[0].items[1] set_base = True if DEBUG_ENCODER: print('parent class: ', parent) D.class_stack.append(name) classinfo = {'name':name,'vars':[], 'methods':{}, 'parent': parent, 'children':[]} if name not in D.classes: D.classes[name] = classinfo else: raise SyntaxError('class is already defined') ##kls = do_classdecl() ##un_tmp(kls) kls = get_tmp() ts = _do_string(name) if set_base and parent.val in D.globals256: code(177, kls, ts, D.globals256.index(parent.val) ) else: code(IFACE, kls, ts ) print("New Class: ", name, " stack-level=", len(D.stack)) if len(D.stack)==1 and len(D.globals256) < 256: assert name not in D.globals256 D.globals256.append(name) code(156, D.globals256.index(name), kls ) else: #ts = _do_string(name) code(GSET,ts,kls) #free_tmp(ts) #REG free_tmp(ts) ##free_tmp(do(Token(tok.pos,'call',None,[ ## Token(tok.pos,'name','setmeta'), ## Token(tok.pos,'reg',kls), ## parent]))) for member in items[1].items: if member.type == 'def': do_def(member,kls) elif member.type == 'symbol' and member.val == '=': do_classvar(member,kls) else: continue D.class_stack.pop() free_reg(kls) #REG def do_classdecl(): r = get_tmp(None) manage_seq(IFACE, r, []) return r def do_classvar(t,r): var = do_string(t.items[0]) val = do(t.items[1]) code(SET,r,var,val) free_reg(var) free_reg(val) ## hartsantler #def do_loop_old_hack(t): # LOOP = int(t.val) + 100 # code(LOOP) # items = t.items # t = stack_tag() # body = do(items[0]) # pop_tag() def do_loop(t): #s = stack_tag() items = t.items #LOOP_NUM = 99 #code(LOOP_NUM, int(t.val)) ## this works, just read from VA #LOOP = 100 #code(LOOP) LOOP = 100 #code(LOOP, int(t.val)) ## this still wont work with large loops, the second arg is register A, and that can only be 8bits #code_16(LOOP, a=0, b=int(t.val)) ## not sure how this works r = do_number(t.num_loops) code(LOOP, r) free_tmp(r) #t = stack_tag() #tag(t,'begin') #raise RuntimeError(items) #t = stack_tag() body = do(items[0]) #free_tmp(body) #tag(t,'end') #pop_tag() def do_while(t): items = t.items t = stack_tag() tag(t,'begin') tag(t,'continue') cond = items[0] old_style = True if cond.type == 'symbol': if cond.val == '<': if cond.items[0].type == 'name' and cond.items[1].type == 'number' and cond.items[0].val not in D.globals: const_addr = D.const_number(cond.items[1].val) if const_addr != -1: if items[1].type == 'statements': old_style = False a, b = cond.items ## look ahead to see if the next operation is to increment a.val by 1 do_increment = 0 do_slower_method = True nextop = items[1].items[0] if nextop.type == 'symbol' and nextop.val == '+=': if nextop.items[0].type == 'name' and nextop.items[1].type == 'number' and nextop.items[1].val.isdigit(): incby = int(nextop.items[1].val) if nextop.items[0].val == a.val and incby >= 1 and incby <= 255: items[1].items = items[1].items[1:] ## remove the next op if incby == 1: ## this is optimized more because of `i++` code(80, a=get_reg(a.val), b=const_addr) do_slower_method = False else: do_increment = incby if do_slower_method: code(81, a=get_reg(a.val), b=const_addr, c=do_increment) elif cond.items[0].type == 'name' and cond.items[1].type == 'name': a,b = cond.items if len(b.val)==1 and b.val in D.globals: if len(a.val)==1 and a.val in D.globals: old_style = False code(82, a=ord(a.val), b=ord(b.val)) elif a.val not in D.globals: old_style = False code(83, a=get_reg(a.val), b=ord(b.val)) if old_style: r = do(items[0]) code(IF,r) free_tmp(r) #REG jump(t,'end') free_tmp(do(items[1])) #REG jump(t,'begin') tag(t,'break') tag(t,'end') pop_tag() def do_for(tok): items = tok.items reg = get_tmp() itr = do(items[1]) i = _do_number('0') t = stack_tag(); tag(t,'loop'); tag(t,'continue') code(ITER,reg,itr,i); jump(t,'end') free_tmp(do_set_ctx(items[0], Token(tok.pos, 'reg', reg))) free_tmp(do(items[2])) #REG jump(t,'loop') tag(t,'break'); tag(t,'end'); pop_tag() free_tmp(itr) #REG free_tmp(i) def do_comp(t,r=None, is_global=False): name = 'comp:'+get_tag() r = do_local(Token(t.pos,'name',name)) if is_global: code(151,r,0,0) else: code(LIST,r,0,0) key = Token(t.pos,'get',None,[ Token(t.pos,'reg',r), Token(t.pos,'symbol','None')]) ap = Token(t.pos,'symbol','=',[key,t.items[0]]) do(Token(t.pos,'for',None,[t.items[1],t.items[2],ap])) return r def do_if(t): items = t.items #if len(items)==1: #if 'None' in str(items): # raise RuntimeError(items) t = get_tag() n = 0 for tt in items: tag(t,n) if tt.type == 'elif': slow_if = True if tt.items[0].type == 'symbol' and tt.items[0].val == '<=': opa, opb = tt.items[0].items if opa.type == 'symbol' and opa.val == '+': opaa, opab = opa.items if opaa.type=='name' and opab.type=='name': if len(opaa.val)==1 and opaa.val in D.globals: if len(opab.val)==1 and opab.val in D.globals: if opb.type == 'number' and opb.val.isdigit(): num = int(opb.val) if num >= 0 and num <= 255: slow_if = False code(128, a=ord(opaa.val), b=ord(opab.val), c=num) if slow_if: a = do(tt.items[0]) code(IF,a) free_tmp(a); jump(t,n+1) free_tmp(do(tt.items[1])) #REG elif tt.type == 'else': free_tmp(do(tt.items[0])) #REG else: raise jump(t,'end') n += 1 tag(t,n) tag(t,'end') def do_try(t): items = t.items t = get_tag() setjmp(t,'except') free_tmp(do(items[0])) #REG code(SETJMP,0) jump(t,'end') tag(t,'except') free_tmp(do(items[1].items[1])) #REG tag(t,'end') def do_return(t): D.in_return = True if t.items: r = do(t.items[0]) else: r = _do_none() code(RETURN,r) free_tmp(r) D.in_return = False return def do_assert(t): if t.items: r = do(t.items[0]) else: r = _do_none() un_tmp(r) v = do_call(Token(t.pos,'call',None,[ Token(t.pos,'name','__assert__'), Token(t.pos, 'reg', r) #REG ] )) free_tmp(v) return def do_raise(t): if t.items: r = do(t.items[0]) ## special case for methods in a class, if it raises NotImplementedError, then it is abstract and polymorphic if t.items[0].type == 'name' and t.items[0].val == 'NotImplementedError': if D.class_stack and D.function_stack: func_name = D.function_stack[-1]['name'] classinfo = D.classes[ D.class_stack[-1] ] if func_name in classinfo['methods']: methinfo = classinfo['methods'][func_name] #methinfo['can_fast_call'] = False methinfo['is_polymorphic'] = True methinfo['is_abstract'] = True else: r = _do_none() code(RAISE,r) free_tmp(r) return def do_statements(t): for tt in t.items: free_tmp(do(tt)) def do_list(t,r=None, is_global=False): r = get_tmp(r) if is_global: manage_seq(151,r,t.items) else: manage_seq(LIST,r,t.items) return r def do_dict(t,r=None, is_global=False): r = get_tmp(r) if is_global: manage_seq(150,r,t.items) else: manage_seq(DICT,r,t.items) return r def do_get(t,r=None): ## foo.bar or foo[bar] items = t.items return infix(GET,items[0],items[1],r) def do_mget(t,r=None): items = t.items return infix(MGET,items[0],items[1],r) def do_break(t): jump(D.tstack[-1],'break') def do_continue(t): jump(D.tstack[-1],'continue') def do_pass(t): code(PASS) ## DEPRECATED def do_info(name='?'): return if '--debug-calls' in sys.argv: code(NAME,free_tmp(_do_string(name))) elif D.nopos: return else: code(FILE,free_tmp(_do_string(D.fname))) code(NAME,free_tmp(_do_string(name))) def do_module(t): do_info() free_tmp(do(t.items[0])) #REG def do_reg(t,r=None): return t.val fmap = { 'loop': do_loop, ## hartsantler, sept21 'module':do_module,'statements':do_statements,'def':do_def, 'return':do_return, 'assert':do_assert, 'while':do_while,'if':do_if, 'break':do_break,'pass':do_pass,'continue':do_continue,'for':do_for, 'class':do_class,'raise':do_raise,'try':do_try,'import':do_import, 'globals':do_globals,'del':do_del,'from':do_from, } rmap = { 'list':do_list, 'tuple':do_list, 'dict':do_dict, 'slice':do_list, 'comp':do_comp, 'name':do_name,'symbol':do_symbol,'number':do_number, 'string':do_string,'get':do_get, 'mget':do_mget, 'call':do_call, 'reg':do_reg, } def do(t,r=None): if t.pos: setpos(t.pos) #try: if 1: if t.type in rmap: return rmap[t.type](t,r) #if r != None: free_reg(r) #REG return fmap[t.type](t) #except: # raise # if D.error: raise # D.error = True # tokenize.u_error('encode',D.code,t.pos) class encode: @classmethod def encode(cls, fname,s,t): t = Token((1,1),'module','module',[t]) global D s = tokenize().clean(s) D = DState(s,fname) D.begin(True) do(t) #D.end() maincodes = list(D.out) D.out = [] finalcodes = [ maincodes[0] ] if D.fast_meth_defs: print( yellow('======= doing fast meth defs - first pass =======') ) mdefs1 = list( D.fast_meth_defs.keys() ) print(mdefs1) for meth_name in mdefs1: tok = D.fast_meth_defs[meth_name] do_fast_meth_def(tok, meth_name) print( yellow('======= doing fast meth defs - second pass =======') ) mdefs2 = list( D.fast_meth_defs.keys() ) for meth_name in mdefs2: if meth_name in mdefs1: continue print(meth_name) tok = D.fast_meth_defs[meth_name] do_fast_meth_def(tok, meth_name) for code in D.out: finalcodes.append(code) finalcodes.extend( maincodes[1:] ) D.out = finalcodes D.end() if '--inspect-bytecode' in sys.argv: for chunk in D.out: print(chunk) map_tags() out = [] for part in D.out: if type(part) is str: out.append(part.encode('latin1')) else: out.append(part) print(green("encode to bytecodes complete")) print( purple("D.strings256:") ) print(D.strings256) D = None # Use a function instead of ''.join() so that bytes and # strings during bootstrap if '--inspect-bytecode' in sys.argv: print(out) return b''.join(out) ## from parse.py ## def check(t,*vs): if vs[0] == None: return True if t.type in vs: return True if t.type == 'symbol' and t.val in vs: return True return False def tweak(k,v): P.stack.append((k,dmap[k])) if v: dmap[k] = omap[k] else: dmap[k] = {'lbp':0,'nud':itself} def restore(): k,v = P.stack.pop() dmap[k] = v def cpy(d): r = {} for k in d: r[k] = d[k] return r class PData: def __init__(self,s,tokens): self.s = s self.tokens = tokens self.pos = 0 self.token = None self.stack = [] self._terminal = 0 self.indent = 0 self.lastline = '' def init(self): global omap,dmap omap = cpy(base_dmap) dmap = cpy(base_dmap) self.advance() ## advance to next string of val def advance(self,val=None): if val is not None and DEBUG_PARSER: if val == 'indent': self.indent += 1 print() print('\t'*self.indent, end='') self.lastline = '' elif val == 'dedent': self.indent -= 1 print() print('\t'*self.indent, end='') self.lastline = '' if not check(self.token,val): error('expected '+val,self.token) if self.pos < len(self.tokens): t = self.tokens[self.pos] self.pos += 1 else: t = Token((0,0),'eof','eof') self.token = parse.do(t) self._terminal += 1 if check(self.token,'nl','eof',';','dedent'): self._terminal = 0 return t def terminal(self): if self._terminal > 1: error('invalid statement',self.token) def error(ctx,t): if P is None: print(t) raise RuntimeError('parser context `P` is None') u_error(ctx,P.s,t.pos) def nud(t): #if 'nud' not in t: #error('no nud',t) return t.nud(t) def led(t,left): #if 'led' not in t: #error('no led',t) return t.led(t,left) def get_lbp(t): #if 'lbp' not in t: #error('no lbp',t) return t.lbp def get_items(t): #if 'items' not in t: #error('no items',t) return t.items def expression(rbp, left=None): if left is None: t = P.token advance() left = nud(t) while rbp < get_lbp(P.token): t = P.token advance() left = led(t,left) return left def infix_led(t,left): t.items = [left,expression(t.bp)] return t def infix_is(t,left): if check(P.token,'not'): t.val = 'isnot' advance('not') t.items = [left,expression(t.bp)] return t def infix_not(t,left): advance('in') t.val = 'notin' t.items = [left,expression(t.bp)] return t def infix_tuple(t,left): r = expression(t.bp) if left.val == ',': left.items.append(r) return left t.items = [left,r] t.type = 'tuple' return t def lst(t): if t == None: return [] if check(t,',','tuple','statements'): return get_items(t) return [t] def ilst(typ,t): return Token(t.pos,typ,typ,lst(t)) def call_led(t,left): r = Token(t.pos,'call','$',[left]) while not check(P.token,')'): tweak(',',0) r.items.append(expression(0)) if P.token.val == ',': advance(',') restore() advance(")") return r def get_led(t,left): r = Token(t.pos,'get','.',[left]) items = [left] more = False while not check(P.token,']'): more = False if check(P.token,':'): items.append(Token(P.token.pos,'symbol','None')) else: items.append(expression(0)) if check(P.token,':'): advance(':') more = True if more: items.append(Token(P.token.pos,'symbol','None')) if len(items) > 2: items = [left,Token(t.pos,'slice',':',items[1:])] r.items = items advance("]") return r def dot_led(t,left): r = expression(t.bp) r.type = 'string' t.items = [left,r] return t def itself(t): return t def paren_nud(t): tweak(',',1) r = expression(0) restore() advance(')') return r def list_nud(t): t.type = 'list' t.val = '[]' t.items = [] next = P.token tweak(',',0) while not check(P.token,'for',']'): r = expression(0) t.items.append(r) if P.token.val == ',': advance(',') if check(P.token,'for'): restore() t.type = 'comp' advance('for') tweak('in',0) tweak(',', 1) t.items.append(expression(0)) advance('in') t.items.append(expression(0)) restore() restore() advance(']') return t def dict_nud(t): t.type='dict' t.val = '{}' t.items = [] tweak(',',0) while not check(P.token,'}'): t.items.append(expression(0)) if check(P.token,':',','): advance() restore() advance('}') return t def advance(t=None): return P.advance(t) def iblock(items): while check(P.token,'nl',';'): advance() while True: items.append(expression(0)) P.terminal() while check(P.token,'nl',';'): advance() if check(P.token,'dedent','eof'): break def block(): items = [] tok = P.token if check(P.token,'nl'): while check(P.token,'nl'): advance() advance('indent') iblock(items) advance('dedent') else: items.append(expression(0)) while check(P.token,';'): advance(';') items.append(expression(0)) P.terminal() while check(P.token,'nl'): advance() if len(items) > 1: return Token(tok.pos,'statements',';',items) return items.pop() def def_nud(t): items = t.items = [] items.append(P.token); advance() advance('(') r = Token(t.pos,'symbol','():',[]) items.append(r) while not check(P.token,')'): tweak(',',0) r.items.append(expression(0)) if check(P.token,','): advance(',') restore() advance(')') advance(':') items.append(block()) return t def loop_nud(t): print(dir(t)) t.items = [] t.items.append(expression(0)) advance(':') print('type:', t.type) print('val:', t.val) print('items:', t.items) print('lbp:', t.lbp) print('nud:', t.nud) print('--------------') print(P.token) assert len(t.items)==1 #raise RuntimeError(t.items[0].val) t.val = t.items[0].val t.num_loops = t.items[0] t.items = [] t.items.append(block()) #advance(';') return t def while_nud(t): items = t.items = [] items.append(expression(0)) advance(':') items.append(block()) return t def class_nud(t): items = t.items = [] items.append(expression(0)) advance(':') items.append(ilst('methods',block())) return t def from_nud(t): items = t.items = [] # relative import s = '' while True: if check(P.token, 'mget'): s = s + '.' advance() elif check(P.token, 'name'): s = s + P.token.val advance() else: break expr = Token(t.pos, 'string', s) items.append(expr) advance('import') items.append(expression(0)) return t def import_nud(t): items = t.items = [] s = '' while True: if check(P.token, 'mget'): s = s + '.' advance() elif check(P.token, 'name'): s = s + P.token.val advance() else: break expr = Token(t.pos, 'string', s) items.append(expr) if check(P.token, 'as'): advance('as') items.append(expression(0)) return t def for_nud(t): items = t.items = [] tweak('in',0) items.append(expression(0)) advance('in') items.append(expression(0)) restore() advance(':') items.append(block()) return t def if_nud(t): items = t.items = [] a = expression(0) advance(':') b = block() items.append(Token(t.pos,'elif','elif',[a,b])) while check(P.token,'elif'): tok = P.token advance('elif') a = expression(0) advance(':') b = block() items.append(Token(tok.pos,'elif','elif',[a,b])) if check(P.token,'else'): tok = P.token advance('else') advance(':') b = block() items.append(Token(tok.pos,'else','else',[b])) return t def try_nud(t): items = t.items = [] advance(':') b = block() items.append(b) while check(P.token,'except'): tok = P.token advance('except') if not check(P.token,':'): a = expression(0) else: a = Token(tok.pos,'symbol','None') advance(':') b = block() items.append(Token(tok.pos,'except','except',[a,b])) #commenting this out, i don't think this next bit is valid syntax?? #if check(P.token,'else'): #tok = P.token #advance('else') #advance(':') #b = block() #items.append(Token(tok.pos,'else','else',[b])) return t def prefix_nud(t): #bp = 70 #if 'bp' in t: bp = t['bp'] bp = t.bp t.items = [expression(bp)] return t def prefix_nud0(t): if check(P.token,'nl',';','eof','dedent'): return t return prefix_nud(t) def prefix_nuds(t): r = expression(0) return ilst(t.type,r) def prefix_neg(t): r = expression(50) if r.type == 'number': r.val = str(-float(r.val)) return r t.items = [Token(t.pos,'number','0'),r] return t def vargs_nud(t): r = prefix_nud(t) t.type = 'args' t.val = '*' return t def nargs_nud(t): r = prefix_nud(t) t.type = 'nargs' t.val = '**' return t base_dmap = { ',':{'lbp':20,'bp':20,'led':infix_tuple}, '+':{'lbp':50,'bp':50,'led':infix_led}, '-':{'lbp':50,'nud':prefix_neg, 'bp':50,'led':infix_led}, 'not':{'lbp':35,'nud':prefix_nud,'bp':35, 'bp':35,'led':infix_not }, '%':{'lbp':60,'bp':60,'led':infix_led}, '*':{'lbp':60,'nud':vargs_nud, 'bp':60,'led':infix_led,}, '**': {'lbp':65,'nud':nargs_nud, 'bp':65,'led':infix_led,}, '/':{'lbp':60,'bp':60,'led':infix_led}, '(':{'lbp':70,'nud':paren_nud, 'bp':80,'led':call_led,}, '[':{'lbp':70,'nud':list_nud, 'bp':80,'led':get_led,}, '{':{'lbp':0,'nud':dict_nud,}, '.':{'lbp':80,'bp':80,'led':dot_led,'type':'mget',}, 'break':{'lbp':0,'nud':itself,'type':'break'}, 'pass':{'lbp':0,'nud':itself,'type':'pass'}, 'continue':{'lbp':0,'nud':itself,'type':'continue'}, 'eof':{'lbp':0,'type':'eof','val':'eof'}, 'def':{'lbp':0,'nud':def_nud,'type':'def',}, 'while':{'lbp':0,'nud':while_nud,'type':'while',}, 'for':{'lbp':0,'nud':for_nud,'type':'for',}, 'try':{'lbp':0,'nud':try_nud,'type':'try',}, 'if':{'lbp':0,'nud':if_nud,'type':'if',}, 'class':{'lbp':0,'nud':class_nud,'type':'class',}, 'raise':{'lbp':0,'nud':prefix_nud0,'type':'raise','bp':20,}, 'return':{'lbp':0,'nud':prefix_nud0,'type':'return','bp':10,}, 'assert':{'lbp':0,'nud':prefix_nud0,'type':'assert','bp':10,}, 'import':{'lbp':0,'nud':import_nud,'type':'import','bp':20,}, 'from':{'lbp':0,'nud':from_nud,'type':'from','bp':20,}, 'del':{'lbp':0,'nud':prefix_nuds,'type':'del','bp':10,}, 'global':{'lbp':0,'nud':prefix_nuds,'type':'globals','bp':20,}, '=':{ 'lbp':10,'bp':9,'led':infix_led, }, ## hartsantler tests 'loop':{'lbp':0,'nud':loop_nud,'type':'loop',}, } def i_infix(bp,led,*vs): for v in vs: base_dmap[v] = {'lbp':bp,'bp':bp,'led':led} i_infix(40,infix_led,'<','>','<=','>=','!=','==') i_infix(40,infix_is,'is','in') i_infix(10,infix_led,'+=','-=','*=','/=', '&=', '|=', '^=') i_infix(32,infix_led,'and','&') i_infix(31,infix_led,'^') i_infix(30,infix_led,'or','|') i_infix(36,infix_led,'<<','>>') def i_terms(*vs): for v in vs: base_dmap[v] = {'lbp':0,'nud':itself} i_terms('as', ')','}',']',';',':','nl','elif','else','True','False','None','name','string','number','indent','dedent','except') base_dmap['nl']['val'] = 'nl' def gmap(t,v): if v not in dmap: print(dmap.keys()) print(t) raise RuntimeError(v) error('unknown "%s"'%v,t) return dmap[v] class parse: @classmethod def parse(cls, s,tokens,wrap=0): global P s = tokenize().clean(s) P=PData(s,tokens) P.init() r = parse.do_module() P = None return r @classmethod def do(cls, t): if DEBUG_PARSER: info = str(t) if info: if info.startswith('\n'): print(info, end='') else: print(info, ' ', end='') P.lastline += info + ' ' if t.type == 'symbol': r = gmap(t, t.val) else: r = gmap(t, t.type) merge(t, r) return t @classmethod def do_module(cls): tok = P.token items = [] iblock(items) advance('eof') if len(items) > 1: return Token(tok.pos,'statements',';',items) return items.pop() ## from tokenize.py ## cNone='\033[00m' cRed='\033[01;31m' cGreen='\033[01;32m' cBlue='\033[01;34m' cYellow='\033[01;33m' cPurple='\033[01;35m' cCyan='\033[01;36m' cWhite='\033[01;37m' cBold='\033[1m' cUnderline='\033[4m' cOrange = '\033[00;33m' def red( s ): return cRed + str(s) + cNone def green( s ): return cGreen + str(s) + cNone def blue( s ): return cBlue + str(s) + cNone def orange( s ): return cOrange + str(s) + cNone def yellow( s ): return cYellow + str(s) + cNone def purple( s ): return cPurple + str(s) + cNone def cyan( s ): return cCyan + str(s) + cNone def white( s ): return cWhite + str(s) + cNone def bold( s ): return cBold + str(s) + cNone def underline( s ): return cUnderline + str(s) + cNone class Token: ## debug printing is done in class parse.do def __init__(self,pos=(0,0),type='symbol',val=None,items=None): self.pos = pos self.type = type self.val = val self.items = items def _format(self, full_debug_mode=True): prefix = '' #if full_debug_mode and hasattr(self, 'nud') and self.nud: # prefix = '<%s>' % self.nud.func_name if self.items is not None and self.val is not None: if self.val in '+-/*<>': return prefix + red('(') + self.val.join([i._format() for i in self.items]) + red(')') elif len(self.val) == 2 and self.val[1]=='=': return prefix + self.val.join([i._format() for i in self.items]) else: prefix += str(self.val) return prefix + red('[') + ', '.join([i._format() for i in self.items]) + red(']') elif self.type == 'nl': #return '\n ## new line' P.lastline = '' return '\n' + ('\t' * P.indent) elif self.type == 'name': if self.val=='self': return underline(self.val) else: return green(self.val) elif self.type == 'symbol': if self.val in ('if', 'elif', 'else'): return purple(self.val) elif self.val =='def': return blue(self.val) elif self.val == 'class': return orange(self.val) elif self.val == 'return': return red(self.val) return cyan(self.val) elif self.type == 'number': return yellow(self.val) elif self.type == 'string': return bold("`" + self.val + "`") elif self.type in ('indent', 'dedent'): # indent printing is done in PData.advance return '' elif self.type == 'eof': return '\n' else: return prefix + red(self.type) + '<' + str(self.val) + '>' def __repr__(self): return self._format() def u_error(ctx,s,i): y,x = i line = s.split('\n')[y-1] p = '' if y < 10: p += ' ' if y < 100: p += ' ' r = p + str(y) + ": " + line + "\n" r += " "+" "*x+"^" +'\n' raise Exception('error: '+ctx+'\n'+r) ISYMBOLS = u'`-=[];,./~!@$%^&*()+{}:<>?|×÷' UNISYMBOLS = { u'×' : '*', u'÷' : '/', } SYMBOLS = [ 'def','class','yield','return','pass','and','or','not','in','as', 'with', 'import', 'is','while','break','for','continue','if','else','elif','try', 'except','raise','assert','True','False','None','global','del','from', '-','+','*','**','/','%','<<','>>', '-=','+=','*=','/=','=','==','!=','<','>', '|=', '&=', '^=', '<=','>=','[',']','{','}','(',')','.',':',',',';','&','|','!', '^' ] B_BEGIN,B_END = ['[','(','{'],[']',')','}'] class TData: def __init__(self): self.y,self.yi,self.nl = 1,0,True self.res,self.indent,self.braces = [],[0],0 def add(self,t,v): self.res.append(Token(self.f,t,v)) class tokenize: def tokenize(self, s): global T s = self.clean(s) T,i,l = TData(),0,len(s) try: return self.do_tokenize(s,i,l) except: u_error('tokenize',s,T.f) def clean(self, s): s = s.replace('\r\n','\n') s = s.replace('\r','\n') return s def do_tokenize(self, s,i,l): global T T.f = (T.y,i-T.yi+1) while i < l: c = s[i]; T.f = (T.y,i-T.yi+1) if T.nl: T.nl = False i = self.do_indent(s,i,l) elif c == '\n': i = self.do_nl(s,i,l) elif c in ISYMBOLS: i = self.do_symbol(s,i,l) #elif c in UNISYMBOLS: i = do_symbol(s,i,l) elif c >= '0' and c <= '9': i = self.do_number(s,i,l) elif (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_': i = self.do_name(s,i,l) elif c=='"' or c=="'": i = self.do_string(s,i,l) elif c=='#': i = self.do_comment(s,i,l) elif c == '\\' and s[i+1] == '\n': i += 2; T.y,T.yi = T.y+1,i elif c == ' ' or c == '\t': i += 1 else: u_error('tokenize',s,T.f) self.indent(0) r = T.res; T = None #for t in r: #print (t.pos,t.type,t.val) return r def do_nl(self, s,i,l): if not T.braces: T.add('nl',None) i,T.nl = i+1,True T.y,T.yi = T.y+1,i return i def do_indent(self, s,i,l): v = 0 while i T.indent[-1]: T.indent.append(v) T.add('indent',v) elif v < T.indent[-1]: n = T.indent.index(v) while len(T.indent) > n+1: v = T.indent.pop() T.add('dedent',v) def do_symbol(self, s,i,l): symbols = [] v,f,i = s[i],i,i+1 if v in UNISYMBOLS: symbols.append( UNISYMBOLS[v] ) elif v in SYMBOLS: symbols.append(v) while i '9') and (c < 'a' or c > 'f') and c != 'x': break v,i = v+c,i+1 if c == '.': v,i = v+c,i+1 while i '9': break v,i = v+c,i+1 T.add('number',v) return i def do_name(self, s,i,l): v,i =s[i],i+1 while i 'z') and (c < 'A' or c > 'Z') and (c < '0' or c > '9') and c != '_': break v,i = v+c,i+1 if v in SYMBOLS: T.add('symbol',v) else: T.add('name',v) return i def do_string(self, s,i,l): v,q,i = '',s[i],i+1 if (l-i) >= 5 and s[i] == q and s[i+1] == q: # """ i += 2 while i