import { parse as python3Parse, parser } from './python3'; import { printNode } from './printNode'; /** * Reset the lexer state after an error. Otherwise, parses after a failed parse can fail too. */ let yy = parser.yy as any; let oldParseError = yy.parseError; oldParseError = function (text: String, hash: any) { this.indents = [0]; this.indent = 0; this.dedents = 0; this.brackets_count = 0; oldParseError.call(this, text, hash); }; /** * This is the main interface for parsing code. * Call this instead of the `parse` method in python3.js. * If the `parse` method gets an error, all later calls will throw an error. * This method resets the state of the `parse` method so that doesn't happen. */ export function parse(program: string): Module { if (program.charCodeAt(0) === 65279) { // eliminate byte order mark program = program.slice(1); } // The parser is fussy about line endings, so remote // carriage returns and make sure we end with a newline. return python3Parse(program.replace(/\r/g, '') + '\n'); } export type SyntaxNode = | Module | Import | From | Decorator | Decorate | Def | Parameter | Assignment | Assert | Pass | Return | Yield | Raise | Continue | Break | Global | Nonlocal | If | Else | While | For | Try | With | Call | Index | Slice | Dot | IfExpr | CompFor | CompIf | Lambda | UnaryOperator | BinaryOperator | Starred | Tuple | ListExpr | SetExpr | DictExpr | Name | Literal | Class; /// Must be consistent with Jison (Bison) naming conventions interface JisonLocation { first_line: number; first_column: number; last_line: number; last_column: number; } export interface Location extends JisonLocation { path?: string; // useful for error messages and other tracking } export function locationString(loc: Location) { return `${loc.path}${loc.last_line}:${loc.first_column}-${loc.last_line}:${loc.last_column}`; } // loc2 is inside loc1 export function locationContains(loc1: Location, loc2: Location) { function contains(loc: Location, line: number, col: number) { return ( (loc.first_line < line || (loc.first_line === line && loc.first_column <= col)) && (line < loc.last_line || (line === loc.last_line && col <= loc.last_column)) ); } return ( contains(loc1, loc2.first_line, loc2.first_column) && contains(loc1, loc2.last_line, loc2.last_column) ); } export interface Locatable { location?: Location; cellId?: string; executionCount?: number; } const LocatableFields = ['location', 'cellId', 'executionCount']; export const MODULE = 'module'; export interface Module extends Locatable { type: typeof MODULE; code: SyntaxNode[]; } export const IMPORT = 'import'; export interface Import extends Locatable { type: typeof IMPORT; names: { path: string; name?: string; location: Location }[]; } export const FROM = 'from'; export interface From extends Locatable { type: typeof FROM; base: string; imports: { path: string; name: string; location: Location }[]; } export const DECORATOR = 'decorator'; export interface Decorator extends Locatable { type: typeof DECORATOR; decorator: string; args: SyntaxNode[]; } export const DECORATE = 'decorate'; export interface Decorate extends Locatable { type: typeof DECORATE; decorators: Decorator[]; def: SyntaxNode; } export const DEF = 'def'; export interface Def extends Locatable { type: typeof DEF; name: string; params: Parameter[]; code: SyntaxNode[]; } export const PARAMETER = 'parameter'; export interface Parameter extends Locatable { type: typeof PARAMETER; name: string; anno: SyntaxNode; default_value: SyntaxNode; star: boolean; starstar: boolean; } export const ASSIGN = 'assign'; export interface Assignment extends Locatable { type: typeof ASSIGN; op: string | undefined; // defined for augment e.g. += targets: SyntaxNode[]; sources: SyntaxNode[]; } export const ASSERT = 'assert'; export interface Assert extends Locatable { type: typeof ASSERT; cond: SyntaxNode; err: SyntaxNode; } export const PASS = 'pass'; export interface Pass extends Locatable { type: typeof PASS; } export const RETURN = 'return'; export interface Return extends Locatable { type: typeof RETURN; values: SyntaxNode[]; } export const YIELD = 'yield'; export interface Yield extends Locatable { type: typeof YIELD; value: SyntaxNode[]; from?: SyntaxNode; } export const RAISE = 'raise'; export interface Raise extends Locatable { type: typeof RAISE; err: SyntaxNode; } export const BREAK = 'break'; export interface Break extends Locatable { type: typeof BREAK; } export const CONTINUE = 'continue'; export interface Continue extends Locatable { type: typeof CONTINUE; } export const GLOBAL = 'global'; export interface Global extends Locatable { type: typeof GLOBAL; names: string[]; } export const NONLOCAL = 'nonlocal'; export interface Nonlocal extends Locatable { type: typeof NONLOCAL; names: string[]; } export const IF = 'if'; export interface If extends Locatable { type: typeof IF; cond: SyntaxNode; code: SyntaxNode[]; elif: { cond: SyntaxNode; code: SyntaxNode[] }[]; else: Else; } export const WHILE = 'while'; export interface While extends Locatable { type: typeof WHILE; cond: SyntaxNode; code: SyntaxNode[]; else: SyntaxNode[]; } export const ELSE = 'else'; export interface Else extends Locatable { type: typeof ELSE; code: SyntaxNode[]; } export const FOR = 'for'; export interface For extends Locatable { type: typeof FOR; target: SyntaxNode[]; iter: SyntaxNode[]; code: SyntaxNode[]; else?: SyntaxNode[]; decl_location: Location; } export const COMPFOR = 'comp_for'; export interface CompFor extends Locatable { type: typeof COMPFOR; for: SyntaxNode[]; in: SyntaxNode; } export const COMPIF = 'comp_if'; export interface CompIf extends Locatable { type: typeof COMPIF; test: SyntaxNode; } export const TRY = 'try'; export interface Try extends Locatable { type: typeof TRY; code: SyntaxNode[]; excepts: { cond: SyntaxNode; name: string; code: SyntaxNode[] }[]; else: SyntaxNode[]; finally: SyntaxNode[]; } export const WITH = 'with'; export interface With extends Locatable { type: typeof WITH; items: { with: SyntaxNode; as: SyntaxNode }[]; code: SyntaxNode[]; } export const CALL = 'call'; export interface Call extends Locatable { type: typeof CALL; func: SyntaxNode; args: Argument[]; } export const ARG = 'arg'; export interface Argument extends Locatable { type: typeof ARG; actual: SyntaxNode; keyword?: SyntaxNode; loop?: CompFor; varargs?: boolean; kwargs?: boolean; } export const INDEX = 'index'; export interface Index extends Locatable { type: typeof INDEX; value: SyntaxNode; args: SyntaxNode[]; } export const SLICE = 'slice'; export interface Slice extends Locatable { type: typeof SLICE; start?: SyntaxNode; stop?: SyntaxNode; step?: SyntaxNode; } export const DOT = 'dot'; export interface Dot extends Locatable { type: typeof DOT; value: SyntaxNode; name: string; } export const IFEXPR = 'ifexpr'; export interface IfExpr extends Locatable { type: typeof IFEXPR; test: SyntaxNode; then: SyntaxNode; else: SyntaxNode; } export const LAMBDA = 'lambda'; export interface Lambda extends Locatable { type: typeof LAMBDA; args: Parameter[]; code: SyntaxNode; } export const UNOP = 'unop'; export interface UnaryOperator extends Locatable { type: typeof UNOP; op: string; operand: SyntaxNode; } export const BINOP = 'binop'; export interface BinaryOperator extends Locatable { type: typeof BINOP; op: string; left: SyntaxNode; right: SyntaxNode; } export const STARRED = 'starred'; export interface Starred extends Locatable { type: typeof STARRED; value: SyntaxNode; } export const TUPLE = 'tuple'; export interface Tuple extends Locatable { type: typeof TUPLE; items: SyntaxNode[]; } export const LIST = 'list'; export interface ListExpr extends Locatable { type: typeof LIST; items: SyntaxNode[]; } export const SET = 'set'; export interface SetExpr extends Locatable { type: typeof SET; entries: SyntaxNode[]; comp_for?: SyntaxNode[]; } export const DICT = 'dict'; export interface DictExpr extends Locatable { type: typeof DICT; entries: { k: SyntaxNode; v: SyntaxNode }[]; comp_for?: SyntaxNode[]; } export const NAME = 'name'; export interface Name extends Locatable { type: typeof NAME; id: string; } export const LITERAL = 'literal'; export interface Literal extends Locatable { type: typeof LITERAL; value: any; } export const CLASS = 'class'; export interface Class extends Locatable { type: typeof CLASS; name: string; extends: SyntaxNode[]; code: SyntaxNode[]; } /* UTILITY FUNCTIONS */ /** * returns whether two syntax nodes are semantically equivalent */ export function isEquivalent(node1: SyntaxNode, node2: SyntaxNode): boolean { if (!node1 || !node2) { return node1 === node2; } return printNode(node1) === printNode(node2); } export function flatten(arrayArrays: T[][]): T[] { return [].concat(...arrayArrays); } /** * Listener for pre-order traversal of the parse tree. */ export interface WalkListener { /** * Called whenever a node is entered. */ onEnterNode?(node: SyntaxNode, ancestors: SyntaxNode[]): void; /** * Called whenever a node is exited. */ onExitNode?(node: SyntaxNode, ancestors: SyntaxNode[]): void; } /** * Preorder tree traversal with optional listener. */ export function walk( node: SyntaxNode, walkListener?: WalkListener ): SyntaxNode[] { return walkRecursive(node, [], walkListener); } /** * Recursive implementation of pre-order tree walk. */ // tslint:disable-next-line: max-func-body-length function walkRecursive( node: SyntaxNode, ancestors?: SyntaxNode[], walkListener?: WalkListener ): SyntaxNode[] { if (!node) { console.error('Node undefined. Ancestors:', ancestors); return []; } ancestors.push(node); if (walkListener && walkListener.onEnterNode) { walkListener.onEnterNode(node, ancestors); } let children: SyntaxNode[] = []; switch (node.type) { case MODULE: case DEF: case CLASS: children = node.code; break; case IF: children = [node.cond] .concat(node.code) .concat( node.elif ? flatten(node.elif.map(e => [e.cond].concat(e.code))) : [] ) .concat(node.else ? [node.else] : []); break; case ELSE: children = node.code; break; case WHILE: children = [node.cond].concat(node.code); break; case WITH: children = flatten(node.items.map(r => [r.with, r.as])).concat(node.code); break; case FOR: children = node.iter.concat(node.target).concat(node.code); break; case TRY: children = node.code .concat( flatten( (node.excepts || []).map(e => (e.cond ? [e.cond] : []).concat(e.code) ) ) ) .concat(node.else || []) .concat(node.finally || []); break; case DECORATE: children = [node.def]; break; case LAMBDA: children = [node.code]; break; case CALL: children = [node.func].concat(node.args.map(a => a.actual)); break; case IFEXPR: children = [node.test, node.then, node.else]; break; case COMPFOR: children = node.for.concat([node.in]); break; case UNOP: children = [node.operand]; break; case BINOP: children = [node.left, node.right]; break; case STARRED: children = [node.value]; break; case SET: children = node.entries.concat(node.comp_for ? node.comp_for : []); break; case LIST: children = node.items; break; case TUPLE: children = node.items; break; case DICT: children = flatten(node.entries.map(p => [p.k, p.v])).concat( node.comp_for ? node.comp_for : [] ); break; case ASSIGN: if (!node.sources) console.log(node); children = node.sources.concat(node.targets); break; case ASSERT: children = [node.cond].concat(node.err ? [node.err] : []); break; case DOT: children = [node.value]; break; case INDEX: children = [node.value].concat(node.args); break; case SLICE: children = (node.start ? [node.start] : []) .concat(node.stop ? [node.stop] : []) .concat(node.step ? [node.step] : []); break; case COMPFOR: children = node.for.concat([node.in]); break; case COMPIF: children = [node.test]; break; case YIELD: children = node.value ? node.value : []; break; case RETURN: children = node.values ? node.values : []; break; case RAISE: children = node.err ? [node.err] : []; break; case IFEXPR: children = [node.test, node.then, node.else]; break; } let nodes = [node]; if (children.some(c => !c)) { console.log('BAD CHILDREN', node); } let subtreeNodes = flatten( children.map(node => walkRecursive(node, ancestors, walkListener)) ); nodes = nodes.concat(subtreeNodes); if (walkListener && walkListener.onExitNode) { walkListener.onExitNode(node, ancestors); } ancestors.pop(); return nodes; }