1 """Parse tree transformation module. 2 3 Transforms Python source code into an abstract syntax tree (AST) 4 defined in the ast module. 5 6 The simplest ways to invoke this module are via parse and parseFile. 7 parse(buf) -> AST 8 parseFile(path) -> AST 9 """ 10 11 # Original version written by Greg Stein (gstein@lyra.org) 12 # and Bill Tutt (rassilon@lima.mudlib.org) 13 # February 1997. 14 # 15 # Modifications and improvements for Python 2.0 by Jeremy Hylton and 16 # Mark Hammond 17 # 18 # Some fixes to try to have correct line number on almost all nodes 19 # (except Module, Discard and Stmt) added by Sylvain Thenault 20 # 21 # Portions of this file are: 22 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. 23 # 24 # This module is provided under a BSD-ish license. See 25 # http://www.opensource.org/licenses/bsd-license.html 26 # and replace OWNER, ORGANIZATION, and YEAR as appropriate. 27 28 from compiler.ast import * 29 import parser 30 import symbol 31 import token 32 import sys 33 34 class WalkerError(StandardError): 35 pass 36 37 from compiler.consts import CO_VARARGS, CO_VARKEYWORDS 38 from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY 39 40 def parseFile(path): 41 f = open(path, "U") 42 # XXX The parser API tolerates files without a trailing newline, 43 # but not strings without a trailing newline. Always add an extra 44 # newline to the file contents, since we're going through the string 45 # version of the API. 46 src = f.read() + "\n" 47 f.close() 48 return parse(src) 49 50 def parse(buf, mode="exec"): 51 if mode == "exec" or mode == "single": 52 return Transformer().parsesuite(buf) 53 elif mode == "eval": 54 return Transformer().parseexpr(buf) 55 else: 56 raise ValueError("compile() arg 3 must be" 57 " 'exec' or 'eval' or 'single'") 58 59 def asList(nodes): 60 l = [] 61 for item in nodes: 62 if hasattr(item, "asList"): 63 l.append(item.asList()) 64 else: 65 if type(item) is type( (None, None) ): 66 l.append(tuple(asList(item))) 67 elif type(item) is type( [] ): 68 l.append(asList(item)) 69 else: 70 l.append(item) 71 return l 72 73 def extractLineNo(ast): 74 if not isinstance(ast[1], tuple): 75 # get a terminal node 76 return ast[2] 77 for child in ast[1:]: 78 if isinstance(child, tuple): 79 lineno = extractLineNo(child) 80 if lineno is not None: 81 return lineno 82 83 def Node(*args): 84 kind = args[0] 85 if nodes.has_key(kind): 86 try: 87 return nodes[kind](*args[1:]) 88 except TypeError: 89 print nodes[kind], len(args), args 90 raise 91 else: 92 raise WalkerError, "Can't find appropriate Node type: %s" % str(args) 93 #return apply(ast.Node, args) 94 95 class Transformer: 96 """Utility object for transforming Python parse trees. 97 98 Exposes the following methods: 99 tree = transform(ast_tree) 100 tree = parsesuite(text) 101 tree = parseexpr(text) 102 tree = parsefile(fileob | filename) 103 """ 104 105 def __init__(self): 106 self._dispatch = {} 107 for value, name in symbol.sym_name.items(): 108 if hasattr(self, name): 109 self._dispatch[value] = getattr(self, name) 110 self._dispatch[token.NEWLINE] = self.com_NEWLINE 111 self._atom_dispatch = {token.LPAR: self.atom_lpar, 112 token.LSQB: self.atom_lsqb, 113 token.LBRACE: self.atom_lbrace, 114 token.BACKQUOTE: self.atom_backquote, 115 token.NUMBER: self.atom_number, 116 token.STRING: self.atom_string, 117 token.NAME: self.atom_name, 118 } 119 self.encoding = None 120 121 def transform(self, tree): 122 """Transform an AST into a modified parse tree.""" 123 if not (isinstance(tree, tuple) or isinstance(tree, list)): 124 tree = parser.ast2tuple(tree, line_info=1) 125 return self.compile_node(tree) 126 127 def parsesuite(self, text): 128 """Return a modified parse tree for the given suite text.""" 129 return self.transform(parser.suite(text)) 130 131 def parseexpr(self, text): 132 """Return a modified parse tree for the given expression text.""" 133 return self.transform(parser.expr(text)) 134 135 def parsefile(self, file): 136 """Return a modified parse tree for the contents of the given file.""" 137 if type(file) == type(''): 138 file = open(file) 139 return self.parsesuite(file.read()) 140 141 # -------------------------------------------------------------- 142 # 143 # PRIVATE METHODS 144 # 145 146 def compile_node(self, node): 147 ### emit a line-number node? 148 n = node[0] 149 150 if n == symbol.encoding_decl: 151 self.encoding = node[2] 152 node = node[1] 153 n = node[0] 154 155 if n == symbol.single_input: 156 return self.single_input(node[1:]) 157 if n == symbol.file_input: 158 return self.file_input(node[1:]) 159 if n == symbol.eval_input: 160 return self.eval_input(node[1:]) 161 if n == symbol.lambdef: 162 return self.lambdef(node[1:]) 163 if n == symbol.funcdef: 164 return self.funcdef(node[1:]) 165 if n == symbol.classdef: 166 return self.classdef(node[1:]) 167 168 raise WalkerError, ('unexpected node type', n) 169 170 def single_input(self, node): 171 ### do we want to do anything about being "interactive" ? 172 173 # NEWLINE | simple_stmt | compound_stmt NEWLINE 174 n = node[0][0] 175 if n != token.NEWLINE: 176 return self.com_stmt(node[0]) 177 178 return Pass() 179 180 def file_input(self, nodelist): 181 doc = self.get_docstring(nodelist, symbol.file_input) 182 if doc is not None: 183 i = 1 184 else: 185 i = 0 186 stmts = [] 187 for node in nodelist[i:]: 188 if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: 189 self.com_append_stmt(stmts, node) 190 return Module(doc, Stmt(stmts)) 191 192 def eval_input(self, nodelist): 193 # from the built-in function input() 194 ### is this sufficient? 195 return Expression(self.com_node(nodelist[0])) 196 197 def decorator_name(self, nodelist): 198 listlen = len(nodelist) 199 assert listlen >= 1 and listlen % 2 == 1 200 201 item = self.atom_name(nodelist) 202 i = 1 203 while i < listlen: 204 assert nodelist[i][0] == token.DOT 205 assert nodelist[i + 1][0] == token.NAME 206 item = Getattr(item, nodelist[i + 1][1]) 207 i += 2 208 209 return item 210 211 def decorator(self, nodelist): 212 # '@' dotted_name [ '(' [arglist] ')' ] 213 assert len(nodelist) in (3, 5, 6) 214 assert nodelist[0][0] == token.AT 215 assert nodelist[-1][0] == token.NEWLINE 216 217 assert nodelist[1][0] == symbol.dotted_name 218 funcname = self.decorator_name(nodelist[1][1:]) 219 220 if len(nodelist) > 3: 221 assert nodelist[2][0] == token.LPAR 222 expr = self.com_call_function(funcname, nodelist[3]) 223 else: 224 expr = funcname 225 226 return expr 227 228 def decorators(self, nodelist): 229 # decorators: decorator ([NEWLINE] decorator)* NEWLINE 230 items = [] 231 for dec_nodelist in nodelist: 232 assert dec_nodelist[0] == symbol.decorator 233 items.append(self.decorator(dec_nodelist[1:])) 234 return Decorators(items) 235 236 def funcdef(self, nodelist): 237 # -6 -5 -4 -3 -2 -1 238 # funcdef: [decorators] 'def' NAME parameters ':' suite 239 # parameters: '(' [varargslist] ')' 240 241 if len(nodelist) == 6: 242 assert nodelist[0][0] == symbol.decorators 243 decorators = self.decorators(nodelist[0][1:]) 244 else: 245 assert len(nodelist) == 5 246 decorators = None 247 248 lineno = nodelist[-4][2] 249 name = nodelist[-4][1] 250 args = nodelist[-3][2] 251 252 if args[0] == symbol.varargslist: 253 names, defaults, flags = self.com_arglist(args[1:]) 254 else: 255 names = defaults = () 256 flags = 0 257 doc = self.get_docstring(nodelist[-1]) 258 259 # code for function 260 code = self.com_node(nodelist[-1]) 261 262 if doc is not None: 263 assert isinstance(code, Stmt) 264 assert isinstance(code.nodes[0], Discard) 265 del code.nodes[0] 266 return Function(decorators, name, names, defaults, flags, doc, code, 267 lineno=lineno) 268 269 def lambdef(self, nodelist): 270 # lambdef: 'lambda' [varargslist] ':' test 271 if nodelist[2][0] == symbol.varargslist: 272 names, defaults, flags = self.com_arglist(nodelist[2][1:]) 273 else: 274 names = defaults = () 275 flags = 0 276 277 # code for lambda 278 code = self.com_node(nodelist[-1]) 279 280 return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) 281 old_lambdef = lambdef 282 283 def classdef(self, nodelist): 284 # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 285 286 name = nodelist[1][1] 287 doc = self.get_docstring(nodelist[-1]) 288 if nodelist[2][0] == token.COLON: 289 bases = [] 290 elif nodelist[3][0] == token.RPAR: 291 bases = [] 292 else: 293 bases = self.com_bases(nodelist[3]) 294 295 # code for class 296 code = self.com_node(nodelist[-1]) 297 298 if doc is not None: 299 assert isinstance(code, Stmt) 300 assert isinstance(code.nodes[0], Discard) 301 del code.nodes[0] 302 303 return Class(name, bases, doc, code, lineno=nodelist[1][2]) 304 305 def stmt(self, nodelist): 306 return self.com_stmt(nodelist[0]) 307 308 small_stmt = stmt 309 flow_stmt = stmt 310 compound_stmt = stmt 311 312 def simple_stmt(self, nodelist): 313 # small_stmt (';' small_stmt)* [';'] NEWLINE 314 stmts = [] 315 for i in range(0, len(nodelist), 2): 316 self.com_append_stmt(stmts, nodelist[i]) 317 return Stmt(stmts) 318 319 def parameters(self, nodelist): 320 raise WalkerError 321 322 def varargslist(self, nodelist): 323 raise WalkerError 324 325 def fpdef(self, nodelist): 326 raise WalkerError 327 328 def fplist(self, nodelist): 329 raise WalkerError 330 331 def dotted_name(self, nodelist): 332 raise WalkerError 333 334 def comp_op(self, nodelist): 335 raise WalkerError 336 337 def trailer(self, nodelist): 338 raise WalkerError 339 340 def sliceop(self, nodelist): 341 raise WalkerError 342 343 def argument(self, nodelist): 344 raise WalkerError 345 346 # -------------------------------------------------------------- 347 # 348 # STATEMENT NODES (invoked by com_node()) 349 # 350 351 def expr_stmt(self, nodelist): 352 # augassign testlist | testlist ('=' testlist)* 353 en = nodelist[-1] 354 exprNode = self.lookup_node(en)(en[1:]) 355 if len(nodelist) == 1: 356 return Discard(exprNode, lineno=exprNode.lineno) 357 if nodelist[1][0] == token.EQUAL: 358 nodesl = [] 359 for i in range(0, len(nodelist) - 2, 2): 360 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) 361 return Assign(nodesl, exprNode, lineno=nodelist[1][2]) 362 else: 363 lval = self.com_augassign(nodelist[0]) 364 op = self.com_augassign_op(nodelist[1]) 365 return AugAssign(lval, op[1], exprNode, lineno=op[2]) 366 raise WalkerError, "can't get here" 367 368 def print_stmt(self, nodelist): 369 # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) 370 items = [] 371 if len(nodelist) == 1: 372 start = 1 373 dest = None 374 elif nodelist[1][0] == token.RIGHTSHIFT: 375 assert len(nodelist) == 3 \ 376 or nodelist[3][0] == token.COMMA 377 dest = self.com_node(nodelist[2]) 378 start = 4 379 else: 380 dest = None 381 start = 1 382 for i in range(start, len(nodelist), 2): 383 items.append(self.com_node(nodelist[i])) 384 if nodelist[-1][0] == token.COMMA: 385 return Print(items, dest, lineno=nodelist[0][2]) 386 return Printnl(items, dest, lineno=nodelist[0][2]) 387 388 def del_stmt(self, nodelist): 389 return self.com_assign(nodelist[1], OP_DELETE) 390 391 def pass_stmt(self, nodelist): 392 return Pass(lineno=nodelist[0][2]) 393 394 def break_stmt(self, nodelist): 395 return Break(lineno=nodelist[0][2]) 396 397 def continue_stmt(self, nodelist): 398 return Continue(lineno=nodelist[0][2]) 399 400 def return_stmt(self, nodelist): 401 # return: [testlist] 402 if len(nodelist) < 2: 403 return Return(Const(None), lineno=nodelist[0][2]) 404 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 405 406 def yield_stmt(self, nodelist): 407 expr = self.com_node(nodelist[0]) 408 return Discard(expr, lineno=expr.lineno) 409 410 def yield_expr(self, nodelist): 411 if len(nodelist) > 1: 412 value = self.com_node(nodelist[1]) 413 else: 414 value = Const(None) 415 return Yield(value, lineno=nodelist[0][2]) 416 417 def raise_stmt(self, nodelist): 418 # raise: [test [',' test [',' test]]] 419 if len(nodelist) > 5: 420 expr3 = self.com_node(nodelist[5]) 421 else: 422 expr3 = None 423 if len(nodelist) > 3: 424 expr2 = self.com_node(nodelist[3]) 425 else: 426 expr2 = None 427 if len(nodelist) > 1: 428 expr1 = self.com_node(nodelist[1]) 429 else: 430 expr1 = None 431 return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) 432 433 def import_stmt(self, nodelist): 434 # import_stmt: import_name | import_from 435 assert len(nodelist) == 1 436 return self.com_node(nodelist[0]) 437 438 def import_name(self, nodelist): 439 # import_name: 'import' dotted_as_names 440 return Import(self.com_dotted_as_names(nodelist[1]), 441 lineno=nodelist[0][2]) 442 443 def import_from(self, nodelist): 444 # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | 445 # '(' import_as_names ')' | import_as_names) 446 assert nodelist[0][1] == 'from' 447 idx = 1 448 while nodelist[idx][1] == '.': 449 idx += 1 450 level = idx - 1 451 if nodelist[idx][0] == symbol.dotted_name: 452 fromname = self.com_dotted_name(nodelist[idx]) 453 idx += 1 454 else: 455 fromname = "" 456 assert nodelist[idx][1] == 'import' 457 if nodelist[idx + 1][0] == token.STAR: 458 return From(fromname, [('*', None)], level, 459 lineno=nodelist[0][2]) 460 else: 461 node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)] 462 return From(fromname, self.com_import_as_names(node), level, 463 lineno=nodelist[0][2]) 464 465 def global_stmt(self, nodelist): 466 # global: NAME (',' NAME)* 467 names = [] 468 for i in range(1, len(nodelist), 2): 469 names.append(nodelist[i][1]) 470 return Global(names, lineno=nodelist[0][2]) 471 472 def exec_stmt(self, nodelist): 473 # exec_stmt: 'exec' expr ['in' expr [',' expr]] 474 expr1 = self.com_node(nodelist[1]) 475 if len(nodelist) >= 4: 476 expr2 = self.com_node(nodelist[3]) 477 if len(nodelist) >= 6: 478 expr3 = self.com_node(nodelist[5]) 479 else: 480 expr3 = None 481 else: 482 expr2 = expr3 = None 483 484 return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) 485 486 def assert_stmt(self, nodelist): 487 # 'assert': test, [',' test] 488 expr1 = self.com_node(nodelist[1]) 489 if (len(nodelist) == 4): 490 expr2 = self.com_node(nodelist[3]) 491 else: 492 expr2 = None 493 return Assert(expr1, expr2, lineno=nodelist[0][2]) 494 495 def if_stmt(self, nodelist): 496 # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 497 tests = [] 498 for i in range(0, len(nodelist) - 3, 4): 499 testNode = self.com_node(nodelist[i + 1]) 500 suiteNode = self.com_node(nodelist[i + 3]) 501 tests.append((testNode, suiteNode)) 502 503 if len(nodelist) % 4 == 3: 504 elseNode = self.com_node(nodelist[-1]) 505 ## elseNode.lineno = nodelist[-1][1][2] 506 else: 507 elseNode = None 508 return If(tests, elseNode, lineno=nodelist[0][2]) 509 510 def while_stmt(self, nodelist): 511 # 'while' test ':' suite ['else' ':' suite] 512 513 testNode = self.com_node(nodelist[1]) 514 bodyNode = self.com_node(nodelist[3]) 515 516 if len(nodelist) > 4: 517 elseNode = self.com_node(nodelist[6]) 518 else: 519 elseNode = None 520 521 return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) 522 523 def for_stmt(self, nodelist): 524 # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] 525 526 assignNode = self.com_assign(nodelist[1], OP_ASSIGN) 527 listNode = self.com_node(nodelist[3]) 528 bodyNode = self.com_node(nodelist[5]) 529 530 if len(nodelist) > 8: 531 elseNode = self.com_node(nodelist[8]) 532 else: 533 elseNode = None 534 535 return For(assignNode, listNode, bodyNode, elseNode, 536 lineno=nodelist[0][2]) 537 538 def try_stmt(self, nodelist): 539 return self.com_try_except_finally(nodelist) 540 541 def with_stmt(self, nodelist): 542 return self.com_with(nodelist) 543 544 def with_var(self, nodelist): 545 return self.com_with_var(nodelist) 546 547 def suite(self, nodelist): 548 # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT 549 if len(nodelist) == 1: 550 return self.com_stmt(nodelist[0]) 551 552 stmts = [] 553 for node in nodelist: 554 if node[0] == symbol.stmt: 555 self.com_append_stmt(stmts, node) 556 return Stmt(stmts) 557 558 # -------------------------------------------------------------- 559 # 560 # EXPRESSION NODES (invoked by com_node()) 561 # 562 563 def testlist(self, nodelist): 564 # testlist: expr (',' expr)* [','] 565 # testlist_safe: test [(',' test)+ [',']] 566 # exprlist: expr (',' expr)* [','] 567 return self.com_binary(Tuple, nodelist) 568 569 testlist_safe = testlist # XXX 570 testlist1 = testlist 571 exprlist = testlist 572 573 def testlist_gexp(self, nodelist): 574 if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: 575 test = self.com_node(nodelist[0]) 576 return self.com_generator_expression(test, nodelist[1]) 577 return self.testlist(nodelist) 578 579 def test(self, nodelist): 580 # or_test ['if' or_test 'else' test] | lambdef 581 if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: 582 return self.lambdef(nodelist[0]) 583 then = self.com_node(nodelist[0]) 584 if len(nodelist) > 1: 585 assert len(nodelist) == 5 586 assert nodelist[1][1] == 'if' 587 assert nodelist[3][1] == 'else' 588 test = self.com_node(nodelist[2]) 589 else_ = self.com_node(nodelist[4]) 590 return IfExp(test, then, else_, lineno=nodelist[1][2]) 591 return then 592 593 def or_test(self, nodelist): 594 # and_test ('or' and_test)* | lambdef 595 if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: 596 return self.lambdef(nodelist[0]) 597 return self.com_binary(Or, nodelist) 598 old_test = or_test 599 600 def and_test(self, nodelist): 601 # not_test ('and' not_test)* 602 return self.com_binary(And, nodelist) 603 604 def not_test(self, nodelist): 605 # 'not' not_test | comparison 606 result = self.com_node(nodelist[-1]) 607 if len(nodelist) == 2: 608 return Not(result, lineno=nodelist[0][2]) 609 return result 610 611 def comparison(self, nodelist): 612 # comparison: expr (comp_op expr)* 613 node = self.com_node(nodelist[0]) 614 if len(nodelist) == 1: 615 return node 616 617 results = [] 618 for i in range(2, len(nodelist), 2): 619 nl = nodelist[i-1] 620 621 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 622 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 623 n = nl[1] 624 if n[0] == token.NAME: 625 type = n[1] 626 if len(nl) == 3: 627 if type == 'not': 628 type = 'not in' 629 else: 630 type = 'is not' 631 else: 632 type = _cmp_types[n[0]] 633 634 lineno = nl[1][2] 635 results.append((type, self.com_node(nodelist[i]))) 636 637 # we need a special "compare" node so that we can distinguish 638 # 3 < x < 5 from (3 < x) < 5 639 # the two have very different semantics and results (note that the 640 # latter form is always true) 641 642 return Compare(node, results, lineno=lineno) 643 644 def expr(self, nodelist): 645 # xor_expr ('|' xor_expr)* 646 return self.com_binary(Bitor, nodelist) 647 648 def xor_expr(self, nodelist): 649 # xor_expr ('^' xor_expr)* 650 return self.com_binary(Bitxor, nodelist) 651 652 def and_expr(self, nodelist): 653 # xor_expr ('&' xor_expr)* 654 return self.com_binary(Bitand, nodelist) 655 656 def shift_expr(self, nodelist): 657 # shift_expr ('<<'|'>>' shift_expr)* 658 node = self.com_node(nodelist[0]) 659 for i in range(2, len(nodelist), 2): 660 right = self.com_node(nodelist[i]) 661 if nodelist[i-1][0] == token.LEFTSHIFT: 662 node = LeftShift([node, right], lineno=nodelist[1][2]) 663 elif nodelist[i-1][0] == token.RIGHTSHIFT: 664 node = RightShift([node, right], lineno=nodelist[1][2]) 665 else: 666 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 667 return node 668 669 def arith_expr(self, nodelist): 670 node = self.com_node(nodelist[0]) 671 for i in range(2, len(nodelist), 2): 672 right = self.com_node(nodelist[i]) 673 if nodelist[i-1][0] == token.PLUS: 674 node = Add([node, right], lineno=nodelist[1][2]) 675 elif nodelist[i-1][0] == token.MINUS: 676 node = Sub([node, right], lineno=nodelist[1][2]) 677 else: 678 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 679 return node 680 681 def term(self, nodelist): 682 node = self.com_node(nodelist[0]) 683 for i in range(2, len(nodelist), 2): 684 right = self.com_node(nodelist[i]) 685 t = nodelist[i-1][0] 686 if t == token.STAR: 687 node = Mul([node, right]) 688 elif t == token.SLASH: 689 node = Div([node, right]) 690 elif t == token.PERCENT: 691 node = Mod([node, right]) 692 elif t == token.DOUBLESLASH: 693 node = FloorDiv([node, right]) 694 else: 695 raise ValueError, "unexpected token: %s" % t 696 node.lineno = nodelist[1][2] 697 return node 698 699 def factor(self, nodelist): 700 elt = nodelist[0] 701 t = elt[0] 702 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) 703 # need to handle (unary op)constant here... 704 if t == token.PLUS: 705 return UnaryAdd(node, lineno=elt[2]) 706 elif t == token.MINUS: 707 return UnarySub(node, lineno=elt[2]) 708 elif t == token.TILDE: 709 node = Invert(node, lineno=elt[2]) 710 return node 711 712 def power(self, nodelist): 713 # power: atom trailer* ('**' factor)* 714 node = self.com_node(nodelist[0]) 715 for i in range(1, len(nodelist)): 716 elt = nodelist[i] 717 if elt[0] == token.DOUBLESTAR: 718 return Power([node, self.com_node(nodelist[i+1])], 719 lineno=elt[2]) 720 721 node = self.com_apply_trailer(node, elt) 722 723 return node 724 725 def atom(self, nodelist): 726 return self._atom_dispatch[nodelist[0][0]](nodelist) 727 728 def atom_lpar(self, nodelist): 729 if nodelist[1][0] == token.RPAR: 730 return Tuple((), lineno=nodelist[0][2]) 731 return self.com_node(nodelist[1]) 732 733 def atom_lsqb(self, nodelist): 734 if nodelist[1][0] == token.RSQB: 735 return List((), lineno=nodelist[0][2]) 736 return self.com_list_constructor(nodelist[1]) 737 738 def atom_lbrace(self, nodelist): 739 if nodelist[1][0] == token.RBRACE: 740 return Dict((), lineno=nodelist[0][2]) 741 return self.com_dictmaker(nodelist[1]) 742 743 def atom_backquote(self, nodelist): 744 return Backquote(self.com_node(nodelist[1])) 745 746 def atom_number(self, nodelist): 747 ### need to verify this matches compile.c 748 k = eval(nodelist[0][1]) 749 return Const(k, lineno=nodelist[0][2]) 750 751 def decode_literal(self, lit): 752 if self.encoding: 753 # this is particularly fragile & a bit of a 754 # hack... changes in compile.c:parsestr and 755 # tokenizer.c must be reflected here. 756 if self.encoding not in ['utf-8', 'iso-8859-1']: 757 lit = unicode(lit, 'utf-8').encode(self.encoding) 758 return eval("# coding: %s\n%s" % (self.encoding, lit)) 759 else: 760 return eval(lit) 761 762 def atom_string(self, nodelist): 763 k = '' 764 for node in nodelist: 765 k += self.decode_literal(node[1]) 766 return Const(k, lineno=nodelist[0][2]) 767 768 def atom_name(self, nodelist): 769 return Name(nodelist[0][1], lineno=nodelist[0][2]) 770 771 # -------------------------------------------------------------- 772 # 773 # INTERNAL PARSING UTILITIES 774 # 775 776 # The use of com_node() introduces a lot of extra stack frames, 777 # enough to cause a stack overflow compiling test.test_parser with 778 # the standard interpreter recursionlimit. The com_node() is a 779 # convenience function that hides the dispatch details, but comes 780 # at a very high cost. It is more efficient to dispatch directly 781 # in the callers. In these cases, use lookup_node() and call the 782 # dispatched node directly. 783 784 def lookup_node(self, node): 785 return self._dispatch[node[0]] 786 787 def com_node(self, node): 788 # Note: compile.c has handling in com_node for del_stmt, pass_stmt, 789 # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, 790 # and compound_stmt. 791 # We'll just dispatch them. 792 return self._dispatch[node[0]](node[1:]) 793 794 def com_NEWLINE(self, *args): 795 # A ';' at the end of a line can make a NEWLINE token appear 796 # here, Render it harmless. (genc discards ('discard', 797 # ('const', xxxx)) Nodes) 798 return Discard(Const(None)) 799 800 def com_arglist(self, nodelist): 801 # varargslist: 802 # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) 803 # | fpdef ['=' test] (',' fpdef ['=' test])* [','] 804 # fpdef: NAME | '(' fplist ')' 805 # fplist: fpdef (',' fpdef)* [','] 806 names = [] 807 defaults = [] 808 flags = 0 809 810 i = 0 811 while i < len(nodelist): 812 node = nodelist[i] 813 if node[0] == token.STAR or node[0] == token.DOUBLESTAR: 814 if node[0] == token.STAR: 815 node = nodelist[i+1] 816 if node[0] == token.NAME: 817 names.append(node[1]) 818 flags = flags | CO_VARARGS 819 i = i + 3 820 821 if i < len(nodelist): 822 # should be DOUBLESTAR 823 t = nodelist[i][0] 824 if t == token.DOUBLESTAR: 825 node = nodelist[i+1] 826 else: 827 raise ValueError, "unexpected token: %s" % t 828 names.append(node[1]) 829 flags = flags | CO_VARKEYWORDS 830 831 break 832 833 # fpdef: NAME | '(' fplist ')' 834 names.append(self.com_fpdef(node)) 835 836 i = i + 1 837 if i < len(nodelist) and nodelist[i][0] == token.EQUAL: 838 defaults.append(self.com_node(nodelist[i + 1])) 839 i = i + 2 840 elif len(defaults): 841 # we have already seen an argument with default, but here 842 # came one without 843 raise SyntaxError, "non-default argument follows default argument" 844 845 # skip the comma 846 i = i + 1 847 848 return names, defaults, flags 849 850 def com_fpdef(self, node): 851 # fpdef: NAME | '(' fplist ')' 852 if node[1][0] == token.LPAR: 853 return self.com_fplist(node[2]) 854 return node[1][1] 855 856 def com_fplist(self, node): 857 # fplist: fpdef (',' fpdef)* [','] 858 if len(node) == 2: 859 return self.com_fpdef(node[1]) 860 list = [] 861 for i in range(1, len(node), 2): 862 list.append(self.com_fpdef(node[i])) 863 return tuple(list) 864 865 def com_dotted_name(self, node): 866 # String together the dotted names and return the string 867 name = "" 868 for n in node: 869 if type(n) == type(()) and n[0] == 1: 870 name = name + n[1] + '.' 871 return name[:-1] 872 873 def com_dotted_as_name(self, node): 874 assert node[0] == symbol.dotted_as_name 875 node = node[1:] 876 dot = self.com_dotted_name(node[0][1:]) 877 if len(node) == 1: 878 return dot, None 879 assert node[1][1] == 'as' 880 assert node[2][0] == token.NAME 881 return dot, node[2][1] 882 883 def com_dotted_as_names(self, node): 884 assert node[0] == symbol.dotted_as_names 885 node = node[1:] 886 names = [self.com_dotted_as_name(node[0])] 887 for i in range(2, len(node), 2): 888 names.append(self.com_dotted_as_name(node[i])) 889 return names 890 891 def com_import_as_name(self, node): 892 assert node[0] == symbol.import_as_name 893 node = node[1:] 894 assert node[0][0] == token.NAME 895 if len(node) == 1: 896 return node[0][1], None 897 assert node[1][1] == 'as', node 898 assert node[2][0] == token.NAME 899 return node[0][1], node[2][1] 900 901 def com_import_as_names(self, node): 902 assert node[0] == symbol.import_as_names 903 node = node[1:] 904 names = [self.com_import_as_name(node[0])] 905 for i in range(2, len(node), 2): 906 names.append(self.com_import_as_name(node[i])) 907 return names 908 909 def com_bases(self, node): 910 bases = [] 911 for i in range(1, len(node), 2): 912 bases.append(self.com_node(node[i])) 913 return bases 914 915 def com_try_except_finally(self, nodelist): 916 # ('try' ':' suite 917 # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] 918 # | 'finally' ':' suite)) 919 920 if nodelist[3][0] == token.NAME: 921 # first clause is a finally clause: only try-finally 922 return TryFinally(self.com_node(nodelist[2]), 923 self.com_node(nodelist[5]), 924 lineno=nodelist[0][2]) 925 926 #tryexcept: [TryNode, [except_clauses], elseNode)] 927 clauses = [] 928 elseNode = None 929 finallyNode = None 930 for i in range(3, len(nodelist), 3): 931 node = nodelist[i] 932 if node[0] == symbol.except_clause: 933 # except_clause: 'except' [expr [',' expr]] */ 934 if len(node) > 2: 935 expr1 = self.com_node(node[2]) 936 if len(node) > 4: 937 expr2 = self.com_assign(node[4], OP_ASSIGN) 938 else: 939 expr2 = None 940 else: 941 expr1 = expr2 = None 942 clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) 943 944 if node[0] == token.NAME: 945 if node[1] == 'else': 946 elseNode = self.com_node(nodelist[i+2]) 947 elif node[1] == 'finally': 948 finallyNode = self.com_node(nodelist[i+2]) 949 try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, 950 lineno=nodelist[0][2]) 951 if finallyNode: 952 return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) 953 else: 954 return try_except 955 956 def com_with(self, nodelist): 957 # with_stmt: 'with' expr [with_var] ':' suite 958 expr = self.com_node(nodelist[1]) 959 body = self.com_node(nodelist[-1]) 960 if nodelist[2][0] == token.COLON: 961 var = None 962 else: 963 var = self.com_assign(nodelist[2][2], OP_ASSIGN) 964 return With(expr, var, body, lineno=nodelist[0][2]) 965 966 def com_with_var(self, nodelist): 967 # with_var: 'as' expr 968 return self.com_node(nodelist[1]) 969 970 def com_augassign_op(self, node): 971 assert node[0] == symbol.augassign 972 return node[1] 973 974 def com_augassign(self, node): 975 """Return node suitable for lvalue of augmented assignment 976 977 Names, slices, and attributes are the only allowable nodes. 978 """ 979 l = self.com_node(node) 980 if l.__class__ in (Name, Slice, Subscript, Getattr): 981 return l 982 raise SyntaxError, "can't assign to %s" % l.__class__.__name__ 983 984 def com_assign(self, node, assigning): 985 # return a node suitable for use as an "lvalue" 986 # loop to avoid trivial recursion 987 while 1: 988 t = node[0] 989 if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp): 990 if len(node) > 2: 991 return self.com_assign_tuple(node, assigning) 992 node = node[1] 993 elif t in _assign_types: 994 if len(node) > 2: 995 raise SyntaxError, "can't assign to operator" 996 node = node[1] 997 elif t == symbol.power: 998 if node[1][0] != symbol.atom: 999 raise SyntaxError, "can't assign to operator" 1000 if len(node) > 2: 1001 primary = self.com_node(node[1]) 1002 for i in range(2, len(node)-1): 1003 ch = node[i] 1004 if ch[0] == token.DOUBLESTAR: 1005 raise SyntaxError, "can't assign to operator" 1006 primary = self.com_apply_trailer(primary, ch) 1007 return self.com_assign_trailer(primary, node[-1], 1008 assigning) 1009 node = node[1] 1010 elif t == symbol.atom: 1011 t = node[1][0] 1012 if t == token.LPAR: 1013 node = node[2] 1014 if node[0] == token.RPAR: 1015 raise SyntaxError, "can't assign to ()" 1016 elif t == token.LSQB: 1017 node = node[2] 1018 if node[0] == token.RSQB: 1019 raise SyntaxError, "can't assign to []" 1020 return self.com_assign_list(node, assigning) 1021 elif t == token.NAME: 1022 return self.com_assign_name(node[1], assigning) 1023 else: 1024 raise SyntaxError, "can't assign to literal" 1025 else: 1026 raise SyntaxError, "bad assignment (%s)" % t 1027 1028 def com_assign_tuple(self, node, assigning): 1029 assigns = [] 1030 for i in range(1, len(node), 2): 1031 assigns.append(self.com_assign(node[i], assigning)) 1032 return AssTuple(assigns, lineno=extractLineNo(node)) 1033 1034 def com_assign_list(self, node, assigning): 1035 assigns = [] 1036 for i in range(1, len(node), 2): 1037 if i + 1 < len(node): 1038 if node[i + 1][0] == symbol.list_for: 1039 raise SyntaxError, "can't assign to list comprehension" 1040 assert node[i + 1][0] == token.COMMA, node[i + 1] 1041 assigns.append(self.com_assign(node[i], assigning)) 1042 return AssList(assigns, lineno=extractLineNo(node)) 1043 1044 def com_assign_name(self, node, assigning): 1045 return AssName(node[1], assigning, lineno=node[2]) 1046 1047 def com_assign_trailer(self, primary, node, assigning): 1048 t = node[1][0] 1049 if t == token.DOT: 1050 return self.com_assign_attr(primary, node[2], assigning) 1051 if t == token.LSQB: 1052 return self.com_subscriptlist(primary, node[2], assigning) 1053 if t == token.LPAR: 1054 raise SyntaxError, "can't assign to function call" 1055 raise SyntaxError, "unknown trailer type: %s" % t 1056 1057 def com_assign_attr(self, primary, node, assigning): 1058 return AssAttr(primary, node[1], assigning, lineno=node[-1]) 1059 1060 def com_binary(self, constructor, nodelist): 1061 "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." 1062 l = len(nodelist) 1063 if l == 1: 1064 n = nodelist[0] 1065 return self.lookup_node(n)(n[1:]) 1066 items = [] 1067 for i in range(0, l, 2): 1068 n = nodelist[i] 1069 items.append(self.lookup_node(n)(n[1:])) 1070 return constructor(items, lineno=extractLineNo(nodelist)) 1071 1072 def com_stmt(self, node): 1073 result = self.lookup_node(node)(node[1:]) 1074 assert result is not None 1075 if isinstance(result, Stmt): 1076 return result 1077 return Stmt([result]) 1078 1079 def com_append_stmt(self, stmts, node): 1080 result = self.lookup_node(node)(node[1:]) 1081 assert result is not None 1082 if isinstance(result, Stmt): 1083 stmts.extend(result.nodes) 1084 else: 1085 stmts.append(result) 1086 1087 if hasattr(symbol, 'list_for'): 1088 def com_list_constructor(self, nodelist): 1089 # listmaker: test ( list_for | (',' test)* [','] ) 1090 values = [] 1091 for i in range(1, len(nodelist)): 1092 if nodelist[i][0] == symbol.list_for: 1093 assert len(nodelist[i:]) == 1 1094 return self.com_list_comprehension(values[0], 1095 nodelist[i]) 1096 elif nodelist[i][0] == token.COMMA: 1097 continue 1098 values.append(self.com_node(nodelist[i])) 1099 return List(values, lineno=values[0].lineno) 1100 1101 def com_list_comprehension(self, expr, node): 1102 # list_iter: list_for | list_if 1103 # list_for: 'for' exprlist 'in' testlist [list_iter] 1104 # list_if: 'if' test [list_iter] 1105 1106 # XXX should raise SyntaxError for assignment 1107 1108 lineno = node[1][2] 1109 fors = [] 1110 while node: 1111 t = node[1][1] 1112 if t == 'for': 1113 assignNode = self.com_assign(node[2], OP_ASSIGN) 1114 listNode = self.com_node(node[4]) 1115 newfor = ListCompFor(assignNode, listNode, []) 1116 newfor.lineno = node[1][2] 1117 fors.append(newfor) 1118 if len(node) == 5: 1119 node = None 1120 else: 1121 node = self.com_list_iter(node[5]) 1122 elif t == 'if': 1123 test = self.com_node(node[2]) 1124 newif = ListCompIf(test, lineno=node[1][2]) 1125 newfor.ifs.append(newif) 1126 if len(node) == 3: 1127 node = None 1128 else: 1129 node = self.com_list_iter(node[3]) 1130 else: 1131 raise SyntaxError, \ 1132 ("unexpected list comprehension element: %s %d" 1133 % (node, lineno)) 1134 return ListComp(expr, fors, lineno=lineno) 1135 1136 def com_list_iter(self, node): 1137 assert node[0] == symbol.list_iter 1138 return node[1] 1139 else: 1140 def com_list_constructor(self, nodelist): 1141 values = [] 1142 for i in range(1, len(nodelist), 2): 1143 values.append(self.com_node(nodelist[i])) 1144 return List(values, lineno=values[0].lineno) 1145 1146 if hasattr(symbol, 'gen_for'): 1147 def com_generator_expression(self, expr, node): 1148 # gen_iter: gen_for | gen_if 1149 # gen_for: 'for' exprlist 'in' test [gen_iter] 1150 # gen_if: 'if' test [gen_iter] 1151 1152 lineno = node[1][2] 1153 fors = [] 1154 while node: 1155 t = node[1][1] 1156 if t == 'for': 1157 assignNode = self.com_assign(node[2], OP_ASSIGN) 1158 genNode = self.com_node(node[4]) 1159 newfor = GenExprFor(assignNode, genNode, [], 1160 lineno=node[1][2]) 1161 fors.append(newfor) 1162 if (len(node)) == 5: 1163 node = None 1164 else: 1165 node = self.com_gen_iter(node[5]) 1166 elif t == 'if': 1167 test = self.com_node(node[2]) 1168 newif = GenExprIf(test, lineno=node[1][2]) 1169 newfor.ifs.append(newif) 1170 if len(node) == 3: 1171 node = None 1172 else: 1173 node = self.com_gen_iter(node[3]) 1174 else: 1175 raise SyntaxError, \ 1176 ("unexpected generator expression element: %s %d" 1177 % (node, lineno)) 1178 fors[0].is_outmost = True 1179 return GenExpr(GenExprInner(expr, fors), lineno=lineno) 1180 1181 def com_gen_iter(self, node): 1182 assert node[0] == symbol.gen_iter 1183 return node[1] 1184 1185 def com_dictmaker(self, nodelist): 1186 # dictmaker: test ':' test (',' test ':' value)* [','] 1187 items = [] 1188 for i in range(1, len(nodelist), 4): 1189 items.append((self.com_node(nodelist[i]), 1190 self.com_node(nodelist[i+2]))) 1191 return Dict(items, lineno=items[0][0].lineno) 1192 1193 def com_apply_trailer(self, primaryNode, nodelist): 1194 t = nodelist[1][0] 1195 if t == token.LPAR: 1196 return self.com_call_function(primaryNode, nodelist[2]) 1197 if t == token.DOT: 1198 return self.com_select_member(primaryNode, nodelist[2]) 1199 if t == token.LSQB: 1200 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) 1201 1202 raise SyntaxError, 'unknown node type: %s' % t 1203 1204 def com_select_member(self, primaryNode, nodelist): 1205 if nodelist[0] != token.NAME: 1206 raise SyntaxError, "member must be a name" 1207 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) 1208 1209 def com_call_function(self, primaryNode, nodelist): 1210 if nodelist[0] == token.RPAR: 1211 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) 1212 args = [] 1213 kw = 0 1214 len_nodelist = len(nodelist) 1215 for i in range(1, len_nodelist, 2): 1216 node = nodelist[i] 1217 if node[0] == token.STAR or node[0] == token.DOUBLESTAR: 1218 break 1219 kw, result = self.com_argument(node, kw) 1220 1221 if len_nodelist != 2 and isinstance(result, GenExpr) \ 1222 and len(node) == 3 and node[2][0] == symbol.gen_for: 1223 # allow f(x for x in y), but reject f(x for x in y, 1) 1224 # should use f((x for x in y), 1) instead of f(x for x in y, 1) 1225 raise SyntaxError, 'generator expression needs parenthesis' 1226 1227 args.append(result) 1228 else: 1229 # No broken by star arg, so skip the last one we processed. 1230 i = i + 1 1231 if i < len_nodelist and nodelist[i][0] == token.COMMA: 1232 # need to accept an application that looks like "f(a, b,)" 1233 i = i + 1 1234 star_node = dstar_node = None 1235 while i < len_nodelist: 1236 tok = nodelist[i] 1237 ch = nodelist[i+1] 1238 i = i + 3 1239 if tok[0]==token.STAR: 1240 if star_node is not None: 1241 raise SyntaxError, 'already have the varargs indentifier' 1242 star_node = self.com_node(ch) 1243 elif tok[0]==token.DOUBLESTAR: 1244 if dstar_node is not None: 1245 raise SyntaxError, 'already have the kwargs indentifier' 1246 dstar_node = self.com_node(ch) 1247 else: 1248 raise SyntaxError, 'unknown node type: %s' % tok 1249 return CallFunc(primaryNode, args, star_node, dstar_node, 1250 lineno=extractLineNo(nodelist)) 1251 1252 def com_argument(self, nodelist, kw): 1253 if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for: 1254 test = self.com_node(nodelist[1]) 1255 return 0, self.com_generator_expression(test, nodelist[2]) 1256 if len(nodelist) == 2: 1257 if kw: 1258 raise SyntaxError, "non-keyword arg after keyword arg" 1259 return 0, self.com_node(nodelist[1]) 1260 result = self.com_node(nodelist[3]) 1261 n = nodelist[1] 1262 while len(n) == 2 and n[0] != token.NAME: 1263 n = n[1] 1264 if n[0] != token.NAME: 1265 raise SyntaxError, "keyword can't be an expression (%s)"%n[0] 1266 node = Keyword(n[1], result, lineno=n[2]) 1267 return 1, node 1268 1269 def com_subscriptlist(self, primary, nodelist, assigning): 1270 # slicing: simple_slicing | extended_slicing 1271 # simple_slicing: primary "[" short_slice "]" 1272 # extended_slicing: primary "[" slice_list "]" 1273 # slice_list: slice_item ("," slice_item)* [","] 1274 1275 # backwards compat slice for '[i:j]' 1276 if len(nodelist) == 2: 1277 sub = nodelist[1] 1278 if (sub[1][0] == token.COLON or \ 1279 (len(sub) > 2 and sub[2][0] == token.COLON)) and \ 1280 sub[-1][0] != symbol.sliceop: 1281 return self.com_slice(primary, sub, assigning) 1282 1283 subscripts = [] 1284 for i in range(1, len(nodelist), 2): 1285 subscripts.append(self.com_subscript(nodelist[i])) 1286 return Subscript(primary, assigning, subscripts, 1287 lineno=extractLineNo(nodelist)) 1288 1289 def com_subscript(self, node): 1290 # slice_item: expression | proper_slice | ellipsis 1291 ch = node[1] 1292 t = ch[0] 1293 if t == token.DOT and node[2][0] == token.DOT: 1294 return Ellipsis() 1295 if t == token.COLON or len(node) > 2: 1296 return self.com_sliceobj(node) 1297 return self.com_node(ch) 1298 1299 def com_sliceobj(self, node): 1300 # proper_slice: short_slice | long_slice 1301 # short_slice: [lower_bound] ":" [upper_bound] 1302 # long_slice: short_slice ":" [stride] 1303 # lower_bound: expression 1304 # upper_bound: expression 1305 # stride: expression 1306 # 1307 # Note: a stride may be further slicing... 1308 1309 items = [] 1310 1311 if node[1][0] == token.COLON: 1312 items.append(Const(None)) 1313 i = 2 1314 else: 1315 items.append(self.com_node(node[1])) 1316 # i == 2 is a COLON 1317 i = 3 1318 1319 if i < len(node) and node[i][0] == symbol.test: 1320 items.append(self.com_node(node[i])) 1321 i = i + 1 1322 else: 1323 items.append(Const(None)) 1324 1325 # a short_slice has been built. look for long_slice now by looking 1326 # for strides... 1327 for j in range(i, len(node)): 1328 ch = node[j] 1329 if len(ch) == 2: 1330 items.append(Const(None)) 1331 else: 1332 items.append(self.com_node(ch[2])) 1333 return Sliceobj(items, lineno=extractLineNo(node)) 1334 1335 def com_slice(self, primary, node, assigning): 1336 # short_slice: [lower_bound] ":" [upper_bound] 1337 lower = upper = None 1338 if len(node) == 3: 1339 if node[1][0] == token.COLON: 1340 upper = self.com_node(node[2]) 1341 else: 1342 lower = self.com_node(node[1]) 1343 elif len(node) == 4: 1344 lower = self.com_node(node[1]) 1345 upper = self.com_node(node[3]) 1346 return Slice(primary, assigning, lower, upper, 1347 lineno=extractLineNo(node)) 1348 1349 def get_docstring(self, node, n=None): 1350 if n is None: 1351 n = node[0] 1352 node = node[1:] 1353 if n == symbol.suite: 1354 if len(node) == 1: 1355 return self.get_docstring(node[0]) 1356 for sub in node: 1357 if sub[0] == symbol.stmt: 1358 return self.get_docstring(sub) 1359 return None 1360 if n == symbol.file_input: 1361 for sub in node: 1362 if sub[0] == symbol.stmt: 1363 return self.get_docstring(sub) 1364 return None 1365 if n == symbol.atom: 1366 if node[0][0] == token.STRING: 1367 s = '' 1368 for t in node: 1369 s = s + eval(t[1]) 1370 return s 1371 return None 1372 if n == symbol.stmt or n == symbol.simple_stmt \ 1373 or n == symbol.small_stmt: 1374 return self.get_docstring(node[0]) 1375 if n in _doc_nodes and len(node) == 1: 1376 return self.get_docstring(node[0]) 1377 return None 1378 1379 1380 _doc_nodes = [ 1381 symbol.expr_stmt, 1382 symbol.testlist, 1383 symbol.testlist_safe, 1384 symbol.test, 1385 symbol.or_test, 1386 symbol.and_test, 1387 symbol.not_test, 1388 symbol.comparison, 1389 symbol.expr, 1390 symbol.xor_expr, 1391 symbol.and_expr, 1392 symbol.shift_expr, 1393 symbol.arith_expr, 1394 symbol.term, 1395 symbol.factor, 1396 symbol.power, 1397 ] 1398 1399 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 1400 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 1401 _cmp_types = { 1402 token.LESS : '<', 1403 token.GREATER : '>', 1404 token.EQEQUAL : '==', 1405 token.EQUAL : '==', 1406 token.LESSEQUAL : '<=', 1407 token.GREATEREQUAL : '>=', 1408 token.NOTEQUAL : '!=', 1409 } 1410 1411 _legal_node_types = [ 1412 symbol.funcdef, 1413 symbol.classdef, 1414 symbol.stmt, 1415 symbol.small_stmt, 1416 symbol.flow_stmt, 1417 symbol.simple_stmt, 1418 symbol.compound_stmt, 1419 symbol.expr_stmt, 1420 symbol.print_stmt, 1421 symbol.del_stmt, 1422 symbol.pass_stmt, 1423 symbol.break_stmt, 1424 symbol.continue_stmt, 1425 symbol.return_stmt, 1426 symbol.raise_stmt, 1427 symbol.import_stmt, 1428 symbol.global_stmt, 1429 symbol.exec_stmt, 1430 symbol.assert_stmt, 1431 symbol.if_stmt, 1432 symbol.while_stmt, 1433 symbol.for_stmt, 1434 symbol.try_stmt, 1435 symbol.with_stmt, 1436 symbol.suite, 1437 symbol.testlist, 1438 symbol.testlist_safe, 1439 symbol.test, 1440 symbol.and_test, 1441 symbol.not_test, 1442 symbol.comparison, 1443 symbol.exprlist, 1444 symbol.expr, 1445 symbol.xor_expr, 1446 symbol.and_expr, 1447 symbol.shift_expr, 1448 symbol.arith_expr, 1449 symbol.term, 1450 symbol.factor, 1451 symbol.power, 1452 symbol.atom, 1453 ] 1454 1455 if hasattr(symbol, 'yield_stmt'): 1456 _legal_node_types.append(symbol.yield_stmt) 1457 if hasattr(symbol, 'yield_expr'): 1458 _legal_node_types.append(symbol.yield_expr) 1459 1460 _assign_types = [ 1461 symbol.test, 1462 symbol.or_test, 1463 symbol.and_test, 1464 symbol.not_test, 1465 symbol.comparison, 1466 symbol.expr, 1467 symbol.xor_expr, 1468 symbol.and_expr, 1469 symbol.shift_expr, 1470 symbol.arith_expr, 1471 symbol.term, 1472 symbol.factor, 1473 ] 1474 1475 _names = {} 1476 for k, v in symbol.sym_name.items(): 1477 _names[k] = v 1478 for k, v in token.tok_name.items(): 1479 _names[k] = v 1480 1481 def debug_tree(tree): 1482 l = [] 1483 for elt in tree: 1484 if isinstance(elt, int): 1485 l.append(_names.get(elt, elt)) 1486 elif isinstance(elt, str): 1487 l.append(elt) 1488 else: 1489 l.append(debug_tree(elt)) 1490 return l