1 """Parse tree transformation module. 2 3 Transforms Python source code into an abstract syntax tree (AST) 4 defined in the ast module. 5 6 The simplest ways to invoke this module are via parse and parseFile. 7 parse(buf) -> AST 8 parseFile(path) -> AST 9 """ 10 11 # Original version written by Greg Stein (gstein@lyra.org) 12 # and Bill Tutt (rassilon@lima.mudlib.org) 13 # February 1997. 14 # 15 # Modifications and improvements for Python 2.0 by Jeremy Hylton and 16 # Mark Hammond 17 # 18 # Some fixes to try to have correct line number on almost all nodes 19 # (except Module, Discard and Stmt) added by Sylvain Thenault 20 # 21 # Portions of this file are: 22 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. 23 # 24 # This module is provided under a BSD-ish license. See 25 # http://www.opensource.org/licenses/bsd-license.html 26 # and replace OWNER, ORGANIZATION, and YEAR as appropriate. 27 28 from compiler.ast import * 29 import parser 30 import symbol 31 import token 32 33 class WalkerError(StandardError): 34 pass 35 36 from compiler.consts import CO_VARARGS, CO_VARKEYWORDS 37 from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY 38 39 def parseFile(path): 40 f = open(path, "U") 41 # XXX The parser API tolerates files without a trailing newline, 42 # but not strings without a trailing newline. Always add an extra 43 # newline to the file contents, since we're going through the string 44 # version of the API. 45 src = f.read() + "\n" 46 f.close() 47 return parse(src) 48 49 def parse(buf, mode="exec"): 50 if mode == "exec" or mode == "single": 51 return Transformer().parsesuite(buf) 52 elif mode == "eval": 53 return Transformer().parseexpr(buf) 54 else: 55 raise ValueError("compile() arg 3 must be" 56 " 'exec' or 'eval' or 'single'") 57 58 def extractLineNo(ast): 59 if not isinstance(ast[1], tuple): 60 # get a terminal node 61 return ast[2] 62 for child in ast[1:]: 63 if isinstance(child, tuple): 64 lineno = extractLineNo(child) 65 if lineno is not None: 66 return lineno 67 68 def Node(*args): 69 kind = args[0] 70 if kind in nodes: 71 try: 72 return nodes[kind](*args[1:]) 73 except TypeError: 74 print nodes[kind], len(args), args 75 raise 76 else: 77 raise WalkerError, "Can't find appropriate Node type: %s" % str(args) 78 #return apply(ast.Node, args) 79 80 class Transformer: 81 """Utility object for transforming Python parse trees. 82 83 Exposes the following methods: 84 tree = transform(ast_tree) 85 tree = parsesuite(text) 86 tree = parseexpr(text) 87 tree = parsefile(fileob | filename) 88 """ 89 90 def __init__(self): 91 self._dispatch = {} 92 for value, name in symbol.sym_name.items(): 93 if hasattr(self, name): 94 self._dispatch[value] = getattr(self, name) 95 self._dispatch[token.NEWLINE] = self.com_NEWLINE 96 self._atom_dispatch = {token.LPAR: self.atom_lpar, 97 token.LSQB: self.atom_lsqb, 98 token.LBRACE: self.atom_lbrace, 99 token.BACKQUOTE: self.atom_backquote, 100 token.NUMBER: self.atom_number, 101 token.STRING: self.atom_string, 102 token.NAME: self.atom_name, 103 } 104 self.encoding = None 105 106 def transform(self, tree): 107 """Transform an AST into a modified parse tree.""" 108 if not (isinstance(tree, tuple) or isinstance(tree, list)): 109 tree = parser.st2tuple(tree, line_info=1) 110 return self.compile_node(tree) 111 112 def parsesuite(self, text): 113 """Return a modified parse tree for the given suite text.""" 114 return self.transform(parser.suite(text)) 115 116 def parseexpr(self, text): 117 """Return a modified parse tree for the given expression text.""" 118 return self.transform(parser.expr(text)) 119 120 def parsefile(self, file): 121 """Return a modified parse tree for the contents of the given file.""" 122 if type(file) == type(''): 123 file = open(file) 124 return self.parsesuite(file.read()) 125 126 # -------------------------------------------------------------- 127 # 128 # PRIVATE METHODS 129 # 130 131 def compile_node(self, node): 132 ### emit a line-number node? 133 n = node[0] 134 135 if n == symbol.encoding_decl: 136 self.encoding = node[2] 137 node = node[1] 138 n = node[0] 139 140 if n == symbol.single_input: 141 return self.single_input(node[1:]) 142 if n == symbol.file_input: 143 return self.file_input(node[1:]) 144 if n == symbol.eval_input: 145 return self.eval_input(node[1:]) 146 if n == symbol.lambdef: 147 return self.lambdef(node[1:]) 148 if n == symbol.funcdef: 149 return self.funcdef(node[1:]) 150 if n == symbol.classdef: 151 return self.classdef(node[1:]) 152 153 raise WalkerError, ('unexpected node type', n) 154 155 def single_input(self, node): 156 ### do we want to do anything about being "interactive" ? 157 158 # NEWLINE | simple_stmt | compound_stmt NEWLINE 159 n = node[0][0] 160 if n != token.NEWLINE: 161 return self.com_stmt(node[0]) 162 163 return Pass() 164 165 def file_input(self, nodelist): 166 doc = self.get_docstring(nodelist, symbol.file_input) 167 if doc is not None: 168 i = 1 169 else: 170 i = 0 171 stmts = [] 172 for node in nodelist[i:]: 173 if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: 174 self.com_append_stmt(stmts, node) 175 return Module(doc, Stmt(stmts)) 176 177 def eval_input(self, nodelist): 178 # from the built-in function input() 179 ### is this sufficient? 180 return Expression(self.com_node(nodelist[0])) 181 182 def decorator_name(self, nodelist): 183 listlen = len(nodelist) 184 assert listlen >= 1 and listlen % 2 == 1 185 186 item = self.atom_name(nodelist) 187 i = 1 188 while i < listlen: 189 assert nodelist[i][0] == token.DOT 190 assert nodelist[i + 1][0] == token.NAME 191 item = Getattr(item, nodelist[i + 1][1]) 192 i += 2 193 194 return item 195 196 def decorator(self, nodelist): 197 # '@' dotted_name [ '(' [arglist] ')' ] 198 assert len(nodelist) in (3, 5, 6) 199 assert nodelist[0][0] == token.AT 200 assert nodelist[-1][0] == token.NEWLINE 201 202 assert nodelist[1][0] == symbol.dotted_name 203 funcname = self.decorator_name(nodelist[1][1:]) 204 205 if len(nodelist) > 3: 206 assert nodelist[2][0] == token.LPAR 207 expr = self.com_call_function(funcname, nodelist[3]) 208 else: 209 expr = funcname 210 211 return expr 212 213 def decorators(self, nodelist): 214 # decorators: decorator ([NEWLINE] decorator)* NEWLINE 215 items = [] 216 for dec_nodelist in nodelist: 217 assert dec_nodelist[0] == symbol.decorator 218 items.append(self.decorator(dec_nodelist[1:])) 219 return Decorators(items) 220 221 def decorated(self, nodelist): 222 assert nodelist[0][0] == symbol.decorators 223 if nodelist[1][0] == symbol.funcdef: 224 n = [nodelist[0]] + list(nodelist[1][1:]) 225 return self.funcdef(n) 226 elif nodelist[1][0] == symbol.classdef: 227 decorators = self.decorators(nodelist[0][1:]) 228 cls = self.classdef(nodelist[1][1:]) 229 cls.decorators = decorators 230 return cls 231 raise WalkerError() 232 233 def funcdef(self, nodelist): 234 # -6 -5 -4 -3 -2 -1 235 # funcdef: [decorators] 'def' NAME parameters ':' suite 236 # parameters: '(' [varargslist] ')' 237 238 if len(nodelist) == 6: 239 assert nodelist[0][0] == symbol.decorators 240 decorators = self.decorators(nodelist[0][1:]) 241 else: 242 assert len(nodelist) == 5 243 decorators = None 244 245 lineno = nodelist[-4][2] 246 name = nodelist[-4][1] 247 args = nodelist[-3][2] 248 249 if args[0] == symbol.varargslist: 250 names, defaults, flags = self.com_arglist(args[1:]) 251 else: 252 names = defaults = () 253 flags = 0 254 doc = self.get_docstring(nodelist[-1]) 255 256 # code for function 257 code = self.com_node(nodelist[-1]) 258 259 if doc is not None: 260 assert isinstance(code, Stmt) 261 assert isinstance(code.nodes[0], Discard) 262 del code.nodes[0] 263 return Function(decorators, name, names, defaults, flags, doc, code, 264 lineno=lineno) 265 266 def lambdef(self, nodelist): 267 # lambdef: 'lambda' [varargslist] ':' test 268 if nodelist[2][0] == symbol.varargslist: 269 names, defaults, flags = self.com_arglist(nodelist[2][1:]) 270 else: 271 names = defaults = () 272 flags = 0 273 274 # code for lambda 275 code = self.com_node(nodelist[-1]) 276 277 return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) 278 old_lambdef = lambdef 279 280 def classdef(self, nodelist): 281 # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 282 283 name = nodelist[1][1] 284 doc = self.get_docstring(nodelist[-1]) 285 if nodelist[2][0] == token.COLON: 286 bases = [] 287 elif nodelist[3][0] == token.RPAR: 288 bases = [] 289 else: 290 bases = self.com_bases(nodelist[3]) 291 292 # code for class 293 code = self.com_node(nodelist[-1]) 294 295 if doc is not None: 296 assert isinstance(code, Stmt) 297 assert isinstance(code.nodes[0], Discard) 298 del code.nodes[0] 299 300 return Class(name, bases, doc, code, lineno=nodelist[1][2]) 301 302 def stmt(self, nodelist): 303 return self.com_stmt(nodelist[0]) 304 305 small_stmt = stmt 306 flow_stmt = stmt 307 compound_stmt = stmt 308 309 def simple_stmt(self, nodelist): 310 # small_stmt (';' small_stmt)* [';'] NEWLINE 311 stmts = [] 312 for i in range(0, len(nodelist), 2): 313 self.com_append_stmt(stmts, nodelist[i]) 314 return Stmt(stmts) 315 316 def parameters(self, nodelist): 317 raise WalkerError 318 319 def varargslist(self, nodelist): 320 raise WalkerError 321 322 def fpdef(self, nodelist): 323 raise WalkerError 324 325 def fplist(self, nodelist): 326 raise WalkerError 327 328 def dotted_name(self, nodelist): 329 raise WalkerError 330 331 def comp_op(self, nodelist): 332 raise WalkerError 333 334 def trailer(self, nodelist): 335 raise WalkerError 336 337 def sliceop(self, nodelist): 338 raise WalkerError 339 340 def argument(self, nodelist): 341 raise WalkerError 342 343 # -------------------------------------------------------------- 344 # 345 # STATEMENT NODES (invoked by com_node()) 346 # 347 348 def expr_stmt(self, nodelist): 349 # augassign testlist | testlist ('=' testlist)* 350 en = nodelist[-1] 351 exprNode = self.lookup_node(en)(en[1:]) 352 if len(nodelist) == 1: 353 return Discard(exprNode, lineno=exprNode.lineno) 354 if nodelist[1][0] == token.EQUAL: 355 nodesl = [] 356 for i in range(0, len(nodelist) - 2, 2): 357 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) 358 return Assign(nodesl, exprNode, lineno=nodelist[1][2]) 359 else: 360 lval = self.com_augassign(nodelist[0]) 361 op = self.com_augassign_op(nodelist[1]) 362 return AugAssign(lval, op[1], exprNode, lineno=op[2]) 363 raise WalkerError, "can't get here" 364 365 def print_stmt(self, nodelist): 366 # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) 367 items = [] 368 if len(nodelist) == 1: 369 start = 1 370 dest = None 371 elif nodelist[1][0] == token.RIGHTSHIFT: 372 assert len(nodelist) == 3 \ 373 or nodelist[3][0] == token.COMMA 374 dest = self.com_node(nodelist[2]) 375 start = 4 376 else: 377 dest = None 378 start = 1 379 for i in range(start, len(nodelist), 2): 380 items.append(self.com_node(nodelist[i])) 381 if nodelist[-1][0] == token.COMMA: 382 return Print(items, dest, lineno=nodelist[0][2]) 383 return Printnl(items, dest, lineno=nodelist[0][2]) 384 385 def del_stmt(self, nodelist): 386 return self.com_assign(nodelist[1], OP_DELETE) 387 388 def pass_stmt(self, nodelist): 389 return Pass(lineno=nodelist[0][2]) 390 391 def break_stmt(self, nodelist): 392 return Break(lineno=nodelist[0][2]) 393 394 def continue_stmt(self, nodelist): 395 return Continue(lineno=nodelist[0][2]) 396 397 def return_stmt(self, nodelist): 398 # return: [testlist] 399 if len(nodelist) < 2: 400 return Return(Const(None), lineno=nodelist[0][2]) 401 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 402 403 def yield_stmt(self, nodelist): 404 expr = self.com_node(nodelist[0]) 405 return Discard(expr, lineno=expr.lineno) 406 407 def yield_expr(self, nodelist): 408 if len(nodelist) > 1: 409 value = self.com_node(nodelist[1]) 410 else: 411 value = Const(None) 412 return Yield(value, lineno=nodelist[0][2]) 413 414 def raise_stmt(self, nodelist): 415 # raise: [test [',' test [',' test]]] 416 if len(nodelist) > 5: 417 expr3 = self.com_node(nodelist[5]) 418 else: 419 expr3 = None 420 if len(nodelist) > 3: 421 expr2 = self.com_node(nodelist[3]) 422 else: 423 expr2 = None 424 if len(nodelist) > 1: 425 expr1 = self.com_node(nodelist[1]) 426 else: 427 expr1 = None 428 return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) 429 430 def import_stmt(self, nodelist): 431 # import_stmt: import_name | import_from 432 assert len(nodelist) == 1 433 return self.com_node(nodelist[0]) 434 435 def import_name(self, nodelist): 436 # import_name: 'import' dotted_as_names 437 return Import(self.com_dotted_as_names(nodelist[1]), 438 lineno=nodelist[0][2]) 439 440 def import_from(self, nodelist): 441 # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | 442 # '(' import_as_names ')' | import_as_names) 443 assert nodelist[0][1] == 'from' 444 idx = 1 445 while nodelist[idx][1] == '.': 446 idx += 1 447 level = idx - 1 448 if nodelist[idx][0] == symbol.dotted_name: 449 fromname = self.com_dotted_name(nodelist[idx]) 450 idx += 1 451 else: 452 fromname = "" 453 assert nodelist[idx][1] == 'import' 454 if nodelist[idx + 1][0] == token.STAR: 455 return From(fromname, [('*', None)], level, 456 lineno=nodelist[0][2]) 457 else: 458 node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)] 459 return From(fromname, self.com_import_as_names(node), level, 460 lineno=nodelist[0][2]) 461 462 def global_stmt(self, nodelist): 463 # global: NAME (',' NAME)* 464 names = [] 465 for i in range(1, len(nodelist), 2): 466 names.append(nodelist[i][1]) 467 return Global(names, lineno=nodelist[0][2]) 468 469 def exec_stmt(self, nodelist): 470 # exec_stmt: 'exec' expr ['in' expr [',' expr]] 471 expr1 = self.com_node(nodelist[1]) 472 if len(nodelist) >= 4: 473 expr2 = self.com_node(nodelist[3]) 474 if len(nodelist) >= 6: 475 expr3 = self.com_node(nodelist[5]) 476 else: 477 expr3 = None 478 else: 479 expr2 = expr3 = None 480 481 return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) 482 483 def assert_stmt(self, nodelist): 484 # 'assert': test, [',' test] 485 expr1 = self.com_node(nodelist[1]) 486 if (len(nodelist) == 4): 487 expr2 = self.com_node(nodelist[3]) 488 else: 489 expr2 = None 490 return Assert(expr1, expr2, lineno=nodelist[0][2]) 491 492 def if_stmt(self, nodelist): 493 # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 494 tests = [] 495 for i in range(0, len(nodelist) - 3, 4): 496 testNode = self.com_node(nodelist[i + 1]) 497 suiteNode = self.com_node(nodelist[i + 3]) 498 tests.append((testNode, suiteNode)) 499 500 if len(nodelist) % 4 == 3: 501 elseNode = self.com_node(nodelist[-1]) 502 ## elseNode.lineno = nodelist[-1][1][2] 503 else: 504 elseNode = None 505 return If(tests, elseNode, lineno=nodelist[0][2]) 506 507 def while_stmt(self, nodelist): 508 # 'while' test ':' suite ['else' ':' suite] 509 510 testNode = self.com_node(nodelist[1]) 511 bodyNode = self.com_node(nodelist[3]) 512 513 if len(nodelist) > 4: 514 elseNode = self.com_node(nodelist[6]) 515 else: 516 elseNode = None 517 518 return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) 519 520 def for_stmt(self, nodelist): 521 # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] 522 523 assignNode = self.com_assign(nodelist[1], OP_ASSIGN) 524 listNode = self.com_node(nodelist[3]) 525 bodyNode = self.com_node(nodelist[5]) 526 527 if len(nodelist) > 8: 528 elseNode = self.com_node(nodelist[8]) 529 else: 530 elseNode = None 531 532 return For(assignNode, listNode, bodyNode, elseNode, 533 lineno=nodelist[0][2]) 534 535 def try_stmt(self, nodelist): 536 return self.com_try_except_finally(nodelist) 537 538 def with_stmt(self, nodelist): 539 return self.com_with(nodelist) 540 541 def suite(self, nodelist): 542 # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT 543 if len(nodelist) == 1: 544 return self.com_stmt(nodelist[0]) 545 546 stmts = [] 547 for node in nodelist: 548 if node[0] == symbol.stmt: 549 self.com_append_stmt(stmts, node) 550 return Stmt(stmts) 551 552 # -------------------------------------------------------------- 553 # 554 # EXPRESSION NODES (invoked by com_node()) 555 # 556 557 def testlist(self, nodelist): 558 # testlist: expr (',' expr)* [','] 559 # testlist_safe: test [(',' test)+ [',']] 560 # exprlist: expr (',' expr)* [','] 561 return self.com_binary(Tuple, nodelist) 562 563 testlist_safe = testlist # XXX 564 testlist1 = testlist 565 exprlist = testlist 566 567 def testlist_comp(self, nodelist): 568 # test ( comp_for | (',' test)* [','] ) 569 assert nodelist[0][0] == symbol.test 570 if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for: 571 test = self.com_node(nodelist[0]) 572 return self.com_generator_expression(test, nodelist[1]) 573 return self.testlist(nodelist) 574 575 def test(self, nodelist): 576 # or_test ['if' or_test 'else' test] | lambdef 577 if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: 578 return self.lambdef(nodelist[0]) 579 then = self.com_node(nodelist[0]) 580 if len(nodelist) > 1: 581 assert len(nodelist) == 5 582 assert nodelist[1][1] == 'if' 583 assert nodelist[3][1] == 'else' 584 test = self.com_node(nodelist[2]) 585 else_ = self.com_node(nodelist[4]) 586 return IfExp(test, then, else_, lineno=nodelist[1][2]) 587 return then 588 589 def or_test(self, nodelist): 590 # and_test ('or' and_test)* | lambdef 591 if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: 592 return self.lambdef(nodelist[0]) 593 return self.com_binary(Or, nodelist) 594 old_test = or_test 595 596 def and_test(self, nodelist): 597 # not_test ('and' not_test)* 598 return self.com_binary(And, nodelist) 599 600 def not_test(self, nodelist): 601 # 'not' not_test | comparison 602 result = self.com_node(nodelist[-1]) 603 if len(nodelist) == 2: 604 return Not(result, lineno=nodelist[0][2]) 605 return result 606 607 def comparison(self, nodelist): 608 # comparison: expr (comp_op expr)* 609 node = self.com_node(nodelist[0]) 610 if len(nodelist) == 1: 611 return node 612 613 results = [] 614 for i in range(2, len(nodelist), 2): 615 nl = nodelist[i-1] 616 617 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 618 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 619 n = nl[1] 620 if n[0] == token.NAME: 621 type = n[1] 622 if len(nl) == 3: 623 if type == 'not': 624 type = 'not in' 625 else: 626 type = 'is not' 627 else: 628 type = _cmp_types[n[0]] 629 630 lineno = nl[1][2] 631 results.append((type, self.com_node(nodelist[i]))) 632 633 # we need a special "compare" node so that we can distinguish 634 # 3 < x < 5 from (3 < x) < 5 635 # the two have very different semantics and results (note that the 636 # latter form is always true) 637 638 return Compare(node, results, lineno=lineno) 639 640 def expr(self, nodelist): 641 # xor_expr ('|' xor_expr)* 642 return self.com_binary(Bitor, nodelist) 643 644 def xor_expr(self, nodelist): 645 # xor_expr ('^' xor_expr)* 646 return self.com_binary(Bitxor, nodelist) 647 648 def and_expr(self, nodelist): 649 # xor_expr ('&' xor_expr)* 650 return self.com_binary(Bitand, nodelist) 651 652 def shift_expr(self, nodelist): 653 # shift_expr ('<<'|'>>' shift_expr)* 654 node = self.com_node(nodelist[0]) 655 for i in range(2, len(nodelist), 2): 656 right = self.com_node(nodelist[i]) 657 if nodelist[i-1][0] == token.LEFTSHIFT: 658 node = LeftShift([node, right], lineno=nodelist[1][2]) 659 elif nodelist[i-1][0] == token.RIGHTSHIFT: 660 node = RightShift([node, right], lineno=nodelist[1][2]) 661 else: 662 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 663 return node 664 665 def arith_expr(self, nodelist): 666 node = self.com_node(nodelist[0]) 667 for i in range(2, len(nodelist), 2): 668 right = self.com_node(nodelist[i]) 669 if nodelist[i-1][0] == token.PLUS: 670 node = Add([node, right], lineno=nodelist[1][2]) 671 elif nodelist[i-1][0] == token.MINUS: 672 node = Sub([node, right], lineno=nodelist[1][2]) 673 else: 674 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 675 return node 676 677 def term(self, nodelist): 678 node = self.com_node(nodelist[0]) 679 for i in range(2, len(nodelist), 2): 680 right = self.com_node(nodelist[i]) 681 t = nodelist[i-1][0] 682 if t == token.STAR: 683 node = Mul([node, right]) 684 elif t == token.SLASH: 685 node = Div([node, right]) 686 elif t == token.PERCENT: 687 node = Mod([node, right]) 688 elif t == token.DOUBLESLASH: 689 node = FloorDiv([node, right]) 690 else: 691 raise ValueError, "unexpected token: %s" % t 692 node.lineno = nodelist[1][2] 693 return node 694 695 def factor(self, nodelist): 696 elt = nodelist[0] 697 t = elt[0] 698 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) 699 # need to handle (unary op)constant here... 700 if t == token.PLUS: 701 return UnaryAdd(node, lineno=elt[2]) 702 elif t == token.MINUS: 703 return UnarySub(node, lineno=elt[2]) 704 elif t == token.TILDE: 705 node = Invert(node, lineno=elt[2]) 706 return node 707 708 def power(self, nodelist): 709 # power: atom trailer* ('**' factor)* 710 node = self.com_node(nodelist[0]) 711 for i in range(1, len(nodelist)): 712 elt = nodelist[i] 713 if elt[0] == token.DOUBLESTAR: 714 return Power([node, self.com_node(nodelist[i+1])], 715 lineno=elt[2]) 716 717 node = self.com_apply_trailer(node, elt) 718 719 return node 720 721 def atom(self, nodelist): 722 return self._atom_dispatch[nodelist[0][0]](nodelist) 723 724 def atom_lpar(self, nodelist): 725 if nodelist[1][0] == token.RPAR: 726 return Tuple((), lineno=nodelist[0][2]) 727 return self.com_node(nodelist[1]) 728 729 def atom_lsqb(self, nodelist): 730 if nodelist[1][0] == token.RSQB: 731 return List((), lineno=nodelist[0][2]) 732 return self.com_list_constructor(nodelist[1]) 733 734 def atom_lbrace(self, nodelist): 735 if nodelist[1][0] == token.RBRACE: 736 return Dict((), lineno=nodelist[0][2]) 737 return self.com_dictorsetmaker(nodelist[1]) 738 739 def atom_backquote(self, nodelist): 740 return Backquote(self.com_node(nodelist[1])) 741 742 def atom_number(self, nodelist): 743 ### need to verify this matches compile.c 744 k = eval(nodelist[0][1]) 745 return Const(k, lineno=nodelist[0][2]) 746 747 def decode_literal(self, lit): 748 if self.encoding: 749 # this is particularly fragile & a bit of a 750 # hack... changes in compile.c:parsestr and 751 # tokenizer.c must be reflected here. 752 if self.encoding not in ['utf-8', 'iso-8859-1']: 753 lit = unicode(lit, 'utf-8').encode(self.encoding) 754 return eval("# coding: %s\n%s" % (self.encoding, lit)) 755 else: 756 return eval(lit) 757 758 def atom_string(self, nodelist): 759 k = '' 760 for node in nodelist: 761 k += self.decode_literal(node[1]) 762 return Const(k, lineno=nodelist[0][2]) 763 764 def atom_name(self, nodelist): 765 return Name(nodelist[0][1], lineno=nodelist[0][2]) 766 767 # -------------------------------------------------------------- 768 # 769 # INTERNAL PARSING UTILITIES 770 # 771 772 # The use of com_node() introduces a lot of extra stack frames, 773 # enough to cause a stack overflow compiling test.test_parser with 774 # the standard interpreter recursionlimit. The com_node() is a 775 # convenience function that hides the dispatch details, but comes 776 # at a very high cost. It is more efficient to dispatch directly 777 # in the callers. In these cases, use lookup_node() and call the 778 # dispatched node directly. 779 780 def lookup_node(self, node): 781 return self._dispatch[node[0]] 782 783 def com_node(self, node): 784 # Note: compile.c has handling in com_node for del_stmt, pass_stmt, 785 # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, 786 # and compound_stmt. 787 # We'll just dispatch them. 788 return self._dispatch[node[0]](node[1:]) 789 790 def com_NEWLINE(self, *args): 791 # A ';' at the end of a line can make a NEWLINE token appear 792 # here, Render it harmless. (genc discards ('discard', 793 # ('const', xxxx)) Nodes) 794 return Discard(Const(None)) 795 796 def com_arglist(self, nodelist): 797 # varargslist: 798 # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) 799 # | fpdef ['=' test] (',' fpdef ['=' test])* [','] 800 # fpdef: NAME | '(' fplist ')' 801 # fplist: fpdef (',' fpdef)* [','] 802 names = [] 803 defaults = [] 804 flags = 0 805 806 i = 0 807 while i < len(nodelist): 808 node = nodelist[i] 809 if node[0] == token.STAR or node[0] == token.DOUBLESTAR: 810 if node[0] == token.STAR: 811 node = nodelist[i+1] 812 if node[0] == token.NAME: 813 names.append(node[1]) 814 flags = flags | CO_VARARGS 815 i = i + 3 816 817 if i < len(nodelist): 818 # should be DOUBLESTAR 819 t = nodelist[i][0] 820 if t == token.DOUBLESTAR: 821 node = nodelist[i+1] 822 else: 823 raise ValueError, "unexpected token: %s" % t 824 names.append(node[1]) 825 flags = flags | CO_VARKEYWORDS 826 827 break 828 829 # fpdef: NAME | '(' fplist ')' 830 names.append(self.com_fpdef(node)) 831 832 i = i + 1 833 if i < len(nodelist) and nodelist[i][0] == token.EQUAL: 834 defaults.append(self.com_node(nodelist[i + 1])) 835 i = i + 2 836 elif len(defaults): 837 # we have already seen an argument with default, but here 838 # came one without 839 raise SyntaxError, "non-default argument follows default argument" 840 841 # skip the comma 842 i = i + 1 843 844 return names, defaults, flags 845 846 def com_fpdef(self, node): 847 # fpdef: NAME | '(' fplist ')' 848 if node[1][0] == token.LPAR: 849 return self.com_fplist(node[2]) 850 return node[1][1] 851 852 def com_fplist(self, node): 853 # fplist: fpdef (',' fpdef)* [','] 854 if len(node) == 2: 855 return self.com_fpdef(node[1]) 856 list = [] 857 for i in range(1, len(node), 2): 858 list.append(self.com_fpdef(node[i])) 859 return tuple(list) 860 861 def com_dotted_name(self, node): 862 # String together the dotted names and return the string 863 name = "" 864 for n in node: 865 if type(n) == type(()) and n[0] == 1: 866 name = name + n[1] + '.' 867 return name[:-1] 868 869 def com_dotted_as_name(self, node): 870 assert node[0] == symbol.dotted_as_name 871 node = node[1:] 872 dot = self.com_dotted_name(node[0][1:]) 873 if len(node) == 1: 874 return dot, None 875 assert node[1][1] == 'as' 876 assert node[2][0] == token.NAME 877 return dot, node[2][1] 878 879 def com_dotted_as_names(self, node): 880 assert node[0] == symbol.dotted_as_names 881 node = node[1:] 882 names = [self.com_dotted_as_name(node[0])] 883 for i in range(2, len(node), 2): 884 names.append(self.com_dotted_as_name(node[i])) 885 return names 886 887 def com_import_as_name(self, node): 888 assert node[0] == symbol.import_as_name 889 node = node[1:] 890 assert node[0][0] == token.NAME 891 if len(node) == 1: 892 return node[0][1], None 893 assert node[1][1] == 'as', node 894 assert node[2][0] == token.NAME 895 return node[0][1], node[2][1] 896 897 def com_import_as_names(self, node): 898 assert node[0] == symbol.import_as_names 899 node = node[1:] 900 names = [self.com_import_as_name(node[0])] 901 for i in range(2, len(node), 2): 902 names.append(self.com_import_as_name(node[i])) 903 return names 904 905 def com_bases(self, node): 906 bases = [] 907 for i in range(1, len(node), 2): 908 bases.append(self.com_node(node[i])) 909 return bases 910 911 def com_try_except_finally(self, nodelist): 912 # ('try' ':' suite 913 # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] 914 # | 'finally' ':' suite)) 915 916 if nodelist[3][0] == token.NAME: 917 # first clause is a finally clause: only try-finally 918 return TryFinally(self.com_node(nodelist[2]), 919 self.com_node(nodelist[5]), 920 lineno=nodelist[0][2]) 921 922 #tryexcept: [TryNode, [except_clauses], elseNode)] 923 clauses = [] 924 elseNode = None 925 finallyNode = None 926 for i in range(3, len(nodelist), 3): 927 node = nodelist[i] 928 if node[0] == symbol.except_clause: 929 # except_clause: 'except' [expr [(',' | 'as') expr]] */ 930 if len(node) > 2: 931 expr1 = self.com_node(node[2]) 932 if len(node) > 4: 933 expr2 = self.com_assign(node[4], OP_ASSIGN) 934 else: 935 expr2 = None 936 else: 937 expr1 = expr2 = None 938 clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) 939 940 if node[0] == token.NAME: 941 if node[1] == 'else': 942 elseNode = self.com_node(nodelist[i+2]) 943 elif node[1] == 'finally': 944 finallyNode = self.com_node(nodelist[i+2]) 945 try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, 946 lineno=nodelist[0][2]) 947 if finallyNode: 948 return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) 949 else: 950 return try_except 951 952 def com_with(self, nodelist): 953 # with_stmt: 'with' with_item (',' with_item)* ':' suite 954 body = self.com_node(nodelist[-1]) 955 for i in range(len(nodelist) - 3, 0, -2): 956 ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) 957 if i == 1: 958 return ret 959 body = ret 960 961 def com_with_item(self, nodelist, body, lineno): 962 # with_item: test ['as' expr] 963 if len(nodelist) == 4: 964 var = self.com_assign(nodelist[3], OP_ASSIGN) 965 else: 966 var = None 967 expr = self.com_node(nodelist[1]) 968 return With(expr, var, body, lineno=lineno) 969 970 def com_augassign_op(self, node): 971 assert node[0] == symbol.augassign 972 return node[1] 973 974 def com_augassign(self, node): 975 """Return node suitable for lvalue of augmented assignment 976 977 Names, slices, and attributes are the only allowable nodes. 978 """ 979 l = self.com_node(node) 980 if l.__class__ in (Name, Slice, Subscript, Getattr): 981 return l 982 raise SyntaxError, "can't assign to %s" % l.__class__.__name__ 983 984 def com_assign(self, node, assigning): 985 # return a node suitable for use as an "lvalue" 986 # loop to avoid trivial recursion 987 while 1: 988 t = node[0] 989 if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp): 990 if len(node) > 2: 991 return self.com_assign_tuple(node, assigning) 992 node = node[1] 993 elif t in _assign_types: 994 if len(node) > 2: 995 raise SyntaxError, "can't assign to operator" 996 node = node[1] 997 elif t == symbol.power: 998 if node[1][0] != symbol.atom: 999 raise SyntaxError, "can't assign to operator" 1000 if len(node) > 2: 1001 primary = self.com_node(node[1]) 1002 for i in range(2, len(node)-1): 1003 ch = node[i] 1004 if ch[0] == token.DOUBLESTAR: 1005 raise SyntaxError, "can't assign to operator" 1006 primary = self.com_apply_trailer(primary, ch) 1007 return self.com_assign_trailer(primary, node[-1], 1008 assigning) 1009 node = node[1] 1010 elif t == symbol.atom: 1011 t = node[1][0] 1012 if t == token.LPAR: 1013 node = node[2] 1014 if node[0] == token.RPAR: 1015 raise SyntaxError, "can't assign to ()" 1016 elif t == token.LSQB: 1017 node = node[2] 1018 if node[0] == token.RSQB: 1019 raise SyntaxError, "can't assign to []" 1020 return self.com_assign_list(node, assigning) 1021 elif t == token.NAME: 1022 return self.com_assign_name(node[1], assigning) 1023 else: 1024 raise SyntaxError, "can't assign to literal" 1025 else: 1026 raise SyntaxError, "bad assignment (%s)" % t 1027 1028 def com_assign_tuple(self, node, assigning): 1029 assigns = [] 1030 for i in range(1, len(node), 2): 1031 assigns.append(self.com_assign(node[i], assigning)) 1032 return AssTuple(assigns, lineno=extractLineNo(node)) 1033 1034 def com_assign_list(self, node, assigning): 1035 assigns = [] 1036 for i in range(1, len(node), 2): 1037 if i + 1 < len(node): 1038 if node[i + 1][0] == symbol.list_for: 1039 raise SyntaxError, "can't assign to list comprehension" 1040 assert node[i + 1][0] == token.COMMA, node[i + 1] 1041 assigns.append(self.com_assign(node[i], assigning)) 1042 return AssList(assigns, lineno=extractLineNo(node)) 1043 1044 def com_assign_name(self, node, assigning): 1045 return AssName(node[1], assigning, lineno=node[2]) 1046 1047 def com_assign_trailer(self, primary, node, assigning): 1048 t = node[1][0] 1049 if t == token.DOT: 1050 return self.com_assign_attr(primary, node[2], assigning) 1051 if t == token.LSQB: 1052 return self.com_subscriptlist(primary, node[2], assigning) 1053 if t == token.LPAR: 1054 raise SyntaxError, "can't assign to function call" 1055 raise SyntaxError, "unknown trailer type: %s" % t 1056 1057 def com_assign_attr(self, primary, node, assigning): 1058 return AssAttr(primary, node[1], assigning, lineno=node[-1]) 1059 1060 def com_binary(self, constructor, nodelist): 1061 "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." 1062 l = len(nodelist) 1063 if l == 1: 1064 n = nodelist[0] 1065 return self.lookup_node(n)(n[1:]) 1066 items = [] 1067 for i in range(0, l, 2): 1068 n = nodelist[i] 1069 items.append(self.lookup_node(n)(n[1:])) 1070 return constructor(items, lineno=extractLineNo(nodelist)) 1071 1072 def com_stmt(self, node): 1073 result = self.lookup_node(node)(node[1:]) 1074 assert result is not None 1075 if isinstance(result, Stmt): 1076 return result 1077 return Stmt([result]) 1078 1079 def com_append_stmt(self, stmts, node): 1080 result = self.lookup_node(node)(node[1:]) 1081 assert result is not None 1082 if isinstance(result, Stmt): 1083 stmts.extend(result.nodes) 1084 else: 1085 stmts.append(result) 1086 1087 def com_list_constructor(self, nodelist): 1088 # listmaker: test ( list_for | (',' test)* [','] ) 1089 values = [] 1090 for i in range(1, len(nodelist)): 1091 if nodelist[i][0] == symbol.list_for: 1092 assert len(nodelist[i:]) == 1 1093 return self.com_list_comprehension(values[0], 1094 nodelist[i]) 1095 elif nodelist[i][0] == token.COMMA: 1096 continue 1097 values.append(self.com_node(nodelist[i])) 1098 return List(values, lineno=values[0].lineno) 1099 1100 def com_list_comprehension(self, expr, node): 1101 return self.com_comprehension(expr, None, node, 'list') 1102 1103 def com_comprehension(self, expr1, expr2, node, type): 1104 # list_iter: list_for | list_if 1105 # list_for: 'for' exprlist 'in' testlist [list_iter] 1106 # list_if: 'if' test [list_iter] 1107 1108 # XXX should raise SyntaxError for assignment 1109 # XXX(avassalotti) Set and dict comprehensions should have generator 1110 # semantics. In other words, they shouldn't leak 1111 # variables outside of the comprehension's scope. 1112 1113 lineno = node[1][2] 1114 fors = [] 1115 while node: 1116 t = node[1][1] 1117 if t == 'for': 1118 assignNode = self.com_assign(node[2], OP_ASSIGN) 1119 compNode = self.com_node(node[4]) 1120 newfor = ListCompFor(assignNode, compNode, []) 1121 newfor.lineno = node[1][2] 1122 fors.append(newfor) 1123 if len(node) == 5: 1124 node = None 1125 elif type == 'list': 1126 node = self.com_list_iter(node[5]) 1127 else: 1128 node = self.com_comp_iter(node[5]) 1129 elif t == 'if': 1130 test = self.com_node(node[2]) 1131 newif = ListCompIf(test, lineno=node[1][2]) 1132 newfor.ifs.append(newif) 1133 if len(node) == 3: 1134 node = None 1135 elif type == 'list': 1136 node = self.com_list_iter(node[3]) 1137 else: 1138 node = self.com_comp_iter(node[3]) 1139 else: 1140 raise SyntaxError, \ 1141 ("unexpected comprehension element: %s %d" 1142 % (node, lineno)) 1143 if type == 'list': 1144 return ListComp(expr1, fors, lineno=lineno) 1145 elif type == 'set': 1146 return SetComp(expr1, fors, lineno=lineno) 1147 elif type == 'dict': 1148 return DictComp(expr1, expr2, fors, lineno=lineno) 1149 else: 1150 raise ValueError("unexpected comprehension type: " + repr(type)) 1151 1152 def com_list_iter(self, node): 1153 assert node[0] == symbol.list_iter 1154 return node[1] 1155 1156 def com_comp_iter(self, node): 1157 assert node[0] == symbol.comp_iter 1158 return node[1] 1159 1160 def com_generator_expression(self, expr, node): 1161 # comp_iter: comp_for | comp_if 1162 # comp_for: 'for' exprlist 'in' test [comp_iter] 1163 # comp_if: 'if' test [comp_iter] 1164 1165 lineno = node[1][2] 1166 fors = [] 1167 while node: 1168 t = node[1][1] 1169 if t == 'for': 1170 assignNode = self.com_assign(node[2], OP_ASSIGN) 1171 genNode = self.com_node(node[4]) 1172 newfor = GenExprFor(assignNode, genNode, [], 1173 lineno=node[1][2]) 1174 fors.append(newfor) 1175 if (len(node)) == 5: 1176 node = None 1177 else: 1178 node = self.com_comp_iter(node[5]) 1179 elif t == 'if': 1180 test = self.com_node(node[2]) 1181 newif = GenExprIf(test, lineno=node[1][2]) 1182 newfor.ifs.append(newif) 1183 if len(node) == 3: 1184 node = None 1185 else: 1186 node = self.com_comp_iter(node[3]) 1187 else: 1188 raise SyntaxError, \ 1189 ("unexpected generator expression element: %s %d" 1190 % (node, lineno)) 1191 fors[0].is_outmost = True 1192 return GenExpr(GenExprInner(expr, fors), lineno=lineno) 1193 1194 def com_dictorsetmaker(self, nodelist): 1195 # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 1196 # (test (comp_for | (',' test)* [','])) ) 1197 assert nodelist[0] == symbol.dictorsetmaker 1198 nodelist = nodelist[1:] 1199 if len(nodelist) == 1 or nodelist[1][0] == token.COMMA: 1200 # set literal 1201 items = [] 1202 for i in range(0, len(nodelist), 2): 1203 items.append(self.com_node(nodelist[i])) 1204 return Set(items, lineno=items[0].lineno) 1205 elif nodelist[1][0] == symbol.comp_for: 1206 # set comprehension 1207 expr = self.com_node(nodelist[0]) 1208 return self.com_comprehension(expr, None, nodelist[1], 'set') 1209 elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for: 1210 # dict comprehension 1211 assert nodelist[1][0] == token.COLON 1212 key = self.com_node(nodelist[0]) 1213 value = self.com_node(nodelist[2]) 1214 return self.com_comprehension(key, value, nodelist[3], 'dict') 1215 else: 1216 # dict literal 1217 items = [] 1218 for i in range(0, len(nodelist), 4): 1219 items.append((self.com_node(nodelist[i]), 1220 self.com_node(nodelist[i+2]))) 1221 return Dict(items, lineno=items[0][0].lineno) 1222 1223 def com_apply_trailer(self, primaryNode, nodelist): 1224 t = nodelist[1][0] 1225 if t == token.LPAR: 1226 return self.com_call_function(primaryNode, nodelist[2]) 1227 if t == token.DOT: 1228 return self.com_select_member(primaryNode, nodelist[2]) 1229 if t == token.LSQB: 1230 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) 1231 1232 raise SyntaxError, 'unknown node type: %s' % t 1233 1234 def com_select_member(self, primaryNode, nodelist): 1235 if nodelist[0] != token.NAME: 1236 raise SyntaxError, "member must be a name" 1237 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) 1238 1239 def com_call_function(self, primaryNode, nodelist): 1240 if nodelist[0] == token.RPAR: 1241 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) 1242 args = [] 1243 kw = 0 1244 star_node = dstar_node = None 1245 len_nodelist = len(nodelist) 1246 i = 1 1247 while i < len_nodelist: 1248 node = nodelist[i] 1249 1250 if node[0]==token.STAR: 1251 if star_node is not None: 1252 raise SyntaxError, 'already have the varargs indentifier' 1253 star_node = self.com_node(nodelist[i+1]) 1254 i = i + 3 1255 continue 1256 elif node[0]==token.DOUBLESTAR: 1257 if dstar_node is not None: 1258 raise SyntaxError, 'already have the kwargs indentifier' 1259 dstar_node = self.com_node(nodelist[i+1]) 1260 i = i + 3 1261 continue 1262 1263 # positional or named parameters 1264 kw, result = self.com_argument(node, kw, star_node) 1265 1266 if len_nodelist != 2 and isinstance(result, GenExpr) \ 1267 and len(node) == 3 and node[2][0] == symbol.comp_for: 1268 # allow f(x for x in y), but reject f(x for x in y, 1) 1269 # should use f((x for x in y), 1) instead of f(x for x in y, 1) 1270 raise SyntaxError, 'generator expression needs parenthesis' 1271 1272 args.append(result) 1273 i = i + 2 1274 1275 return CallFunc(primaryNode, args, star_node, dstar_node, 1276 lineno=extractLineNo(nodelist)) 1277 1278 def com_argument(self, nodelist, kw, star_node): 1279 if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for: 1280 test = self.com_node(nodelist[1]) 1281 return 0, self.com_generator_expression(test, nodelist[2]) 1282 if len(nodelist) == 2: 1283 if kw: 1284 raise SyntaxError, "non-keyword arg after keyword arg" 1285 if star_node: 1286 raise SyntaxError, "only named arguments may follow *expression" 1287 return 0, self.com_node(nodelist[1]) 1288 result = self.com_node(nodelist[3]) 1289 n = nodelist[1] 1290 while len(n) == 2 and n[0] != token.NAME: 1291 n = n[1] 1292 if n[0] != token.NAME: 1293 raise SyntaxError, "keyword can't be an expression (%s)"%n[0] 1294 node = Keyword(n[1], result, lineno=n[2]) 1295 return 1, node 1296 1297 def com_subscriptlist(self, primary, nodelist, assigning): 1298 # slicing: simple_slicing | extended_slicing 1299 # simple_slicing: primary "[" short_slice "]" 1300 # extended_slicing: primary "[" slice_list "]" 1301 # slice_list: slice_item ("," slice_item)* [","] 1302 1303 # backwards compat slice for '[i:j]' 1304 if len(nodelist) == 2: 1305 sub = nodelist[1] 1306 if (sub[1][0] == token.COLON or \ 1307 (len(sub) > 2 and sub[2][0] == token.COLON)) and \ 1308 sub[-1][0] != symbol.sliceop: 1309 return self.com_slice(primary, sub, assigning) 1310 1311 subscripts = [] 1312 for i in range(1, len(nodelist), 2): 1313 subscripts.append(self.com_subscript(nodelist[i])) 1314 return Subscript(primary, assigning, subscripts, 1315 lineno=extractLineNo(nodelist)) 1316 1317 def com_subscript(self, node): 1318 # slice_item: expression | proper_slice | ellipsis 1319 ch = node[1] 1320 t = ch[0] 1321 if t == token.DOT and node[2][0] == token.DOT: 1322 return Ellipsis() 1323 if t == token.COLON or len(node) > 2: 1324 return self.com_sliceobj(node) 1325 return self.com_node(ch) 1326 1327 def com_sliceobj(self, node): 1328 # proper_slice: short_slice | long_slice 1329 # short_slice: [lower_bound] ":" [upper_bound] 1330 # long_slice: short_slice ":" [stride] 1331 # lower_bound: expression 1332 # upper_bound: expression 1333 # stride: expression 1334 # 1335 # Note: a stride may be further slicing... 1336 1337 items = [] 1338 1339 if node[1][0] == token.COLON: 1340 items.append(Const(None)) 1341 i = 2 1342 else: 1343 items.append(self.com_node(node[1])) 1344 # i == 2 is a COLON 1345 i = 3 1346 1347 if i < len(node) and node[i][0] == symbol.test: 1348 items.append(self.com_node(node[i])) 1349 i = i + 1 1350 else: 1351 items.append(Const(None)) 1352 1353 # a short_slice has been built. look for long_slice now by looking 1354 # for strides... 1355 for j in range(i, len(node)): 1356 ch = node[j] 1357 if len(ch) == 2: 1358 items.append(Const(None)) 1359 else: 1360 items.append(self.com_node(ch[2])) 1361 return Sliceobj(items, lineno=extractLineNo(node)) 1362 1363 def com_slice(self, primary, node, assigning): 1364 # short_slice: [lower_bound] ":" [upper_bound] 1365 lower = upper = None 1366 if len(node) == 3: 1367 if node[1][0] == token.COLON: 1368 upper = self.com_node(node[2]) 1369 else: 1370 lower = self.com_node(node[1]) 1371 elif len(node) == 4: 1372 lower = self.com_node(node[1]) 1373 upper = self.com_node(node[3]) 1374 return Slice(primary, assigning, lower, upper, 1375 lineno=extractLineNo(node)) 1376 1377 def get_docstring(self, node, n=None): 1378 if n is None: 1379 n = node[0] 1380 node = node[1:] 1381 if n == symbol.suite: 1382 if len(node) == 1: 1383 return self.get_docstring(node[0]) 1384 for sub in node: 1385 if sub[0] == symbol.stmt: 1386 return self.get_docstring(sub) 1387 return None 1388 if n == symbol.file_input: 1389 for sub in node: 1390 if sub[0] == symbol.stmt: 1391 return self.get_docstring(sub) 1392 return None 1393 if n == symbol.atom: 1394 if node[0][0] == token.STRING: 1395 s = '' 1396 for t in node: 1397 s = s + eval(t[1]) 1398 return s 1399 return None 1400 if n == symbol.stmt or n == symbol.simple_stmt \ 1401 or n == symbol.small_stmt: 1402 return self.get_docstring(node[0]) 1403 if n in _doc_nodes and len(node) == 1: 1404 return self.get_docstring(node[0]) 1405 return None 1406 1407 1408 _doc_nodes = [ 1409 symbol.expr_stmt, 1410 symbol.testlist, 1411 symbol.testlist_safe, 1412 symbol.test, 1413 symbol.or_test, 1414 symbol.and_test, 1415 symbol.not_test, 1416 symbol.comparison, 1417 symbol.expr, 1418 symbol.xor_expr, 1419 symbol.and_expr, 1420 symbol.shift_expr, 1421 symbol.arith_expr, 1422 symbol.term, 1423 symbol.factor, 1424 symbol.power, 1425 ] 1426 1427 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 1428 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 1429 _cmp_types = { 1430 token.LESS : '<', 1431 token.GREATER : '>', 1432 token.EQEQUAL : '==', 1433 token.EQUAL : '==', 1434 token.LESSEQUAL : '<=', 1435 token.GREATEREQUAL : '>=', 1436 token.NOTEQUAL : '!=', 1437 } 1438 1439 _legal_node_types = [ 1440 symbol.funcdef, 1441 symbol.classdef, 1442 symbol.stmt, 1443 symbol.small_stmt, 1444 symbol.flow_stmt, 1445 symbol.simple_stmt, 1446 symbol.compound_stmt, 1447 symbol.expr_stmt, 1448 symbol.print_stmt, 1449 symbol.del_stmt, 1450 symbol.pass_stmt, 1451 symbol.break_stmt, 1452 symbol.continue_stmt, 1453 symbol.return_stmt, 1454 symbol.raise_stmt, 1455 symbol.import_stmt, 1456 symbol.global_stmt, 1457 symbol.exec_stmt, 1458 symbol.assert_stmt, 1459 symbol.if_stmt, 1460 symbol.while_stmt, 1461 symbol.for_stmt, 1462 symbol.try_stmt, 1463 symbol.with_stmt, 1464 symbol.suite, 1465 symbol.testlist, 1466 symbol.testlist_safe, 1467 symbol.test, 1468 symbol.and_test, 1469 symbol.not_test, 1470 symbol.comparison, 1471 symbol.exprlist, 1472 symbol.expr, 1473 symbol.xor_expr, 1474 symbol.and_expr, 1475 symbol.shift_expr, 1476 symbol.arith_expr, 1477 symbol.term, 1478 symbol.factor, 1479 symbol.power, 1480 symbol.atom, 1481 symbol.yield_stmt, 1482 symbol.yield_expr, 1483 ] 1484 1485 _assign_types = [ 1486 symbol.test, 1487 symbol.or_test, 1488 symbol.and_test, 1489 symbol.not_test, 1490 symbol.comparison, 1491 symbol.expr, 1492 symbol.xor_expr, 1493 symbol.and_expr, 1494 symbol.shift_expr, 1495 symbol.arith_expr, 1496 symbol.term, 1497 symbol.factor, 1498 ] 1499 1500 _names = {} 1501 for k, v in symbol.sym_name.items(): 1502 _names[k] = v 1503 for k, v in token.tok_name.items(): 1504 _names[k] = v 1505 1506 def debug_tree(tree): 1507 l = [] 1508 for elt in tree: 1509 if isinstance(elt, int): 1510 l.append(_names.get(elt, elt)) 1511 elif isinstance(elt, str): 1512 l.append(elt) 1513 else: 1514 l.append(debug_tree(elt)) 1515 return l