1 #!/usr/bin/env python 2 3 """ 4 Simplify AST structures for easier type propagation and analysis. The code in 5 this module processes AST trees originating from the compiler module and 6 produces a result tree consisting of instruction-oriented program nodes. 7 8 Copyright (C) 2006, 2007 Paul Boddie <paul@boddie.org.uk> 9 10 This software is free software; you can redistribute it and/or 11 modify it under the terms of the GNU General Public License as 12 published by the Free Software Foundation; either version 2 of 13 the License, or (at your option) any later version. 14 15 This software is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU General Public License for more details. 19 20 You should have received a copy of the GNU General Public 21 License along with this library; see the file LICENCE.txt 22 If not, write to the Free Software Foundation, Inc., 23 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA 24 25 -------- 26 27 To use this module, the easiest approach is to use the simplify function: 28 29 simplify(filename) 30 31 The more complicated approach involves first instantiating a Simplifier object: 32 33 simplifier = Simplifier() 34 35 Then, applying the simplifier to an AST tree: 36 37 module = compiler.parseFile(filename) 38 simplifier.process(module) 39 """ 40 41 from simplified import * 42 import compiler.ast 43 import os 44 45 class Simplifier(Visitor): 46 47 """ 48 A simplifying visitor for AST nodes. 49 50 Covered: Add, And, AssAttr, AssList, AssName, AssTuple, Assign, AugAssign, 51 Break, CallFunc, Class, Compare, Const, Continue, Dict, Discard, 52 Div, FloorDiv, For, From, Function, Getattr, Global, If, Import, 53 Invert, Keyword, Lambda, List, Mod, Module, Mul, Name, Not, Or, 54 Pass, Power, Print, Printnl, Raise, Return, Slice, Stmt, Sub, 55 Subscript, TryExcept, TryFinally, Tuple, While, UnaryAdd, UnarySub. 56 57 Missing: Assert, Backquote, Bitand, Bitor, Bitxor, Decorators, Ellipsis, 58 Exec, LeftShift, ListComp, ListCompFor, ListCompIf, RightShift, 59 Sliceobj, Yield. 60 """ 61 62 def __init__(self, builtins=0): 63 64 """ 65 Initialise the simplifier with the optional 'builtins' parameter 66 indicating whether the module contains the built-in classes and 67 functions. 68 """ 69 70 Visitor.__init__(self) 71 self.subprograms = [] # Subprograms outside the tree. 72 self.structures = [] # Structures/classes. 73 self.constants = {} # Constants. 74 self.current_subprograms = [] # Current subprograms being processed. 75 self.current_structures = [] # Current structures being processed. 76 self.within_class = 0 # Whether a class is being defined. 77 self.builtins = builtins # Whether the builtins are being processed. 78 79 # Convenience attributes. 80 81 self.subnames = {} 82 83 # For compiler package mechanisms. 84 85 self.visitor = self 86 87 def process(self, node, name): 88 result = self.dispatch(node, name) 89 result.simplifier = self 90 return result 91 92 def dispatch_or_none(self, node, *args): 93 if node is not None: 94 return self.dispatch(node, *args) 95 else: 96 return LoadName(node, name="None") 97 98 # Top-level transformation. 99 100 def visitModule(self, module, name=None): 101 102 """ 103 Process the given 'module', producing a Module object which contains the 104 resulting program nodes. If the optional 'name' is provided, the 'name' 105 attribute is set on the Module object using a value other than None. 106 """ 107 108 result = self.module = Module(module, 1, name=name) 109 module_code = self.dispatch(module.node) 110 111 # NOTE: Constant initialisation necessary for annotation but perhaps 112 # NOTE: redundant in the program. 113 114 init_code = [] 115 for value, constant in self.constants.items(): 116 init_code.append( 117 StoreAttr( 118 lvalue=LoadRef(ref=constant), 119 name="__class__", 120 expr=LoadName(name=constant.typename) 121 ) 122 ) 123 124 # NOTE: Hack to ensure correct initialisation of constants. 125 126 if self.builtins: 127 result.code = module_code + init_code 128 else: 129 result.code = init_code + module_code 130 return result 131 132 # Node transformations. 133 134 def visitAdd(self, add): 135 return self._visitBinary(add, "__add__", "__radd__") 136 137 def visitAnd(self, and_): 138 139 """ 140 Make a subprogram for the 'and_' node and record its contents inside the 141 subprogram. Convert... 142 143 And (test) 144 (test) 145 ... 146 147 ...to: 148 149 Subprogram -> Conditional (test) -> ReturnFromBlock ... 150 (else) -> Conditional (test) -> ReturnFromBlock ... 151 (else) -> ... 152 """ 153 154 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) 155 self.current_subprograms.append(subprogram) 156 157 # In the subprogram, make instructions which store each operand, test 158 # for each operand's truth status, and if appropriate return from the 159 # subprogram with the value of the operand. 160 161 last = and_.nodes[-1] 162 results = nodes = [] 163 164 for node in and_.nodes: 165 expr = self.dispatch(node) 166 167 # Return from the subprogram where the test is not satisfied. 168 169 if node is not last: 170 nodes += [ 171 StoreTemp(expr=expr), 172 Conditional( 173 test=self._visitNot(LoadTemp()), 174 body=[ 175 ReturnFromBlock( 176 expr=LoadTemp() 177 ) 178 ], 179 else_=[ 180 ReleaseTemp() 181 # Subsequent operations go here! 182 ] 183 ) 184 ] 185 186 # Put subsequent operations in the else section of this conditional. 187 188 nodes = nodes[-1].else_ 189 190 # For the last operation, return the result. 191 192 else: 193 nodes.append(ReturnFromBlock(expr=expr)) 194 195 # Finish the subprogram definition. 196 197 subprogram.code = results 198 199 self.current_subprograms.pop() 200 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 201 202 # Make an invocation of the subprogram. 203 204 result = InvokeBlock(and_, 1, produces_result=1) 205 result.expr = LoadRef(ref=subprogram) 206 return result 207 208 # Assignments. 209 210 def visitAssAttr(self, assattr, in_sequence=0): 211 expr = self._visitAssNameOrAttr(assattr, in_sequence) 212 lvalue = self.dispatch(assattr.expr) 213 result = StoreAttr(assattr, 1, name=assattr.attrname, lvalue=lvalue, expr=expr) 214 return result 215 216 def visitAssign(self, assign): 217 result = Assign(assign, 1) 218 store = StoreTemp(expr=self.dispatch(assign.expr)) 219 release = ReleaseTemp() 220 result.code = [store] + self.dispatches(assign.nodes, 0) + [release] 221 return result 222 223 def visitAssList(self, asslist, in_sequence=0): 224 if not in_sequence: 225 expr = LoadTemp() 226 else: 227 expr = InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next")) 228 result = Assign(asslist, 1) 229 store = StoreTemp(expr=InvokeFunction(expr=LoadAttr(name="__iter__", expr=expr))) 230 release = ReleaseTemp() 231 result.code = [store] + self.dispatches(asslist.nodes, 1) + [release] 232 return result 233 234 visitAssTuple = visitAssList 235 236 def _visitAssNameOrAttr(self, node, in_sequence): 237 if not in_sequence: 238 return LoadTemp() 239 else: 240 return InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next")) 241 242 def visitAssName(self, assname, in_sequence=0): 243 expr = self._visitAssNameOrAttr(assname, in_sequence) 244 result = StoreName(assname, 1, name=assname.name, expr=expr) 245 return result 246 247 augassign_methods = { 248 "+=" : "__iadd__", "-=" : "__isub__", "*=" : "__imul__", "/=" : "__idiv__", 249 "%=" : "__imod__", "**=" : "__ipow__", "<<=" : "__ilshift__", ">>=" : "__irshift__", 250 "&=" : "__iand__", "^=" : "__ixor__", "|=" : "__ior__" 251 } 252 253 def visitAugAssign(self, augassign): 254 255 """ 256 Convert the augmented assignment... 257 258 AugAssign (node) -> Name | Getattr | Slice | Subscript 259 (op) 260 (expr) 261 262 ...to: 263 264 Assign (code) -> StoreTemp (expr) -> InvokeFunction (expr) -> LoadAttr (expr) -> <name> 265 (name) -> <op> 266 StoreName (name) -> <name> 267 (expr) -> LoadTemp 268 ReleaseTemp 269 """ 270 271 result = Assign(augassign, 1) 272 expr = self.dispatch(augassign.expr) 273 274 # Simple augmented assignment: name += expr 275 276 if isinstance(augassign.node, compiler.ast.Name): 277 result.code = [ 278 StoreTemp( 279 expr=InvokeFunction( # referenced below 280 args=[expr], 281 star=None, 282 dstar=None, 283 expr=LoadAttr( 284 expr=self.dispatch(augassign.node), 285 name=self.augassign_methods[augassign.op] 286 ) 287 ) 288 ), 289 StoreName( 290 expr=LoadTemp(), 291 name=augassign.node.name), 292 ReleaseTemp() 293 ] 294 295 # Make nice annotations for the viewer. 296 297 augassign._op_call = result.code[0].expr 298 299 # Complicated augmented assignment: lvalue.attr += expr 300 301 elif isinstance(augassign.node, compiler.ast.Getattr): 302 303 # <lvalue> -> setattr(<lvalue>, getattr(<lvalue>, "attr").__xxx__(expr)) 304 305 result.code = [ 306 StoreTemp( 307 index="expr", 308 expr=self.dispatch(augassign.node.expr) 309 ), 310 StoreTemp( 311 expr=InvokeFunction( # referenced below 312 args=[expr], star=None, dstar=None, 313 expr=LoadAttr( 314 expr=LoadAttr(augassign.node, 1, 315 expr=LoadTemp(index="expr"), 316 name=augassign.node.attrname 317 ), 318 name=self.augassign_methods[augassign.op] 319 ) 320 ) 321 ), 322 StoreAttr( 323 expr=LoadTemp(), 324 lvalue=LoadTemp(index="expr"), 325 name=augassign.node.attrname 326 ), 327 ReleaseTemp(index="expr"), 328 ReleaseTemp() 329 ] 330 331 # Make nice annotations for the viewer. 332 333 augassign._op_call = result.code[1].expr 334 335 # Complicated augassign using slices: lvalue[lower:upper] += expr 336 337 elif isinstance(augassign.node, compiler.ast.Slice): 338 339 # <lvalue>, <lower>, <upper> -> <lvalue>.__setslice__(<lower>, <upper>, <lvalue>.__getslice__(<lower>, <upper>).__xxx__(expr)) 340 341 result.code = [ 342 StoreTemp( 343 index="expr", 344 expr=self.dispatch(augassign.node.expr) 345 ), 346 StoreTemp( 347 index="lower", 348 expr=self.dispatch_or_none(augassign.node.lower) 349 ), 350 StoreTemp( 351 index="upper", 352 expr=self.dispatch_or_none(augassign.node.upper) 353 ), 354 StoreTemp( 355 expr=InvokeFunction( # referenced below 356 args=[expr], star=None, dstar=None, 357 expr=LoadAttr( 358 expr=self._visitSlice( 359 augassign.node, 360 LoadTemp(index="expr"), 361 LoadTemp(index="lower"), 362 LoadTemp(index="upper"), 363 "OP_APPLY"), 364 name=self.augassign_methods[augassign.op] 365 ) 366 ) 367 ), 368 self._visitSlice( 369 augassign.node, 370 LoadTemp(index="expr"), 371 LoadTemp(index="lower"), 372 LoadTemp(index="upper"), 373 "OP_ASSIGN", 374 LoadTemp() 375 ), 376 ReleaseTemp(index="expr"), 377 ReleaseTemp(index="lower"), 378 ReleaseTemp(index="upper"), 379 ReleaseTemp() 380 ] 381 382 # Make nice annotations for the viewer. 383 384 augassign._op_call = result.code[3].expr 385 386 # Complicated augassign using subscripts: lvalue[subs] += expr 387 388 elif isinstance(augassign.node, compiler.ast.Subscript): 389 390 # <lvalue>, <subs> -> <lvalue>.__setitem__(<subs>, <lvalue>.__getitem__(<subs>).__xxx__(expr)) 391 392 result.code = [ 393 StoreTemp(index="expr", expr=self.dispatch(augassign.node.expr)), 394 StoreTemp(index="subs", expr=self._visitSubscriptSubs(augassign.node, augassign.node.subs)), 395 StoreTemp( 396 expr=InvokeFunction( # referenced below 397 args=[expr], star=None, dstar=None, 398 expr=LoadAttr( 399 expr=self._visitSubscript( 400 augassign.node, 401 LoadTemp(index="expr"), 402 LoadTemp(index="subs"), 403 "OP_APPLY" 404 ), 405 name=self.augassign_methods[augassign.op] 406 ) 407 ) 408 ), 409 self._visitSubscript( 410 augassign.node, 411 LoadTemp(index="expr"), 412 LoadTemp(index="subs"), 413 "OP_ASSIGN", 414 LoadTemp() 415 ), 416 ReleaseTemp(index="expr"), 417 ReleaseTemp(index="subs"), 418 ReleaseTemp() 419 ] 420 421 # Make nice annotations for the viewer. 422 423 augassign._op_call = result.code[2].expr 424 425 else: 426 raise NotImplementedError, augassign.node.__class__ 427 428 return result 429 430 def visitBreak(self, break_): 431 result = ReturnFromBlock(break_, 1) 432 return result 433 434 def visitCallFunc(self, callfunc): 435 result = InvokeFunction(callfunc, 1, star=None, dstar=None, args=self.dispatches(callfunc.args)) 436 if callfunc.star_args is not None: 437 result.star = self.dispatch(callfunc.star_args) 438 if callfunc.dstar_args is not None: 439 result.dstar = self.dispatch(callfunc.dstar_args) 440 result.expr = self.dispatch(callfunc.node) 441 return result 442 443 def visitClass(self, class_): 444 445 # Add "object" if the class is not "object" and has an empty bases list. 446 447 if class_.name != "object" and not class_.bases: 448 bases = [compiler.ast.Name("object")] 449 else: 450 bases = class_.bases 451 452 structure = Class(name=class_.name, module=self.module, bases=self.dispatches(bases)) 453 self.structures.append(structure) 454 within_class = self.within_class 455 self.within_class = 1 456 457 # Make a subprogram which initialises the class structure. 458 459 subprogram = Subprogram(name=None, module=self.module, structure=structure, params=[], star=None, dstar=None) 460 self.current_subprograms.append(subprogram) 461 self.current_structures.append(structure) # mostly for name construction 462 463 # The class is initialised using the code found inside. 464 465 subprogram.code = self.dispatch(class_.code) + [ReturnFromBlock()] 466 467 self.within_class = within_class 468 self.current_structures.pop() 469 self.current_subprograms.pop() 470 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 471 472 # Make a definition of the class associating it with a name. 473 474 result = Assign( 475 code=[ 476 StoreName(class_, 1, # defines the class 477 name=class_.name, 478 expr=LoadRef(ref=structure) 479 ), 480 InvokeBlock( 481 share_locals=0, # override the local sharing usually in InvokeBlock 482 expr=LoadRef(ref=subprogram) 483 ) 484 ] 485 ) 486 return result 487 488 comparison_methods = { 489 "==" : "__eq__", "!=" : "__ne__", "<" : "__lt__", "<=" : "__le__", 490 ">=" : "__ge__", ">" : "__gt__", "is" : None, "is not" : None 491 } 492 493 def visitCompare(self, compare): 494 495 """ 496 Make a subprogram for the 'compare' node and record its contents inside 497 the subprogram. Convert... 498 499 Compare (expr) 500 (name/node) 501 ... 502 503 ...to: 504 505 InvokeBlock -> Subprogram -> Conditional (test) -> (body) 506 (else) -> Conditional (test) -> (body) 507 (else) -> ... 508 """ 509 510 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) 511 self.current_subprograms.append(subprogram) 512 513 # In the subprogram, make instructions which invoke a method on the 514 # first operand of each operand pair and, if appropriate, return with 515 # the value from that method. 516 517 last = compare.ops[-1] 518 previous = self.dispatch(compare.expr) 519 results = nodes = [] 520 521 # For viewing purposes, record invocations on the AST node. 522 523 compare._ops = [] 524 525 for op in compare.ops: 526 op_name, node = op 527 expr = self.dispatch(node) 528 529 # Identify the operation and produce the appropriate method call. 530 531 method_name = self.comparison_methods[op_name] 532 if method_name: 533 invocation = InvokeFunction( 534 expr=LoadAttr( 535 expr=previous, 536 name=method_name), 537 args=[expr], 538 star=None, 539 dstar=None) 540 541 elif op_name == "is": 542 invocation = InvokeFunction( 543 expr=LoadName(name="__is__"), 544 args=[previous, expr], 545 star=None, 546 dstar=None) 547 548 elif op_name == "is not": 549 invocation = Not( 550 expr=InvokeFunction( 551 expr=LoadName(name="__is__"), 552 args=[previous, expr], 553 star=None, 554 dstar=None) 555 ) 556 else: 557 raise NotImplementedError, op_name 558 559 nodes.append(StoreTemp(expr=invocation)) 560 compare._ops.append(invocation) 561 562 # Return from the subprogram where the test is not satisfied. 563 564 if op is not last: 565 nodes.append( 566 Conditional( 567 test=self._visitNot(LoadTemp()), 568 body=[ 569 ReturnFromBlock(expr=LoadTemp()) 570 ], 571 else_=[ 572 ReleaseTemp() 573 # Subsequent operations go here! 574 ] 575 ) 576 ) 577 578 # Put subsequent operations in the else section of this conditional. 579 580 nodes = nodes[-1].else_ 581 582 # For the last operation, return the result. 583 584 else: 585 nodes.append( 586 ReturnFromBlock(expr=LoadTemp(release=1)) 587 ) 588 589 previous = expr 590 591 # Finish the subprogram definition. 592 593 subprogram.code = results 594 595 self.current_subprograms.pop() 596 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 597 598 # Make an invocation of the subprogram. 599 600 result = InvokeBlock(compare, 1, produces_result=1) 601 result.expr = LoadRef(ref=subprogram) 602 return result 603 604 def visitConst(self, const): 605 key = "%s-%s" % (const.value.__class__.__name__, const.value) 606 if not self.constants.has_key(key): 607 self.constants[key] = Constant(name=repr(const.value), value=const.value) 608 result = LoadRef(const, 1, ref=self.constants[key]) 609 return result 610 611 def visitContinue(self, continue_): 612 result = InvokeBlock(continue_, 1, 613 expr=LoadRef(ref=self.current_subprograms[-1]) 614 ) 615 return result 616 617 def visitDict(self, dict): 618 result = InvokeFunction(dict, 1, expr=LoadName(name="dict"), star=None, dstar=None) 619 args = [] 620 for key, value in dict.items: 621 tuple = InvokeFunction(expr=LoadName(name="tuple"), star=None, dstar=None) 622 tuple.set_args([self.dispatch(key), self.dispatch(value)]) 623 args.append(tuple) 624 result.set_args(args) 625 return result 626 627 def visitDiscard(self, discard): 628 return self.dispatch(discard.expr) 629 630 def visitDiv(self, div): 631 return self._visitBinary(div, "__div__", "__rdiv__") 632 633 def visitFloorDiv(self, floordiv): 634 return self._visitBinary(floordiv, "__floordiv__", "__rfloordiv__") 635 636 def visitFor(self, for_): 637 638 """ 639 Make a subprogram for the 'for_' node and record its contents inside the 640 subprogram. Convert... 641 642 For (assign) 643 (body) 644 (else) 645 646 ...to: 647 648 Assign (assign #1) 649 Invoke -> Subprogram -> Try (body) -> (assign #2) 650 (body) 651 Invoke subprogram 652 (handler) -> ... 653 (else) -> ... 654 """ 655 656 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None) 657 self.current_subprograms.append(subprogram) 658 659 # Always return from conditional sections/subprograms. 660 661 if for_.else_ is not None: 662 else_stmt = self.dispatch(for_.else_) + [ReturnFromBlock()] 663 else: 664 else_stmt = [ReturnFromBlock()] 665 666 # Wrap the assignment in a try...except statement. 667 # Inside the body, add a recursive invocation to the subprogram. 668 669 subprogram.code = [ 670 Try( 671 body=[ 672 Assign( 673 code=[ 674 StoreTemp(expr=InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next"))), 675 self.dispatch(for_.assign), 676 ReleaseTemp() 677 ]) 678 ] + self.dispatch(for_.body) + [ 679 InvokeBlock( 680 expr=LoadRef(ref=subprogram) 681 ) 682 ], 683 handler=[ 684 Conditional( 685 test=InvokeFunction( 686 expr=LoadName(name="isinstance"), 687 args=[LoadExc(), LoadName(name="StopIteration")], 688 star=None, 689 dstar=None), 690 body=else_stmt, 691 else_=[Raise(expr=LoadExc())] 692 ) 693 ], 694 else_=[], 695 finally_=[] 696 ), 697 ReturnFromBlock() 698 ] 699 700 # Finish the subprogram definition. 701 702 self.current_subprograms.pop() 703 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 704 705 # Obtain an iterator for the sequence involved. 706 # Then, make an invocation of the subprogram. 707 708 result = Assign(for_, 1, 709 code=[ 710 StoreTemp( 711 expr=InvokeFunction( 712 expr=LoadAttr( 713 name="__iter__", 714 expr=self.dispatch(for_.list) 715 ) 716 ) 717 ), 718 InvokeBlock(expr=LoadRef(ref=subprogram)), 719 ReleaseTemp() 720 ] 721 ) 722 723 # Make nice annotations for the viewer. 724 725 for_._iter_call = result.code[0].expr 726 for_._next_call = subprogram.code[0].body[0].code[0].expr 727 728 return result 729 730 def visitFrom(self, from_): 731 result = Assign(from_, 1) 732 code = [] 733 _names = [] 734 code.append( 735 StoreTemp( 736 expr=Import(name=from_.modname, alias=1) 737 ) 738 ) 739 from_._modname = code[-1].expr 740 for name, alias in from_.names: 741 code.append( 742 StoreName( 743 expr=LoadAttr( 744 expr=LoadTemp(), 745 name=name), 746 name=(alias or name) 747 ) 748 ) 749 _names.append(code[-1].expr) 750 code.append(ReleaseTemp()) 751 result.code = code 752 from_._names = _names 753 return result 754 755 def _visitFunction(self, function, subprogram): 756 757 """ 758 A common function generator which transforms the given 'function' node 759 and initialises the given 'subprogram' appropriately. 760 """ 761 762 # Discover star and dstar parameters. 763 764 if function.flags & 4 != 0: 765 has_star = 1 766 else: 767 has_star = 0 768 if function.flags & 8 != 0: 769 has_dstar = 1 770 else: 771 has_dstar = 0 772 773 # Discover the number of defaults and positional parameters. 774 775 ndefaults = len(function.defaults) 776 npositional = len(function.argnames) - has_star - has_dstar 777 778 # Produce star and dstar parameters with appropriate defaults. 779 780 if has_star: 781 star = ( 782 function.argnames[npositional], 783 InvokeFunction(expr=LoadName(name="list")) 784 ) 785 else: 786 star = None 787 if has_dstar: 788 dstar = ( 789 function.argnames[npositional + has_star], 790 InvokeFunction(expr=LoadName(name="dict")) 791 ) 792 else: 793 dstar = None 794 795 params = [] 796 for i in range(0, npositional - ndefaults): 797 params.append((function.argnames[i], None)) 798 799 # Process defaults. 800 801 for i in range(0, ndefaults): 802 default = function.defaults[i] 803 if default is not None: 804 params.append((function.argnames[npositional - ndefaults + i], self.dispatch(default))) 805 else: 806 params.append((function.argnames[npositional - ndefaults + i], None)) 807 808 subprogram.params = params 809 subprogram.star = star 810 subprogram.dstar = dstar 811 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 812 813 def visitFunction(self, function): 814 815 """ 816 Make a subprogram for the 'function' and record it outside the main 817 tree. Produce something like the following: 818 819 StoreName (name) 820 (expr) -> LoadRef (ref) -> Subprogram (params) 821 (star) 822 (dstar) 823 """ 824 825 subprogram = Subprogram(name=function.name, module=self.module, structures=self.current_structures[:], 826 internal=0, returns_value=1, star=None, dstar=None, is_method=self.within_class, original_def=function) 827 828 self.current_subprograms.append(subprogram) 829 within_class = self.within_class 830 self.within_class = 0 831 832 subprogram.code = self.dispatch(function.code) + [ReturnFromFunction()] 833 834 self.within_class = within_class 835 self.current_subprograms.pop() 836 self._visitFunction(function, subprogram) 837 838 # Make a definition of the function associating it with a name. 839 840 result = StoreName(function, 1, name=function.name, expr=LoadRef(ref=subprogram)) 841 return result 842 843 def visitGetattr(self, getattr): 844 result = LoadAttr(getattr, 1, 845 name=getattr.attrname, 846 expr=self.dispatch(getattr.expr) 847 ) 848 return result 849 850 def visitGlobal(self, global_): 851 result = Global(global_, 1, 852 names=global_.names 853 ) 854 return result 855 856 def visitIf(self, if_): 857 858 """ 859 Make conditionals for each test from an 'if_' AST node, adding the body 860 and putting each subsequent test as part of the conditional's else 861 section. 862 863 Convert... 864 865 If (test/body) 866 (test/body) 867 ... 868 (else/body) 869 870 ...to: 871 872 Conditional (test) -> (body) 873 (else) -> Conditional (test) -> (body) 874 (else) -> ... 875 """ 876 877 878 results = nodes = [] 879 880 # Produce something like... 881 # expr.__bool__() ? body 882 883 first = 1 884 for compare, stmt in if_.tests: 885 886 # Set the first as the defining node. 887 888 test = Conditional(if_, first, 889 test=InvokeFunction( 890 expr=LoadAttr( 891 expr=self.dispatch(compare), 892 name="__bool__" 893 ), 894 ) 895 ) 896 test.body = self.dispatch(stmt) 897 nodes.append(test) 898 nodes = test.else_ = [] 899 first = 0 900 901 # Add the compound statement from any else clause to the end. 902 903 if if_.else_ is not None: 904 nodes += self.dispatch(if_.else_) 905 906 result = results[0] 907 return result 908 909 def visitImport(self, import_): 910 result = Assign(import_, 1) 911 code = [] 912 _names = [] 913 for path, alias in import_.names: 914 importer = Import(name=path, alias=alias) 915 top = alias or path.split(".")[0] 916 code.append(StoreName(expr=importer, name=top)) 917 _names.append(code[-1].expr) 918 result.code = code 919 import_._names = _names 920 return result 921 922 def visitInvert(self, invert): 923 return self._visitUnary(invert, "__invert__") 924 925 def visitKeyword(self, keyword): 926 result = Keyword(keyword, 1, 927 name=keyword.name, 928 expr=self.dispatch(keyword.expr) 929 ) 930 return result 931 932 def visitLambda(self, lambda_): 933 934 # Make a subprogram for the function and record it outside the main 935 # tree. 936 937 subprogram = Subprogram(name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=lambda_) 938 self.current_subprograms.append(subprogram) 939 subprogram.code = [ReturnFromFunction(expr=self.dispatch(lambda_.code))] 940 self.current_subprograms.pop() 941 self._visitFunction(lambda_, subprogram) 942 943 # Get the subprogram reference to the lambda. 944 945 return LoadRef(lambda_, 1, ref=subprogram) 946 947 def visitList(self, list): 948 return self._visitBuiltin(list, "list") 949 950 def visitMod(self, mod): 951 return self._visitBinary(mod, "__mod__", "__rmod__") 952 953 def visitMul(self, mul): 954 return self._visitBinary(mul, "__mul__", "__rmul__") 955 956 def visitName(self, name): 957 result = LoadName(name, 1, name=name.name) 958 return result 959 960 def _visitNot(self, expr, not_=None): 961 invocation = InvokeFunction( 962 expr=LoadAttr( 963 expr=expr, 964 name="__bool__" 965 ), 966 ) 967 if not_ is not None: 968 result = Not(not_, 1, expr=invocation) 969 else: 970 result = Not(expr=invocation) 971 return result 972 973 def visitNot(self, not_): 974 return self._visitNot(self.dispatch(not_.expr), not_) 975 976 def visitOr(self, or_): 977 978 """ 979 Make a subprogram for the 'or_' node and record its contents inside the 980 subprogram. Convert... 981 982 Or (test) 983 (test) 984 ... 985 986 ...to: 987 988 Subprogram -> Conditional (test) -> ReturnFromBlock ... 989 (else) -> Conditional (test) -> ReturnFromBlock ... 990 (else) -> ... 991 """ 992 993 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) 994 self.current_subprograms.append(subprogram) 995 996 # In the subprogram, make instructions which store each operand, test 997 # for each operand's truth status, and if appropriate return from the 998 # subprogram with the value of the operand. 999 1000 last = or_.nodes[-1] 1001 results = nodes = [] 1002 1003 for node in or_.nodes: 1004 expr = self.dispatch(node) 1005 1006 # Return from the subprogram where the test is satisfied. 1007 1008 if node is not last: 1009 nodes.append(StoreTemp(expr=expr)) 1010 invocation = InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="__bool__")) 1011 test = Conditional(test=invocation, body=[ReturnFromBlock(expr=LoadTemp())]) 1012 nodes.append(test) 1013 1014 # Put subsequent operations in the else section of this conditional. 1015 1016 nodes = test.else_ = [ReleaseTemp()] 1017 1018 # For the last operation, return the result. 1019 1020 else: 1021 nodes.append( 1022 ReturnFromBlock(expr=expr) 1023 ) 1024 1025 # Finish the subprogram definition. 1026 1027 subprogram.code = results 1028 1029 self.current_subprograms.pop() 1030 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 1031 1032 # Make an invocation of the subprogram. 1033 1034 result = InvokeBlock(or_, 1, produces_result=1) 1035 result.expr = LoadRef(ref=subprogram) 1036 return result 1037 1038 def visitPass(self, pass_): 1039 return Pass(pass_, 1) 1040 1041 def visitPower(self, power): 1042 return self._visitBinary(power, "__pow__", "__rpow__") 1043 1044 def visitPrint(self, print_): 1045 1046 """ 1047 Convert... 1048 1049 Print (dest) -> 1050 (nodes) 1051 1052 ...to: 1053 1054 StoreTemp (index) -> "print" 1055 (expr) -> LoadAttr (expr) -> (dest) 1056 (name) -> "write" 1057 InvokeFunction (expr) -> LoadTemp (index) -> "print" 1058 (args) -> [(node)] 1059 ReleaseTemp (index) -> "print" 1060 """ 1061 1062 if print_.dest is not None: 1063 dest = self.dispatch(print_.dest) 1064 else: 1065 dest = self.dispatch(compiler.ast.Name("stdout")) 1066 1067 result = Assign(print_, 1, 1068 code=[ 1069 StoreTemp( 1070 index="print", 1071 expr=LoadAttr( 1072 expr=dest, 1073 name="write" 1074 ) 1075 ) 1076 ] 1077 ) 1078 1079 for node in print_.nodes: 1080 result.code.append( 1081 InvokeFunction( 1082 expr=LoadTemp(index="print"), 1083 args=[self.dispatch(node)], 1084 star=None, 1085 dstar=None 1086 ) 1087 ) 1088 1089 result.code.append( 1090 ReleaseTemp(index="print") 1091 ) 1092 1093 return result 1094 1095 def visitPrintnl(self, printnl): 1096 result = self.visitPrint(printnl) 1097 result.code.insert( 1098 len(result.code) - 1, 1099 InvokeFunction( 1100 expr=LoadTemp(index="print"), 1101 args=[self.dispatch(compiler.ast.Const("\n"))], 1102 star=None, 1103 dstar=None 1104 ) 1105 ) 1106 return result 1107 1108 def visitRaise(self, raise_): 1109 result = Raise(raise_, 1) 1110 if raise_.expr2 is None: 1111 result.expr = self.dispatch(raise_.expr1) 1112 else: 1113 result.expr = InvokeFunction( 1114 expr=self.dispatch(raise_.expr1), 1115 args=[self.dispatch(raise_.expr2)], 1116 star=None, 1117 dstar=None 1118 ) 1119 if raise_.expr3 is not None: 1120 result.traceback = self.dispatch(raise_.expr3) 1121 else: 1122 result.traceback = None 1123 return result 1124 1125 def visitReturn(self, return_): 1126 result = ReturnFromFunction(return_, 1, 1127 expr=self.dispatch(return_.value) 1128 ) 1129 return result 1130 1131 def _visitSlice(self, slice, expr, lower, upper, flags, value=None): 1132 if flags == "OP_ASSIGN": 1133 result = InvokeFunction(slice, 1, 1134 expr=LoadAttr( 1135 expr=expr, 1136 name="__setslice__" 1137 ), 1138 star=None, 1139 dstar=None, 1140 args=[lower, upper, value] 1141 ) 1142 elif flags == "OP_APPLY": 1143 args = [] 1144 result = InvokeFunction(slice, 1, 1145 expr=LoadAttr( 1146 expr=expr, 1147 name="__getslice__" 1148 ), 1149 star=None, 1150 dstar=None, 1151 args=[lower, upper] 1152 ) 1153 elif flags == "OP_DELETE": 1154 args = [] 1155 result = InvokeFunction(slice, 1, 1156 expr=LoadAttr( 1157 expr=expr, 1158 name="__delslice__" 1159 ), 1160 star=None, 1161 dstar=None, 1162 args=[lower, upper] 1163 ) 1164 else: 1165 raise NotImplementedError, flags 1166 1167 return result 1168 1169 def visitSlice(self, slice, in_sequence=0): 1170 return self._visitSlice(slice, self.dispatch(slice.expr), self.dispatch_or_none(slice.lower), 1171 self.dispatch_or_none(slice.upper), slice.flags, self._visitAssNameOrAttr(slice, in_sequence)) 1172 1173 def visitStmt(self, stmt): 1174 return self.dispatches(stmt.nodes) 1175 1176 def visitSub(self, sub): 1177 return self._visitBinary(sub, "__sub__", "__rsub__") 1178 1179 def _visitSubscript(self, subscript, expr, subs, flags, value=None): 1180 if flags == "OP_ASSIGN": 1181 result = InvokeFunction(subscript, 1, 1182 expr=LoadAttr( 1183 expr=expr, 1184 name="__setitem__" 1185 ), 1186 star=None, 1187 dstar=None, 1188 args=[subs, value] 1189 ) 1190 elif flags == "OP_APPLY": 1191 args = [] 1192 result = InvokeFunction(subscript, 1, 1193 expr=LoadAttr( 1194 expr=expr, 1195 name="__getitem__" 1196 ), 1197 star=None, 1198 dstar=None, 1199 args=[subs] 1200 ) 1201 elif flags == "OP_DELETE": 1202 args = [] 1203 result = InvokeFunction(subscript, 1, 1204 expr=LoadAttr( 1205 expr=expr, 1206 name="__delitem__" 1207 ), 1208 star=None, 1209 dstar=None, 1210 args=[subs] 1211 ) 1212 else: 1213 raise NotImplementedError, flags 1214 1215 return result 1216 1217 def _visitSubscriptSubs(self, node, subs): 1218 if len(subs) == 1: 1219 return self.dispatch(subs[0]) 1220 else: 1221 return InvokeFunction(node, 1, 1222 expr=LoadName(name="tuple"), 1223 args=self.dispatches(subs), 1224 star=None, 1225 dstar=None 1226 ) 1227 1228 def visitSubscript(self, subscript, in_sequence=0): 1229 return self._visitSubscript( 1230 subscript, self.dispatch(subscript.expr), self._visitSubscriptSubs(subscript, subscript.subs), subscript.flags, 1231 self._visitAssNameOrAttr(subscript, in_sequence) 1232 ) 1233 1234 def visitTryExcept(self, tryexcept): 1235 1236 """ 1237 Make conditionals for each handler associated with a 'tryexcept' node. 1238 1239 Convert... 1240 1241 TryExcept (body) 1242 (else) 1243 (spec/assign/stmt) 1244 ... 1245 1246 ...to: 1247 1248 Try (body) 1249 (else) 1250 (handler) -> Conditional (test) -> (stmt) 1251 (body) -> ... 1252 (else) -> Conditional (test) -> (stmt) 1253 (body) -> ... 1254 (else) -> ... 1255 """ 1256 1257 result = Try(tryexcept, 1, body=[], else_=[], finally_=[]) 1258 1259 if tryexcept.body is not None: 1260 result.body = self.dispatch(tryexcept.body) 1261 if tryexcept.else_ is not None: 1262 result.else_ = self.dispatch(tryexcept.else_) 1263 1264 results = nodes = [] 1265 catch_all = 0 1266 1267 for spec, assign, stmt in tryexcept.handlers: 1268 1269 # If no specification exists, produce an unconditional block. 1270 1271 if spec is None: 1272 nodes += self.dispatch(stmt) 1273 catch_all = 1 1274 1275 # Produce an exception value check. 1276 1277 else: 1278 test = Conditional( 1279 isolate_test=1, 1280 test=CheckExc(expr=LoadExc(), choices=self._visitTryExcept(spec)) 1281 ) 1282 test.body = [] 1283 1284 if assign is not None: 1285 test.body.append( 1286 Assign( 1287 code=[ 1288 StoreTemp(expr=LoadExc()), 1289 self.dispatch(assign), 1290 ReleaseTemp() 1291 ] 1292 ) 1293 ) 1294 1295 test.body += self.dispatch(stmt) 1296 nodes.append(test) 1297 nodes = test.else_ = [] 1298 1299 # Add a raise operation to deal with unhandled exceptions. 1300 1301 if not catch_all: 1302 nodes.append( 1303 Raise( 1304 expr=LoadExc()) 1305 ) 1306 1307 result.handler = results 1308 return result 1309 1310 def _visitTryExcept(self, spec): 1311 1312 "Return a list of nodes for the given exception type 'spec'." 1313 1314 if isinstance(spec, compiler.ast.Tuple): 1315 nodes = [] 1316 for node in spec.nodes: 1317 nodes += self._visitTryExcept(node) 1318 else: 1319 nodes = [self.dispatch(spec)] 1320 return nodes 1321 1322 def visitTryFinally(self, tryfinally): 1323 result = Try(tryfinally, 1, body=[], else_=[], finally_=[]) 1324 if tryfinally.body is not None: 1325 result.body = self.dispatch(tryfinally.body) 1326 if tryfinally.final is not None: 1327 result.finally_ = self.dispatch(tryfinally.final) 1328 return result 1329 1330 def visitTuple(self, tuple): 1331 return self._visitBuiltin(tuple, "tuple") 1332 1333 def visitUnaryAdd(self, unaryadd): 1334 return self._visitUnary(unaryadd, "__pos__") 1335 1336 def visitUnarySub(self, unarysub): 1337 return self._visitUnary(unarysub, "__neg__") 1338 1339 def visitWhile(self, while_): 1340 1341 """ 1342 Make a subprogram for the 'while' node and record its contents inside the 1343 subprogram. Convert... 1344 1345 While (test) -> (body) 1346 (else) 1347 1348 ...to: 1349 1350 Subprogram -> Conditional (test) -> (body) -> Invoke subprogram 1351 (else) -> Conditional (test) -> ReturnFromBlock ... 1352 (else) -> ... 1353 """ 1354 1355 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None) 1356 self.current_subprograms.append(subprogram) 1357 1358 # Include a conditional statement in the subprogram. 1359 # Inside the conditional, add a recursive invocation to the subprogram 1360 # if the test condition was satisfied. 1361 # Return within the main section of the loop. 1362 1363 test = Conditional( 1364 test=InvokeFunction( 1365 expr=LoadAttr( 1366 expr=self.dispatch(while_.test), 1367 name="__bool__"), 1368 ), 1369 body=self.dispatch(while_.body) + [ 1370 InvokeBlock( 1371 expr=LoadRef(ref=subprogram) 1372 ), 1373 ReturnFromBlock() 1374 ], 1375 else_=[] 1376 ) 1377 1378 # Provide the else section, if present, along with an explicit return. 1379 1380 if while_.else_ is not None: 1381 test.else_ = self.dispatch(while_.else_) + [ReturnFromBlock()] 1382 1383 # Finish the subprogram definition. 1384 1385 subprogram.code = [test] 1386 1387 self.current_subprograms.pop() 1388 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 1389 1390 # Make an invocation of the subprogram. 1391 1392 result = InvokeBlock(while_, 1, 1393 expr=LoadRef(ref=subprogram) 1394 ) 1395 1396 # Make nice annotations for the viewer. 1397 1398 while_._test_call = subprogram.code[0].test 1399 1400 return result 1401 1402 # Convenience methods. 1403 1404 def _visitBinary(self, binary, left_name, right_name): 1405 1406 """ 1407 Emulate the current mechanisms by producing nodes as follows: 1408 1409 InvokeBlock -> Subprogram -> Try (body) -> ReturnFromBlock (expr) -> x.__add__(y) 1410 (else) 1411 (handler) -> Conditional (test) -> CheckExc (expr) -> LoadExc 1412 (choices) -> LoadName TypeError 1413 (body) -> ReturnFromBlock (expr) -> y.__radd__(x) 1414 (else) 1415 """ 1416 1417 subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) 1418 self.current_subprograms.append(subprogram) 1419 1420 subprogram.code = [ 1421 Try(binary, 1, 1422 body=[ 1423 ReturnFromBlock( 1424 expr=InvokeFunction( 1425 expr=LoadAttr(expr=self.dispatch(binary.left), name=left_name), 1426 args=[self.dispatch(binary.right)], 1427 star=None, 1428 dstar=None) 1429 ) 1430 ], 1431 else_=[], 1432 finally_=[], 1433 handler=[ 1434 Conditional( 1435 test=CheckExc(expr=LoadExc(), choices=[LoadName(name="TypeError")]), 1436 body=[ 1437 ReturnFromBlock( 1438 expr=InvokeFunction( 1439 expr=LoadAttr(expr=self.dispatch(binary.right), name=right_name), 1440 args=[self.dispatch(binary.left)], 1441 star=None, 1442 dstar=None) 1443 ) 1444 ], 1445 else_=[] 1446 ) 1447 ] 1448 ) 1449 ] 1450 1451 self.current_subprograms.pop() 1452 self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram 1453 1454 result = InvokeBlock( 1455 produces_result=1, 1456 expr=LoadRef(ref=subprogram) 1457 ) 1458 1459 # Make nice annotations for the viewer. 1460 1461 binary._left_call = subprogram.code[0].body[0].expr 1462 binary._right_call = subprogram.code[0].handler[0].body[0].expr 1463 1464 return result 1465 1466 def _visitBuiltin(self, builtin, name): 1467 result = InvokeFunction(builtin, 1, expr=LoadName(name=name), args=self.dispatches(builtin.nodes), star=None, dstar=None) 1468 return result 1469 1470 def _visitUnary(self, unary, name): 1471 result = InvokeFunction(unary, 1, 1472 expr=LoadAttr( 1473 expr=self.dispatch(unary.expr), 1474 name=name 1475 ) 1476 ) 1477 1478 # Make nice annotations for the viewer. 1479 1480 unary._unary_call = result 1481 1482 return result 1483 1484 # Convenience functions. 1485 1486 def simplify(filename, builtins=0, module_name=None): 1487 1488 """ 1489 Simplify the module stored in the file with the given 'filename'. 1490 1491 If the optional 'builtins' parameter is set to a true value (the default 1492 being a false value), then the module is considered as the builtins module. 1493 """ 1494 1495 simplifier = Simplifier(builtins) 1496 module = compiler.parseFile(filename) 1497 compiler.misc.set_filename(filename, module) 1498 if builtins: 1499 name = module_name or "__builtins__" 1500 else: 1501 path, ext = os.path.splitext(filename) 1502 path, name = os.path.split(path) 1503 name = module_name or name 1504 simplified = simplifier.process(module, name) 1505 return simplified 1506 1507 # vim: tabstop=4 expandtab shiftwidth=4