simplify

simplify.py

178:81c7b9708818
2007-01-22 paulb Added node copying, used when accessing class attributes for instances where such attributes refer to subprograms (ie. methods); added the acquisition of such copied method attributes in the get_attributes function. Added the original_def attribute for certain kinds of subprograms (functions, lambda expressions) which is used to "paint" copied sections of the simplified node tree with an AST node which can help identify distinct sites for object instantiation. Moved the Self and Attribute classes to the simplified module.
     1 #!/usr/bin/env python     2      3 """     4 Simplify AST structures for easier type propagation and analysis. The code in     5 this module processes AST trees originating from the compiler module and     6 produces a result tree consisting of instruction-oriented program nodes.     7      8 Copyright (C) 2006, 2007 Paul Boddie <paul@boddie.org.uk>     9     10 This software is free software; you can redistribute it and/or    11 modify it under the terms of the GNU General Public License as    12 published by the Free Software Foundation; either version 2 of    13 the License, or (at your option) any later version.    14     15 This software is distributed in the hope that it will be useful,    16 but WITHOUT ANY WARRANTY; without even the implied warranty of    17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    18 GNU General Public License for more details.    19     20 You should have received a copy of the GNU General Public    21 License along with this library; see the file LICENCE.txt    22 If not, write to the Free Software Foundation, Inc.,    23 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA    24     25 --------    26     27 To use this module, the easiest approach is to use the simplify function:    28     29 simplify(filename)    30     31 The more complicated approach involves first instantiating a Simplifier object:    32     33 simplifier = Simplifier()    34     35 Then, applying the simplifier to an AST tree:    36     37 module = compiler.parseFile(filename)    38 simplifier.process(module)    39 """    40     41 from simplified import *    42 import compiler.ast    43 import os    44     45 class Simplifier(Visitor):    46     47     """    48     A simplifying visitor for AST nodes.    49     50     Covered: Add, And, AssAttr, AssList, AssName, AssTuple, Assign, AugAssign,    51              Break, CallFunc, Class, Compare, Const, Continue, Dict, Discard,    52              Div, FloorDiv, For, From, Function, Getattr, Global, If, Import,    53              Invert, Keyword, Lambda, List, Mod, Module, Mul, Name, Not, Or,    54              Pass, Power, Print, Printnl, Raise, Return, Slice, Stmt, Sub,    55              Subscript, TryExcept, TryFinally, Tuple, While, UnaryAdd, UnarySub.    56     57     Missing: Assert, Backquote, Bitand, Bitor, Bitxor, Decorators, Ellipsis,    58              Exec, LeftShift, ListComp, ListCompFor, ListCompIf, RightShift,    59              Sliceobj, Yield.    60     """    61     62     def __init__(self, builtins=0):    63     64         """    65         Initialise the simplifier with the optional 'builtins' parameter    66         indicating whether the module contains the built-in classes and    67         functions.    68         """    69     70         Visitor.__init__(self)    71         self.subprograms = []           # Subprograms outside the tree.    72         self.structures = []            # Structures/classes.    73         self.constants = {}             # Constants.    74         self.current_subprograms = []   # Current subprograms being processed.    75         self.current_structures = []    # Current structures being processed.    76         self.within_class = 0           # Whether a class is being defined.    77         self.builtins = builtins        # Whether the builtins are being processed.    78     79         # Convenience attributes.    80     81         self.subnames = {}    82     83         # For compiler package mechanisms.    84     85         self.visitor = self    86     87     def process(self, node, name):    88         result = self.dispatch(node, name)    89         result.simplifier = self    90         return result    91     92     def dispatch_or_none(self, node, *args):    93         if node is not None:    94             return self.dispatch(node, *args)    95         else:    96             return LoadName(node, name="None")    97     98     # Top-level transformation.    99    100     def visitModule(self, module, name=None):   101    102         """   103         Process the given 'module', producing a Module object which contains the   104         resulting program nodes. If the optional 'name' is provided, the 'name'   105         attribute is set on the Module object using a value other than None.   106         """   107    108         result = self.module = Module(module, 1, name=name)   109         module_code = self.dispatch(module.node)   110    111         # NOTE: Constant initialisation necessary for annotation but perhaps   112         # NOTE: redundant in the program.   113    114         init_code = []   115         for value, constant in self.constants.items():   116             init_code.append(   117                 StoreAttr(   118                     lvalue=LoadRef(ref=constant),   119                     name="__class__",   120                     expr=LoadName(name=constant.typename)   121                     )   122                 )   123    124         # NOTE: Hack to ensure correct initialisation of constants.   125    126         if self.builtins:   127             result.code = module_code + init_code   128         else:   129             result.code = init_code + module_code    130         return result   131    132     # Node transformations.   133    134     def visitAdd(self, add):   135         return self._visitBinary(add, "__add__", "__radd__")   136    137     def visitAnd(self, and_):   138    139         """   140         Make a subprogram for the 'and_' node and record its contents inside the   141         subprogram. Convert...   142    143         And (test)   144             (test)   145             ...   146    147         ...to:   148    149         Subprogram -> Conditional (test) -> ReturnFromBlock ...   150                                   (else) -> Conditional (test) -> ReturnFromBlock ...   151                                                         (else) -> ...   152         """   153    154         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None)   155         self.current_subprograms.append(subprogram)   156    157         # In the subprogram, make instructions which store each operand, test   158         # for each operand's truth status, and if appropriate return from the   159         # subprogram with the value of the operand.   160    161         last = and_.nodes[-1]   162         results = nodes = []   163    164         for node in and_.nodes:   165             expr = self.dispatch(node)   166    167             # Return from the subprogram where the test is not satisfied.   168    169             if node is not last:   170                 nodes += [   171                     StoreTemp(expr=expr),   172                     Conditional(   173                         test=self._visitNot(LoadTemp()),   174                         body=[   175                             ReturnFromBlock(   176                                 expr=LoadTemp()   177                                 )   178                             ],   179                         else_=[   180                             ReleaseTemp()   181                             # Subsequent operations go here!   182                             ]   183                         )   184                     ]   185    186                 # Put subsequent operations in the else section of this conditional.   187    188                 nodes = nodes[-1].else_   189    190             # For the last operation, return the result.   191    192             else:   193                 nodes.append(ReturnFromBlock(expr=expr))   194    195         # Finish the subprogram definition.   196    197         subprogram.code = results   198    199         self.current_subprograms.pop()   200         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram   201    202         # Make an invocation of the subprogram.   203    204         result = InvokeBlock(and_, 1, produces_result=1)   205         result.expr = LoadRef(ref=subprogram)   206         return result   207    208     # Assignments.   209    210     def visitAssAttr(self, assattr, in_sequence=0):   211         expr = self._visitAssNameOrAttr(assattr, in_sequence)   212         lvalue = self.dispatch(assattr.expr)   213         result = StoreAttr(assattr, 1, name=assattr.attrname, lvalue=lvalue, expr=expr)   214         return result   215    216     def visitAssign(self, assign):   217         result = Assign(assign, 1)   218         store = StoreTemp(expr=self.dispatch(assign.expr))   219         release = ReleaseTemp()   220         result.code = [store] + self.dispatches(assign.nodes, 0) + [release]   221         return result   222    223     def visitAssList(self, asslist, in_sequence=0):   224         if not in_sequence:   225             expr = LoadTemp()   226         else:   227             expr = InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next"))   228         result = Assign(asslist, 1)   229         store = StoreTemp(expr=InvokeFunction(expr=LoadAttr(name="__iter__", expr=expr)))   230         release = ReleaseTemp()   231         result.code = [store] + self.dispatches(asslist.nodes, 1) + [release]   232         return result   233    234     visitAssTuple = visitAssList   235    236     def _visitAssNameOrAttr(self, node, in_sequence):   237         if not in_sequence:   238             return LoadTemp()   239         else:   240             return InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next"))   241    242     def visitAssName(self, assname, in_sequence=0):   243         expr = self._visitAssNameOrAttr(assname, in_sequence)   244         result = StoreName(assname, 1, name=assname.name, expr=expr)   245         return result   246    247     augassign_methods = {   248         "+=" : "__iadd__", "-=" : "__isub__", "*=" : "__imul__", "/=" : "__idiv__",   249         "%=" : "__imod__", "**=" : "__ipow__", "<<=" : "__ilshift__", ">>=" : "__irshift__",   250         "&=" : "__iand__", "^=" : "__ixor__", "|=" : "__ior__"   251         }   252    253     def visitAugAssign(self, augassign):   254    255         """   256         Convert the augmented assignment...   257    258         AugAssign (node) -> Name | Getattr | Slice | Subscript   259                   (op)   260                   (expr)   261    262         ...to:   263    264         Assign (code) -> StoreTemp (expr) -> InvokeFunction (expr) -> LoadAttr (expr) -> <name>   265                                                                                (name) -> <op>   266                          StoreName (name) -> <name>   267                                    (expr) -> LoadTemp   268                          ReleaseTemp   269         """   270    271         result = Assign(augassign, 1)   272         expr = self.dispatch(augassign.expr)   273    274         # Simple augmented assignment: name += expr   275    276         if isinstance(augassign.node, compiler.ast.Name):   277             result.code = [   278                 StoreTemp(   279                     expr=InvokeFunction( # referenced below   280                         args=[expr],   281                         star=None,   282                         dstar=None,   283                         expr=LoadAttr(   284                             expr=self.dispatch(augassign.node),   285                             name=self.augassign_methods[augassign.op]   286                             )   287                         )   288                     ),   289                 StoreName(   290                     expr=LoadTemp(),   291                     name=augassign.node.name),   292                 ReleaseTemp()   293                 ]   294    295             # Make nice annotations for the viewer.   296    297             augassign._op_call = result.code[0].expr   298    299         # Complicated augmented assignment: lvalue.attr += expr   300    301         elif isinstance(augassign.node, compiler.ast.Getattr):   302    303             # <lvalue> -> setattr(<lvalue>, getattr(<lvalue>, "attr").__xxx__(expr))   304    305             result.code = [   306                 StoreTemp(   307                     index="expr",   308                     expr=self.dispatch(augassign.node.expr)   309                     ),   310                 StoreTemp(   311                     expr=InvokeFunction( # referenced below   312                         args=[expr], star=None, dstar=None,   313                         expr=LoadAttr(   314                             expr=LoadAttr(augassign.node, 1,   315                                 expr=LoadTemp(index="expr"),   316                                 name=augassign.node.attrname   317                                 ),   318                             name=self.augassign_methods[augassign.op]   319                             )   320                         )   321                     ),   322                 StoreAttr(   323                     expr=LoadTemp(),   324                     lvalue=LoadTemp(index="expr"),   325                     name=augassign.node.attrname   326                     ),   327                 ReleaseTemp(index="expr"),   328                 ReleaseTemp()   329                 ]   330    331             # Make nice annotations for the viewer.   332    333             augassign._op_call = result.code[1].expr   334    335         # Complicated augassign using slices: lvalue[lower:upper] += expr   336    337         elif isinstance(augassign.node, compiler.ast.Slice):   338    339             # <lvalue>, <lower>, <upper> -> <lvalue>.__setslice__(<lower>, <upper>, <lvalue>.__getslice__(<lower>, <upper>).__xxx__(expr))   340    341             result.code = [   342                 StoreTemp(   343                     index="expr",   344                     expr=self.dispatch(augassign.node.expr)   345                     ),   346                 StoreTemp(   347                     index="lower",   348                     expr=self.dispatch_or_none(augassign.node.lower)   349                     ),   350                 StoreTemp(   351                     index="upper",   352                     expr=self.dispatch_or_none(augassign.node.upper)   353                     ),   354                 StoreTemp(   355                     expr=InvokeFunction( # referenced below   356                         args=[expr], star=None, dstar=None,   357                         expr=LoadAttr(   358                             expr=self._visitSlice(   359                                 augassign.node,   360                                 LoadTemp(index="expr"),   361                                 LoadTemp(index="lower"),   362                                 LoadTemp(index="upper"),   363                                 "OP_APPLY"),   364                             name=self.augassign_methods[augassign.op]   365                             )   366                         )   367                     ),   368                 self._visitSlice(   369                     augassign.node,   370                     LoadTemp(index="expr"),   371                     LoadTemp(index="lower"),   372                     LoadTemp(index="upper"),   373                     "OP_ASSIGN",   374                     LoadTemp()   375                     ),   376                 ReleaseTemp(index="expr"),   377                 ReleaseTemp(index="lower"),   378                 ReleaseTemp(index="upper"),   379                 ReleaseTemp()   380                 ]   381    382             # Make nice annotations for the viewer.   383    384             augassign._op_call = result.code[3].expr   385    386         # Complicated augassign using subscripts: lvalue[subs] += expr   387    388         elif isinstance(augassign.node, compiler.ast.Subscript):   389    390             # <lvalue>, <subs> -> <lvalue>.__setitem__(<subs>, <lvalue>.__getitem__(<subs>).__xxx__(expr))   391    392             result.code = [   393                 StoreTemp(index="expr", expr=self.dispatch(augassign.node.expr)),   394                 StoreTemp(index="subs", expr=self._visitSubscriptSubs(augassign.node, augassign.node.subs)),   395                 StoreTemp(   396                     expr=InvokeFunction( # referenced below   397                         args=[expr], star=None, dstar=None,   398                         expr=LoadAttr(   399                             expr=self._visitSubscript(   400                                 augassign.node,   401                                 LoadTemp(index="expr"),   402                                 LoadTemp(index="subs"),   403                                 "OP_APPLY"   404                                 ),   405                             name=self.augassign_methods[augassign.op]   406                             )   407                         )   408                     ),   409                 self._visitSubscript(   410                     augassign.node,   411                     LoadTemp(index="expr"),   412                     LoadTemp(index="subs"),   413                     "OP_ASSIGN",   414                     LoadTemp()   415                     ),   416                 ReleaseTemp(index="expr"),   417                 ReleaseTemp(index="subs"),   418                 ReleaseTemp()   419                 ]   420    421             # Make nice annotations for the viewer.   422    423             augassign._op_call = result.code[2].expr   424    425         else:   426             raise NotImplementedError, augassign.node.__class__   427    428         return result   429    430     def visitBreak(self, break_):   431         result = ReturnFromBlock(break_, 1)   432         return result   433    434     def visitCallFunc(self, callfunc):   435         result = InvokeFunction(callfunc, 1, star=None, dstar=None, args=self.dispatches(callfunc.args))   436         if callfunc.star_args is not None:   437             result.star = self.dispatch(callfunc.star_args)   438         if callfunc.dstar_args is not None:   439             result.dstar = self.dispatch(callfunc.dstar_args)   440         result.expr = self.dispatch(callfunc.node)   441         return result   442    443     def visitClass(self, class_):   444    445         # Add "object" if the class is not "object" and has an empty bases list.   446    447         if class_.name != "object" and not class_.bases:   448             bases = [compiler.ast.Name("object")]   449         else:   450             bases = class_.bases   451    452         structure = Class(name=class_.name, module=self.module, bases=self.dispatches(bases))   453         self.structures.append(structure)   454         within_class = self.within_class   455         self.within_class = 1   456    457         # Make a subprogram which initialises the class structure.   458    459         subprogram = Subprogram(name=None, module=self.module, structure=structure, params=[], star=None, dstar=None)   460         self.current_subprograms.append(subprogram)   461         self.current_structures.append(structure) # mostly for name construction   462    463         # The class is initialised using the code found inside.   464    465         subprogram.code = self.dispatch(class_.code) + [ReturnFromBlock()]   466    467         self.within_class = within_class   468         self.current_structures.pop()   469         self.current_subprograms.pop()   470         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram   471    472         # Make a definition of the class associating it with a name.   473    474         result = Assign(   475             code=[   476                 StoreName(class_, 1, # defines the class   477                     name=class_.name,   478                     expr=LoadRef(ref=structure)   479                     ),   480                 InvokeBlock(   481                     share_locals=0, # override the local sharing usually in InvokeBlock   482                     expr=LoadRef(ref=subprogram)   483                     )   484                 ]   485             )   486         return result   487    488     comparison_methods = {   489         "==" : "__eq__", "!=" : "__ne__", "<" : "__lt__", "<=" : "__le__",   490         ">=" : "__ge__", ">" : "__gt__", "is" : None, "is not" : None   491         }   492    493     def visitCompare(self, compare):   494    495         """   496         Make a subprogram for the 'compare' node and record its contents inside   497         the subprogram. Convert...   498    499         Compare (expr)   500                 (name/node)   501                 ...   502    503         ...to:   504    505         InvokeBlock -> Subprogram -> Conditional (test) -> (body)   506                                                  (else) -> Conditional (test) -> (body)   507                                                                        (else) -> ...   508         """   509    510         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None)   511         self.current_subprograms.append(subprogram)   512    513         # In the subprogram, make instructions which invoke a method on the   514         # first operand of each operand pair and, if appropriate, return with   515         # the value from that method.   516    517         last = compare.ops[-1]   518         previous = self.dispatch(compare.expr)   519         results = nodes = []   520    521         # For viewing purposes, record invocations on the AST node.   522    523         compare._ops = []   524    525         for op in compare.ops:   526             op_name, node = op   527             expr = self.dispatch(node)   528    529             # Identify the operation and produce the appropriate method call.   530    531             method_name = self.comparison_methods[op_name]   532             if method_name:   533                 invocation = InvokeFunction(   534                     expr=LoadAttr(   535                         expr=previous,   536                         name=method_name),   537                     args=[expr],   538                     star=None,   539                     dstar=None)   540    541             elif op_name == "is":   542                 invocation = InvokeFunction(   543                     expr=LoadName(name="__is__"),   544                     args=[previous, expr],   545                     star=None,   546                     dstar=None)   547    548             elif op_name == "is not":   549                 invocation = Not(   550                     expr=InvokeFunction(   551                         expr=LoadName(name="__is__"),   552                         args=[previous, expr],   553                         star=None,   554                         dstar=None)   555                     )   556             else:   557                 raise NotImplementedError, op_name   558    559             nodes.append(StoreTemp(expr=invocation))   560             compare._ops.append(invocation)   561    562             # Return from the subprogram where the test is not satisfied.   563    564             if op is not last:   565                 nodes.append(   566                     Conditional(   567                         test=self._visitNot(LoadTemp()),   568                         body=[   569                             ReturnFromBlock(expr=LoadTemp())   570                             ],   571                         else_=[   572                             ReleaseTemp()   573                             # Subsequent operations go here!   574                             ]   575                         )   576                     )   577    578                 # Put subsequent operations in the else section of this conditional.   579    580                 nodes = nodes[-1].else_   581    582             # For the last operation, return the result.   583    584             else:   585                 nodes.append(   586                     ReturnFromBlock(expr=LoadTemp(release=1))   587                     )   588    589             previous = expr   590    591         # Finish the subprogram definition.   592    593         subprogram.code = results   594    595         self.current_subprograms.pop()   596         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram   597    598         # Make an invocation of the subprogram.   599    600         result = InvokeBlock(compare, 1, produces_result=1)   601         result.expr = LoadRef(ref=subprogram)   602         return result   603    604     def visitConst(self, const):   605         key = "%s-%s" % (const.value.__class__.__name__, const.value)   606         if not self.constants.has_key(key):   607             self.constants[key] = Constant(name=repr(const.value), value=const.value)   608         result = LoadRef(const, 1, ref=self.constants[key])   609         return result   610    611     def visitContinue(self, continue_):   612         result = InvokeBlock(continue_, 1,   613             expr=LoadRef(ref=self.current_subprograms[-1])   614             )   615         return result   616    617     def visitDict(self, dict):   618         result = InvokeFunction(dict, 1, expr=LoadName(name="dict"), star=None, dstar=None)   619         args = []   620         for key, value in dict.items:   621             tuple = InvokeFunction(expr=LoadName(name="tuple"), star=None, dstar=None)   622             tuple.set_args([self.dispatch(key), self.dispatch(value)])   623             args.append(tuple)   624         result.set_args(args)   625         return result   626    627     def visitDiscard(self, discard):   628         return self.dispatch(discard.expr)   629    630     def visitDiv(self, div):   631         return self._visitBinary(div, "__div__", "__rdiv__")   632    633     def visitFloorDiv(self, floordiv):   634         return self._visitBinary(floordiv, "__floordiv__", "__rfloordiv__")   635    636     def visitFor(self, for_):   637    638         """   639         Make a subprogram for the 'for_' node and record its contents inside the   640         subprogram. Convert...   641    642         For (assign)   643             (body)   644             (else)   645    646         ...to:   647    648         Assign (assign #1)   649         Invoke -> Subprogram -> Try (body) -> (assign #2)   650                                               (body)   651                                               Invoke subprogram   652                                     (handler) -> ...   653                                     (else) -> ...   654         """   655    656         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None)   657         self.current_subprograms.append(subprogram)   658    659         # Always return from conditional sections/subprograms.   660    661         if for_.else_ is not None:   662             else_stmt = self.dispatch(for_.else_) + [ReturnFromBlock()]   663         else:   664             else_stmt = [ReturnFromBlock()]   665    666         # Wrap the assignment in a try...except statement.   667         # Inside the body, add a recursive invocation to the subprogram.   668    669         subprogram.code = [   670             Try(   671                 body=[   672                     Assign(   673                         code=[   674                             StoreTemp(expr=InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="next"))),   675                             self.dispatch(for_.assign),   676                             ReleaseTemp()   677                             ])   678                     ] + self.dispatch(for_.body) + [   679                     InvokeBlock(   680                         expr=LoadRef(ref=subprogram)   681                         )   682                     ],   683                 handler=[   684                     Conditional(   685                         test=InvokeFunction(   686                             expr=LoadName(name="isinstance"),   687                             args=[LoadExc(), LoadName(name="StopIteration")],   688                             star=None,   689                             dstar=None),   690                         body=else_stmt,   691                         else_=[Raise(expr=LoadExc())]   692                         )   693                     ],   694                 else_=[],   695                 finally_=[]   696                 ),   697             ReturnFromBlock()   698             ]   699    700         # Finish the subprogram definition.   701    702         self.current_subprograms.pop()   703         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram   704    705         # Obtain an iterator for the sequence involved.   706         # Then, make an invocation of the subprogram.   707    708         result = Assign(for_, 1,   709             code=[   710                 StoreTemp(   711                     expr=InvokeFunction(   712                         expr=LoadAttr(   713                             name="__iter__",   714                             expr=self.dispatch(for_.list)   715                             )   716                         )   717                     ),   718                 InvokeBlock(expr=LoadRef(ref=subprogram)),   719                 ReleaseTemp()   720                 ]   721             )   722    723         # Make nice annotations for the viewer.   724    725         for_._iter_call = result.code[0].expr   726         for_._next_call = subprogram.code[0].body[0].code[0].expr   727    728         return result   729    730     def visitFrom(self, from_):   731         result = Assign(from_, 1)   732         code = []   733         _names = []   734         code.append(   735             StoreTemp(   736                 expr=Import(name=from_.modname, alias=1)   737                 )   738             )   739         from_._modname = code[-1].expr   740         for name, alias in from_.names:   741             code.append(   742                 StoreName(   743                     expr=LoadAttr(   744                         expr=LoadTemp(),   745                         name=name),   746                     name=(alias or name)   747                     )   748                 )   749             _names.append(code[-1].expr)   750         code.append(ReleaseTemp())   751         result.code = code   752         from_._names = _names   753         return result   754    755     def _visitFunction(self, function, subprogram):   756    757         """   758         A common function generator which transforms the given 'function' node   759         and initialises the given 'subprogram' appropriately.   760         """   761    762         # Discover star and dstar parameters.   763    764         if function.flags & 4 != 0:   765             has_star = 1   766         else:   767             has_star = 0   768         if function.flags & 8 != 0:   769             has_dstar = 1   770         else:   771             has_dstar = 0   772    773         # Discover the number of defaults and positional parameters.   774    775         ndefaults = len(function.defaults)   776         npositional = len(function.argnames) - has_star - has_dstar   777    778         # Produce star and dstar parameters with appropriate defaults.   779    780         if has_star:   781             star = (   782                 function.argnames[npositional],   783                 InvokeFunction(expr=LoadName(name="list"))   784                 )   785         else:   786             star = None   787         if has_dstar:   788             dstar = (   789                 function.argnames[npositional + has_star],   790                 InvokeFunction(expr=LoadName(name="dict"))   791                 )   792         else:   793             dstar = None   794    795         params = []   796         for i in range(0, npositional - ndefaults):   797             params.append((function.argnames[i], None))   798    799         # Process defaults.   800    801         for i in range(0, ndefaults):   802             default = function.defaults[i]   803             if default is not None:   804                 params.append((function.argnames[npositional - ndefaults + i], self.dispatch(default)))   805             else:   806                 params.append((function.argnames[npositional - ndefaults + i], None))   807    808         subprogram.params = params   809         subprogram.star = star   810         subprogram.dstar = dstar   811         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram   812    813     def visitFunction(self, function):   814    815         """   816         Make a subprogram for the 'function' and record it outside the main   817         tree. Produce something like the following:   818    819         StoreName (name)   820                   (expr) -> LoadRef (ref) -> Subprogram (params)   821                                                         (star)   822                                                         (dstar)   823         """   824    825         subprogram = Subprogram(name=function.name, module=self.module, structures=self.current_structures[:],   826             internal=0, returns_value=1, star=None, dstar=None, is_method=self.within_class, original_def=function)   827    828         self.current_subprograms.append(subprogram)   829         within_class = self.within_class   830         self.within_class = 0   831    832         subprogram.code = self.dispatch(function.code) + [ReturnFromFunction()]   833    834         self.within_class = within_class   835         self.current_subprograms.pop()   836         self._visitFunction(function, subprogram)   837    838         # Make a definition of the function associating it with a name.   839    840         result = StoreName(function, 1, name=function.name, expr=LoadRef(ref=subprogram))   841         return result   842    843     def visitGetattr(self, getattr):   844         result = LoadAttr(getattr, 1,   845             name=getattr.attrname,   846             expr=self.dispatch(getattr.expr)   847             )   848         return result   849    850     def visitGlobal(self, global_):   851         result = Global(global_, 1,   852             names=global_.names   853             )   854         return result   855    856     def visitIf(self, if_):   857    858         """   859         Make conditionals for each test from an 'if_' AST node, adding the body   860         and putting each subsequent test as part of the conditional's else   861         section.   862    863         Convert...   864    865         If (test/body)   866            (test/body)   867            ...   868            (else/body)   869    870         ...to:   871    872         Conditional (test) -> (body)   873                     (else) -> Conditional (test) -> (body)   874                                           (else) -> ...   875         """   876    877    878         results = nodes = []   879    880         # Produce something like...   881         # expr.__bool__() ? body   882    883         first = 1   884         for compare, stmt in if_.tests:   885    886             # Set the first as the defining node.   887    888             test = Conditional(if_, first,   889                 test=InvokeFunction(   890                     expr=LoadAttr(   891                         expr=self.dispatch(compare),   892                         name="__bool__"   893                         ),   894                     )   895                 )   896             test.body = self.dispatch(stmt)   897             nodes.append(test)   898             nodes = test.else_ = []   899             first = 0   900    901         # Add the compound statement from any else clause to the end.   902    903         if if_.else_ is not None:   904             nodes += self.dispatch(if_.else_)   905    906         result = results[0]   907         return result   908    909     def visitImport(self, import_):   910         result = Assign(import_, 1)   911         code = []   912         _names = []   913         for path, alias in import_.names:   914             importer = Import(name=path, alias=alias)   915             top = alias or path.split(".")[0]   916             code.append(StoreName(expr=importer, name=top))   917             _names.append(code[-1].expr)   918         result.code = code   919         import_._names = _names   920         return result   921    922     def visitInvert(self, invert):   923         return self._visitUnary(invert, "__invert__")   924    925     def visitKeyword(self, keyword):   926         result = Keyword(keyword, 1,   927             name=keyword.name,   928             expr=self.dispatch(keyword.expr)   929             )   930         return result   931    932     def visitLambda(self, lambda_):   933    934         # Make a subprogram for the function and record it outside the main   935         # tree.   936    937         subprogram = Subprogram(name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=lambda_)   938         self.current_subprograms.append(subprogram)   939         subprogram.code = [ReturnFromFunction(expr=self.dispatch(lambda_.code))]   940         self.current_subprograms.pop()   941         self._visitFunction(lambda_, subprogram)   942    943         # Get the subprogram reference to the lambda.   944    945         return LoadRef(lambda_, 1, ref=subprogram)   946    947     def visitList(self, list):   948         return self._visitBuiltin(list, "list")   949    950     def visitMod(self, mod):   951         return self._visitBinary(mod, "__mod__", "__rmod__")   952    953     def visitMul(self, mul):   954         return self._visitBinary(mul, "__mul__", "__rmul__")   955    956     def visitName(self, name):   957         result = LoadName(name, 1, name=name.name)   958         return result   959    960     def _visitNot(self, expr, not_=None):   961         invocation = InvokeFunction(   962             expr=LoadAttr(   963                 expr=expr,   964                 name="__bool__"   965                 ),   966             )   967         if not_ is not None:   968             result = Not(not_, 1, expr=invocation)   969         else:   970             result = Not(expr=invocation)   971         return result   972    973     def visitNot(self, not_):   974         return self._visitNot(self.dispatch(not_.expr), not_)   975    976     def visitOr(self, or_):   977    978         """   979         Make a subprogram for the 'or_' node and record its contents inside the   980         subprogram. Convert...   981    982         Or (test)   983            (test)   984            ...   985    986         ...to:   987    988         Subprogram -> Conditional (test) -> ReturnFromBlock ...   989                                   (else) -> Conditional (test) -> ReturnFromBlock ...   990                                                         (else) -> ...   991         """   992    993         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None)   994         self.current_subprograms.append(subprogram)   995    996         # In the subprogram, make instructions which store each operand, test   997         # for each operand's truth status, and if appropriate return from the   998         # subprogram with the value of the operand.   999   1000         last = or_.nodes[-1]  1001         results = nodes = []  1002   1003         for node in or_.nodes:  1004             expr = self.dispatch(node)  1005   1006             # Return from the subprogram where the test is satisfied.  1007   1008             if node is not last:  1009                 nodes.append(StoreTemp(expr=expr))  1010                 invocation = InvokeFunction(expr=LoadAttr(expr=LoadTemp(), name="__bool__"))  1011                 test = Conditional(test=invocation, body=[ReturnFromBlock(expr=LoadTemp())])  1012                 nodes.append(test)  1013   1014                 # Put subsequent operations in the else section of this conditional.  1015   1016                 nodes = test.else_ = [ReleaseTemp()]  1017   1018             # For the last operation, return the result.  1019   1020             else:  1021                 nodes.append(  1022                     ReturnFromBlock(expr=expr)  1023                     )  1024   1025         # Finish the subprogram definition.  1026   1027         subprogram.code = results  1028   1029         self.current_subprograms.pop()  1030         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram  1031   1032         # Make an invocation of the subprogram.  1033   1034         result = InvokeBlock(or_, 1, produces_result=1)  1035         result.expr = LoadRef(ref=subprogram)  1036         return result  1037   1038     def visitPass(self, pass_):  1039         return Pass(pass_, 1)  1040   1041     def visitPower(self, power):  1042         return self._visitBinary(power, "__pow__", "__rpow__")  1043   1044     def visitPrint(self, print_):  1045   1046         """  1047         Convert...  1048   1049         Print (dest) ->  1050               (nodes)  1051   1052         ...to:  1053   1054         StoreTemp (index) -> "print"  1055                   (expr) -> LoadAttr (expr) -> (dest)  1056                                      (name) -> "write"  1057         InvokeFunction (expr) -> LoadTemp (index) -> "print"  1058                        (args) -> [(node)]  1059         ReleaseTemp (index) -> "print"  1060         """  1061   1062         if print_.dest is not None:  1063             dest = self.dispatch(print_.dest)  1064         else:  1065             dest = self.dispatch(compiler.ast.Name("stdout"))  1066   1067         result = Assign(print_, 1,  1068             code=[  1069                 StoreTemp(  1070                     index="print",  1071                     expr=LoadAttr(  1072                         expr=dest,  1073                         name="write"  1074                         )  1075                     )  1076                 ]  1077             )  1078   1079         for node in print_.nodes:  1080             result.code.append(  1081                 InvokeFunction(  1082                     expr=LoadTemp(index="print"),  1083                     args=[self.dispatch(node)],  1084                     star=None,  1085                     dstar=None  1086                     )  1087                 )  1088   1089         result.code.append(  1090             ReleaseTemp(index="print")  1091             )  1092   1093         return result  1094   1095     def visitPrintnl(self, printnl):  1096         result = self.visitPrint(printnl)  1097         result.code.insert(  1098             len(result.code) - 1,  1099             InvokeFunction(  1100                 expr=LoadTemp(index="print"),  1101                 args=[self.dispatch(compiler.ast.Const("\n"))],  1102                 star=None,  1103                 dstar=None  1104                 )  1105             )  1106         return result  1107   1108     def visitRaise(self, raise_):  1109         result = Raise(raise_, 1)  1110         if raise_.expr2 is None:  1111             result.expr = self.dispatch(raise_.expr1)  1112         else:  1113             result.expr = InvokeFunction(  1114                 expr=self.dispatch(raise_.expr1),  1115                 args=[self.dispatch(raise_.expr2)],  1116                 star=None,  1117                 dstar=None  1118                 )  1119         if raise_.expr3 is not None:  1120             result.traceback = self.dispatch(raise_.expr3)  1121         else:  1122             result.traceback = None  1123         return result  1124   1125     def visitReturn(self, return_):  1126         result = ReturnFromFunction(return_, 1,  1127             expr=self.dispatch(return_.value)  1128             )  1129         return result  1130   1131     def _visitSlice(self, slice, expr, lower, upper, flags, value=None):  1132         if flags == "OP_ASSIGN":  1133             result = InvokeFunction(slice, 1,  1134                 expr=LoadAttr(  1135                     expr=expr,  1136                     name="__setslice__"  1137                     ),  1138                 star=None,  1139                 dstar=None,  1140                 args=[lower, upper, value]  1141                 )  1142         elif flags == "OP_APPLY":  1143             args = []  1144             result = InvokeFunction(slice, 1,  1145                 expr=LoadAttr(  1146                     expr=expr,  1147                     name="__getslice__"  1148                     ),  1149                 star=None,  1150                 dstar=None,  1151                 args=[lower, upper]  1152                 )  1153         elif flags == "OP_DELETE":  1154             args = []  1155             result = InvokeFunction(slice, 1,  1156                 expr=LoadAttr(  1157                     expr=expr,  1158                     name="__delslice__"  1159                     ),  1160                 star=None,  1161                 dstar=None,  1162                 args=[lower, upper]  1163                 )  1164         else:  1165             raise NotImplementedError, flags  1166   1167         return result  1168   1169     def visitSlice(self, slice, in_sequence=0):  1170         return self._visitSlice(slice, self.dispatch(slice.expr), self.dispatch_or_none(slice.lower),  1171             self.dispatch_or_none(slice.upper), slice.flags, self._visitAssNameOrAttr(slice, in_sequence))  1172   1173     def visitStmt(self, stmt):  1174         return self.dispatches(stmt.nodes)  1175   1176     def visitSub(self, sub):  1177         return self._visitBinary(sub, "__sub__", "__rsub__")  1178   1179     def _visitSubscript(self, subscript, expr, subs, flags, value=None):  1180         if flags == "OP_ASSIGN":  1181             result = InvokeFunction(subscript, 1,  1182                 expr=LoadAttr(  1183                     expr=expr,  1184                     name="__setitem__"  1185                     ),  1186                 star=None,  1187                 dstar=None,  1188                 args=[subs, value]  1189                 )  1190         elif flags == "OP_APPLY":  1191             args = []  1192             result = InvokeFunction(subscript, 1,  1193                 expr=LoadAttr(  1194                     expr=expr,  1195                     name="__getitem__"  1196                     ),  1197                 star=None,  1198                 dstar=None,  1199                 args=[subs]  1200                 )  1201         elif flags == "OP_DELETE":  1202             args = []  1203             result = InvokeFunction(subscript, 1,  1204                 expr=LoadAttr(  1205                     expr=expr,  1206                     name="__delitem__"  1207                     ),  1208                 star=None,  1209                 dstar=None,  1210                 args=[subs]  1211                 )  1212         else:  1213             raise NotImplementedError, flags  1214   1215         return result  1216   1217     def _visitSubscriptSubs(self, node, subs):  1218         if len(subs) == 1:  1219             return self.dispatch(subs[0])  1220         else:  1221             return InvokeFunction(node, 1,  1222                 expr=LoadName(name="tuple"),  1223                 args=self.dispatches(subs),  1224                 star=None,  1225                 dstar=None  1226                 )  1227   1228     def visitSubscript(self, subscript, in_sequence=0):  1229         return self._visitSubscript(  1230             subscript, self.dispatch(subscript.expr), self._visitSubscriptSubs(subscript, subscript.subs), subscript.flags,  1231             self._visitAssNameOrAttr(subscript, in_sequence)  1232             )  1233   1234     def visitTryExcept(self, tryexcept):  1235   1236         """  1237         Make conditionals for each handler associated with a 'tryexcept' node.  1238   1239         Convert...  1240   1241         TryExcept (body)  1242                   (else)  1243                   (spec/assign/stmt)  1244                   ...  1245   1246         ...to:  1247   1248         Try (body)  1249             (else)  1250             (handler) -> Conditional (test) -> (stmt)  1251                                      (body) -> ...  1252                                      (else) -> Conditional (test) -> (stmt)  1253                                                            (body) -> ...  1254                                                            (else) -> ...  1255         """  1256                     1257         result = Try(tryexcept, 1, body=[], else_=[], finally_=[])  1258   1259         if tryexcept.body is not None:  1260             result.body = self.dispatch(tryexcept.body)  1261         if tryexcept.else_ is not None:  1262             result.else_ = self.dispatch(tryexcept.else_)  1263   1264         results = nodes = []  1265         catch_all = 0  1266   1267         for spec, assign, stmt in tryexcept.handlers:  1268   1269             # If no specification exists, produce an unconditional block.  1270   1271             if spec is None:  1272                 nodes += self.dispatch(stmt)  1273                 catch_all = 1  1274   1275             # Produce an exception value check.  1276   1277             else:  1278                 test = Conditional(  1279                     isolate_test=1,  1280                     test=CheckExc(expr=LoadExc(), choices=self._visitTryExcept(spec))  1281                     )  1282                 test.body = []  1283   1284                 if assign is not None:  1285                     test.body.append(  1286                         Assign(  1287                             code=[  1288                                 StoreTemp(expr=LoadExc()),  1289                                 self.dispatch(assign),  1290                                 ReleaseTemp()  1291                                 ]  1292                             )  1293                         )  1294   1295                 test.body += self.dispatch(stmt)  1296                 nodes.append(test)  1297                 nodes = test.else_ = []  1298   1299         # Add a raise operation to deal with unhandled exceptions.  1300   1301         if not catch_all:  1302             nodes.append(  1303                 Raise(  1304                     expr=LoadExc())  1305                 )  1306   1307         result.handler = results  1308         return result  1309   1310     def _visitTryExcept(self, spec):  1311   1312         "Return a list of nodes for the given exception type 'spec'."  1313   1314         if isinstance(spec, compiler.ast.Tuple):  1315             nodes = []  1316             for node in spec.nodes:  1317                 nodes += self._visitTryExcept(node)  1318         else:  1319             nodes = [self.dispatch(spec)]  1320         return nodes  1321   1322     def visitTryFinally(self, tryfinally):  1323         result = Try(tryfinally, 1, body=[], else_=[], finally_=[])  1324         if tryfinally.body is not None:  1325             result.body = self.dispatch(tryfinally.body)  1326         if tryfinally.final is not None:  1327             result.finally_ = self.dispatch(tryfinally.final)  1328         return result  1329   1330     def visitTuple(self, tuple):  1331         return self._visitBuiltin(tuple, "tuple")  1332   1333     def visitUnaryAdd(self, unaryadd):  1334         return self._visitUnary(unaryadd, "__pos__")  1335   1336     def visitUnarySub(self, unarysub):  1337         return self._visitUnary(unarysub, "__neg__")  1338   1339     def visitWhile(self, while_):  1340   1341         """  1342         Make a subprogram for the 'while' node and record its contents inside the  1343         subprogram. Convert...  1344   1345         While (test) -> (body)  1346               (else)  1347   1348         ...to:  1349   1350         Subprogram -> Conditional (test) -> (body) -> Invoke subprogram  1351                                   (else) -> Conditional (test) -> ReturnFromBlock ...  1352                                                         (else) -> ...  1353         """  1354   1355         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None)  1356         self.current_subprograms.append(subprogram)  1357   1358         # Include a conditional statement in the subprogram.  1359         # Inside the conditional, add a recursive invocation to the subprogram  1360         # if the test condition was satisfied.  1361         # Return within the main section of the loop.  1362   1363         test = Conditional(  1364             test=InvokeFunction(  1365                 expr=LoadAttr(  1366                     expr=self.dispatch(while_.test),  1367                     name="__bool__"),  1368                 ),  1369             body=self.dispatch(while_.body) + [  1370                 InvokeBlock(  1371                     expr=LoadRef(ref=subprogram)  1372                     ),  1373                 ReturnFromBlock()  1374                 ],  1375             else_=[]  1376             )  1377   1378         # Provide the else section, if present, along with an explicit return.  1379   1380         if while_.else_ is not None:  1381             test.else_ = self.dispatch(while_.else_) + [ReturnFromBlock()]  1382   1383         # Finish the subprogram definition.  1384   1385         subprogram.code = [test]  1386   1387         self.current_subprograms.pop()  1388         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram  1389   1390         # Make an invocation of the subprogram.  1391   1392         result = InvokeBlock(while_, 1,  1393             expr=LoadRef(ref=subprogram)  1394             )  1395   1396         # Make nice annotations for the viewer.  1397   1398         while_._test_call = subprogram.code[0].test  1399   1400         return result  1401   1402     # Convenience methods.  1403   1404     def _visitBinary(self, binary, left_name, right_name):  1405   1406         """  1407         Emulate the current mechanisms by producing nodes as follows:  1408   1409         InvokeBlock -> Subprogram -> Try (body) -> ReturnFromBlock (expr) -> x.__add__(y)  1410                                          (else)  1411                                          (handler) -> Conditional (test) -> CheckExc (expr) -> LoadExc  1412                                                                                      (choices) -> LoadName TypeError  1413                                                                   (body) -> ReturnFromBlock (expr) -> y.__radd__(x)  1414                                                                   (else)  1415         """  1416   1417         subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None)  1418         self.current_subprograms.append(subprogram)  1419   1420         subprogram.code = [  1421             Try(binary, 1,  1422                 body=[  1423                     ReturnFromBlock(  1424                         expr=InvokeFunction(  1425                             expr=LoadAttr(expr=self.dispatch(binary.left), name=left_name),  1426                             args=[self.dispatch(binary.right)],  1427                             star=None,  1428                             dstar=None)  1429                         )  1430                     ],  1431                 else_=[],  1432                 finally_=[],  1433                 handler=[  1434                     Conditional(  1435                         test=CheckExc(expr=LoadExc(), choices=[LoadName(name="TypeError")]),  1436                         body=[  1437                             ReturnFromBlock(  1438                                 expr=InvokeFunction(  1439                                     expr=LoadAttr(expr=self.dispatch(binary.right), name=right_name),  1440                                     args=[self.dispatch(binary.left)],  1441                                     star=None,  1442                                     dstar=None)  1443                                 )  1444                             ],  1445                         else_=[]  1446                         )  1447                     ]  1448                 )  1449             ]  1450   1451         self.current_subprograms.pop()  1452         self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram  1453   1454         result = InvokeBlock(  1455             produces_result=1,  1456             expr=LoadRef(ref=subprogram)  1457             )  1458   1459         # Make nice annotations for the viewer.  1460   1461         binary._left_call = subprogram.code[0].body[0].expr  1462         binary._right_call = subprogram.code[0].handler[0].body[0].expr  1463   1464         return result  1465   1466     def _visitBuiltin(self, builtin, name):  1467         result = InvokeFunction(builtin, 1, expr=LoadName(name=name), args=self.dispatches(builtin.nodes), star=None, dstar=None)  1468         return result  1469   1470     def _visitUnary(self, unary, name):  1471         result = InvokeFunction(unary, 1,  1472             expr=LoadAttr(  1473                 expr=self.dispatch(unary.expr),  1474                 name=name  1475                 )  1476             )  1477   1478         # Make nice annotations for the viewer.  1479   1480         unary._unary_call = result  1481   1482         return result  1483   1484 # Convenience functions.  1485   1486 def simplify(filename, builtins=0, module_name=None):  1487   1488     """  1489     Simplify the module stored in the file with the given 'filename'.  1490   1491     If the optional 'builtins' parameter is set to a true value (the default  1492     being a false value), then the module is considered as the builtins module.  1493     """  1494   1495     simplifier = Simplifier(builtins)  1496     module = compiler.parseFile(filename)  1497     compiler.misc.set_filename(filename, module)  1498     if builtins:  1499         name = module_name or "__builtins__"  1500     else:  1501         path, ext = os.path.splitext(filename)  1502         path, name = os.path.split(path)  1503         name = module_name or name  1504     simplified = simplifier.process(module, name)  1505     return simplified  1506   1507 # vim: tabstop=4 expandtab shiftwidth=4