Lichen

inspector.py

852:7c314a9d1724
2018-07-12 Paul Boddie Attempt to reintroduce function acquisition optimisations by extending context tests related to invocations of access results.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from branching import BranchTracker    24 from common import CommonModule, get_argnames, init_item, predefined_constants    25 from modules import BasicModule, CacheWritingModule, InspectionNaming    26 from errors import InspectError    27 from referencing import Reference    28 from resolving import NameResolving    29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    30                     LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \    31                     Result, VariableRef    32 import compiler    33 import sys    34     35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    36     37     "A module inspector."    38     39     def __init__(self, name, importer):    40     41         "Initialise the module with basic details."    42     43         BasicModule.__init__(self, name, importer)    44     45         self.in_class = False    46         self.in_conditional = False    47     48         # Accesses to global attributes.    49     50         self.global_attr_accesses = {}    51     52         # Usage tracking.    53     54         self.trackers = []    55         self.attr_accessor_branches = {}    56     57     def __repr__(self):    58         return "InspectedModule(%r, %r)" % (self.name, self.importer)    59     60     # Principal methods.    61     62     def parse(self, filename):    63     64         "Parse the file having the given 'filename'."    65     66         self.parse_file(filename)    67     68         # Inspect the module.    69     70         self.start_tracking_in_module()    71     72         # Detect and record imports and globals declared in the module.    73     74         self.process_structure(self.astnode)    75     76         # Set the class of the module after the definition has occurred.    77     78         ref = self.get_builtin("module")    79         self.set_name("__class__", ref)    80         self.set_name("__name__", self.get_constant("string", self.name).reference())    81         self.set_name("__file__", self.get_constant("string", filename).reference())    82     83         # Reserve a constant for the encoding.    84     85         if self.encoding:    86             self.get_constant("string", self.encoding)    87     88         # Get module-level attribute usage details.    89     90         self.stop_tracking_in_module()    91     92         # Collect external name references.    93     94         self.collect_names()    95     96     def complete(self):    97     98         "Complete the module inspection."    99    100         # Resolve names not definitively mapped to objects.   101    102         self.resolve()   103    104         # Propagate to the importer information needed in subsequent activities.   105    106         self.propagate()   107    108     # Accessory methods.   109    110     def collect_names(self):   111    112         "Collect the names used by each scope."   113    114         for path in self.names_used.keys():   115             self.collect_names_for_path(path)   116    117     def collect_names_for_path(self, path):   118    119         """   120         Collect the names used by the given 'path'. These are propagated to the   121         importer in advance of any dependency resolution.   122         """   123    124         names = self.names_used.get(path)   125         if not names:   126             return   127    128         in_function = self.function_locals.has_key(path)   129    130         for name in names:   131             if in_function and name in self.function_locals[path]:   132                 continue   133    134             key = "%s.%s" % (path, name)   135    136             # Find local definitions.   137    138             ref = self.get_resolved_object(key, True)   139             if ref:   140                 self.set_name_reference(key, ref)   141                 continue   142    143             # Find global.   144    145             ref = self.get_global(name)   146             if ref:   147                 self.set_name_reference(key, ref)   148                 continue   149    150             # Find presumed built-in definitions.   151    152             ref = self.get_builtin(name)   153             self.set_name_reference(key, ref)   154    155     def set_name_reference(self, path, ref):   156    157         "Map the given name 'path' to 'ref'."   158    159         self.importer.all_name_references[path] = self.name_references[path] = ref   160    161     # Module structure traversal.   162    163     def process_structure_node(self, n):   164    165         "Process the individual node 'n'."   166    167         path = self.get_namespace_path()   168    169         # Module global detection.   170    171         if isinstance(n, compiler.ast.Global):   172             self.process_global_node(n)   173    174         # Module import declarations.   175    176         elif isinstance(n, compiler.ast.From):   177             self.process_from_node(n)   178    179         elif isinstance(n, compiler.ast.Import):   180             self.process_import_node(n)   181    182         # Nodes using operator module functions.   183    184         elif isinstance(n, compiler.ast.Operator):   185             return self.process_operator_node(n)   186    187         elif isinstance(n, compiler.ast.AugAssign):   188             self.process_augassign_node(n)   189    190         elif isinstance(n, compiler.ast.Compare):   191             return self.process_compare_node(n)   192    193         elif isinstance(n, compiler.ast.Slice):   194             return self.process_slice_node(n)   195    196         elif isinstance(n, compiler.ast.Sliceobj):   197             return self.process_sliceobj_node(n)   198    199         elif isinstance(n, compiler.ast.Subscript):   200             return self.process_subscript_node(n)   201    202         # Namespaces within modules.   203    204         elif isinstance(n, compiler.ast.Class):   205             self.process_class_node(n)   206    207         elif isinstance(n, compiler.ast.Function):   208             self.process_function_node(n, n.name)   209    210         elif isinstance(n, compiler.ast.Lambda):   211             return self.process_lambda_node(n)   212    213         # Assignments.   214    215         elif isinstance(n, compiler.ast.Assign):   216    217             # Handle each assignment node.   218    219             for node in n.nodes:   220                 self.process_assignment_node(node, n.expr)   221    222         # Assignments within non-Assign nodes.   223    224         elif isinstance(n, compiler.ast.AssName):   225             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   226    227         elif isinstance(n, compiler.ast.AssAttr):   228             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   229    230         # Accesses.   231    232         elif isinstance(n, compiler.ast.Getattr):   233             return self.process_attribute_access(n)   234    235         # Name recording for later testing.   236    237         elif isinstance(n, compiler.ast.Name):   238             return self.process_name_node(n)   239    240         # Conditional statement tracking.   241    242         elif isinstance(n, compiler.ast.For):   243             self.process_for_node(n)   244    245         elif isinstance(n, compiler.ast.While):   246             self.process_while_node(n)   247    248         elif isinstance(n, compiler.ast.If):   249             self.process_if_node(n)   250    251         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   252             return self.process_logical_node(n)   253    254         # Exception control-flow tracking.   255    256         elif isinstance(n, compiler.ast.TryExcept):   257             self.process_try_node(n)   258    259         elif isinstance(n, compiler.ast.TryFinally):   260             self.process_try_finally_node(n)   261    262         # Control-flow modification statements.   263    264         elif isinstance(n, compiler.ast.Break):   265             self.trackers[-1].suspend_broken_branch()   266    267         elif isinstance(n, compiler.ast.Continue):   268             self.trackers[-1].suspend_continuing_branch()   269    270         elif isinstance(n, compiler.ast.Raise):   271             self.process_structure(n)   272             self.trackers[-1].abandon_branch()   273    274         elif isinstance(n, compiler.ast.Return):   275             self.record_return_value(self.process_structure_node(n.value))   276             self.trackers[-1].abandon_returning_branch()   277    278         # Print statements.   279    280         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   281             self.process_print_node(n)   282    283         # Invocations.   284    285         elif isinstance(n, compiler.ast.CallFunc):   286             return self.process_invocation_node(n)   287    288         # Constant usage.   289    290         elif isinstance(n, compiler.ast.Const):   291             return self.get_literal_instance(n)   292    293         elif isinstance(n, compiler.ast.Dict):   294             return self.get_literal_instance(n, "dict")   295    296         elif isinstance(n, compiler.ast.List):   297             return self.get_literal_instance(n, "list")   298    299         elif isinstance(n, compiler.ast.Tuple):   300             return self.get_literal_instance(n, "tuple")   301    302         # All other nodes are processed depth-first.   303    304         else:   305             self.process_structure(n)   306    307         # By default, no expression details are returned.   308    309         return None   310    311     # Specific node handling.   312    313     def process_assignment_node(self, n, expr):   314    315         "Process the individual node 'n' to be assigned the contents of 'expr'."   316    317         # Names and attributes are assigned the entire expression.   318    319         if isinstance(n, compiler.ast.AssName):   320             if n.name == "self":   321                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   322    323             name_ref = expr and self.process_structure_node(expr)   324    325             # Name assignments populate either function namespaces or the   326             # general namespace hierarchy.   327    328             self.assign_general_local(n.name, name_ref)   329    330             # Record usage of the name.   331    332             self.record_name(n.name)   333    334         elif isinstance(n, compiler.ast.AssAttr):   335             if expr:   336                 expr = self.process_structure_node(expr)   337    338             in_assignment = self.in_assignment   339             self.in_assignment = True   340             self.process_attribute_access(n)   341             self.in_assignment = in_assignment   342    343         # Lists and tuples are matched against the expression and their   344         # items assigned to expression items.   345    346         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   347             self.process_assignment_node_items(n, expr)   348    349         # Slices and subscripts are permitted within assignment nodes.   350    351         elif isinstance(n, compiler.ast.Slice):   352             self.process_slice_node(n, expr)   353    354         elif isinstance(n, compiler.ast.Subscript):   355             self.process_subscript_node(n, expr)   356    357     def process_attribute_access(self, n):   358    359         "Process the given attribute access node 'n'."   360    361         # Obtain any completed chain and return the reference to it.   362    363         name_ref = self.process_attribute_chain(n)   364    365         if self.have_access_expression(n):   366             return name_ref   367    368         # Where the start of the chain of attributes has been reached, determine   369         # the complete access.   370    371         # Given a non-access node, this chain can be handled in its entirety,   372         # either being name-based and thus an access rooted on a name, or being   373         # based on some other node and thus an anonymous access of some kind.   374    375         path = self.get_namespace_path()   376    377         # Start with the the full attribute chain.   378    379         remaining = self.attrs   380         attrnames = ".".join(remaining)   381    382         # If the accessor cannot be identified, or where attributes   383         # remain in an attribute chain, record the anonymous accesses.   384    385         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   386    387             init_item(self.attr_accesses, path, set)   388             self.attr_accesses[path].add(attrnames)   389    390             self.record_access_details(None, attrnames, self.in_assignment,   391                 self.in_invocation)   392             del self.attrs[0]   393             return   394    395         # Name-based accesses will handle the first attribute in a   396         # chain.   397    398         else:   399             attrname = remaining[0]   400    401             # Attribute assignments are used to identify instance attributes.   402    403             if isinstance(n, compiler.ast.AssAttr) and \   404                 self.in_class and self.in_function and n.expr.name == "self":   405    406                 self.set_instance_attr(attrname)   407    408             # Record attribute usage using any name local to this namespace,   409             # if assigned in the namespace, or using an external name   410             # (presently just globals within classes).   411    412             name = self.get_name_for_tracking(name_ref.name, name_ref)   413             tracker = self.trackers[-1]   414    415             immediate_access = len(self.attrs) == 1   416             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   417    418             # Record global-based chains for subsequent resolution.   419    420             if name_ref.is_global_name():   421                 self.record_global_access_details(name, attrnames)   422    423             # Make sure the name is being tracked: global names will not   424             # already be initialised in a branch and must be added   425             # explicitly.   426    427             if not tracker.have_name(name):   428                 tracker.assign_names([name])   429                 if self.in_function:   430                     self.scope_globals[path].add(name)   431    432             # Record attribute usage in the tracker, and record the branch   433             # information for the access.   434    435             branches = tracker.use_attribute(name, attrname,   436                 self.in_invocation is not None, assignment)   437    438             if not branches:   439                 raise InspectError("Name %s is accessed using %s before an assignment." % (   440                     name, attrname), path, n)   441    442             self.record_branches_for_access(branches, name, attrnames)   443             access_number = self.record_access_details(name, attrnames,   444                 self.in_assignment, self.in_invocation)   445    446             del self.attrs[0]   447             return AccessRef(name, attrnames, access_number)   448    449     def process_class_node(self, n):   450    451         "Process the given class node 'n'."   452    453         path = self.get_namespace_path()   454    455         # To avoid notions of class "versions" where the same definition   456         # might be parameterised with different state and be referenced   457         # elsewhere (as base classes, for example), classes in functions or   458         # conditions are forbidden.   459    460         if self.in_function or self.in_conditional:   461             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   462                 path, n.name)   463             return   464    465         # Resolve base classes.   466    467         bases = []   468    469         for base in n.bases:   470             base_class = self.get_class(base)   471    472             if not base_class:   473                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   474                     path, n.name, base)   475                 return   476             else:   477                 bases.append(base_class)   478    479         # Detect conflicting definitions. Such definitions cause conflicts in   480         # the storage of namespace-related information.   481    482         class_name = self.get_object_path(n.name)   483         ref = self.get_object(class_name, defer=False)   484    485         if ref and ref.static():   486             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   487    488         # Record bases for the class and retain the class name.   489         # Note that the function class does not inherit from the object class.   490    491         if not bases and class_name != "__builtins__.core.object" and \   492                          class_name != "__builtins__.core.function":   493    494             ref = self.get_object("__builtins__.object")   495             bases.append(ref)   496    497         self.importer.classes[class_name] = self.classes[class_name] = bases   498         self.importer.subclasses[class_name] = set()   499         self.scope_globals[class_name] = set()   500    501         # Set the definition before entering the namespace rather than   502         # afterwards because methods may reference it. In normal Python,   503         # a class is not accessible until the definition is complete, but   504         # methods can generally reference it since upon being called the   505         # class will already exist.   506    507         self.set_definition(n.name, "<class>")   508    509         in_class = self.in_class   510         self.in_class = class_name   511         self.set_instance_attr("__class__", Reference("<class>", class_name))   512         self.enter_namespace(n.name)   513    514         # Do not provide the special instantiator attributes on the function   515         # class. Function instances provide these attributes.   516    517         if class_name != "__builtins__.core.function":   518    519             self.set_name("__fn__") # special instantiator attribute   520             self.set_name("__args__") # special instantiator attribute   521    522         # Provide leafname, parent and context attributes.   523    524         parent, leafname = class_name.rsplit(".", 1)   525         self.set_name("__name__", self.get_constant("string", leafname).reference())   526    527         if class_name != "__builtins__.core.function":   528             self.set_name("__parent__")   529    530         self.process_structure_node(n.code)   531         self.exit_namespace()   532         self.in_class = in_class   533    534     def process_from_node(self, n):   535    536         "Process the given node 'n', importing from another module."   537    538         path = self.get_namespace_path()   539    540         module_name, names = self.get_module_name(n)   541         if module_name == self.name:   542             raise InspectError("Cannot import from the current module.", path, n)   543    544         self.queue_module(module_name)   545    546         # Attempt to obtain the referenced objects.   547    548         for name, alias in n.names:   549             if name == "*":   550                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   551    552             # Explicit names.   553    554             ref = self.import_name_from_module(name, module_name)   555             value = ResolvedNameRef(alias or name, ref)   556             self.set_general_local(alias or name, value)   557    558     def process_function_node(self, n, name):   559    560         """   561         Process the given function or lambda node 'n' with the given 'name'.   562         """   563    564         is_lambda = isinstance(n, compiler.ast.Lambda)   565    566         # Where a function is declared conditionally, use a separate name for   567         # the definition, and assign the definition to the stated name.   568    569         if (self.in_conditional or self.in_function) and not is_lambda:   570             original_name = name   571             name = self.get_lambda_name()   572         else:   573             original_name = None   574    575         # Detect conflicting definitions. Such definitions cause conflicts in   576         # the storage of namespace-related information.   577    578         function_name = self.get_object_path(name)   579         ref = self.get_object(function_name, defer=False)   580    581         if ref and ref.static():   582             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   583    584         # Initialise argument and local records.   585    586         argnames = get_argnames(n.argnames)   587         is_method = self.in_class and not self.in_function   588    589         # Remove explicit "self" from method parameters.   590    591         if is_method and argnames and argnames[0] == "self":   592             del argnames[0]   593    594         # Convert .name entries in the parameters, provided this is a method.   595    596         l = []   597         attr_initialisers = []   598    599         for argname in argnames:   600             if argname[0] == ".":   601                 if not is_method:   602                     raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n)   603    604                 argname = argname[1:]   605                 attr_initialisers.append(argname)   606    607             if argname in l:   608                 raise InspectError("Argument name %s is repeated." % argname, function_name, n)   609    610             l.append(argname)   611    612         argnames = l   613    614         # Copy and propagate the parameters.   615    616         self.importer.function_parameters[function_name] = \   617             self.function_parameters[function_name] = argnames[:]   618    619         self.importer.function_attr_initialisers[function_name] = \   620             self.function_attr_initialisers[function_name] = attr_initialisers   621    622         # Define all arguments/parameters in the local namespace.   623    624         locals = \   625             self.importer.function_locals[function_name] = \   626             self.function_locals[function_name] = {}   627    628         # Insert "self" into method locals.   629    630         if is_method:   631             argnames.insert(0, "self")   632    633         # Define "self" in terms of the class if in a method.   634         # This does not diminish the need for type-narrowing in the deducer.   635    636         if argnames:   637             if self.in_class and not self.in_function and argnames[0] == "self":   638                 locals[argnames[0]] = Reference("<instance>", self.in_class)   639             else:   640                 locals[argnames[0]] = Reference("<var>")   641    642         for argname in argnames[1:]:   643             locals[argname] = Reference("<var>")   644    645         globals = self.scope_globals[function_name] = set()   646    647         # Process the defaults.   648    649         defaults = self.importer.function_defaults[function_name] = \   650                    self.function_defaults[function_name] = []   651    652         for argname, default in compiler.ast.get_defaults(n):   653             if default:   654    655                 # Obtain any reference for the default.   656    657                 name_ref = self.process_structure_node(default)   658                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   659    660         # Reset conditional tracking to focus on the function contents.   661    662         in_conditional = self.in_conditional   663         self.in_conditional = False   664    665         in_function = self.in_function   666         self.in_function = function_name   667    668         self.enter_namespace(name)   669    670         # Define a leafname attribute value for the function instance.   671    672         ref = self.get_builtin_class("string")   673         self.reserve_constant(function_name, name, ref.get_origin())   674    675         # Track attribute usage within the namespace.   676    677         path = self.get_namespace_path()   678         self.start_tracking(locals)   679    680         # Establish attributes for .name entries, provided this is a method.   681    682         for argname in attr_initialisers:   683             self.process_assignment_node(   684                     compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"),   685                     compiler.ast.Name(argname))   686    687         self.process_structure_node(n.code)   688         returns_value = self.stop_tracking()   689    690         # Record any null result.   691    692         is_initialiser = is_method and name == "__init__"   693    694         if not returns_value and not is_initialiser:   695             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   696    697         # Exit to the parent.   698    699         self.exit_namespace()   700    701         # Update flags.   702    703         self.in_function = in_function   704         self.in_conditional = in_conditional   705    706         # Define the function using the appropriate name.   707    708         self.set_definition(name, "<function>")   709    710         # Where a function is set conditionally, assign the name.   711    712         if original_name:   713             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   714    715     def process_global_node(self, n):   716    717         """   718         Process the given "global" node 'n'.   719         """   720    721         path = self.get_namespace_path()   722    723         if path != self.name:   724             self.scope_globals[path].update(n.names)   725    726     def process_if_node(self, n):   727    728         """   729         Process the given "if" node 'n'.   730         """   731    732         tracker = self.trackers[-1]   733         tracker.new_branchpoint()   734    735         for test, body in n.tests:   736             self.process_structure_node(test)   737    738             tracker.new_branch()   739    740             in_conditional = self.in_conditional   741             self.in_conditional = True   742             self.process_structure_node(body)   743             self.in_conditional = in_conditional   744    745             tracker.shelve_branch()   746    747         # Maintain a branch for the else clause.   748    749         tracker.new_branch()   750         if n.else_:   751             self.process_structure_node(n.else_)   752         tracker.shelve_branch()   753    754         tracker.merge_branches()   755    756     def process_import_node(self, n):   757    758         "Process the given import node 'n'."   759    760         path = self.get_namespace_path()   761    762         # Load the mentioned module.   763    764         for name, alias in n.names:   765             if name == self.name:   766                 raise InspectError("Cannot import the current module.", path, n)   767    768             self.set_module(alias or name.split(".")[-1], name)   769             self.queue_module(name, True)   770    771     def process_invocation_node(self, n):   772    773         "Process the given invocation node 'n'."   774    775         path = self.get_namespace_path()   776    777         in_invocation = self.in_invocation   778         self.in_invocation = None   779    780         # Process the arguments.   781    782         keywords = set()   783    784         for arg in n.args:   785             self.process_structure_node(arg)   786             if isinstance(arg, compiler.ast.Keyword):   787                 keywords.add(arg.name)   788    789         keywords = list(keywords)   790         keywords.sort()   791    792         # Communicate to the invocation target expression that it forms the   793         # target of an invocation, potentially affecting attribute accesses.   794    795         self.in_invocation = len(n.args), keywords   796    797         # Process the expression, obtaining any identified reference.   798    799         name_ref = self.process_structure_node(n.node)   800         self.in_invocation = in_invocation   801    802         # Detect class invocations.   803    804         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   805             return InstanceRef(name_ref.reference().instance_of())   806    807         elif isinstance(name_ref, (NameRef, AccessRef)):   808             return InvocationRef(name_ref)   809    810         # Provide a general reference to indicate that something is produced   811         # by the invocation, useful for retaining assignment expression   812         # details.   813    814         return VariableRef()   815    816     def process_lambda_node(self, n):   817    818         "Process the given lambda node 'n'."   819    820         name = self.get_lambda_name()   821         self.process_function_node(n, name)   822    823         origin = self.get_object_path(name)   824    825         if self.function_defaults.get(origin):   826             return None   827         else:   828             return ResolvedNameRef(name, Reference("<function>", origin))   829    830     def process_logical_node(self, n):   831    832         "Process the given operator node 'n'."   833    834         return self.process_operator_chain(n.nodes)   835    836     def process_name_node(self, n):   837    838         "Process the given name node 'n'."   839    840         path = self.get_namespace_path()   841    842         # Find predefined constant names before anything else.   843    844         if n.name in predefined_constants:   845             ref = self.get_builtin(n.name)   846             value = ResolvedNameRef(n.name, ref)   847             return value   848    849         # Special names that have already been identified.   850    851         if n.name.startswith("$"):   852             value = self.get_special(n.name)   853             if value:   854                 return value   855    856         # Special case for operator functions introduced through code   857         # transformations.   858    859         if n.name.startswith("$op"):   860    861             # Obtain the location of the actual function defined in the operator   862             # package.   863    864             op = n.name[len("$op"):]   865    866             # Attempt to get a reference.   867    868             ref = self.import_name_from_module(op, "operator")   869    870             # Record the imported name and provide the resolved name reference.   871    872             value = ResolvedNameRef(n.name, ref)   873             self.set_special(n.name, value)   874             return value   875    876         # Special case for sequence length testing.   877    878         elif n.name.startswith("$seq"):   879             op = n.name[len("$seq"):]   880             ref = self.import_name_from_module(op, "__builtins__.sequence")   881             value = ResolvedNameRef(n.name, ref)   882             self.set_special(n.name, value)   883             return value   884    885         # Special case for print operations.   886    887         elif n.name.startswith("$print"):   888    889             # Attempt to get a reference.   890    891             ref = self.get_builtin("print_")   892    893             # Record the imported name and provide the resolved name reference.   894    895             value = ResolvedNameRef(n.name, ref)   896             self.set_special(n.name, value)   897             return value   898    899         # Test for self usage, which is only allowed in methods.   900    901         if n.name == "self" and not (self.in_function and self.in_class):   902             raise InspectError("Use of self is only allowed in methods.", path, n)   903    904         # Record usage of the name.   905    906         self.record_name(n.name)   907    908         # Search for unknown names in non-function scopes immediately.   909         # Temporary names should not be re-used between scopes.   910         # External names in functions are resolved later.   911    912         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   913    914         if ref:   915             self.record_name_access(n.name, True)   916             return ResolvedNameRef(n.name, ref, is_global=True)   917    918         # Explicitly-declared global names.   919    920         elif self.in_function and n.name in self.scope_globals[path]:   921             self.record_name_access(n.name, True)   922             return NameRef(n.name, is_global=True)   923    924         # Examine other names.   925    926         else:   927    928             # Check local names.   929    930             access_number = self.record_name_access(n.name)   931    932             # Local name.   933    934             if access_number is not None:   935                 return LocalNameRef(n.name, access_number)   936    937             # Possible global or built-in name.   938    939             else:   940                 self.record_name_access(n.name, True)   941                 return NameRef(n.name, is_global=True)   942    943     def record_name_access(self, name, is_global=False):   944    945         """   946         Record an access involving 'name' if the name is being tracked, using   947         'is_global' to indicate whether the name is global.   948         """   949    950         name = self.get_name_for_tracking(name, is_global=is_global)   951         branches = self.trackers[-1].tracking_name(name)   952         if branches:   953             self.record_branches_for_access(branches, name, None)   954             return self.record_access_details(name, None, self.in_assignment,   955                                               self.in_invocation)   956         return None   957    958     def process_operator_chain(self, nodes):   959    960         """   961         Process the given chain of 'nodes', processing each node or item.   962         Each node starts a new conditional region, effectively making a deeply-   963         nested collection of if-like statements.   964         """   965    966         results = []   967    968         tracker = self.trackers[-1]   969    970         for item in nodes:   971             tracker.new_branchpoint()   972             tracker.new_branch()   973             result = self.process_structure_node(item)   974             if result:   975                 results.append(result)   976    977         for item in nodes[:-1]:   978             tracker.shelve_branch()   979             tracker.new_branch()   980             tracker.shelve_branch()   981             tracker.merge_branches()   982    983         tracker.shelve_branch()   984         tracker.merge_branches()   985    986         return MultipleRef(results)   987    988     def process_try_node(self, n):   989    990         """   991         Process the given "try...except" node 'n'.   992         """   993    994         self.record_exception_handler()   995    996         tracker = self.trackers[-1]   997         tracker.new_branchpoint()   998    999         self.process_structure_node(n.body)  1000   1001         for name, var, handler in n.handlers:  1002             if name is not None:  1003                 self.process_structure_node(name)  1004   1005             # Any abandoned branches from the body can now be resumed in a new  1006             # branch.  1007   1008             tracker.resume_abandoned_branches()  1009   1010             # Establish the local for the handler.  1011   1012             if var is not None:  1013                 self.process_assignment_node(var, None)  1014             if handler is not None:  1015                 self.process_structure_node(handler)  1016   1017             tracker.shelve_branch()  1018   1019         # The else clause maintains the usage from the body but without the  1020         # abandoned branches since they would never lead to the else clause  1021         # being executed.  1022   1023         if n.else_:  1024             tracker.new_branch()  1025             self.process_structure_node(n.else_)  1026             tracker.shelve_branch()  1027   1028         # Without an else clause, a null branch propagates the successful  1029         # outcome.  1030   1031         else:  1032             tracker.new_branch()  1033             tracker.shelve_branch()  1034   1035         tracker.merge_branches()  1036   1037     def process_try_finally_node(self, n):  1038   1039         """  1040         Process the given "try...finally" node 'n'.  1041         """  1042   1043         self.record_exception_handler()  1044   1045         tracker = self.trackers[-1]  1046         self.process_structure_node(n.body)  1047   1048         # Any abandoned branches from the body can now be resumed.  1049   1050         branches = tracker.resume_all_abandoned_branches()  1051         self.process_structure_node(n.final)  1052   1053         # At the end of the finally clause, abandoned branches are discarded.  1054   1055         tracker.restore_active_branches(branches)  1056   1057     def process_while_node(self, n):  1058   1059         "Process the given while node 'n'."  1060   1061         tracker = self.trackers[-1]  1062         tracker.new_branchpoint(loop_node=True)  1063   1064         # Evaluate any test or iterator outside the loop.  1065   1066         self.process_structure_node(n.test)  1067   1068         # Propagate attribute usage to branches.  1069   1070         tracker.new_branch(loop_node=True)  1071   1072         # Enter the loop.  1073   1074         in_conditional = self.in_conditional  1075         self.in_conditional = True  1076         self.process_structure_node(n.body)  1077         self.in_conditional = in_conditional  1078   1079         # Continuing branches are resumed before any test.  1080   1081         tracker.resume_continuing_branches()  1082   1083         # Evaluate any continuation test within the body.  1084   1085         self.process_structure_node(n.test)  1086   1087         tracker.shelve_branch(loop_node=True)  1088   1089         # Support the non-looping condition.  1090   1091         tracker.new_branch()  1092         tracker.shelve_branch()  1093   1094         tracker.merge_branches()  1095   1096         # Evaluate any else clause outside branches.  1097   1098         if n.else_:  1099             self.process_structure_node(n.else_)  1100   1101         # Connect broken branches to the code after any loop.  1102   1103         tracker.resume_broken_branches()  1104   1105     # Branch tracking methods.  1106   1107     def start_tracking(self, names):  1108   1109         """  1110         Start tracking attribute usage for names in the current namespace,  1111         immediately registering the given 'names'.  1112         """  1113   1114         path = self.get_namespace_path()  1115         parent = self.trackers[-1]  1116         tracker = BranchTracker()  1117         self.trackers.append(tracker)  1118   1119         # Record the given names established as new branches.  1120   1121         tracker.assign_names(names)  1122   1123     def assign_name(self, name, name_ref):  1124   1125         "Assign to 'name' the given 'name_ref' in the current namespace."  1126   1127         name = self.get_name_for_tracking(name)  1128         self.trackers[-1].assign_names([name], [name_ref])  1129   1130     def stop_tracking(self):  1131   1132         """  1133         Stop tracking attribute usage, recording computed usage for the current  1134         namespace. Indicate whether a value is always returned from the  1135         namespace.  1136         """  1137   1138         path = self.get_namespace_path()  1139         tracker = self.trackers.pop()  1140         self.record_assignments_for_access(tracker)  1141   1142         self.attr_usage[path] = tracker.get_all_usage()  1143         self.name_initialisers[path] = tracker.get_all_values()  1144   1145         return tracker.returns_value()  1146   1147     def start_tracking_in_module(self):  1148   1149         "Start tracking attribute usage in the module."  1150   1151         tracker = BranchTracker()  1152         self.trackers.append(tracker)  1153   1154     def stop_tracking_in_module(self):  1155   1156         "Stop tracking attribute usage in the module."  1157   1158         tracker = self.trackers[0]  1159         self.record_assignments_for_access(tracker)  1160         self.attr_usage[self.name] = tracker.get_all_usage()  1161         self.name_initialisers[self.name] = tracker.get_all_values()  1162   1163     def record_assignments_for_access(self, tracker):  1164   1165         """  1166         For the current path, use the given 'tracker' to record assignment  1167         version information for attribute accesses.  1168         """  1169   1170         path = self.get_path_for_access()  1171   1172         if not self.attr_accessor_branches.has_key(path):  1173             return  1174   1175         init_item(self.attr_accessors, path, dict)  1176         attr_accessors = self.attr_accessors[path]  1177   1178         # Obtain the branches applying during each access.  1179   1180         for access, all_branches in self.attr_accessor_branches[path].items():  1181             name, attrnames = access  1182             init_item(attr_accessors, access, list)  1183   1184             # Obtain the assignments applying to each branch.  1185   1186             for branches in all_branches:  1187                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1188   1189                 # Detect missing name information.  1190   1191                 if None in positions:  1192                     globals = self.global_attr_accesses.get(path)  1193                     accesses = globals and globals.get(name)  1194                     if not accesses:  1195                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1196                             self.get_namespace_path(), name)  1197                     positions.remove(None)  1198   1199                 attr_accessors[access].append(positions)  1200   1201     def record_branches_for_access(self, branches, name, attrnames):  1202   1203         """  1204         Record the given 'branches' for an access involving the given 'name' and  1205         'attrnames'.  1206         """  1207   1208         access = name, attrnames  1209         path = self.get_path_for_access()  1210   1211         init_item(self.attr_accessor_branches, path, dict)  1212         attr_accessor_branches = self.attr_accessor_branches[path]  1213   1214         init_item(attr_accessor_branches, access, list)  1215         attr_accessor_branches[access].append(branches)  1216   1217     def record_access_details(self, name, attrnames, assignment, invocation):  1218   1219         """  1220         For the given 'name' and 'attrnames', record an access indicating  1221         whether an 'assignment' or an 'invocation' is occurring.  1222   1223         These details correspond to accesses otherwise recorded by the attribute  1224         accessor and attribute access dictionaries.  1225         """  1226   1227         access = name, attrnames  1228         path = self.get_path_for_access()  1229   1230         init_item(self.attr_access_modifiers, path, dict)  1231         init_item(self.attr_access_modifiers[path], access, list)  1232   1233         access_number = len(self.attr_access_modifiers[path][access])  1234         self.attr_access_modifiers[path][access].append((assignment, invocation))  1235         return access_number  1236   1237     def record_global_access_details(self, name, attrnames):  1238   1239         """  1240         Record details of a global access via the given 'name' involving the  1241         indicated 'attrnames'.  1242         """  1243   1244         path = self.get_namespace_path()  1245   1246         init_item(self.global_attr_accesses, path, dict)  1247         init_item(self.global_attr_accesses[path], name, set)  1248         self.global_attr_accesses[path][name].add(attrnames)  1249   1250     # Namespace modification.  1251   1252     def record_name(self, name):  1253   1254         "Record the use of 'name' in a namespace."  1255   1256         path = self.get_namespace_path()  1257         init_item(self.names_used, path, set)  1258         self.names_used[path].add(name)  1259   1260     def set_module(self, name, module_name):  1261   1262         """  1263         Set a module in the current namespace using the given 'name' associated  1264         with the corresponding 'module_name'.  1265         """  1266   1267         if name:  1268             self.set_general_local(name, Reference("<module>", module_name))  1269   1270     def set_definition(self, name, kind):  1271   1272         """  1273         Set the definition having the given 'name' and 'kind'.  1274   1275         Definitions are set in the static namespace hierarchy, but they can also  1276         be recorded for function locals.  1277         """  1278   1279         if self.is_global(name):  1280             print >>sys.stderr, "In %s, %s is defined as being global." % (  1281                 self.get_namespace_path(), name)  1282   1283         path = self.get_object_path(name)  1284         self.set_object(path, kind)  1285   1286         ref = self.get_object(path)  1287         if ref.get_kind() == "<var>":  1288             print >>sys.stderr, "In %s, %s is defined more than once." % (  1289                 self.get_namespace_path(), name)  1290   1291         if not self.is_global(name) and self.in_function:  1292             self.set_function_local(name, ref)  1293   1294     def set_function_local(self, name, ref=None):  1295   1296         "Set the local with the given 'name' and optional 'ref'."  1297   1298         path = self.get_namespace_path()  1299         locals = self.function_locals[path]  1300         used = self.names_used.get(path)  1301   1302         if not locals.has_key(name) and used and name in used:  1303             raise InspectError("Name %s assigned locally but used previously." % name, path)  1304   1305         multiple = not ref or locals.has_key(name) and locals[name] != ref  1306         locals[name] = multiple and Reference("<var>") or ref  1307   1308     def assign_general_local(self, name, name_ref):  1309   1310         """  1311         Set for 'name' the given 'name_ref', recording the name for attribute  1312         usage tracking.  1313         """  1314   1315         self.set_general_local(name, name_ref)  1316         self.assign_name(name, name_ref)  1317   1318     def set_general_local(self, name, value=None):  1319   1320         """  1321         Set the 'name' with optional 'value' in any kind of local namespace,  1322         where the 'value' should be a reference if specified.  1323         """  1324   1325         init_value = self.get_initialising_value(value)  1326   1327         # Module global names.  1328   1329         if self.is_global(name):  1330             path = self.get_global_path(name)  1331             self.set_object(path, init_value)  1332   1333         # Function local names.  1334   1335         elif self.in_function:  1336             self.set_function_local(name, init_value)  1337   1338         # Other namespaces (classes).  1339   1340         else:  1341             self.set_name(name, init_value)  1342   1343     def set_name(self, name, ref=None):  1344   1345         "Attach the 'name' with optional 'ref' to the current namespace."  1346   1347         self.set_object(self.get_object_path(name), ref)  1348   1349     def set_instance_attr(self, name, ref=None):  1350   1351         """  1352         Add an instance attribute of the given 'name' to the current class,  1353         using the optional 'ref'.  1354         """  1355   1356         self._set_instance_attr(self.in_class, name, ref)  1357   1358     def _set_instance_attr(self, path, name, ref=None):  1359   1360         init_item(self.instance_attrs, path, set)  1361         self.instance_attrs[path].add(name)  1362   1363         if ref:  1364             init_item(self.instance_attr_constants, path, dict)  1365             self.instance_attr_constants[path][name] = ref  1366   1367     def get_initialising_value(self, value):  1368   1369         "Return a suitable initialiser reference for 'value'."  1370   1371         if isinstance(value, Result):  1372             return value.reference()  1373         else:  1374             return value  1375   1376     # Static, program-relative naming.  1377   1378     def find_name(self, name):  1379   1380         """  1381         Return the qualified name for the given 'name' used in the current  1382         non-function namespace.  1383         """  1384   1385         path = self.get_namespace_path()  1386         ref = None  1387   1388         if not self.in_function and name not in predefined_constants:  1389             if self.in_class:  1390                 ref = self.get_object(self.get_object_path(name), False)  1391             if not ref:  1392                 ref = self.get_global_or_builtin(name)  1393   1394         return ref  1395   1396     def get_class(self, node):  1397   1398         """  1399         Use the given 'node' to obtain the identity of a class. Return a  1400         reference for the class. Unresolved dependencies are permitted and must  1401         be resolved later.  1402         """  1403   1404         ref = self._get_class(node)  1405         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1406   1407     def _get_class(self, node):  1408   1409         """  1410         Use the given 'node' to find a class definition. Return a reference to  1411         the class.  1412         """  1413   1414         if isinstance(node, compiler.ast.Getattr):  1415   1416             # Obtain the identity of the access target.  1417   1418             ref = self._get_class(node.expr)  1419   1420             # Where the target is a class or module, obtain the identity of the  1421             # attribute.  1422   1423             if ref.has_kind(["<function>", "<var>"]):  1424                 return None  1425             else:  1426                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1427                 return self.get_object(attrname)  1428   1429         # Names can be module-level or built-in.  1430   1431         elif isinstance(node, compiler.ast.Name):  1432   1433             # Record usage of the name and attempt to identify it.  1434   1435             self.record_name(node.name)  1436             return self.find_name(node.name)  1437         else:  1438             return None  1439   1440     def get_constant(self, name, value):  1441   1442         "Return a constant reference for the given type 'name' and 'value'."  1443   1444         ref = self.get_builtin_class(name)  1445         return self.get_constant_reference(ref, value)  1446   1447     def get_literal_instance(self, n, name=None):  1448   1449         """  1450         For node 'n', return a reference to an instance of 'name', or if 'name'  1451         is not specified, deduce the type from the value.  1452         """  1453   1454         # Handle stray None constants (Sliceobj seems to produce them).  1455   1456         if name == "NoneType":  1457             return self.process_name_node(compiler.ast.Name("None"))  1458   1459         # Obtain the details of the literal itself.  1460         # An alias to the type is generated for sequences.  1461   1462         if name in ("dict", "list", "tuple"):  1463             ref = self.get_builtin_class(name)  1464             self.set_special_literal(name, ref)  1465             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1466   1467         # Constant values are independently recorded.  1468   1469         else:  1470             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1471             ref = self.get_builtin_class(typename)  1472             return self.get_constant_reference(ref, value, encoding)  1473   1474     # Special names.  1475   1476     def get_special(self, name):  1477   1478         "Return any stored value for the given special 'name'."  1479   1480         value = self.special.get(name)  1481         if value:  1482             ref, paths = value  1483         else:  1484             ref = None  1485         return ref  1486   1487     def set_special(self, name, value):  1488   1489         """  1490         Set a special 'name' that merely tracks the use of an implicit object  1491         'value'.  1492         """  1493   1494         if not self.special.has_key(name):  1495             paths = set()  1496             self.special[name] = value, paths  1497         else:  1498             _ref, paths = self.special[name]  1499   1500         paths.add(self.get_namespace_path())  1501   1502     def set_special_literal(self, name, ref):  1503   1504         """  1505         Set a special name for the literal type 'name' having type 'ref'. Such  1506         special names provide a way of referring to literal object types.  1507         """  1508   1509         literal_name = "$L%s" % name  1510         value = ResolvedNameRef(literal_name, ref)  1511         self.set_special(literal_name, value)  1512   1513     # Exceptions.  1514   1515     def record_exception_handler(self):  1516   1517         "Record the current namespace as employing an exception handler."  1518   1519         self.exception_namespaces.add(self.get_namespace_path())  1520   1521     # Return values.  1522   1523     def record_return_value(self, expr):  1524   1525         "Record the given return 'expr'."  1526   1527         path = self.get_namespace_path()  1528         l = init_item(self.return_values, path, list)  1529         l.append(expr)  1530         if not self.importer.all_return_values.has_key(path):  1531             self.importer.all_return_values[path] = l  1532   1533 # vim: tabstop=4 expandtab shiftwidth=4