Lichen

inspector.py

792:d70932955645
2017-03-31 Paul Boddie Fixed non-recognition of deferred references in non-module, non-function scopes.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from branching import BranchTracker    24 from common import CommonModule, get_argnames, init_item, predefined_constants    25 from modules import BasicModule, CacheWritingModule, InspectionNaming    26 from errors import InspectError    27 from referencing import Reference    28 from resolving import NameResolving    29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    30                     LocalNameRef, NameRef, ResolvedNameRef, VariableRef    31 import compiler    32 import sys    33     34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    35     36     "A module inspector."    37     38     def __init__(self, name, importer):    39     40         "Initialise the module with basic details."    41     42         BasicModule.__init__(self, name, importer)    43     44         self.in_class = False    45         self.in_conditional = False    46     47         # Accesses to global attributes.    48     49         self.global_attr_accesses = {}    50     51         # Usage tracking.    52     53         self.trackers = []    54         self.attr_accessor_branches = {}    55     56     def __repr__(self):    57         return "InspectedModule(%r, %r)" % (self.name, self.importer)    58     59     # Principal methods.    60     61     def parse(self, filename):    62     63         "Parse the file having the given 'filename'."    64     65         self.parse_file(filename)    66     67         # Inspect the module.    68     69         self.start_tracking_in_module()    70     71         # Detect and record imports and globals declared in the module.    72     73         self.process_structure(self.astnode)    74     75         # Set the class of the module after the definition has occurred.    76     77         ref = self.get_builtin("module")    78         self.set_name("__class__", ref)    79         self.set_name("__name__", self.get_constant("string", self.name).reference())    80         self.set_name("__file__", self.get_constant("string", filename).reference())    81     82         # Reserve a constant for the encoding.    83     84         if self.encoding:    85             self.get_constant("string", self.encoding)    86     87         # Get module-level attribute usage details.    88     89         self.stop_tracking_in_module()    90     91         # Collect external name references.    92     93         self.collect_names()    94     95     def complete(self):    96     97         "Complete the module inspection."    98     99         # Resolve names not definitively mapped to objects.   100    101         self.resolve()   102    103         # Propagate to the importer information needed in subsequent activities.   104    105         self.propagate()   106    107     # Accessory methods.   108    109     def collect_names(self):   110    111         "Collect the names used by each scope."   112    113         for path in self.names_used.keys():   114             self.collect_names_for_path(path)   115    116     def collect_names_for_path(self, path):   117    118         """   119         Collect the names used by the given 'path'. These are propagated to the   120         importer in advance of any dependency resolution.   121         """   122    123         names = self.names_used.get(path)   124         if not names:   125             return   126    127         in_function = self.function_locals.has_key(path)   128    129         for name in names:   130             if in_function and name in self.function_locals[path]:   131                 continue   132    133             key = "%s.%s" % (path, name)   134    135             # Find local definitions.   136    137             ref = self.get_resolved_object(key, True)   138             if ref:   139                 self.set_name_reference(key, ref)   140                 continue   141    142             # Find global.   143    144             ref = self.get_global(name)   145             if ref:   146                 self.set_name_reference(key, ref)   147                 continue   148    149             # Find presumed built-in definitions.   150    151             ref = self.get_builtin(name)   152             self.set_name_reference(key, ref)   153    154     def set_name_reference(self, path, ref):   155    156         "Map the given name 'path' to 'ref'."   157    158         self.importer.all_name_references[path] = self.name_references[path] = ref   159    160     # Module structure traversal.   161    162     def process_structure_node(self, n):   163    164         "Process the individual node 'n'."   165    166         path = self.get_namespace_path()   167    168         # Module global detection.   169    170         if isinstance(n, compiler.ast.Global):   171             self.process_global_node(n)   172    173         # Module import declarations.   174    175         elif isinstance(n, compiler.ast.From):   176             self.process_from_node(n)   177    178         elif isinstance(n, compiler.ast.Import):   179             self.process_import_node(n)   180    181         # Nodes using operator module functions.   182    183         elif isinstance(n, compiler.ast.Operator):   184             return self.process_operator_node(n)   185    186         elif isinstance(n, compiler.ast.AugAssign):   187             self.process_augassign_node(n)   188    189         elif isinstance(n, compiler.ast.Compare):   190             return self.process_compare_node(n)   191    192         elif isinstance(n, compiler.ast.Slice):   193             return self.process_slice_node(n)   194    195         elif isinstance(n, compiler.ast.Sliceobj):   196             return self.process_sliceobj_node(n)   197    198         elif isinstance(n, compiler.ast.Subscript):   199             return self.process_subscript_node(n)   200    201         # Namespaces within modules.   202    203         elif isinstance(n, compiler.ast.Class):   204             self.process_class_node(n)   205    206         elif isinstance(n, compiler.ast.Function):   207             self.process_function_node(n, n.name)   208    209         elif isinstance(n, compiler.ast.Lambda):   210             return self.process_lambda_node(n)   211    212         # Assignments.   213    214         elif isinstance(n, compiler.ast.Assign):   215    216             # Handle each assignment node.   217    218             for node in n.nodes:   219                 self.process_assignment_node(node, n.expr)   220    221         # Assignments within non-Assign nodes.   222    223         elif isinstance(n, compiler.ast.AssName):   224             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   225    226         elif isinstance(n, compiler.ast.AssAttr):   227             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   228    229         # Accesses.   230    231         elif isinstance(n, compiler.ast.Getattr):   232             return self.process_attribute_access(n)   233    234         # Name recording for later testing.   235    236         elif isinstance(n, compiler.ast.Name):   237             return self.process_name_node(n)   238    239         # Conditional statement tracking.   240    241         elif isinstance(n, compiler.ast.For):   242             self.process_for_node(n)   243    244         elif isinstance(n, compiler.ast.While):   245             self.process_while_node(n)   246    247         elif isinstance(n, compiler.ast.If):   248             self.process_if_node(n)   249    250         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   251             return self.process_logical_node(n)   252    253         # Exception control-flow tracking.   254    255         elif isinstance(n, compiler.ast.TryExcept):   256             self.process_try_node(n)   257    258         elif isinstance(n, compiler.ast.TryFinally):   259             self.process_try_finally_node(n)   260    261         # Control-flow modification statements.   262    263         elif isinstance(n, compiler.ast.Break):   264             self.trackers[-1].suspend_broken_branch()   265    266         elif isinstance(n, compiler.ast.Continue):   267             self.trackers[-1].suspend_continuing_branch()   268    269         elif isinstance(n, compiler.ast.Raise):   270             self.process_structure(n)   271             self.trackers[-1].abandon_branch()   272    273         elif isinstance(n, compiler.ast.Return):   274             self.record_return_value(self.process_structure_node(n.value))   275             self.trackers[-1].abandon_returning_branch()   276    277         # Print statements.   278    279         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   280             self.process_print_node(n)   281    282         # Invocations.   283    284         elif isinstance(n, compiler.ast.CallFunc):   285             return self.process_invocation_node(n)   286    287         # Constant usage.   288    289         elif isinstance(n, compiler.ast.Const):   290             return self.get_literal_instance(n)   291    292         elif isinstance(n, compiler.ast.Dict):   293             return self.get_literal_instance(n, "dict")   294    295         elif isinstance(n, compiler.ast.List):   296             return self.get_literal_instance(n, "list")   297    298         elif isinstance(n, compiler.ast.Tuple):   299             return self.get_literal_instance(n, "tuple")   300    301         # All other nodes are processed depth-first.   302    303         else:   304             self.process_structure(n)   305    306         # By default, no expression details are returned.   307    308         return None   309    310     # Specific node handling.   311    312     def process_assignment_node(self, n, expr):   313    314         "Process the individual node 'n' to be assigned the contents of 'expr'."   315    316         # Names and attributes are assigned the entire expression.   317    318         if isinstance(n, compiler.ast.AssName):   319             if n.name == "self":   320                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   321    322             name_ref = expr and self.process_structure_node(expr)   323    324             # Name assignments populate either function namespaces or the   325             # general namespace hierarchy.   326    327             self.assign_general_local(n.name, name_ref)   328    329             # Record usage of the name.   330    331             self.record_name(n.name)   332    333         elif isinstance(n, compiler.ast.AssAttr):   334             if expr:   335                 expr = self.process_structure_node(expr)   336    337             in_assignment = self.in_assignment   338             self.in_assignment = True   339             self.process_attribute_access(n)   340             self.in_assignment = in_assignment   341    342         # Lists and tuples are matched against the expression and their   343         # items assigned to expression items.   344    345         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   346             self.process_assignment_node_items(n, expr)   347    348         # Slices and subscripts are permitted within assignment nodes.   349    350         elif isinstance(n, compiler.ast.Slice):   351             self.process_slice_node(n, expr)   352    353         elif isinstance(n, compiler.ast.Subscript):   354             self.process_subscript_node(n, expr)   355    356     def process_attribute_access(self, n):   357    358         "Process the given attribute access node 'n'."   359    360         # Obtain any completed chain and return the reference to it.   361    362         name_ref = self.process_attribute_chain(n)   363    364         if self.have_access_expression(n):   365             return name_ref   366    367         # Where the start of the chain of attributes has been reached, determine   368         # the complete access.   369    370         # Given a non-access node, this chain can be handled in its entirety,   371         # either being name-based and thus an access rooted on a name, or being   372         # based on some other node and thus an anonymous access of some kind.   373    374         path = self.get_namespace_path()   375    376         # Start with the the full attribute chain.   377    378         remaining = self.attrs   379         attrnames = ".".join(remaining)   380    381         # If the accessor cannot be identified, or where attributes   382         # remain in an attribute chain, record the anonymous accesses.   383    384         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   385    386             init_item(self.attr_accesses, path, set)   387             self.attr_accesses[path].add(attrnames)   388    389             self.record_access_details(None, attrnames, self.in_assignment,   390                 self.in_invocation)   391             del self.attrs[0]   392             return   393    394         # Name-based accesses will handle the first attribute in a   395         # chain.   396    397         else:   398             attrname = remaining[0]   399    400             # Attribute assignments are used to identify instance attributes.   401    402             if isinstance(n, compiler.ast.AssAttr) and \   403                 self.in_class and self.in_function and n.expr.name == "self":   404    405                 self.set_instance_attr(attrname)   406    407             # Record attribute usage using any name local to this namespace,   408             # if assigned in the namespace, or using an external name   409             # (presently just globals within classes).   410    411             name = self.get_name_for_tracking(name_ref.name, name_ref)   412             tracker = self.trackers[-1]   413    414             immediate_access = len(self.attrs) == 1   415             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   416    417             # Record global-based chains for subsequent resolution.   418    419             if name_ref.is_global_name():   420                 self.record_global_access_details(name, attrnames)   421    422             # Make sure the name is being tracked: global names will not   423             # already be initialised in a branch and must be added   424             # explicitly.   425    426             if not tracker.have_name(name):   427                 tracker.assign_names([name])   428                 if self.in_function:   429                     self.scope_globals[path].add(name)   430    431             # Record attribute usage in the tracker, and record the branch   432             # information for the access.   433    434             branches = tracker.use_attribute(name, attrname,   435                 self.in_invocation is not None, assignment)   436    437             if not branches:   438                 raise InspectError("Name %s is accessed using %s before an assignment." % (   439                     name, attrname), path, n)   440    441             self.record_branches_for_access(branches, name, attrnames)   442             access_number = self.record_access_details(name, attrnames,   443                 self.in_assignment, self.in_invocation)   444    445             del self.attrs[0]   446             return AccessRef(name, attrnames, access_number)   447    448     def process_class_node(self, n):   449    450         "Process the given class node 'n'."   451    452         path = self.get_namespace_path()   453    454         # To avoid notions of class "versions" where the same definition   455         # might be parameterised with different state and be referenced   456         # elsewhere (as base classes, for example), classes in functions or   457         # conditions are forbidden.   458    459         if self.in_function or self.in_conditional:   460             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   461                 path, n.name)   462             return   463    464         # Resolve base classes.   465    466         bases = []   467    468         for base in n.bases:   469             base_class = self.get_class(base)   470    471             if not base_class:   472                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   473                     path, n.name, base)   474                 return   475             else:   476                 bases.append(base_class)   477    478         # Detect conflicting definitions. Such definitions cause conflicts in   479         # the storage of namespace-related information.   480    481         class_name = self.get_object_path(n.name)   482         ref = self.get_object(class_name, defer=False)   483    484         if ref and ref.static():   485             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   486    487         # Record bases for the class and retain the class name.   488         # Note that the function class does not inherit from the object class.   489    490         if not bases and class_name != "__builtins__.core.object" and \   491                          class_name != "__builtins__.core.function":   492    493             ref = self.get_object("__builtins__.object")   494             bases.append(ref)   495    496         self.importer.classes[class_name] = self.classes[class_name] = bases   497         self.importer.subclasses[class_name] = set()   498         self.scope_globals[class_name] = set()   499    500         # Set the definition before entering the namespace rather than   501         # afterwards because methods may reference it. In normal Python,   502         # a class is not accessible until the definition is complete, but   503         # methods can generally reference it since upon being called the   504         # class will already exist.   505    506         self.set_definition(n.name, "<class>")   507    508         in_class = self.in_class   509         self.in_class = class_name   510         self.set_instance_attr("__class__", Reference("<class>", class_name))   511         self.enter_namespace(n.name)   512    513         # Do not provide the special instantiator attributes on the function   514         # class. Function instances provide these attributes.   515    516         if class_name != "__builtins__.core.function":   517    518             self.set_name("__fn__") # special instantiator attribute   519             self.set_name("__args__") # special instantiator attribute   520    521         # Provide leafname, parent and context attributes.   522    523         parent, leafname = class_name.rsplit(".", 1)   524         self.set_name("__name__", self.get_constant("string", leafname).reference())   525    526         if class_name != "__builtins__.core.function":   527             self.set_name("__parent__")   528    529         self.process_structure_node(n.code)   530         self.exit_namespace()   531         self.in_class = in_class   532    533     def process_from_node(self, n):   534    535         "Process the given node 'n', importing from another module."   536    537         path = self.get_namespace_path()   538    539         module_name, names = self.get_module_name(n)   540         if module_name == self.name:   541             raise InspectError("Cannot import from the current module.", path, n)   542    543         self.queue_module(module_name)   544    545         # Attempt to obtain the referenced objects.   546    547         for name, alias in n.names:   548             if name == "*":   549                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   550    551             # Explicit names.   552    553             ref = self.import_name_from_module(name, module_name)   554             value = ResolvedNameRef(alias or name, ref)   555             self.set_general_local(alias or name, value)   556    557     def process_function_node(self, n, name):   558    559         """   560         Process the given function or lambda node 'n' with the given 'name'.   561         """   562    563         is_lambda = isinstance(n, compiler.ast.Lambda)   564    565         # Where a function is declared conditionally, use a separate name for   566         # the definition, and assign the definition to the stated name.   567    568         if (self.in_conditional or self.in_function) and not is_lambda:   569             original_name = name   570             name = self.get_lambda_name()   571         else:   572             original_name = None   573    574         # Detect conflicting definitions. Such definitions cause conflicts in   575         # the storage of namespace-related information.   576    577         function_name = self.get_object_path(name)   578         ref = self.get_object(function_name, defer=False)   579    580         if ref and ref.static():   581             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   582    583         # Initialise argument and local records.   584    585         argnames = get_argnames(n.argnames)   586         is_method = self.in_class and not self.in_function   587    588         # Remove explicit "self" from method parameters.   589    590         if is_method and argnames and argnames[0] == "self":   591             del argnames[0]   592    593         # Copy and propagate the parameters.   594    595         self.importer.function_parameters[function_name] = \   596             self.function_parameters[function_name] = argnames[:]   597    598         # Define all arguments/parameters in the local namespace.   599    600         locals = \   601             self.importer.function_locals[function_name] = \   602             self.function_locals[function_name] = {}   603    604         # Insert "self" into method locals.   605    606         if is_method:   607             argnames.insert(0, "self")   608    609         # Define "self" in terms of the class if in a method.   610         # This does not diminish the need for type-narrowing in the deducer.   611    612         if argnames:   613             if self.in_class and not self.in_function and argnames[0] == "self":   614                 locals[argnames[0]] = Reference("<instance>", self.in_class)   615             else:   616                 locals[argnames[0]] = Reference("<var>")   617    618         for argname in argnames[1:]:   619             locals[argname] = Reference("<var>")   620    621         globals = self.scope_globals[function_name] = set()   622    623         # Process the defaults.   624    625         defaults = self.importer.function_defaults[function_name] = \   626                    self.function_defaults[function_name] = []   627    628         for argname, default in compiler.ast.get_defaults(n):   629             if default:   630    631                 # Obtain any reference for the default.   632    633                 name_ref = self.process_structure_node(default)   634                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   635    636         # Reset conditional tracking to focus on the function contents.   637    638         in_conditional = self.in_conditional   639         self.in_conditional = False   640    641         in_function = self.in_function   642         self.in_function = function_name   643    644         self.enter_namespace(name)   645    646         # Define a leafname attribute value for the function instance.   647    648         ref = self.get_builtin_class("string")   649         self.reserve_constant(function_name, name, ref.get_origin())   650    651         # Track attribute usage within the namespace.   652    653         path = self.get_namespace_path()   654    655         self.start_tracking(locals)   656         self.process_structure_node(n.code)   657         returns_value = self.stop_tracking()   658    659         # Record any null result.   660    661         is_initialiser = is_method and name == "__init__"   662    663         if not returns_value and not is_initialiser:   664             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   665    666         # Exit to the parent.   667    668         self.exit_namespace()   669    670         # Update flags.   671    672         self.in_function = in_function   673         self.in_conditional = in_conditional   674    675         # Define the function using the appropriate name.   676    677         self.set_definition(name, "<function>")   678    679         # Where a function is set conditionally, assign the name.   680    681         if original_name:   682             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   683    684     def process_global_node(self, n):   685    686         """   687         Process the given "global" node 'n'.   688         """   689    690         path = self.get_namespace_path()   691    692         if path != self.name:   693             self.scope_globals[path].update(n.names)   694    695     def process_if_node(self, n):   696    697         """   698         Process the given "if" node 'n'.   699         """   700    701         tracker = self.trackers[-1]   702         tracker.new_branchpoint()   703    704         for test, body in n.tests:   705             self.process_structure_node(test)   706    707             tracker.new_branch()   708    709             in_conditional = self.in_conditional   710             self.in_conditional = True   711             self.process_structure_node(body)   712             self.in_conditional = in_conditional   713    714             tracker.shelve_branch()   715    716         # Maintain a branch for the else clause.   717    718         tracker.new_branch()   719         if n.else_:   720             self.process_structure_node(n.else_)   721         tracker.shelve_branch()   722    723         tracker.merge_branches()   724    725     def process_import_node(self, n):   726    727         "Process the given import node 'n'."   728    729         path = self.get_namespace_path()   730    731         # Load the mentioned module.   732    733         for name, alias in n.names:   734             if name == self.name:   735                 raise InspectError("Cannot import the current module.", path, n)   736    737             self.set_module(alias or name.split(".")[-1], name)   738             self.queue_module(name, True)   739    740     def process_invocation_node(self, n):   741    742         "Process the given invocation node 'n'."   743    744         path = self.get_namespace_path()   745    746         in_invocation = self.in_invocation   747         self.in_invocation = None   748    749         # Process the arguments.   750    751         keywords = set()   752    753         for arg in n.args:   754             self.process_structure_node(arg)   755             if isinstance(arg, compiler.ast.Keyword):   756                 keywords.add(arg.name)   757    758         keywords = list(keywords)   759         keywords.sort()   760    761         # Communicate to the invocation target expression that it forms the   762         # target of an invocation, potentially affecting attribute accesses.   763    764         self.in_invocation = len(n.args), keywords   765    766         # Process the expression, obtaining any identified reference.   767    768         name_ref = self.process_structure_node(n.node)   769         self.in_invocation = in_invocation   770    771         # Detect class invocations.   772    773         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   774             return InstanceRef(name_ref.reference().instance_of())   775    776         elif isinstance(name_ref, (NameRef, AccessRef)):   777             return InvocationRef(name_ref)   778    779         # Provide a general reference to indicate that something is produced   780         # by the invocation, useful for retaining assignment expression   781         # details.   782    783         return VariableRef()   784    785     def process_lambda_node(self, n):   786    787         "Process the given lambda node 'n'."   788    789         name = self.get_lambda_name()   790         self.process_function_node(n, name)   791    792         origin = self.get_object_path(name)   793    794         if self.function_defaults.get(origin):   795             return None   796         else:   797             return ResolvedNameRef(name, Reference("<function>", origin))   798    799     def process_logical_node(self, n):   800    801         "Process the given operator node 'n'."   802    803         self.process_operator_chain(n.nodes, self.process_structure_node)   804    805     def process_name_node(self, n):   806    807         "Process the given name node 'n'."   808    809         path = self.get_namespace_path()   810    811         # Find predefined constant names before anything else.   812    813         if n.name in predefined_constants:   814             ref = self.get_builtin(n.name)   815             value = ResolvedNameRef(n.name, ref)   816             return value   817    818         # Special names that have already been identified.   819    820         if n.name.startswith("$"):   821             value = self.get_special(n.name)   822             if value:   823                 return value   824    825         # Special case for operator functions introduced through code   826         # transformations.   827    828         if n.name.startswith("$op"):   829    830             # Obtain the location of the actual function defined in the operator   831             # package.   832    833             op = n.name[len("$op"):]   834    835             # Attempt to get a reference.   836    837             ref = self.import_name_from_module(op, "operator")   838    839             # Record the imported name and provide the resolved name reference.   840    841             value = ResolvedNameRef(n.name, ref)   842             self.set_special(n.name, value)   843             return value   844    845         # Special case for print operations.   846    847         elif n.name.startswith("$print"):   848    849             # Attempt to get a reference.   850    851             ref = self.get_builtin("print_")   852    853             # Record the imported name and provide the resolved name reference.   854    855             value = ResolvedNameRef(n.name, ref)   856             self.set_special(n.name, value)   857             return value   858    859         # Test for self usage, which is only allowed in methods.   860    861         if n.name == "self" and not (self.in_function and self.in_class):   862             raise InspectError("Use of self is only allowed in methods.", path, n)   863    864         # Record usage of the name.   865    866         self.record_name(n.name)   867    868         # Search for unknown names in non-function scopes immediately.   869         # Temporary names should not be re-used between scopes.   870         # External names in functions are resolved later.   871    872         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   873    874         if ref:   875             self.record_name_access(n.name, True)   876             return ResolvedNameRef(n.name, ref, is_global=True)   877    878         # Explicitly-declared global names.   879    880         elif self.in_function and n.name in self.scope_globals[path]:   881             self.record_name_access(n.name, True)   882             return NameRef(n.name, is_global=True)   883    884         # Examine other names.   885    886         else:   887    888             # Check local names.   889    890             access_number = self.record_name_access(n.name)   891    892             # Local name.   893    894             if access_number is not None:   895                 return LocalNameRef(n.name, access_number)   896    897             # Possible global or built-in name.   898    899             else:   900                 self.record_name_access(n.name, True)   901                 return NameRef(n.name, is_global=True)   902    903     def record_name_access(self, name, is_global=False):   904    905         """   906         Record an access involving 'name' if the name is being tracked, using   907         'is_global' to indicate whether the name is global.   908         """   909    910         name = self.get_name_for_tracking(name, is_global=is_global)   911         branches = self.trackers[-1].tracking_name(name)   912         if branches:   913             self.record_branches_for_access(branches, name, None)   914             return self.record_access_details(name, None, self.in_assignment,   915                                               self.in_invocation)   916         return None   917    918     def process_operator_chain(self, nodes, fn):   919    920         """   921         Process the given chain of 'nodes', applying 'fn' to each node or item.   922         Each node starts a new conditional region, effectively making a deeply-   923         nested collection of if-like statements.   924         """   925    926         tracker = self.trackers[-1]   927    928         for item in nodes:   929             tracker.new_branchpoint()   930             tracker.new_branch()   931             fn(item)   932    933         for item in nodes[:-1]:   934             tracker.shelve_branch()   935             tracker.new_branch()   936             tracker.shelve_branch()   937             tracker.merge_branches()   938    939         tracker.shelve_branch()   940         tracker.merge_branches()   941    942     def process_try_node(self, n):   943    944         """   945         Process the given "try...except" node 'n'.   946         """   947    948         self.record_exception_handler()   949    950         tracker = self.trackers[-1]   951         tracker.new_branchpoint()   952    953         self.process_structure_node(n.body)   954    955         for name, var, handler in n.handlers:   956             if name is not None:   957                 self.process_structure_node(name)   958    959             # Any abandoned branches from the body can now be resumed in a new   960             # branch.   961    962             tracker.resume_abandoned_branches()   963    964             # Establish the local for the handler.   965    966             if var is not None:   967                 self.process_assignment_node(var, None)   968             if handler is not None:   969                 self.process_structure_node(handler)   970    971             tracker.shelve_branch()   972    973         # The else clause maintains the usage from the body but without the   974         # abandoned branches since they would never lead to the else clause   975         # being executed.   976    977         if n.else_:   978             tracker.new_branch()   979             self.process_structure_node(n.else_)   980             tracker.shelve_branch()   981    982         # Without an else clause, a null branch propagates the successful   983         # outcome.   984    985         else:   986             tracker.new_branch()   987             tracker.shelve_branch()   988    989         tracker.merge_branches()   990    991     def process_try_finally_node(self, n):   992    993         """   994         Process the given "try...finally" node 'n'.   995         """   996    997         self.record_exception_handler()   998    999         tracker = self.trackers[-1]  1000         self.process_structure_node(n.body)  1001   1002         # Any abandoned branches from the body can now be resumed.  1003   1004         branches = tracker.resume_all_abandoned_branches()  1005         self.process_structure_node(n.final)  1006   1007         # At the end of the finally clause, abandoned branches are discarded.  1008   1009         tracker.restore_active_branches(branches)  1010   1011     def process_while_node(self, n):  1012   1013         "Process the given while node 'n'."  1014   1015         tracker = self.trackers[-1]  1016         tracker.new_branchpoint(loop_node=True)  1017   1018         # Evaluate any test or iterator outside the loop.  1019   1020         self.process_structure_node(n.test)  1021   1022         # Propagate attribute usage to branches.  1023   1024         tracker.new_branch(loop_node=True)  1025   1026         # Enter the loop.  1027   1028         in_conditional = self.in_conditional  1029         self.in_conditional = True  1030         self.process_structure_node(n.body)  1031         self.in_conditional = in_conditional  1032   1033         # Continuing branches are resumed before any test.  1034   1035         tracker.resume_continuing_branches()  1036   1037         # Evaluate any continuation test within the body.  1038   1039         self.process_structure_node(n.test)  1040   1041         tracker.shelve_branch(loop_node=True)  1042   1043         # Support the non-looping condition.  1044   1045         tracker.new_branch()  1046         tracker.shelve_branch()  1047   1048         tracker.merge_branches()  1049   1050         # Evaluate any else clause outside branches.  1051   1052         if n.else_:  1053             self.process_structure_node(n.else_)  1054   1055         # Connect broken branches to the code after any loop.  1056   1057         tracker.resume_broken_branches()  1058   1059     # Branch tracking methods.  1060   1061     def start_tracking(self, names):  1062   1063         """  1064         Start tracking attribute usage for names in the current namespace,  1065         immediately registering the given 'names'.  1066         """  1067   1068         path = self.get_namespace_path()  1069         parent = self.trackers[-1]  1070         tracker = BranchTracker()  1071         self.trackers.append(tracker)  1072   1073         # Record the given names established as new branches.  1074   1075         tracker.assign_names(names)  1076   1077     def assign_name(self, name, name_ref):  1078   1079         "Assign to 'name' the given 'name_ref' in the current namespace."  1080   1081         name = self.get_name_for_tracking(name)  1082         self.trackers[-1].assign_names([name], [name_ref])  1083   1084     def stop_tracking(self):  1085   1086         """  1087         Stop tracking attribute usage, recording computed usage for the current  1088         namespace. Indicate whether a value is always returned from the  1089         namespace.  1090         """  1091   1092         path = self.get_namespace_path()  1093         tracker = self.trackers.pop()  1094         self.record_assignments_for_access(tracker)  1095   1096         self.attr_usage[path] = tracker.get_all_usage()  1097         self.name_initialisers[path] = tracker.get_all_values()  1098   1099         return tracker.returns_value()  1100   1101     def start_tracking_in_module(self):  1102   1103         "Start tracking attribute usage in the module."  1104   1105         tracker = BranchTracker()  1106         self.trackers.append(tracker)  1107   1108     def stop_tracking_in_module(self):  1109   1110         "Stop tracking attribute usage in the module."  1111   1112         tracker = self.trackers[0]  1113         self.record_assignments_for_access(tracker)  1114         self.attr_usage[self.name] = tracker.get_all_usage()  1115         self.name_initialisers[self.name] = tracker.get_all_values()  1116   1117     def record_assignments_for_access(self, tracker):  1118   1119         """  1120         For the current path, use the given 'tracker' to record assignment  1121         version information for attribute accesses.  1122         """  1123   1124         path = self.get_path_for_access()  1125   1126         if not self.attr_accessor_branches.has_key(path):  1127             return  1128   1129         init_item(self.attr_accessors, path, dict)  1130         attr_accessors = self.attr_accessors[path]  1131   1132         # Obtain the branches applying during each access.  1133   1134         for access, all_branches in self.attr_accessor_branches[path].items():  1135             name, attrnames = access  1136             init_item(attr_accessors, access, list)  1137   1138             # Obtain the assignments applying to each branch.  1139   1140             for branches in all_branches:  1141                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1142   1143                 # Detect missing name information.  1144   1145                 if None in positions:  1146                     globals = self.global_attr_accesses.get(path)  1147                     accesses = globals and globals.get(name)  1148                     if not accesses:  1149                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1150                             self.get_namespace_path(), name)  1151                     positions.remove(None)  1152   1153                 attr_accessors[access].append(positions)  1154   1155     def record_branches_for_access(self, branches, name, attrnames):  1156   1157         """  1158         Record the given 'branches' for an access involving the given 'name' and  1159         'attrnames'.  1160         """  1161   1162         access = name, attrnames  1163         path = self.get_path_for_access()  1164   1165         init_item(self.attr_accessor_branches, path, dict)  1166         attr_accessor_branches = self.attr_accessor_branches[path]  1167   1168         init_item(attr_accessor_branches, access, list)  1169         attr_accessor_branches[access].append(branches)  1170   1171     def record_access_details(self, name, attrnames, assignment, invocation):  1172   1173         """  1174         For the given 'name' and 'attrnames', record an access indicating  1175         whether an 'assignment' or an 'invocation' is occurring.  1176   1177         These details correspond to accesses otherwise recorded by the attribute  1178         accessor and attribute access dictionaries.  1179         """  1180   1181         access = name, attrnames  1182         path = self.get_path_for_access()  1183   1184         init_item(self.attr_access_modifiers, path, dict)  1185         init_item(self.attr_access_modifiers[path], access, list)  1186   1187         access_number = len(self.attr_access_modifiers[path][access])  1188         self.attr_access_modifiers[path][access].append((assignment, invocation))  1189         return access_number  1190   1191     def record_global_access_details(self, name, attrnames):  1192   1193         """  1194         Record details of a global access via the given 'name' involving the  1195         indicated 'attrnames'.  1196         """  1197   1198         path = self.get_namespace_path()  1199   1200         init_item(self.global_attr_accesses, path, dict)  1201         init_item(self.global_attr_accesses[path], name, set)  1202         self.global_attr_accesses[path][name].add(attrnames)  1203   1204     # Namespace modification.  1205   1206     def record_name(self, name):  1207   1208         "Record the use of 'name' in a namespace."  1209   1210         path = self.get_namespace_path()  1211         init_item(self.names_used, path, set)  1212         self.names_used[path].add(name)  1213   1214     def set_module(self, name, module_name):  1215   1216         """  1217         Set a module in the current namespace using the given 'name' associated  1218         with the corresponding 'module_name'.  1219         """  1220   1221         if name:  1222             self.set_general_local(name, Reference("<module>", module_name))  1223   1224     def set_definition(self, name, kind):  1225   1226         """  1227         Set the definition having the given 'name' and 'kind'.  1228   1229         Definitions are set in the static namespace hierarchy, but they can also  1230         be recorded for function locals.  1231         """  1232   1233         if self.is_global(name):  1234             print >>sys.stderr, "In %s, %s is defined as being global." % (  1235                 self.get_namespace_path(), name)  1236   1237         path = self.get_object_path(name)  1238         self.set_object(path, kind)  1239   1240         ref = self.get_object(path)  1241         if ref.get_kind() == "<var>":  1242             print >>sys.stderr, "In %s, %s is defined more than once." % (  1243                 self.get_namespace_path(), name)  1244   1245         if not self.is_global(name) and self.in_function:  1246             self.set_function_local(name, ref)  1247   1248     def set_function_local(self, name, ref=None):  1249   1250         "Set the local with the given 'name' and optional 'ref'."  1251   1252         path = self.get_namespace_path()  1253         locals = self.function_locals[path]  1254         used = self.names_used.get(path)  1255   1256         if not locals.has_key(name) and used and name in used:  1257             raise InspectError("Name %s assigned locally but used previously." % name, path)  1258   1259         multiple = not ref or locals.has_key(name) and locals[name] != ref  1260         locals[name] = multiple and Reference("<var>") or ref  1261   1262     def assign_general_local(self, name, name_ref):  1263   1264         """  1265         Set for 'name' the given 'name_ref', recording the name for attribute  1266         usage tracking.  1267         """  1268   1269         self.set_general_local(name, name_ref)  1270         self.assign_name(name, name_ref)  1271   1272     def set_general_local(self, name, value=None):  1273   1274         """  1275         Set the 'name' with optional 'value' in any kind of local namespace,  1276         where the 'value' should be a reference if specified.  1277         """  1278   1279         init_value = self.get_initialising_value(value)  1280   1281         # Module global names.  1282   1283         if self.is_global(name):  1284             path = self.get_global_path(name)  1285             self.set_object(path, init_value)  1286   1287         # Function local names.  1288   1289         elif self.in_function:  1290             self.set_function_local(name, init_value)  1291   1292         # Other namespaces (classes).  1293   1294         else:  1295             self.set_name(name, init_value)  1296   1297     def set_name(self, name, ref=None):  1298   1299         "Attach the 'name' with optional 'ref' to the current namespace."  1300   1301         self.set_object(self.get_object_path(name), ref)  1302   1303     def set_instance_attr(self, name, ref=None):  1304   1305         """  1306         Add an instance attribute of the given 'name' to the current class,  1307         using the optional 'ref'.  1308         """  1309   1310         self._set_instance_attr(self.in_class, name, ref)  1311   1312     def _set_instance_attr(self, path, name, ref=None):  1313   1314         init_item(self.instance_attrs, path, set)  1315         self.instance_attrs[path].add(name)  1316   1317         if ref:  1318             init_item(self.instance_attr_constants, path, dict)  1319             self.instance_attr_constants[path][name] = ref  1320   1321     def get_initialising_value(self, value):  1322   1323         "Return a suitable initialiser reference for 'value'."  1324   1325         # Includes LiteralSequenceRef, ResolvedNameRef...  1326   1327         if isinstance(value, (NameRef, AccessRef, InstanceRef)):  1328             return value.reference()  1329   1330         # In general, invocations do not produce known results. However, the  1331         # name initialisers are resolved once a module has been inspected.  1332   1333         elif isinstance(value, InvocationRef):  1334             return value.reference()  1335   1336         # Variable references are unknown results.  1337   1338         elif isinstance(value, VariableRef):  1339             return value.reference()  1340   1341         else:  1342             return value  1343   1344     # Static, program-relative naming.  1345   1346     def find_name(self, name):  1347   1348         """  1349         Return the qualified name for the given 'name' used in the current  1350         non-function namespace.  1351         """  1352   1353         path = self.get_namespace_path()  1354         ref = None  1355   1356         if not self.in_function and name not in predefined_constants:  1357             if self.in_class:  1358                 ref = self.get_object(self.get_object_path(name), False)  1359             if not ref:  1360                 ref = self.get_global_or_builtin(name)  1361   1362         return ref  1363   1364     def get_class(self, node):  1365   1366         """  1367         Use the given 'node' to obtain the identity of a class. Return a  1368         reference for the class. Unresolved dependencies are permitted and must  1369         be resolved later.  1370         """  1371   1372         ref = self._get_class(node)  1373         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1374   1375     def _get_class(self, node):  1376   1377         """  1378         Use the given 'node' to find a class definition. Return a reference to  1379         the class.  1380         """  1381   1382         if isinstance(node, compiler.ast.Getattr):  1383   1384             # Obtain the identity of the access target.  1385   1386             ref = self._get_class(node.expr)  1387   1388             # Where the target is a class or module, obtain the identity of the  1389             # attribute.  1390   1391             if ref.has_kind(["<function>", "<var>"]):  1392                 return None  1393             else:  1394                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1395                 return self.get_object(attrname)  1396   1397         # Names can be module-level or built-in.  1398   1399         elif isinstance(node, compiler.ast.Name):  1400   1401             # Record usage of the name and attempt to identify it.  1402   1403             self.record_name(node.name)  1404             return self.find_name(node.name)  1405         else:  1406             return None  1407   1408     def get_constant(self, name, value):  1409   1410         "Return a constant reference for the given type 'name' and 'value'."  1411   1412         ref = self.get_builtin_class(name)  1413         return self.get_constant_reference(ref, value)  1414   1415     def get_literal_instance(self, n, name=None):  1416   1417         """  1418         For node 'n', return a reference to an instance of 'name', or if 'name'  1419         is not specified, deduce the type from the value.  1420         """  1421   1422         # Handle stray None constants (Sliceobj seems to produce them).  1423   1424         if name == "NoneType":  1425             return self.process_name_node(compiler.ast.Name("None"))  1426   1427         # Obtain the details of the literal itself.  1428         # An alias to the type is generated for sequences.  1429   1430         if name in ("dict", "list", "tuple"):  1431             ref = self.get_builtin_class(name)  1432             self.set_special_literal(name, ref)  1433             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1434   1435         # Constant values are independently recorded.  1436   1437         else:  1438             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1439             ref = self.get_builtin_class(typename)  1440             return self.get_constant_reference(ref, value, encoding)  1441   1442     # Special names.  1443   1444     def get_special(self, name):  1445   1446         "Return any stored value for the given special 'name'."  1447   1448         value = self.special.get(name)  1449         if value:  1450             ref, paths = value  1451         else:  1452             ref = None  1453         return ref  1454   1455     def set_special(self, name, value):  1456   1457         """  1458         Set a special 'name' that merely tracks the use of an implicit object  1459         'value'.  1460         """  1461   1462         if not self.special.has_key(name):  1463             paths = set()  1464             self.special[name] = value, paths  1465         else:  1466             _ref, paths = self.special[name]  1467   1468         paths.add(self.get_namespace_path())  1469   1470     def set_special_literal(self, name, ref):  1471   1472         """  1473         Set a special name for the literal type 'name' having type 'ref'. Such  1474         special names provide a way of referring to literal object types.  1475         """  1476   1477         literal_name = "$L%s" % name  1478         value = ResolvedNameRef(literal_name, ref)  1479         self.set_special(literal_name, value)  1480   1481     # Exceptions.  1482   1483     def record_exception_handler(self):  1484   1485         "Record the current namespace as employing an exception handler."  1486   1487         self.exception_namespaces.add(self.get_namespace_path())  1488   1489     # Return values.  1490   1491     def record_return_value(self, expr):  1492   1493         "Record the given return 'expr'."  1494   1495         path = self.get_namespace_path()  1496         l = init_item(self.return_values, path, list)  1497         l.append(expr)  1498         if not self.importer.all_return_values.has_key(path):  1499             self.importer.all_return_values[path] = l  1500   1501 # vim: tabstop=4 expandtab shiftwidth=4