Lichen

inspector.py

795:345fe332fcf1
2017-03-31 Paul Boddie Added example return values to provide reference details from native functions.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from branching import BranchTracker    24 from common import CommonModule, get_argnames, init_item, predefined_constants    25 from modules import BasicModule, CacheWritingModule, InspectionNaming    26 from errors import InspectError    27 from referencing import Reference    28 from resolving import NameResolving    29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    30                     LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \    31                     Result, VariableRef    32 import compiler    33 import sys    34     35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    36     37     "A module inspector."    38     39     def __init__(self, name, importer):    40     41         "Initialise the module with basic details."    42     43         BasicModule.__init__(self, name, importer)    44     45         self.in_class = False    46         self.in_conditional = False    47     48         # Accesses to global attributes.    49     50         self.global_attr_accesses = {}    51     52         # Usage tracking.    53     54         self.trackers = []    55         self.attr_accessor_branches = {}    56     57     def __repr__(self):    58         return "InspectedModule(%r, %r)" % (self.name, self.importer)    59     60     # Principal methods.    61     62     def parse(self, filename):    63     64         "Parse the file having the given 'filename'."    65     66         self.parse_file(filename)    67     68         # Inspect the module.    69     70         self.start_tracking_in_module()    71     72         # Detect and record imports and globals declared in the module.    73     74         self.process_structure(self.astnode)    75     76         # Set the class of the module after the definition has occurred.    77     78         ref = self.get_builtin("module")    79         self.set_name("__class__", ref)    80         self.set_name("__name__", self.get_constant("string", self.name).reference())    81         self.set_name("__file__", self.get_constant("string", filename).reference())    82     83         # Reserve a constant for the encoding.    84     85         if self.encoding:    86             self.get_constant("string", self.encoding)    87     88         # Get module-level attribute usage details.    89     90         self.stop_tracking_in_module()    91     92         # Collect external name references.    93     94         self.collect_names()    95     96     def complete(self):    97     98         "Complete the module inspection."    99    100         # Resolve names not definitively mapped to objects.   101    102         self.resolve()   103    104         # Propagate to the importer information needed in subsequent activities.   105    106         self.propagate()   107    108     # Accessory methods.   109    110     def collect_names(self):   111    112         "Collect the names used by each scope."   113    114         for path in self.names_used.keys():   115             self.collect_names_for_path(path)   116    117     def collect_names_for_path(self, path):   118    119         """   120         Collect the names used by the given 'path'. These are propagated to the   121         importer in advance of any dependency resolution.   122         """   123    124         names = self.names_used.get(path)   125         if not names:   126             return   127    128         in_function = self.function_locals.has_key(path)   129    130         for name in names:   131             if in_function and name in self.function_locals[path]:   132                 continue   133    134             key = "%s.%s" % (path, name)   135    136             # Find local definitions.   137    138             ref = self.get_resolved_object(key, True)   139             if ref:   140                 self.set_name_reference(key, ref)   141                 continue   142    143             # Find global.   144    145             ref = self.get_global(name)   146             if ref:   147                 self.set_name_reference(key, ref)   148                 continue   149    150             # Find presumed built-in definitions.   151    152             ref = self.get_builtin(name)   153             self.set_name_reference(key, ref)   154    155     def set_name_reference(self, path, ref):   156    157         "Map the given name 'path' to 'ref'."   158    159         self.importer.all_name_references[path] = self.name_references[path] = ref   160    161     # Module structure traversal.   162    163     def process_structure_node(self, n):   164    165         "Process the individual node 'n'."   166    167         path = self.get_namespace_path()   168    169         # Module global detection.   170    171         if isinstance(n, compiler.ast.Global):   172             self.process_global_node(n)   173    174         # Module import declarations.   175    176         elif isinstance(n, compiler.ast.From):   177             self.process_from_node(n)   178    179         elif isinstance(n, compiler.ast.Import):   180             self.process_import_node(n)   181    182         # Nodes using operator module functions.   183    184         elif isinstance(n, compiler.ast.Operator):   185             return self.process_operator_node(n)   186    187         elif isinstance(n, compiler.ast.AugAssign):   188             self.process_augassign_node(n)   189    190         elif isinstance(n, compiler.ast.Compare):   191             return self.process_compare_node(n)   192    193         elif isinstance(n, compiler.ast.Slice):   194             return self.process_slice_node(n)   195    196         elif isinstance(n, compiler.ast.Sliceobj):   197             return self.process_sliceobj_node(n)   198    199         elif isinstance(n, compiler.ast.Subscript):   200             return self.process_subscript_node(n)   201    202         # Namespaces within modules.   203    204         elif isinstance(n, compiler.ast.Class):   205             self.process_class_node(n)   206    207         elif isinstance(n, compiler.ast.Function):   208             self.process_function_node(n, n.name)   209    210         elif isinstance(n, compiler.ast.Lambda):   211             return self.process_lambda_node(n)   212    213         # Assignments.   214    215         elif isinstance(n, compiler.ast.Assign):   216    217             # Handle each assignment node.   218    219             for node in n.nodes:   220                 self.process_assignment_node(node, n.expr)   221    222         # Assignments within non-Assign nodes.   223    224         elif isinstance(n, compiler.ast.AssName):   225             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   226    227         elif isinstance(n, compiler.ast.AssAttr):   228             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   229    230         # Accesses.   231    232         elif isinstance(n, compiler.ast.Getattr):   233             return self.process_attribute_access(n)   234    235         # Name recording for later testing.   236    237         elif isinstance(n, compiler.ast.Name):   238             return self.process_name_node(n)   239    240         # Conditional statement tracking.   241    242         elif isinstance(n, compiler.ast.For):   243             self.process_for_node(n)   244    245         elif isinstance(n, compiler.ast.While):   246             self.process_while_node(n)   247    248         elif isinstance(n, compiler.ast.If):   249             self.process_if_node(n)   250    251         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   252             return self.process_logical_node(n)   253    254         # Exception control-flow tracking.   255    256         elif isinstance(n, compiler.ast.TryExcept):   257             self.process_try_node(n)   258    259         elif isinstance(n, compiler.ast.TryFinally):   260             self.process_try_finally_node(n)   261    262         # Control-flow modification statements.   263    264         elif isinstance(n, compiler.ast.Break):   265             self.trackers[-1].suspend_broken_branch()   266    267         elif isinstance(n, compiler.ast.Continue):   268             self.trackers[-1].suspend_continuing_branch()   269    270         elif isinstance(n, compiler.ast.Raise):   271             self.process_structure(n)   272             self.trackers[-1].abandon_branch()   273    274         elif isinstance(n, compiler.ast.Return):   275             self.record_return_value(self.process_structure_node(n.value))   276             self.trackers[-1].abandon_returning_branch()   277    278         # Print statements.   279    280         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   281             self.process_print_node(n)   282    283         # Invocations.   284    285         elif isinstance(n, compiler.ast.CallFunc):   286             return self.process_invocation_node(n)   287    288         # Constant usage.   289    290         elif isinstance(n, compiler.ast.Const):   291             return self.get_literal_instance(n)   292    293         elif isinstance(n, compiler.ast.Dict):   294             return self.get_literal_instance(n, "dict")   295    296         elif isinstance(n, compiler.ast.List):   297             return self.get_literal_instance(n, "list")   298    299         elif isinstance(n, compiler.ast.Tuple):   300             return self.get_literal_instance(n, "tuple")   301    302         # All other nodes are processed depth-first.   303    304         else:   305             self.process_structure(n)   306    307         # By default, no expression details are returned.   308    309         return None   310    311     # Specific node handling.   312    313     def process_assignment_node(self, n, expr):   314    315         "Process the individual node 'n' to be assigned the contents of 'expr'."   316    317         # Names and attributes are assigned the entire expression.   318    319         if isinstance(n, compiler.ast.AssName):   320             if n.name == "self":   321                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   322    323             name_ref = expr and self.process_structure_node(expr)   324    325             # Name assignments populate either function namespaces or the   326             # general namespace hierarchy.   327    328             self.assign_general_local(n.name, name_ref)   329    330             # Record usage of the name.   331    332             self.record_name(n.name)   333    334         elif isinstance(n, compiler.ast.AssAttr):   335             if expr:   336                 expr = self.process_structure_node(expr)   337    338             in_assignment = self.in_assignment   339             self.in_assignment = True   340             self.process_attribute_access(n)   341             self.in_assignment = in_assignment   342    343         # Lists and tuples are matched against the expression and their   344         # items assigned to expression items.   345    346         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   347             self.process_assignment_node_items(n, expr)   348    349         # Slices and subscripts are permitted within assignment nodes.   350    351         elif isinstance(n, compiler.ast.Slice):   352             self.process_slice_node(n, expr)   353    354         elif isinstance(n, compiler.ast.Subscript):   355             self.process_subscript_node(n, expr)   356    357     def process_attribute_access(self, n):   358    359         "Process the given attribute access node 'n'."   360    361         # Obtain any completed chain and return the reference to it.   362    363         name_ref = self.process_attribute_chain(n)   364    365         if self.have_access_expression(n):   366             return name_ref   367    368         # Where the start of the chain of attributes has been reached, determine   369         # the complete access.   370    371         # Given a non-access node, this chain can be handled in its entirety,   372         # either being name-based and thus an access rooted on a name, or being   373         # based on some other node and thus an anonymous access of some kind.   374    375         path = self.get_namespace_path()   376    377         # Start with the the full attribute chain.   378    379         remaining = self.attrs   380         attrnames = ".".join(remaining)   381    382         # If the accessor cannot be identified, or where attributes   383         # remain in an attribute chain, record the anonymous accesses.   384    385         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   386    387             init_item(self.attr_accesses, path, set)   388             self.attr_accesses[path].add(attrnames)   389    390             self.record_access_details(None, attrnames, self.in_assignment,   391                 self.in_invocation)   392             del self.attrs[0]   393             return   394    395         # Name-based accesses will handle the first attribute in a   396         # chain.   397    398         else:   399             attrname = remaining[0]   400    401             # Attribute assignments are used to identify instance attributes.   402    403             if isinstance(n, compiler.ast.AssAttr) and \   404                 self.in_class and self.in_function and n.expr.name == "self":   405    406                 self.set_instance_attr(attrname)   407    408             # Record attribute usage using any name local to this namespace,   409             # if assigned in the namespace, or using an external name   410             # (presently just globals within classes).   411    412             name = self.get_name_for_tracking(name_ref.name, name_ref)   413             tracker = self.trackers[-1]   414    415             immediate_access = len(self.attrs) == 1   416             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   417    418             # Record global-based chains for subsequent resolution.   419    420             if name_ref.is_global_name():   421                 self.record_global_access_details(name, attrnames)   422    423             # Make sure the name is being tracked: global names will not   424             # already be initialised in a branch and must be added   425             # explicitly.   426    427             if not tracker.have_name(name):   428                 tracker.assign_names([name])   429                 if self.in_function:   430                     self.scope_globals[path].add(name)   431    432             # Record attribute usage in the tracker, and record the branch   433             # information for the access.   434    435             branches = tracker.use_attribute(name, attrname,   436                 self.in_invocation is not None, assignment)   437    438             if not branches:   439                 raise InspectError("Name %s is accessed using %s before an assignment." % (   440                     name, attrname), path, n)   441    442             self.record_branches_for_access(branches, name, attrnames)   443             access_number = self.record_access_details(name, attrnames,   444                 self.in_assignment, self.in_invocation)   445    446             del self.attrs[0]   447             return AccessRef(name, attrnames, access_number)   448    449     def process_class_node(self, n):   450    451         "Process the given class node 'n'."   452    453         path = self.get_namespace_path()   454    455         # To avoid notions of class "versions" where the same definition   456         # might be parameterised with different state and be referenced   457         # elsewhere (as base classes, for example), classes in functions or   458         # conditions are forbidden.   459    460         if self.in_function or self.in_conditional:   461             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   462                 path, n.name)   463             return   464    465         # Resolve base classes.   466    467         bases = []   468    469         for base in n.bases:   470             base_class = self.get_class(base)   471    472             if not base_class:   473                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   474                     path, n.name, base)   475                 return   476             else:   477                 bases.append(base_class)   478    479         # Detect conflicting definitions. Such definitions cause conflicts in   480         # the storage of namespace-related information.   481    482         class_name = self.get_object_path(n.name)   483         ref = self.get_object(class_name, defer=False)   484    485         if ref and ref.static():   486             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   487    488         # Record bases for the class and retain the class name.   489         # Note that the function class does not inherit from the object class.   490    491         if not bases and class_name != "__builtins__.core.object" and \   492                          class_name != "__builtins__.core.function":   493    494             ref = self.get_object("__builtins__.object")   495             bases.append(ref)   496    497         self.importer.classes[class_name] = self.classes[class_name] = bases   498         self.importer.subclasses[class_name] = set()   499         self.scope_globals[class_name] = set()   500    501         # Set the definition before entering the namespace rather than   502         # afterwards because methods may reference it. In normal Python,   503         # a class is not accessible until the definition is complete, but   504         # methods can generally reference it since upon being called the   505         # class will already exist.   506    507         self.set_definition(n.name, "<class>")   508    509         in_class = self.in_class   510         self.in_class = class_name   511         self.set_instance_attr("__class__", Reference("<class>", class_name))   512         self.enter_namespace(n.name)   513    514         # Do not provide the special instantiator attributes on the function   515         # class. Function instances provide these attributes.   516    517         if class_name != "__builtins__.core.function":   518    519             self.set_name("__fn__") # special instantiator attribute   520             self.set_name("__args__") # special instantiator attribute   521    522         # Provide leafname, parent and context attributes.   523    524         parent, leafname = class_name.rsplit(".", 1)   525         self.set_name("__name__", self.get_constant("string", leafname).reference())   526    527         if class_name != "__builtins__.core.function":   528             self.set_name("__parent__")   529    530         self.process_structure_node(n.code)   531         self.exit_namespace()   532         self.in_class = in_class   533    534     def process_from_node(self, n):   535    536         "Process the given node 'n', importing from another module."   537    538         path = self.get_namespace_path()   539    540         module_name, names = self.get_module_name(n)   541         if module_name == self.name:   542             raise InspectError("Cannot import from the current module.", path, n)   543    544         self.queue_module(module_name)   545    546         # Attempt to obtain the referenced objects.   547    548         for name, alias in n.names:   549             if name == "*":   550                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   551    552             # Explicit names.   553    554             ref = self.import_name_from_module(name, module_name)   555             value = ResolvedNameRef(alias or name, ref)   556             self.set_general_local(alias or name, value)   557    558     def process_function_node(self, n, name):   559    560         """   561         Process the given function or lambda node 'n' with the given 'name'.   562         """   563    564         is_lambda = isinstance(n, compiler.ast.Lambda)   565    566         # Where a function is declared conditionally, use a separate name for   567         # the definition, and assign the definition to the stated name.   568    569         if (self.in_conditional or self.in_function) and not is_lambda:   570             original_name = name   571             name = self.get_lambda_name()   572         else:   573             original_name = None   574    575         # Detect conflicting definitions. Such definitions cause conflicts in   576         # the storage of namespace-related information.   577    578         function_name = self.get_object_path(name)   579         ref = self.get_object(function_name, defer=False)   580    581         if ref and ref.static():   582             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   583    584         # Initialise argument and local records.   585    586         argnames = get_argnames(n.argnames)   587         is_method = self.in_class and not self.in_function   588    589         # Remove explicit "self" from method parameters.   590    591         if is_method and argnames and argnames[0] == "self":   592             del argnames[0]   593    594         # Copy and propagate the parameters.   595    596         self.importer.function_parameters[function_name] = \   597             self.function_parameters[function_name] = argnames[:]   598    599         # Define all arguments/parameters in the local namespace.   600    601         locals = \   602             self.importer.function_locals[function_name] = \   603             self.function_locals[function_name] = {}   604    605         # Insert "self" into method locals.   606    607         if is_method:   608             argnames.insert(0, "self")   609    610         # Define "self" in terms of the class if in a method.   611         # This does not diminish the need for type-narrowing in the deducer.   612    613         if argnames:   614             if self.in_class and not self.in_function and argnames[0] == "self":   615                 locals[argnames[0]] = Reference("<instance>", self.in_class)   616             else:   617                 locals[argnames[0]] = Reference("<var>")   618    619         for argname in argnames[1:]:   620             locals[argname] = Reference("<var>")   621    622         globals = self.scope_globals[function_name] = set()   623    624         # Process the defaults.   625    626         defaults = self.importer.function_defaults[function_name] = \   627                    self.function_defaults[function_name] = []   628    629         for argname, default in compiler.ast.get_defaults(n):   630             if default:   631    632                 # Obtain any reference for the default.   633    634                 name_ref = self.process_structure_node(default)   635                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   636    637         # Reset conditional tracking to focus on the function contents.   638    639         in_conditional = self.in_conditional   640         self.in_conditional = False   641    642         in_function = self.in_function   643         self.in_function = function_name   644    645         self.enter_namespace(name)   646    647         # Define a leafname attribute value for the function instance.   648    649         ref = self.get_builtin_class("string")   650         self.reserve_constant(function_name, name, ref.get_origin())   651    652         # Track attribute usage within the namespace.   653    654         path = self.get_namespace_path()   655    656         self.start_tracking(locals)   657         self.process_structure_node(n.code)   658         returns_value = self.stop_tracking()   659    660         # Record any null result.   661    662         is_initialiser = is_method and name == "__init__"   663    664         if not returns_value and not is_initialiser:   665             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   666    667         # Exit to the parent.   668    669         self.exit_namespace()   670    671         # Update flags.   672    673         self.in_function = in_function   674         self.in_conditional = in_conditional   675    676         # Define the function using the appropriate name.   677    678         self.set_definition(name, "<function>")   679    680         # Where a function is set conditionally, assign the name.   681    682         if original_name:   683             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   684    685     def process_global_node(self, n):   686    687         """   688         Process the given "global" node 'n'.   689         """   690    691         path = self.get_namespace_path()   692    693         if path != self.name:   694             self.scope_globals[path].update(n.names)   695    696     def process_if_node(self, n):   697    698         """   699         Process the given "if" node 'n'.   700         """   701    702         tracker = self.trackers[-1]   703         tracker.new_branchpoint()   704    705         for test, body in n.tests:   706             self.process_structure_node(test)   707    708             tracker.new_branch()   709    710             in_conditional = self.in_conditional   711             self.in_conditional = True   712             self.process_structure_node(body)   713             self.in_conditional = in_conditional   714    715             tracker.shelve_branch()   716    717         # Maintain a branch for the else clause.   718    719         tracker.new_branch()   720         if n.else_:   721             self.process_structure_node(n.else_)   722         tracker.shelve_branch()   723    724         tracker.merge_branches()   725    726     def process_import_node(self, n):   727    728         "Process the given import node 'n'."   729    730         path = self.get_namespace_path()   731    732         # Load the mentioned module.   733    734         for name, alias in n.names:   735             if name == self.name:   736                 raise InspectError("Cannot import the current module.", path, n)   737    738             self.set_module(alias or name.split(".")[-1], name)   739             self.queue_module(name, True)   740    741     def process_invocation_node(self, n):   742    743         "Process the given invocation node 'n'."   744    745         path = self.get_namespace_path()   746    747         in_invocation = self.in_invocation   748         self.in_invocation = None   749    750         # Process the arguments.   751    752         keywords = set()   753    754         for arg in n.args:   755             self.process_structure_node(arg)   756             if isinstance(arg, compiler.ast.Keyword):   757                 keywords.add(arg.name)   758    759         keywords = list(keywords)   760         keywords.sort()   761    762         # Communicate to the invocation target expression that it forms the   763         # target of an invocation, potentially affecting attribute accesses.   764    765         self.in_invocation = len(n.args), keywords   766    767         # Process the expression, obtaining any identified reference.   768    769         name_ref = self.process_structure_node(n.node)   770         self.in_invocation = in_invocation   771    772         # Detect class invocations.   773    774         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   775             return InstanceRef(name_ref.reference().instance_of())   776    777         elif isinstance(name_ref, (NameRef, AccessRef)):   778             return InvocationRef(name_ref)   779    780         # Provide a general reference to indicate that something is produced   781         # by the invocation, useful for retaining assignment expression   782         # details.   783    784         return VariableRef()   785    786     def process_lambda_node(self, n):   787    788         "Process the given lambda node 'n'."   789    790         name = self.get_lambda_name()   791         self.process_function_node(n, name)   792    793         origin = self.get_object_path(name)   794    795         if self.function_defaults.get(origin):   796             return None   797         else:   798             return ResolvedNameRef(name, Reference("<function>", origin))   799    800     def process_logical_node(self, n):   801    802         "Process the given operator node 'n'."   803    804         return self.process_operator_chain(n.nodes)   805    806     def process_name_node(self, n):   807    808         "Process the given name node 'n'."   809    810         path = self.get_namespace_path()   811    812         # Find predefined constant names before anything else.   813    814         if n.name in predefined_constants:   815             ref = self.get_builtin(n.name)   816             value = ResolvedNameRef(n.name, ref)   817             return value   818    819         # Special names that have already been identified.   820    821         if n.name.startswith("$"):   822             value = self.get_special(n.name)   823             if value:   824                 return value   825    826         # Special case for operator functions introduced through code   827         # transformations.   828    829         if n.name.startswith("$op"):   830    831             # Obtain the location of the actual function defined in the operator   832             # package.   833    834             op = n.name[len("$op"):]   835    836             # Attempt to get a reference.   837    838             ref = self.import_name_from_module(op, "operator")   839    840             # Record the imported name and provide the resolved name reference.   841    842             value = ResolvedNameRef(n.name, ref)   843             self.set_special(n.name, value)   844             return value   845    846         # Special case for print operations.   847    848         elif n.name.startswith("$print"):   849    850             # Attempt to get a reference.   851    852             ref = self.get_builtin("print_")   853    854             # Record the imported name and provide the resolved name reference.   855    856             value = ResolvedNameRef(n.name, ref)   857             self.set_special(n.name, value)   858             return value   859    860         # Test for self usage, which is only allowed in methods.   861    862         if n.name == "self" and not (self.in_function and self.in_class):   863             raise InspectError("Use of self is only allowed in methods.", path, n)   864    865         # Record usage of the name.   866    867         self.record_name(n.name)   868    869         # Search for unknown names in non-function scopes immediately.   870         # Temporary names should not be re-used between scopes.   871         # External names in functions are resolved later.   872    873         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   874    875         if ref:   876             self.record_name_access(n.name, True)   877             return ResolvedNameRef(n.name, ref, is_global=True)   878    879         # Explicitly-declared global names.   880    881         elif self.in_function and n.name in self.scope_globals[path]:   882             self.record_name_access(n.name, True)   883             return NameRef(n.name, is_global=True)   884    885         # Examine other names.   886    887         else:   888    889             # Check local names.   890    891             access_number = self.record_name_access(n.name)   892    893             # Local name.   894    895             if access_number is not None:   896                 return LocalNameRef(n.name, access_number)   897    898             # Possible global or built-in name.   899    900             else:   901                 self.record_name_access(n.name, True)   902                 return NameRef(n.name, is_global=True)   903    904     def record_name_access(self, name, is_global=False):   905    906         """   907         Record an access involving 'name' if the name is being tracked, using   908         'is_global' to indicate whether the name is global.   909         """   910    911         name = self.get_name_for_tracking(name, is_global=is_global)   912         branches = self.trackers[-1].tracking_name(name)   913         if branches:   914             self.record_branches_for_access(branches, name, None)   915             return self.record_access_details(name, None, self.in_assignment,   916                                               self.in_invocation)   917         return None   918    919     def process_operator_chain(self, nodes):   920    921         """   922         Process the given chain of 'nodes', processing each node or item.   923         Each node starts a new conditional region, effectively making a deeply-   924         nested collection of if-like statements.   925         """   926    927         results = []   928    929         tracker = self.trackers[-1]   930    931         for item in nodes:   932             tracker.new_branchpoint()   933             tracker.new_branch()   934             result = self.process_structure_node(item)   935             if result:   936                 results.append(result)   937    938         for item in nodes[:-1]:   939             tracker.shelve_branch()   940             tracker.new_branch()   941             tracker.shelve_branch()   942             tracker.merge_branches()   943    944         tracker.shelve_branch()   945         tracker.merge_branches()   946    947         return MultipleRef(results)   948    949     def process_try_node(self, n):   950    951         """   952         Process the given "try...except" node 'n'.   953         """   954    955         self.record_exception_handler()   956    957         tracker = self.trackers[-1]   958         tracker.new_branchpoint()   959    960         self.process_structure_node(n.body)   961    962         for name, var, handler in n.handlers:   963             if name is not None:   964                 self.process_structure_node(name)   965    966             # Any abandoned branches from the body can now be resumed in a new   967             # branch.   968    969             tracker.resume_abandoned_branches()   970    971             # Establish the local for the handler.   972    973             if var is not None:   974                 self.process_assignment_node(var, None)   975             if handler is not None:   976                 self.process_structure_node(handler)   977    978             tracker.shelve_branch()   979    980         # The else clause maintains the usage from the body but without the   981         # abandoned branches since they would never lead to the else clause   982         # being executed.   983    984         if n.else_:   985             tracker.new_branch()   986             self.process_structure_node(n.else_)   987             tracker.shelve_branch()   988    989         # Without an else clause, a null branch propagates the successful   990         # outcome.   991    992         else:   993             tracker.new_branch()   994             tracker.shelve_branch()   995    996         tracker.merge_branches()   997    998     def process_try_finally_node(self, n):   999   1000         """  1001         Process the given "try...finally" node 'n'.  1002         """  1003   1004         self.record_exception_handler()  1005   1006         tracker = self.trackers[-1]  1007         self.process_structure_node(n.body)  1008   1009         # Any abandoned branches from the body can now be resumed.  1010   1011         branches = tracker.resume_all_abandoned_branches()  1012         self.process_structure_node(n.final)  1013   1014         # At the end of the finally clause, abandoned branches are discarded.  1015   1016         tracker.restore_active_branches(branches)  1017   1018     def process_while_node(self, n):  1019   1020         "Process the given while node 'n'."  1021   1022         tracker = self.trackers[-1]  1023         tracker.new_branchpoint(loop_node=True)  1024   1025         # Evaluate any test or iterator outside the loop.  1026   1027         self.process_structure_node(n.test)  1028   1029         # Propagate attribute usage to branches.  1030   1031         tracker.new_branch(loop_node=True)  1032   1033         # Enter the loop.  1034   1035         in_conditional = self.in_conditional  1036         self.in_conditional = True  1037         self.process_structure_node(n.body)  1038         self.in_conditional = in_conditional  1039   1040         # Continuing branches are resumed before any test.  1041   1042         tracker.resume_continuing_branches()  1043   1044         # Evaluate any continuation test within the body.  1045   1046         self.process_structure_node(n.test)  1047   1048         tracker.shelve_branch(loop_node=True)  1049   1050         # Support the non-looping condition.  1051   1052         tracker.new_branch()  1053         tracker.shelve_branch()  1054   1055         tracker.merge_branches()  1056   1057         # Evaluate any else clause outside branches.  1058   1059         if n.else_:  1060             self.process_structure_node(n.else_)  1061   1062         # Connect broken branches to the code after any loop.  1063   1064         tracker.resume_broken_branches()  1065   1066     # Branch tracking methods.  1067   1068     def start_tracking(self, names):  1069   1070         """  1071         Start tracking attribute usage for names in the current namespace,  1072         immediately registering the given 'names'.  1073         """  1074   1075         path = self.get_namespace_path()  1076         parent = self.trackers[-1]  1077         tracker = BranchTracker()  1078         self.trackers.append(tracker)  1079   1080         # Record the given names established as new branches.  1081   1082         tracker.assign_names(names)  1083   1084     def assign_name(self, name, name_ref):  1085   1086         "Assign to 'name' the given 'name_ref' in the current namespace."  1087   1088         name = self.get_name_for_tracking(name)  1089         self.trackers[-1].assign_names([name], [name_ref])  1090   1091     def stop_tracking(self):  1092   1093         """  1094         Stop tracking attribute usage, recording computed usage for the current  1095         namespace. Indicate whether a value is always returned from the  1096         namespace.  1097         """  1098   1099         path = self.get_namespace_path()  1100         tracker = self.trackers.pop()  1101         self.record_assignments_for_access(tracker)  1102   1103         self.attr_usage[path] = tracker.get_all_usage()  1104         self.name_initialisers[path] = tracker.get_all_values()  1105   1106         return tracker.returns_value()  1107   1108     def start_tracking_in_module(self):  1109   1110         "Start tracking attribute usage in the module."  1111   1112         tracker = BranchTracker()  1113         self.trackers.append(tracker)  1114   1115     def stop_tracking_in_module(self):  1116   1117         "Stop tracking attribute usage in the module."  1118   1119         tracker = self.trackers[0]  1120         self.record_assignments_for_access(tracker)  1121         self.attr_usage[self.name] = tracker.get_all_usage()  1122         self.name_initialisers[self.name] = tracker.get_all_values()  1123   1124     def record_assignments_for_access(self, tracker):  1125   1126         """  1127         For the current path, use the given 'tracker' to record assignment  1128         version information for attribute accesses.  1129         """  1130   1131         path = self.get_path_for_access()  1132   1133         if not self.attr_accessor_branches.has_key(path):  1134             return  1135   1136         init_item(self.attr_accessors, path, dict)  1137         attr_accessors = self.attr_accessors[path]  1138   1139         # Obtain the branches applying during each access.  1140   1141         for access, all_branches in self.attr_accessor_branches[path].items():  1142             name, attrnames = access  1143             init_item(attr_accessors, access, list)  1144   1145             # Obtain the assignments applying to each branch.  1146   1147             for branches in all_branches:  1148                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1149   1150                 # Detect missing name information.  1151   1152                 if None in positions:  1153                     globals = self.global_attr_accesses.get(path)  1154                     accesses = globals and globals.get(name)  1155                     if not accesses:  1156                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1157                             self.get_namespace_path(), name)  1158                     positions.remove(None)  1159   1160                 attr_accessors[access].append(positions)  1161   1162     def record_branches_for_access(self, branches, name, attrnames):  1163   1164         """  1165         Record the given 'branches' for an access involving the given 'name' and  1166         'attrnames'.  1167         """  1168   1169         access = name, attrnames  1170         path = self.get_path_for_access()  1171   1172         init_item(self.attr_accessor_branches, path, dict)  1173         attr_accessor_branches = self.attr_accessor_branches[path]  1174   1175         init_item(attr_accessor_branches, access, list)  1176         attr_accessor_branches[access].append(branches)  1177   1178     def record_access_details(self, name, attrnames, assignment, invocation):  1179   1180         """  1181         For the given 'name' and 'attrnames', record an access indicating  1182         whether an 'assignment' or an 'invocation' is occurring.  1183   1184         These details correspond to accesses otherwise recorded by the attribute  1185         accessor and attribute access dictionaries.  1186         """  1187   1188         access = name, attrnames  1189         path = self.get_path_for_access()  1190   1191         init_item(self.attr_access_modifiers, path, dict)  1192         init_item(self.attr_access_modifiers[path], access, list)  1193   1194         access_number = len(self.attr_access_modifiers[path][access])  1195         self.attr_access_modifiers[path][access].append((assignment, invocation))  1196         return access_number  1197   1198     def record_global_access_details(self, name, attrnames):  1199   1200         """  1201         Record details of a global access via the given 'name' involving the  1202         indicated 'attrnames'.  1203         """  1204   1205         path = self.get_namespace_path()  1206   1207         init_item(self.global_attr_accesses, path, dict)  1208         init_item(self.global_attr_accesses[path], name, set)  1209         self.global_attr_accesses[path][name].add(attrnames)  1210   1211     # Namespace modification.  1212   1213     def record_name(self, name):  1214   1215         "Record the use of 'name' in a namespace."  1216   1217         path = self.get_namespace_path()  1218         init_item(self.names_used, path, set)  1219         self.names_used[path].add(name)  1220   1221     def set_module(self, name, module_name):  1222   1223         """  1224         Set a module in the current namespace using the given 'name' associated  1225         with the corresponding 'module_name'.  1226         """  1227   1228         if name:  1229             self.set_general_local(name, Reference("<module>", module_name))  1230   1231     def set_definition(self, name, kind):  1232   1233         """  1234         Set the definition having the given 'name' and 'kind'.  1235   1236         Definitions are set in the static namespace hierarchy, but they can also  1237         be recorded for function locals.  1238         """  1239   1240         if self.is_global(name):  1241             print >>sys.stderr, "In %s, %s is defined as being global." % (  1242                 self.get_namespace_path(), name)  1243   1244         path = self.get_object_path(name)  1245         self.set_object(path, kind)  1246   1247         ref = self.get_object(path)  1248         if ref.get_kind() == "<var>":  1249             print >>sys.stderr, "In %s, %s is defined more than once." % (  1250                 self.get_namespace_path(), name)  1251   1252         if not self.is_global(name) and self.in_function:  1253             self.set_function_local(name, ref)  1254   1255     def set_function_local(self, name, ref=None):  1256   1257         "Set the local with the given 'name' and optional 'ref'."  1258   1259         path = self.get_namespace_path()  1260         locals = self.function_locals[path]  1261         used = self.names_used.get(path)  1262   1263         if not locals.has_key(name) and used and name in used:  1264             raise InspectError("Name %s assigned locally but used previously." % name, path)  1265   1266         multiple = not ref or locals.has_key(name) and locals[name] != ref  1267         locals[name] = multiple and Reference("<var>") or ref  1268   1269     def assign_general_local(self, name, name_ref):  1270   1271         """  1272         Set for 'name' the given 'name_ref', recording the name for attribute  1273         usage tracking.  1274         """  1275   1276         self.set_general_local(name, name_ref)  1277         self.assign_name(name, name_ref)  1278   1279     def set_general_local(self, name, value=None):  1280   1281         """  1282         Set the 'name' with optional 'value' in any kind of local namespace,  1283         where the 'value' should be a reference if specified.  1284         """  1285   1286         init_value = self.get_initialising_value(value)  1287   1288         # Module global names.  1289   1290         if self.is_global(name):  1291             path = self.get_global_path(name)  1292             self.set_object(path, init_value)  1293   1294         # Function local names.  1295   1296         elif self.in_function:  1297             self.set_function_local(name, init_value)  1298   1299         # Other namespaces (classes).  1300   1301         else:  1302             self.set_name(name, init_value)  1303   1304     def set_name(self, name, ref=None):  1305   1306         "Attach the 'name' with optional 'ref' to the current namespace."  1307   1308         self.set_object(self.get_object_path(name), ref)  1309   1310     def set_instance_attr(self, name, ref=None):  1311   1312         """  1313         Add an instance attribute of the given 'name' to the current class,  1314         using the optional 'ref'.  1315         """  1316   1317         self._set_instance_attr(self.in_class, name, ref)  1318   1319     def _set_instance_attr(self, path, name, ref=None):  1320   1321         init_item(self.instance_attrs, path, set)  1322         self.instance_attrs[path].add(name)  1323   1324         if ref:  1325             init_item(self.instance_attr_constants, path, dict)  1326             self.instance_attr_constants[path][name] = ref  1327   1328     def get_initialising_value(self, value):  1329   1330         "Return a suitable initialiser reference for 'value'."  1331   1332         if isinstance(value, Result):  1333             return value.reference()  1334         else:  1335             return value  1336   1337     # Static, program-relative naming.  1338   1339     def find_name(self, name):  1340   1341         """  1342         Return the qualified name for the given 'name' used in the current  1343         non-function namespace.  1344         """  1345   1346         path = self.get_namespace_path()  1347         ref = None  1348   1349         if not self.in_function and name not in predefined_constants:  1350             if self.in_class:  1351                 ref = self.get_object(self.get_object_path(name), False)  1352             if not ref:  1353                 ref = self.get_global_or_builtin(name)  1354   1355         return ref  1356   1357     def get_class(self, node):  1358   1359         """  1360         Use the given 'node' to obtain the identity of a class. Return a  1361         reference for the class. Unresolved dependencies are permitted and must  1362         be resolved later.  1363         """  1364   1365         ref = self._get_class(node)  1366         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1367   1368     def _get_class(self, node):  1369   1370         """  1371         Use the given 'node' to find a class definition. Return a reference to  1372         the class.  1373         """  1374   1375         if isinstance(node, compiler.ast.Getattr):  1376   1377             # Obtain the identity of the access target.  1378   1379             ref = self._get_class(node.expr)  1380   1381             # Where the target is a class or module, obtain the identity of the  1382             # attribute.  1383   1384             if ref.has_kind(["<function>", "<var>"]):  1385                 return None  1386             else:  1387                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1388                 return self.get_object(attrname)  1389   1390         # Names can be module-level or built-in.  1391   1392         elif isinstance(node, compiler.ast.Name):  1393   1394             # Record usage of the name and attempt to identify it.  1395   1396             self.record_name(node.name)  1397             return self.find_name(node.name)  1398         else:  1399             return None  1400   1401     def get_constant(self, name, value):  1402   1403         "Return a constant reference for the given type 'name' and 'value'."  1404   1405         ref = self.get_builtin_class(name)  1406         return self.get_constant_reference(ref, value)  1407   1408     def get_literal_instance(self, n, name=None):  1409   1410         """  1411         For node 'n', return a reference to an instance of 'name', or if 'name'  1412         is not specified, deduce the type from the value.  1413         """  1414   1415         # Handle stray None constants (Sliceobj seems to produce them).  1416   1417         if name == "NoneType":  1418             return self.process_name_node(compiler.ast.Name("None"))  1419   1420         # Obtain the details of the literal itself.  1421         # An alias to the type is generated for sequences.  1422   1423         if name in ("dict", "list", "tuple"):  1424             ref = self.get_builtin_class(name)  1425             self.set_special_literal(name, ref)  1426             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1427   1428         # Constant values are independently recorded.  1429   1430         else:  1431             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1432             ref = self.get_builtin_class(typename)  1433             return self.get_constant_reference(ref, value, encoding)  1434   1435     # Special names.  1436   1437     def get_special(self, name):  1438   1439         "Return any stored value for the given special 'name'."  1440   1441         value = self.special.get(name)  1442         if value:  1443             ref, paths = value  1444         else:  1445             ref = None  1446         return ref  1447   1448     def set_special(self, name, value):  1449   1450         """  1451         Set a special 'name' that merely tracks the use of an implicit object  1452         'value'.  1453         """  1454   1455         if not self.special.has_key(name):  1456             paths = set()  1457             self.special[name] = value, paths  1458         else:  1459             _ref, paths = self.special[name]  1460   1461         paths.add(self.get_namespace_path())  1462   1463     def set_special_literal(self, name, ref):  1464   1465         """  1466         Set a special name for the literal type 'name' having type 'ref'. Such  1467         special names provide a way of referring to literal object types.  1468         """  1469   1470         literal_name = "$L%s" % name  1471         value = ResolvedNameRef(literal_name, ref)  1472         self.set_special(literal_name, value)  1473   1474     # Exceptions.  1475   1476     def record_exception_handler(self):  1477   1478         "Record the current namespace as employing an exception handler."  1479   1480         self.exception_namespaces.add(self.get_namespace_path())  1481   1482     # Return values.  1483   1484     def record_return_value(self, expr):  1485   1486         "Record the given return 'expr'."  1487   1488         path = self.get_namespace_path()  1489         l = init_item(self.return_values, path, list)  1490         l.append(expr)  1491         if not self.importer.all_return_values.has_key(path):  1492             self.importer.all_return_values[path] = l  1493   1494 # vim: tabstop=4 expandtab shiftwidth=4