Lichen

inspector.py

820:be4f3575f605
2018-01-08 Paul Boddie Added tests for attribute initialisation plus testing for duplicate parameters.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from branching import BranchTracker    24 from common import CommonModule, get_argnames, init_item, predefined_constants    25 from modules import BasicModule, CacheWritingModule, InspectionNaming    26 from errors import InspectError    27 from referencing import Reference    28 from resolving import NameResolving    29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    30                     LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \    31                     Result, VariableRef    32 import compiler    33 import sys    34     35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    36     37     "A module inspector."    38     39     def __init__(self, name, importer):    40     41         "Initialise the module with basic details."    42     43         BasicModule.__init__(self, name, importer)    44     45         self.in_class = False    46         self.in_conditional = False    47     48         # Accesses to global attributes.    49     50         self.global_attr_accesses = {}    51     52         # Usage tracking.    53     54         self.trackers = []    55         self.attr_accessor_branches = {}    56     57     def __repr__(self):    58         return "InspectedModule(%r, %r)" % (self.name, self.importer)    59     60     # Principal methods.    61     62     def parse(self, filename):    63     64         "Parse the file having the given 'filename'."    65     66         self.parse_file(filename)    67     68         # Inspect the module.    69     70         self.start_tracking_in_module()    71     72         # Detect and record imports and globals declared in the module.    73     74         self.process_structure(self.astnode)    75     76         # Set the class of the module after the definition has occurred.    77     78         ref = self.get_builtin("module")    79         self.set_name("__class__", ref)    80         self.set_name("__name__", self.get_constant("string", self.name).reference())    81         self.set_name("__file__", self.get_constant("string", filename).reference())    82     83         # Reserve a constant for the encoding.    84     85         if self.encoding:    86             self.get_constant("string", self.encoding)    87     88         # Get module-level attribute usage details.    89     90         self.stop_tracking_in_module()    91     92         # Collect external name references.    93     94         self.collect_names()    95     96     def complete(self):    97     98         "Complete the module inspection."    99    100         # Resolve names not definitively mapped to objects.   101    102         self.resolve()   103    104         # Propagate to the importer information needed in subsequent activities.   105    106         self.propagate()   107    108     # Accessory methods.   109    110     def collect_names(self):   111    112         "Collect the names used by each scope."   113    114         for path in self.names_used.keys():   115             self.collect_names_for_path(path)   116    117     def collect_names_for_path(self, path):   118    119         """   120         Collect the names used by the given 'path'. These are propagated to the   121         importer in advance of any dependency resolution.   122         """   123    124         names = self.names_used.get(path)   125         if not names:   126             return   127    128         in_function = self.function_locals.has_key(path)   129    130         for name in names:   131             if in_function and name in self.function_locals[path]:   132                 continue   133    134             key = "%s.%s" % (path, name)   135    136             # Find local definitions.   137    138             ref = self.get_resolved_object(key, True)   139             if ref:   140                 self.set_name_reference(key, ref)   141                 continue   142    143             # Find global.   144    145             ref = self.get_global(name)   146             if ref:   147                 self.set_name_reference(key, ref)   148                 continue   149    150             # Find presumed built-in definitions.   151    152             ref = self.get_builtin(name)   153             self.set_name_reference(key, ref)   154    155     def set_name_reference(self, path, ref):   156    157         "Map the given name 'path' to 'ref'."   158    159         self.importer.all_name_references[path] = self.name_references[path] = ref   160    161     # Module structure traversal.   162    163     def process_structure_node(self, n):   164    165         "Process the individual node 'n'."   166    167         path = self.get_namespace_path()   168    169         # Module global detection.   170    171         if isinstance(n, compiler.ast.Global):   172             self.process_global_node(n)   173    174         # Module import declarations.   175    176         elif isinstance(n, compiler.ast.From):   177             self.process_from_node(n)   178    179         elif isinstance(n, compiler.ast.Import):   180             self.process_import_node(n)   181    182         # Nodes using operator module functions.   183    184         elif isinstance(n, compiler.ast.Operator):   185             return self.process_operator_node(n)   186    187         elif isinstance(n, compiler.ast.AugAssign):   188             self.process_augassign_node(n)   189    190         elif isinstance(n, compiler.ast.Compare):   191             return self.process_compare_node(n)   192    193         elif isinstance(n, compiler.ast.Slice):   194             return self.process_slice_node(n)   195    196         elif isinstance(n, compiler.ast.Sliceobj):   197             return self.process_sliceobj_node(n)   198    199         elif isinstance(n, compiler.ast.Subscript):   200             return self.process_subscript_node(n)   201    202         # Namespaces within modules.   203    204         elif isinstance(n, compiler.ast.Class):   205             self.process_class_node(n)   206    207         elif isinstance(n, compiler.ast.Function):   208             self.process_function_node(n, n.name)   209    210         elif isinstance(n, compiler.ast.Lambda):   211             return self.process_lambda_node(n)   212    213         # Assignments.   214    215         elif isinstance(n, compiler.ast.Assign):   216    217             # Handle each assignment node.   218    219             for node in n.nodes:   220                 self.process_assignment_node(node, n.expr)   221    222         # Assignments within non-Assign nodes.   223    224         elif isinstance(n, compiler.ast.AssName):   225             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   226    227         elif isinstance(n, compiler.ast.AssAttr):   228             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   229    230         # Accesses.   231    232         elif isinstance(n, compiler.ast.Getattr):   233             return self.process_attribute_access(n)   234    235         # Name recording for later testing.   236    237         elif isinstance(n, compiler.ast.Name):   238             return self.process_name_node(n)   239    240         # Conditional statement tracking.   241    242         elif isinstance(n, compiler.ast.For):   243             self.process_for_node(n)   244    245         elif isinstance(n, compiler.ast.While):   246             self.process_while_node(n)   247    248         elif isinstance(n, compiler.ast.If):   249             self.process_if_node(n)   250    251         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   252             return self.process_logical_node(n)   253    254         # Exception control-flow tracking.   255    256         elif isinstance(n, compiler.ast.TryExcept):   257             self.process_try_node(n)   258    259         elif isinstance(n, compiler.ast.TryFinally):   260             self.process_try_finally_node(n)   261    262         # Control-flow modification statements.   263    264         elif isinstance(n, compiler.ast.Break):   265             self.trackers[-1].suspend_broken_branch()   266    267         elif isinstance(n, compiler.ast.Continue):   268             self.trackers[-1].suspend_continuing_branch()   269    270         elif isinstance(n, compiler.ast.Raise):   271             self.process_structure(n)   272             self.trackers[-1].abandon_branch()   273    274         elif isinstance(n, compiler.ast.Return):   275             self.record_return_value(self.process_structure_node(n.value))   276             self.trackers[-1].abandon_returning_branch()   277    278         # Print statements.   279    280         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   281             self.process_print_node(n)   282    283         # Invocations.   284    285         elif isinstance(n, compiler.ast.CallFunc):   286             return self.process_invocation_node(n)   287    288         # Constant usage.   289    290         elif isinstance(n, compiler.ast.Const):   291             return self.get_literal_instance(n)   292    293         elif isinstance(n, compiler.ast.Dict):   294             return self.get_literal_instance(n, "dict")   295    296         elif isinstance(n, compiler.ast.List):   297             return self.get_literal_instance(n, "list")   298    299         elif isinstance(n, compiler.ast.Tuple):   300             return self.get_literal_instance(n, "tuple")   301    302         # All other nodes are processed depth-first.   303    304         else:   305             self.process_structure(n)   306    307         # By default, no expression details are returned.   308    309         return None   310    311     # Specific node handling.   312    313     def process_assignment_node(self, n, expr):   314    315         "Process the individual node 'n' to be assigned the contents of 'expr'."   316    317         # Names and attributes are assigned the entire expression.   318    319         if isinstance(n, compiler.ast.AssName):   320             if n.name == "self":   321                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   322    323             name_ref = expr and self.process_structure_node(expr)   324    325             # Name assignments populate either function namespaces or the   326             # general namespace hierarchy.   327    328             self.assign_general_local(n.name, name_ref)   329    330             # Record usage of the name.   331    332             self.record_name(n.name)   333    334         elif isinstance(n, compiler.ast.AssAttr):   335             if expr:   336                 expr = self.process_structure_node(expr)   337    338             in_assignment = self.in_assignment   339             self.in_assignment = True   340             self.process_attribute_access(n)   341             self.in_assignment = in_assignment   342    343         # Lists and tuples are matched against the expression and their   344         # items assigned to expression items.   345    346         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   347             self.process_assignment_node_items(n, expr)   348    349         # Slices and subscripts are permitted within assignment nodes.   350    351         elif isinstance(n, compiler.ast.Slice):   352             self.process_slice_node(n, expr)   353    354         elif isinstance(n, compiler.ast.Subscript):   355             self.process_subscript_node(n, expr)   356    357     def process_attribute_access(self, n):   358    359         "Process the given attribute access node 'n'."   360    361         # Obtain any completed chain and return the reference to it.   362    363         name_ref = self.process_attribute_chain(n)   364    365         if self.have_access_expression(n):   366             return name_ref   367    368         # Where the start of the chain of attributes has been reached, determine   369         # the complete access.   370    371         # Given a non-access node, this chain can be handled in its entirety,   372         # either being name-based and thus an access rooted on a name, or being   373         # based on some other node and thus an anonymous access of some kind.   374    375         path = self.get_namespace_path()   376    377         # Start with the the full attribute chain.   378    379         remaining = self.attrs   380         attrnames = ".".join(remaining)   381    382         # If the accessor cannot be identified, or where attributes   383         # remain in an attribute chain, record the anonymous accesses.   384    385         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   386    387             init_item(self.attr_accesses, path, set)   388             self.attr_accesses[path].add(attrnames)   389    390             self.record_access_details(None, attrnames, self.in_assignment,   391                 self.in_invocation)   392             del self.attrs[0]   393             return   394    395         # Name-based accesses will handle the first attribute in a   396         # chain.   397    398         else:   399             attrname = remaining[0]   400    401             # Attribute assignments are used to identify instance attributes.   402    403             if isinstance(n, compiler.ast.AssAttr) and \   404                 self.in_class and self.in_function and n.expr.name == "self":   405    406                 self.set_instance_attr(attrname)   407    408             # Record attribute usage using any name local to this namespace,   409             # if assigned in the namespace, or using an external name   410             # (presently just globals within classes).   411    412             name = self.get_name_for_tracking(name_ref.name, name_ref)   413             tracker = self.trackers[-1]   414    415             immediate_access = len(self.attrs) == 1   416             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   417    418             # Record global-based chains for subsequent resolution.   419    420             if name_ref.is_global_name():   421                 self.record_global_access_details(name, attrnames)   422    423             # Make sure the name is being tracked: global names will not   424             # already be initialised in a branch and must be added   425             # explicitly.   426    427             if not tracker.have_name(name):   428                 tracker.assign_names([name])   429                 if self.in_function:   430                     self.scope_globals[path].add(name)   431    432             # Record attribute usage in the tracker, and record the branch   433             # information for the access.   434    435             branches = tracker.use_attribute(name, attrname,   436                 self.in_invocation is not None, assignment)   437    438             if not branches:   439                 raise InspectError("Name %s is accessed using %s before an assignment." % (   440                     name, attrname), path, n)   441    442             self.record_branches_for_access(branches, name, attrnames)   443             access_number = self.record_access_details(name, attrnames,   444                 self.in_assignment, self.in_invocation)   445    446             del self.attrs[0]   447             return AccessRef(name, attrnames, access_number)   448    449     def process_class_node(self, n):   450    451         "Process the given class node 'n'."   452    453         path = self.get_namespace_path()   454    455         # To avoid notions of class "versions" where the same definition   456         # might be parameterised with different state and be referenced   457         # elsewhere (as base classes, for example), classes in functions or   458         # conditions are forbidden.   459    460         if self.in_function or self.in_conditional:   461             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   462                 path, n.name)   463             return   464    465         # Resolve base classes.   466    467         bases = []   468    469         for base in n.bases:   470             base_class = self.get_class(base)   471    472             if not base_class:   473                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   474                     path, n.name, base)   475                 return   476             else:   477                 bases.append(base_class)   478    479         # Detect conflicting definitions. Such definitions cause conflicts in   480         # the storage of namespace-related information.   481    482         class_name = self.get_object_path(n.name)   483         ref = self.get_object(class_name, defer=False)   484    485         if ref and ref.static():   486             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   487    488         # Record bases for the class and retain the class name.   489         # Note that the function class does not inherit from the object class.   490    491         if not bases and class_name != "__builtins__.core.object" and \   492                          class_name != "__builtins__.core.function":   493    494             ref = self.get_object("__builtins__.object")   495             bases.append(ref)   496    497         self.importer.classes[class_name] = self.classes[class_name] = bases   498         self.importer.subclasses[class_name] = set()   499         self.scope_globals[class_name] = set()   500    501         # Set the definition before entering the namespace rather than   502         # afterwards because methods may reference it. In normal Python,   503         # a class is not accessible until the definition is complete, but   504         # methods can generally reference it since upon being called the   505         # class will already exist.   506    507         self.set_definition(n.name, "<class>")   508    509         in_class = self.in_class   510         self.in_class = class_name   511         self.set_instance_attr("__class__", Reference("<class>", class_name))   512         self.enter_namespace(n.name)   513    514         # Do not provide the special instantiator attributes on the function   515         # class. Function instances provide these attributes.   516    517         if class_name != "__builtins__.core.function":   518    519             self.set_name("__fn__") # special instantiator attribute   520             self.set_name("__args__") # special instantiator attribute   521    522         # Provide leafname, parent and context attributes.   523    524         parent, leafname = class_name.rsplit(".", 1)   525         self.set_name("__name__", self.get_constant("string", leafname).reference())   526    527         if class_name != "__builtins__.core.function":   528             self.set_name("__parent__")   529    530         self.process_structure_node(n.code)   531         self.exit_namespace()   532         self.in_class = in_class   533    534     def process_from_node(self, n):   535    536         "Process the given node 'n', importing from another module."   537    538         path = self.get_namespace_path()   539    540         module_name, names = self.get_module_name(n)   541         if module_name == self.name:   542             raise InspectError("Cannot import from the current module.", path, n)   543    544         self.queue_module(module_name)   545    546         # Attempt to obtain the referenced objects.   547    548         for name, alias in n.names:   549             if name == "*":   550                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   551    552             # Explicit names.   553    554             ref = self.import_name_from_module(name, module_name)   555             value = ResolvedNameRef(alias or name, ref)   556             self.set_general_local(alias or name, value)   557    558     def process_function_node(self, n, name):   559    560         """   561         Process the given function or lambda node 'n' with the given 'name'.   562         """   563    564         is_lambda = isinstance(n, compiler.ast.Lambda)   565    566         # Where a function is declared conditionally, use a separate name for   567         # the definition, and assign the definition to the stated name.   568    569         if (self.in_conditional or self.in_function) and not is_lambda:   570             original_name = name   571             name = self.get_lambda_name()   572         else:   573             original_name = None   574    575         # Detect conflicting definitions. Such definitions cause conflicts in   576         # the storage of namespace-related information.   577    578         function_name = self.get_object_path(name)   579         ref = self.get_object(function_name, defer=False)   580    581         if ref and ref.static():   582             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   583    584         # Initialise argument and local records.   585    586         argnames = get_argnames(n.argnames)   587         is_method = self.in_class and not self.in_function   588    589         # Remove explicit "self" from method parameters.   590    591         if is_method and argnames and argnames[0] == "self":   592             del argnames[0]   593    594         # Convert .name entries in the parameters, provided this is a method.   595    596         l = []   597         attr_initialisers = []   598    599         for argname in argnames:   600             if argname[0] == ".":   601                 if not is_method:   602                     raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n)   603    604                 argname = argname[1:]   605                 attr_initialisers.append(argname)   606    607             if argname in l:   608                 raise InspectError("Argument name %s is repeated." % argname, function_name, n)   609    610             l.append(argname)   611    612         argnames = l   613    614         # Copy and propagate the parameters.   615    616         self.importer.function_parameters[function_name] = \   617             self.function_parameters[function_name] = argnames[:]   618    619         self.importer.function_attr_initialisers[function_name] = \   620             self.function_attr_initialisers[function_name] = attr_initialisers   621    622         # Define all arguments/parameters in the local namespace.   623    624         locals = \   625             self.importer.function_locals[function_name] = \   626             self.function_locals[function_name] = {}   627    628         # Insert "self" into method locals.   629    630         if is_method:   631             argnames.insert(0, "self")   632    633         # Define "self" in terms of the class if in a method.   634         # This does not diminish the need for type-narrowing in the deducer.   635    636         if argnames:   637             if self.in_class and not self.in_function and argnames[0] == "self":   638                 locals[argnames[0]] = Reference("<instance>", self.in_class)   639             else:   640                 locals[argnames[0]] = Reference("<var>")   641    642         for argname in argnames[1:]:   643             locals[argname] = Reference("<var>")   644    645         globals = self.scope_globals[function_name] = set()   646    647         # Process the defaults.   648    649         defaults = self.importer.function_defaults[function_name] = \   650                    self.function_defaults[function_name] = []   651    652         for argname, default in compiler.ast.get_defaults(n):   653             if default:   654    655                 # Obtain any reference for the default.   656    657                 name_ref = self.process_structure_node(default)   658                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   659    660         # Reset conditional tracking to focus on the function contents.   661    662         in_conditional = self.in_conditional   663         self.in_conditional = False   664    665         in_function = self.in_function   666         self.in_function = function_name   667    668         self.enter_namespace(name)   669    670         # Define a leafname attribute value for the function instance.   671    672         ref = self.get_builtin_class("string")   673         self.reserve_constant(function_name, name, ref.get_origin())   674    675         # Track attribute usage within the namespace.   676    677         path = self.get_namespace_path()   678         self.start_tracking(locals)   679    680         # Establish attributes for .name entries, provided this is a method.   681    682         for argname in attr_initialisers:   683             self.process_assignment_node(   684                     compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"),   685                     compiler.ast.Name(argname))   686    687         self.process_structure_node(n.code)   688         returns_value = self.stop_tracking()   689    690         # Record any null result.   691    692         is_initialiser = is_method and name == "__init__"   693    694         if not returns_value and not is_initialiser:   695             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   696    697         # Exit to the parent.   698    699         self.exit_namespace()   700    701         # Update flags.   702    703         self.in_function = in_function   704         self.in_conditional = in_conditional   705    706         # Define the function using the appropriate name.   707    708         self.set_definition(name, "<function>")   709    710         # Where a function is set conditionally, assign the name.   711    712         if original_name:   713             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   714    715     def process_global_node(self, n):   716    717         """   718         Process the given "global" node 'n'.   719         """   720    721         path = self.get_namespace_path()   722    723         if path != self.name:   724             self.scope_globals[path].update(n.names)   725    726     def process_if_node(self, n):   727    728         """   729         Process the given "if" node 'n'.   730         """   731    732         tracker = self.trackers[-1]   733         tracker.new_branchpoint()   734    735         for test, body in n.tests:   736             self.process_structure_node(test)   737    738             tracker.new_branch()   739    740             in_conditional = self.in_conditional   741             self.in_conditional = True   742             self.process_structure_node(body)   743             self.in_conditional = in_conditional   744    745             tracker.shelve_branch()   746    747         # Maintain a branch for the else clause.   748    749         tracker.new_branch()   750         if n.else_:   751             self.process_structure_node(n.else_)   752         tracker.shelve_branch()   753    754         tracker.merge_branches()   755    756     def process_import_node(self, n):   757    758         "Process the given import node 'n'."   759    760         path = self.get_namespace_path()   761    762         # Load the mentioned module.   763    764         for name, alias in n.names:   765             if name == self.name:   766                 raise InspectError("Cannot import the current module.", path, n)   767    768             self.set_module(alias or name.split(".")[-1], name)   769             self.queue_module(name, True)   770    771     def process_invocation_node(self, n):   772    773         "Process the given invocation node 'n'."   774    775         path = self.get_namespace_path()   776    777         in_invocation = self.in_invocation   778         self.in_invocation = None   779    780         # Process the arguments.   781    782         keywords = set()   783    784         for arg in n.args:   785             self.process_structure_node(arg)   786             if isinstance(arg, compiler.ast.Keyword):   787                 keywords.add(arg.name)   788    789         keywords = list(keywords)   790         keywords.sort()   791    792         # Communicate to the invocation target expression that it forms the   793         # target of an invocation, potentially affecting attribute accesses.   794    795         self.in_invocation = len(n.args), keywords   796    797         # Process the expression, obtaining any identified reference.   798    799         name_ref = self.process_structure_node(n.node)   800         self.in_invocation = in_invocation   801    802         # Detect class invocations.   803    804         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   805             return InstanceRef(name_ref.reference().instance_of())   806    807         elif isinstance(name_ref, (NameRef, AccessRef)):   808             return InvocationRef(name_ref)   809    810         # Provide a general reference to indicate that something is produced   811         # by the invocation, useful for retaining assignment expression   812         # details.   813    814         return VariableRef()   815    816     def process_lambda_node(self, n):   817    818         "Process the given lambda node 'n'."   819    820         name = self.get_lambda_name()   821         self.process_function_node(n, name)   822    823         origin = self.get_object_path(name)   824    825         if self.function_defaults.get(origin):   826             return None   827         else:   828             return ResolvedNameRef(name, Reference("<function>", origin))   829    830     def process_logical_node(self, n):   831    832         "Process the given operator node 'n'."   833    834         return self.process_operator_chain(n.nodes)   835    836     def process_name_node(self, n):   837    838         "Process the given name node 'n'."   839    840         path = self.get_namespace_path()   841    842         # Find predefined constant names before anything else.   843    844         if n.name in predefined_constants:   845             ref = self.get_builtin(n.name)   846             value = ResolvedNameRef(n.name, ref)   847             return value   848    849         # Special names that have already been identified.   850    851         if n.name.startswith("$"):   852             value = self.get_special(n.name)   853             if value:   854                 return value   855    856         # Special case for operator functions introduced through code   857         # transformations.   858    859         if n.name.startswith("$op"):   860    861             # Obtain the location of the actual function defined in the operator   862             # package.   863    864             op = n.name[len("$op"):]   865    866             # Attempt to get a reference.   867    868             ref = self.import_name_from_module(op, "operator")   869    870             # Record the imported name and provide the resolved name reference.   871    872             value = ResolvedNameRef(n.name, ref)   873             self.set_special(n.name, value)   874             return value   875    876         # Special case for print operations.   877    878         elif n.name.startswith("$print"):   879    880             # Attempt to get a reference.   881    882             ref = self.get_builtin("print_")   883    884             # Record the imported name and provide the resolved name reference.   885    886             value = ResolvedNameRef(n.name, ref)   887             self.set_special(n.name, value)   888             return value   889    890         # Test for self usage, which is only allowed in methods.   891    892         if n.name == "self" and not (self.in_function and self.in_class):   893             raise InspectError("Use of self is only allowed in methods.", path, n)   894    895         # Record usage of the name.   896    897         self.record_name(n.name)   898    899         # Search for unknown names in non-function scopes immediately.   900         # Temporary names should not be re-used between scopes.   901         # External names in functions are resolved later.   902    903         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   904    905         if ref:   906             self.record_name_access(n.name, True)   907             return ResolvedNameRef(n.name, ref, is_global=True)   908    909         # Explicitly-declared global names.   910    911         elif self.in_function and n.name in self.scope_globals[path]:   912             self.record_name_access(n.name, True)   913             return NameRef(n.name, is_global=True)   914    915         # Examine other names.   916    917         else:   918    919             # Check local names.   920    921             access_number = self.record_name_access(n.name)   922    923             # Local name.   924    925             if access_number is not None:   926                 return LocalNameRef(n.name, access_number)   927    928             # Possible global or built-in name.   929    930             else:   931                 self.record_name_access(n.name, True)   932                 return NameRef(n.name, is_global=True)   933    934     def record_name_access(self, name, is_global=False):   935    936         """   937         Record an access involving 'name' if the name is being tracked, using   938         'is_global' to indicate whether the name is global.   939         """   940    941         name = self.get_name_for_tracking(name, is_global=is_global)   942         branches = self.trackers[-1].tracking_name(name)   943         if branches:   944             self.record_branches_for_access(branches, name, None)   945             return self.record_access_details(name, None, self.in_assignment,   946                                               self.in_invocation)   947         return None   948    949     def process_operator_chain(self, nodes):   950    951         """   952         Process the given chain of 'nodes', processing each node or item.   953         Each node starts a new conditional region, effectively making a deeply-   954         nested collection of if-like statements.   955         """   956    957         results = []   958    959         tracker = self.trackers[-1]   960    961         for item in nodes:   962             tracker.new_branchpoint()   963             tracker.new_branch()   964             result = self.process_structure_node(item)   965             if result:   966                 results.append(result)   967    968         for item in nodes[:-1]:   969             tracker.shelve_branch()   970             tracker.new_branch()   971             tracker.shelve_branch()   972             tracker.merge_branches()   973    974         tracker.shelve_branch()   975         tracker.merge_branches()   976    977         return MultipleRef(results)   978    979     def process_try_node(self, n):   980    981         """   982         Process the given "try...except" node 'n'.   983         """   984    985         self.record_exception_handler()   986    987         tracker = self.trackers[-1]   988         tracker.new_branchpoint()   989    990         self.process_structure_node(n.body)   991    992         for name, var, handler in n.handlers:   993             if name is not None:   994                 self.process_structure_node(name)   995    996             # Any abandoned branches from the body can now be resumed in a new   997             # branch.   998    999             tracker.resume_abandoned_branches()  1000   1001             # Establish the local for the handler.  1002   1003             if var is not None:  1004                 self.process_assignment_node(var, None)  1005             if handler is not None:  1006                 self.process_structure_node(handler)  1007   1008             tracker.shelve_branch()  1009   1010         # The else clause maintains the usage from the body but without the  1011         # abandoned branches since they would never lead to the else clause  1012         # being executed.  1013   1014         if n.else_:  1015             tracker.new_branch()  1016             self.process_structure_node(n.else_)  1017             tracker.shelve_branch()  1018   1019         # Without an else clause, a null branch propagates the successful  1020         # outcome.  1021   1022         else:  1023             tracker.new_branch()  1024             tracker.shelve_branch()  1025   1026         tracker.merge_branches()  1027   1028     def process_try_finally_node(self, n):  1029   1030         """  1031         Process the given "try...finally" node 'n'.  1032         """  1033   1034         self.record_exception_handler()  1035   1036         tracker = self.trackers[-1]  1037         self.process_structure_node(n.body)  1038   1039         # Any abandoned branches from the body can now be resumed.  1040   1041         branches = tracker.resume_all_abandoned_branches()  1042         self.process_structure_node(n.final)  1043   1044         # At the end of the finally clause, abandoned branches are discarded.  1045   1046         tracker.restore_active_branches(branches)  1047   1048     def process_while_node(self, n):  1049   1050         "Process the given while node 'n'."  1051   1052         tracker = self.trackers[-1]  1053         tracker.new_branchpoint(loop_node=True)  1054   1055         # Evaluate any test or iterator outside the loop.  1056   1057         self.process_structure_node(n.test)  1058   1059         # Propagate attribute usage to branches.  1060   1061         tracker.new_branch(loop_node=True)  1062   1063         # Enter the loop.  1064   1065         in_conditional = self.in_conditional  1066         self.in_conditional = True  1067         self.process_structure_node(n.body)  1068         self.in_conditional = in_conditional  1069   1070         # Continuing branches are resumed before any test.  1071   1072         tracker.resume_continuing_branches()  1073   1074         # Evaluate any continuation test within the body.  1075   1076         self.process_structure_node(n.test)  1077   1078         tracker.shelve_branch(loop_node=True)  1079   1080         # Support the non-looping condition.  1081   1082         tracker.new_branch()  1083         tracker.shelve_branch()  1084   1085         tracker.merge_branches()  1086   1087         # Evaluate any else clause outside branches.  1088   1089         if n.else_:  1090             self.process_structure_node(n.else_)  1091   1092         # Connect broken branches to the code after any loop.  1093   1094         tracker.resume_broken_branches()  1095   1096     # Branch tracking methods.  1097   1098     def start_tracking(self, names):  1099   1100         """  1101         Start tracking attribute usage for names in the current namespace,  1102         immediately registering the given 'names'.  1103         """  1104   1105         path = self.get_namespace_path()  1106         parent = self.trackers[-1]  1107         tracker = BranchTracker()  1108         self.trackers.append(tracker)  1109   1110         # Record the given names established as new branches.  1111   1112         tracker.assign_names(names)  1113   1114     def assign_name(self, name, name_ref):  1115   1116         "Assign to 'name' the given 'name_ref' in the current namespace."  1117   1118         name = self.get_name_for_tracking(name)  1119         self.trackers[-1].assign_names([name], [name_ref])  1120   1121     def stop_tracking(self):  1122   1123         """  1124         Stop tracking attribute usage, recording computed usage for the current  1125         namespace. Indicate whether a value is always returned from the  1126         namespace.  1127         """  1128   1129         path = self.get_namespace_path()  1130         tracker = self.trackers.pop()  1131         self.record_assignments_for_access(tracker)  1132   1133         self.attr_usage[path] = tracker.get_all_usage()  1134         self.name_initialisers[path] = tracker.get_all_values()  1135   1136         return tracker.returns_value()  1137   1138     def start_tracking_in_module(self):  1139   1140         "Start tracking attribute usage in the module."  1141   1142         tracker = BranchTracker()  1143         self.trackers.append(tracker)  1144   1145     def stop_tracking_in_module(self):  1146   1147         "Stop tracking attribute usage in the module."  1148   1149         tracker = self.trackers[0]  1150         self.record_assignments_for_access(tracker)  1151         self.attr_usage[self.name] = tracker.get_all_usage()  1152         self.name_initialisers[self.name] = tracker.get_all_values()  1153   1154     def record_assignments_for_access(self, tracker):  1155   1156         """  1157         For the current path, use the given 'tracker' to record assignment  1158         version information for attribute accesses.  1159         """  1160   1161         path = self.get_path_for_access()  1162   1163         if not self.attr_accessor_branches.has_key(path):  1164             return  1165   1166         init_item(self.attr_accessors, path, dict)  1167         attr_accessors = self.attr_accessors[path]  1168   1169         # Obtain the branches applying during each access.  1170   1171         for access, all_branches in self.attr_accessor_branches[path].items():  1172             name, attrnames = access  1173             init_item(attr_accessors, access, list)  1174   1175             # Obtain the assignments applying to each branch.  1176   1177             for branches in all_branches:  1178                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1179   1180                 # Detect missing name information.  1181   1182                 if None in positions:  1183                     globals = self.global_attr_accesses.get(path)  1184                     accesses = globals and globals.get(name)  1185                     if not accesses:  1186                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1187                             self.get_namespace_path(), name)  1188                     positions.remove(None)  1189   1190                 attr_accessors[access].append(positions)  1191   1192     def record_branches_for_access(self, branches, name, attrnames):  1193   1194         """  1195         Record the given 'branches' for an access involving the given 'name' and  1196         'attrnames'.  1197         """  1198   1199         access = name, attrnames  1200         path = self.get_path_for_access()  1201   1202         init_item(self.attr_accessor_branches, path, dict)  1203         attr_accessor_branches = self.attr_accessor_branches[path]  1204   1205         init_item(attr_accessor_branches, access, list)  1206         attr_accessor_branches[access].append(branches)  1207   1208     def record_access_details(self, name, attrnames, assignment, invocation):  1209   1210         """  1211         For the given 'name' and 'attrnames', record an access indicating  1212         whether an 'assignment' or an 'invocation' is occurring.  1213   1214         These details correspond to accesses otherwise recorded by the attribute  1215         accessor and attribute access dictionaries.  1216         """  1217   1218         access = name, attrnames  1219         path = self.get_path_for_access()  1220   1221         init_item(self.attr_access_modifiers, path, dict)  1222         init_item(self.attr_access_modifiers[path], access, list)  1223   1224         access_number = len(self.attr_access_modifiers[path][access])  1225         self.attr_access_modifiers[path][access].append((assignment, invocation))  1226         return access_number  1227   1228     def record_global_access_details(self, name, attrnames):  1229   1230         """  1231         Record details of a global access via the given 'name' involving the  1232         indicated 'attrnames'.  1233         """  1234   1235         path = self.get_namespace_path()  1236   1237         init_item(self.global_attr_accesses, path, dict)  1238         init_item(self.global_attr_accesses[path], name, set)  1239         self.global_attr_accesses[path][name].add(attrnames)  1240   1241     # Namespace modification.  1242   1243     def record_name(self, name):  1244   1245         "Record the use of 'name' in a namespace."  1246   1247         path = self.get_namespace_path()  1248         init_item(self.names_used, path, set)  1249         self.names_used[path].add(name)  1250   1251     def set_module(self, name, module_name):  1252   1253         """  1254         Set a module in the current namespace using the given 'name' associated  1255         with the corresponding 'module_name'.  1256         """  1257   1258         if name:  1259             self.set_general_local(name, Reference("<module>", module_name))  1260   1261     def set_definition(self, name, kind):  1262   1263         """  1264         Set the definition having the given 'name' and 'kind'.  1265   1266         Definitions are set in the static namespace hierarchy, but they can also  1267         be recorded for function locals.  1268         """  1269   1270         if self.is_global(name):  1271             print >>sys.stderr, "In %s, %s is defined as being global." % (  1272                 self.get_namespace_path(), name)  1273   1274         path = self.get_object_path(name)  1275         self.set_object(path, kind)  1276   1277         ref = self.get_object(path)  1278         if ref.get_kind() == "<var>":  1279             print >>sys.stderr, "In %s, %s is defined more than once." % (  1280                 self.get_namespace_path(), name)  1281   1282         if not self.is_global(name) and self.in_function:  1283             self.set_function_local(name, ref)  1284   1285     def set_function_local(self, name, ref=None):  1286   1287         "Set the local with the given 'name' and optional 'ref'."  1288   1289         path = self.get_namespace_path()  1290         locals = self.function_locals[path]  1291         used = self.names_used.get(path)  1292   1293         if not locals.has_key(name) and used and name in used:  1294             raise InspectError("Name %s assigned locally but used previously." % name, path)  1295   1296         multiple = not ref or locals.has_key(name) and locals[name] != ref  1297         locals[name] = multiple and Reference("<var>") or ref  1298   1299     def assign_general_local(self, name, name_ref):  1300   1301         """  1302         Set for 'name' the given 'name_ref', recording the name for attribute  1303         usage tracking.  1304         """  1305   1306         self.set_general_local(name, name_ref)  1307         self.assign_name(name, name_ref)  1308   1309     def set_general_local(self, name, value=None):  1310   1311         """  1312         Set the 'name' with optional 'value' in any kind of local namespace,  1313         where the 'value' should be a reference if specified.  1314         """  1315   1316         init_value = self.get_initialising_value(value)  1317   1318         # Module global names.  1319   1320         if self.is_global(name):  1321             path = self.get_global_path(name)  1322             self.set_object(path, init_value)  1323   1324         # Function local names.  1325   1326         elif self.in_function:  1327             self.set_function_local(name, init_value)  1328   1329         # Other namespaces (classes).  1330   1331         else:  1332             self.set_name(name, init_value)  1333   1334     def set_name(self, name, ref=None):  1335   1336         "Attach the 'name' with optional 'ref' to the current namespace."  1337   1338         self.set_object(self.get_object_path(name), ref)  1339   1340     def set_instance_attr(self, name, ref=None):  1341   1342         """  1343         Add an instance attribute of the given 'name' to the current class,  1344         using the optional 'ref'.  1345         """  1346   1347         self._set_instance_attr(self.in_class, name, ref)  1348   1349     def _set_instance_attr(self, path, name, ref=None):  1350   1351         init_item(self.instance_attrs, path, set)  1352         self.instance_attrs[path].add(name)  1353   1354         if ref:  1355             init_item(self.instance_attr_constants, path, dict)  1356             self.instance_attr_constants[path][name] = ref  1357   1358     def get_initialising_value(self, value):  1359   1360         "Return a suitable initialiser reference for 'value'."  1361   1362         if isinstance(value, Result):  1363             return value.reference()  1364         else:  1365             return value  1366   1367     # Static, program-relative naming.  1368   1369     def find_name(self, name):  1370   1371         """  1372         Return the qualified name for the given 'name' used in the current  1373         non-function namespace.  1374         """  1375   1376         path = self.get_namespace_path()  1377         ref = None  1378   1379         if not self.in_function and name not in predefined_constants:  1380             if self.in_class:  1381                 ref = self.get_object(self.get_object_path(name), False)  1382             if not ref:  1383                 ref = self.get_global_or_builtin(name)  1384   1385         return ref  1386   1387     def get_class(self, node):  1388   1389         """  1390         Use the given 'node' to obtain the identity of a class. Return a  1391         reference for the class. Unresolved dependencies are permitted and must  1392         be resolved later.  1393         """  1394   1395         ref = self._get_class(node)  1396         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1397   1398     def _get_class(self, node):  1399   1400         """  1401         Use the given 'node' to find a class definition. Return a reference to  1402         the class.  1403         """  1404   1405         if isinstance(node, compiler.ast.Getattr):  1406   1407             # Obtain the identity of the access target.  1408   1409             ref = self._get_class(node.expr)  1410   1411             # Where the target is a class or module, obtain the identity of the  1412             # attribute.  1413   1414             if ref.has_kind(["<function>", "<var>"]):  1415                 return None  1416             else:  1417                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1418                 return self.get_object(attrname)  1419   1420         # Names can be module-level or built-in.  1421   1422         elif isinstance(node, compiler.ast.Name):  1423   1424             # Record usage of the name and attempt to identify it.  1425   1426             self.record_name(node.name)  1427             return self.find_name(node.name)  1428         else:  1429             return None  1430   1431     def get_constant(self, name, value):  1432   1433         "Return a constant reference for the given type 'name' and 'value'."  1434   1435         ref = self.get_builtin_class(name)  1436         return self.get_constant_reference(ref, value)  1437   1438     def get_literal_instance(self, n, name=None):  1439   1440         """  1441         For node 'n', return a reference to an instance of 'name', or if 'name'  1442         is not specified, deduce the type from the value.  1443         """  1444   1445         # Handle stray None constants (Sliceobj seems to produce them).  1446   1447         if name == "NoneType":  1448             return self.process_name_node(compiler.ast.Name("None"))  1449   1450         # Obtain the details of the literal itself.  1451         # An alias to the type is generated for sequences.  1452   1453         if name in ("dict", "list", "tuple"):  1454             ref = self.get_builtin_class(name)  1455             self.set_special_literal(name, ref)  1456             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1457   1458         # Constant values are independently recorded.  1459   1460         else:  1461             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1462             ref = self.get_builtin_class(typename)  1463             return self.get_constant_reference(ref, value, encoding)  1464   1465     # Special names.  1466   1467     def get_special(self, name):  1468   1469         "Return any stored value for the given special 'name'."  1470   1471         value = self.special.get(name)  1472         if value:  1473             ref, paths = value  1474         else:  1475             ref = None  1476         return ref  1477   1478     def set_special(self, name, value):  1479   1480         """  1481         Set a special 'name' that merely tracks the use of an implicit object  1482         'value'.  1483         """  1484   1485         if not self.special.has_key(name):  1486             paths = set()  1487             self.special[name] = value, paths  1488         else:  1489             _ref, paths = self.special[name]  1490   1491         paths.add(self.get_namespace_path())  1492   1493     def set_special_literal(self, name, ref):  1494   1495         """  1496         Set a special name for the literal type 'name' having type 'ref'. Such  1497         special names provide a way of referring to literal object types.  1498         """  1499   1500         literal_name = "$L%s" % name  1501         value = ResolvedNameRef(literal_name, ref)  1502         self.set_special(literal_name, value)  1503   1504     # Exceptions.  1505   1506     def record_exception_handler(self):  1507   1508         "Record the current namespace as employing an exception handler."  1509   1510         self.exception_namespaces.add(self.get_namespace_path())  1511   1512     # Return values.  1513   1514     def record_return_value(self, expr):  1515   1516         "Record the given return 'expr'."  1517   1518         path = self.get_namespace_path()  1519         l = init_item(self.return_values, path, list)  1520         l.append(expr)  1521         if not self.importer.all_return_values.has_key(path):  1522             self.importer.all_return_values[path] = l  1523   1524 # vim: tabstop=4 expandtab shiftwidth=4