Lichen

importer.py

838:00fd67211882
2018-07-03 Paul Boddie Take advantage of context testing when acquiring functions for invocation. Since a context whose nature has been determined will have been tested, either as a consequence of deduction or at run-time, use of __get_function (which tests the context again) is unnecessary.
     1 #!/usr/bin/env python     2      3 """     4 Import logic.     5      6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from errors import ProgramError    24 from os.path import exists, extsep, getmtime, join    25 from os import listdir, makedirs, remove    26 from common import init_item, order_dependencies, readfile, writefile    27 from modules import CachedModule    28 from referencing import Reference    29 import inspector    30 import sys    31     32 class Importer:    33     34     "An import machine, searching for and loading modules."    35     36     special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__")    37     38     def __init__(self, path, cache=None, verbose=False, warnings=None):    39     40         """    41         Initialise the importer with the given search 'path' - a list of    42         directories to search for Python modules.    43     44         The optional 'cache' should be the name of a directory used to store    45         cached module information.    46     47         The optional 'verbose' parameter causes output concerning the activities    48         of the object to be produced if set to a true value (not the default).    49     50         The optional 'warnings' parameter may indicate classes of warnings to be    51         produced.    52         """    53     54         self.path = path    55         self.cache = cache    56         self.verbose = verbose    57         self.warnings = warnings    58     59         # Module importing queue, required modules, removed modules and active    60         # modules in the final program.    61     62         self.to_import = set()    63         self.required = set(["__main__"])    64         self.removed = {}    65         self.modules = {}    66     67         # Module relationships and invalidated cached modules.    68     69         self.accessing_modules = {}    70         self.invalidated = set()    71     72         # Object relationships and dependencies.    73     74         self.depends = {}    75         self.module_depends = {}    76     77         # Basic program information.    78     79         self.objects = {}    80         self.classes = {}    81         self.function_parameters = {}    82         self.function_attr_initialisers = {}    83         self.function_defaults = {}    84         self.function_locals = {}    85     86         # Unresolved names.    87     88         self.missing = set()    89     90         # Derived information.    91     92         self.subclasses = {}    93     94         # Attributes of different object types.    95     96         self.all_class_attrs = {}    97         self.all_instance_attrs = {}    98         self.all_instance_attr_constants = {}    99         self.all_combined_attrs = {}   100         self.all_module_attrs = {}   101         self.all_shadowed_attrs = {}   102    103         # References to external names and aliases within program units.   104    105         self.all_name_references = {}   106         self.all_initialised_names = {}   107         self.all_aliased_names = {}   108    109         # General attribute accesses.   110    111         self.all_attr_accesses = {}   112         self.all_const_accesses = {}   113         self.all_attr_access_modifiers = {}   114    115         # Constant literals and values.   116    117         self.all_constants = {}   118         self.all_constant_values = {}   119    120         # Common function value collection used during inspection.   121    122         self.all_return_values = {}   123    124         self.make_cache()   125    126     def give_warning(self, name):   127    128         "Return whether the indicated warning 'name' should be given."   129    130         return self.warnings and (name in self.warnings or "all" in self.warnings)   131    132     def make_cache(self):   133    134         "Make a cache directory if it does not already exist."   135    136         if self.cache and not exists(self.cache):   137             makedirs(self.cache)   138    139     def check_cache(self, details):   140    141         """   142         Check whether the cache applies for the given 'details', invalidating it   143         if it does not.   144         """   145    146         recorded_details = self.get_cache_details()   147    148         if recorded_details != details:   149             self.remove_cache()   150    151         writefile(self.get_cache_details_filename(), details)   152    153     def get_cache_details_filename(self):   154    155         "Return the filename for the cache details."   156    157         return join(self.cache, "$details")   158    159     def get_cache_details(self):   160    161         "Return details of the cache."   162    163         details_filename = self.get_cache_details_filename()   164    165         if not exists(details_filename):   166             return None   167         else:   168             return readfile(details_filename)   169    170     def remove_cache(self):   171    172         "Remove the contents of the cache."   173    174         for filename in listdir(self.cache):   175             remove(join(self.cache, filename))   176    177     def to_cache(self):   178    179         "Write modules to the cache."   180    181         if self.cache:   182             for module_name, module in self.modules.items():   183                 module.to_cache(join(self.cache, module_name))   184    185     # Object retrieval and storage.   186    187     def get_object(self, name):   188    189         """   190         Return a reference for the given 'name' or None if no such object   191         exists.   192         """   193    194         return self.objects.get(name)   195    196     def set_object(self, name, value=None):   197    198         "Set the object with the given 'name' and the given 'value'."   199    200         if isinstance(value, Reference):   201             ref = value.alias(name)   202         else:   203             ref = Reference(value, name)   204    205         self.objects[name] = ref   206    207     # Identification of both stored object names and name references.   208    209     def identify(self, name):   210    211         "Identify 'name' using stored object and external name records."   212    213         return self.objects.get(name) or self.all_name_references.get(name)   214    215     # Indirect object retrieval.   216    217     def get_attributes(self, ref, attrname):   218    219         """   220         Return attributes provided by 'ref' for 'attrname'. Class attributes   221         may be provided by instances.   222         """   223    224         kind = ref.get_kind()   225         if kind == "<class>":   226             ref = self.get_class_attribute(ref.get_origin(), attrname)   227             return ref and set([ref]) or set()   228         elif kind == "<instance>":   229             return self.get_combined_attributes(ref.get_origin(), attrname)   230         elif kind == "<module>":   231             ref = self.get_module_attribute(ref.get_origin(), attrname)   232             return ref and set([ref]) or set()   233         else:   234             return set()   235    236     def get_class_attribute(self, object_type, attrname):   237    238         "Return from 'object_type' the details of class attribute 'attrname'."   239    240         attrs = self.all_class_attrs.get(object_type)   241         attr = attrs and attrs.get(attrname)   242         return attr and self.get_object(attr)   243    244     def get_instance_attributes(self, object_type, attrname):   245    246         """   247         Return from 'object_type' the details of instance attribute 'attrname'.   248         """   249    250         consts = self.all_instance_attr_constants.get(object_type)   251         attrs = set()   252         for attr in self.all_instance_attrs[object_type].get(attrname, []):   253             attrs.add(consts and consts.get(attrname) or Reference("<var>", attr))   254         return attrs   255    256     def get_combined_attributes(self, object_type, attrname):   257    258         """   259         Return from 'object_type' the details of class or instance attribute   260         'attrname'.   261         """   262    263         ref = self.get_class_attribute(object_type, attrname)   264         refs = ref and set([ref]) or set()   265         refs.update(self.get_instance_attributes(object_type, attrname))   266         return refs   267    268     def get_module_attribute(self, object_type, attrname):   269    270         "Return from 'object_type' the details of module attribute 'attrname'."   271    272         if attrname in self.all_module_attrs[object_type]:   273             return self.get_object("%s.%s" % (object_type, attrname))   274         else:   275             return None   276    277     # Convenience methods for deducing which kind of object provided an   278     # attribute.   279    280     def get_attribute_provider(self, ref, attrname):   281    282         """   283         Return the kind of provider of the attribute accessed via 'ref' using   284         'attrname'.   285         """   286    287         kind = ref.get_kind()   288    289         if kind in ["<class>", "<module>"]:   290             return kind   291         else:   292             return self.get_instance_attribute_provider(ref.get_origin(), attrname)   293    294     def get_instance_attribute_provider(self, object_type, attrname):   295    296         """   297         Return the kind of provider of the attribute accessed via an instance of   298         'object_type' using 'attrname'.   299         """   300    301         if self.get_class_attribute(object_type, attrname):   302             return "<class>"   303         else:   304             return "<instance>"   305    306     # Module management.   307    308     def queue_module(self, name, accessor, required=False):   309    310         """   311         Queue the module with the given 'name' for import from the given   312         'accessor' module. If 'required' is true (it is false by default), the   313         module will be required in the final program.   314         """   315    316         if not self.modules.has_key(name):   317             self.to_import.add(name)   318    319         if required:   320             self.required.add(name)   321    322         init_item(self.accessing_modules, name, set)   323         self.accessing_modules[name].add(accessor.name)   324    325     def get_modules(self):   326    327         "Return all modules known to the importer."   328    329         return self.modules.values()   330    331     def get_module(self, name):   332    333         "Return the module with the given 'name'."   334    335         if not self.modules.has_key(name):   336             return None   337    338         return self.modules[name]   339    340     # Program operations.   341    342     def initialise(self, filename, reset=False):   343    344         """   345         Initialise a program whose main module is 'filename', resetting the   346         cache if 'reset' is true. Return the main module.   347         """   348    349         if reset:   350             self.remove_cache()   351         self.check_cache(filename)   352    353         # Load the program itself.   354    355         m = self.load_from_file(filename)   356    357         # Load any queued modules.   358    359         while self.to_import:   360             for name in list(self.to_import): # avoid mutation issue   361                 self.load(name)   362    363         # Resolve dependencies between modules.   364    365         self.resolve()   366    367         # Record the type of all classes.   368    369         self.type_ref = self.get_object("__builtins__.type")   370    371         # Resolve dependencies within the program.   372    373         for module in self.modules.values():   374             module.complete()   375    376         # Remove unneeded modules.   377    378         all_modules = self.modules.items()   379    380         for name, module in all_modules:   381             if name not in self.required:   382                 module.unpropagate()   383                 del self.modules[name]   384                 self.removed[name] = module   385    386         # Collect redundant objects.   387    388         for module in self.removed.values():   389             module.collect()   390    391         # Assert module objects where aliases have been removed.   392    393         for name in self.required:   394             if not self.objects.has_key(name):   395                 self.objects[name] = Reference("<module>", name)   396    397         return m   398    399     def finalise(self):   400    401         """   402         Finalise the inspected program, returning whether the program could be   403         finalised.   404         """   405    406         self.finalise_classes()   407         self.add_init_dependencies()   408         self.to_cache()   409    410         if self.missing:   411             return False   412    413         self.set_class_types()   414         self.define_instantiators()   415         self.collect_constants()   416    417         return True   418    419     # Supporting operations.   420    421     def resolve(self):   422    423         "Resolve dependencies between modules."   424    425         self.waiting = {}   426    427         for module in self.modules.values():   428    429             # Resolve all deferred references in each module.   430    431             original_deferred = []   432    433             for ref in module.deferred:   434    435                 # Retain original references for caching.   436    437                 original_deferred.append(ref.copy())   438    439                 # Update references throughout the program.   440    441                 found = self.find_dependency(ref)   442                 if not found:   443                     self.missing.add((module.name, ref.get_origin()))   444    445                 # Record the resolved names and identify required modules.   446    447                 else:   448                     # Find the providing module of this reference.   449                     # Where definitive details of the origin cannot be found,   450                     # identify the provider using the deferred reference.   451                     # NOTE: This may need to test for static origins.   452    453                     provider = self.get_module_provider(found.unresolved() and ref or found)   454                     ref.mutate(found)   455    456                     # Record any external dependency.   457    458                     if provider and provider != module.name:   459    460                         # Handle built-in modules accidentally referenced by   461                         # names.   462    463                         if provider == "__builtins__" and found.has_kind("<module>"):   464                             raise ProgramError("Name %s, used by %s, refers to module %s." %   465                                                (found.leaf(), module.name, found.get_origin()))   466    467                         # Record the provider dependency.   468    469                         module.required.add(provider)   470                         self.accessing_modules[provider].add(module.name)   471    472                         # Postpone any inclusion of the provider until this   473                         # module becomes required.   474    475                         if module.name not in self.required:   476                             init_item(self.waiting, module.name, set)   477                             self.waiting[module.name].add(provider)   478                             if self.verbose:   479                                 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name   480    481                         # Make this module required in the accessing module.   482    483                         elif provider not in self.required:   484                             self.required.add(provider)   485                             if self.verbose:   486                                 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name   487    488                         # Record a module ordering dependency.   489    490                         if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found):   491                             self.add_module_dependency(module.name, provider)   492    493             # Restore the original references so that they may be read back in   494             # and produce the same results.   495    496             module.deferred = original_deferred   497    498         # Check modules again to see if they are now required and should now   499         # cause the inclusion of other modules providing objects to the program.   500    501         for module_name in self.waiting.keys():   502             self.require_providers(module_name)   503    504         self.add_special_dependencies()   505         self.add_module_dependencies()   506    507     def require_providers(self, module_name):   508    509         """   510         Test if 'module_name' is itself required and, if so, require modules   511         containing objects provided to the module.   512         """   513    514         if module_name in self.required and self.waiting.has_key(module_name):   515             for provider in self.waiting[module_name]:   516                 if provider not in self.required:   517                     self.required.add(provider)   518                     if self.verbose:   519                         print >>sys.stderr, "Requiring", provider   520                     self.require_providers(provider)   521    522     def add_special_dependencies(self):   523    524         "Add dependencies due to the use of special names in namespaces."   525    526         for module in self.modules.values():   527             for ref, paths in module.special.values():   528                 for path in paths:   529                     self.add_dependency(path, ref.get_origin())   530    531     def add_init_dependencies(self):   532    533         "Add dependencies related to object initialisation."   534    535         for name in self.classes.keys():   536             if self.is_dynamic_class(name):   537    538                 # Make subclasses depend on any class with non-static   539                 # attributes, plus its module for the initialisation.   540    541                 for subclass in self.subclasses[name]:   542                     ref = Reference("<class>", subclass)   543                     self.add_dependency(subclass, name)   544                     self.add_dependency(subclass, self.get_module_provider(ref))   545    546                 # Also make the class dependent on its module for   547                 # initialisation.   548    549                 ref = Reference("<class>", name)   550                 self.add_dependency(name, self.get_module_provider(ref))   551    552         for name in self.function_defaults.keys():   553             if self.is_dynamic_callable(name):   554    555                 # Make functions with defaults requiring initialisation depend   556                 # on the parent scope, if a function, or the module scope.   557    558                 ref = Reference("<function>", name)   559                 parent_ref = self.get_object(ref.parent())   560    561                 # Function no longer present in the program.   562    563                 if not parent_ref:   564                     continue   565    566                 if parent_ref.has_kind("<class>"):   567                     parent = self.get_module_provider(parent_ref)   568                 else:   569                     parent = parent_ref.get_origin()   570    571                 self.add_dependency(name, parent)   572    573     def add_module_dependencies(self):   574    575         "Record module-based dependencies."   576    577         for module_name, providers in self.module_depends.items():   578             if self.modules.has_key(module_name):   579                 for provider in providers:   580                     if self.modules.has_key(provider):   581                         self.add_dependency(module_name, provider)   582    583     def add_dependency(self, path, origin):   584    585         "Add dependency details for 'path' involving 'origin'."   586    587         if origin and not origin.startswith("%s." % path):   588             init_item(self.depends, path, set)   589             self.depends[path].add(origin)   590    591     def add_module_dependency(self, module_name, provider):   592    593         "Add dependency details for 'module_name' involving 'provider'."   594    595         if provider:   596             init_item(self.module_depends, module_name, set)   597             self.module_depends[module_name].add(provider)   598    599     def condense_dependencies(self):   600    601         """   602         Condense the dependencies by removing all functions that do not need   603         initialisation.   604         """   605    606         d = {}   607         for path, depends in self.depends.items():   608             d[path] = {}   609             d[path] = self.condense_dependency_entry(depends, d)   610    611         self.depends = {}   612    613         for path, depends in d.items():   614             if depends:   615                 self.depends[path] = depends   616    617     def condense_dependency_entry(self, depends, d):   618         l = set()   619         for depend in depends:   620             if self.modules.has_key(depend) or self.classes.has_key(depend) or \   621                self.is_dynamic_callable(depend):   622    623                 l.add(depend)   624             else:   625                 deps = d.get(depend)   626                 if deps:   627                     l.update(self.condense_dependency_entry(deps, d))   628         return l   629    630     def is_dynamic(self, ref):   631         return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value()   632    633     def is_dynamic_class(self, name):   634    635         """   636         Return whether 'name' refers to a class with members that must be   637         initialised dynamically.   638         """   639    640         attrs = self.all_class_attrs.get(name)   641    642         if not attrs:   643             return False   644    645         for attrname, attr in attrs.items():   646             if attrname in self.special_attributes:   647                 continue   648             ref = attr and self.get_object(attr)   649             if self.is_dynamic(ref):   650                 return True   651    652         return False   653    654     def is_dynamic_callable(self, name):   655    656         """   657         Return whether 'name' refers to a callable employing defaults that may   658         need initialising before the callable can be used.   659         """   660    661         # Find any defaults for the function or method.   662    663         defaults = self.function_defaults.get(name)   664         if not defaults:   665             return False   666    667         # Identify non-constant defaults.   668    669         for name, ref in defaults:   670             if self.is_dynamic(ref):   671                 return True   672    673         return False   674    675     def order_objects(self):   676    677         "Produce an object initialisation ordering."   678    679         self.condense_dependencies()   680    681         try:   682             ordered = order_dependencies(self.depends)   683         except ValueError, exc:   684             raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(exc.args[0].keys()))   685    686         if "__main__" in ordered:   687             ordered.remove("__main__")   688    689         ordered.append("__main__")   690         return ordered   691    692     def order_modules(self):   693    694         "Produce a module initialisation ordering."   695    696         ordered = self.order_objects()   697         filtered = []   698    699         for module_name in self.modules.keys():   700             if module_name not in ordered:   701                 filtered.append(module_name)   702    703         for path in ordered:   704             if self.modules.has_key(path):   705                 filtered.append(path)   706    707         return filtered   708    709     def find_dependency(self, ref):   710    711         "Find the ultimate dependency for 'ref'."   712    713         found = set()   714         while ref and ref.has_kind("<depends>") and not ref in found:   715             found.add(ref)   716             ref = self.identify(ref.get_origin())   717         return ref   718    719     def get_module_provider(self, ref):   720    721         "Identify the provider of the given 'ref'."   722    723         for ancestor in ref.ancestors():   724             if self.modules.has_key(ancestor):   725                 return ancestor   726         return None   727    728     def finalise_classes(self):   729    730         "Finalise the class relationships and attributes."   731    732         self.derive_inherited_attrs()   733         self.derive_subclasses()   734         self.derive_shadowed_attrs()   735    736     def derive_inherited_attrs(self):   737    738         "Derive inherited attributes for classes throughout the program."   739    740         for name in self.classes.keys():   741             self.propagate_attrs_for_class(name)   742    743     def propagate_attrs_for_class(self, name, visited=None):   744    745         "Propagate inherited attributes for class 'name'."   746    747         # Visit classes only once.   748    749         if self.all_combined_attrs.has_key(name):   750             return   751    752         visited = visited or []   753    754         if name in visited:   755             raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name)   756    757         visited.append(name)   758    759         class_attrs = {}   760         instance_attrs = {}   761    762         # Aggregate the attributes from base classes, recording the origins of   763         # applicable attributes.   764    765         for base in self.classes[name][::-1]:   766    767             # Get the identity of the class from the reference.   768    769             base = base.get_origin()   770    771             # Define the base class completely before continuing with this   772             # class.   773    774             self.propagate_attrs_for_class(base, visited)   775             class_attrs.update(self.all_class_attrs[base])   776    777             # Instance attribute origins are combined if different.   778    779             for key, values in self.all_instance_attrs[base].items():   780                 init_item(instance_attrs, key, set)   781                 instance_attrs[key].update(values)   782    783         # Class attributes override those defined earlier in the hierarchy.   784    785         class_attrs.update(self.all_class_attrs.get(name, {}))   786    787         # Instance attributes are merely added if not already defined.   788    789         for key in self.all_instance_attrs.get(name, []):   790             if not instance_attrs.has_key(key):   791                 instance_attrs[key] = set(["%s.%s" % (name, key)])   792    793         self.all_class_attrs[name] = class_attrs   794         self.all_instance_attrs[name] = instance_attrs   795         self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys())   796    797     def derive_subclasses(self):   798    799         "Derive subclass details for classes."   800    801         for name, bases in self.classes.items():   802             for base in bases:   803    804                 # Get the identity of the class from the reference.   805    806                 base = base.get_origin()   807                 self.subclasses[base].add(name)   808    809     def derive_shadowed_attrs(self):   810    811         "Derive shadowed attributes for classes."   812    813         for name, attrs in self.all_instance_attrs.items():   814             attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys())   815             if attrs:   816                 self.all_shadowed_attrs[name] = attrs   817    818     def set_class_types(self):   819    820         "Set the type of each class."   821    822         for attrs in self.all_class_attrs.values():   823             attrs["__class__"] = self.type_ref.get_origin()   824    825     def define_instantiators(self):   826    827         """   828         Consolidate parameter and default details, incorporating initialiser   829         details to define instantiator signatures.   830         """   831    832         for cls, attrs in self.all_class_attrs.items():   833             initialiser = attrs["__init__"]   834             self.function_parameters[cls] = self.function_parameters[initialiser]   835             self.function_defaults[cls] = self.function_defaults[initialiser]   836    837     def collect_constants(self):   838    839         "Get constants from all active modules."   840    841         for module in self.modules.values():   842             self.all_constants.update(module.constants)   843    844     # Import methods.   845    846     def find_in_path(self, name):   847    848         """   849         Find the given module 'name' in the search path, returning None where no   850         such module could be found, or a 2-tuple from the 'find' method   851         otherwise.   852         """   853    854         for d in self.path:   855             m = self.find(d, name)   856             if m: return m   857         return None   858    859     def find(self, d, name):   860    861         """   862         In the directory 'd', find the given module 'name', where 'name' can   863         either refer to a single file module or to a package. Return None if the   864         'name' cannot be associated with either a file or a package directory,   865         or a 2-tuple from '_find_package' or '_find_module' otherwise.   866         """   867    868         m = self._find_package(d, name)   869         if m: return m   870         m = self._find_module(d, name)   871         if m: return m   872         return None   873    874     def _find_module(self, d, name):   875    876         """   877         In the directory 'd', find the given module 'name', returning None where   878         no suitable file exists in the directory, or a 2-tuple consisting of   879         None (indicating that no package directory is involved) and a filename   880         indicating the location of the module.   881         """   882    883         name_py = name + extsep + "py"   884         filename = self._find_file(d, name_py)   885         if filename:   886             return None, filename   887         return None   888    889     def _find_package(self, d, name):   890    891         """   892         In the directory 'd', find the given package 'name', returning None   893         where no suitable package directory exists, or a 2-tuple consisting of   894         a directory (indicating the location of the package directory itself)   895         and a filename indicating the location of the __init__.py module which   896         declares the package's top-level contents.   897         """   898    899         filename = self._find_file(d, name)   900         if filename:   901             init_py = "__init__" + extsep + "py"   902             init_py_filename = self._find_file(filename, init_py)   903             if init_py_filename:   904                 return filename, init_py_filename   905         return None   906    907     def _find_file(self, d, filename):   908    909         """   910         Return the filename obtained when searching the directory 'd' for the   911         given 'filename', or None if no actual file exists for the filename.   912         """   913    914         filename = join(d, filename)   915         if exists(filename):   916             return filename   917         else:   918             return None   919    920     def load(self, name):   921    922         """   923         Load the module or package with the given 'name'. Return an object   924         referencing the loaded module or package, or None if no such module or   925         package exists.   926         """   927    928         # Loaded modules are returned immediately.   929         # Modules may be known but not yet loading (having been registered as   930         # submodules), loading, loaded, or completely unknown.   931    932         module = self.get_module(name)   933    934         if module:   935             return self.modules[name]   936    937         # Otherwise, modules are loaded.   938    939         # Split the name into path components, and try to find the uppermost in   940         # the search path.   941    942         path = name.split(".")   943         path_so_far = []   944         module = None   945    946         for p in path:   947    948             # Get the module's filesystem details.   949    950             if not path_so_far:   951                 m = self.find_in_path(p)   952             elif d:   953                 m = self.find(d, p)   954             else:   955                 m = None   956    957             path_so_far.append(p)   958             module_name = ".".join(path_so_far)   959    960             # Raise an exception if the module could not be located.   961    962             if not m:   963                 raise ProgramError("Module not found: %s" % name)   964    965             # Get the directory and module filename.   966    967             d, filename = m   968    969         # Get the module itself.   970    971         return self.load_from_file(filename, module_name)   972    973     def load_from_file(self, filename, module_name=None):   974    975         "Load the module from the given 'filename'."   976    977         if module_name is None:   978             module_name = "__main__"   979    980         module = self.modules.get(module_name)   981    982         if not module:   983    984             # Try to load from cache.   985    986             module = self.load_from_cache(filename, module_name)   987             if module:   988                 return module   989    990             # If no cache entry exists, load from file.   991    992             module = inspector.InspectedModule(module_name, self)   993             self.add_module(module_name, module)   994             self.update_cache_validity(module)   995    996             self._load(module, module_name, lambda m: m.parse, filename)   997    998         return module   999   1000     def update_cache_validity(self, module):  1001   1002         "Make 'module' valid in the cache, but invalidate accessing modules."  1003   1004         accessing = self.accessing_modules.get(module.name)  1005         if accessing:  1006             self.invalidated.update(accessing)  1007         if module.name in self.invalidated:  1008             self.invalidated.remove(module.name)  1009   1010     def source_is_new(self, filename, module_name):  1011   1012         "Return whether 'filename' is newer than the cached 'module_name'."  1013   1014         if self.cache:  1015             cache_filename = join(self.cache, module_name)  1016             return not exists(cache_filename) or \  1017                 getmtime(filename) > getmtime(cache_filename) or \  1018                 module_name in self.invalidated  1019         else:  1020             return True  1021   1022     def load_from_cache(self, filename, module_name):  1023   1024         "Return a module residing in the cache."  1025   1026         module = self.modules.get(module_name)  1027   1028         if not module and not self.source_is_new(filename, module_name):  1029             module = CachedModule(module_name, self)  1030             self.add_module(module_name, module)  1031   1032             filename = join(self.cache, module_name)  1033             self._load(module, module_name, lambda m: m.from_cache, filename)  1034   1035         return module  1036   1037     def _load(self, module, module_name, fn, filename):  1038   1039         """  1040         Load 'module' for the given 'module_name', and with 'fn' performing an  1041         invocation on the module with the given 'filename'.  1042         """  1043   1044         # Load the module.  1045   1046         if self.verbose:  1047             print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename  1048         fn(module)(filename)  1049   1050         # Add the module object if not already defined.  1051   1052         if not self.objects.has_key(module_name):  1053             self.objects[module_name] = Reference("<module>", module_name)  1054   1055     def add_module(self, module_name, module):  1056   1057         """  1058         Return the module with the given 'module_name', adding a new module  1059         object if one does not already exist.  1060         """  1061   1062         self.modules[module_name] = module  1063         if module_name in self.to_import:  1064             self.to_import.remove(module_name)  1065   1066 # vim: tabstop=4 expandtab shiftwidth=4