Lichen

importer.py

814:30e58152063b
2017-04-11 Paul Boddie Added tag rel-0-1 for changeset a80de062b595
     1 #!/usr/bin/env python     2      3 """     4 Import logic.     5      6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,     7               2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from errors import ProgramError    24 from os.path import exists, extsep, getmtime, join    25 from os import listdir, makedirs, remove    26 from common import init_item, order_dependencies, readfile, writefile    27 from modules import CachedModule    28 from referencing import Reference    29 import inspector    30 import sys    31     32 class Importer:    33     34     "An import machine, searching for and loading modules."    35     36     special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__")    37     38     def __init__(self, path, cache=None, verbose=False, warnings=None):    39     40         """    41         Initialise the importer with the given search 'path' - a list of    42         directories to search for Python modules.    43     44         The optional 'cache' should be the name of a directory used to store    45         cached module information.    46     47         The optional 'verbose' parameter causes output concerning the activities    48         of the object to be produced if set to a true value (not the default).    49     50         The optional 'warnings' parameter may indicate classes of warnings to be    51         produced.    52         """    53     54         self.path = path    55         self.cache = cache    56         self.verbose = verbose    57         self.warnings = warnings    58     59         # Module importing queue, required modules, removed modules and active    60         # modules in the final program.    61     62         self.to_import = set()    63         self.required = set(["__main__"])    64         self.removed = {}    65         self.modules = {}    66     67         # Module relationships and invalidated cached modules.    68     69         self.accessing_modules = {}    70         self.invalidated = set()    71     72         # Object relationships and dependencies.    73     74         self.depends = {}    75         self.module_depends = {}    76     77         # Basic program information.    78     79         self.objects = {}    80         self.classes = {}    81         self.function_parameters = {}    82         self.function_defaults = {}    83         self.function_locals = {}    84     85         # Unresolved names.    86     87         self.missing = set()    88     89         # Derived information.    90     91         self.subclasses = {}    92     93         # Attributes of different object types.    94     95         self.all_class_attrs = {}    96         self.all_instance_attrs = {}    97         self.all_instance_attr_constants = {}    98         self.all_combined_attrs = {}    99         self.all_module_attrs = {}   100         self.all_shadowed_attrs = {}   101    102         # References to external names and aliases within program units.   103    104         self.all_name_references = {}   105         self.all_initialised_names = {}   106         self.all_aliased_names = {}   107    108         # General attribute accesses.   109    110         self.all_attr_accesses = {}   111         self.all_const_accesses = {}   112         self.all_attr_access_modifiers = {}   113    114         # Constant literals and values.   115    116         self.all_constants = {}   117         self.all_constant_values = {}   118    119         # Common function value collection used during inspection.   120    121         self.all_return_values = {}   122    123         self.make_cache()   124    125     def give_warning(self, name):   126    127         "Return whether the indicated warning 'name' should be given."   128    129         return self.warnings and (name in self.warnings or "all" in self.warnings)   130    131     def make_cache(self):   132    133         "Make a cache directory if it does not already exist."   134    135         if self.cache and not exists(self.cache):   136             makedirs(self.cache)   137    138     def check_cache(self, details):   139    140         """   141         Check whether the cache applies for the given 'details', invalidating it   142         if it does not.   143         """   144    145         recorded_details = self.get_cache_details()   146    147         if recorded_details != details:   148             self.remove_cache()   149    150         writefile(self.get_cache_details_filename(), details)   151    152     def get_cache_details_filename(self):   153    154         "Return the filename for the cache details."   155    156         return join(self.cache, "$details")   157    158     def get_cache_details(self):   159    160         "Return details of the cache."   161    162         details_filename = self.get_cache_details_filename()   163    164         if not exists(details_filename):   165             return None   166         else:   167             return readfile(details_filename)   168    169     def remove_cache(self):   170    171         "Remove the contents of the cache."   172    173         for filename in listdir(self.cache):   174             remove(join(self.cache, filename))   175    176     def to_cache(self):   177    178         "Write modules to the cache."   179    180         if self.cache:   181             for module_name, module in self.modules.items():   182                 module.to_cache(join(self.cache, module_name))   183    184     # Object retrieval and storage.   185    186     def get_object(self, name):   187    188         """   189         Return a reference for the given 'name' or None if no such object   190         exists.   191         """   192    193         return self.objects.get(name)   194    195     def set_object(self, name, value=None):   196    197         "Set the object with the given 'name' and the given 'value'."   198    199         if isinstance(value, Reference):   200             ref = value.alias(name)   201         else:   202             ref = Reference(value, name)   203    204         self.objects[name] = ref   205    206     # Identification of both stored object names and name references.   207    208     def identify(self, name):   209    210         "Identify 'name' using stored object and external name records."   211    212         return self.objects.get(name) or self.all_name_references.get(name)   213    214     # Indirect object retrieval.   215    216     def get_attributes(self, ref, attrname):   217    218         """   219         Return attributes provided by 'ref' for 'attrname'. Class attributes   220         may be provided by instances.   221         """   222    223         kind = ref.get_kind()   224         if kind == "<class>":   225             ref = self.get_class_attribute(ref.get_origin(), attrname)   226             return ref and set([ref]) or set()   227         elif kind == "<instance>":   228             return self.get_combined_attributes(ref.get_origin(), attrname)   229         elif kind == "<module>":   230             ref = self.get_module_attribute(ref.get_origin(), attrname)   231             return ref and set([ref]) or set()   232         else:   233             return set()   234    235     def get_class_attribute(self, object_type, attrname):   236    237         "Return from 'object_type' the details of class attribute 'attrname'."   238    239         attrs = self.all_class_attrs.get(object_type)   240         attr = attrs and attrs.get(attrname)   241         return attr and self.get_object(attr)   242    243     def get_instance_attributes(self, object_type, attrname):   244    245         """   246         Return from 'object_type' the details of instance attribute 'attrname'.   247         """   248    249         consts = self.all_instance_attr_constants.get(object_type)   250         attrs = set()   251         for attr in self.all_instance_attrs[object_type].get(attrname, []):   252             attrs.add(consts and consts.get(attrname) or Reference("<var>", attr))   253         return attrs   254    255     def get_combined_attributes(self, object_type, attrname):   256    257         """   258         Return from 'object_type' the details of class or instance attribute   259         'attrname'.   260         """   261    262         ref = self.get_class_attribute(object_type, attrname)   263         refs = ref and set([ref]) or set()   264         refs.update(self.get_instance_attributes(object_type, attrname))   265         return refs   266    267     def get_module_attribute(self, object_type, attrname):   268    269         "Return from 'object_type' the details of module attribute 'attrname'."   270    271         if attrname in self.all_module_attrs[object_type]:   272             return self.get_object("%s.%s" % (object_type, attrname))   273         else:   274             return None   275    276     # Convenience methods for deducing which kind of object provided an   277     # attribute.   278    279     def get_attribute_provider(self, ref, attrname):   280    281         """   282         Return the kind of provider of the attribute accessed via 'ref' using   283         'attrname'.   284         """   285    286         kind = ref.get_kind()   287    288         if kind in ["<class>", "<module>"]:   289             return kind   290         else:   291             return self.get_instance_attribute_provider(ref.get_origin(), attrname)   292    293     def get_instance_attribute_provider(self, object_type, attrname):   294    295         """   296         Return the kind of provider of the attribute accessed via an instance of   297         'object_type' using 'attrname'.   298         """   299    300         if self.get_class_attribute(object_type, attrname):   301             return "<class>"   302         else:   303             return "<instance>"   304    305     # Module management.   306    307     def queue_module(self, name, accessor, required=False):   308    309         """   310         Queue the module with the given 'name' for import from the given   311         'accessor' module. If 'required' is true (it is false by default), the   312         module will be required in the final program.   313         """   314    315         if not self.modules.has_key(name):   316             self.to_import.add(name)   317    318         if required:   319             self.required.add(name)   320    321         init_item(self.accessing_modules, name, set)   322         self.accessing_modules[name].add(accessor.name)   323    324     def get_modules(self):   325    326         "Return all modules known to the importer."   327    328         return self.modules.values()   329    330     def get_module(self, name):   331    332         "Return the module with the given 'name'."   333    334         if not self.modules.has_key(name):   335             return None   336    337         return self.modules[name]   338    339     # Program operations.   340    341     def initialise(self, filename, reset=False):   342    343         """   344         Initialise a program whose main module is 'filename', resetting the   345         cache if 'reset' is true. Return the main module.   346         """   347    348         if reset:   349             self.remove_cache()   350         self.check_cache(filename)   351    352         # Load the program itself.   353    354         m = self.load_from_file(filename)   355    356         # Load any queued modules.   357    358         while self.to_import:   359             for name in list(self.to_import): # avoid mutation issue   360                 self.load(name)   361    362         # Resolve dependencies between modules.   363    364         self.resolve()   365    366         # Record the type of all classes.   367    368         self.type_ref = self.get_object("__builtins__.type")   369    370         # Resolve dependencies within the program.   371    372         for module in self.modules.values():   373             module.complete()   374    375         # Remove unneeded modules.   376    377         all_modules = self.modules.items()   378    379         for name, module in all_modules:   380             if name not in self.required:   381                 module.unpropagate()   382                 del self.modules[name]   383                 self.removed[name] = module   384    385         # Collect redundant objects.   386    387         for module in self.removed.values():   388             module.collect()   389    390         # Assert module objects where aliases have been removed.   391    392         for name in self.required:   393             if not self.objects.has_key(name):   394                 self.objects[name] = Reference("<module>", name)   395    396         return m   397    398     def finalise(self):   399    400         """   401         Finalise the inspected program, returning whether the program could be   402         finalised.   403         """   404    405         self.finalise_classes()   406         self.add_init_dependencies()   407         self.to_cache()   408    409         if self.missing:   410             return False   411    412         self.set_class_types()   413         self.define_instantiators()   414         self.collect_constants()   415    416         return True   417    418     # Supporting operations.   419    420     def resolve(self):   421    422         "Resolve dependencies between modules."   423    424         self.waiting = {}   425    426         for module in self.modules.values():   427    428             # Resolve all deferred references in each module.   429    430             original_deferred = []   431    432             for ref in module.deferred:   433    434                 # Retain original references for caching.   435    436                 original_deferred.append(ref.copy())   437    438                 # Update references throughout the program.   439    440                 found = self.find_dependency(ref)   441                 if not found:   442                     self.missing.add((module.name, ref.get_origin()))   443    444                 # Record the resolved names and identify required modules.   445    446                 else:   447                     # Find the providing module of this reference.   448                     # Where definitive details of the origin cannot be found,   449                     # identify the provider using the deferred reference.   450                     # NOTE: This may need to test for static origins.   451    452                     provider = self.get_module_provider(found.unresolved() and ref or found)   453                     ref.mutate(found)   454    455                     # Record any external dependency.   456    457                     if provider and provider != module.name:   458    459                         # Handle built-in modules accidentally referenced by   460                         # names.   461    462                         if provider == "__builtins__" and found.has_kind("<module>"):   463                             raise ProgramError("Name %s, used by %s, refers to module %s." %   464                                                (found.leaf(), module.name, found.get_origin()))   465    466                         # Record the provider dependency.   467    468                         module.required.add(provider)   469                         self.accessing_modules[provider].add(module.name)   470    471                         # Postpone any inclusion of the provider until this   472                         # module becomes required.   473    474                         if module.name not in self.required:   475                             init_item(self.waiting, module.name, set)   476                             self.waiting[module.name].add(provider)   477                             if self.verbose:   478                                 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name   479    480                         # Make this module required in the accessing module.   481    482                         elif provider not in self.required:   483                             self.required.add(provider)   484                             if self.verbose:   485                                 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name   486    487                         # Record a module ordering dependency.   488    489                         if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found):   490                             self.add_module_dependency(module.name, provider)   491    492             # Restore the original references so that they may be read back in   493             # and produce the same results.   494    495             module.deferred = original_deferred   496    497         # Check modules again to see if they are now required and should now   498         # cause the inclusion of other modules providing objects to the program.   499    500         for module_name in self.waiting.keys():   501             self.require_providers(module_name)   502    503         self.add_special_dependencies()   504         self.add_module_dependencies()   505    506     def require_providers(self, module_name):   507    508         """   509         Test if 'module_name' is itself required and, if so, require modules   510         containing objects provided to the module.   511         """   512    513         if module_name in self.required and self.waiting.has_key(module_name):   514             for provider in self.waiting[module_name]:   515                 if provider not in self.required:   516                     self.required.add(provider)   517                     if self.verbose:   518                         print >>sys.stderr, "Requiring", provider   519                     self.require_providers(provider)   520    521     def add_special_dependencies(self):   522    523         "Add dependencies due to the use of special names in namespaces."   524    525         for module in self.modules.values():   526             for ref, paths in module.special.values():   527                 for path in paths:   528                     self.add_dependency(path, ref.get_origin())   529    530     def add_init_dependencies(self):   531    532         "Add dependencies related to object initialisation."   533    534         for name in self.classes.keys():   535             if self.is_dynamic_class(name):   536    537                 # Make subclasses depend on any class with non-static   538                 # attributes, plus its module for the initialisation.   539    540                 for subclass in self.subclasses[name]:   541                     ref = Reference("<class>", subclass)   542                     self.add_dependency(subclass, name)   543                     self.add_dependency(subclass, self.get_module_provider(ref))   544    545                 # Also make the class dependent on its module for   546                 # initialisation.   547    548                 ref = Reference("<class>", name)   549                 self.add_dependency(name, self.get_module_provider(ref))   550    551         for name in self.function_defaults.keys():   552             if self.is_dynamic_callable(name):   553    554                 # Make functions with defaults requiring initialisation depend   555                 # on the parent scope, if a function, or the module scope.   556    557                 ref = Reference("<function>", name)   558                 parent_ref = self.get_object(ref.parent())   559    560                 # Function no longer present in the program.   561    562                 if not parent_ref:   563                     continue   564    565                 if parent_ref.has_kind("<class>"):   566                     parent = self.get_module_provider(parent_ref)   567                 else:   568                     parent = parent_ref.get_origin()   569    570                 self.add_dependency(name, parent)   571    572     def add_module_dependencies(self):   573    574         "Record module-based dependencies."   575    576         for module_name, providers in self.module_depends.items():   577             if self.modules.has_key(module_name):   578                 for provider in providers:   579                     if self.modules.has_key(provider):   580                         self.add_dependency(module_name, provider)   581    582     def add_dependency(self, path, origin):   583    584         "Add dependency details for 'path' involving 'origin'."   585    586         if origin and not origin.startswith("%s." % path):   587             init_item(self.depends, path, set)   588             self.depends[path].add(origin)   589    590     def add_module_dependency(self, module_name, provider):   591    592         "Add dependency details for 'module_name' involving 'provider'."   593    594         if provider:   595             init_item(self.module_depends, module_name, set)   596             self.module_depends[module_name].add(provider)   597    598     def condense_dependencies(self):   599    600         """   601         Condense the dependencies by removing all functions that do not need   602         initialisation.   603         """   604    605         d = {}   606         for path, depends in self.depends.items():   607             d[path] = {}   608             d[path] = self.condense_dependency_entry(depends, d)   609    610         self.depends = {}   611    612         for path, depends in d.items():   613             if depends:   614                 self.depends[path] = depends   615    616     def condense_dependency_entry(self, depends, d):   617         l = set()   618         for depend in depends:   619             if self.modules.has_key(depend) or self.classes.has_key(depend) or \   620                self.is_dynamic_callable(depend):   621    622                 l.add(depend)   623             else:   624                 deps = d.get(depend)   625                 if deps:   626                     l.update(self.condense_dependency_entry(deps, d))   627         return l   628    629     def is_dynamic(self, ref):   630         return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value()   631    632     def is_dynamic_class(self, name):   633    634         """   635         Return whether 'name' refers to a class with members that must be   636         initialised dynamically.   637         """   638    639         attrs = self.all_class_attrs.get(name)   640    641         if not attrs:   642             return False   643    644         for attrname, attr in attrs.items():   645             if attrname in self.special_attributes:   646                 continue   647             ref = attr and self.get_object(attr)   648             if self.is_dynamic(ref):   649                 return True   650    651         return False   652    653     def is_dynamic_callable(self, name):   654    655         """   656         Return whether 'name' refers to a callable employing defaults that may   657         need initialising before the callable can be used.   658         """   659    660         # Find any defaults for the function or method.   661    662         defaults = self.function_defaults.get(name)   663         if not defaults:   664             return False   665    666         # Identify non-constant defaults.   667    668         for name, ref in defaults:   669             if self.is_dynamic(ref):   670                 return True   671    672         return False   673    674     def order_objects(self):   675    676         "Produce an object initialisation ordering."   677    678         self.condense_dependencies()   679    680         try:   681             ordered = order_dependencies(self.depends)   682         except ValueError, exc:   683             raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(exc.args[0].keys()))   684    685         if "__main__" in ordered:   686             ordered.remove("__main__")   687    688         ordered.append("__main__")   689         return ordered   690    691     def order_modules(self):   692    693         "Produce a module initialisation ordering."   694    695         ordered = self.order_objects()   696         filtered = []   697    698         for module_name in self.modules.keys():   699             if module_name not in ordered:   700                 filtered.append(module_name)   701    702         for path in ordered:   703             if self.modules.has_key(path):   704                 filtered.append(path)   705    706         return filtered   707    708     def find_dependency(self, ref):   709    710         "Find the ultimate dependency for 'ref'."   711    712         found = set()   713         while ref and ref.has_kind("<depends>") and not ref in found:   714             found.add(ref)   715             ref = self.identify(ref.get_origin())   716         return ref   717    718     def get_module_provider(self, ref):   719    720         "Identify the provider of the given 'ref'."   721    722         for ancestor in ref.ancestors():   723             if self.modules.has_key(ancestor):   724                 return ancestor   725         return None   726    727     def finalise_classes(self):   728    729         "Finalise the class relationships and attributes."   730    731         self.derive_inherited_attrs()   732         self.derive_subclasses()   733         self.derive_shadowed_attrs()   734    735     def derive_inherited_attrs(self):   736    737         "Derive inherited attributes for classes throughout the program."   738    739         for name in self.classes.keys():   740             self.propagate_attrs_for_class(name)   741    742     def propagate_attrs_for_class(self, name, visited=None):   743    744         "Propagate inherited attributes for class 'name'."   745    746         # Visit classes only once.   747    748         if self.all_combined_attrs.has_key(name):   749             return   750    751         visited = visited or []   752    753         if name in visited:   754             raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name)   755    756         visited.append(name)   757    758         class_attrs = {}   759         instance_attrs = {}   760    761         # Aggregate the attributes from base classes, recording the origins of   762         # applicable attributes.   763    764         for base in self.classes[name][::-1]:   765    766             # Get the identity of the class from the reference.   767    768             base = base.get_origin()   769    770             # Define the base class completely before continuing with this   771             # class.   772    773             self.propagate_attrs_for_class(base, visited)   774             class_attrs.update(self.all_class_attrs[base])   775    776             # Instance attribute origins are combined if different.   777    778             for key, values in self.all_instance_attrs[base].items():   779                 init_item(instance_attrs, key, set)   780                 instance_attrs[key].update(values)   781    782         # Class attributes override those defined earlier in the hierarchy.   783    784         class_attrs.update(self.all_class_attrs.get(name, {}))   785    786         # Instance attributes are merely added if not already defined.   787    788         for key in self.all_instance_attrs.get(name, []):   789             if not instance_attrs.has_key(key):   790                 instance_attrs[key] = set(["%s.%s" % (name, key)])   791    792         self.all_class_attrs[name] = class_attrs   793         self.all_instance_attrs[name] = instance_attrs   794         self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys())   795    796     def derive_subclasses(self):   797    798         "Derive subclass details for classes."   799    800         for name, bases in self.classes.items():   801             for base in bases:   802    803                 # Get the identity of the class from the reference.   804    805                 base = base.get_origin()   806                 self.subclasses[base].add(name)   807    808     def derive_shadowed_attrs(self):   809    810         "Derive shadowed attributes for classes."   811    812         for name, attrs in self.all_instance_attrs.items():   813             attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys())   814             if attrs:   815                 self.all_shadowed_attrs[name] = attrs   816    817     def set_class_types(self):   818    819         "Set the type of each class."   820    821         for attrs in self.all_class_attrs.values():   822             attrs["__class__"] = self.type_ref.get_origin()   823    824     def define_instantiators(self):   825    826         """   827         Consolidate parameter and default details, incorporating initialiser   828         details to define instantiator signatures.   829         """   830    831         for cls, attrs in self.all_class_attrs.items():   832             initialiser = attrs["__init__"]   833             self.function_parameters[cls] = self.function_parameters[initialiser]   834             self.function_defaults[cls] = self.function_defaults[initialiser]   835    836     def collect_constants(self):   837    838         "Get constants from all active modules."   839    840         for module in self.modules.values():   841             self.all_constants.update(module.constants)   842    843     # Import methods.   844    845     def find_in_path(self, name):   846    847         """   848         Find the given module 'name' in the search path, returning None where no   849         such module could be found, or a 2-tuple from the 'find' method   850         otherwise.   851         """   852    853         for d in self.path:   854             m = self.find(d, name)   855             if m: return m   856         return None   857    858     def find(self, d, name):   859    860         """   861         In the directory 'd', find the given module 'name', where 'name' can   862         either refer to a single file module or to a package. Return None if the   863         'name' cannot be associated with either a file or a package directory,   864         or a 2-tuple from '_find_package' or '_find_module' otherwise.   865         """   866    867         m = self._find_package(d, name)   868         if m: return m   869         m = self._find_module(d, name)   870         if m: return m   871         return None   872    873     def _find_module(self, d, name):   874    875         """   876         In the directory 'd', find the given module 'name', returning None where   877         no suitable file exists in the directory, or a 2-tuple consisting of   878         None (indicating that no package directory is involved) and a filename   879         indicating the location of the module.   880         """   881    882         name_py = name + extsep + "py"   883         filename = self._find_file(d, name_py)   884         if filename:   885             return None, filename   886         return None   887    888     def _find_package(self, d, name):   889    890         """   891         In the directory 'd', find the given package 'name', returning None   892         where no suitable package directory exists, or a 2-tuple consisting of   893         a directory (indicating the location of the package directory itself)   894         and a filename indicating the location of the __init__.py module which   895         declares the package's top-level contents.   896         """   897    898         filename = self._find_file(d, name)   899         if filename:   900             init_py = "__init__" + extsep + "py"   901             init_py_filename = self._find_file(filename, init_py)   902             if init_py_filename:   903                 return filename, init_py_filename   904         return None   905    906     def _find_file(self, d, filename):   907    908         """   909         Return the filename obtained when searching the directory 'd' for the   910         given 'filename', or None if no actual file exists for the filename.   911         """   912    913         filename = join(d, filename)   914         if exists(filename):   915             return filename   916         else:   917             return None   918    919     def load(self, name):   920    921         """   922         Load the module or package with the given 'name'. Return an object   923         referencing the loaded module or package, or None if no such module or   924         package exists.   925         """   926    927         # Loaded modules are returned immediately.   928         # Modules may be known but not yet loading (having been registered as   929         # submodules), loading, loaded, or completely unknown.   930    931         module = self.get_module(name)   932    933         if module:   934             return self.modules[name]   935    936         # Otherwise, modules are loaded.   937    938         # Split the name into path components, and try to find the uppermost in   939         # the search path.   940    941         path = name.split(".")   942         path_so_far = []   943         module = None   944    945         for p in path:   946    947             # Get the module's filesystem details.   948    949             if not path_so_far:   950                 m = self.find_in_path(p)   951             elif d:   952                 m = self.find(d, p)   953             else:   954                 m = None   955    956             path_so_far.append(p)   957             module_name = ".".join(path_so_far)   958    959             # Raise an exception if the module could not be located.   960    961             if not m:   962                 raise ProgramError("Module not found: %s" % name)   963    964             # Get the directory and module filename.   965    966             d, filename = m   967    968         # Get the module itself.   969    970         return self.load_from_file(filename, module_name)   971    972     def load_from_file(self, filename, module_name=None):   973    974         "Load the module from the given 'filename'."   975    976         if module_name is None:   977             module_name = "__main__"   978    979         module = self.modules.get(module_name)   980    981         if not module:   982    983             # Try to load from cache.   984    985             module = self.load_from_cache(filename, module_name)   986             if module:   987                 return module   988    989             # If no cache entry exists, load from file.   990    991             module = inspector.InspectedModule(module_name, self)   992             self.add_module(module_name, module)   993             self.update_cache_validity(module)   994    995             self._load(module, module_name, lambda m: m.parse, filename)   996    997         return module   998    999     def update_cache_validity(self, module):  1000   1001         "Make 'module' valid in the cache, but invalidate accessing modules."  1002   1003         accessing = self.accessing_modules.get(module.name)  1004         if accessing:  1005             self.invalidated.update(accessing)  1006         if module.name in self.invalidated:  1007             self.invalidated.remove(module.name)  1008   1009     def source_is_new(self, filename, module_name):  1010   1011         "Return whether 'filename' is newer than the cached 'module_name'."  1012   1013         if self.cache:  1014             cache_filename = join(self.cache, module_name)  1015             return not exists(cache_filename) or \  1016                 getmtime(filename) > getmtime(cache_filename) or \  1017                 module_name in self.invalidated  1018         else:  1019             return True  1020   1021     def load_from_cache(self, filename, module_name):  1022   1023         "Return a module residing in the cache."  1024   1025         module = self.modules.get(module_name)  1026   1027         if not module and not self.source_is_new(filename, module_name):  1028             module = CachedModule(module_name, self)  1029             self.add_module(module_name, module)  1030   1031             filename = join(self.cache, module_name)  1032             self._load(module, module_name, lambda m: m.from_cache, filename)  1033   1034         return module  1035   1036     def _load(self, module, module_name, fn, filename):  1037   1038         """  1039         Load 'module' for the given 'module_name', and with 'fn' performing an  1040         invocation on the module with the given 'filename'.  1041         """  1042   1043         # Load the module.  1044   1045         if self.verbose:  1046             print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename  1047         fn(module)(filename)  1048   1049         # Add the module object if not already defined.  1050   1051         if not self.objects.has_key(module_name):  1052             self.objects[module_name] = Reference("<module>", module_name)  1053   1054     def add_module(self, module_name, module):  1055   1056         """  1057         Return the module with the given 'module_name', adding a new module  1058         object if one does not already exist.  1059         """  1060   1061         self.modules[module_name] = module  1062         if module_name in self.to_import:  1063             self.to_import.remove(module_name)  1064   1065 # vim: tabstop=4 expandtab shiftwidth=4