1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, order_dependencies, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__") 37 38 def __init__(self, path, cache=None, verbose=False, warnings=None): 39 40 """ 41 Initialise the importer with the given search 'path' - a list of 42 directories to search for Python modules. 43 44 The optional 'cache' should be the name of a directory used to store 45 cached module information. 46 47 The optional 'verbose' parameter causes output concerning the activities 48 of the object to be produced if set to a true value (not the default). 49 50 The optional 'warnings' parameter may indicate classes of warnings to be 51 produced. 52 """ 53 54 self.path = path 55 self.cache = cache 56 self.verbose = verbose 57 self.warnings = warnings 58 59 # Module importing queue, required modules, removed modules and active 60 # modules in the final program. 61 62 self.to_import = set() 63 self.required = set(["__main__"]) 64 self.removed = {} 65 self.modules = {} 66 67 # Module relationships and invalidated cached modules. 68 69 self.accessing_modules = {} 70 self.invalidated = set() 71 72 # Object relationships and dependencies. 73 74 self.depends = {} 75 self.module_depends = {} 76 77 # Basic program information. 78 79 self.objects = {} 80 self.classes = {} 81 self.function_parameters = {} 82 self.function_defaults = {} 83 self.function_locals = {} 84 85 # Unresolved names. 86 87 self.missing = set() 88 89 # Derived information. 90 91 self.subclasses = {} 92 93 # Attributes of different object types. 94 95 self.all_class_attrs = {} 96 self.all_instance_attrs = {} 97 self.all_instance_attr_constants = {} 98 self.all_combined_attrs = {} 99 self.all_module_attrs = {} 100 self.all_shadowed_attrs = {} 101 102 # References to external names and aliases within program units. 103 104 self.all_name_references = {} 105 self.all_initialised_names = {} 106 self.all_aliased_names = {} 107 108 # General attribute accesses. 109 110 self.all_attr_accesses = {} 111 self.all_const_accesses = {} 112 self.all_attr_access_modifiers = {} 113 114 # Constant literals and values. 115 116 self.all_constants = {} 117 self.all_constant_values = {} 118 119 # Common function value collection used during inspection. 120 121 self.all_return_values = {} 122 123 self.make_cache() 124 125 def give_warning(self, name): 126 127 "Return whether the indicated warning 'name' should be given." 128 129 return self.warnings and (name in self.warnings or "all" in self.warnings) 130 131 def make_cache(self): 132 133 "Make a cache directory if it does not already exist." 134 135 if self.cache and not exists(self.cache): 136 makedirs(self.cache) 137 138 def check_cache(self, details): 139 140 """ 141 Check whether the cache applies for the given 'details', invalidating it 142 if it does not. 143 """ 144 145 recorded_details = self.get_cache_details() 146 147 if recorded_details != details: 148 self.remove_cache() 149 150 writefile(self.get_cache_details_filename(), details) 151 152 def get_cache_details_filename(self): 153 154 "Return the filename for the cache details." 155 156 return join(self.cache, "$details") 157 158 def get_cache_details(self): 159 160 "Return details of the cache." 161 162 details_filename = self.get_cache_details_filename() 163 164 if not exists(details_filename): 165 return None 166 else: 167 return readfile(details_filename) 168 169 def remove_cache(self): 170 171 "Remove the contents of the cache." 172 173 for filename in listdir(self.cache): 174 remove(join(self.cache, filename)) 175 176 def to_cache(self): 177 178 "Write modules to the cache." 179 180 if self.cache: 181 for module_name, module in self.modules.items(): 182 module.to_cache(join(self.cache, module_name)) 183 184 # Object retrieval and storage. 185 186 def get_object(self, name): 187 188 """ 189 Return a reference for the given 'name' or None if no such object 190 exists. 191 """ 192 193 return self.objects.get(name) 194 195 def set_object(self, name, value=None): 196 197 "Set the object with the given 'name' and the given 'value'." 198 199 if isinstance(value, Reference): 200 ref = value.alias(name) 201 else: 202 ref = Reference(value, name) 203 204 self.objects[name] = ref 205 206 # Identification of both stored object names and name references. 207 208 def identify(self, name): 209 210 "Identify 'name' using stored object and external name records." 211 212 return self.objects.get(name) or self.all_name_references.get(name) 213 214 # Indirect object retrieval. 215 216 def get_attributes(self, ref, attrname): 217 218 """ 219 Return attributes provided by 'ref' for 'attrname'. Class attributes 220 may be provided by instances. 221 """ 222 223 kind = ref.get_kind() 224 if kind == "<class>": 225 ref = self.get_class_attribute(ref.get_origin(), attrname) 226 return ref and set([ref]) or set() 227 elif kind == "<instance>": 228 return self.get_combined_attributes(ref.get_origin(), attrname) 229 elif kind == "<module>": 230 ref = self.get_module_attribute(ref.get_origin(), attrname) 231 return ref and set([ref]) or set() 232 else: 233 return set() 234 235 def get_class_attribute(self, object_type, attrname): 236 237 "Return from 'object_type' the details of class attribute 'attrname'." 238 239 attrs = self.all_class_attrs.get(object_type) 240 attr = attrs and attrs.get(attrname) 241 return attr and self.get_object(attr) 242 243 def get_instance_attributes(self, object_type, attrname): 244 245 """ 246 Return from 'object_type' the details of instance attribute 'attrname'. 247 """ 248 249 consts = self.all_instance_attr_constants.get(object_type) 250 attrs = set() 251 for attr in self.all_instance_attrs[object_type].get(attrname, []): 252 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 253 return attrs 254 255 def get_combined_attributes(self, object_type, attrname): 256 257 """ 258 Return from 'object_type' the details of class or instance attribute 259 'attrname'. 260 """ 261 262 ref = self.get_class_attribute(object_type, attrname) 263 refs = ref and set([ref]) or set() 264 refs.update(self.get_instance_attributes(object_type, attrname)) 265 return refs 266 267 def get_module_attribute(self, object_type, attrname): 268 269 "Return from 'object_type' the details of module attribute 'attrname'." 270 271 if attrname in self.all_module_attrs[object_type]: 272 return self.get_object("%s.%s" % (object_type, attrname)) 273 else: 274 return None 275 276 # Convenience methods for deducing which kind of object provided an 277 # attribute. 278 279 def get_attribute_provider(self, ref, attrname): 280 281 """ 282 Return the kind of provider of the attribute accessed via 'ref' using 283 'attrname'. 284 """ 285 286 kind = ref.get_kind() 287 288 if kind in ["<class>", "<module>"]: 289 return kind 290 else: 291 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 292 293 def get_instance_attribute_provider(self, object_type, attrname): 294 295 """ 296 Return the kind of provider of the attribute accessed via an instance of 297 'object_type' using 'attrname'. 298 """ 299 300 if self.get_class_attribute(object_type, attrname): 301 return "<class>" 302 else: 303 return "<instance>" 304 305 # Module management. 306 307 def queue_module(self, name, accessor, required=False): 308 309 """ 310 Queue the module with the given 'name' for import from the given 311 'accessor' module. If 'required' is true (it is false by default), the 312 module will be required in the final program. 313 """ 314 315 if not self.modules.has_key(name): 316 self.to_import.add(name) 317 318 if required: 319 self.required.add(name) 320 321 init_item(self.accessing_modules, name, set) 322 self.accessing_modules[name].add(accessor.name) 323 324 def get_modules(self): 325 326 "Return all modules known to the importer." 327 328 return self.modules.values() 329 330 def get_module(self, name): 331 332 "Return the module with the given 'name'." 333 334 if not self.modules.has_key(name): 335 return None 336 337 return self.modules[name] 338 339 # Program operations. 340 341 def initialise(self, filename, reset=False): 342 343 """ 344 Initialise a program whose main module is 'filename', resetting the 345 cache if 'reset' is true. Return the main module. 346 """ 347 348 if reset: 349 self.remove_cache() 350 self.check_cache(filename) 351 352 # Load the program itself. 353 354 m = self.load_from_file(filename) 355 356 # Load any queued modules. 357 358 while self.to_import: 359 for name in list(self.to_import): # avoid mutation issue 360 self.load(name) 361 362 # Resolve dependencies between modules. 363 364 self.resolve() 365 366 # Record the type of all classes. 367 368 self.type_ref = self.get_object("__builtins__.type") 369 370 # Resolve dependencies within the program. 371 372 for module in self.modules.values(): 373 module.complete() 374 375 # Remove unneeded modules. 376 377 all_modules = self.modules.items() 378 379 for name, module in all_modules: 380 if name not in self.required: 381 module.unpropagate() 382 del self.modules[name] 383 self.removed[name] = module 384 385 # Collect redundant objects. 386 387 for module in self.removed.values(): 388 module.collect() 389 390 # Assert module objects where aliases have been removed. 391 392 for name in self.required: 393 if not self.objects.has_key(name): 394 self.objects[name] = Reference("<module>", name) 395 396 return m 397 398 def finalise(self): 399 400 """ 401 Finalise the inspected program, returning whether the program could be 402 finalised. 403 """ 404 405 self.finalise_classes() 406 self.add_init_dependencies() 407 self.to_cache() 408 409 if self.missing: 410 return False 411 412 self.set_class_types() 413 self.define_instantiators() 414 self.collect_constants() 415 416 return True 417 418 # Supporting operations. 419 420 def resolve(self): 421 422 "Resolve dependencies between modules." 423 424 self.waiting = {} 425 426 for module in self.modules.values(): 427 428 # Resolve all deferred references in each module. 429 430 original_deferred = [] 431 432 for ref in module.deferred: 433 434 # Retain original references for caching. 435 436 original_deferred.append(ref.copy()) 437 438 # Update references throughout the program. 439 440 found = self.find_dependency(ref) 441 if not found: 442 self.missing.add((module.name, ref.get_origin())) 443 444 # Record the resolved names and identify required modules. 445 446 else: 447 # Find the providing module of this reference. 448 # Where definitive details of the origin cannot be found, 449 # identify the provider using the deferred reference. 450 # NOTE: This may need to test for static origins. 451 452 provider = self.get_module_provider(found.unresolved() and ref or found) 453 ref.mutate(found) 454 455 # Record any external dependency. 456 457 if provider and provider != module.name: 458 459 # Handle built-in modules accidentally referenced by 460 # names. 461 462 if provider == "__builtins__" and found.has_kind("<module>"): 463 raise ProgramError("Name %s, used by %s, refers to module %s." % 464 (found.leaf(), module.name, found.get_origin())) 465 466 # Record the provider dependency. 467 468 module.required.add(provider) 469 self.accessing_modules[provider].add(module.name) 470 471 # Postpone any inclusion of the provider until this 472 # module becomes required. 473 474 if module.name not in self.required: 475 init_item(self.waiting, module.name, set) 476 self.waiting[module.name].add(provider) 477 if self.verbose: 478 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 479 480 # Make this module required in the accessing module. 481 482 elif provider not in self.required: 483 self.required.add(provider) 484 if self.verbose: 485 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 486 487 # Record a module ordering dependency. 488 489 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 490 self.add_module_dependency(module.name, provider) 491 492 # Restore the original references so that they may be read back in 493 # and produce the same results. 494 495 module.deferred = original_deferred 496 497 # Check modules again to see if they are now required and should now 498 # cause the inclusion of other modules providing objects to the program. 499 500 for module_name in self.waiting.keys(): 501 self.require_providers(module_name) 502 503 self.add_special_dependencies() 504 self.add_module_dependencies() 505 506 def require_providers(self, module_name): 507 508 """ 509 Test if 'module_name' is itself required and, if so, require modules 510 containing objects provided to the module. 511 """ 512 513 if module_name in self.required and self.waiting.has_key(module_name): 514 for provider in self.waiting[module_name]: 515 if provider not in self.required: 516 self.required.add(provider) 517 if self.verbose: 518 print >>sys.stderr, "Requiring", provider 519 self.require_providers(provider) 520 521 def add_special_dependencies(self): 522 523 "Add dependencies due to the use of special names in namespaces." 524 525 for module in self.modules.values(): 526 for ref, paths in module.special.values(): 527 for path in paths: 528 self.add_dependency(path, ref.get_origin()) 529 530 def add_init_dependencies(self): 531 532 "Add dependencies related to object initialisation." 533 534 for name in self.classes.keys(): 535 if self.is_dynamic_class(name): 536 537 # Make subclasses depend on any class with non-static 538 # attributes, plus its module for the initialisation. 539 540 for subclass in self.subclasses[name]: 541 ref = Reference("<class>", subclass) 542 self.add_dependency(subclass, name) 543 self.add_dependency(subclass, self.get_module_provider(ref)) 544 545 # Also make the class dependent on its module for 546 # initialisation. 547 548 ref = Reference("<class>", name) 549 self.add_dependency(name, self.get_module_provider(ref)) 550 551 for name in self.function_defaults.keys(): 552 if self.is_dynamic_callable(name): 553 554 # Make functions with defaults requiring initialisation depend 555 # on the parent scope, if a function, or the module scope. 556 557 ref = Reference("<function>", name) 558 parent_ref = self.get_object(ref.parent()) 559 560 # Function no longer present in the program. 561 562 if not parent_ref: 563 continue 564 565 if parent_ref.has_kind("<class>"): 566 parent = self.get_module_provider(parent_ref) 567 else: 568 parent = parent_ref.get_origin() 569 570 self.add_dependency(name, parent) 571 572 def add_module_dependencies(self): 573 574 "Record module-based dependencies." 575 576 for module_name, providers in self.module_depends.items(): 577 if self.modules.has_key(module_name): 578 for provider in providers: 579 if self.modules.has_key(provider): 580 self.add_dependency(module_name, provider) 581 582 def add_dependency(self, path, origin): 583 584 "Add dependency details for 'path' involving 'origin'." 585 586 if origin and not origin.startswith("%s." % path): 587 init_item(self.depends, path, set) 588 self.depends[path].add(origin) 589 590 def add_module_dependency(self, module_name, provider): 591 592 "Add dependency details for 'module_name' involving 'provider'." 593 594 if provider: 595 init_item(self.module_depends, module_name, set) 596 self.module_depends[module_name].add(provider) 597 598 def condense_dependencies(self): 599 600 """ 601 Condense the dependencies by removing all functions that do not need 602 initialisation. 603 """ 604 605 d = {} 606 for path, depends in self.depends.items(): 607 d[path] = {} 608 d[path] = self.condense_dependency_entry(depends, d) 609 610 self.depends = {} 611 612 for path, depends in d.items(): 613 if depends: 614 self.depends[path] = depends 615 616 def condense_dependency_entry(self, depends, d): 617 l = set() 618 for depend in depends: 619 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 620 self.is_dynamic_callable(depend): 621 622 l.add(depend) 623 else: 624 deps = d.get(depend) 625 if deps: 626 l.update(self.condense_dependency_entry(deps, d)) 627 return l 628 629 def is_dynamic(self, ref): 630 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 631 632 def is_dynamic_class(self, name): 633 634 """ 635 Return whether 'name' refers to a class with members that must be 636 initialised dynamically. 637 """ 638 639 attrs = self.all_class_attrs.get(name) 640 641 if not attrs: 642 return False 643 644 for attrname, attr in attrs.items(): 645 if attrname in self.special_attributes: 646 continue 647 ref = attr and self.get_object(attr) 648 if self.is_dynamic(ref): 649 return True 650 651 return False 652 653 def is_dynamic_callable(self, name): 654 655 """ 656 Return whether 'name' refers to a callable employing defaults that may 657 need initialising before the callable can be used. 658 """ 659 660 # Find any defaults for the function or method. 661 662 defaults = self.function_defaults.get(name) 663 if not defaults: 664 return False 665 666 # Identify non-constant defaults. 667 668 for name, ref in defaults: 669 if self.is_dynamic(ref): 670 return True 671 672 return False 673 674 def order_objects(self): 675 676 "Produce an object initialisation ordering." 677 678 self.condense_dependencies() 679 680 try: 681 ordered = order_dependencies(self.depends) 682 except ValueError, exc: 683 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(exc.args[0].keys())) 684 685 if "__main__" in ordered: 686 ordered.remove("__main__") 687 688 ordered.append("__main__") 689 return ordered 690 691 def order_modules(self): 692 693 "Produce a module initialisation ordering." 694 695 ordered = self.order_objects() 696 filtered = [] 697 698 for module_name in self.modules.keys(): 699 if module_name not in ordered: 700 filtered.append(module_name) 701 702 for path in ordered: 703 if self.modules.has_key(path): 704 filtered.append(path) 705 706 return filtered 707 708 def find_dependency(self, ref): 709 710 "Find the ultimate dependency for 'ref'." 711 712 found = set() 713 while ref and ref.has_kind("<depends>") and not ref in found: 714 found.add(ref) 715 ref = self.identify(ref.get_origin()) 716 return ref 717 718 def get_module_provider(self, ref): 719 720 "Identify the provider of the given 'ref'." 721 722 for ancestor in ref.ancestors(): 723 if self.modules.has_key(ancestor): 724 return ancestor 725 return None 726 727 def finalise_classes(self): 728 729 "Finalise the class relationships and attributes." 730 731 self.derive_inherited_attrs() 732 self.derive_subclasses() 733 self.derive_shadowed_attrs() 734 735 def derive_inherited_attrs(self): 736 737 "Derive inherited attributes for classes throughout the program." 738 739 for name in self.classes.keys(): 740 self.propagate_attrs_for_class(name) 741 742 def propagate_attrs_for_class(self, name, visited=None): 743 744 "Propagate inherited attributes for class 'name'." 745 746 # Visit classes only once. 747 748 if self.all_combined_attrs.has_key(name): 749 return 750 751 visited = visited or [] 752 753 if name in visited: 754 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 755 756 visited.append(name) 757 758 class_attrs = {} 759 instance_attrs = {} 760 761 # Aggregate the attributes from base classes, recording the origins of 762 # applicable attributes. 763 764 for base in self.classes[name][::-1]: 765 766 # Get the identity of the class from the reference. 767 768 base = base.get_origin() 769 770 # Define the base class completely before continuing with this 771 # class. 772 773 self.propagate_attrs_for_class(base, visited) 774 class_attrs.update(self.all_class_attrs[base]) 775 776 # Instance attribute origins are combined if different. 777 778 for key, values in self.all_instance_attrs[base].items(): 779 init_item(instance_attrs, key, set) 780 instance_attrs[key].update(values) 781 782 # Class attributes override those defined earlier in the hierarchy. 783 784 class_attrs.update(self.all_class_attrs.get(name, {})) 785 786 # Instance attributes are merely added if not already defined. 787 788 for key in self.all_instance_attrs.get(name, []): 789 if not instance_attrs.has_key(key): 790 instance_attrs[key] = set(["%s.%s" % (name, key)]) 791 792 self.all_class_attrs[name] = class_attrs 793 self.all_instance_attrs[name] = instance_attrs 794 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 795 796 def derive_subclasses(self): 797 798 "Derive subclass details for classes." 799 800 for name, bases in self.classes.items(): 801 for base in bases: 802 803 # Get the identity of the class from the reference. 804 805 base = base.get_origin() 806 self.subclasses[base].add(name) 807 808 def derive_shadowed_attrs(self): 809 810 "Derive shadowed attributes for classes." 811 812 for name, attrs in self.all_instance_attrs.items(): 813 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 814 if attrs: 815 self.all_shadowed_attrs[name] = attrs 816 817 def set_class_types(self): 818 819 "Set the type of each class." 820 821 for attrs in self.all_class_attrs.values(): 822 attrs["__class__"] = self.type_ref.get_origin() 823 824 def define_instantiators(self): 825 826 """ 827 Consolidate parameter and default details, incorporating initialiser 828 details to define instantiator signatures. 829 """ 830 831 for cls, attrs in self.all_class_attrs.items(): 832 initialiser = attrs["__init__"] 833 self.function_parameters[cls] = self.function_parameters[initialiser] 834 self.function_defaults[cls] = self.function_defaults[initialiser] 835 836 def collect_constants(self): 837 838 "Get constants from all active modules." 839 840 for module in self.modules.values(): 841 self.all_constants.update(module.constants) 842 843 # Import methods. 844 845 def find_in_path(self, name): 846 847 """ 848 Find the given module 'name' in the search path, returning None where no 849 such module could be found, or a 2-tuple from the 'find' method 850 otherwise. 851 """ 852 853 for d in self.path: 854 m = self.find(d, name) 855 if m: return m 856 return None 857 858 def find(self, d, name): 859 860 """ 861 In the directory 'd', find the given module 'name', where 'name' can 862 either refer to a single file module or to a package. Return None if the 863 'name' cannot be associated with either a file or a package directory, 864 or a 2-tuple from '_find_package' or '_find_module' otherwise. 865 """ 866 867 m = self._find_package(d, name) 868 if m: return m 869 m = self._find_module(d, name) 870 if m: return m 871 return None 872 873 def _find_module(self, d, name): 874 875 """ 876 In the directory 'd', find the given module 'name', returning None where 877 no suitable file exists in the directory, or a 2-tuple consisting of 878 None (indicating that no package directory is involved) and a filename 879 indicating the location of the module. 880 """ 881 882 name_py = name + extsep + "py" 883 filename = self._find_file(d, name_py) 884 if filename: 885 return None, filename 886 return None 887 888 def _find_package(self, d, name): 889 890 """ 891 In the directory 'd', find the given package 'name', returning None 892 where no suitable package directory exists, or a 2-tuple consisting of 893 a directory (indicating the location of the package directory itself) 894 and a filename indicating the location of the __init__.py module which 895 declares the package's top-level contents. 896 """ 897 898 filename = self._find_file(d, name) 899 if filename: 900 init_py = "__init__" + extsep + "py" 901 init_py_filename = self._find_file(filename, init_py) 902 if init_py_filename: 903 return filename, init_py_filename 904 return None 905 906 def _find_file(self, d, filename): 907 908 """ 909 Return the filename obtained when searching the directory 'd' for the 910 given 'filename', or None if no actual file exists for the filename. 911 """ 912 913 filename = join(d, filename) 914 if exists(filename): 915 return filename 916 else: 917 return None 918 919 def load(self, name): 920 921 """ 922 Load the module or package with the given 'name'. Return an object 923 referencing the loaded module or package, or None if no such module or 924 package exists. 925 """ 926 927 # Loaded modules are returned immediately. 928 # Modules may be known but not yet loading (having been registered as 929 # submodules), loading, loaded, or completely unknown. 930 931 module = self.get_module(name) 932 933 if module: 934 return self.modules[name] 935 936 # Otherwise, modules are loaded. 937 938 # Split the name into path components, and try to find the uppermost in 939 # the search path. 940 941 path = name.split(".") 942 path_so_far = [] 943 module = None 944 945 for p in path: 946 947 # Get the module's filesystem details. 948 949 if not path_so_far: 950 m = self.find_in_path(p) 951 elif d: 952 m = self.find(d, p) 953 else: 954 m = None 955 956 path_so_far.append(p) 957 module_name = ".".join(path_so_far) 958 959 # Raise an exception if the module could not be located. 960 961 if not m: 962 raise ProgramError("Module not found: %s" % name) 963 964 # Get the directory and module filename. 965 966 d, filename = m 967 968 # Get the module itself. 969 970 return self.load_from_file(filename, module_name) 971 972 def load_from_file(self, filename, module_name=None): 973 974 "Load the module from the given 'filename'." 975 976 if module_name is None: 977 module_name = "__main__" 978 979 module = self.modules.get(module_name) 980 981 if not module: 982 983 # Try to load from cache. 984 985 module = self.load_from_cache(filename, module_name) 986 if module: 987 return module 988 989 # If no cache entry exists, load from file. 990 991 module = inspector.InspectedModule(module_name, self) 992 self.add_module(module_name, module) 993 self.update_cache_validity(module) 994 995 self._load(module, module_name, lambda m: m.parse, filename) 996 997 return module 998 999 def update_cache_validity(self, module): 1000 1001 "Make 'module' valid in the cache, but invalidate accessing modules." 1002 1003 accessing = self.accessing_modules.get(module.name) 1004 if accessing: 1005 self.invalidated.update(accessing) 1006 if module.name in self.invalidated: 1007 self.invalidated.remove(module.name) 1008 1009 def source_is_new(self, filename, module_name): 1010 1011 "Return whether 'filename' is newer than the cached 'module_name'." 1012 1013 if self.cache: 1014 cache_filename = join(self.cache, module_name) 1015 return not exists(cache_filename) or \ 1016 getmtime(filename) > getmtime(cache_filename) or \ 1017 module_name in self.invalidated 1018 else: 1019 return True 1020 1021 def load_from_cache(self, filename, module_name): 1022 1023 "Return a module residing in the cache." 1024 1025 module = self.modules.get(module_name) 1026 1027 if not module and not self.source_is_new(filename, module_name): 1028 module = CachedModule(module_name, self) 1029 self.add_module(module_name, module) 1030 1031 filename = join(self.cache, module_name) 1032 self._load(module, module_name, lambda m: m.from_cache, filename) 1033 1034 return module 1035 1036 def _load(self, module, module_name, fn, filename): 1037 1038 """ 1039 Load 'module' for the given 'module_name', and with 'fn' performing an 1040 invocation on the module with the given 'filename'. 1041 """ 1042 1043 # Load the module. 1044 1045 if self.verbose: 1046 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1047 fn(module)(filename) 1048 1049 # Add the module object if not already defined. 1050 1051 if not self.objects.has_key(module_name): 1052 self.objects[module_name] = Reference("<module>", module_name) 1053 1054 def add_module(self, module_name, module): 1055 1056 """ 1057 Return the module with the given 'module_name', adding a new module 1058 object if one does not already exist. 1059 """ 1060 1061 self.modules[module_name] = module 1062 if module_name in self.to_import: 1063 self.to_import.remove(module_name) 1064 1065 # vim: tabstop=4 expandtab shiftwidth=4