1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 def __init__(self, path, cache=None, verbose=False): 37 38 """ 39 Initialise the importer with the given search 'path' - a list of 40 directories to search for Python modules. 41 42 The optional 'cache' should be the name of a directory used to store 43 cached module information. 44 45 The optional 'verbose' parameter causes output concerning the activities 46 of the object to be produced if set to a true value (not the default). 47 """ 48 49 self.path = path 50 self.cache = cache 51 self.verbose = verbose 52 53 # Module importing queue, required modules, removed modules and active 54 # modules in the final program. 55 56 self.to_import = set() 57 self.required = set(["__main__"]) 58 self.removed = {} 59 self.modules = {} 60 61 # Module relationships and invalidated cached modules. 62 63 self.accessing_modules = {} 64 self.invalidated = set() 65 66 # Object relationships and dependencies. 67 68 self.depends = {} 69 self.module_depends = {} 70 71 # Basic program information. 72 73 self.objects = {} 74 self.classes = {} 75 self.function_parameters = {} 76 self.function_defaults = {} 77 self.function_locals = {} 78 self.function_targets = {} 79 self.function_arguments = {} 80 81 # Unresolved names. 82 83 self.missing = set() 84 85 # Derived information. 86 87 self.subclasses = {} 88 89 # Attributes of different object types. 90 91 self.all_class_attrs = {} 92 self.all_instance_attrs = {} 93 self.all_instance_attr_constants = {} 94 self.all_combined_attrs = {} 95 self.all_module_attrs = {} 96 self.all_shadowed_attrs = {} 97 98 # References to external names and aliases within program units. 99 100 self.all_name_references = {} 101 self.all_initialised_names = {} 102 self.all_aliased_names = {} 103 104 # General attribute accesses. 105 106 self.all_attr_accesses = {} 107 self.all_const_accesses = {} 108 self.all_attr_access_modifiers = {} 109 110 # Constant literals and values. 111 112 self.all_constants = {} 113 self.all_constant_values = {} 114 115 self.make_cache() 116 117 def make_cache(self): 118 119 "Make a cache directory if it does not already exist." 120 121 if self.cache and not exists(self.cache): 122 makedirs(self.cache) 123 124 def check_cache(self, details): 125 126 """ 127 Check whether the cache applies for the given 'details', invalidating it 128 if it does not. 129 """ 130 131 recorded_details = self.get_cache_details() 132 133 if recorded_details != details: 134 self.remove_cache() 135 136 writefile(self.get_cache_details_filename(), details) 137 138 def get_cache_details_filename(self): 139 140 "Return the filename for the cache details." 141 142 return join(self.cache, "$details") 143 144 def get_cache_details(self): 145 146 "Return details of the cache." 147 148 details_filename = self.get_cache_details_filename() 149 150 if not exists(details_filename): 151 return None 152 else: 153 return readfile(details_filename) 154 155 def remove_cache(self): 156 157 "Remove the contents of the cache." 158 159 for filename in listdir(self.cache): 160 remove(join(self.cache, filename)) 161 162 def to_cache(self): 163 164 "Write modules to the cache." 165 166 if self.cache: 167 for module_name, module in self.modules.items(): 168 module.to_cache(join(self.cache, module_name)) 169 170 # Object retrieval and storage. 171 172 def get_object(self, name): 173 174 """ 175 Return a reference for the given 'name' or None if no such object 176 exists. 177 """ 178 179 return self.objects.get(name) 180 181 def set_object(self, name, value=None): 182 183 "Set the object with the given 'name' and the given 'value'." 184 185 if isinstance(value, Reference): 186 ref = value.alias(name) 187 else: 188 ref = Reference(value, name) 189 190 self.objects[name] = ref 191 192 # Identification of both stored object names and name references. 193 194 def identify(self, name): 195 196 "Identify 'name' using stored object and external name records." 197 198 return self.objects.get(name) or self.all_name_references.get(name) 199 200 # Indirect object retrieval. 201 202 def get_attributes(self, ref, attrname): 203 204 """ 205 Return attributes provided by 'ref' for 'attrname'. Class attributes 206 may be provided by instances. 207 """ 208 209 kind = ref.get_kind() 210 if kind == "<class>": 211 ref = self.get_class_attribute(ref.get_origin(), attrname) 212 return ref and set([ref]) or set() 213 elif kind == "<instance>": 214 return self.get_combined_attributes(ref.get_origin(), attrname) 215 elif kind == "<module>": 216 ref = self.get_module_attribute(ref.get_origin(), attrname) 217 return ref and set([ref]) or set() 218 else: 219 return set() 220 221 def get_class_attribute(self, object_type, attrname): 222 223 "Return from 'object_type' the details of class attribute 'attrname'." 224 225 attrs = self.all_class_attrs.get(object_type) 226 attr = attrs and attrs.get(attrname) 227 return attr and self.get_object(attr) 228 229 def get_instance_attributes(self, object_type, attrname): 230 231 """ 232 Return from 'object_type' the details of instance attribute 'attrname'. 233 """ 234 235 consts = self.all_instance_attr_constants.get(object_type) 236 attrs = set() 237 for attr in self.all_instance_attrs[object_type].get(attrname, []): 238 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 239 return attrs 240 241 def get_combined_attributes(self, object_type, attrname): 242 243 """ 244 Return from 'object_type' the details of class or instance attribute 245 'attrname'. 246 """ 247 248 ref = self.get_class_attribute(object_type, attrname) 249 refs = ref and set([ref]) or set() 250 refs.update(self.get_instance_attributes(object_type, attrname)) 251 return refs 252 253 def get_module_attribute(self, object_type, attrname): 254 255 "Return from 'object_type' the details of module attribute 'attrname'." 256 257 if attrname in self.all_module_attrs[object_type]: 258 return self.get_object("%s.%s" % (object_type, attrname)) 259 else: 260 return None 261 262 # Convenience methods for deducing which kind of object provided an 263 # attribute. 264 265 def get_attribute_provider(self, ref, attrname): 266 267 """ 268 Return the kind of provider of the attribute accessed via 'ref' using 269 'attrname'. 270 """ 271 272 kind = ref.get_kind() 273 274 if kind in ["<class>", "<module>"]: 275 return kind 276 else: 277 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 278 279 def get_instance_attribute_provider(self, object_type, attrname): 280 281 """ 282 Return the kind of provider of the attribute accessed via an instance of 283 'object_type' using 'attrname'. 284 """ 285 286 if self.get_class_attribute(object_type, attrname): 287 return "<class>" 288 else: 289 return "<instance>" 290 291 # Module management. 292 293 def queue_module(self, name, accessor, required=False): 294 295 """ 296 Queue the module with the given 'name' for import from the given 297 'accessor' module. If 'required' is true (it is false by default), the 298 module will be required in the final program. 299 """ 300 301 if not self.modules.has_key(name): 302 self.to_import.add(name) 303 304 if required: 305 self.required.add(name) 306 307 init_item(self.accessing_modules, name, set) 308 self.accessing_modules[name].add(accessor.name) 309 310 def get_modules(self): 311 312 "Return all modules known to the importer." 313 314 return self.modules.values() 315 316 def get_module(self, name): 317 318 "Return the module with the given 'name'." 319 320 if not self.modules.has_key(name): 321 return None 322 323 return self.modules[name] 324 325 # Program operations. 326 327 def initialise(self, filename, reset=False): 328 329 """ 330 Initialise a program whose main module is 'filename', resetting the 331 cache if 'reset' is true. Return the main module. 332 """ 333 334 if reset: 335 self.remove_cache() 336 self.check_cache(filename) 337 338 # Load the program itself. 339 340 m = self.load_from_file(filename) 341 342 # Load any queued modules. 343 344 while self.to_import: 345 for name in list(self.to_import): # avoid mutation issue 346 self.load(name) 347 348 # Resolve dependencies between modules. 349 350 self.resolve() 351 352 # Record the type of all classes. 353 354 self.type_ref = self.get_object("__builtins__.type") 355 356 # Resolve dependencies within the program. 357 358 for module in self.modules.values(): 359 module.complete() 360 361 # Remove unneeded modules. 362 363 all_modules = self.modules.items() 364 365 for name, module in all_modules: 366 if name not in self.required: 367 module.unpropagate() 368 del self.modules[name] 369 self.removed[name] = module 370 371 # Collect redundant objects. 372 373 for module in self.removed.values(): 374 module.collect() 375 376 # Assert module objects where aliases have been removed. 377 378 for name in self.required: 379 if not self.objects.has_key(name): 380 self.objects[name] = Reference("<module>", name) 381 382 return m 383 384 def finalise(self): 385 386 """ 387 Finalise the inspected program, returning whether the program could be 388 finalised. 389 """ 390 391 self.finalise_classes() 392 self.add_init_dependencies() 393 self.to_cache() 394 395 if self.missing: 396 return False 397 398 self.set_class_types() 399 self.define_instantiators() 400 self.collect_constants() 401 402 return True 403 404 # Supporting operations. 405 406 def resolve(self): 407 408 "Resolve dependencies between modules." 409 410 self.waiting = {} 411 412 for module in self.modules.values(): 413 414 # Resolve all deferred references in each module. 415 416 original_deferred = [] 417 418 for ref in module.deferred: 419 420 # Retain original references for caching. 421 422 original_deferred.append(ref.copy()) 423 424 # Update references throughout the program. 425 426 found = self.find_dependency(ref) 427 if not found: 428 self.missing.add((module.name, ref.get_origin())) 429 430 # Record the resolved names and identify required modules. 431 432 else: 433 # Find the providing module of this reference. 434 # Where definitive details of the origin cannot be found, 435 # identify the provider using the deferred reference. 436 # NOTE: This may need to test for static origins. 437 438 provider = self.get_module_provider(found.unresolved() and ref or found) 439 ref.mutate(found) 440 441 # Record any external dependency. 442 443 if provider and provider != module.name: 444 445 # Record the provider dependency. 446 447 module.required.add(provider) 448 self.accessing_modules[provider].add(module.name) 449 450 # Postpone any inclusion of the provider until this 451 # module becomes required. 452 453 if module.name not in self.required: 454 init_item(self.waiting, module.name, set) 455 self.waiting[module.name].add(provider) 456 if self.verbose: 457 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 458 459 # Make this module required in the accessing module. 460 461 elif provider not in self.required: 462 self.required.add(provider) 463 if self.verbose: 464 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 465 466 # Record a module ordering dependency. 467 468 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 469 self.add_module_dependency(module.name, provider) 470 471 # Restore the original references so that they may be read back in 472 # and produce the same results. 473 474 module.deferred = original_deferred 475 476 # Check modules again to see if they are now required and should now 477 # cause the inclusion of other modules providing objects to the program. 478 479 for module_name in self.waiting.keys(): 480 self.require_providers(module_name) 481 482 self.add_special_dependencies() 483 self.add_module_dependencies() 484 485 def require_providers(self, module_name): 486 487 """ 488 Test if 'module_name' is itself required and, if so, require modules 489 containing objects provided to the module. 490 """ 491 492 if module_name in self.required and self.waiting.has_key(module_name): 493 for provider in self.waiting[module_name]: 494 if provider not in self.required: 495 self.required.add(provider) 496 if self.verbose: 497 print >>sys.stderr, "Requiring", provider 498 self.require_providers(provider) 499 500 def add_special_dependencies(self): 501 502 "Add dependencies due to the use of special names in namespaces." 503 504 for module in self.modules.values(): 505 for ref, paths in module.special.values(): 506 for path in paths: 507 self.add_dependency(path, ref.get_origin()) 508 509 def add_init_dependencies(self): 510 511 "Add dependencies related to object initialisation." 512 513 for name in self.classes.keys(): 514 if self.is_dynamic_class(name): 515 516 # Make subclasses depend on any class with non-static 517 # attributes, plus its module for the initialisation. 518 519 for subclass in self.subclasses[name]: 520 ref = Reference("<class>", subclass) 521 self.add_dependency(subclass, name) 522 self.add_dependency(subclass, self.get_module_provider(ref)) 523 524 # Also make the class dependent on its module for 525 # initialisation. 526 527 ref = Reference("<class>", name) 528 self.add_dependency(name, self.get_module_provider(ref)) 529 530 for name in self.function_defaults.keys(): 531 if self.is_dynamic_callable(name): 532 533 # Make functions with defaults requiring initialisation depend 534 # on the parent scope, if a function, or the module scope. 535 536 ref = Reference("<function>", name) 537 parent_ref = self.get_object(ref.parent()) 538 539 # Function no longer present in the program. 540 541 if not parent_ref: 542 continue 543 544 if parent_ref.has_kind("<class>"): 545 parent = self.get_module_provider(parent_ref) 546 else: 547 parent = parent_ref.get_origin() 548 549 self.add_dependency(name, parent) 550 551 def add_module_dependencies(self): 552 553 "Record module-based dependencies." 554 555 for module_name, providers in self.module_depends.items(): 556 if self.modules.has_key(module_name): 557 for provider in providers: 558 if self.modules.has_key(provider): 559 self.add_dependency(module_name, provider) 560 561 def add_dependency(self, path, origin): 562 563 "Add dependency details for 'path' involving 'origin'." 564 565 if origin and not origin.startswith("%s." % path): 566 init_item(self.depends, path, set) 567 self.depends[path].add(origin) 568 569 def add_module_dependency(self, module_name, provider): 570 571 "Add dependency details for 'module_name' involving 'provider'." 572 573 if provider: 574 init_item(self.module_depends, module_name, set) 575 self.module_depends[module_name].add(provider) 576 577 def condense_dependencies(self): 578 579 """ 580 Condense the dependencies by removing all functions that do not need 581 initialisation. 582 """ 583 584 d = {} 585 for path, depends in self.depends.items(): 586 d[path] = {} 587 d[path] = self.condense_dependency_entry(depends, d) 588 589 self.depends = {} 590 591 for path, depends in d.items(): 592 if depends: 593 self.depends[path] = depends 594 595 def condense_dependency_entry(self, depends, d): 596 l = set() 597 for depend in depends: 598 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 599 self.is_dynamic_callable(depend): 600 601 l.add(depend) 602 else: 603 deps = d.get(depend) 604 if deps: 605 l.update(self.condense_dependency_entry(deps, d)) 606 return l 607 608 # NOTE: Consolidate this information in a common location. 609 610 special_attributes = ("__args__", "__file__", "__fn__", "__fname__", "__mname__", "__name__") 611 612 def is_dynamic(self, ref): 613 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 614 615 def is_dynamic_class(self, name): 616 617 """ 618 Return whether 'name' refers to a class with members that must be 619 initialised dynamically. 620 """ 621 622 attrs = self.all_class_attrs.get(name) 623 624 if not attrs: 625 return False 626 627 for attrname, attr in attrs.items(): 628 if attrname in self.special_attributes: 629 continue 630 ref = attr and self.get_object(attr) 631 if self.is_dynamic(ref): 632 return True 633 634 return False 635 636 def is_dynamic_callable(self, name): 637 638 """ 639 Return whether 'name' refers to a callable employing defaults that may 640 need initialising before the callable can be used. 641 """ 642 643 # Find any defaults for the function or method. 644 645 defaults = self.function_defaults.get(name) 646 if not defaults: 647 return False 648 649 # Identify non-constant defaults. 650 651 for name, ref in defaults: 652 if self.is_dynamic(ref): 653 return True 654 655 return False 656 657 def order_objects(self): 658 659 "Produce an object initialisation ordering." 660 661 self.condense_dependencies() 662 663 # Record the number of modules using or depending on each module. 664 665 usage = {} 666 667 # Record path-based dependencies. 668 669 for path in self.depends.keys(): 670 usage[path] = set() 671 672 for path, depends in self.depends.items(): 673 for origin in depends: 674 init_item(usage, origin, set) 675 usage[origin].add(path) 676 677 # Produce an ordering by obtaining exposed modules (required by modules 678 # already processed) and putting them at the start of the list. 679 680 ordered = [] 681 682 while usage: 683 have_next = False 684 685 for path, n in usage.items(): 686 if not n: 687 ordered.insert(0, path) 688 depends = self.depends.get(path) 689 690 # Reduce usage of the referenced objects. 691 692 if depends: 693 for origin in depends: 694 usage[origin].remove(path) 695 696 del usage[path] 697 have_next = True 698 699 if not have_next: 700 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(usage.keys())) 701 702 if "__main__" in ordered: 703 ordered.remove("__main__") 704 705 ordered.append("__main__") 706 return ordered 707 708 def order_modules(self): 709 710 "Produce a module initialisation ordering." 711 712 ordered = self.order_objects() 713 filtered = [] 714 715 for module_name in self.modules.keys(): 716 if module_name not in ordered: 717 filtered.append(module_name) 718 719 for path in ordered: 720 if self.modules.has_key(path): 721 filtered.append(path) 722 723 return filtered 724 725 def find_dependency(self, ref): 726 727 "Find the ultimate dependency for 'ref'." 728 729 found = set() 730 while ref and ref.has_kind("<depends>") and not ref in found: 731 found.add(ref) 732 ref = self.identify(ref.get_origin()) 733 return ref 734 735 def get_module_provider(self, ref): 736 737 "Identify the provider of the given 'ref'." 738 739 for ancestor in ref.ancestors(): 740 if self.modules.has_key(ancestor): 741 return ancestor 742 return None 743 744 def finalise_classes(self): 745 746 "Finalise the class relationships and attributes." 747 748 self.derive_inherited_attrs() 749 self.derive_subclasses() 750 self.derive_shadowed_attrs() 751 752 def derive_inherited_attrs(self): 753 754 "Derive inherited attributes for classes throughout the program." 755 756 for name in self.classes.keys(): 757 self.propagate_attrs_for_class(name) 758 759 def propagate_attrs_for_class(self, name, visited=None): 760 761 "Propagate inherited attributes for class 'name'." 762 763 # Visit classes only once. 764 765 if self.all_combined_attrs.has_key(name): 766 return 767 768 visited = visited or [] 769 770 if name in visited: 771 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 772 773 visited.append(name) 774 775 class_attrs = {} 776 instance_attrs = {} 777 778 # Aggregate the attributes from base classes, recording the origins of 779 # applicable attributes. 780 781 for base in self.classes[name][::-1]: 782 783 # Get the identity of the class from the reference. 784 785 base = base.get_origin() 786 787 # Define the base class completely before continuing with this 788 # class. 789 790 self.propagate_attrs_for_class(base, visited) 791 class_attrs.update(self.all_class_attrs[base]) 792 793 # Instance attribute origins are combined if different. 794 795 for key, values in self.all_instance_attrs[base].items(): 796 init_item(instance_attrs, key, set) 797 instance_attrs[key].update(values) 798 799 # Class attributes override those defined earlier in the hierarchy. 800 801 class_attrs.update(self.all_class_attrs.get(name, {})) 802 803 # Instance attributes are merely added if not already defined. 804 805 for key in self.all_instance_attrs.get(name, []): 806 if not instance_attrs.has_key(key): 807 instance_attrs[key] = set(["%s.%s" % (name, key)]) 808 809 self.all_class_attrs[name] = class_attrs 810 self.all_instance_attrs[name] = instance_attrs 811 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 812 813 def derive_subclasses(self): 814 815 "Derive subclass details for classes." 816 817 for name, bases in self.classes.items(): 818 for base in bases: 819 820 # Get the identity of the class from the reference. 821 822 base = base.get_origin() 823 self.subclasses[base].add(name) 824 825 def derive_shadowed_attrs(self): 826 827 "Derive shadowed attributes for classes." 828 829 for name, attrs in self.all_instance_attrs.items(): 830 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 831 if attrs: 832 self.all_shadowed_attrs[name] = attrs 833 834 def set_class_types(self): 835 836 "Set the type of each class." 837 838 for attrs in self.all_class_attrs.values(): 839 attrs["__class__"] = self.type_ref.get_origin() 840 841 def define_instantiators(self): 842 843 """ 844 Consolidate parameter and default details, incorporating initialiser 845 details to define instantiator signatures. 846 """ 847 848 for cls, attrs in self.all_class_attrs.items(): 849 initialiser = attrs["__init__"] 850 self.function_parameters[cls] = self.function_parameters[initialiser] 851 self.function_defaults[cls] = self.function_defaults[initialiser] 852 853 def collect_constants(self): 854 855 "Get constants from all active modules." 856 857 for module in self.modules.values(): 858 self.all_constants.update(module.constants) 859 860 # Import methods. 861 862 def find_in_path(self, name): 863 864 """ 865 Find the given module 'name' in the search path, returning None where no 866 such module could be found, or a 2-tuple from the 'find' method 867 otherwise. 868 """ 869 870 for d in self.path: 871 m = self.find(d, name) 872 if m: return m 873 return None 874 875 def find(self, d, name): 876 877 """ 878 In the directory 'd', find the given module 'name', where 'name' can 879 either refer to a single file module or to a package. Return None if the 880 'name' cannot be associated with either a file or a package directory, 881 or a 2-tuple from '_find_package' or '_find_module' otherwise. 882 """ 883 884 m = self._find_package(d, name) 885 if m: return m 886 m = self._find_module(d, name) 887 if m: return m 888 return None 889 890 def _find_module(self, d, name): 891 892 """ 893 In the directory 'd', find the given module 'name', returning None where 894 no suitable file exists in the directory, or a 2-tuple consisting of 895 None (indicating that no package directory is involved) and a filename 896 indicating the location of the module. 897 """ 898 899 name_py = name + extsep + "py" 900 filename = self._find_file(d, name_py) 901 if filename: 902 return None, filename 903 return None 904 905 def _find_package(self, d, name): 906 907 """ 908 In the directory 'd', find the given package 'name', returning None 909 where no suitable package directory exists, or a 2-tuple consisting of 910 a directory (indicating the location of the package directory itself) 911 and a filename indicating the location of the __init__.py module which 912 declares the package's top-level contents. 913 """ 914 915 filename = self._find_file(d, name) 916 if filename: 917 init_py = "__init__" + extsep + "py" 918 init_py_filename = self._find_file(filename, init_py) 919 if init_py_filename: 920 return filename, init_py_filename 921 return None 922 923 def _find_file(self, d, filename): 924 925 """ 926 Return the filename obtained when searching the directory 'd' for the 927 given 'filename', or None if no actual file exists for the filename. 928 """ 929 930 filename = join(d, filename) 931 if exists(filename): 932 return filename 933 else: 934 return None 935 936 def load(self, name): 937 938 """ 939 Load the module or package with the given 'name'. Return an object 940 referencing the loaded module or package, or None if no such module or 941 package exists. 942 """ 943 944 # Loaded modules are returned immediately. 945 # Modules may be known but not yet loading (having been registered as 946 # submodules), loading, loaded, or completely unknown. 947 948 module = self.get_module(name) 949 950 if module: 951 return self.modules[name] 952 953 # Otherwise, modules are loaded. 954 955 # Split the name into path components, and try to find the uppermost in 956 # the search path. 957 958 path = name.split(".") 959 path_so_far = [] 960 module = None 961 962 for p in path: 963 964 # Get the module's filesystem details. 965 966 if not path_so_far: 967 m = self.find_in_path(p) 968 elif d: 969 m = self.find(d, p) 970 else: 971 m = None 972 973 path_so_far.append(p) 974 module_name = ".".join(path_so_far) 975 976 # Raise an exception if the module could not be located. 977 978 if not m: 979 raise ProgramError("Module not found: %s" % name) 980 981 # Get the directory and module filename. 982 983 d, filename = m 984 985 # Get the module itself. 986 987 return self.load_from_file(filename, module_name) 988 989 def load_from_file(self, filename, module_name=None): 990 991 "Load the module from the given 'filename'." 992 993 if module_name is None: 994 module_name = "__main__" 995 996 module = self.modules.get(module_name) 997 998 if not module: 999 1000 # Try to load from cache. 1001 1002 module = self.load_from_cache(filename, module_name) 1003 if module: 1004 return module 1005 1006 # If no cache entry exists, load from file. 1007 1008 module = inspector.InspectedModule(module_name, self) 1009 self.add_module(module_name, module) 1010 self.update_cache_validity(module) 1011 1012 self._load(module, module_name, lambda m: m.parse, filename) 1013 1014 return module 1015 1016 def update_cache_validity(self, module): 1017 1018 "Make 'module' valid in the cache, but invalidate accessing modules." 1019 1020 accessing = self.accessing_modules.get(module.name) 1021 if accessing: 1022 self.invalidated.update(accessing) 1023 if module.name in self.invalidated: 1024 self.invalidated.remove(module.name) 1025 1026 def source_is_new(self, filename, module_name): 1027 1028 "Return whether 'filename' is newer than the cached 'module_name'." 1029 1030 if self.cache: 1031 cache_filename = join(self.cache, module_name) 1032 return not exists(cache_filename) or \ 1033 getmtime(filename) > getmtime(cache_filename) or \ 1034 module_name in self.invalidated 1035 else: 1036 return True 1037 1038 def load_from_cache(self, filename, module_name): 1039 1040 "Return a module residing in the cache." 1041 1042 module = self.modules.get(module_name) 1043 1044 if not module and not self.source_is_new(filename, module_name): 1045 module = CachedModule(module_name, self) 1046 self.add_module(module_name, module) 1047 1048 filename = join(self.cache, module_name) 1049 self._load(module, module_name, lambda m: m.from_cache, filename) 1050 1051 return module 1052 1053 def _load(self, module, module_name, fn, filename): 1054 1055 """ 1056 Load 'module' for the given 'module_name', and with 'fn' performing an 1057 invocation on the module with the given 'filename'. 1058 """ 1059 1060 # Load the module. 1061 1062 if self.verbose: 1063 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1064 fn(module)(filename) 1065 1066 # Add the module object if not already defined. 1067 1068 if not self.objects.has_key(module_name): 1069 self.objects[module_name] = Reference("<module>", module_name) 1070 1071 def add_module(self, module_name, module): 1072 1073 """ 1074 Return the module with the given 'module_name', adding a new module 1075 object if one does not already exist. 1076 """ 1077 1078 self.modules[module_name] = module 1079 if module_name in self.to_import: 1080 self.to_import.remove(module_name) 1081 1082 # vim: tabstop=4 expandtab shiftwidth=4