1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, order_dependencies, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__") 37 38 def __init__(self, path, cache=None, verbose=False, warnings=None): 39 40 """ 41 Initialise the importer with the given search 'path' - a list of 42 directories to search for Python modules. 43 44 The optional 'cache' should be the name of a directory used to store 45 cached module information. 46 47 The optional 'verbose' parameter causes output concerning the activities 48 of the object to be produced if set to a true value (not the default). 49 50 The optional 'warnings' parameter may indicate classes of warnings to be 51 produced. 52 """ 53 54 self.path = path 55 self.cache = cache 56 self.verbose = verbose 57 self.warnings = warnings 58 59 # Module importing queue, required modules, removed modules and active 60 # modules in the final program. 61 62 self.to_import = set() 63 self.required = set(["__main__"]) 64 self.removed = {} 65 self.modules = {} 66 67 # Module relationships and invalidated cached modules. 68 69 self.accessing_modules = {} 70 self.invalidated = set() 71 72 # Object relationships and dependencies. 73 74 self.depends = {} 75 self.module_depends = {} 76 77 # Basic program information. 78 79 self.objects = {} 80 self.classes = {} 81 self.function_parameters = {} 82 self.function_attr_initialisers = {} 83 self.function_defaults = {} 84 self.function_locals = {} 85 86 # Unresolved names. 87 88 self.missing = set() 89 90 # Derived information. 91 92 self.subclasses = {} 93 94 # Attributes of different object types. 95 96 self.all_class_attrs = {} 97 self.all_instance_attrs = {} 98 self.all_instance_attr_constants = {} 99 self.all_combined_attrs = {} 100 self.all_module_attrs = {} 101 self.all_shadowed_attrs = {} 102 103 # References to external names and aliases within program units. 104 105 self.all_name_references = {} 106 self.all_initialised_names = {} 107 self.all_aliased_names = {} 108 109 # General attribute accesses. 110 111 self.all_attr_accesses = {} 112 self.all_const_accesses = {} 113 self.all_attr_access_modifiers = {} 114 115 # Constant literals and values. 116 117 self.all_constants = {} 118 self.all_constant_values = {} 119 120 # Common function value collection used during inspection. 121 122 self.all_return_values = {} 123 124 self.make_cache() 125 126 def give_warning(self, name): 127 128 "Return whether the indicated warning 'name' should be given." 129 130 return self.warnings and (name in self.warnings or "all" in self.warnings) 131 132 def make_cache(self): 133 134 "Make a cache directory if it does not already exist." 135 136 if self.cache and not exists(self.cache): 137 makedirs(self.cache) 138 139 def check_cache(self, details): 140 141 """ 142 Check whether the cache applies for the given 'details', invalidating it 143 if it does not. 144 """ 145 146 recorded_details = self.get_cache_details() 147 148 if recorded_details != details: 149 self.remove_cache() 150 151 writefile(self.get_cache_details_filename(), details) 152 153 def get_cache_details_filename(self): 154 155 "Return the filename for the cache details." 156 157 return join(self.cache, "$details") 158 159 def get_cache_details(self): 160 161 "Return details of the cache." 162 163 details_filename = self.get_cache_details_filename() 164 165 if not exists(details_filename): 166 return None 167 else: 168 return readfile(details_filename) 169 170 def remove_cache(self): 171 172 "Remove the contents of the cache." 173 174 for filename in listdir(self.cache): 175 remove(join(self.cache, filename)) 176 177 def to_cache(self): 178 179 "Write modules to the cache." 180 181 if self.cache: 182 for module_name, module in self.modules.items(): 183 module.to_cache(join(self.cache, module_name)) 184 185 # Object retrieval and storage. 186 187 def get_object(self, name): 188 189 """ 190 Return a reference for the given 'name' or None if no such object 191 exists. 192 """ 193 194 return self.objects.get(name) 195 196 def set_object(self, name, value=None): 197 198 "Set the object with the given 'name' and the given 'value'." 199 200 if isinstance(value, Reference): 201 ref = value.alias(name) 202 else: 203 ref = Reference(value, name) 204 205 self.objects[name] = ref 206 207 # Identification of both stored object names and name references. 208 209 def identify(self, name): 210 211 "Identify 'name' using stored object and external name records." 212 213 return self.objects.get(name) or self.all_name_references.get(name) 214 215 # Indirect object retrieval. 216 217 def get_attributes(self, ref, attrname): 218 219 """ 220 Return attributes provided by 'ref' for 'attrname'. Class attributes 221 may be provided by instances. 222 """ 223 224 kind = ref.get_kind() 225 if kind == "<class>": 226 ref = self.get_class_attribute(ref.get_origin(), attrname) 227 return ref and set([ref]) or set() 228 elif kind == "<instance>": 229 return self.get_combined_attributes(ref.get_origin(), attrname) 230 elif kind == "<module>": 231 ref = self.get_module_attribute(ref.get_origin(), attrname) 232 return ref and set([ref]) or set() 233 else: 234 return set() 235 236 def get_class_attribute(self, object_type, attrname): 237 238 "Return from 'object_type' the details of class attribute 'attrname'." 239 240 attrs = self.all_class_attrs.get(object_type) 241 attr = attrs and attrs.get(attrname) 242 return attr and self.get_object(attr) 243 244 def get_instance_attributes(self, object_type, attrname): 245 246 """ 247 Return from 'object_type' the details of instance attribute 'attrname'. 248 """ 249 250 consts = self.all_instance_attr_constants.get(object_type) 251 attrs = set() 252 for attr in self.all_instance_attrs[object_type].get(attrname, []): 253 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 254 return attrs 255 256 def get_combined_attributes(self, object_type, attrname): 257 258 """ 259 Return from 'object_type' the details of class or instance attribute 260 'attrname'. 261 """ 262 263 ref = self.get_class_attribute(object_type, attrname) 264 refs = ref and set([ref]) or set() 265 refs.update(self.get_instance_attributes(object_type, attrname)) 266 return refs 267 268 def get_module_attribute(self, object_type, attrname): 269 270 "Return from 'object_type' the details of module attribute 'attrname'." 271 272 if attrname in self.all_module_attrs[object_type]: 273 return self.get_object("%s.%s" % (object_type, attrname)) 274 else: 275 return None 276 277 # Convenience methods for deducing which kind of object provided an 278 # attribute. 279 280 def get_attribute_provider(self, ref, attrname): 281 282 """ 283 Return the kind of provider of the attribute accessed via 'ref' using 284 'attrname'. 285 """ 286 287 kind = ref.get_kind() 288 289 if kind in ["<class>", "<module>"]: 290 return kind 291 else: 292 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 293 294 def get_instance_attribute_provider(self, object_type, attrname): 295 296 """ 297 Return the kind of provider of the attribute accessed via an instance of 298 'object_type' using 'attrname'. 299 """ 300 301 if self.get_class_attribute(object_type, attrname): 302 return "<class>" 303 else: 304 return "<instance>" 305 306 # Module management. 307 308 def queue_module(self, name, accessor, required=False): 309 310 """ 311 Queue the module with the given 'name' for import from the given 312 'accessor' module. If 'required' is true (it is false by default), the 313 module will be required in the final program. 314 """ 315 316 if not self.modules.has_key(name): 317 self.to_import.add(name) 318 319 if required: 320 self.required.add(name) 321 322 init_item(self.accessing_modules, name, set) 323 self.accessing_modules[name].add(accessor.name) 324 325 def get_modules(self): 326 327 "Return all modules known to the importer." 328 329 return self.modules.values() 330 331 def get_module(self, name): 332 333 "Return the module with the given 'name'." 334 335 if not self.modules.has_key(name): 336 return None 337 338 return self.modules[name] 339 340 # Program operations. 341 342 def initialise(self, filename, reset=False): 343 344 """ 345 Initialise a program whose main module is 'filename', resetting the 346 cache if 'reset' is true. Return the main module. 347 """ 348 349 if reset: 350 self.remove_cache() 351 self.check_cache(filename) 352 353 # Load the program itself. 354 355 m = self.load_from_file(filename) 356 357 # Load any queued modules. 358 359 while self.to_import: 360 for name in list(self.to_import): # avoid mutation issue 361 self.load(name) 362 363 # Resolve dependencies between modules. 364 365 self.resolve() 366 367 # Record the type of all classes. 368 369 self.type_ref = self.get_object("__builtins__.type") 370 371 # Resolve dependencies within the program. 372 373 for module in self.modules.values(): 374 module.complete() 375 376 # Remove unneeded modules. 377 378 all_modules = self.modules.items() 379 380 for name, module in all_modules: 381 if name not in self.required: 382 module.unpropagate() 383 del self.modules[name] 384 self.removed[name] = module 385 386 # Collect redundant objects. 387 388 for module in self.removed.values(): 389 module.collect() 390 391 # Assert module objects where aliases have been removed. 392 393 for name in self.required: 394 if not self.objects.has_key(name): 395 self.objects[name] = Reference("<module>", name) 396 397 return m 398 399 def finalise(self): 400 401 """ 402 Finalise the inspected program, returning whether the program could be 403 finalised. 404 """ 405 406 self.finalise_classes() 407 self.add_init_dependencies() 408 self.to_cache() 409 410 if self.missing: 411 return False 412 413 self.set_class_types() 414 self.define_instantiators() 415 self.collect_constants() 416 417 return True 418 419 # Supporting operations. 420 421 def resolve(self): 422 423 "Resolve dependencies between modules." 424 425 self.waiting = {} 426 427 for module in self.modules.values(): 428 429 # Resolve all deferred references in each module. 430 431 original_deferred = [] 432 433 for ref in module.deferred: 434 435 # Retain original references for caching. 436 437 original_deferred.append(ref.copy()) 438 439 # Update references throughout the program. 440 441 found = self.find_dependency(ref) 442 if not found: 443 self.missing.add((module.name, ref.get_origin())) 444 445 # Record the resolved names and identify required modules. 446 447 else: 448 # Find the providing module of this reference. 449 # Where definitive details of the origin cannot be found, 450 # identify the provider using the deferred reference. 451 # NOTE: This may need to test for static origins. 452 453 provider = self.get_module_provider(found.unresolved() and ref or found) 454 ref.mutate(found) 455 456 # Record any external dependency. 457 458 if provider and provider != module.name: 459 460 # Handle built-in modules accidentally referenced by 461 # names. 462 463 if provider == "__builtins__" and found.has_kind("<module>"): 464 raise ProgramError("Name %s, used by %s, refers to module %s." % 465 (found.leaf(), module.name, found.get_origin())) 466 467 # Record the provider dependency. 468 469 module.required.add(provider) 470 self.accessing_modules[provider].add(module.name) 471 472 # Postpone any inclusion of the provider until this 473 # module becomes required. 474 475 if module.name not in self.required: 476 init_item(self.waiting, module.name, set) 477 self.waiting[module.name].add(provider) 478 if self.verbose: 479 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 480 481 # Make this module required in the accessing module. 482 483 elif provider not in self.required: 484 self.required.add(provider) 485 if self.verbose: 486 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 487 488 # Record a module ordering dependency. 489 490 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 491 self.add_module_dependency(module.name, provider) 492 493 # Restore the original references so that they may be read back in 494 # and produce the same results. 495 496 module.deferred = original_deferred 497 498 # Check modules again to see if they are now required and should now 499 # cause the inclusion of other modules providing objects to the program. 500 501 for module_name in self.waiting.keys(): 502 self.require_providers(module_name) 503 504 self.add_special_dependencies() 505 self.add_module_dependencies() 506 507 def require_providers(self, module_name): 508 509 """ 510 Test if 'module_name' is itself required and, if so, require modules 511 containing objects provided to the module. 512 """ 513 514 if module_name in self.required and self.waiting.has_key(module_name): 515 for provider in self.waiting[module_name]: 516 if provider not in self.required: 517 self.required.add(provider) 518 if self.verbose: 519 print >>sys.stderr, "Requiring", provider 520 self.require_providers(provider) 521 522 def add_special_dependencies(self): 523 524 "Add dependencies due to the use of special names in namespaces." 525 526 for module in self.modules.values(): 527 for ref, paths in module.special.values(): 528 for path in paths: 529 self.add_dependency(path, ref.get_origin()) 530 531 def add_init_dependencies(self): 532 533 "Add dependencies related to object initialisation." 534 535 for name in self.classes.keys(): 536 if self.is_dynamic_class(name): 537 538 # Make subclasses depend on any class with non-static 539 # attributes, plus its module for the initialisation. 540 541 for subclass in self.subclasses[name]: 542 ref = Reference("<class>", subclass) 543 self.add_dependency(subclass, name) 544 self.add_dependency(subclass, self.get_module_provider(ref)) 545 546 # Also make the class dependent on its module for 547 # initialisation. 548 549 ref = Reference("<class>", name) 550 self.add_dependency(name, self.get_module_provider(ref)) 551 552 for name in self.function_defaults.keys(): 553 if self.is_dynamic_callable(name): 554 555 # Make functions with defaults requiring initialisation depend 556 # on the parent scope, if a function, or the module scope. 557 558 ref = Reference("<function>", name) 559 parent_ref = self.get_object(ref.parent()) 560 561 # Function no longer present in the program. 562 563 if not parent_ref: 564 continue 565 566 if parent_ref.has_kind("<class>"): 567 parent = self.get_module_provider(parent_ref) 568 else: 569 parent = parent_ref.get_origin() 570 571 self.add_dependency(name, parent) 572 573 def add_module_dependencies(self): 574 575 "Record module-based dependencies." 576 577 for module_name, providers in self.module_depends.items(): 578 if self.modules.has_key(module_name): 579 for provider in providers: 580 if self.modules.has_key(provider): 581 self.add_dependency(module_name, provider) 582 583 def add_dependency(self, path, origin): 584 585 "Add dependency details for 'path' involving 'origin'." 586 587 if origin and not origin.startswith("%s." % path): 588 init_item(self.depends, path, set) 589 self.depends[path].add(origin) 590 591 def add_module_dependency(self, module_name, provider): 592 593 "Add dependency details for 'module_name' involving 'provider'." 594 595 if provider: 596 init_item(self.module_depends, module_name, set) 597 self.module_depends[module_name].add(provider) 598 599 def condense_dependencies(self): 600 601 """ 602 Condense the dependencies by removing all functions that do not need 603 initialisation. 604 """ 605 606 d = {} 607 for path, depends in self.depends.items(): 608 d[path] = {} 609 d[path] = self.condense_dependency_entry(depends, d) 610 611 self.depends = {} 612 613 for path, depends in d.items(): 614 if depends: 615 self.depends[path] = depends 616 617 def condense_dependency_entry(self, depends, d): 618 l = set() 619 for depend in depends: 620 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 621 self.is_dynamic_callable(depend): 622 623 l.add(depend) 624 else: 625 deps = d.get(depend) 626 if deps: 627 l.update(self.condense_dependency_entry(deps, d)) 628 return l 629 630 def is_dynamic(self, ref): 631 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 632 633 def is_dynamic_class(self, name): 634 635 """ 636 Return whether 'name' refers to a class with members that must be 637 initialised dynamically. 638 """ 639 640 attrs = self.all_class_attrs.get(name) 641 642 if not attrs: 643 return False 644 645 for attrname, attr in attrs.items(): 646 if attrname in self.special_attributes: 647 continue 648 ref = attr and self.get_object(attr) 649 if self.is_dynamic(ref): 650 return True 651 652 return False 653 654 def is_dynamic_callable(self, name): 655 656 """ 657 Return whether 'name' refers to a callable employing defaults that may 658 need initialising before the callable can be used. 659 """ 660 661 # Find any defaults for the function or method. 662 663 defaults = self.function_defaults.get(name) 664 if not defaults: 665 return False 666 667 # Identify non-constant defaults. 668 669 for name, ref in defaults: 670 if self.is_dynamic(ref): 671 return True 672 673 return False 674 675 def order_objects(self): 676 677 "Produce an object initialisation ordering." 678 679 self.condense_dependencies() 680 681 try: 682 ordered = order_dependencies(self.depends) 683 except ValueError, exc: 684 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(exc.args[0].keys())) 685 686 if "__main__" in ordered: 687 ordered.remove("__main__") 688 689 ordered.append("__main__") 690 return ordered 691 692 def order_modules(self): 693 694 "Produce a module initialisation ordering." 695 696 ordered = self.order_objects() 697 filtered = [] 698 699 for module_name in self.modules.keys(): 700 if module_name not in ordered: 701 filtered.append(module_name) 702 703 for path in ordered: 704 if self.modules.has_key(path): 705 filtered.append(path) 706 707 return filtered 708 709 def find_dependency(self, ref): 710 711 "Find the ultimate dependency for 'ref'." 712 713 found = set() 714 while ref and ref.has_kind("<depends>") and not ref in found: 715 found.add(ref) 716 ref = self.identify(ref.get_origin()) 717 return ref 718 719 def get_module_provider(self, ref): 720 721 "Identify the provider of the given 'ref'." 722 723 for ancestor in ref.ancestors(): 724 if self.modules.has_key(ancestor): 725 return ancestor 726 return None 727 728 def finalise_classes(self): 729 730 "Finalise the class relationships and attributes." 731 732 self.derive_inherited_attrs() 733 self.derive_subclasses() 734 self.derive_shadowed_attrs() 735 736 def derive_inherited_attrs(self): 737 738 "Derive inherited attributes for classes throughout the program." 739 740 for name in self.classes.keys(): 741 self.propagate_attrs_for_class(name) 742 743 def propagate_attrs_for_class(self, name, visited=None): 744 745 "Propagate inherited attributes for class 'name'." 746 747 # Visit classes only once. 748 749 if self.all_combined_attrs.has_key(name): 750 return 751 752 visited = visited or [] 753 754 if name in visited: 755 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 756 757 visited.append(name) 758 759 class_attrs = {} 760 instance_attrs = {} 761 762 # Aggregate the attributes from base classes, recording the origins of 763 # applicable attributes. 764 765 for base in self.classes[name][::-1]: 766 767 # Get the identity of the class from the reference. 768 769 base = base.get_origin() 770 771 # Define the base class completely before continuing with this 772 # class. 773 774 self.propagate_attrs_for_class(base, visited) 775 class_attrs.update(self.all_class_attrs[base]) 776 777 # Instance attribute origins are combined if different. 778 779 for key, values in self.all_instance_attrs[base].items(): 780 init_item(instance_attrs, key, set) 781 instance_attrs[key].update(values) 782 783 # Class attributes override those defined earlier in the hierarchy. 784 785 class_attrs.update(self.all_class_attrs.get(name, {})) 786 787 # Instance attributes are merely added if not already defined. 788 789 for key in self.all_instance_attrs.get(name, []): 790 if not instance_attrs.has_key(key): 791 instance_attrs[key] = set(["%s.%s" % (name, key)]) 792 793 self.all_class_attrs[name] = class_attrs 794 self.all_instance_attrs[name] = instance_attrs 795 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 796 797 def derive_subclasses(self): 798 799 "Derive subclass details for classes." 800 801 for name, bases in self.classes.items(): 802 for base in bases: 803 804 # Get the identity of the class from the reference. 805 806 base = base.get_origin() 807 self.subclasses[base].add(name) 808 809 def derive_shadowed_attrs(self): 810 811 "Derive shadowed attributes for classes." 812 813 for name, attrs in self.all_instance_attrs.items(): 814 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 815 if attrs: 816 self.all_shadowed_attrs[name] = attrs 817 818 def set_class_types(self): 819 820 "Set the type of each class." 821 822 for attrs in self.all_class_attrs.values(): 823 attrs["__class__"] = self.type_ref.get_origin() 824 825 def define_instantiators(self): 826 827 """ 828 Consolidate parameter and default details, incorporating initialiser 829 details to define instantiator signatures. 830 """ 831 832 for cls, attrs in self.all_class_attrs.items(): 833 initialiser = attrs["__init__"] 834 self.function_parameters[cls] = self.function_parameters[initialiser] 835 self.function_defaults[cls] = self.function_defaults[initialiser] 836 837 def collect_constants(self): 838 839 "Get constants from all active modules." 840 841 for module in self.modules.values(): 842 self.all_constants.update(module.constants) 843 844 # Import methods. 845 846 def find_in_path(self, name): 847 848 """ 849 Find the given module 'name' in the search path, returning None where no 850 such module could be found, or a 2-tuple from the 'find' method 851 otherwise. 852 """ 853 854 for d in self.path: 855 m = self.find(d, name) 856 if m: return m 857 return None 858 859 def find(self, d, name): 860 861 """ 862 In the directory 'd', find the given module 'name', where 'name' can 863 either refer to a single file module or to a package. Return None if the 864 'name' cannot be associated with either a file or a package directory, 865 or a 2-tuple from '_find_package' or '_find_module' otherwise. 866 """ 867 868 m = self._find_package(d, name) 869 if m: return m 870 m = self._find_module(d, name) 871 if m: return m 872 return None 873 874 def _find_module(self, d, name): 875 876 """ 877 In the directory 'd', find the given module 'name', returning None where 878 no suitable file exists in the directory, or a 2-tuple consisting of 879 None (indicating that no package directory is involved) and a filename 880 indicating the location of the module. 881 """ 882 883 name_py = name + extsep + "py" 884 filename = self._find_file(d, name_py) 885 if filename: 886 return None, filename 887 return None 888 889 def _find_package(self, d, name): 890 891 """ 892 In the directory 'd', find the given package 'name', returning None 893 where no suitable package directory exists, or a 2-tuple consisting of 894 a directory (indicating the location of the package directory itself) 895 and a filename indicating the location of the __init__.py module which 896 declares the package's top-level contents. 897 """ 898 899 filename = self._find_file(d, name) 900 if filename: 901 init_py = "__init__" + extsep + "py" 902 init_py_filename = self._find_file(filename, init_py) 903 if init_py_filename: 904 return filename, init_py_filename 905 return None 906 907 def _find_file(self, d, filename): 908 909 """ 910 Return the filename obtained when searching the directory 'd' for the 911 given 'filename', or None if no actual file exists for the filename. 912 """ 913 914 filename = join(d, filename) 915 if exists(filename): 916 return filename 917 else: 918 return None 919 920 def load(self, name): 921 922 """ 923 Load the module or package with the given 'name'. Return an object 924 referencing the loaded module or package, or None if no such module or 925 package exists. 926 """ 927 928 # Loaded modules are returned immediately. 929 # Modules may be known but not yet loading (having been registered as 930 # submodules), loading, loaded, or completely unknown. 931 932 module = self.get_module(name) 933 934 if module: 935 return self.modules[name] 936 937 # Otherwise, modules are loaded. 938 939 # Split the name into path components, and try to find the uppermost in 940 # the search path. 941 942 path = name.split(".") 943 path_so_far = [] 944 module = None 945 946 for p in path: 947 948 # Get the module's filesystem details. 949 950 if not path_so_far: 951 m = self.find_in_path(p) 952 elif d: 953 m = self.find(d, p) 954 else: 955 m = None 956 957 path_so_far.append(p) 958 module_name = ".".join(path_so_far) 959 960 # Raise an exception if the module could not be located. 961 962 if not m: 963 raise ProgramError("Module not found: %s" % name) 964 965 # Get the directory and module filename. 966 967 d, filename = m 968 969 # Get the module itself. 970 971 return self.load_from_file(filename, module_name) 972 973 def load_from_file(self, filename, module_name=None): 974 975 "Load the module from the given 'filename'." 976 977 if module_name is None: 978 module_name = "__main__" 979 980 module = self.modules.get(module_name) 981 982 if not module: 983 984 # Try to load from cache. 985 986 module = self.load_from_cache(filename, module_name) 987 if module: 988 return module 989 990 # If no cache entry exists, load from file. 991 992 module = inspector.InspectedModule(module_name, self) 993 self.add_module(module_name, module) 994 self.update_cache_validity(module) 995 996 self._load(module, module_name, lambda m: m.parse, filename) 997 998 return module 999 1000 def update_cache_validity(self, module): 1001 1002 "Make 'module' valid in the cache, but invalidate accessing modules." 1003 1004 accessing = self.accessing_modules.get(module.name) 1005 if accessing: 1006 self.invalidated.update(accessing) 1007 if module.name in self.invalidated: 1008 self.invalidated.remove(module.name) 1009 1010 def source_is_new(self, filename, module_name): 1011 1012 "Return whether 'filename' is newer than the cached 'module_name'." 1013 1014 if self.cache: 1015 cache_filename = join(self.cache, module_name) 1016 return not exists(cache_filename) or \ 1017 getmtime(filename) > getmtime(cache_filename) or \ 1018 module_name in self.invalidated 1019 else: 1020 return True 1021 1022 def load_from_cache(self, filename, module_name): 1023 1024 "Return a module residing in the cache." 1025 1026 module = self.modules.get(module_name) 1027 1028 if not module and not self.source_is_new(filename, module_name): 1029 module = CachedModule(module_name, self) 1030 self.add_module(module_name, module) 1031 1032 filename = join(self.cache, module_name) 1033 self._load(module, module_name, lambda m: m.from_cache, filename) 1034 1035 return module 1036 1037 def _load(self, module, module_name, fn, filename): 1038 1039 """ 1040 Load 'module' for the given 'module_name', and with 'fn' performing an 1041 invocation on the module with the given 'filename'. 1042 """ 1043 1044 # Load the module. 1045 1046 if self.verbose: 1047 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1048 fn(module)(filename) 1049 1050 # Add the module object if not already defined. 1051 1052 if not self.objects.has_key(module_name): 1053 self.objects[module_name] = Reference("<module>", module_name) 1054 1055 def add_module(self, module_name, module): 1056 1057 """ 1058 Return the module with the given 'module_name', adding a new module 1059 object if one does not already exist. 1060 """ 1061 1062 self.modules[module_name] = module 1063 if module_name in self.to_import: 1064 self.to_import.remove(module_name) 1065 1066 # vim: tabstop=4 expandtab shiftwidth=4