1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__") 37 38 def __init__(self, path, cache=None, verbose=False): 39 40 """ 41 Initialise the importer with the given search 'path' - a list of 42 directories to search for Python modules. 43 44 The optional 'cache' should be the name of a directory used to store 45 cached module information. 46 47 The optional 'verbose' parameter causes output concerning the activities 48 of the object to be produced if set to a true value (not the default). 49 """ 50 51 self.path = path 52 self.cache = cache 53 self.verbose = verbose 54 55 # Module importing queue, required modules, removed modules and active 56 # modules in the final program. 57 58 self.to_import = set() 59 self.required = set(["__main__"]) 60 self.removed = {} 61 self.modules = {} 62 63 # Module relationships and invalidated cached modules. 64 65 self.accessing_modules = {} 66 self.invalidated = set() 67 68 # Object relationships and dependencies. 69 70 self.depends = {} 71 self.module_depends = {} 72 73 # Basic program information. 74 75 self.objects = {} 76 self.classes = {} 77 self.function_parameters = {} 78 self.function_defaults = {} 79 self.function_locals = {} 80 self.function_targets = {} 81 self.function_arguments = {} 82 83 # Unresolved names. 84 85 self.missing = set() 86 87 # Derived information. 88 89 self.subclasses = {} 90 91 # Attributes of different object types. 92 93 self.all_class_attrs = {} 94 self.all_instance_attrs = {} 95 self.all_instance_attr_constants = {} 96 self.all_combined_attrs = {} 97 self.all_module_attrs = {} 98 self.all_shadowed_attrs = {} 99 100 # References to external names and aliases within program units. 101 102 self.all_name_references = {} 103 self.all_initialised_names = {} 104 self.all_aliased_names = {} 105 106 # General attribute accesses. 107 108 self.all_attr_accesses = {} 109 self.all_const_accesses = {} 110 self.all_attr_access_modifiers = {} 111 112 # Constant literals and values. 113 114 self.all_constants = {} 115 self.all_constant_values = {} 116 117 self.make_cache() 118 119 def make_cache(self): 120 121 "Make a cache directory if it does not already exist." 122 123 if self.cache and not exists(self.cache): 124 makedirs(self.cache) 125 126 def check_cache(self, details): 127 128 """ 129 Check whether the cache applies for the given 'details', invalidating it 130 if it does not. 131 """ 132 133 recorded_details = self.get_cache_details() 134 135 if recorded_details != details: 136 self.remove_cache() 137 138 writefile(self.get_cache_details_filename(), details) 139 140 def get_cache_details_filename(self): 141 142 "Return the filename for the cache details." 143 144 return join(self.cache, "$details") 145 146 def get_cache_details(self): 147 148 "Return details of the cache." 149 150 details_filename = self.get_cache_details_filename() 151 152 if not exists(details_filename): 153 return None 154 else: 155 return readfile(details_filename) 156 157 def remove_cache(self): 158 159 "Remove the contents of the cache." 160 161 for filename in listdir(self.cache): 162 remove(join(self.cache, filename)) 163 164 def to_cache(self): 165 166 "Write modules to the cache." 167 168 if self.cache: 169 for module_name, module in self.modules.items(): 170 module.to_cache(join(self.cache, module_name)) 171 172 # Object retrieval and storage. 173 174 def get_object(self, name): 175 176 """ 177 Return a reference for the given 'name' or None if no such object 178 exists. 179 """ 180 181 return self.objects.get(name) 182 183 def set_object(self, name, value=None): 184 185 "Set the object with the given 'name' and the given 'value'." 186 187 if isinstance(value, Reference): 188 ref = value.alias(name) 189 else: 190 ref = Reference(value, name) 191 192 self.objects[name] = ref 193 194 # Identification of both stored object names and name references. 195 196 def identify(self, name): 197 198 "Identify 'name' using stored object and external name records." 199 200 return self.objects.get(name) or self.all_name_references.get(name) 201 202 # Indirect object retrieval. 203 204 def get_attributes(self, ref, attrname): 205 206 """ 207 Return attributes provided by 'ref' for 'attrname'. Class attributes 208 may be provided by instances. 209 """ 210 211 kind = ref.get_kind() 212 if kind == "<class>": 213 ref = self.get_class_attribute(ref.get_origin(), attrname) 214 return ref and set([ref]) or set() 215 elif kind == "<instance>": 216 return self.get_combined_attributes(ref.get_origin(), attrname) 217 elif kind == "<module>": 218 ref = self.get_module_attribute(ref.get_origin(), attrname) 219 return ref and set([ref]) or set() 220 else: 221 return set() 222 223 def get_class_attribute(self, object_type, attrname): 224 225 "Return from 'object_type' the details of class attribute 'attrname'." 226 227 attrs = self.all_class_attrs.get(object_type) 228 attr = attrs and attrs.get(attrname) 229 return attr and self.get_object(attr) 230 231 def get_instance_attributes(self, object_type, attrname): 232 233 """ 234 Return from 'object_type' the details of instance attribute 'attrname'. 235 """ 236 237 consts = self.all_instance_attr_constants.get(object_type) 238 attrs = set() 239 for attr in self.all_instance_attrs[object_type].get(attrname, []): 240 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 241 return attrs 242 243 def get_combined_attributes(self, object_type, attrname): 244 245 """ 246 Return from 'object_type' the details of class or instance attribute 247 'attrname'. 248 """ 249 250 ref = self.get_class_attribute(object_type, attrname) 251 refs = ref and set([ref]) or set() 252 refs.update(self.get_instance_attributes(object_type, attrname)) 253 return refs 254 255 def get_module_attribute(self, object_type, attrname): 256 257 "Return from 'object_type' the details of module attribute 'attrname'." 258 259 if attrname in self.all_module_attrs[object_type]: 260 return self.get_object("%s.%s" % (object_type, attrname)) 261 else: 262 return None 263 264 # Convenience methods for deducing which kind of object provided an 265 # attribute. 266 267 def get_attribute_provider(self, ref, attrname): 268 269 """ 270 Return the kind of provider of the attribute accessed via 'ref' using 271 'attrname'. 272 """ 273 274 kind = ref.get_kind() 275 276 if kind in ["<class>", "<module>"]: 277 return kind 278 else: 279 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 280 281 def get_instance_attribute_provider(self, object_type, attrname): 282 283 """ 284 Return the kind of provider of the attribute accessed via an instance of 285 'object_type' using 'attrname'. 286 """ 287 288 if self.get_class_attribute(object_type, attrname): 289 return "<class>" 290 else: 291 return "<instance>" 292 293 # Module management. 294 295 def queue_module(self, name, accessor, required=False): 296 297 """ 298 Queue the module with the given 'name' for import from the given 299 'accessor' module. If 'required' is true (it is false by default), the 300 module will be required in the final program. 301 """ 302 303 if not self.modules.has_key(name): 304 self.to_import.add(name) 305 306 if required: 307 self.required.add(name) 308 309 init_item(self.accessing_modules, name, set) 310 self.accessing_modules[name].add(accessor.name) 311 312 def get_modules(self): 313 314 "Return all modules known to the importer." 315 316 return self.modules.values() 317 318 def get_module(self, name): 319 320 "Return the module with the given 'name'." 321 322 if not self.modules.has_key(name): 323 return None 324 325 return self.modules[name] 326 327 # Program operations. 328 329 def initialise(self, filename, reset=False): 330 331 """ 332 Initialise a program whose main module is 'filename', resetting the 333 cache if 'reset' is true. Return the main module. 334 """ 335 336 if reset: 337 self.remove_cache() 338 self.check_cache(filename) 339 340 # Load the program itself. 341 342 m = self.load_from_file(filename) 343 344 # Load any queued modules. 345 346 while self.to_import: 347 for name in list(self.to_import): # avoid mutation issue 348 self.load(name) 349 350 # Resolve dependencies between modules. 351 352 self.resolve() 353 354 # Record the type of all classes. 355 356 self.type_ref = self.get_object("__builtins__.type") 357 358 # Resolve dependencies within the program. 359 360 for module in self.modules.values(): 361 module.complete() 362 363 # Remove unneeded modules. 364 365 all_modules = self.modules.items() 366 367 for name, module in all_modules: 368 if name not in self.required: 369 module.unpropagate() 370 del self.modules[name] 371 self.removed[name] = module 372 373 # Collect redundant objects. 374 375 for module in self.removed.values(): 376 module.collect() 377 378 # Assert module objects where aliases have been removed. 379 380 for name in self.required: 381 if not self.objects.has_key(name): 382 self.objects[name] = Reference("<module>", name) 383 384 return m 385 386 def finalise(self): 387 388 """ 389 Finalise the inspected program, returning whether the program could be 390 finalised. 391 """ 392 393 self.finalise_classes() 394 self.add_init_dependencies() 395 self.to_cache() 396 397 if self.missing: 398 return False 399 400 self.set_class_types() 401 self.define_instantiators() 402 self.collect_constants() 403 404 return True 405 406 # Supporting operations. 407 408 def resolve(self): 409 410 "Resolve dependencies between modules." 411 412 self.waiting = {} 413 414 for module in self.modules.values(): 415 416 # Resolve all deferred references in each module. 417 418 original_deferred = [] 419 420 for ref in module.deferred: 421 422 # Retain original references for caching. 423 424 original_deferred.append(ref.copy()) 425 426 # Update references throughout the program. 427 428 found = self.find_dependency(ref) 429 if not found: 430 self.missing.add((module.name, ref.get_origin())) 431 432 # Record the resolved names and identify required modules. 433 434 else: 435 # Find the providing module of this reference. 436 # Where definitive details of the origin cannot be found, 437 # identify the provider using the deferred reference. 438 # NOTE: This may need to test for static origins. 439 440 provider = self.get_module_provider(found.unresolved() and ref or found) 441 ref.mutate(found) 442 443 # Record any external dependency. 444 445 if provider and provider != module.name: 446 447 # Handle built-in modules accidentally referenced by 448 # names. 449 450 if provider == "__builtins__" and found.has_kind("<module>"): 451 raise ProgramError("Name %s, used by %s, refers to module %s." % 452 (found.leaf(), module.name, found.get_origin())) 453 454 # Record the provider dependency. 455 456 module.required.add(provider) 457 self.accessing_modules[provider].add(module.name) 458 459 # Postpone any inclusion of the provider until this 460 # module becomes required. 461 462 if module.name not in self.required: 463 init_item(self.waiting, module.name, set) 464 self.waiting[module.name].add(provider) 465 if self.verbose: 466 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 467 468 # Make this module required in the accessing module. 469 470 elif provider not in self.required: 471 self.required.add(provider) 472 if self.verbose: 473 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 474 475 # Record a module ordering dependency. 476 477 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 478 self.add_module_dependency(module.name, provider) 479 480 # Restore the original references so that they may be read back in 481 # and produce the same results. 482 483 module.deferred = original_deferred 484 485 # Check modules again to see if they are now required and should now 486 # cause the inclusion of other modules providing objects to the program. 487 488 for module_name in self.waiting.keys(): 489 self.require_providers(module_name) 490 491 self.add_special_dependencies() 492 self.add_module_dependencies() 493 494 def require_providers(self, module_name): 495 496 """ 497 Test if 'module_name' is itself required and, if so, require modules 498 containing objects provided to the module. 499 """ 500 501 if module_name in self.required and self.waiting.has_key(module_name): 502 for provider in self.waiting[module_name]: 503 if provider not in self.required: 504 self.required.add(provider) 505 if self.verbose: 506 print >>sys.stderr, "Requiring", provider 507 self.require_providers(provider) 508 509 def add_special_dependencies(self): 510 511 "Add dependencies due to the use of special names in namespaces." 512 513 for module in self.modules.values(): 514 for ref, paths in module.special.values(): 515 for path in paths: 516 self.add_dependency(path, ref.get_origin()) 517 518 def add_init_dependencies(self): 519 520 "Add dependencies related to object initialisation." 521 522 for name in self.classes.keys(): 523 if self.is_dynamic_class(name): 524 525 # Make subclasses depend on any class with non-static 526 # attributes, plus its module for the initialisation. 527 528 for subclass in self.subclasses[name]: 529 ref = Reference("<class>", subclass) 530 self.add_dependency(subclass, name) 531 self.add_dependency(subclass, self.get_module_provider(ref)) 532 533 # Also make the class dependent on its module for 534 # initialisation. 535 536 ref = Reference("<class>", name) 537 self.add_dependency(name, self.get_module_provider(ref)) 538 539 for name in self.function_defaults.keys(): 540 if self.is_dynamic_callable(name): 541 542 # Make functions with defaults requiring initialisation depend 543 # on the parent scope, if a function, or the module scope. 544 545 ref = Reference("<function>", name) 546 parent_ref = self.get_object(ref.parent()) 547 548 # Function no longer present in the program. 549 550 if not parent_ref: 551 continue 552 553 if parent_ref.has_kind("<class>"): 554 parent = self.get_module_provider(parent_ref) 555 else: 556 parent = parent_ref.get_origin() 557 558 self.add_dependency(name, parent) 559 560 def add_module_dependencies(self): 561 562 "Record module-based dependencies." 563 564 for module_name, providers in self.module_depends.items(): 565 if self.modules.has_key(module_name): 566 for provider in providers: 567 if self.modules.has_key(provider): 568 self.add_dependency(module_name, provider) 569 570 def add_dependency(self, path, origin): 571 572 "Add dependency details for 'path' involving 'origin'." 573 574 if origin and not origin.startswith("%s." % path): 575 init_item(self.depends, path, set) 576 self.depends[path].add(origin) 577 578 def add_module_dependency(self, module_name, provider): 579 580 "Add dependency details for 'module_name' involving 'provider'." 581 582 if provider: 583 init_item(self.module_depends, module_name, set) 584 self.module_depends[module_name].add(provider) 585 586 def condense_dependencies(self): 587 588 """ 589 Condense the dependencies by removing all functions that do not need 590 initialisation. 591 """ 592 593 d = {} 594 for path, depends in self.depends.items(): 595 d[path] = {} 596 d[path] = self.condense_dependency_entry(depends, d) 597 598 self.depends = {} 599 600 for path, depends in d.items(): 601 if depends: 602 self.depends[path] = depends 603 604 def condense_dependency_entry(self, depends, d): 605 l = set() 606 for depend in depends: 607 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 608 self.is_dynamic_callable(depend): 609 610 l.add(depend) 611 else: 612 deps = d.get(depend) 613 if deps: 614 l.update(self.condense_dependency_entry(deps, d)) 615 return l 616 617 def is_dynamic(self, ref): 618 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 619 620 def is_dynamic_class(self, name): 621 622 """ 623 Return whether 'name' refers to a class with members that must be 624 initialised dynamically. 625 """ 626 627 attrs = self.all_class_attrs.get(name) 628 629 if not attrs: 630 return False 631 632 for attrname, attr in attrs.items(): 633 if attrname in self.special_attributes: 634 continue 635 ref = attr and self.get_object(attr) 636 if self.is_dynamic(ref): 637 return True 638 639 return False 640 641 def is_dynamic_callable(self, name): 642 643 """ 644 Return whether 'name' refers to a callable employing defaults that may 645 need initialising before the callable can be used. 646 """ 647 648 # Find any defaults for the function or method. 649 650 defaults = self.function_defaults.get(name) 651 if not defaults: 652 return False 653 654 # Identify non-constant defaults. 655 656 for name, ref in defaults: 657 if self.is_dynamic(ref): 658 return True 659 660 return False 661 662 def order_objects(self): 663 664 "Produce an object initialisation ordering." 665 666 self.condense_dependencies() 667 668 # Record the number of modules using or depending on each module. 669 670 usage = {} 671 672 # Record path-based dependencies. 673 674 for path in self.depends.keys(): 675 usage[path] = set() 676 677 for path, depends in self.depends.items(): 678 for origin in depends: 679 init_item(usage, origin, set) 680 usage[origin].add(path) 681 682 # Produce an ordering by obtaining exposed modules (required by modules 683 # already processed) and putting them at the start of the list. 684 685 ordered = [] 686 687 while usage: 688 have_next = False 689 690 for path, n in usage.items(): 691 if not n: 692 ordered.insert(0, path) 693 depends = self.depends.get(path) 694 695 # Reduce usage of the referenced objects. 696 697 if depends: 698 for origin in depends: 699 usage[origin].remove(path) 700 701 del usage[path] 702 have_next = True 703 704 if not have_next: 705 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(usage.keys())) 706 707 if "__main__" in ordered: 708 ordered.remove("__main__") 709 710 ordered.append("__main__") 711 return ordered 712 713 def order_modules(self): 714 715 "Produce a module initialisation ordering." 716 717 ordered = self.order_objects() 718 filtered = [] 719 720 for module_name in self.modules.keys(): 721 if module_name not in ordered: 722 filtered.append(module_name) 723 724 for path in ordered: 725 if self.modules.has_key(path): 726 filtered.append(path) 727 728 return filtered 729 730 def find_dependency(self, ref): 731 732 "Find the ultimate dependency for 'ref'." 733 734 found = set() 735 while ref and ref.has_kind("<depends>") and not ref in found: 736 found.add(ref) 737 ref = self.identify(ref.get_origin()) 738 return ref 739 740 def get_module_provider(self, ref): 741 742 "Identify the provider of the given 'ref'." 743 744 for ancestor in ref.ancestors(): 745 if self.modules.has_key(ancestor): 746 return ancestor 747 return None 748 749 def finalise_classes(self): 750 751 "Finalise the class relationships and attributes." 752 753 self.derive_inherited_attrs() 754 self.derive_subclasses() 755 self.derive_shadowed_attrs() 756 757 def derive_inherited_attrs(self): 758 759 "Derive inherited attributes for classes throughout the program." 760 761 for name in self.classes.keys(): 762 self.propagate_attrs_for_class(name) 763 764 def propagate_attrs_for_class(self, name, visited=None): 765 766 "Propagate inherited attributes for class 'name'." 767 768 # Visit classes only once. 769 770 if self.all_combined_attrs.has_key(name): 771 return 772 773 visited = visited or [] 774 775 if name in visited: 776 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 777 778 visited.append(name) 779 780 class_attrs = {} 781 instance_attrs = {} 782 783 # Aggregate the attributes from base classes, recording the origins of 784 # applicable attributes. 785 786 for base in self.classes[name][::-1]: 787 788 # Get the identity of the class from the reference. 789 790 base = base.get_origin() 791 792 # Define the base class completely before continuing with this 793 # class. 794 795 self.propagate_attrs_for_class(base, visited) 796 class_attrs.update(self.all_class_attrs[base]) 797 798 # Instance attribute origins are combined if different. 799 800 for key, values in self.all_instance_attrs[base].items(): 801 init_item(instance_attrs, key, set) 802 instance_attrs[key].update(values) 803 804 # Class attributes override those defined earlier in the hierarchy. 805 806 class_attrs.update(self.all_class_attrs.get(name, {})) 807 808 # Instance attributes are merely added if not already defined. 809 810 for key in self.all_instance_attrs.get(name, []): 811 if not instance_attrs.has_key(key): 812 instance_attrs[key] = set(["%s.%s" % (name, key)]) 813 814 self.all_class_attrs[name] = class_attrs 815 self.all_instance_attrs[name] = instance_attrs 816 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 817 818 def derive_subclasses(self): 819 820 "Derive subclass details for classes." 821 822 for name, bases in self.classes.items(): 823 for base in bases: 824 825 # Get the identity of the class from the reference. 826 827 base = base.get_origin() 828 self.subclasses[base].add(name) 829 830 def derive_shadowed_attrs(self): 831 832 "Derive shadowed attributes for classes." 833 834 for name, attrs in self.all_instance_attrs.items(): 835 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 836 if attrs: 837 self.all_shadowed_attrs[name] = attrs 838 839 def set_class_types(self): 840 841 "Set the type of each class." 842 843 for attrs in self.all_class_attrs.values(): 844 attrs["__class__"] = self.type_ref.get_origin() 845 846 def define_instantiators(self): 847 848 """ 849 Consolidate parameter and default details, incorporating initialiser 850 details to define instantiator signatures. 851 """ 852 853 for cls, attrs in self.all_class_attrs.items(): 854 initialiser = attrs["__init__"] 855 self.function_parameters[cls] = self.function_parameters[initialiser] 856 self.function_defaults[cls] = self.function_defaults[initialiser] 857 858 def collect_constants(self): 859 860 "Get constants from all active modules." 861 862 for module in self.modules.values(): 863 self.all_constants.update(module.constants) 864 865 # Import methods. 866 867 def find_in_path(self, name): 868 869 """ 870 Find the given module 'name' in the search path, returning None where no 871 such module could be found, or a 2-tuple from the 'find' method 872 otherwise. 873 """ 874 875 for d in self.path: 876 m = self.find(d, name) 877 if m: return m 878 return None 879 880 def find(self, d, name): 881 882 """ 883 In the directory 'd', find the given module 'name', where 'name' can 884 either refer to a single file module or to a package. Return None if the 885 'name' cannot be associated with either a file or a package directory, 886 or a 2-tuple from '_find_package' or '_find_module' otherwise. 887 """ 888 889 m = self._find_package(d, name) 890 if m: return m 891 m = self._find_module(d, name) 892 if m: return m 893 return None 894 895 def _find_module(self, d, name): 896 897 """ 898 In the directory 'd', find the given module 'name', returning None where 899 no suitable file exists in the directory, or a 2-tuple consisting of 900 None (indicating that no package directory is involved) and a filename 901 indicating the location of the module. 902 """ 903 904 name_py = name + extsep + "py" 905 filename = self._find_file(d, name_py) 906 if filename: 907 return None, filename 908 return None 909 910 def _find_package(self, d, name): 911 912 """ 913 In the directory 'd', find the given package 'name', returning None 914 where no suitable package directory exists, or a 2-tuple consisting of 915 a directory (indicating the location of the package directory itself) 916 and a filename indicating the location of the __init__.py module which 917 declares the package's top-level contents. 918 """ 919 920 filename = self._find_file(d, name) 921 if filename: 922 init_py = "__init__" + extsep + "py" 923 init_py_filename = self._find_file(filename, init_py) 924 if init_py_filename: 925 return filename, init_py_filename 926 return None 927 928 def _find_file(self, d, filename): 929 930 """ 931 Return the filename obtained when searching the directory 'd' for the 932 given 'filename', or None if no actual file exists for the filename. 933 """ 934 935 filename = join(d, filename) 936 if exists(filename): 937 return filename 938 else: 939 return None 940 941 def load(self, name): 942 943 """ 944 Load the module or package with the given 'name'. Return an object 945 referencing the loaded module or package, or None if no such module or 946 package exists. 947 """ 948 949 # Loaded modules are returned immediately. 950 # Modules may be known but not yet loading (having been registered as 951 # submodules), loading, loaded, or completely unknown. 952 953 module = self.get_module(name) 954 955 if module: 956 return self.modules[name] 957 958 # Otherwise, modules are loaded. 959 960 # Split the name into path components, and try to find the uppermost in 961 # the search path. 962 963 path = name.split(".") 964 path_so_far = [] 965 module = None 966 967 for p in path: 968 969 # Get the module's filesystem details. 970 971 if not path_so_far: 972 m = self.find_in_path(p) 973 elif d: 974 m = self.find(d, p) 975 else: 976 m = None 977 978 path_so_far.append(p) 979 module_name = ".".join(path_so_far) 980 981 # Raise an exception if the module could not be located. 982 983 if not m: 984 raise ProgramError("Module not found: %s" % name) 985 986 # Get the directory and module filename. 987 988 d, filename = m 989 990 # Get the module itself. 991 992 return self.load_from_file(filename, module_name) 993 994 def load_from_file(self, filename, module_name=None): 995 996 "Load the module from the given 'filename'." 997 998 if module_name is None: 999 module_name = "__main__" 1000 1001 module = self.modules.get(module_name) 1002 1003 if not module: 1004 1005 # Try to load from cache. 1006 1007 module = self.load_from_cache(filename, module_name) 1008 if module: 1009 return module 1010 1011 # If no cache entry exists, load from file. 1012 1013 module = inspector.InspectedModule(module_name, self) 1014 self.add_module(module_name, module) 1015 self.update_cache_validity(module) 1016 1017 self._load(module, module_name, lambda m: m.parse, filename) 1018 1019 return module 1020 1021 def update_cache_validity(self, module): 1022 1023 "Make 'module' valid in the cache, but invalidate accessing modules." 1024 1025 accessing = self.accessing_modules.get(module.name) 1026 if accessing: 1027 self.invalidated.update(accessing) 1028 if module.name in self.invalidated: 1029 self.invalidated.remove(module.name) 1030 1031 def source_is_new(self, filename, module_name): 1032 1033 "Return whether 'filename' is newer than the cached 'module_name'." 1034 1035 if self.cache: 1036 cache_filename = join(self.cache, module_name) 1037 return not exists(cache_filename) or \ 1038 getmtime(filename) > getmtime(cache_filename) or \ 1039 module_name in self.invalidated 1040 else: 1041 return True 1042 1043 def load_from_cache(self, filename, module_name): 1044 1045 "Return a module residing in the cache." 1046 1047 module = self.modules.get(module_name) 1048 1049 if not module and not self.source_is_new(filename, module_name): 1050 module = CachedModule(module_name, self) 1051 self.add_module(module_name, module) 1052 1053 filename = join(self.cache, module_name) 1054 self._load(module, module_name, lambda m: m.from_cache, filename) 1055 1056 return module 1057 1058 def _load(self, module, module_name, fn, filename): 1059 1060 """ 1061 Load 'module' for the given 'module_name', and with 'fn' performing an 1062 invocation on the module with the given 'filename'. 1063 """ 1064 1065 # Load the module. 1066 1067 if self.verbose: 1068 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1069 fn(module)(filename) 1070 1071 # Add the module object if not already defined. 1072 1073 if not self.objects.has_key(module_name): 1074 self.objects[module_name] = Reference("<module>", module_name) 1075 1076 def add_module(self, module_name, module): 1077 1078 """ 1079 Return the module with the given 'module_name', adding a new module 1080 object if one does not already exist. 1081 """ 1082 1083 self.modules[module_name] = module 1084 if module_name in self.to_import: 1085 self.to_import.remove(module_name) 1086 1087 # vim: tabstop=4 expandtab shiftwidth=4