1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__") 37 38 def __init__(self, path, cache=None, verbose=False): 39 40 """ 41 Initialise the importer with the given search 'path' - a list of 42 directories to search for Python modules. 43 44 The optional 'cache' should be the name of a directory used to store 45 cached module information. 46 47 The optional 'verbose' parameter causes output concerning the activities 48 of the object to be produced if set to a true value (not the default). 49 """ 50 51 self.path = path 52 self.cache = cache 53 self.verbose = verbose 54 55 # Module importing queue, required modules, removed modules and active 56 # modules in the final program. 57 58 self.to_import = set() 59 self.required = set(["__main__"]) 60 self.removed = {} 61 self.modules = {} 62 63 # Module relationships and invalidated cached modules. 64 65 self.accessing_modules = {} 66 self.invalidated = set() 67 68 # Object relationships and dependencies. 69 70 self.depends = {} 71 self.module_depends = {} 72 73 # Basic program information. 74 75 self.objects = {} 76 self.classes = {} 77 self.function_parameters = {} 78 self.function_defaults = {} 79 self.function_locals = {} 80 self.function_targets = {} 81 self.function_arguments = {} 82 83 # Unresolved names. 84 85 self.missing = set() 86 87 # Derived information. 88 89 self.subclasses = {} 90 91 # Attributes of different object types. 92 93 self.all_class_attrs = {} 94 self.all_instance_attrs = {} 95 self.all_instance_attr_constants = {} 96 self.all_combined_attrs = {} 97 self.all_module_attrs = {} 98 self.all_shadowed_attrs = {} 99 100 # References to external names and aliases within program units. 101 102 self.all_name_references = {} 103 self.all_initialised_names = {} 104 self.all_aliased_names = {} 105 106 # General attribute accesses. 107 108 self.all_attr_accesses = {} 109 self.all_const_accesses = {} 110 self.all_attr_access_modifiers = {} 111 112 # Constant literals and values. 113 114 self.all_constants = {} 115 self.all_constant_values = {} 116 117 self.make_cache() 118 119 def make_cache(self): 120 121 "Make a cache directory if it does not already exist." 122 123 if self.cache and not exists(self.cache): 124 makedirs(self.cache) 125 126 def check_cache(self, details): 127 128 """ 129 Check whether the cache applies for the given 'details', invalidating it 130 if it does not. 131 """ 132 133 recorded_details = self.get_cache_details() 134 135 if recorded_details != details: 136 self.remove_cache() 137 138 writefile(self.get_cache_details_filename(), details) 139 140 def get_cache_details_filename(self): 141 142 "Return the filename for the cache details." 143 144 return join(self.cache, "$details") 145 146 def get_cache_details(self): 147 148 "Return details of the cache." 149 150 details_filename = self.get_cache_details_filename() 151 152 if not exists(details_filename): 153 return None 154 else: 155 return readfile(details_filename) 156 157 def remove_cache(self): 158 159 "Remove the contents of the cache." 160 161 for filename in listdir(self.cache): 162 remove(join(self.cache, filename)) 163 164 def to_cache(self): 165 166 "Write modules to the cache." 167 168 if self.cache: 169 for module_name, module in self.modules.items(): 170 module.to_cache(join(self.cache, module_name)) 171 172 # Object retrieval and storage. 173 174 def get_object(self, name): 175 176 """ 177 Return a reference for the given 'name' or None if no such object 178 exists. 179 """ 180 181 return self.objects.get(name) 182 183 def set_object(self, name, value=None): 184 185 "Set the object with the given 'name' and the given 'value'." 186 187 if isinstance(value, Reference): 188 ref = value.alias(name) 189 else: 190 ref = Reference(value, name) 191 192 self.objects[name] = ref 193 194 # Identification of both stored object names and name references. 195 196 def identify(self, name): 197 198 "Identify 'name' using stored object and external name records." 199 200 return self.objects.get(name) or self.all_name_references.get(name) 201 202 # Indirect object retrieval. 203 204 def get_attributes(self, ref, attrname): 205 206 """ 207 Return attributes provided by 'ref' for 'attrname'. Class attributes 208 may be provided by instances. 209 """ 210 211 kind = ref.get_kind() 212 if kind == "<class>": 213 ref = self.get_class_attribute(ref.get_origin(), attrname) 214 return ref and set([ref]) or set() 215 elif kind == "<instance>": 216 return self.get_combined_attributes(ref.get_origin(), attrname) 217 elif kind == "<module>": 218 ref = self.get_module_attribute(ref.get_origin(), attrname) 219 return ref and set([ref]) or set() 220 else: 221 return set() 222 223 def get_class_attribute(self, object_type, attrname): 224 225 "Return from 'object_type' the details of class attribute 'attrname'." 226 227 attrs = self.all_class_attrs.get(object_type) 228 attr = attrs and attrs.get(attrname) 229 return attr and self.get_object(attr) 230 231 def get_instance_attributes(self, object_type, attrname): 232 233 """ 234 Return from 'object_type' the details of instance attribute 'attrname'. 235 """ 236 237 consts = self.all_instance_attr_constants.get(object_type) 238 attrs = set() 239 for attr in self.all_instance_attrs[object_type].get(attrname, []): 240 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 241 return attrs 242 243 def get_combined_attributes(self, object_type, attrname): 244 245 """ 246 Return from 'object_type' the details of class or instance attribute 247 'attrname'. 248 """ 249 250 ref = self.get_class_attribute(object_type, attrname) 251 refs = ref and set([ref]) or set() 252 refs.update(self.get_instance_attributes(object_type, attrname)) 253 return refs 254 255 def get_module_attribute(self, object_type, attrname): 256 257 "Return from 'object_type' the details of module attribute 'attrname'." 258 259 if attrname in self.all_module_attrs[object_type]: 260 return self.get_object("%s.%s" % (object_type, attrname)) 261 else: 262 return None 263 264 # Convenience methods for deducing which kind of object provided an 265 # attribute. 266 267 def get_attribute_provider(self, ref, attrname): 268 269 """ 270 Return the kind of provider of the attribute accessed via 'ref' using 271 'attrname'. 272 """ 273 274 kind = ref.get_kind() 275 276 if kind in ["<class>", "<module>"]: 277 return kind 278 else: 279 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 280 281 def get_instance_attribute_provider(self, object_type, attrname): 282 283 """ 284 Return the kind of provider of the attribute accessed via an instance of 285 'object_type' using 'attrname'. 286 """ 287 288 if self.get_class_attribute(object_type, attrname): 289 return "<class>" 290 else: 291 return "<instance>" 292 293 # Module management. 294 295 def queue_module(self, name, accessor, required=False): 296 297 """ 298 Queue the module with the given 'name' for import from the given 299 'accessor' module. If 'required' is true (it is false by default), the 300 module will be required in the final program. 301 """ 302 303 if not self.modules.has_key(name): 304 self.to_import.add(name) 305 306 if required: 307 self.required.add(name) 308 309 init_item(self.accessing_modules, name, set) 310 self.accessing_modules[name].add(accessor.name) 311 312 def get_modules(self): 313 314 "Return all modules known to the importer." 315 316 return self.modules.values() 317 318 def get_module(self, name): 319 320 "Return the module with the given 'name'." 321 322 if not self.modules.has_key(name): 323 return None 324 325 return self.modules[name] 326 327 # Program operations. 328 329 def initialise(self, filename, reset=False): 330 331 """ 332 Initialise a program whose main module is 'filename', resetting the 333 cache if 'reset' is true. Return the main module. 334 """ 335 336 if reset: 337 self.remove_cache() 338 self.check_cache(filename) 339 340 # Load the program itself. 341 342 m = self.load_from_file(filename) 343 344 # Load any queued modules. 345 346 while self.to_import: 347 for name in list(self.to_import): # avoid mutation issue 348 self.load(name) 349 350 # Resolve dependencies between modules. 351 352 self.resolve() 353 354 # Record the type of all classes. 355 356 self.type_ref = self.get_object("__builtins__.type") 357 358 # Resolve dependencies within the program. 359 360 for module in self.modules.values(): 361 module.complete() 362 363 # Remove unneeded modules. 364 365 all_modules = self.modules.items() 366 367 for name, module in all_modules: 368 if name not in self.required: 369 module.unpropagate() 370 del self.modules[name] 371 self.removed[name] = module 372 373 # Collect redundant objects. 374 375 for module in self.removed.values(): 376 module.collect() 377 378 # Assert module objects where aliases have been removed. 379 380 for name in self.required: 381 if not self.objects.has_key(name): 382 self.objects[name] = Reference("<module>", name) 383 384 return m 385 386 def finalise(self): 387 388 """ 389 Finalise the inspected program, returning whether the program could be 390 finalised. 391 """ 392 393 self.finalise_classes() 394 self.add_init_dependencies() 395 self.to_cache() 396 397 if self.missing: 398 return False 399 400 self.set_class_types() 401 self.define_instantiators() 402 self.collect_constants() 403 404 return True 405 406 # Supporting operations. 407 408 def resolve(self): 409 410 "Resolve dependencies between modules." 411 412 self.waiting = {} 413 414 for module in self.modules.values(): 415 416 # Resolve all deferred references in each module. 417 418 original_deferred = [] 419 420 for ref in module.deferred: 421 422 # Retain original references for caching. 423 424 original_deferred.append(ref.copy()) 425 426 # Update references throughout the program. 427 428 found = self.find_dependency(ref) 429 if not found: 430 self.missing.add((module.name, ref.get_origin())) 431 432 # Record the resolved names and identify required modules. 433 434 else: 435 # Find the providing module of this reference. 436 # Where definitive details of the origin cannot be found, 437 # identify the provider using the deferred reference. 438 # NOTE: This may need to test for static origins. 439 440 provider = self.get_module_provider(found.unresolved() and ref or found) 441 ref.mutate(found) 442 443 # Record any external dependency. 444 445 if provider and provider != module.name: 446 447 # Record the provider dependency. 448 449 module.required.add(provider) 450 self.accessing_modules[provider].add(module.name) 451 452 # Postpone any inclusion of the provider until this 453 # module becomes required. 454 455 if module.name not in self.required: 456 init_item(self.waiting, module.name, set) 457 self.waiting[module.name].add(provider) 458 if self.verbose: 459 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 460 461 # Make this module required in the accessing module. 462 463 elif provider not in self.required: 464 self.required.add(provider) 465 if self.verbose: 466 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 467 468 # Record a module ordering dependency. 469 470 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 471 self.add_module_dependency(module.name, provider) 472 473 # Restore the original references so that they may be read back in 474 # and produce the same results. 475 476 module.deferred = original_deferred 477 478 # Check modules again to see if they are now required and should now 479 # cause the inclusion of other modules providing objects to the program. 480 481 for module_name in self.waiting.keys(): 482 self.require_providers(module_name) 483 484 self.add_special_dependencies() 485 self.add_module_dependencies() 486 487 def require_providers(self, module_name): 488 489 """ 490 Test if 'module_name' is itself required and, if so, require modules 491 containing objects provided to the module. 492 """ 493 494 if module_name in self.required and self.waiting.has_key(module_name): 495 for provider in self.waiting[module_name]: 496 if provider not in self.required: 497 self.required.add(provider) 498 if self.verbose: 499 print >>sys.stderr, "Requiring", provider 500 self.require_providers(provider) 501 502 def add_special_dependencies(self): 503 504 "Add dependencies due to the use of special names in namespaces." 505 506 for module in self.modules.values(): 507 for ref, paths in module.special.values(): 508 for path in paths: 509 self.add_dependency(path, ref.get_origin()) 510 511 def add_init_dependencies(self): 512 513 "Add dependencies related to object initialisation." 514 515 for name in self.classes.keys(): 516 if self.is_dynamic_class(name): 517 518 # Make subclasses depend on any class with non-static 519 # attributes, plus its module for the initialisation. 520 521 for subclass in self.subclasses[name]: 522 ref = Reference("<class>", subclass) 523 self.add_dependency(subclass, name) 524 self.add_dependency(subclass, self.get_module_provider(ref)) 525 526 # Also make the class dependent on its module for 527 # initialisation. 528 529 ref = Reference("<class>", name) 530 self.add_dependency(name, self.get_module_provider(ref)) 531 532 for name in self.function_defaults.keys(): 533 if self.is_dynamic_callable(name): 534 535 # Make functions with defaults requiring initialisation depend 536 # on the parent scope, if a function, or the module scope. 537 538 ref = Reference("<function>", name) 539 parent_ref = self.get_object(ref.parent()) 540 541 # Function no longer present in the program. 542 543 if not parent_ref: 544 continue 545 546 if parent_ref.has_kind("<class>"): 547 parent = self.get_module_provider(parent_ref) 548 else: 549 parent = parent_ref.get_origin() 550 551 self.add_dependency(name, parent) 552 553 def add_module_dependencies(self): 554 555 "Record module-based dependencies." 556 557 for module_name, providers in self.module_depends.items(): 558 if self.modules.has_key(module_name): 559 for provider in providers: 560 if self.modules.has_key(provider): 561 self.add_dependency(module_name, provider) 562 563 def add_dependency(self, path, origin): 564 565 "Add dependency details for 'path' involving 'origin'." 566 567 if origin and not origin.startswith("%s." % path): 568 init_item(self.depends, path, set) 569 self.depends[path].add(origin) 570 571 def add_module_dependency(self, module_name, provider): 572 573 "Add dependency details for 'module_name' involving 'provider'." 574 575 if provider: 576 init_item(self.module_depends, module_name, set) 577 self.module_depends[module_name].add(provider) 578 579 def condense_dependencies(self): 580 581 """ 582 Condense the dependencies by removing all functions that do not need 583 initialisation. 584 """ 585 586 d = {} 587 for path, depends in self.depends.items(): 588 d[path] = {} 589 d[path] = self.condense_dependency_entry(depends, d) 590 591 self.depends = {} 592 593 for path, depends in d.items(): 594 if depends: 595 self.depends[path] = depends 596 597 def condense_dependency_entry(self, depends, d): 598 l = set() 599 for depend in depends: 600 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 601 self.is_dynamic_callable(depend): 602 603 l.add(depend) 604 else: 605 deps = d.get(depend) 606 if deps: 607 l.update(self.condense_dependency_entry(deps, d)) 608 return l 609 610 def is_dynamic(self, ref): 611 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 612 613 def is_dynamic_class(self, name): 614 615 """ 616 Return whether 'name' refers to a class with members that must be 617 initialised dynamically. 618 """ 619 620 attrs = self.all_class_attrs.get(name) 621 622 if not attrs: 623 return False 624 625 for attrname, attr in attrs.items(): 626 if attrname in self.special_attributes: 627 continue 628 ref = attr and self.get_object(attr) 629 if self.is_dynamic(ref): 630 return True 631 632 return False 633 634 def is_dynamic_callable(self, name): 635 636 """ 637 Return whether 'name' refers to a callable employing defaults that may 638 need initialising before the callable can be used. 639 """ 640 641 # Find any defaults for the function or method. 642 643 defaults = self.function_defaults.get(name) 644 if not defaults: 645 return False 646 647 # Identify non-constant defaults. 648 649 for name, ref in defaults: 650 if self.is_dynamic(ref): 651 return True 652 653 return False 654 655 def order_objects(self): 656 657 "Produce an object initialisation ordering." 658 659 self.condense_dependencies() 660 661 # Record the number of modules using or depending on each module. 662 663 usage = {} 664 665 # Record path-based dependencies. 666 667 for path in self.depends.keys(): 668 usage[path] = set() 669 670 for path, depends in self.depends.items(): 671 for origin in depends: 672 init_item(usage, origin, set) 673 usage[origin].add(path) 674 675 # Produce an ordering by obtaining exposed modules (required by modules 676 # already processed) and putting them at the start of the list. 677 678 ordered = [] 679 680 while usage: 681 have_next = False 682 683 for path, n in usage.items(): 684 if not n: 685 ordered.insert(0, path) 686 depends = self.depends.get(path) 687 688 # Reduce usage of the referenced objects. 689 690 if depends: 691 for origin in depends: 692 usage[origin].remove(path) 693 694 del usage[path] 695 have_next = True 696 697 if not have_next: 698 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(usage.keys())) 699 700 if "__main__" in ordered: 701 ordered.remove("__main__") 702 703 ordered.append("__main__") 704 return ordered 705 706 def order_modules(self): 707 708 "Produce a module initialisation ordering." 709 710 ordered = self.order_objects() 711 filtered = [] 712 713 for module_name in self.modules.keys(): 714 if module_name not in ordered: 715 filtered.append(module_name) 716 717 for path in ordered: 718 if self.modules.has_key(path): 719 filtered.append(path) 720 721 return filtered 722 723 def find_dependency(self, ref): 724 725 "Find the ultimate dependency for 'ref'." 726 727 found = set() 728 while ref and ref.has_kind("<depends>") and not ref in found: 729 found.add(ref) 730 ref = self.identify(ref.get_origin()) 731 return ref 732 733 def get_module_provider(self, ref): 734 735 "Identify the provider of the given 'ref'." 736 737 for ancestor in ref.ancestors(): 738 if self.modules.has_key(ancestor): 739 return ancestor 740 return None 741 742 def finalise_classes(self): 743 744 "Finalise the class relationships and attributes." 745 746 self.derive_inherited_attrs() 747 self.derive_subclasses() 748 self.derive_shadowed_attrs() 749 750 def derive_inherited_attrs(self): 751 752 "Derive inherited attributes for classes throughout the program." 753 754 for name in self.classes.keys(): 755 self.propagate_attrs_for_class(name) 756 757 def propagate_attrs_for_class(self, name, visited=None): 758 759 "Propagate inherited attributes for class 'name'." 760 761 # Visit classes only once. 762 763 if self.all_combined_attrs.has_key(name): 764 return 765 766 visited = visited or [] 767 768 if name in visited: 769 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 770 771 visited.append(name) 772 773 class_attrs = {} 774 instance_attrs = {} 775 776 # Aggregate the attributes from base classes, recording the origins of 777 # applicable attributes. 778 779 for base in self.classes[name][::-1]: 780 781 # Get the identity of the class from the reference. 782 783 base = base.get_origin() 784 785 # Define the base class completely before continuing with this 786 # class. 787 788 self.propagate_attrs_for_class(base, visited) 789 class_attrs.update(self.all_class_attrs[base]) 790 791 # Instance attribute origins are combined if different. 792 793 for key, values in self.all_instance_attrs[base].items(): 794 init_item(instance_attrs, key, set) 795 instance_attrs[key].update(values) 796 797 # Class attributes override those defined earlier in the hierarchy. 798 799 class_attrs.update(self.all_class_attrs.get(name, {})) 800 801 # Instance attributes are merely added if not already defined. 802 803 for key in self.all_instance_attrs.get(name, []): 804 if not instance_attrs.has_key(key): 805 instance_attrs[key] = set(["%s.%s" % (name, key)]) 806 807 self.all_class_attrs[name] = class_attrs 808 self.all_instance_attrs[name] = instance_attrs 809 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 810 811 def derive_subclasses(self): 812 813 "Derive subclass details for classes." 814 815 for name, bases in self.classes.items(): 816 for base in bases: 817 818 # Get the identity of the class from the reference. 819 820 base = base.get_origin() 821 self.subclasses[base].add(name) 822 823 def derive_shadowed_attrs(self): 824 825 "Derive shadowed attributes for classes." 826 827 for name, attrs in self.all_instance_attrs.items(): 828 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 829 if attrs: 830 self.all_shadowed_attrs[name] = attrs 831 832 def set_class_types(self): 833 834 "Set the type of each class." 835 836 for attrs in self.all_class_attrs.values(): 837 attrs["__class__"] = self.type_ref.get_origin() 838 839 def define_instantiators(self): 840 841 """ 842 Consolidate parameter and default details, incorporating initialiser 843 details to define instantiator signatures. 844 """ 845 846 for cls, attrs in self.all_class_attrs.items(): 847 initialiser = attrs["__init__"] 848 self.function_parameters[cls] = self.function_parameters[initialiser] 849 self.function_defaults[cls] = self.function_defaults[initialiser] 850 851 def collect_constants(self): 852 853 "Get constants from all active modules." 854 855 for module in self.modules.values(): 856 self.all_constants.update(module.constants) 857 858 # Import methods. 859 860 def find_in_path(self, name): 861 862 """ 863 Find the given module 'name' in the search path, returning None where no 864 such module could be found, or a 2-tuple from the 'find' method 865 otherwise. 866 """ 867 868 for d in self.path: 869 m = self.find(d, name) 870 if m: return m 871 return None 872 873 def find(self, d, name): 874 875 """ 876 In the directory 'd', find the given module 'name', where 'name' can 877 either refer to a single file module or to a package. Return None if the 878 'name' cannot be associated with either a file or a package directory, 879 or a 2-tuple from '_find_package' or '_find_module' otherwise. 880 """ 881 882 m = self._find_package(d, name) 883 if m: return m 884 m = self._find_module(d, name) 885 if m: return m 886 return None 887 888 def _find_module(self, d, name): 889 890 """ 891 In the directory 'd', find the given module 'name', returning None where 892 no suitable file exists in the directory, or a 2-tuple consisting of 893 None (indicating that no package directory is involved) and a filename 894 indicating the location of the module. 895 """ 896 897 name_py = name + extsep + "py" 898 filename = self._find_file(d, name_py) 899 if filename: 900 return None, filename 901 return None 902 903 def _find_package(self, d, name): 904 905 """ 906 In the directory 'd', find the given package 'name', returning None 907 where no suitable package directory exists, or a 2-tuple consisting of 908 a directory (indicating the location of the package directory itself) 909 and a filename indicating the location of the __init__.py module which 910 declares the package's top-level contents. 911 """ 912 913 filename = self._find_file(d, name) 914 if filename: 915 init_py = "__init__" + extsep + "py" 916 init_py_filename = self._find_file(filename, init_py) 917 if init_py_filename: 918 return filename, init_py_filename 919 return None 920 921 def _find_file(self, d, filename): 922 923 """ 924 Return the filename obtained when searching the directory 'd' for the 925 given 'filename', or None if no actual file exists for the filename. 926 """ 927 928 filename = join(d, filename) 929 if exists(filename): 930 return filename 931 else: 932 return None 933 934 def load(self, name): 935 936 """ 937 Load the module or package with the given 'name'. Return an object 938 referencing the loaded module or package, or None if no such module or 939 package exists. 940 """ 941 942 # Loaded modules are returned immediately. 943 # Modules may be known but not yet loading (having been registered as 944 # submodules), loading, loaded, or completely unknown. 945 946 module = self.get_module(name) 947 948 if module: 949 return self.modules[name] 950 951 # Otherwise, modules are loaded. 952 953 # Split the name into path components, and try to find the uppermost in 954 # the search path. 955 956 path = name.split(".") 957 path_so_far = [] 958 module = None 959 960 for p in path: 961 962 # Get the module's filesystem details. 963 964 if not path_so_far: 965 m = self.find_in_path(p) 966 elif d: 967 m = self.find(d, p) 968 else: 969 m = None 970 971 path_so_far.append(p) 972 module_name = ".".join(path_so_far) 973 974 # Raise an exception if the module could not be located. 975 976 if not m: 977 raise ProgramError("Module not found: %s" % name) 978 979 # Get the directory and module filename. 980 981 d, filename = m 982 983 # Get the module itself. 984 985 return self.load_from_file(filename, module_name) 986 987 def load_from_file(self, filename, module_name=None): 988 989 "Load the module from the given 'filename'." 990 991 if module_name is None: 992 module_name = "__main__" 993 994 module = self.modules.get(module_name) 995 996 if not module: 997 998 # Try to load from cache. 999 1000 module = self.load_from_cache(filename, module_name) 1001 if module: 1002 return module 1003 1004 # If no cache entry exists, load from file. 1005 1006 module = inspector.InspectedModule(module_name, self) 1007 self.add_module(module_name, module) 1008 self.update_cache_validity(module) 1009 1010 self._load(module, module_name, lambda m: m.parse, filename) 1011 1012 return module 1013 1014 def update_cache_validity(self, module): 1015 1016 "Make 'module' valid in the cache, but invalidate accessing modules." 1017 1018 accessing = self.accessing_modules.get(module.name) 1019 if accessing: 1020 self.invalidated.update(accessing) 1021 if module.name in self.invalidated: 1022 self.invalidated.remove(module.name) 1023 1024 def source_is_new(self, filename, module_name): 1025 1026 "Return whether 'filename' is newer than the cached 'module_name'." 1027 1028 if self.cache: 1029 cache_filename = join(self.cache, module_name) 1030 return not exists(cache_filename) or \ 1031 getmtime(filename) > getmtime(cache_filename) or \ 1032 module_name in self.invalidated 1033 else: 1034 return True 1035 1036 def load_from_cache(self, filename, module_name): 1037 1038 "Return a module residing in the cache." 1039 1040 module = self.modules.get(module_name) 1041 1042 if not module and not self.source_is_new(filename, module_name): 1043 module = CachedModule(module_name, self) 1044 self.add_module(module_name, module) 1045 1046 filename = join(self.cache, module_name) 1047 self._load(module, module_name, lambda m: m.from_cache, filename) 1048 1049 return module 1050 1051 def _load(self, module, module_name, fn, filename): 1052 1053 """ 1054 Load 'module' for the given 'module_name', and with 'fn' performing an 1055 invocation on the module with the given 'filename'. 1056 """ 1057 1058 # Load the module. 1059 1060 if self.verbose: 1061 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1062 fn(module)(filename) 1063 1064 # Add the module object if not already defined. 1065 1066 if not self.objects.has_key(module_name): 1067 self.objects[module_name] = Reference("<module>", module_name) 1068 1069 def add_module(self, module_name, module): 1070 1071 """ 1072 Return the module with the given 'module_name', adding a new module 1073 object if one does not already exist. 1074 """ 1075 1076 self.modules[module_name] = module 1077 if module_name in self.to_import: 1078 self.to_import.remove(module_name) 1079 1080 # vim: tabstop=4 expandtab shiftwidth=4