1 #!/usr/bin/env python 2 3 """ 4 Module abstractions. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from common import * 24 from encoders import decode_modifier_term, encode_modifiers, encode_usage 25 from referencing import decode_reference, Reference 26 import sys 27 28 class BasicModule(CommonModule): 29 30 "The basic module information." 31 32 def __init__(self, name, importer): 33 CommonModule.__init__(self, name, importer) 34 35 # Import machinery links. 36 37 self.loaded = False 38 39 # Module dependencies. 40 41 self.imported = set() 42 self.imported_hidden = set() 43 self.imported_names = {} 44 self.revealed = set() 45 self.accessing_modules = set() 46 47 # Global name information. 48 49 self.objects = {} 50 self.special = {} 51 52 # Class relationships. 53 54 self.classes = {} 55 56 # Attributes. 57 58 self.class_attrs = {} 59 self.instance_attrs = {} 60 self.instance_attr_constants = {} 61 self.module_attrs = set() 62 63 # Names used and missing. 64 65 self.names_used = {} 66 self.names_missing = {} 67 self.name_references = {} # references to globals 68 69 # Function details. 70 71 self.function_parameters = {} 72 self.function_defaults = {} 73 self.function_locals = {} 74 self.scope_globals = {} 75 76 # Invocation details. 77 78 self.function_targets = {} 79 self.function_arguments = {} 80 81 # Attribute usage at module and function levels. 82 83 self.attr_usage = {} 84 self.name_initialisers = {} 85 86 # General attribute access expressions. 87 88 self.attr_accesses = {} 89 self.const_accesses = {} 90 91 # Attribute accessor definition details. 92 93 self.attr_accessors = {} 94 95 # Assignment details for accesses. 96 97 self.attr_access_modifiers = {} 98 99 # Initialisation-related details. 100 101 self.initialised_names = {} 102 self.aliased_names = {} 103 104 def __repr__(self): 105 return "BasicModule(%r, %r)" % (self.name, self.importer) 106 107 def resolve(self): 108 109 "Resolve dependencies and complete definitions." 110 111 self.resolve_class_bases() 112 self.check_special() 113 self.check_names_used() 114 self.resolve_members() 115 self.resolve_initialisers() 116 self.resolve_literals() 117 self.remove_redundant_accessors() 118 self.set_invocation_usage() 119 120 # Propagate to the importer information needed in subsequent activities. 121 122 self.propagate() 123 124 # Derived information methods. 125 126 def propagate(self): 127 128 "Finalise and propagate module information." 129 130 self.propagate_attrs() 131 self.propagate_name_references() 132 self.propagate_attr_accesses() 133 self.propagate_constants() 134 135 def unpropagate(self): 136 137 """ 138 Retract information from the importer including information about this 139 module derived by the importer. 140 """ 141 142 del self.importer.all_module_attrs[self.name] 143 144 for name in self.classes.keys(): 145 del self.importer.all_class_attrs[name] 146 del self.importer.all_instance_attrs[name] 147 del self.importer.all_combined_attrs[name] 148 del self.importer.all_instance_attr_constants[name] 149 150 for name, bases in self.classes.items(): 151 for base in bases: 152 153 # Get the identity of the class from the reference. 154 155 base = base.get_origin() 156 157 try: 158 self.importer.subclasses[base].remove(name) 159 except (KeyError, ValueError): 160 pass 161 162 remove_items(self.importer.all_name_references, self.name_references) 163 remove_items(self.importer.all_initialised_names, self.initialised_names) 164 remove_items(self.importer.all_aliased_names, self.aliased_names) 165 remove_items(self.importer.all_attr_accesses, self.attr_accesses) 166 remove_items(self.importer.all_const_accesses, self.const_accesses) 167 remove_items(self.importer.all_attr_access_modifiers, self.attr_access_modifiers) 168 remove_items(self.importer.all_constants, self.constants) 169 remove_items(self.importer.all_constant_values, self.constant_values) 170 171 # Remove this module's objects from the importer. Objects are 172 # automatically propagated when defined. 173 174 for name, ref in self.objects.items(): 175 if ref.provided_by_module(self.name) or name in self.importer.hidden: 176 if ref.get_kind() != "<module>": 177 del self.importer.objects[name] 178 179 def resolve_class_bases(self): 180 181 "Resolve all class bases since some of them may have been deferred." 182 183 for name, bases in self.classes.items(): 184 resolved = [] 185 bad = [] 186 187 for base in bases: 188 189 # Resolve dependencies. 190 191 if base.has_kind("<depends>"): 192 ref = self.importer.get_object(base.get_origin()) 193 else: 194 ref = base 195 196 # Obtain the origin of the base class reference. 197 198 if not ref or not ref.has_kind("<class>"): 199 bad.append(base) 200 break 201 202 resolved.append(ref) 203 204 if bad: 205 print >>sys.stderr, "Bases of class %s were not classes." % (name, ", ".join(map(str, bad))) 206 else: 207 self.importer.classes[name] = self.classes[name] = resolved 208 209 def propagate_attrs(self): 210 211 "Derive attributes from the class and module member details." 212 213 # Initialise class attribute records for all classes. 214 215 for name in self.classes.keys(): 216 self.importer.all_class_attrs[name] = self.class_attrs[name] = {} 217 218 # Separate the objects into module and class attributes. 219 220 for name in self.objects.keys(): 221 if "." in name: 222 parent, attrname = name.rsplit(".", 1) 223 if self.classes.has_key(parent): 224 self.class_attrs[parent][attrname] = name 225 elif parent == self.name: 226 self.module_attrs.add(attrname) 227 228 # Propagate the module attributes. 229 230 self.importer.all_module_attrs[self.name] = self.module_attrs 231 232 def propagate_name_references(self): 233 234 "Propagate name references for the module." 235 236 self.importer.all_name_references.update(self.name_references) 237 self.importer.all_initialised_names.update(self.initialised_names) 238 self.importer.all_aliased_names.update(self.aliased_names) 239 240 def propagate_attr_accesses(self): 241 242 "Propagate attribute accesses for the module." 243 244 self.importer.all_attr_accesses.update(self.attr_accesses) 245 self.importer.all_const_accesses.update(self.const_accesses) 246 self.importer.all_attr_access_modifiers.update(self.attr_access_modifiers) 247 248 def propagate_constants(self): 249 250 "Propagate constant values and aliases for the module." 251 252 self.importer.all_constants.update(self.constants) 253 self.importer.all_constant_values.update(self.constant_values) 254 255 for name in self.classes.keys(): 256 self.importer.all_instance_attrs[name] = self.instance_attrs.get(name) or {} 257 self.importer.all_instance_attr_constants[name] = self.instance_attr_constants.get(name) or {} 258 259 # Module-relative naming. 260 261 def is_global(self, name): 262 263 """ 264 Return whether 'name' is registered as a global in the current 265 namespace. 266 """ 267 268 path = self.get_namespace_path() 269 return name in self.scope_globals.get(path, []) 270 271 def get_globals(self): 272 273 """ 274 Get the globals from this module, returning a dictionary mapping names 275 to references incorporating original definition details. 276 """ 277 278 l = [] 279 for name, value in self.objects.items(): 280 parent, attrname = name.rsplit(".", 1) 281 if parent == self.name: 282 l.append((attrname, value)) 283 return dict(l) 284 285 def get_global(self, name): 286 287 """ 288 Get the global of the given 'name' from this module, returning a 289 reference incorporating the original definition details. 290 """ 291 292 path = self.get_global_path(name) 293 return self.objects.get(path) 294 295 # Name definition discovery. 296 297 def get_global_or_builtin(self, name): 298 299 """ 300 Return the object recorded the given 'name' from this module or from the 301 __builtins__ module. If no object has yet been defined, perhaps due to 302 circular module references, None is returned. 303 """ 304 305 return self.get_global(name) or self.get_builtin(name) 306 307 def get_builtin(self, name): 308 309 "Return the object providing the given built-in 'name'." 310 311 path = "__builtins__.%s" % name 312 313 # Obtain __builtins__. 314 315 module = self.ensure_builtins(path) 316 317 # Attempt to find the named object within __builtins__. 318 319 if module: 320 self.find_imported_name(name, module.name, module) 321 322 # Return the path and any reference for the named object. 323 324 return self.importer.get_object(path) 325 326 def ensure_builtins(self, path): 327 328 """ 329 If 'path' is a reference to an object within __builtins__, return the 330 __builtins__ module. 331 """ 332 333 if path.split(".", 1)[0] == "__builtins__": 334 return self.importer.load("__builtins__", True, True) 335 else: 336 return None 337 338 def get_object(self, path): 339 340 """ 341 Get the details of an object with the given 'path', either from this 342 module or from the whole program. Return a tuple containing the path and 343 any object found. 344 """ 345 346 if self.objects.has_key(path): 347 return self.objects[path] 348 else: 349 self.ensure_builtins(path) 350 return self.importer.get_object(path) 351 352 def set_object(self, name, value=None): 353 354 "Set an object with the given 'name' and the given 'value'." 355 356 ref = decode_reference(value, name) 357 multiple = self.objects.has_key(name) and self.objects[name].get_kind() != ref.get_kind() 358 self.importer.objects[name] = self.objects[name] = multiple and ref.as_var() or ref 359 360 # Special names. 361 362 def get_special(self, name): 363 364 "Return any stored value for the given special 'name'." 365 366 return self.special.get(name) 367 368 def set_special(self, name, value): 369 370 """ 371 Set a special 'name' that merely tracks the use of an implicit object 372 'value'. 373 """ 374 375 self.special[name] = value 376 377 def set_special_literal(self, name, ref): 378 379 """ 380 Set a special name for the literal type 'name' having type 'ref'. Such 381 special names provide a way of referring to literal object types. 382 """ 383 384 literal_name = "$L%s" % name 385 value = ResolvedNameRef(literal_name, ref) 386 self.set_special(literal_name, value) 387 388 # Revealing modules by tracking name imports across modules. 389 390 def set_imported_name(self, name, module_name, alias=None, path=None): 391 392 """ 393 Record 'name' as being imported from the given 'module_name', employing 394 the given 'alias' in the local namespace if specified. 395 """ 396 397 path = path or self.get_namespace_path() 398 init_item(self.imported_names, path, dict) 399 self.imported_names[path][alias or name] = (name, module_name) 400 401 def get_imported_name(self, name, path): 402 403 "Get details of any imported 'name' within the namespace at 'path'." 404 405 if self.imported_names.has_key(path): 406 return self.imported_names[path].get(name) 407 else: 408 return None 409 410 def find_imported_name(self, name, path, module=None): 411 412 """ 413 Find details of the imported 'name' within the namespace at 'path', 414 starting within the given 'module' if indicated, or within this module 415 otherwise. 416 """ 417 418 module = module or self 419 420 # Obtain any module required by the name. 421 422 name_modname = module.get_imported_name(name, path) 423 424 if name_modname: 425 name, modname = name_modname 426 module = self._find_imported_name(name, modname, module) 427 428 # Obtain the name from the final module, revealing it if appropriate. 429 430 if module: 431 init_item(self.importer.revealing, module.name, set) 432 self.importer.set_revealing(module, name, self) 433 434 def _find_imported_name(self, name, modname, module): 435 436 """ 437 Traverse details for 'name' via 'modname' to the module providing the 438 name, tentatively revealing the module even if the module is not yet 439 loaded and cannot provide the details of the object recorded for the 440 name. 441 """ 442 443 _name = name 444 445 # Obtain any modules referenced by each required module. 446 447 while True: 448 449 # Get the module directly or traverse possibly-aliased names. 450 451 module = self.get_module_direct(modname, True) 452 if not module: 453 top, module = self.get_module(modname, True) 454 name_modname = module.get_imported_name(_name, module.name) 455 if not name_modname: 456 break 457 else: 458 _name, modname = name_modname 459 460 return module 461 462 def reveal_referenced(self): 463 464 """ 465 Reveal modules referenced by this module. 466 """ 467 468 for path, names in self.imported_names.items(): 469 for alias, (name, modname) in names.items(): 470 module = self._find_imported_name(name, modname, self) 471 self.reveal_module(module) 472 473 def reveal_module(self, module): 474 475 """ 476 Reveal the given 'module', recording the revealed modules on this 477 module. 478 """ 479 480 if module is not self: 481 self.importer.reveal_module(module) 482 self.revealed.add(module) 483 module.accessing_modules.add(self.name) 484 485 # Module loading. 486 487 def get_module_direct(self, modname, hidden=False): 488 489 """ 490 Return 'modname' without traversing parent modules, keeping the module 491 'hidden' if set to a true value, loading the module if not already 492 loaded. 493 """ 494 495 return self.importer.get_module(modname, True) or self.importer.load(modname, hidden=hidden) 496 497 def get_module(self, name, hidden=False): 498 499 """ 500 Use the given 'name' to obtain the identity of a module. Return module 501 objects or None if the module cannot be found. This method is required 502 when aliases are used to refer to modules and where a module "path" does 503 not correspond to the actual module path. 504 505 A tuple is returned containing the top or base module and the deepest or 506 leaf module involved. 507 """ 508 509 path_so_far = [] 510 top = module = None 511 parts = name.split(".") 512 513 for i, part in enumerate(parts): 514 path_so_far.append(part) 515 module_name = ".".join(path_so_far) 516 ref = self.get_object(module_name) 517 518 # If no known object exists, attempt to load it. 519 520 if not ref: 521 module = self.importer.load(module_name, True, hidden) 522 if not module: 523 return None 524 525 # Rewrite the path to use the actual module details. 526 527 path_so_far = module.name.split(".") 528 529 # If the object exists and is not a module, stop. 530 531 elif ref.has_kind(["<class>", "<function>", "<var>"]): 532 return None 533 534 else: 535 module = self.importer.get_module(ref.get_origin(), hidden) 536 537 if not top: 538 top = module 539 540 return top, module 541 542 class CachedModule(BasicModule): 543 544 "A cached module." 545 546 def __repr__(self): 547 return "CachedModule(%r, %r)" % (self.name, self.importer) 548 549 def resolve(self): 550 pass 551 552 def to_cache(self, filename): 553 554 "Not actually writing the module back to 'filename'." 555 556 pass 557 558 def from_cache(self, filename): 559 560 """ 561 Read a module's details from the file with the given 'filename' as 562 described in the to_cache method of InspectedModule. 563 """ 564 565 f = open(filename) 566 try: 567 self.filename = f.readline().rstrip() 568 569 accessing_modules = f.readline().split(": ", 1)[-1].rstrip() 570 571 module_names = f.readline().split(": ", 1)[-1].rstrip() 572 if module_names: 573 for module_name in module_names.split(", "): 574 self.imported.add(self.importer.load(module_name)) 575 576 module_names = f.readline().split(": ", 1)[-1].rstrip() 577 if module_names: 578 for module_name in module_names.split(", "): 579 self.imported_hidden.add(self.importer.load(module_name, hidden=True)) 580 581 module_names = f.readline().split(": ", 1)[-1].rstrip() 582 if module_names: 583 for module_name in module_names.split(", "): 584 module = self.importer.load(module_name, True, True) 585 self.reveal_module(module) 586 587 f.readline() # (empty line) 588 589 self._get_imported_names(f) 590 self._get_members(f) 591 self._get_class_relationships(f) 592 self._get_instance_attrs(f) 593 self._get_instance_attr_constants(f) 594 self.from_lines(f, self.names_used) # "names used:" 595 self.from_lines(f, self.names_missing) # "names missing:" 596 self._get_name_references(f) 597 self._get_initialised_names(f) 598 self._get_aliased_names(f) 599 self._get_function_parameters(f) 600 self._get_function_defaults(f) 601 self._get_function_locals(f) 602 self.from_lines(f, self.scope_globals) # "scope globals:" 603 self._get_function_targets(f) 604 self._get_function_arguments(f) 605 self._get_attribute_usage(f) 606 self._get_attr_accesses(f) 607 self._get_const_accesses(f) 608 self._get_attr_accessors(f) 609 self._get_attr_access_modifiers(f) 610 self._get_constant_literals(f) 611 self._get_constant_values(f) 612 613 finally: 614 f.close() 615 616 self.loaded = True 617 618 def resolve(self): 619 self.propagate() 620 621 def _get_imported_names(self, f): 622 f.readline() # "imported names:" 623 line = f.readline().rstrip() 624 while line: 625 path, alias_or_name, name, module_name = self._get_fields(line, 4) 626 init_item(self.imported_names, path, dict) 627 self.imported_names[path][alias_or_name] = (name, module_name) 628 line = f.readline().rstrip() 629 630 def _get_members(self, f): 631 f.readline() # "members:" 632 line = f.readline().rstrip() 633 while line: 634 name, ref = line.split(" ", 1) 635 self.set_object(name, ref) 636 line = f.readline().rstrip() 637 638 def _get_class_relationships(self, f): 639 f.readline() # "class relationships:" 640 line = f.readline().rstrip() 641 while line: 642 name, value = self._get_fields(line) 643 values = value and value.split(", ") or [] 644 self.importer.classes[name] = self.classes[name] = map(decode_reference, values) 645 self.importer.subclasses[name] = set() 646 line = f.readline().rstrip() 647 648 def _get_instance_attrs(self, f): 649 f.readline() # "instance attributes:" 650 line = f.readline().rstrip() 651 while line: 652 name, value = self._get_fields(line) 653 self.importer.all_instance_attrs[name] = self.instance_attrs[name] = set(value and value.split(", ") or []) 654 line = f.readline().rstrip() 655 656 def _get_instance_attr_constants(self, f): 657 f.readline() # "instance attribute constants:" 658 line = f.readline().rstrip() 659 while line: 660 name, attrname, ref = self._get_fields(line, 3) 661 init_item(self.instance_attr_constants, name, dict) 662 self.instance_attr_constants[name][attrname] = decode_reference(ref) 663 line = f.readline().rstrip() 664 665 def _get_name_references(self, f): 666 f.readline() # "name references:" 667 line = f.readline().rstrip() 668 while line: 669 name, value = self._get_fields(line) 670 self.name_references[name] = value 671 line = f.readline().rstrip() 672 673 def _get_initialised_names(self, f): 674 f.readline() # "initialised names:" 675 line = f.readline().rstrip() 676 while line: 677 name, version, value = self._get_fields(line, 3) 678 init_item(self.initialised_names, name, dict) 679 self.initialised_names[name][int(version)] = decode_reference(value) 680 line = f.readline().rstrip() 681 682 def _get_aliased_names(self, f): 683 f.readline() # "aliased names:" 684 line = f.readline().rstrip() 685 while line: 686 name, version, original_name, attrnames, number = self._get_fields(line, 5) 687 init_item(self.aliased_names, name, dict) 688 if number == "{}": number = None 689 else: number = int(number) 690 self.aliased_names[name][int(version)] = (original_name, attrnames != "{}" and attrnames or None, number) 691 line = f.readline().rstrip() 692 693 def _get_function_parameters(self, f): 694 f.readline() # "function parameters:" 695 line = f.readline().rstrip() 696 while line: 697 function, names = self._get_fields(line) 698 self.importer.function_parameters[function] = \ 699 self.function_parameters[function] = names and names.split(", ") or [] 700 line = f.readline().rstrip() 701 702 def _get_function_defaults(self, f): 703 f.readline() # "function default parameters:" 704 line = f.readline().rstrip() 705 while line: 706 function, defaults = self._get_fields(line) 707 self.importer.function_defaults[function] = \ 708 self.function_defaults[function] = l = [] 709 if defaults != "{}": 710 for value in defaults.split(", "): 711 name, default = value.split("=") 712 default = decode_reference(default) 713 l.append((name, default)) 714 line = f.readline().rstrip() 715 716 def _get_function_locals(self, f): 717 f.readline() # "function locals:" 718 line = f.readline().rstrip() 719 while line: 720 function, name, value = self._get_fields(line, 3) 721 init_item(self.function_locals, function, dict) 722 if name != "{}": 723 self.function_locals[function][name] = decode_reference(value) 724 line = f.readline().rstrip() 725 726 def _get_function_targets(self, f): 727 f.readline() # "function targets:" 728 line = f.readline().rstrip() 729 while line: 730 function, n = self._get_fields(line) 731 self.importer.function_targets[function] = \ 732 self.function_targets[function] = int(n) 733 line = f.readline().rstrip() 734 735 def _get_function_arguments(self, f): 736 f.readline() # "function arguments:" 737 line = f.readline().rstrip() 738 while line: 739 function, n = self._get_fields(line) 740 self.importer.function_arguments[function] = \ 741 self.function_arguments[function] = int(n) 742 line = f.readline().rstrip() 743 744 def _get_attribute_usage(self, f): 745 f.readline() # "attribute usage:" 746 line = f.readline().rstrip() 747 while line: 748 unit, value = self._get_fields(line) 749 init_item(self.attr_usage, unit, dict) 750 self.usage_from_cache(value, self.attr_usage[unit]) 751 line = f.readline().rstrip() 752 753 def _get_attr_accesses(self, f): 754 f.readline() # "attribute accesses:" 755 line = f.readline().rstrip() 756 while line: 757 name, value = self._get_fields(line) 758 self.attr_accesses[name] = set(value.split(", ")) 759 line = f.readline().rstrip() 760 761 def _get_const_accesses(self, f): 762 f.readline() # "constant accesses:" 763 line = f.readline().rstrip() 764 while line: 765 name, original_name, attrnames, objpath, ref, remaining = self._get_fields(line, 6) 766 if attrnames == "{}": attrnames = None 767 init_item(self.const_accesses, name, dict) 768 self.const_accesses[name][(original_name, attrnames)] = (objpath, decode_reference(ref), remaining != "{}" and remaining or "") 769 line = f.readline().rstrip() 770 771 def _get_attr_accessors(self, f): 772 f.readline() # "attribute access usage:" 773 line = f.readline().rstrip() 774 while line: 775 objpath, name, attrname, value = self._get_fields(line, 4) 776 if attrname == "{}": attrname = None 777 access = name, attrname 778 init_item(self.attr_accessors, objpath, dict) 779 init_item(self.attr_accessors[objpath], access, list) 780 positions = map(int, value.split(", ")) 781 self.attr_accessors[objpath][access].append(positions) 782 line = f.readline().rstrip() 783 784 def _get_attr_access_modifiers(self, f): 785 f.readline() # "attribute access modifiers:" 786 line = f.readline().rstrip() 787 while line: 788 objpath, name, attrnames, value = self._get_fields(line, 4) 789 if name == "{}": name = None 790 if attrnames == "{}": attrnames = None 791 access = name, attrnames 792 init_item(self.attr_access_modifiers, objpath, dict) 793 init_item(self.attr_access_modifiers[objpath], access, list) 794 modifiers = [decode_modifier_term(s) for s in value] 795 self.attr_access_modifiers[objpath][access] = modifiers 796 line = f.readline().rstrip() 797 798 def _get_constant_literals(self, f): 799 f.readline() # "constant literals:" 800 line = f.readline().rstrip() 801 last_path = None 802 n = None 803 while line: 804 path, constant = self._get_fields(line) 805 if path != last_path: 806 n = 0 807 last_path = path 808 else: 809 n += 1 810 init_item(self.constants, path, dict) 811 self.constants[path][eval(constant)] = n 812 line = f.readline().rstrip() 813 814 def _get_constant_values(self, f): 815 f.readline() # "constant values:" 816 line = f.readline().rstrip() 817 while line: 818 name, value_type, value = self._get_fields(line, 3) 819 self.constant_values[name] = eval(value), value_type 820 line = f.readline().rstrip() 821 822 # Generic parsing methods. 823 824 def from_lines(self, f, d): 825 826 "Read lines from 'f', populating 'd'." 827 828 f.readline() # section heading 829 line = f.readline().rstrip() 830 while line: 831 name, value = self._get_fields(line) 832 d[name] = set(value and value.split(", ") or []) 833 line = f.readline().rstrip() 834 835 def usage_from_cache(self, value, mapping): 836 837 """ 838 Interpret the given 'value' containing name and usage information, 839 storing the information in the given 'mapping'. 840 """ 841 842 local, usage = self._get_fields(value) 843 init_item(mapping, local, list) 844 self._usage_from_cache(mapping[local], usage) 845 846 def _usage_from_cache(self, d, usage): 847 848 # Interpret descriptions of each version of the name. 849 850 all_usages = set() 851 for attrnames in usage.split("; "): 852 if attrnames == "{}": 853 all_attrnames = () 854 else: 855 # Decode attribute details for each usage description. 856 857 all_attrnames = set() 858 for attrname_str in attrnames.split(", "): 859 all_attrnames.add(attrname_str) 860 861 all_attrnames = list(all_attrnames) 862 all_attrnames.sort() 863 864 all_usages.add(tuple(all_attrnames)) 865 866 d.append(all_usages) 867 868 def _get_fields(self, s, n=2): 869 result = s.split(" ", n-1) 870 if len(result) == n: 871 return result 872 else: 873 return tuple(result) + tuple([""] * (n - len(result))) 874 875 class CacheWritingModule: 876 877 """ 878 A mix-in providing cache-writing support, to be combined with BasicModule. 879 """ 880 881 def to_cache(self, filename): 882 883 """ 884 Write a cached representation of the inspected module with the following 885 format to the file having the given 'filename': 886 887 filename 888 "accessed by:" accessing module names during this module's import 889 "imports:" imported module names 890 "hidden imports:" imported module names 891 "reveals:" revealed module names 892 (empty line) 893 "imported names:" 894 zero or more: qualified name " " name " " module name 895 (empty line) 896 "members:" 897 zero or more: qualified name " " reference 898 (empty line) 899 "class relationships:" 900 zero or more: qualified class name " " base class references 901 (empty line) 902 "instance attributes:" 903 zero or more: qualified class name " " instance attribute names 904 (empty line) 905 "instance attribute constants:" 906 zero or more: qualified class name " " attribute name " " reference 907 (empty line) 908 "names used:" 909 zero or more: qualified class/function/module name " " names 910 (empty line) 911 "names missing:" 912 zero or more: qualified class/function/module name " " names 913 (empty line) 914 "name references:" 915 zero or more: qualified name " " reference 916 (empty line) 917 "initialised names:" 918 zero or more: qualified name " " definition version " " reference 919 (empty line) 920 "aliased names:" 921 zero or more: qualified name " " definition version " " original name " " attribute names " " access number 922 (empty line) 923 "function parameters:" 924 zero or more: qualified function name " " parameter names 925 (empty line) 926 "function default parameters:" 927 zero or more: qualified function name " " parameter names with defaults 928 (empty line) 929 "function locals:" 930 zero or more: qualified function name " " local variable name " " reference 931 (empty line) 932 "scope globals:" 933 zero or more: qualified function name " " global variable names 934 (empty line) 935 "function targets:" 936 zero or more: qualified function name " " maximum number of targets allocated 937 (empty line) 938 "function arguments:" 939 zero or more: qualified function name " " maximum number of arguments allocated 940 (empty line) 941 "attribute usage:" 942 zero or more: qualified scope name " " local/global/qualified variable name " " usages 943 (empty line) 944 "attribute accesses:" 945 zero or more: qualified scope name " " attribute-chains 946 (empty line) 947 "constant accesses:" 948 zero or more: qualified function name " " attribute-chain " " reference " " remaining attribute-chain 949 (empty line) 950 "attribute access usage:" 951 zero or more: qualified function name " " local/global variable name " " attribute name " " definition versions 952 (empty line) 953 "attribute access modifiers:" 954 zero or more: qualified function name " " local/global variable name " " attribute name " " access modifiers 955 "constant literals:" 956 zero or more: qualified scope name " " constant literal 957 "constant values:" 958 zero or more: qualified name " " value type " " constant literal 959 960 All collections of names are separated by ", " characters. 961 962 References can be "<var>", a module name, or one of "<class>" or 963 "<function>" followed optionally by a ":" character and a qualified 964 name. 965 966 Parameter names with defaults are separated by ", " characters, with 967 each name followed by "=" and then followed by a reference. If "{}" is 968 indicated, no defaults are defined for the function. Similarly, function 969 locals may be indicated as "{}" meaning that there are no locals. 970 971 All usages (attribute usage sets) are separated by "; " characters, with 972 the special string "{}" representing an empty set. 973 974 Each usage is a collection of names separated by ", " characters, with 975 assigned attribute names suffixed with a "*" character. 976 977 Each attribute-chain expression is a dot-separated chain of attribute 978 names, with assignments suffixed with a "*" character. 979 980 Definition versions are separated by ", " characters and indicate the 981 name definition version associated with the access. 982 983 Access modifiers are separated by ", " characters and indicate features 984 of each access, with multiple accesses described on a single line. 985 """ 986 987 f = open(filename, "w") 988 try: 989 print >>f, self.filename 990 991 accessing_modules = list(self.accessing_modules) 992 accessing_modules.sort() 993 print >>f, "accessed by:", ", ".join(accessing_modules) 994 995 module_names = [m.name for m in self.imported] 996 module_names.sort() 997 print >>f, "imports:", ", ".join(module_names) 998 999 module_names = [m.name for m in self.imported_hidden] 1000 module_names.sort() 1001 print >>f, "hidden imports:", ", ".join(module_names) 1002 1003 module_names = [m.name for m in self.revealed] 1004 module_names.sort() 1005 print >>f, "reveals:", ", ".join(module_names) 1006 1007 print >>f 1008 print >>f, "imported names:" 1009 paths = self.imported_names.keys() 1010 paths.sort() 1011 for path in paths: 1012 names = self.imported_names[path].keys() 1013 names.sort() 1014 for alias_or_name in names: 1015 name, modname = self.imported_names[path][alias_or_name] 1016 print >>f, path, alias_or_name, name, modname 1017 1018 print >>f 1019 print >>f, "members:" 1020 objects = self.objects.keys() 1021 objects.sort() 1022 for name in objects: 1023 print >>f, name, self.objects[name] 1024 1025 print >>f 1026 print >>f, "class relationships:" 1027 classes = self.classes.keys() 1028 classes.sort() 1029 for class_ in classes: 1030 bases = self.classes[class_] 1031 if bases: 1032 print >>f, class_, ", ".join(map(str, bases)) 1033 else: 1034 print >>f, class_ 1035 1036 self.to_lines(f, "instance attributes:", self.instance_attrs) 1037 1038 print >>f 1039 print >>f, "instance attribute constants:" 1040 classes = self.instance_attr_constants.items() 1041 classes.sort() 1042 for name, attrs in classes: 1043 attrs = attrs.items() 1044 attrs.sort() 1045 for attrname, ref in attrs: 1046 print >>f, name, attrname, ref 1047 1048 self.to_lines(f, "names used:", self.names_used) 1049 self.to_lines(f, "names missing:", self.names_missing) 1050 1051 print >>f 1052 print >>f, "name references:" 1053 refs = self.name_references.items() 1054 refs.sort() 1055 for name, ref in refs: 1056 print >>f, name, ref 1057 1058 print >>f 1059 print >>f, "initialised names:" 1060 assignments = self.initialised_names.items() 1061 assignments.sort() 1062 for name, refs in assignments: 1063 versions = refs.items() 1064 versions.sort() 1065 for version, ref in versions: 1066 print >>f, name, version, ref 1067 1068 print >>f 1069 print >>f, "aliased names:" 1070 assignments = self.aliased_names.items() 1071 assignments.sort() 1072 for name, aliases in assignments: 1073 versions = aliases.items() 1074 versions.sort() 1075 for version, alias in versions: 1076 original_name, attrnames, number = alias 1077 print >>f, name, version, original_name, attrnames or "{}", number is None and "{}" or number 1078 1079 print >>f 1080 print >>f, "function parameters:" 1081 functions = self.function_parameters.keys() 1082 functions.sort() 1083 for function in functions: 1084 print >>f, function, ", ".join(self.function_parameters[function]) 1085 1086 print >>f 1087 print >>f, "function default parameters:" 1088 functions = self.function_defaults.keys() 1089 functions.sort() 1090 for function in functions: 1091 parameters = self.function_defaults[function] 1092 if parameters: 1093 print >>f, function, ", ".join([("%s=%s" % (name, default)) for (name, default) in parameters]) 1094 else: 1095 print >>f, function, "{}" 1096 1097 print >>f 1098 print >>f, "function locals:" 1099 functions = self.function_locals.keys() 1100 functions.sort() 1101 for function in functions: 1102 names = self.function_locals[function].items() 1103 if names: 1104 names.sort() 1105 for name, value in names: 1106 print >>f, function, name, value 1107 else: 1108 print >>f, function, "{}" 1109 1110 self.to_lines(f, "scope globals:", self.scope_globals) 1111 1112 print >>f 1113 print >>f, "function targets:" 1114 functions = self.function_targets.keys() 1115 functions.sort() 1116 for function in functions: 1117 print >>f, function, self.function_targets[function] 1118 1119 print >>f 1120 print >>f, "function arguments:" 1121 functions = self.function_arguments.keys() 1122 functions.sort() 1123 for function in functions: 1124 print >>f, function, self.function_arguments[function] 1125 1126 print >>f 1127 print >>f, "attribute usage:" 1128 units = self.attr_usage.keys() 1129 units.sort() 1130 for unit in units: 1131 d = self.attr_usage[unit] 1132 self.usage_to_cache(d, f, unit) 1133 1134 print >>f 1135 print >>f, "attribute accesses:" 1136 paths = self.attr_accesses.keys() 1137 paths.sort() 1138 for path in paths: 1139 accesses = list(self.attr_accesses[path]) 1140 accesses.sort() 1141 print >>f, path, ", ".join(accesses) 1142 1143 print >>f 1144 print >>f, "constant accesses:" 1145 paths = self.const_accesses.keys() 1146 paths.sort() 1147 for path in paths: 1148 accesses = self.const_accesses[path].items() 1149 accesses.sort() 1150 for (original_name, attrnames), (objpath, ref, remaining_attrnames) in accesses: 1151 print >>f, path, original_name, attrnames, objpath, ref, remaining_attrnames or "{}" 1152 1153 print >>f 1154 print >>f, "attribute access usage:" 1155 paths = self.attr_accessors.keys() 1156 paths.sort() 1157 for path in paths: 1158 all_accesses = self.attr_accessors[path].items() 1159 all_accesses.sort() 1160 for (name, attrname), accesses in all_accesses: 1161 for positions in accesses: 1162 positions = map(str, positions) 1163 print >>f, path, name, attrname or "{}", ", ".join(positions) 1164 1165 print >>f 1166 print >>f, "attribute access modifiers:" 1167 paths = self.attr_access_modifiers.keys() 1168 paths.sort() 1169 for path in paths: 1170 all_accesses = self.attr_access_modifiers[path].items() 1171 all_accesses.sort() 1172 for (name, attrnames), modifiers in all_accesses: 1173 print >>f, path, name or "{}", attrnames or "{}", encode_modifiers(modifiers) 1174 1175 print >>f 1176 print >>f, "constant literals:" 1177 paths = self.constants.keys() 1178 paths.sort() 1179 for path in paths: 1180 constants = [(v, k) for (k, v) in self.constants[path].items()] 1181 constants.sort() 1182 for n, constant in constants: 1183 print >>f, path, repr(constant) 1184 1185 print >>f 1186 print >>f, "constant values:" 1187 names = self.constant_values.keys() 1188 names.sort() 1189 for name in names: 1190 value, value_type = self.constant_values[name] 1191 print >>f, name, value_type, repr(value) 1192 1193 finally: 1194 f.close() 1195 1196 def to_lines(self, f, heading, d): 1197 1198 "Write lines to 'f' with the given 'heading', using 'd'." 1199 1200 print >>f 1201 print >>f, heading 1202 keys = d.keys() 1203 keys.sort() 1204 for key in keys: 1205 attrs = list(d[key]) 1206 if attrs: 1207 attrs.sort() 1208 print >>f, key, ", ".join(attrs) 1209 1210 def usage_to_cache(self, details, f, prefix): 1211 1212 "Write the given namespace usage details to the cache." 1213 1214 names = list(details.keys()) 1215 if names: 1216 names.sort() 1217 for name in names: 1218 if details[name]: 1219 1220 # Produce descriptions for each version of the name. 1221 1222 for version in details[name]: 1223 all_usages = [] 1224 for usage in version: 1225 all_usages.append(encode_usage(usage)) 1226 1227 print >>f, "%s %s %s" % (prefix, name, "; ".join(all_usages)) 1228 1229 # vim: tabstop=4 expandtab shiftwidth=4