1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 47 # Accesses to global attributes. 48 49 self.global_attr_accesses = {} 50 51 # Usage tracking. 52 53 self.trackers = [] 54 self.attr_accessor_branches = {} 55 56 def __repr__(self): 57 return "InspectedModule(%r, %r)" % (self.name, self.importer) 58 59 # Principal methods. 60 61 def parse(self, filename): 62 63 "Parse the file having the given 'filename'." 64 65 self.parse_file(filename) 66 67 # Inspect the module. 68 69 self.start_tracking_in_module() 70 71 # Detect and record imports and globals declared in the module. 72 73 self.assign_general_local("__name__", self.get_constant("str", self.name)) 74 self.assign_general_local("__file__", self.get_constant("str", filename)) 75 self.process_structure(self.astnode) 76 77 # Set the class of the module after the definition has occurred. 78 79 ref = self.get_builtin("object") 80 self.set_name("__class__", ref) 81 82 # Get module-level attribute usage details. 83 84 self.stop_tracking_in_module() 85 86 # Collect external name references. 87 88 self.collect_names() 89 90 def complete(self): 91 92 "Complete the module inspection." 93 94 # Resolve names not definitively mapped to objects. 95 96 self.resolve() 97 98 # Define the invocation requirements in each namespace. 99 100 self.set_invocation_usage() 101 102 # Propagate to the importer information needed in subsequent activities. 103 104 self.propagate() 105 106 # Accessory methods. 107 108 def collect_names(self): 109 110 "Collect the names used by each scope." 111 112 for path in self.names_used.keys(): 113 self.collect_names_for_path(path) 114 115 def collect_names_for_path(self, path): 116 117 """ 118 Collect the names used by the given 'path'. These are propagated to the 119 importer in advance of any dependency resolution. 120 """ 121 122 names = self.names_used.get(path) 123 if not names: 124 return 125 126 in_function = self.function_locals.has_key(path) 127 128 for name in names: 129 if in_function and name in self.function_locals[path]: 130 continue 131 132 key = "%s.%s" % (path, name) 133 134 # Find predefined constant names before anything else. 135 136 if name in predefined_constants: 137 ref = self.get_builtin(name) 138 self.set_name_reference(key, ref) 139 continue 140 141 # Find local definitions (within dynamic namespaces). 142 143 ref = self.get_resolved_object(key) 144 if ref: 145 self.set_name_reference(key, ref) 146 continue 147 148 # Find global. 149 150 ref = self.get_global(name) 151 if ref: 152 self.set_name_reference(key, ref) 153 continue 154 155 # Find presumed built-in definitions. 156 157 ref = self.get_builtin(name) 158 self.set_name_reference(key, ref) 159 160 def set_name_reference(self, path, ref): 161 162 "Map the given name 'path' to 'ref'." 163 164 self.importer.all_name_references[path] = self.name_references[path] = ref 165 166 # Module structure traversal. 167 168 def process_structure_node(self, n): 169 170 "Process the individual node 'n'." 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 self.process_assignment_node(n, None) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 self.process_attribute_access(n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Invocations. 282 283 elif isinstance(n, compiler.ast.CallFunc): 284 return self.process_invocation_node(n) 285 286 # Constant usage. 287 288 elif isinstance(n, compiler.ast.Const): 289 return self.get_literal_instance(n, n.value.__class__.__name__) 290 291 elif isinstance(n, compiler.ast.Dict): 292 return self.get_literal_instance(n, "dict") 293 294 elif isinstance(n, compiler.ast.List): 295 return self.get_literal_instance(n, "list") 296 297 elif isinstance(n, compiler.ast.Tuple): 298 return self.get_literal_instance(n, "tuple") 299 300 # Unsupported nodes. 301 302 elif isinstance(n, compiler.ast.GenExpr): 303 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 304 305 elif isinstance(n, compiler.ast.IfExp): 306 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 307 308 elif isinstance(n, compiler.ast.ListComp): 309 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 310 311 # All other nodes are processed depth-first. 312 313 else: 314 self.process_structure(n) 315 316 # By default, no expression details are returned. 317 318 return None 319 320 # Specific node handling. 321 322 def process_assignment_node(self, n, expr): 323 324 "Process the individual node 'n' to be assigned the contents of 'expr'." 325 326 # Names and attributes are assigned the entire expression. 327 328 if isinstance(n, compiler.ast.AssName): 329 if n.name == "self": 330 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 331 332 name_ref = expr and self.process_structure_node(expr) 333 334 # Name assignments populate either function namespaces or the 335 # general namespace hierarchy. 336 337 self.assign_general_local(n.name, name_ref) 338 339 # Record usage of the name. 340 341 self.record_name(n.name) 342 343 elif isinstance(n, compiler.ast.AssAttr): 344 if expr: 345 expr = self.process_structure_node(expr) 346 347 in_assignment = self.in_assignment 348 self.in_assignment = expr 349 self.process_attribute_access(n) 350 self.in_assignment = in_assignment 351 352 # Lists and tuples are matched against the expression and their 353 # items assigned to expression items. 354 355 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 356 self.process_assignment_node_items(n, expr) 357 358 # Slices and subscripts are permitted within assignment nodes. 359 360 elif isinstance(n, compiler.ast.Slice): 361 self.process_slice_node(n, expr) 362 363 elif isinstance(n, compiler.ast.Subscript): 364 self.process_subscript_node(n, expr) 365 366 def process_attribute_access(self, n): 367 368 "Process the given attribute access node 'n'." 369 370 # Obtain any completed chain and return the reference to it. 371 372 name_ref = self.process_attribute_chain(n) 373 374 if self.have_access_expression(n): 375 return name_ref 376 377 # Where the start of the chain of attributes has been reached, determine 378 # the complete access. 379 380 # Given a non-access node, this chain can be handled in its entirety, 381 # either being name-based and thus an access rooted on a name, or being 382 # based on some other node and thus an anonymous access of some kind. 383 384 path = self.get_namespace_path() 385 386 # Start with the the full attribute chain. 387 388 remaining = self.attrs 389 attrnames = ".".join(remaining) 390 391 # If the accessor cannot be identified, or where attributes 392 # remain in an attribute chain, record the anonymous accesses. 393 394 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 395 396 init_item(self.attr_accesses, path, set) 397 self.attr_accesses[path].add(attrnames) 398 399 self.record_access_details(None, attrnames, self.in_assignment, 400 self.in_invocation) 401 del self.attrs[0] 402 return 403 404 # Name-based accesses will handle the first attribute in a 405 # chain. 406 407 else: 408 attrname = remaining[0] 409 410 # Attribute assignments are used to identify instance attributes. 411 412 if isinstance(n, compiler.ast.AssAttr) and \ 413 self.in_class and self.in_function and n.expr.name == "self": 414 415 self.set_instance_attr(attrname) 416 417 # Record attribute usage using any name local to this namespace, 418 # if assigned in the namespace, or using an external name 419 # (presently just globals within classes). 420 421 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 422 tracker = self.trackers[-1] 423 424 immediate_access = len(self.attrs) == 1 425 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 426 427 # Record global-based chains for subsequent resolution. 428 429 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 430 not self.in_function 431 432 if is_global: 433 self.record_global_access_details(name, attrnames) 434 435 # Make sure the name is being tracked: global names will not 436 # already be initialised in a branch and must be added 437 # explicitly. 438 439 if not tracker.have_name(name): 440 tracker.assign_names([name]) 441 if self.in_function: 442 self.scope_globals[path].add(name) 443 444 # Record attribute usage in the tracker, and record the branch 445 # information for the access. 446 447 branches = tracker.use_attribute(name, attrname, self.in_invocation, assignment) 448 449 if not branches: 450 raise InspectError("Name %s is accessed using %s before an assignment." % ( 451 name, attrname), path, n) 452 453 self.record_branches_for_access(branches, name, attrnames) 454 access_number = self.record_access_details(name, attrnames, 455 self.in_assignment, self.in_invocation) 456 457 del self.attrs[0] 458 return AccessRef(name, attrnames, access_number) 459 460 def process_class_node(self, n): 461 462 "Process the given class node 'n'." 463 464 path = self.get_namespace_path() 465 466 # To avoid notions of class "versions" where the same definition 467 # might be parameterised with different state and be referenced 468 # elsewhere (as base classes, for example), classes in functions or 469 # conditions are forbidden. 470 471 if self.in_function or self.in_conditional: 472 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 473 path, n.name) 474 return 475 476 # Resolve base classes. 477 478 bases = [] 479 480 for base in n.bases: 481 base_class = self.get_class(base) 482 483 if not base_class: 484 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 485 path, n.name, base) 486 return 487 else: 488 bases.append(base_class) 489 490 # Record bases for the class and retain the class name. 491 # Note that the function class does not inherit from the object class. 492 493 class_name = self.get_object_path(n.name) 494 495 if not bases and class_name != "__builtins__.core.object" and \ 496 class_name != "__builtins__.core.function": 497 498 ref = self.get_object("__builtins__.object") 499 bases.append(ref) 500 501 self.importer.classes[class_name] = self.classes[class_name] = bases 502 self.importer.subclasses[class_name] = set() 503 self.scope_globals[class_name] = set() 504 505 # Set the definition before entering the namespace rather than 506 # afterwards because methods may reference it. In normal Python, 507 # a class is not accessible until the definition is complete, but 508 # methods can generally reference it since upon being called the 509 # class will already exist. 510 511 self.set_definition(n.name, "<class>") 512 513 in_class = self.in_class 514 self.in_class = class_name 515 self.set_instance_attr("__class__", Reference("<class>", class_name)) 516 self.enter_namespace(n.name) 517 518 # Do not provide the special instantiator attributes on the function 519 # class. Function instances provide these attributes. 520 521 if class_name != "__builtins__.core.function": 522 self.set_name("__fn__") # special instantiator attribute 523 self.set_name("__args__") # special instantiator attribute 524 525 self.assign_general_local("__name__", self.get_constant("str", class_name)) 526 self.process_structure_node(n.code) 527 self.exit_namespace() 528 self.in_class = in_class 529 530 def process_from_node(self, n): 531 532 "Process the given node 'n', importing from another module." 533 534 path = self.get_namespace_path() 535 536 module_name, names = self.get_module_name(n) 537 if module_name == self.name: 538 raise InspectError("Cannot import from the current module.", path, n) 539 540 self.queue_module(module_name) 541 542 # Attempt to obtain the referenced objects. 543 544 for name, alias in n.names: 545 if name == "*": 546 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 547 548 # Explicit names. 549 550 ref = self.import_name_from_module(name, module_name) 551 value = ResolvedNameRef(alias or name, ref) 552 self.set_general_local(alias or name, value) 553 554 def process_function_node(self, n, name): 555 556 """ 557 Process the given function or lambda node 'n' with the given 'name'. 558 """ 559 560 is_lambda = isinstance(n, compiler.ast.Lambda) 561 562 # Where a function is declared conditionally, use a separate name for 563 # the definition, and assign the definition to the stated name. 564 565 if (self.in_conditional or self.in_function) and not is_lambda: 566 original_name = name 567 name = self.get_lambda_name() 568 else: 569 original_name = None 570 571 # Initialise argument and local records. 572 573 function_name = self.get_object_path(name) 574 argnames = get_argnames(n.argnames) 575 is_method = self.in_class and not self.in_function 576 577 # Remove explicit "self" from method parameters. 578 579 if is_method and argnames and argnames[0] == "self": 580 del argnames[0] 581 582 # Copy and propagate the parameters. 583 584 self.importer.function_parameters[function_name] = \ 585 self.function_parameters[function_name] = argnames[:] 586 587 # Define all arguments/parameters in the local namespace. 588 589 locals = \ 590 self.importer.function_locals[function_name] = \ 591 self.function_locals[function_name] = {} 592 593 # Insert "self" into method locals. 594 595 if is_method: 596 argnames.insert(0, "self") 597 598 # Define "self" in terms of the class if in a method. 599 # This does not diminish the need for type-narrowing in the deducer. 600 601 if argnames: 602 if self.in_class and not self.in_function and argnames[0] == "self": 603 locals[argnames[0]] = Reference("<instance>", self.in_class) 604 else: 605 locals[argnames[0]] = Reference("<var>") 606 607 for argname in argnames[1:]: 608 locals[argname] = Reference("<var>") 609 610 globals = self.scope_globals[function_name] = set() 611 612 # Process the defaults. 613 614 defaults = self.importer.function_defaults[function_name] = \ 615 self.function_defaults[function_name] = [] 616 617 for argname, default in compiler.ast.get_defaults(n): 618 if default: 619 620 # Obtain any reference for the default. 621 622 name_ref = self.process_structure_node(default) 623 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 624 625 # Reset conditional tracking to focus on the function contents. 626 627 in_conditional = self.in_conditional 628 self.in_conditional = False 629 630 in_function = self.in_function 631 self.in_function = function_name 632 633 self.enter_namespace(name) 634 635 # Track attribute usage within the namespace. 636 637 path = self.get_namespace_path() 638 639 self.start_tracking(locals) 640 self.process_structure_node(n.code) 641 self.stop_tracking() 642 643 # Exit to the parent. 644 645 self.exit_namespace() 646 647 # Update flags. 648 649 self.in_function = in_function 650 self.in_conditional = in_conditional 651 652 # Define the function using the appropriate name. 653 654 self.set_definition(name, "<function>") 655 656 # Where a function is set conditionally, assign the name. 657 658 if original_name: 659 self.process_assignment_for_function(original_name, name) 660 661 def process_global_node(self, n): 662 663 """ 664 Process the given "global" node 'n'. 665 """ 666 667 path = self.get_namespace_path() 668 669 if path != self.name: 670 self.scope_globals[path].update(n.names) 671 672 def process_if_node(self, n): 673 674 """ 675 Process the given "if" node 'n'. 676 """ 677 678 tracker = self.trackers[-1] 679 tracker.new_branchpoint() 680 681 for test, body in n.tests: 682 self.process_structure_node(test) 683 684 tracker.new_branch() 685 686 in_conditional = self.in_conditional 687 self.in_conditional = True 688 self.process_structure_node(body) 689 self.in_conditional = in_conditional 690 691 tracker.shelve_branch() 692 693 # Maintain a branch for the else clause. 694 695 tracker.new_branch() 696 if n.else_: 697 self.process_structure_node(n.else_) 698 tracker.shelve_branch() 699 700 tracker.merge_branches() 701 702 def process_import_node(self, n): 703 704 "Process the given import node 'n'." 705 706 path = self.get_namespace_path() 707 708 # Load the mentioned module. 709 710 for name, alias in n.names: 711 if name == self.name: 712 raise InspectError("Cannot import the current module.", path, n) 713 714 self.set_module(alias or name.split(".")[-1], name) 715 self.queue_module(name, True) 716 717 def process_invocation_node(self, n): 718 719 "Process the given invocation node 'n'." 720 721 path = self.get_namespace_path() 722 723 self.allocate_arguments(path, n.args) 724 725 try: 726 # Communicate to the invocation target expression that it forms the 727 # target of an invocation, potentially affecting attribute accesses. 728 729 in_invocation = self.in_invocation 730 self.in_invocation = True 731 732 # Process the expression, obtaining any identified reference. 733 734 name_ref = self.process_structure_node(n.node) 735 self.in_invocation = in_invocation 736 737 # Process the arguments. 738 739 for arg in n.args: 740 self.process_structure_node(arg) 741 742 # Detect class invocations. 743 744 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 745 return InstanceRef(name_ref.reference().instance_of()) 746 747 elif isinstance(name_ref, NameRef): 748 return InvocationRef(name_ref) 749 750 return None 751 752 finally: 753 self.deallocate_arguments(path, n.args) 754 755 def process_lambda_node(self, n): 756 757 "Process the given lambda node 'n'." 758 759 name = self.get_lambda_name() 760 self.process_function_node(n, name) 761 762 origin = self.get_object_path(name) 763 return ResolvedNameRef(name, Reference("<function>", origin)) 764 765 def process_logical_node(self, n): 766 767 "Process the given operator node 'n'." 768 769 self.process_operator_chain(n.nodes, self.process_structure_node) 770 771 def process_name_node(self, n): 772 773 "Process the given name node 'n'." 774 775 path = self.get_namespace_path() 776 777 # Special names. 778 779 if n.name.startswith("$"): 780 value = self.get_special(n.name) 781 if value: 782 return value 783 784 # Special case for operator functions introduced through code 785 # transformations. 786 787 if n.name.startswith("$op"): 788 789 # Obtain the location of the actual function defined in the operator 790 # package. 791 792 op = n.name[len("$op"):] 793 794 # Attempt to get a reference. 795 796 ref = self.import_name_from_module(op, "operator") 797 798 # Record the imported name and provide the resolved name reference. 799 800 value = ResolvedNameRef(n.name, ref) 801 self.set_special(n.name, value) 802 return value 803 804 # Test for self usage, which is only allowed in methods. 805 806 if n.name == "self" and not (self.in_function and self.in_class): 807 raise InspectError("Use of self is only allowed in methods.", path, n) 808 809 # Record usage of the name. 810 811 self.record_name(n.name) 812 813 # Search for unknown names in non-function scopes immediately. 814 # External names in functions are resolved later. 815 816 ref = self.find_name(n.name) 817 if ref: 818 return ResolvedNameRef(n.name, ref) 819 820 # Explicitly-declared global names. 821 822 elif self.in_function and n.name in self.scope_globals[path]: 823 return NameRef(n.name) 824 825 # Examine other names. 826 827 else: 828 tracker = self.trackers[-1] 829 830 # Check local names. 831 832 branches = tracker.tracking_name(n.name) 833 834 # Local name. 835 836 if branches: 837 self.record_branches_for_access(branches, n.name, None) 838 access_number = self.record_access_details(n.name, None, False, False) 839 return LocalNameRef(n.name, access_number) 840 841 # Possible global or built-in name. 842 843 else: 844 return NameRef(n.name) 845 846 def process_operator_chain(self, nodes, fn): 847 848 """ 849 Process the given chain of 'nodes', applying 'fn' to each node or item. 850 Each node starts a new conditional region, effectively making a deeply- 851 nested collection of if-like statements. 852 """ 853 854 tracker = self.trackers[-1] 855 856 for item in nodes: 857 tracker.new_branchpoint() 858 tracker.new_branch() 859 fn(item) 860 861 for item in nodes[:-1]: 862 tracker.shelve_branch() 863 tracker.new_branch() 864 tracker.shelve_branch() 865 tracker.merge_branches() 866 867 tracker.shelve_branch() 868 tracker.merge_branches() 869 870 def process_try_node(self, n): 871 872 """ 873 Process the given "try...except" node 'n'. 874 """ 875 876 tracker = self.trackers[-1] 877 tracker.new_branchpoint() 878 879 self.process_structure_node(n.body) 880 881 for name, var, handler in n.handlers: 882 if name is not None: 883 self.process_structure_node(name) 884 885 # Any abandoned branches from the body can now be resumed in a new 886 # branch. 887 888 tracker.resume_abandoned_branches() 889 890 # Establish the local for the handler. 891 892 if var is not None: 893 self.process_structure_node(var) 894 if handler is not None: 895 self.process_structure_node(handler) 896 897 tracker.shelve_branch() 898 899 # The else clause maintains the usage from the body but without the 900 # abandoned branches since they would never lead to the else clause 901 # being executed. 902 903 if n.else_: 904 tracker.new_branch() 905 self.process_structure_node(n.else_) 906 tracker.shelve_branch() 907 908 # Without an else clause, a null branch propagates the successful 909 # outcome. 910 911 else: 912 tracker.new_branch() 913 tracker.shelve_branch() 914 915 tracker.merge_branches() 916 917 def process_try_finally_node(self, n): 918 919 """ 920 Process the given "try...finally" node 'n'. 921 """ 922 923 tracker = self.trackers[-1] 924 self.process_structure_node(n.body) 925 926 # Any abandoned branches from the body can now be resumed. 927 928 branches = tracker.resume_all_abandoned_branches() 929 self.process_structure_node(n.final) 930 931 # At the end of the finally clause, abandoned branches are discarded. 932 933 tracker.restore_active_branches(branches) 934 935 def process_while_node(self, n): 936 937 "Process the given while node 'n'." 938 939 tracker = self.trackers[-1] 940 tracker.new_branchpoint(loop_node=True) 941 942 # Evaluate any test or iterator outside the loop. 943 944 self.process_structure_node(n.test) 945 946 # Propagate attribute usage to branches. 947 948 tracker.new_branch(loop_node=True) 949 950 # Enter the loop. 951 952 in_conditional = self.in_conditional 953 self.in_conditional = True 954 self.process_structure_node(n.body) 955 self.in_conditional = in_conditional 956 957 # Continuing branches are resumed before any test. 958 959 tracker.resume_continuing_branches() 960 961 # Evaluate any continuation test within the body. 962 963 self.process_structure_node(n.test) 964 965 tracker.shelve_branch(loop_node=True) 966 967 # Support the non-looping condition. 968 969 tracker.new_branch() 970 tracker.shelve_branch() 971 972 tracker.merge_branches() 973 974 # Evaluate any else clause outside branches. 975 976 if n.else_: 977 self.process_structure_node(n.else_) 978 979 # Connect broken branches to the code after any loop. 980 981 tracker.resume_broken_branches() 982 983 # Branch tracking methods. 984 985 def start_tracking(self, names): 986 987 """ 988 Start tracking attribute usage for names in the current namespace, 989 immediately registering the given 'names'. 990 """ 991 992 path = self.get_namespace_path() 993 parent = self.trackers[-1] 994 tracker = BranchTracker() 995 self.trackers.append(tracker) 996 997 # Record the given names established as new branches. 998 999 tracker.assign_names(names) 1000 1001 def assign_name(self, name, name_ref): 1002 1003 "Assign to 'name' the given 'name_ref' in the current namespace." 1004 1005 name = self.get_name_for_tracking(name) 1006 self.trackers[-1].assign_names([name], [name_ref]) 1007 1008 def stop_tracking(self): 1009 1010 """ 1011 Stop tracking attribute usage, recording computed usage for the current 1012 namespace. 1013 """ 1014 1015 path = self.get_namespace_path() 1016 tracker = self.trackers.pop() 1017 self.record_assignments_for_access(tracker) 1018 1019 self.attr_usage[path] = tracker.get_all_usage() 1020 self.name_initialisers[path] = tracker.get_all_values() 1021 1022 def start_tracking_in_module(self): 1023 1024 "Start tracking attribute usage in the module." 1025 1026 tracker = BranchTracker() 1027 self.trackers.append(tracker) 1028 1029 def stop_tracking_in_module(self): 1030 1031 "Stop tracking attribute usage in the module." 1032 1033 tracker = self.trackers[0] 1034 self.record_assignments_for_access(tracker) 1035 self.attr_usage[self.name] = tracker.get_all_usage() 1036 self.name_initialisers[self.name] = tracker.get_all_values() 1037 1038 def record_assignments_for_access(self, tracker): 1039 1040 """ 1041 For the current path, use the given 'tracker' to record assignment 1042 version information for attribute accesses. 1043 """ 1044 1045 path = self.get_path_for_access() 1046 1047 if not self.attr_accessor_branches.has_key(path): 1048 return 1049 1050 init_item(self.attr_accessors, path, dict) 1051 attr_accessors = self.attr_accessors[path] 1052 1053 # Obtain the branches applying during each access. 1054 1055 for access, all_branches in self.attr_accessor_branches[path].items(): 1056 name, attrnames = access 1057 init_item(attr_accessors, access, list) 1058 1059 # Obtain the assignments applying to each branch. 1060 1061 for branches in all_branches: 1062 positions = tracker.get_assignment_positions_for_branches(name, branches) 1063 1064 # Detect missing name information. 1065 1066 if None in positions: 1067 globals = self.global_attr_accesses.get(path) 1068 accesses = globals and globals.get(name) 1069 if not accesses: 1070 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1071 self.get_namespace_path(), name) 1072 positions.remove(None) 1073 1074 attr_accessors[access].append(positions) 1075 1076 def record_branches_for_access(self, branches, name, attrnames): 1077 1078 """ 1079 Record the given 'branches' for an access involving the given 'name' and 1080 'attrnames'. 1081 """ 1082 1083 access = name, attrnames 1084 path = self.get_path_for_access() 1085 1086 init_item(self.attr_accessor_branches, path, dict) 1087 attr_accessor_branches = self.attr_accessor_branches[path] 1088 1089 init_item(attr_accessor_branches, access, list) 1090 attr_accessor_branches[access].append(branches) 1091 1092 def record_access_details(self, name, attrnames, assignment, invocation): 1093 1094 """ 1095 For the given 'name' and 'attrnames', record an access indicating 1096 whether 'assignment' is occurring. 1097 1098 These details correspond to accesses otherwise recorded by the attribute 1099 accessor and attribute access dictionaries. 1100 """ 1101 1102 access = name, attrnames 1103 path = self.get_path_for_access() 1104 1105 init_item(self.attr_access_modifiers, path, dict) 1106 init_item(self.attr_access_modifiers[path], access, list) 1107 1108 access_number = len(self.attr_access_modifiers[path][access]) 1109 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1110 return access_number 1111 1112 def record_global_access_details(self, name, attrnames): 1113 1114 """ 1115 Record details of a global access via the given 'name' involving the 1116 indicated 'attrnames'. 1117 """ 1118 1119 path = self.get_namespace_path() 1120 1121 init_item(self.global_attr_accesses, path, dict) 1122 init_item(self.global_attr_accesses[path], name, set) 1123 self.global_attr_accesses[path][name].add(attrnames) 1124 1125 # Namespace modification. 1126 1127 def record_name(self, name): 1128 1129 "Record the use of 'name' in a namespace." 1130 1131 path = self.get_namespace_path() 1132 init_item(self.names_used, path, set) 1133 self.names_used[path].add(name) 1134 1135 def set_module(self, name, module_name): 1136 1137 """ 1138 Set a module in the current namespace using the given 'name' associated 1139 with the corresponding 'module_name'. 1140 """ 1141 1142 if name: 1143 self.set_general_local(name, Reference("<module>", module_name)) 1144 1145 def set_definition(self, name, kind): 1146 1147 """ 1148 Set the definition having the given 'name' and 'kind'. 1149 1150 Definitions are set in the static namespace hierarchy, but they can also 1151 be recorded for function locals. 1152 """ 1153 1154 if self.is_global(name): 1155 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1156 self.get_namespace_path(), name) 1157 1158 path = self.get_object_path(name) 1159 self.set_object(path, kind) 1160 1161 ref = self.get_object(path) 1162 if ref.get_kind() == "<var>": 1163 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1164 self.get_namespace_path(), name) 1165 1166 if not self.is_global(name) and self.in_function: 1167 self.set_function_local(name, ref) 1168 1169 def set_function_local(self, name, ref=None): 1170 1171 "Set the local with the given 'name' and optional 'ref'." 1172 1173 locals = self.function_locals[self.get_namespace_path()] 1174 multiple = not ref or locals.has_key(name) and locals[name] != ref 1175 locals[name] = multiple and Reference("<var>") or ref 1176 1177 def assign_general_local(self, name, name_ref): 1178 1179 """ 1180 Set for 'name' the given 'name_ref', recording the name for attribute 1181 usage tracking. 1182 """ 1183 1184 self.set_general_local(name, name_ref) 1185 self.assign_name(name, name_ref) 1186 1187 def set_general_local(self, name, value=None): 1188 1189 """ 1190 Set the 'name' with optional 'value' in any kind of local namespace, 1191 where the 'value' should be a reference if specified. 1192 """ 1193 1194 init_value = self.get_initialising_value(value) 1195 1196 # Module global names. 1197 1198 if self.is_global(name): 1199 path = self.get_global_path(name) 1200 self.set_object(path, init_value) 1201 1202 # Function local names. 1203 1204 elif self.in_function: 1205 path = self.get_object_path(name) 1206 self.set_function_local(name, init_value) 1207 1208 # Other namespaces (classes). 1209 1210 else: 1211 path = self.get_object_path(name) 1212 self.set_name(name, init_value) 1213 1214 def set_name(self, name, ref=None): 1215 1216 "Attach the 'name' with optional 'ref' to the current namespace." 1217 1218 self.set_object(self.get_object_path(name), ref) 1219 1220 def set_instance_attr(self, name, ref=None): 1221 1222 """ 1223 Add an instance attribute of the given 'name' to the current class, 1224 using the optional 'ref'. 1225 """ 1226 1227 init_item(self.instance_attrs, self.in_class, set) 1228 self.instance_attrs[self.in_class].add(name) 1229 1230 if ref: 1231 init_item(self.instance_attr_constants, self.in_class, dict) 1232 self.instance_attr_constants[self.in_class][name] = ref 1233 1234 def get_initialising_value(self, value): 1235 1236 "Return a suitable initialiser reference for 'value'." 1237 1238 # Includes LiteralSequenceRef, ResolvedNameRef... 1239 1240 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1241 return value.reference() 1242 1243 # In general, invocations do not produce known results. However, the 1244 # name initialisers are resolved once a module has been inspected. 1245 1246 elif isinstance(value, InvocationRef): 1247 return value.reference() 1248 1249 else: 1250 return value 1251 1252 # Static, program-relative naming. 1253 1254 def find_name(self, name): 1255 1256 """ 1257 Return the qualified name for the given 'name' used in the current 1258 non-function namespace. 1259 """ 1260 1261 path = self.get_namespace_path() 1262 ref = None 1263 1264 if not self.in_function and name not in predefined_constants: 1265 if self.in_class: 1266 ref = self.get_object(self.get_object_path(name), False) 1267 if not ref: 1268 ref = self.get_global_or_builtin(name) 1269 1270 return ref 1271 1272 def get_class(self, node): 1273 1274 """ 1275 Use the given 'node' to obtain the identity of a class. Return a 1276 reference for the class. Unresolved dependencies are permitted and must 1277 be resolved later. 1278 """ 1279 1280 ref = self._get_class(node) 1281 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1282 1283 def _get_class(self, node): 1284 1285 """ 1286 Use the given 'node' to find a class definition. Return a reference to 1287 the class. 1288 """ 1289 1290 if isinstance(node, compiler.ast.Getattr): 1291 1292 # Obtain the identity of the access target. 1293 1294 ref = self._get_class(node.expr) 1295 1296 # Where the target is a class or module, obtain the identity of the 1297 # attribute. 1298 1299 if ref.has_kind(["<function>", "<var>"]): 1300 return None 1301 else: 1302 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1303 return self.get_object(attrname) 1304 1305 # Names can be module-level or built-in. 1306 1307 elif isinstance(node, compiler.ast.Name): 1308 1309 # Record usage of the name and attempt to identify it. 1310 1311 self.record_name(node.name) 1312 return self.find_name(node.name) 1313 else: 1314 return None 1315 1316 def get_constant(self, name, value): 1317 1318 "Return a constant reference for the given type 'name' and 'value'." 1319 1320 ref = self.get_builtin_class(name) 1321 return self.get_constant_reference(ref, value) 1322 1323 def get_literal_instance(self, n, name): 1324 1325 "For node 'n', return a reference to an instance of 'name'." 1326 1327 # Get a reference to the built-in class. 1328 1329 ref = self.get_builtin_class(name) 1330 1331 # Obtain the details of the literal itself. 1332 # An alias to the type is generated for sequences. 1333 1334 if name in ("dict", "list", "tuple"): 1335 self.set_special_literal(name, ref) 1336 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1337 1338 # Constant values are independently recorded. 1339 1340 else: 1341 return self.get_constant_reference(ref, n.value) 1342 1343 # Special names. 1344 1345 def get_special(self, name): 1346 1347 "Return any stored value for the given special 'name'." 1348 1349 return self.special.get(name) 1350 1351 def set_special(self, name, value): 1352 1353 """ 1354 Set a special 'name' that merely tracks the use of an implicit object 1355 'value'. 1356 """ 1357 1358 self.special[name] = value 1359 1360 def set_special_literal(self, name, ref): 1361 1362 """ 1363 Set a special name for the literal type 'name' having type 'ref'. Such 1364 special names provide a way of referring to literal object types. 1365 """ 1366 1367 literal_name = "$L%s" % name 1368 value = ResolvedNameRef(literal_name, ref) 1369 self.set_special(literal_name, value) 1370 1371 # Functions and invocations. 1372 1373 def set_invocation_usage(self): 1374 1375 """ 1376 Discard the current invocation storage figures, retaining the maximum 1377 values. 1378 """ 1379 1380 for path, (current, maximum) in self.function_targets.items(): 1381 self.importer.function_targets[path] = self.function_targets[path] = maximum 1382 1383 for path, (current, maximum) in self.function_arguments.items(): 1384 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1385 1386 def allocate_arguments(self, path, args): 1387 1388 """ 1389 Allocate temporary argument storage using current and maximum 1390 requirements for the given 'path' and 'args'. 1391 """ 1392 1393 init_item(self.function_targets, path, lambda: [0, 0]) 1394 t = self.function_targets[path] 1395 t[0] += 1 1396 t[1] = max(t[0], t[1]) 1397 1398 init_item(self.function_arguments, path, lambda: [0, 0]) 1399 t = self.function_arguments[path] 1400 t[0] += len(args) + 1 1401 t[1] = max(t[0], t[1]) 1402 1403 def deallocate_arguments(self, path, args): 1404 1405 "Deallocate temporary argument storage for the given 'path' and 'args'." 1406 1407 self.function_targets[path][0] -= 1 1408 self.function_arguments[path][0] -= len(args) + 1 1409 1410 # vim: tabstop=4 expandtab shiftwidth=4