1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 47 # Accesses to global attributes. 48 49 self.global_attr_accesses = {} 50 51 # Usage tracking. 52 53 self.trackers = [] 54 self.attr_accessor_branches = {} 55 56 def __repr__(self): 57 return "InspectedModule(%r, %r)" % (self.name, self.importer) 58 59 # Principal methods. 60 61 def parse(self, filename): 62 63 "Parse the file having the given 'filename'." 64 65 self.parse_file(filename) 66 67 # Inspect the module. 68 69 self.start_tracking_in_module() 70 71 # Detect and record imports and globals declared in the module. 72 73 self.assign_general_local("__name__", self.get_constant("str", self.name)) 74 self.assign_general_local("__file__", self.get_constant("str", filename)) 75 self.process_structure(self.astnode) 76 77 # Set the class of the module after the definition has occurred. 78 79 ref = self.get_builtin("object") 80 self.set_name("__class__", ref) 81 82 # Get module-level attribute usage details. 83 84 self.stop_tracking_in_module() 85 86 # Collect external name references. 87 88 self.collect_names() 89 90 def complete(self): 91 92 "Complete the module inspection." 93 94 # Resolve names not definitively mapped to objects. 95 96 self.resolve() 97 98 # Define the invocation requirements in each namespace. 99 100 self.set_invocation_usage() 101 102 # Propagate to the importer information needed in subsequent activities. 103 104 self.propagate() 105 106 # Accessory methods. 107 108 def collect_names(self): 109 110 "Collect the names used by each scope." 111 112 for path in self.names_used.keys(): 113 self.collect_names_for_path(path) 114 115 def collect_names_for_path(self, path): 116 117 """ 118 Collect the names used by the given 'path'. These are propagated to the 119 importer in advance of any dependency resolution. 120 """ 121 122 names = self.names_used.get(path) 123 if not names: 124 return 125 126 in_function = self.function_locals.has_key(path) 127 128 for name in names: 129 if in_function and name in self.function_locals[path]: 130 continue 131 132 key = "%s.%s" % (path, name) 133 134 # Find predefined constant names before anything else. 135 136 if name in predefined_constants: 137 ref = self.get_builtin(name) 138 self.set_name_reference(key, ref) 139 continue 140 141 # Find local definitions (within dynamic namespaces). 142 143 ref = self.get_resolved_object(key) 144 if ref: 145 self.set_name_reference(key, ref) 146 continue 147 148 # Find global. 149 150 ref = self.get_global(name) 151 if ref: 152 self.set_name_reference(key, ref) 153 continue 154 155 # Find presumed built-in definitions. 156 157 ref = self.get_builtin(name) 158 self.set_name_reference(key, ref) 159 160 def set_name_reference(self, path, ref): 161 162 "Map the given name 'path' to 'ref'." 163 164 self.importer.all_name_references[path] = self.name_references[path] = ref 165 166 # Module structure traversal. 167 168 def process_structure_node(self, n): 169 170 "Process the individual node 'n'." 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 self.process_assignment_node(n, None) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 self.process_attribute_access(n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Invocations. 282 283 elif isinstance(n, compiler.ast.CallFunc): 284 return self.process_invocation_node(n) 285 286 # Constant usage. 287 288 elif isinstance(n, compiler.ast.Const): 289 return self.get_literal_instance(n, n.value.__class__.__name__) 290 291 elif isinstance(n, compiler.ast.Dict): 292 return self.get_literal_instance(n, "dict") 293 294 elif isinstance(n, compiler.ast.List): 295 return self.get_literal_instance(n, "list") 296 297 elif isinstance(n, compiler.ast.Tuple): 298 return self.get_literal_instance(n, "tuple") 299 300 # Unsupported nodes. 301 302 elif isinstance(n, compiler.ast.GenExpr): 303 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 304 305 elif isinstance(n, compiler.ast.IfExp): 306 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 307 308 elif isinstance(n, compiler.ast.ListComp): 309 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 310 311 # All other nodes are processed depth-first. 312 313 else: 314 self.process_structure(n) 315 316 # By default, no expression details are returned. 317 318 return None 319 320 # Specific node handling. 321 322 def process_assignment_node(self, n, expr): 323 324 "Process the individual node 'n' to be assigned the contents of 'expr'." 325 326 # Names and attributes are assigned the entire expression. 327 328 if isinstance(n, compiler.ast.AssName): 329 if n.name == "self": 330 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 331 332 name_ref = expr and self.process_structure_node(expr) 333 334 # Name assignments populate either function namespaces or the 335 # general namespace hierarchy. 336 337 self.assign_general_local(n.name, name_ref) 338 339 # Record usage of the name. 340 341 self.record_name(n.name) 342 343 elif isinstance(n, compiler.ast.AssAttr): 344 if expr: 345 expr = self.process_structure_node(expr) 346 347 in_assignment = self.in_assignment 348 self.in_assignment = expr 349 self.process_attribute_access(n) 350 self.in_assignment = in_assignment 351 352 # Lists and tuples are matched against the expression and their 353 # items assigned to expression items. 354 355 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 356 self.process_assignment_node_items(n, expr) 357 358 # Slices and subscripts are permitted within assignment nodes. 359 360 elif isinstance(n, compiler.ast.Slice): 361 self.process_slice_node(n, expr) 362 363 elif isinstance(n, compiler.ast.Subscript): 364 self.process_subscript_node(n, expr) 365 366 def process_attribute_access(self, n): 367 368 "Process the given attribute access node 'n'." 369 370 # Obtain any completed chain and return the reference to it. 371 372 name_ref = self.process_attribute_chain(n) 373 374 if self.have_access_expression(n): 375 return name_ref 376 377 # Where the start of the chain of attributes has been reached, determine 378 # the complete access. 379 380 # Given a non-access node, this chain can be handled in its entirety, 381 # either being name-based and thus an access rooted on a name, or being 382 # based on some other node and thus an anonymous access of some kind. 383 384 path = self.get_namespace_path() 385 386 # Start with the the full attribute chain. 387 388 remaining = self.attrs 389 attrnames = ".".join(remaining) 390 391 # If the accessor cannot be identified, or where attributes 392 # remain in an attribute chain, record the anonymous accesses. 393 394 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 395 396 init_item(self.attr_accesses, path, set) 397 self.attr_accesses[path].add(attrnames) 398 399 self.record_access_details(None, attrnames, self.in_assignment, 400 self.in_invocation) 401 del self.attrs[0] 402 return 403 404 # Name-based accesses will handle the first attribute in a 405 # chain. 406 407 else: 408 attrname = remaining[0] 409 410 # Attribute assignments are used to identify instance attributes. 411 412 if isinstance(n, compiler.ast.AssAttr) and \ 413 self.in_class and self.in_function and n.expr.name == "self": 414 415 self.set_instance_attr(attrname) 416 417 # Record attribute usage using any name local to this namespace, 418 # if assigned in the namespace, or using an external name 419 # (presently just globals within classes). 420 421 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 422 tracker = self.trackers[-1] 423 424 immediate_access = len(self.attrs) == 1 425 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 426 427 # Record global-based chains for subsequent resolution. 428 429 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 430 not self.in_function 431 432 if is_global: 433 self.record_global_access_details(name, attrnames) 434 435 # Make sure the name is being tracked: global names will not 436 # already be initialised in a branch and must be added 437 # explicitly. 438 439 if not tracker.have_name(name): 440 tracker.assign_names([name]) 441 if self.in_function: 442 self.scope_globals[path].add(name) 443 444 # Record attribute usage in the tracker, and record the branch 445 # information for the access. 446 447 branches = tracker.use_attribute(name, attrname, self.in_invocation, assignment) 448 449 if not branches: 450 raise InspectError("Name %s is accessed using %s before an assignment." % ( 451 name, attrname), path, n) 452 453 self.record_branches_for_access(branches, name, attrnames) 454 access_number = self.record_access_details(name, attrnames, 455 self.in_assignment, self.in_invocation) 456 457 del self.attrs[0] 458 return AccessRef(name, attrnames, access_number) 459 460 def process_class_node(self, n): 461 462 "Process the given class node 'n'." 463 464 path = self.get_namespace_path() 465 466 # To avoid notions of class "versions" where the same definition 467 # might be parameterised with different state and be referenced 468 # elsewhere (as base classes, for example), classes in functions or 469 # conditions are forbidden. 470 471 if self.in_function or self.in_conditional: 472 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 473 path, n.name) 474 return 475 476 # Resolve base classes. 477 478 bases = [] 479 480 for base in n.bases: 481 base_class = self.get_class(base) 482 483 if not base_class: 484 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 485 path, n.name, base) 486 return 487 else: 488 bases.append(base_class) 489 490 # Record bases for the class and retain the class name. 491 # Note that the function class does not inherit from the object class. 492 493 class_name = self.get_object_path(n.name) 494 495 if not bases and class_name != "__builtins__.core.object" and \ 496 class_name != "__builtins__.core.function": 497 498 ref = self.get_object("__builtins__.object") 499 bases.append(ref) 500 501 self.importer.classes[class_name] = self.classes[class_name] = bases 502 self.importer.subclasses[class_name] = set() 503 self.scope_globals[class_name] = set() 504 505 # Set the definition before entering the namespace rather than 506 # afterwards because methods may reference it. In normal Python, 507 # a class is not accessible until the definition is complete, but 508 # methods can generally reference it since upon being called the 509 # class will already exist. 510 511 self.set_definition(n.name, "<class>") 512 513 in_class = self.in_class 514 self.in_class = class_name 515 self.set_instance_attr("__class__", Reference("<class>", class_name)) 516 self.enter_namespace(n.name) 517 518 # Do not provide the special instantiator attributes on the function 519 # class. Function instances provide these attributes. 520 521 if class_name != "__builtins__.core.function": 522 self.set_name("__fn__") # special instantiator attribute 523 self.set_name("__args__") # special instantiator attribute 524 525 self.assign_general_local("__name__", self.get_constant("str", class_name)) 526 self.process_structure_node(n.code) 527 self.exit_namespace() 528 self.in_class = in_class 529 530 def process_from_node(self, n): 531 532 "Process the given node 'n', importing from another module." 533 534 path = self.get_namespace_path() 535 536 module_name, names = self.get_module_name(n) 537 if module_name == self.name: 538 raise InspectError("Cannot import from the current module.", path, n) 539 540 self.queue_module(module_name) 541 542 # Attempt to obtain the referenced objects. 543 544 for name, alias in n.names: 545 if name == "*": 546 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 547 548 # Explicit names. 549 550 ref = self.import_name_from_module(name, module_name) 551 value = ResolvedNameRef(alias or name, ref) 552 self.set_general_local(alias or name, value) 553 554 def process_function_node(self, n, name): 555 556 """ 557 Process the given function or lambda node 'n' with the given 'name'. 558 """ 559 560 is_lambda = isinstance(n, compiler.ast.Lambda) 561 562 # Where a function is declared conditionally, use a separate name for 563 # the definition, and assign the definition to the stated name. 564 565 if (self.in_conditional or self.in_function) and not is_lambda: 566 original_name = name 567 name = self.get_lambda_name() 568 else: 569 original_name = None 570 571 # Initialise argument and local records. 572 573 function_name = self.get_object_path(name) 574 argnames = get_argnames(n.argnames) 575 is_method = self.in_class and not self.in_function 576 577 # Remove explicit "self" from method parameters. 578 579 if is_method and argnames and argnames[0] == "self": 580 del argnames[0] 581 582 # Copy and propagate the parameters. 583 584 self.importer.function_parameters[function_name] = \ 585 self.function_parameters[function_name] = argnames[:] 586 587 # Define all arguments/parameters in the local namespace. 588 589 locals = \ 590 self.importer.function_locals[function_name] = \ 591 self.function_locals[function_name] = {} 592 593 # Insert "self" into method locals. 594 595 if is_method: 596 argnames.insert(0, "self") 597 598 # Define "self" in terms of the class if in a method. 599 # This does not diminish the need for type-narrowing in the deducer. 600 601 if argnames: 602 if self.in_class and not self.in_function and argnames[0] == "self": 603 locals[argnames[0]] = Reference("<instance>", self.in_class) 604 else: 605 locals[argnames[0]] = Reference("<var>") 606 607 for argname in argnames[1:]: 608 locals[argname] = Reference("<var>") 609 610 globals = self.scope_globals[function_name] = set() 611 612 # Process the defaults. 613 614 defaults = self.importer.function_defaults[function_name] = \ 615 self.function_defaults[function_name] = [] 616 617 for argname, default in compiler.ast.get_defaults(n): 618 if default: 619 620 # Obtain any reference for the default. 621 622 name_ref = self.process_structure_node(default) 623 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 624 625 # Reset conditional tracking to focus on the function contents. 626 627 in_conditional = self.in_conditional 628 self.in_conditional = False 629 630 in_function = self.in_function 631 self.in_function = function_name 632 633 self.enter_namespace(name) 634 635 # Track attribute usage within the namespace. 636 637 path = self.get_namespace_path() 638 639 self.start_tracking(locals) 640 self.process_structure_node(n.code) 641 self.stop_tracking() 642 643 # Exit to the parent. 644 645 self.exit_namespace() 646 647 # Update flags. 648 649 self.in_function = in_function 650 self.in_conditional = in_conditional 651 652 # Define the function using the appropriate name. 653 654 self.set_definition(name, "<function>") 655 656 # Where a function is set conditionally, assign the name. 657 658 if original_name: 659 self.process_assignment_for_function(original_name, name) 660 661 def process_global_node(self, n): 662 663 """ 664 Process the given "global" node 'n'. 665 """ 666 667 path = self.get_namespace_path() 668 669 if path != self.name: 670 self.scope_globals[path].update(n.names) 671 672 def process_if_node(self, n): 673 674 """ 675 Process the given "if" node 'n'. 676 """ 677 678 tracker = self.trackers[-1] 679 tracker.new_branchpoint() 680 681 for test, body in n.tests: 682 self.process_structure_node(test) 683 684 tracker.new_branch() 685 686 in_conditional = self.in_conditional 687 self.in_conditional = True 688 self.process_structure_node(body) 689 self.in_conditional = in_conditional 690 691 tracker.shelve_branch() 692 693 # Maintain a branch for the else clause. 694 695 tracker.new_branch() 696 if n.else_: 697 self.process_structure_node(n.else_) 698 tracker.shelve_branch() 699 700 tracker.merge_branches() 701 702 def process_import_node(self, n): 703 704 "Process the given import node 'n'." 705 706 path = self.get_namespace_path() 707 708 # Load the mentioned module. 709 710 for name, alias in n.names: 711 if name == self.name: 712 raise InspectError("Cannot import the current module.", path, n) 713 714 self.set_module(alias or name.split(".")[-1], name) 715 self.queue_module(name, True) 716 717 def process_invocation_node(self, n): 718 719 "Process the given invocation node 'n'." 720 721 path = self.get_namespace_path() 722 723 self.allocate_arguments(path, n.args) 724 725 try: 726 # Communicate to the invocation target expression that it forms the 727 # target of an invocation, potentially affecting attribute accesses. 728 729 in_invocation = self.in_invocation 730 self.in_invocation = True 731 732 # Process the expression, obtaining any identified reference. 733 734 name_ref = self.process_structure_node(n.node) 735 self.in_invocation = in_invocation 736 737 # Process the arguments. 738 739 for arg in n.args: 740 self.process_structure_node(arg) 741 742 # Detect class invocations. 743 744 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 745 return InstanceRef(name_ref.reference().instance_of()) 746 747 elif isinstance(name_ref, NameRef): 748 return InvocationRef(name_ref) 749 750 return None 751 752 finally: 753 self.deallocate_arguments(path, n.args) 754 755 def process_lambda_node(self, n): 756 757 "Process the given lambda node 'n'." 758 759 name = self.get_lambda_name() 760 self.process_function_node(n, name) 761 762 origin = self.get_object_path(name) 763 return ResolvedNameRef(name, Reference("<function>", origin)) 764 765 def process_logical_node(self, n): 766 767 "Process the given operator node 'n'." 768 769 self.process_operator_chain(n.nodes, self.process_structure_node) 770 771 def process_name_node(self, n): 772 773 "Process the given name node 'n'." 774 775 path = self.get_namespace_path() 776 777 # Special names. 778 779 if n.name.startswith("$"): 780 value = self.get_special(n.name) 781 if value: 782 return value 783 784 # Special case for operator functions introduced through code 785 # transformations. 786 787 if n.name.startswith("$op"): 788 789 # Obtain the location of the actual function defined in the operator 790 # package. 791 792 op = n.name[len("$op"):] 793 794 # Attempt to get a reference. 795 796 ref = self.import_name_from_module(op, "operator") 797 self.add_deferred(ref) 798 799 # Record the imported name and provide the resolved name reference. 800 801 value = ResolvedNameRef(n.name, ref) 802 self.set_special(n.name, value) 803 return value 804 805 # Test for self usage, which is only allowed in methods. 806 807 if n.name == "self" and not (self.in_function and self.in_class): 808 raise InspectError("Use of self is only allowed in methods.", path, n) 809 810 # Record usage of the name. 811 812 self.record_name(n.name) 813 814 # Search for unknown names in non-function scopes immediately. 815 # External names in functions are resolved later. 816 817 ref = self.find_name(n.name) 818 if ref: 819 return ResolvedNameRef(n.name, ref) 820 821 # Explicitly-declared global names. 822 823 elif self.in_function and n.name in self.scope_globals[path]: 824 return NameRef(n.name) 825 826 # Examine other names. 827 828 else: 829 tracker = self.trackers[-1] 830 831 # Check local names. 832 833 branches = tracker.tracking_name(n.name) 834 835 # Local name. 836 837 if branches: 838 self.record_branches_for_access(branches, n.name, None) 839 access_number = self.record_access_details(n.name, None, False, False) 840 return LocalNameRef(n.name, access_number) 841 842 # Possible global or built-in name. 843 844 else: 845 return NameRef(n.name) 846 847 def process_operator_chain(self, nodes, fn): 848 849 """ 850 Process the given chain of 'nodes', applying 'fn' to each node or item. 851 Each node starts a new conditional region, effectively making a deeply- 852 nested collection of if-like statements. 853 """ 854 855 tracker = self.trackers[-1] 856 857 for item in nodes: 858 tracker.new_branchpoint() 859 tracker.new_branch() 860 fn(item) 861 862 for item in nodes[:-1]: 863 tracker.shelve_branch() 864 tracker.new_branch() 865 tracker.shelve_branch() 866 tracker.merge_branches() 867 868 tracker.shelve_branch() 869 tracker.merge_branches() 870 871 def process_try_node(self, n): 872 873 """ 874 Process the given "try...except" node 'n'. 875 """ 876 877 tracker = self.trackers[-1] 878 tracker.new_branchpoint() 879 880 self.process_structure_node(n.body) 881 882 for name, var, handler in n.handlers: 883 if name is not None: 884 self.process_structure_node(name) 885 886 # Any abandoned branches from the body can now be resumed in a new 887 # branch. 888 889 tracker.resume_abandoned_branches() 890 891 # Establish the local for the handler. 892 893 if var is not None: 894 self.process_structure_node(var) 895 if handler is not None: 896 self.process_structure_node(handler) 897 898 tracker.shelve_branch() 899 900 # The else clause maintains the usage from the body but without the 901 # abandoned branches since they would never lead to the else clause 902 # being executed. 903 904 if n.else_: 905 tracker.new_branch() 906 self.process_structure_node(n.else_) 907 tracker.shelve_branch() 908 909 # Without an else clause, a null branch propagates the successful 910 # outcome. 911 912 else: 913 tracker.new_branch() 914 tracker.shelve_branch() 915 916 tracker.merge_branches() 917 918 def process_try_finally_node(self, n): 919 920 """ 921 Process the given "try...finally" node 'n'. 922 """ 923 924 tracker = self.trackers[-1] 925 self.process_structure_node(n.body) 926 927 # Any abandoned branches from the body can now be resumed. 928 929 branches = tracker.resume_all_abandoned_branches() 930 self.process_structure_node(n.final) 931 932 # At the end of the finally clause, abandoned branches are discarded. 933 934 tracker.restore_active_branches(branches) 935 936 def process_while_node(self, n): 937 938 "Process the given while node 'n'." 939 940 tracker = self.trackers[-1] 941 tracker.new_branchpoint(loop_node=True) 942 943 # Evaluate any test or iterator outside the loop. 944 945 self.process_structure_node(n.test) 946 947 # Propagate attribute usage to branches. 948 949 tracker.new_branch(loop_node=True) 950 951 # Enter the loop. 952 953 in_conditional = self.in_conditional 954 self.in_conditional = True 955 self.process_structure_node(n.body) 956 self.in_conditional = in_conditional 957 958 # Continuing branches are resumed before any test. 959 960 tracker.resume_continuing_branches() 961 962 # Evaluate any continuation test within the body. 963 964 self.process_structure_node(n.test) 965 966 tracker.shelve_branch(loop_node=True) 967 968 # Support the non-looping condition. 969 970 tracker.new_branch() 971 tracker.shelve_branch() 972 973 tracker.merge_branches() 974 975 # Evaluate any else clause outside branches. 976 977 if n.else_: 978 self.process_structure_node(n.else_) 979 980 # Connect broken branches to the code after any loop. 981 982 tracker.resume_broken_branches() 983 984 # Branch tracking methods. 985 986 def start_tracking(self, names): 987 988 """ 989 Start tracking attribute usage for names in the current namespace, 990 immediately registering the given 'names'. 991 """ 992 993 path = self.get_namespace_path() 994 parent = self.trackers[-1] 995 tracker = BranchTracker() 996 self.trackers.append(tracker) 997 998 # Record the given names established as new branches. 999 1000 tracker.assign_names(names) 1001 1002 def assign_name(self, name, name_ref): 1003 1004 "Assign to 'name' the given 'name_ref' in the current namespace." 1005 1006 name = self.get_name_for_tracking(name) 1007 self.trackers[-1].assign_names([name], [name_ref]) 1008 1009 def stop_tracking(self): 1010 1011 """ 1012 Stop tracking attribute usage, recording computed usage for the current 1013 namespace. 1014 """ 1015 1016 path = self.get_namespace_path() 1017 tracker = self.trackers.pop() 1018 self.record_assignments_for_access(tracker) 1019 1020 self.attr_usage[path] = tracker.get_all_usage() 1021 self.name_initialisers[path] = tracker.get_all_values() 1022 1023 def start_tracking_in_module(self): 1024 1025 "Start tracking attribute usage in the module." 1026 1027 tracker = BranchTracker() 1028 self.trackers.append(tracker) 1029 1030 def stop_tracking_in_module(self): 1031 1032 "Stop tracking attribute usage in the module." 1033 1034 tracker = self.trackers[0] 1035 self.record_assignments_for_access(tracker) 1036 self.attr_usage[self.name] = tracker.get_all_usage() 1037 self.name_initialisers[self.name] = tracker.get_all_values() 1038 1039 def record_assignments_for_access(self, tracker): 1040 1041 """ 1042 For the current path, use the given 'tracker' to record assignment 1043 version information for attribute accesses. 1044 """ 1045 1046 path = self.get_path_for_access() 1047 1048 if not self.attr_accessor_branches.has_key(path): 1049 return 1050 1051 init_item(self.attr_accessors, path, dict) 1052 attr_accessors = self.attr_accessors[path] 1053 1054 # Obtain the branches applying during each access. 1055 1056 for access, all_branches in self.attr_accessor_branches[path].items(): 1057 name, attrnames = access 1058 init_item(attr_accessors, access, list) 1059 1060 # Obtain the assignments applying to each branch. 1061 1062 for branches in all_branches: 1063 positions = tracker.get_assignment_positions_for_branches(name, branches) 1064 1065 # Detect missing name information. 1066 1067 if None in positions: 1068 globals = self.global_attr_accesses.get(path) 1069 accesses = globals and globals.get(name) 1070 if not accesses: 1071 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1072 self.get_namespace_path(), name) 1073 positions.remove(None) 1074 1075 attr_accessors[access].append(positions) 1076 1077 def record_branches_for_access(self, branches, name, attrnames): 1078 1079 """ 1080 Record the given 'branches' for an access involving the given 'name' and 1081 'attrnames'. 1082 """ 1083 1084 access = name, attrnames 1085 path = self.get_path_for_access() 1086 1087 init_item(self.attr_accessor_branches, path, dict) 1088 attr_accessor_branches = self.attr_accessor_branches[path] 1089 1090 init_item(attr_accessor_branches, access, list) 1091 attr_accessor_branches[access].append(branches) 1092 1093 def record_access_details(self, name, attrnames, assignment, invocation): 1094 1095 """ 1096 For the given 'name' and 'attrnames', record an access indicating 1097 whether 'assignment' is occurring. 1098 1099 These details correspond to accesses otherwise recorded by the attribute 1100 accessor and attribute access dictionaries. 1101 """ 1102 1103 access = name, attrnames 1104 path = self.get_path_for_access() 1105 1106 init_item(self.attr_access_modifiers, path, dict) 1107 init_item(self.attr_access_modifiers[path], access, list) 1108 1109 access_number = len(self.attr_access_modifiers[path][access]) 1110 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1111 return access_number 1112 1113 def record_global_access_details(self, name, attrnames): 1114 1115 """ 1116 Record details of a global access via the given 'name' involving the 1117 indicated 'attrnames'. 1118 """ 1119 1120 path = self.get_namespace_path() 1121 1122 init_item(self.global_attr_accesses, path, dict) 1123 init_item(self.global_attr_accesses[path], name, set) 1124 self.global_attr_accesses[path][name].add(attrnames) 1125 1126 # Namespace modification. 1127 1128 def record_name(self, name): 1129 1130 "Record the use of 'name' in a namespace." 1131 1132 path = self.get_namespace_path() 1133 init_item(self.names_used, path, set) 1134 self.names_used[path].add(name) 1135 1136 def set_module(self, name, module_name): 1137 1138 """ 1139 Set a module in the current namespace using the given 'name' associated 1140 with the corresponding 'module_name'. 1141 """ 1142 1143 if name: 1144 self.set_general_local(name, Reference("<module>", module_name)) 1145 1146 def set_definition(self, name, kind): 1147 1148 """ 1149 Set the definition having the given 'name' and 'kind'. 1150 1151 Definitions are set in the static namespace hierarchy, but they can also 1152 be recorded for function locals. 1153 """ 1154 1155 if self.is_global(name): 1156 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1157 self.get_namespace_path(), name) 1158 1159 path = self.get_object_path(name) 1160 self.set_object(path, kind) 1161 1162 ref = self.get_object(path) 1163 if ref.get_kind() == "<var>": 1164 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1165 self.get_namespace_path(), name) 1166 1167 if not self.is_global(name) and self.in_function: 1168 self.set_function_local(name, ref) 1169 1170 def set_function_local(self, name, ref=None): 1171 1172 "Set the local with the given 'name' and optional 'ref'." 1173 1174 locals = self.function_locals[self.get_namespace_path()] 1175 multiple = not ref or locals.has_key(name) and locals[name] != ref 1176 locals[name] = multiple and Reference("<var>") or ref 1177 1178 def assign_general_local(self, name, name_ref): 1179 1180 """ 1181 Set for 'name' the given 'name_ref', recording the name for attribute 1182 usage tracking. 1183 """ 1184 1185 self.set_general_local(name, name_ref) 1186 self.assign_name(name, name_ref) 1187 1188 def set_general_local(self, name, value=None): 1189 1190 """ 1191 Set the 'name' with optional 'value' in any kind of local namespace, 1192 where the 'value' should be a reference if specified. 1193 """ 1194 1195 init_value = self.get_initialising_value(value) 1196 1197 # Module global names. 1198 1199 if self.is_global(name): 1200 path = self.get_global_path(name) 1201 self.set_object(path, init_value) 1202 1203 # Function local names. 1204 1205 elif self.in_function: 1206 path = self.get_object_path(name) 1207 self.set_function_local(name, init_value) 1208 1209 # Other namespaces (classes). 1210 1211 else: 1212 path = self.get_object_path(name) 1213 self.set_name(name, init_value) 1214 1215 def set_name(self, name, ref=None): 1216 1217 "Attach the 'name' with optional 'ref' to the current namespace." 1218 1219 self.set_object(self.get_object_path(name), ref) 1220 1221 def set_instance_attr(self, name, ref=None): 1222 1223 """ 1224 Add an instance attribute of the given 'name' to the current class, 1225 using the optional 'ref'. 1226 """ 1227 1228 init_item(self.instance_attrs, self.in_class, set) 1229 self.instance_attrs[self.in_class].add(name) 1230 1231 if ref: 1232 init_item(self.instance_attr_constants, self.in_class, dict) 1233 self.instance_attr_constants[self.in_class][name] = ref 1234 1235 def get_initialising_value(self, value): 1236 1237 "Return a suitable initialiser reference for 'value'." 1238 1239 # Includes LiteralSequenceRef, ResolvedNameRef... 1240 1241 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1242 return value.reference() 1243 1244 # In general, invocations do not produce known results. However, the 1245 # name initialisers are resolved once a module has been inspected. 1246 1247 elif isinstance(value, InvocationRef): 1248 return value.reference() 1249 1250 else: 1251 return value 1252 1253 # Static, program-relative naming. 1254 1255 def find_name(self, name): 1256 1257 """ 1258 Return the qualified name for the given 'name' used in the current 1259 non-function namespace. 1260 """ 1261 1262 path = self.get_namespace_path() 1263 ref = None 1264 1265 if not self.in_function and name not in predefined_constants: 1266 if self.in_class: 1267 ref = self.get_object(self.get_object_path(name)) 1268 if not ref: 1269 ref = self.get_global_or_builtin(name) 1270 1271 return ref 1272 1273 def get_class(self, node): 1274 1275 """ 1276 Use the given 'node' to obtain the identity of a class. Return a 1277 reference for the class. Unresolved dependencies are permitted and must 1278 be resolved later. 1279 """ 1280 1281 ref = self._get_class(node) 1282 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1283 1284 def _get_class(self, node): 1285 1286 """ 1287 Use the given 'node' to find a class definition. Return a reference to 1288 the class. 1289 """ 1290 1291 if isinstance(node, compiler.ast.Getattr): 1292 1293 # Obtain the identity of the access target. 1294 1295 ref = self._get_class(node.expr) 1296 1297 # Where the target is a class or module, obtain the identity of the 1298 # attribute. 1299 1300 if ref.has_kind(["<function>", "<var>"]): 1301 return None 1302 else: 1303 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1304 return self.get_object(attrname) 1305 1306 # Names can be module-level or built-in. 1307 1308 elif isinstance(node, compiler.ast.Name): 1309 1310 # Record usage of the name and attempt to identify it. 1311 1312 self.record_name(node.name) 1313 return self.find_name(node.name) 1314 else: 1315 return None 1316 1317 def get_constant(self, name, value): 1318 1319 "Return a constant reference for the given type 'name' and 'value'." 1320 1321 ref = self.get_builtin_class(name) 1322 return self.get_constant_reference(ref, value) 1323 1324 def get_literal_instance(self, n, name): 1325 1326 "For node 'n', return a reference to an instance of 'name'." 1327 1328 # Get a reference to the built-in class. 1329 1330 ref = self.get_builtin_class(name) 1331 1332 # Obtain the details of the literal itself. 1333 # An alias to the type is generated for sequences. 1334 1335 if name in ("dict", "list", "tuple"): 1336 self.set_special_literal(name, ref) 1337 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1338 1339 # Constant values are independently recorded. 1340 1341 else: 1342 return self.get_constant_reference(ref, n.value) 1343 1344 # Special names. 1345 1346 def get_special(self, name): 1347 1348 "Return any stored value for the given special 'name'." 1349 1350 return self.special.get(name) 1351 1352 def set_special(self, name, value): 1353 1354 """ 1355 Set a special 'name' that merely tracks the use of an implicit object 1356 'value'. 1357 """ 1358 1359 self.special[name] = value 1360 1361 def set_special_literal(self, name, ref): 1362 1363 """ 1364 Set a special name for the literal type 'name' having type 'ref'. Such 1365 special names provide a way of referring to literal object types. 1366 """ 1367 1368 literal_name = "$L%s" % name 1369 value = ResolvedNameRef(literal_name, ref) 1370 self.set_special(literal_name, value) 1371 1372 # Functions and invocations. 1373 1374 def set_invocation_usage(self): 1375 1376 """ 1377 Discard the current invocation storage figures, retaining the maximum 1378 values. 1379 """ 1380 1381 for path, (current, maximum) in self.function_targets.items(): 1382 self.importer.function_targets[path] = self.function_targets[path] = maximum 1383 1384 for path, (current, maximum) in self.function_arguments.items(): 1385 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1386 1387 def allocate_arguments(self, path, args): 1388 1389 """ 1390 Allocate temporary argument storage using current and maximum 1391 requirements for the given 'path' and 'args'. 1392 """ 1393 1394 init_item(self.function_targets, path, lambda: [0, 0]) 1395 t = self.function_targets[path] 1396 t[0] += 1 1397 t[1] = max(t[0], t[1]) 1398 1399 init_item(self.function_arguments, path, lambda: [0, 0]) 1400 t = self.function_arguments[path] 1401 t[0] += len(args) + 1 1402 t[1] = max(t[0], t[1]) 1403 1404 def deallocate_arguments(self, path, args): 1405 1406 "Deallocate temporary argument storage for the given 'path' and 'args'." 1407 1408 self.function_targets[path][0] -= 1 1409 self.function_arguments[path][0] -= len(args) + 1 1410 1411 # vim: tabstop=4 expandtab shiftwidth=4