1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 self.in_invocation = False 47 self.global_attr_accesses = {} 48 49 # Usage tracking. 50 51 self.trackers = [] 52 self.attr_accessor_branches = {} 53 54 def __repr__(self): 55 return "InspectedModule(%r, %r)" % (self.name, self.importer) 56 57 # Principal methods. 58 59 def parse(self, filename): 60 61 "Parse the file having the given 'filename'." 62 63 self.parse_file(filename) 64 65 # Inspect the module. 66 67 self.start_tracking_in_module() 68 69 # Detect and record imports and globals declared in the module. 70 71 self.assign_general_local("__name__", self.get_constant("str", self.name)) 72 self.assign_general_local("__file__", self.get_constant("str", filename)) 73 self.process_structure(self.astnode) 74 75 # Set the class of the module after the definition has occurred. 76 77 ref = self.get_builtin("object") 78 self.set_name("__class__", ref) 79 80 # Get module-level attribute usage details. 81 82 self.stop_tracking_in_module() 83 84 # Collect external name references. 85 86 self.collect_names() 87 88 def complete(self): 89 90 "Complete the module inspection." 91 92 # Resolve names not definitively mapped to objects. 93 94 self.resolve() 95 96 # Define the invocation requirements in each namespace. 97 98 self.set_invocation_usage() 99 100 # Propagate to the importer information needed in subsequent activities. 101 102 self.propagate() 103 104 # Accessory methods. 105 106 def collect_names(self): 107 108 "Collect the names used by each scope." 109 110 for path in self.names_used.keys(): 111 self.collect_names_for_path(path) 112 113 def collect_names_for_path(self, path): 114 115 """ 116 Collect the names used by the given 'path'. These are propagated to the 117 importer in advance of any dependency resolution. 118 """ 119 120 names = self.names_used.get(path) 121 if not names: 122 return 123 124 in_function = self.function_locals.has_key(path) 125 126 for name in names: 127 if name in predefined_constants or in_function and name in self.function_locals[path]: 128 continue 129 130 # Find local definitions (within dynamic namespaces). 131 132 key = "%s.%s" % (path, name) 133 ref = self.get_resolved_object(key) 134 if ref: 135 self.set_name_reference(key, ref) 136 continue 137 138 # Find global or known built-in definitions. 139 140 ref = self.get_resolved_global_or_builtin(name) 141 if ref: 142 self.set_name_reference(key, ref) 143 continue 144 145 # Find presumed built-in definitions. 146 147 ref = self.get_builtin(name) 148 self.set_name_reference(key, ref) 149 150 def set_name_reference(self, path, ref): 151 152 "Map the given name 'path' to 'ref'." 153 154 self.importer.all_name_references[path] = self.name_references[path] = ref 155 156 def get_resolved_global_or_builtin(self, name): 157 158 "Return the resolved global or built-in object with the given 'name'." 159 160 # In some circumstances, the name is neither global nor recognised by 161 # the importer. It is then assumed to be a general built-in. 162 163 return self.get_global(name) or \ 164 self.importer.get_object("__builtins__.%s" % name) 165 166 # Module structure traversal. 167 168 def process_structure_node(self, n): 169 170 "Process the individual node 'n'." 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 self.process_assignment_node(n, None) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 self.process_attribute_access(n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Invocations. 282 283 elif isinstance(n, compiler.ast.CallFunc): 284 return self.process_invocation_node(n) 285 286 # Constant usage. 287 288 elif isinstance(n, compiler.ast.Const): 289 return self.get_literal_instance(n, n.value.__class__.__name__) 290 291 elif isinstance(n, compiler.ast.Dict): 292 return self.get_literal_instance(n, "dict") 293 294 elif isinstance(n, compiler.ast.List): 295 return self.get_literal_instance(n, "list") 296 297 elif isinstance(n, compiler.ast.Tuple): 298 return self.get_literal_instance(n, "tuple") 299 300 # Unsupported nodes. 301 302 elif isinstance(n, compiler.ast.GenExpr): 303 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 304 305 elif isinstance(n, compiler.ast.IfExp): 306 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 307 308 elif isinstance(n, compiler.ast.ListComp): 309 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 310 311 # All other nodes are processed depth-first. 312 313 else: 314 self.process_structure(n) 315 316 # By default, no expression details are returned. 317 318 return None 319 320 # Specific node handling. 321 322 def process_assignment_node(self, n, expr): 323 324 "Process the individual node 'n' to be assigned the contents of 'expr'." 325 326 # Names and attributes are assigned the entire expression. 327 328 if isinstance(n, compiler.ast.AssName): 329 if n.name == "self": 330 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 331 332 name_ref = expr and self.process_structure_node(expr) 333 334 # Name assignments populate either function namespaces or the 335 # general namespace hierarchy. 336 337 self.assign_general_local(n.name, name_ref) 338 339 # Record usage of the name. 340 341 self.record_name(n.name) 342 343 elif isinstance(n, compiler.ast.AssAttr): 344 if expr: self.process_structure_node(expr) 345 self.process_attribute_access(n) 346 347 # Lists and tuples are matched against the expression and their 348 # items assigned to expression items. 349 350 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 351 self.process_assignment_node_items(n, expr) 352 353 # Slices and subscripts are permitted within assignment nodes. 354 355 elif isinstance(n, compiler.ast.Slice): 356 self.process_slice_node(n, expr) 357 358 elif isinstance(n, compiler.ast.Subscript): 359 self.process_subscript_node(n, expr) 360 361 def process_attribute_access(self, n): 362 363 "Process the given attribute access node 'n'." 364 365 # Obtain any completed chain and return the reference to it. 366 367 in_invocation = self.in_invocation 368 self.in_invocation = False 369 name_ref = self.process_attribute_chain(n) 370 self.in_invocation = in_invocation 371 372 if self.have_access_expression(n): 373 return name_ref 374 375 # Where the start of the chain of attributes has been reached, determine 376 # the complete access. 377 378 # Given a non-access node, this chain can be handled in its entirety, 379 # either being name-based and thus an access rooted on a name, or being 380 # based on some other node and thus an anonymous access of some kind. 381 382 path = self.get_namespace_path() 383 384 # Start with the the full attribute chain. 385 386 remaining = self.attrs 387 attrnames = ".".join(remaining) 388 389 # If the accessor cannot be identified, or where attributes 390 # remain in an attribute chain, record the anonymous accesses. 391 392 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 393 394 assignment = isinstance(n, compiler.ast.AssAttr) 395 396 init_item(self.attr_accesses, path, set) 397 self.attr_accesses[path].add(attrnames) 398 399 self.record_access_details(None, attrnames, assignment) 400 del self.attrs[0] 401 return 402 403 # Name-based accesses will handle the first attribute in a 404 # chain. 405 406 else: 407 attrname = remaining[0] 408 409 # Attribute assignments are used to identify instance attributes. 410 411 if isinstance(n, compiler.ast.AssAttr) and \ 412 self.in_class and self.in_function and n.expr.name == "self": 413 414 self.set_instance_attr(attrname) 415 416 # Record attribute usage using any name local to this namespace, 417 # if assigned in the namespace, or using an external name 418 # (presently just globals within classes). 419 420 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 421 tracker = self.trackers[-1] 422 423 immediate_access = len(self.attrs) == 1 424 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 425 426 del self.attrs[0] 427 428 # Record global-based chains for subsequent resolution. 429 430 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 431 not self.in_function 432 433 if is_global: 434 self.record_global_access_details(name, attrnames) 435 436 # Make sure the name is being tracked: global names will not 437 # already be initialised in a branch and must be added 438 # explicitly. 439 440 if not tracker.have_name(name): 441 tracker.assign_names([name]) 442 if self.in_function: 443 self.scope_globals[path].add(name) 444 445 # Record attribute usage in the tracker, and record the branch 446 # information for the access. 447 448 branches = tracker.use_attribute(name, attrname, self.in_invocation) 449 450 if not branches: 451 raise InspectError("Name %s is accessed using %s before an assignment." % ( 452 name, attrname), path, n) 453 454 self.record_branches_for_access(branches, name, attrnames) 455 access_number = self.record_access_details(name, attrnames, assignment) 456 return AccessRef(name, attrnames, access_number) 457 458 def process_class_node(self, n): 459 460 "Process the given class node 'n'." 461 462 path = self.get_namespace_path() 463 464 # To avoid notions of class "versions" where the same definition 465 # might be parameterised with different state and be referenced 466 # elsewhere (as base classes, for example), classes in functions or 467 # conditions are forbidden. 468 469 if self.in_function or self.in_conditional: 470 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 471 path, n.name) 472 return 473 474 # Resolve base classes. 475 476 bases = [] 477 478 for base in n.bases: 479 base_class = self.get_class(base) 480 481 if not base_class: 482 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 483 path, n.name, base) 484 return 485 else: 486 bases.append(base_class) 487 488 # Record bases for the class and retain the class name. 489 490 class_name = self.get_object_path(n.name) 491 492 if not bases and class_name != "__builtins__.core.object": 493 ref = self.get_object("__builtins__.object") 494 bases.append(ref) 495 496 self.importer.classes[class_name] = self.classes[class_name] = bases 497 self.importer.subclasses[class_name] = set() 498 self.scope_globals[class_name] = set() 499 500 # Set the definition before entering the namespace rather than 501 # afterwards because methods may reference it. In normal Python, 502 # a class is not accessible until the definition is complete, but 503 # methods can generally reference it since upon being called the 504 # class will already exist. 505 506 self.set_definition(n.name, "<class>") 507 508 in_class = self.in_class 509 self.in_class = class_name 510 self.set_instance_attr("__class__", Reference("<class>", class_name)) 511 self.enter_namespace(n.name) 512 self.set_name("__fn__") # special instantiator attribute 513 self.set_name("__args__") # special instantiator attribute 514 self.assign_general_local("__name__", self.get_constant("str", class_name)) 515 self.process_structure_node(n.code) 516 self.exit_namespace() 517 self.in_class = in_class 518 519 def process_from_node(self, n): 520 521 "Process the given node 'n', importing from another module." 522 523 path = self.get_namespace_path() 524 525 module_name, names = self.get_module_name(n) 526 if module_name == self.name: 527 raise InspectError("Cannot import from the current module.", path, n) 528 529 self.queue_module(module_name) 530 531 # Attempt to obtain the referenced objects. 532 533 for name, alias in n.names: 534 if name == "*": 535 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 536 537 # Explicit names. 538 539 ref = self.import_name_from_module(name, module_name) 540 value = ResolvedNameRef(alias or name, ref) 541 self.set_general_local(alias or name, value) 542 543 def process_function_node(self, n, name): 544 545 """ 546 Process the given function or lambda node 'n' with the given 'name'. 547 """ 548 549 is_lambda = isinstance(n, compiler.ast.Lambda) 550 551 # Where a function is declared conditionally, use a separate name for 552 # the definition, and assign the definition to the stated name. 553 554 if (self.in_conditional or self.in_function) and not is_lambda: 555 original_name = name 556 name = self.get_lambda_name() 557 else: 558 original_name = None 559 560 # Initialise argument and local records. 561 562 function_name = self.get_object_path(name) 563 argnames = get_argnames(n.argnames) 564 is_method = self.in_class and not self.in_function 565 566 # Remove explicit "self" from method parameters. 567 568 if is_method and argnames and argnames[0] == "self": 569 del argnames[0] 570 571 # Copy and propagate the parameters. 572 573 self.importer.function_parameters[function_name] = \ 574 self.function_parameters[function_name] = argnames[:] 575 576 # Define all arguments/parameters in the local namespace. 577 578 locals = self.function_locals[function_name] = {} 579 580 # Insert "self" into method locals. 581 582 if is_method: 583 argnames.insert(0, "self") 584 585 # Define "self" in terms of the class if in a method. 586 # This does not diminish the need for type-narrowing in the deducer. 587 588 if argnames: 589 if self.in_class and not self.in_function and argnames[0] == "self": 590 locals[argnames[0]] = Reference("<instance>", self.in_class) 591 else: 592 locals[argnames[0]] = Reference("<var>") 593 594 for argname in argnames[1:]: 595 locals[argname] = Reference("<var>") 596 597 globals = self.scope_globals[function_name] = set() 598 599 # Process the defaults. 600 601 defaults = self.importer.function_defaults[function_name] = \ 602 self.function_defaults[function_name] = [] 603 604 for argname, default in compiler.ast.get_defaults(n): 605 if default: 606 607 # Obtain any reference for the default. 608 609 name_ref = self.process_structure_node(default) 610 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 611 612 # Reset conditional tracking to focus on the function contents. 613 614 in_conditional = self.in_conditional 615 self.in_conditional = False 616 617 in_function = self.in_function 618 self.in_function = function_name 619 620 self.enter_namespace(name) 621 622 # Track attribute usage within the namespace. 623 624 path = self.get_namespace_path() 625 626 self.start_tracking(locals) 627 self.process_structure_node(n.code) 628 self.stop_tracking() 629 630 # Exit to the parent. 631 632 self.exit_namespace() 633 634 # Update flags. 635 636 self.in_function = in_function 637 self.in_conditional = in_conditional 638 639 # Define the function using the appropriate name. 640 641 self.set_definition(name, "<function>") 642 643 # Where a function is set conditionally, assign the name. 644 645 if original_name: 646 self.process_assignment_for_function(original_name, name) 647 648 def process_global_node(self, n): 649 650 """ 651 Process the given "global" node 'n'. 652 """ 653 654 path = self.get_namespace_path() 655 656 if path != self.name: 657 self.scope_globals[path].update(n.names) 658 659 def process_if_node(self, n): 660 661 """ 662 Process the given "if" node 'n'. 663 """ 664 665 tracker = self.trackers[-1] 666 tracker.new_branchpoint() 667 668 for test, body in n.tests: 669 self.process_structure_node(test) 670 671 tracker.new_branch() 672 673 in_conditional = self.in_conditional 674 self.in_conditional = True 675 self.process_structure_node(body) 676 self.in_conditional = in_conditional 677 678 tracker.shelve_branch() 679 680 # Maintain a branch for the else clause. 681 682 tracker.new_branch() 683 if n.else_: 684 self.process_structure_node(n.else_) 685 tracker.shelve_branch() 686 687 tracker.merge_branches() 688 689 def process_import_node(self, n): 690 691 "Process the given import node 'n'." 692 693 path = self.get_namespace_path() 694 695 # Load the mentioned module. 696 697 for name, alias in n.names: 698 if name == self.name: 699 raise InspectError("Cannot import the current module.", path, n) 700 701 self.set_module(alias or name.split(".")[-1], name) 702 self.queue_module(name, True) 703 704 def process_invocation_node(self, n): 705 706 "Process the given invocation node 'n'." 707 708 path = self.get_namespace_path() 709 710 self.allocate_arguments(path, n.args) 711 712 try: 713 # Process the expression, obtaining any identified reference. 714 715 in_invocation = self.in_invocation 716 self.in_invocation = True 717 name_ref = self.process_structure_node(n.node) 718 self.in_invocation = in_invocation 719 720 # Process the arguments. 721 722 for arg in n.args: 723 self.process_structure_node(arg) 724 725 # Detect class invocations. 726 727 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 728 return InstanceRef(name_ref.reference().instance_of()) 729 730 elif isinstance(name_ref, NameRef): 731 return InvocationRef(name_ref) 732 733 return None 734 735 finally: 736 self.deallocate_arguments(path, n.args) 737 738 def process_lambda_node(self, n): 739 740 "Process the given lambda node 'n'." 741 742 name = self.get_lambda_name() 743 self.process_function_node(n, name) 744 745 origin = self.get_object_path(name) 746 return ResolvedNameRef(name, Reference("<function>", origin)) 747 748 def process_logical_node(self, n): 749 750 "Process the given operator node 'n'." 751 752 self.process_operator_chain(n.nodes, self.process_structure_node) 753 754 def process_name_node(self, n): 755 756 "Process the given name node 'n'." 757 758 path = self.get_namespace_path() 759 760 # Special names. 761 762 if n.name.startswith("$"): 763 value = self.get_special(n.name) 764 if value: 765 return value 766 767 # Special case for operator functions introduced through code 768 # transformations. 769 770 if n.name.startswith("$op"): 771 772 # Obtain the location of the actual function defined in the operator 773 # package. 774 775 op = n.name[len("$op"):] 776 777 # Attempt to get a reference. 778 779 ref = self.import_name_from_module(op, "operator") 780 self.add_deferred(ref) 781 782 # Record the imported name and provide the resolved name reference. 783 784 value = ResolvedNameRef(n.name, ref) 785 self.set_special(n.name, value) 786 return value 787 788 # Test for self usage, which is only allowed in methods. 789 790 if n.name == "self" and not (self.in_function and self.in_class): 791 raise InspectError("Use of self is only allowed in methods.", path, n) 792 793 # Record usage of the name. 794 795 self.record_name(n.name) 796 797 # Search for unknown names in non-function scopes immediately. 798 # External names in functions are resolved later. 799 800 ref = self.find_name(n.name) 801 if ref: 802 return ResolvedNameRef(n.name, ref) 803 804 # Explicitly-declared global names. 805 806 elif self.in_function and n.name in self.scope_globals[path]: 807 return NameRef(n.name) 808 809 # Examine other names. 810 811 else: 812 tracker = self.trackers[-1] 813 814 # Check local names. 815 816 branches = tracker.tracking_name(n.name) 817 818 # Local name. 819 820 if branches: 821 self.record_branches_for_access(branches, n.name, None) 822 access_number = self.record_access_details(n.name, None, False) 823 return LocalNameRef(n.name, access_number) 824 825 # Possible global or built-in name. 826 827 else: 828 return NameRef(n.name) 829 830 def process_operator_chain(self, nodes, fn): 831 832 """ 833 Process the given chain of 'nodes', applying 'fn' to each node or item. 834 Each node starts a new conditional region, effectively making a deeply- 835 nested collection of if-like statements. 836 """ 837 838 tracker = self.trackers[-1] 839 840 for item in nodes: 841 tracker.new_branchpoint() 842 tracker.new_branch() 843 fn(item) 844 845 for item in nodes[:-1]: 846 tracker.shelve_branch() 847 tracker.new_branch() 848 tracker.shelve_branch() 849 tracker.merge_branches() 850 851 tracker.shelve_branch() 852 tracker.merge_branches() 853 854 def process_try_node(self, n): 855 856 """ 857 Process the given "try...except" node 'n'. 858 """ 859 860 tracker = self.trackers[-1] 861 tracker.new_branchpoint() 862 863 self.process_structure_node(n.body) 864 865 for name, var, handler in n.handlers: 866 if name is not None: 867 self.process_structure_node(name) 868 869 # Any abandoned branches from the body can now be resumed in a new 870 # branch. 871 872 tracker.resume_abandoned_branches() 873 874 # Establish the local for the handler. 875 876 if var is not None: 877 self.process_structure_node(var) 878 if handler is not None: 879 self.process_structure_node(handler) 880 881 tracker.shelve_branch() 882 883 # The else clause maintains the usage from the body but without the 884 # abandoned branches since they would never lead to the else clause 885 # being executed. 886 887 if n.else_: 888 tracker.new_branch() 889 self.process_structure_node(n.else_) 890 tracker.shelve_branch() 891 892 # Without an else clause, a null branch propagates the successful 893 # outcome. 894 895 else: 896 tracker.new_branch() 897 tracker.shelve_branch() 898 899 tracker.merge_branches() 900 901 def process_try_finally_node(self, n): 902 903 """ 904 Process the given "try...finally" node 'n'. 905 """ 906 907 tracker = self.trackers[-1] 908 self.process_structure_node(n.body) 909 910 # Any abandoned branches from the body can now be resumed. 911 912 branches = tracker.resume_all_abandoned_branches() 913 self.process_structure_node(n.final) 914 915 # At the end of the finally clause, abandoned branches are discarded. 916 917 tracker.restore_active_branches(branches) 918 919 def process_while_node(self, n): 920 921 "Process the given while node 'n'." 922 923 tracker = self.trackers[-1] 924 tracker.new_branchpoint(loop_node=True) 925 926 # Evaluate any test or iterator outside the loop. 927 928 self.process_structure_node(n.test) 929 930 # Propagate attribute usage to branches. 931 932 tracker.new_branch(loop_node=True) 933 934 # Enter the loop. 935 936 in_conditional = self.in_conditional 937 self.in_conditional = True 938 self.process_structure_node(n.body) 939 self.in_conditional = in_conditional 940 941 # Continuing branches are resumed before any test. 942 943 tracker.resume_continuing_branches() 944 945 # Evaluate any continuation test within the body. 946 947 self.process_structure_node(n.test) 948 949 tracker.shelve_branch(loop_node=True) 950 951 # Support the non-looping condition. 952 953 tracker.new_branch() 954 tracker.shelve_branch() 955 956 tracker.merge_branches() 957 958 # Evaluate any else clause outside branches. 959 960 if n.else_: 961 self.process_structure_node(n.else_) 962 963 # Connect broken branches to the code after any loop. 964 965 tracker.resume_broken_branches() 966 967 # Branch tracking methods. 968 969 def start_tracking(self, names): 970 971 """ 972 Start tracking attribute usage for names in the current namespace, 973 immediately registering the given 'names'. 974 """ 975 976 path = self.get_namespace_path() 977 parent = self.trackers[-1] 978 tracker = BranchTracker() 979 self.trackers.append(tracker) 980 981 # Record the given names established as new branches. 982 983 tracker.assign_names(names) 984 985 def assign_name(self, name, name_ref): 986 987 "Assign to 'name' the given 'name_ref' in the current namespace." 988 989 name = self.get_name_for_tracking(name) 990 self.trackers[-1].assign_names([name], [name_ref]) 991 992 def stop_tracking(self): 993 994 """ 995 Stop tracking attribute usage, recording computed usage for the current 996 namespace. 997 """ 998 999 path = self.get_namespace_path() 1000 tracker = self.trackers.pop() 1001 self.record_assignments_for_access(tracker) 1002 1003 self.attr_usage[path] = tracker.get_all_usage() 1004 self.name_initialisers[path] = tracker.get_all_values() 1005 1006 def start_tracking_in_module(self): 1007 1008 "Start tracking attribute usage in the module." 1009 1010 tracker = BranchTracker() 1011 self.trackers.append(tracker) 1012 1013 def stop_tracking_in_module(self): 1014 1015 "Stop tracking attribute usage in the module." 1016 1017 tracker = self.trackers[0] 1018 self.record_assignments_for_access(tracker) 1019 self.attr_usage[self.name] = tracker.get_all_usage() 1020 self.name_initialisers[self.name] = tracker.get_all_values() 1021 1022 def record_assignments_for_access(self, tracker): 1023 1024 """ 1025 For the current path, use the given 'tracker' to record assignment 1026 version information for attribute accesses. 1027 """ 1028 1029 path = self.get_path_for_access() 1030 1031 if not self.attr_accessor_branches.has_key(path): 1032 return 1033 1034 init_item(self.attr_accessors, path, dict) 1035 attr_accessors = self.attr_accessors[path] 1036 1037 # Obtain the branches applying during each access. 1038 1039 for access, all_branches in self.attr_accessor_branches[path].items(): 1040 name, attrnames = access 1041 init_item(attr_accessors, access, list) 1042 1043 # Obtain the assignments applying to each branch. 1044 1045 for branches in all_branches: 1046 positions = tracker.get_assignment_positions_for_branches(name, branches) 1047 1048 # Detect missing name information. 1049 1050 if None in positions: 1051 globals = self.global_attr_accesses.get(path) 1052 accesses = globals and globals.get(name) 1053 if not accesses: 1054 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1055 self.get_namespace_path(), name) 1056 positions.remove(None) 1057 1058 attr_accessors[access].append(positions) 1059 1060 def record_branches_for_access(self, branches, name, attrnames): 1061 1062 """ 1063 Record the given 'branches' for an access involving the given 'name' and 1064 'attrnames'. 1065 """ 1066 1067 access = name, attrnames 1068 path = self.get_path_for_access() 1069 1070 init_item(self.attr_accessor_branches, path, dict) 1071 attr_accessor_branches = self.attr_accessor_branches[path] 1072 1073 init_item(attr_accessor_branches, access, list) 1074 attr_accessor_branches[access].append(branches) 1075 1076 def record_access_details(self, name, attrnames, assignment): 1077 1078 """ 1079 For the given 'name' and 'attrnames', record an access indicating 1080 whether 'assignment' is occurring. 1081 1082 These details correspond to accesses otherwise recorded by the attribute 1083 accessor and attribute access dictionaries. 1084 """ 1085 1086 access = name, attrnames 1087 path = self.get_path_for_access() 1088 1089 init_item(self.attr_access_modifiers, path, dict) 1090 init_item(self.attr_access_modifiers[path], access, list) 1091 1092 access_number = len(self.attr_access_modifiers[path][access]) 1093 self.attr_access_modifiers[path][access].append(assignment) 1094 return access_number 1095 1096 def record_global_access_details(self, name, attrnames): 1097 1098 """ 1099 Record details of a global access via the given 'name' involving the 1100 indicated 'attrnames'. 1101 """ 1102 1103 path = self.get_namespace_path() 1104 1105 init_item(self.global_attr_accesses, path, dict) 1106 init_item(self.global_attr_accesses[path], name, set) 1107 self.global_attr_accesses[path][name].add(attrnames) 1108 1109 # Namespace modification. 1110 1111 def record_name(self, name): 1112 1113 "Record the use of 'name' in a namespace." 1114 1115 path = self.get_namespace_path() 1116 init_item(self.names_used, path, set) 1117 self.names_used[path].add(name) 1118 1119 def set_module(self, name, module_name): 1120 1121 """ 1122 Set a module in the current namespace using the given 'name' associated 1123 with the corresponding 'module_name'. 1124 """ 1125 1126 if name: 1127 self.set_general_local(name, Reference("<module>", module_name)) 1128 1129 def set_definition(self, name, kind): 1130 1131 """ 1132 Set the definition having the given 'name' and 'kind'. 1133 1134 Definitions are set in the static namespace hierarchy, but they can also 1135 be recorded for function locals. 1136 """ 1137 1138 if self.is_global(name): 1139 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1140 self.get_namespace_path(), name) 1141 1142 path = self.get_object_path(name) 1143 self.set_object(path, kind) 1144 1145 ref = self.get_object(path) 1146 if ref.get_kind() == "<var>": 1147 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1148 self.get_namespace_path(), name) 1149 1150 if not self.is_global(name) and self.in_function: 1151 self.set_function_local(name, ref) 1152 1153 def set_function_local(self, name, ref=None): 1154 1155 "Set the local with the given 'name' and optional 'ref'." 1156 1157 locals = self.function_locals[self.get_namespace_path()] 1158 multiple = not ref or locals.has_key(name) and locals[name] != ref 1159 locals[name] = multiple and Reference("<var>") or ref 1160 1161 def assign_general_local(self, name, name_ref): 1162 1163 """ 1164 Set for 'name' the given 'name_ref', recording the name for attribute 1165 usage tracking. 1166 """ 1167 1168 self.set_general_local(name, name_ref) 1169 self.assign_name(name, name_ref) 1170 1171 def set_general_local(self, name, value=None): 1172 1173 """ 1174 Set the 'name' with optional 'value' in any kind of local namespace, 1175 where the 'value' should be a reference if specified. 1176 """ 1177 1178 init_value = self.get_initialising_value(value) 1179 1180 # Module global names. 1181 1182 if self.is_global(name): 1183 path = self.get_global_path(name) 1184 self.set_object(path, init_value) 1185 1186 # Function local names. 1187 1188 elif self.in_function: 1189 path = self.get_object_path(name) 1190 self.set_function_local(name, init_value) 1191 1192 # Other namespaces (classes). 1193 1194 else: 1195 path = self.get_object_path(name) 1196 self.set_name(name, init_value) 1197 1198 def set_name(self, name, ref=None): 1199 1200 "Attach the 'name' with optional 'ref' to the current namespace." 1201 1202 self.set_object(self.get_object_path(name), ref) 1203 1204 def set_instance_attr(self, name, ref=None): 1205 1206 """ 1207 Add an instance attribute of the given 'name' to the current class, 1208 using the optional 'ref'. 1209 """ 1210 1211 init_item(self.instance_attrs, self.in_class, set) 1212 self.instance_attrs[self.in_class].add(name) 1213 1214 if ref: 1215 init_item(self.instance_attr_constants, self.in_class, dict) 1216 self.instance_attr_constants[self.in_class][name] = ref 1217 1218 def get_initialising_value(self, value): 1219 1220 "Return a suitable initialiser reference for 'value'." 1221 1222 # Includes LiteralSequenceRef, ResolvedNameRef... 1223 1224 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1225 return value.reference() 1226 1227 # In general, invocations do not produce known results. However, the 1228 # name initialisers are resolved once a module has been inspected. 1229 1230 elif isinstance(value, InvocationRef): 1231 return value.reference() 1232 1233 else: 1234 return value 1235 1236 # Static, program-relative naming. 1237 1238 def find_name(self, name): 1239 1240 """ 1241 Return the qualified name for the given 'name' used in the current 1242 non-function namespace. 1243 """ 1244 1245 path = self.get_namespace_path() 1246 ref = None 1247 1248 if not self.in_function and name not in predefined_constants: 1249 if self.in_class: 1250 ref = self.get_object(self.get_object_path(name)) 1251 if not ref: 1252 ref = self.get_global_or_builtin(name) 1253 1254 return ref 1255 1256 def get_class(self, node): 1257 1258 """ 1259 Use the given 'node' to obtain the identity of a class. Return a 1260 reference for the class. Unresolved dependencies are permitted and must 1261 be resolved later. 1262 """ 1263 1264 ref = self._get_class(node) 1265 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1266 1267 def _get_class(self, node): 1268 1269 """ 1270 Use the given 'node' to find a class definition. Return a reference to 1271 the class. 1272 """ 1273 1274 if isinstance(node, compiler.ast.Getattr): 1275 1276 # Obtain the identity of the access target. 1277 1278 ref = self._get_class(node.expr) 1279 1280 # Where the target is a class or module, obtain the identity of the 1281 # attribute. 1282 1283 if ref.has_kind(["<function>", "<var>"]): 1284 return None 1285 else: 1286 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1287 return self.get_object(attrname) 1288 1289 # Names can be module-level or built-in. 1290 1291 elif isinstance(node, compiler.ast.Name): 1292 1293 # Record usage of the name and attempt to identify it. 1294 1295 self.record_name(node.name) 1296 return self.find_name(node.name) 1297 else: 1298 return None 1299 1300 def get_constant(self, name, value): 1301 1302 "Return a constant reference for the given type 'name' and 'value'." 1303 1304 ref = self.get_builtin_class(name) 1305 return self.get_constant_reference(ref, value) 1306 1307 def get_literal_instance(self, n, name): 1308 1309 "For node 'n', return a reference to an instance of 'name'." 1310 1311 # Get a reference to the built-in class. 1312 1313 ref = self.get_builtin_class(name) 1314 1315 # Obtain the details of the literal itself. 1316 # An alias to the type is generated for sequences. 1317 1318 if name in ("dict", "list", "tuple"): 1319 self.set_special_literal(name, ref) 1320 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1321 1322 # Constant values are independently recorded. 1323 1324 else: 1325 return self.get_constant_reference(ref, n.value) 1326 1327 # Special names. 1328 1329 def get_special(self, name): 1330 1331 "Return any stored value for the given special 'name'." 1332 1333 return self.special.get(name) 1334 1335 def set_special(self, name, value): 1336 1337 """ 1338 Set a special 'name' that merely tracks the use of an implicit object 1339 'value'. 1340 """ 1341 1342 self.special[name] = value 1343 1344 def set_special_literal(self, name, ref): 1345 1346 """ 1347 Set a special name for the literal type 'name' having type 'ref'. Such 1348 special names provide a way of referring to literal object types. 1349 """ 1350 1351 literal_name = "$L%s" % name 1352 value = ResolvedNameRef(literal_name, ref) 1353 self.set_special(literal_name, value) 1354 1355 # Functions and invocations. 1356 1357 def set_invocation_usage(self): 1358 1359 """ 1360 Discard the current invocation storage figures, retaining the maximum 1361 values. 1362 """ 1363 1364 for path, (current, maximum) in self.function_targets.items(): 1365 self.importer.function_targets[path] = self.function_targets[path] = maximum 1366 1367 for path, (current, maximum) in self.function_arguments.items(): 1368 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1369 1370 def allocate_arguments(self, path, args): 1371 1372 """ 1373 Allocate temporary argument storage using current and maximum 1374 requirements for the given 'path' and 'args'. 1375 """ 1376 1377 init_item(self.function_targets, path, lambda: [0, 0]) 1378 t = self.function_targets[path] 1379 t[0] += 1 1380 t[1] = max(t[0], t[1]) 1381 1382 init_item(self.function_arguments, path, lambda: [0, 0]) 1383 t = self.function_arguments[path] 1384 t[0] += len(args) + 1 1385 t[1] = max(t[0], t[1]) 1386 1387 def deallocate_arguments(self, path, args): 1388 1389 "Deallocate temporary argument storage for the given 'path' and 'args'." 1390 1391 self.function_targets[path][0] -= 1 1392 self.function_arguments[path][0] -= len(args) + 1 1393 1394 # vim: tabstop=4 expandtab shiftwidth=4