1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 self.in_invocation = False 47 self.global_attr_accesses = {} 48 49 # Usage tracking. 50 51 self.trackers = [] 52 self.attr_accessor_branches = {} 53 54 def __repr__(self): 55 return "InspectedModule(%r, %r)" % (self.name, self.importer) 56 57 # Principal methods. 58 59 def parse(self, filename): 60 61 "Parse the file having the given 'filename'." 62 63 self.parse_file(filename) 64 65 # Inspect the module. 66 67 self.start_tracking_in_module() 68 69 # Detect and record imports and globals declared in the module. 70 71 self.assign_general_local("__name__", self.get_constant("str", self.name)) 72 self.assign_general_local("__file__", self.get_constant("str", filename)) 73 self.process_structure(self.astnode) 74 75 # Set the class of the module after the definition has occurred. 76 77 ref = self.get_builtin("object") 78 self.set_name("__class__", ref) 79 80 # Get module-level attribute usage details. 81 82 self.stop_tracking_in_module() 83 84 # Collect external name references. 85 86 self.collect_names() 87 88 def complete(self): 89 90 "Complete the module inspection." 91 92 # Resolve names not definitively mapped to objects. 93 94 self.resolve() 95 96 # Define the invocation requirements in each namespace. 97 98 self.set_invocation_usage() 99 100 # Propagate to the importer information needed in subsequent activities. 101 102 self.propagate() 103 104 # Accessory methods. 105 106 def collect_names(self): 107 108 "Collect the names used by each scope." 109 110 for path in self.names_used.keys(): 111 self.collect_names_for_path(path) 112 113 def collect_names_for_path(self, path): 114 115 """ 116 Collect the names used by the given 'path'. These are propagated to the 117 importer in advance of any dependency resolution. 118 """ 119 120 names = self.names_used.get(path) 121 if not names: 122 return 123 124 in_function = self.function_locals.has_key(path) 125 126 for name in names: 127 if name in predefined_constants or in_function and name in self.function_locals[path]: 128 continue 129 130 # Find local definitions (within dynamic namespaces). 131 132 key = "%s.%s" % (path, name) 133 ref = self.get_resolved_object(key) 134 if ref: 135 self.set_name_reference(key, ref) 136 continue 137 138 # Find global or known built-in definitions. 139 140 ref = self.get_resolved_global_or_builtin(name) 141 if ref: 142 self.set_name_reference(key, ref) 143 continue 144 145 # Find presumed built-in definitions. 146 147 ref = self.get_builtin(name) 148 self.set_name_reference(key, ref) 149 150 def set_name_reference(self, path, ref): 151 152 "Map the given name 'path' to 'ref'." 153 154 self.importer.all_name_references[path] = self.name_references[path] = ref 155 156 def get_resolved_global_or_builtin(self, name): 157 158 "Return the resolved global or built-in object with the given 'name'." 159 160 # In some circumstances, the name is neither global nor recognised by 161 # the importer. It is then assumed to be a general built-in. 162 163 return self.get_global(name) or \ 164 self.importer.get_object("__builtins__.%s" % name) 165 166 # Module structure traversal. 167 168 def process_structure_node(self, n): 169 170 "Process the individual node 'n'." 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 self.process_assignment_node(n, None) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 self.process_attribute_access(n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Invocations. 282 283 elif isinstance(n, compiler.ast.CallFunc): 284 return self.process_invocation_node(n) 285 286 # Constant usage. 287 288 elif isinstance(n, compiler.ast.Const): 289 return self.get_literal_instance(n, n.value.__class__.__name__) 290 291 elif isinstance(n, compiler.ast.Dict): 292 return self.get_literal_instance(n, "dict") 293 294 elif isinstance(n, compiler.ast.List): 295 return self.get_literal_instance(n, "list") 296 297 elif isinstance(n, compiler.ast.Tuple): 298 return self.get_literal_instance(n, "tuple") 299 300 # Unsupported nodes. 301 302 elif isinstance(n, compiler.ast.GenExpr): 303 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 304 305 elif isinstance(n, compiler.ast.IfExp): 306 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 307 308 elif isinstance(n, compiler.ast.ListComp): 309 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 310 311 # All other nodes are processed depth-first. 312 313 else: 314 self.process_structure(n) 315 316 # By default, no expression details are returned. 317 318 return None 319 320 # Specific node handling. 321 322 def process_assignment_node(self, n, expr): 323 324 "Process the individual node 'n' to be assigned the contents of 'expr'." 325 326 # Names and attributes are assigned the entire expression. 327 328 if isinstance(n, compiler.ast.AssName): 329 if n.name == "self": 330 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 331 332 name_ref = expr and self.process_structure_node(expr) 333 334 # Name assignments populate either function namespaces or the 335 # general namespace hierarchy. 336 337 self.assign_general_local(n.name, name_ref) 338 339 # Record usage of the name. 340 341 self.record_name(n.name) 342 343 elif isinstance(n, compiler.ast.AssAttr): 344 if expr: self.process_structure_node(expr) 345 self.process_attribute_access(n) 346 347 # Lists and tuples are matched against the expression and their 348 # items assigned to expression items. 349 350 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 351 self.process_assignment_node_items(n, expr) 352 353 # Slices and subscripts are permitted within assignment nodes. 354 355 elif isinstance(n, compiler.ast.Slice): 356 self.process_slice_node(n, expr) 357 358 elif isinstance(n, compiler.ast.Subscript): 359 self.process_subscript_node(n, expr) 360 361 def process_attribute_access(self, n): 362 363 "Process the given attribute access node 'n'." 364 365 # Obtain any completed chain and return the reference to it. 366 367 name_ref = self.process_attribute_chain(n) 368 if self.have_access_expression(n): 369 return name_ref 370 371 # Where the start of the chain of attributes has been reached, determine 372 # the complete access. 373 374 # Given a non-access node, this chain can be handled in its entirety, 375 # either being name-based and thus an access rooted on a name, or being 376 # based on some other node and thus an anonymous access of some kind. 377 378 path = self.get_namespace_path() 379 380 # Start with the the full attribute chain. 381 382 remaining = self.attrs 383 attrnames = ".".join(remaining) 384 385 # If the accessor cannot be identified, or where attributes 386 # remain in an attribute chain, record the anonymous accesses. 387 388 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 389 390 assignment = isinstance(n, compiler.ast.AssAttr) 391 392 init_item(self.attr_accesses, path, set) 393 self.attr_accesses[path].add(attrnames) 394 395 self.record_access_details(None, attrnames, assignment) 396 del self.attrs[0] 397 return 398 399 # Name-based accesses will handle the first attribute in a 400 # chain. 401 402 else: 403 attrname = remaining[0] 404 405 # Attribute assignments are used to identify instance attributes. 406 407 if isinstance(n, compiler.ast.AssAttr) and \ 408 self.in_class and self.in_function and n.expr.name == "self": 409 410 self.set_instance_attr(attrname) 411 412 # Record attribute usage using any name local to this namespace, 413 # if assigned in the namespace, or using an external name 414 # (presently just globals within classes). 415 416 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 417 tracker = self.trackers[-1] 418 419 immediate_access = len(self.attrs) == 1 420 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 421 422 del self.attrs[0] 423 424 # Record global-based chains for subsequent resolution. 425 426 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 427 not self.in_function 428 429 if is_global: 430 self.record_global_access_details(name, attrnames) 431 432 # Make sure the name is being tracked: global names will not 433 # already be initialised in a branch and must be added 434 # explicitly. 435 436 if not tracker.have_name(name): 437 tracker.assign_names([name]) 438 if self.in_function: 439 self.scope_globals[path].add(name) 440 441 # Record attribute usage in the tracker, and record the branch 442 # information for the access. 443 444 branches = tracker.use_attribute(name, attrname) 445 446 if not branches: 447 print >>sys.stderr, "In %s, name %s is accessed using %s before an assignment." % ( 448 path, name, attrname) 449 branches = tracker.use_attribute(name, attrname) 450 451 self.record_branches_for_access(branches, name, attrnames) 452 access_number = self.record_access_details(name, attrnames, assignment) 453 return AccessRef(name, attrnames, access_number) 454 455 def process_class_node(self, n): 456 457 "Process the given class node 'n'." 458 459 path = self.get_namespace_path() 460 461 # To avoid notions of class "versions" where the same definition 462 # might be parameterised with different state and be referenced 463 # elsewhere (as base classes, for example), classes in functions or 464 # conditions are forbidden. 465 466 if self.in_function or self.in_conditional: 467 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 468 path, n.name) 469 return 470 471 # Resolve base classes. 472 473 bases = [] 474 475 for base in n.bases: 476 base_class = self.get_class(base) 477 478 if not base_class: 479 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 480 path, n.name, base) 481 return 482 else: 483 bases.append(base_class) 484 485 # Record bases for the class and retain the class name. 486 487 class_name = self.get_object_path(n.name) 488 489 if not bases and class_name != "__builtins__.core.object": 490 ref = self.get_object("__builtins__.object") 491 bases.append(ref) 492 493 self.importer.classes[class_name] = self.classes[class_name] = bases 494 self.importer.subclasses[class_name] = set() 495 self.scope_globals[class_name] = set() 496 497 # Set the definition before entering the namespace rather than 498 # afterwards because methods may reference it. In normal Python, 499 # a class is not accessible until the definition is complete, but 500 # methods can generally reference it since upon being called the 501 # class will already exist. 502 503 self.set_definition(n.name, "<class>") 504 505 in_class = self.in_class 506 self.in_class = class_name 507 self.set_instance_attr("__class__", Reference("<class>", class_name)) 508 self.enter_namespace(n.name) 509 self.set_name("__fn__") # special instantiator attribute 510 self.set_name("__args__") # special instantiator attribute 511 self.assign_general_local("__name__", self.get_constant("str", class_name)) 512 self.process_structure_node(n.code) 513 self.exit_namespace() 514 self.in_class = in_class 515 516 def process_from_node(self, n): 517 518 "Process the given node 'n', importing from another module." 519 520 path = self.get_namespace_path() 521 522 module_name, names = self.get_module_name(n) 523 if module_name == self.name: 524 raise InspectError("Cannot import from the current module.", path, n) 525 526 self.queue_module(module_name) 527 528 # Attempt to obtain the referenced objects. 529 530 for name, alias in n.names: 531 if name == "*": 532 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 533 534 # Explicit names. 535 536 ref = self.import_name_from_module(name, module_name) 537 value = ResolvedNameRef(alias or name, ref) 538 self.set_general_local(alias or name, value) 539 540 def process_function_node(self, n, name): 541 542 """ 543 Process the given function or lambda node 'n' with the given 'name'. 544 """ 545 546 is_lambda = isinstance(n, compiler.ast.Lambda) 547 548 # Where a function is declared conditionally, use a separate name for 549 # the definition, and assign the definition to the stated name. 550 551 if (self.in_conditional or self.in_function) and not is_lambda: 552 original_name = name 553 name = self.get_lambda_name() 554 else: 555 original_name = None 556 557 # Initialise argument and local records. 558 559 function_name = self.get_object_path(name) 560 argnames = get_argnames(n.argnames) 561 is_method = self.in_class and not self.in_function 562 563 # Remove explicit "self" from method parameters. 564 565 if is_method and argnames and argnames[0] == "self": 566 del argnames[0] 567 568 # Copy and propagate the parameters. 569 570 self.importer.function_parameters[function_name] = \ 571 self.function_parameters[function_name] = argnames[:] 572 573 # Define all arguments/parameters in the local namespace. 574 575 locals = self.function_locals[function_name] = {} 576 577 # Insert "self" into method locals. 578 579 if is_method: 580 argnames.insert(0, "self") 581 582 # Define "self" in terms of the class if in a method. 583 # This does not diminish the need for type-narrowing in the deducer. 584 585 if argnames: 586 if self.in_class and not self.in_function and argnames[0] == "self": 587 locals[argnames[0]] = Reference("<instance>", self.in_class) 588 else: 589 locals[argnames[0]] = Reference("<var>") 590 591 for argname in argnames[1:]: 592 locals[argname] = Reference("<var>") 593 594 globals = self.scope_globals[function_name] = set() 595 596 # Process the defaults. 597 598 defaults = self.importer.function_defaults[function_name] = \ 599 self.function_defaults[function_name] = [] 600 601 for argname, default in compiler.ast.get_defaults(n): 602 if default: 603 604 # Obtain any reference for the default. 605 606 name_ref = self.process_structure_node(default) 607 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 608 609 # Reset conditional tracking to focus on the function contents. 610 611 in_conditional = self.in_conditional 612 self.in_conditional = False 613 614 in_function = self.in_function 615 self.in_function = function_name 616 617 self.enter_namespace(name) 618 619 # Track attribute usage within the namespace. 620 621 path = self.get_namespace_path() 622 623 self.start_tracking(locals) 624 self.process_structure_node(n.code) 625 self.stop_tracking() 626 627 # Exit to the parent. 628 629 self.exit_namespace() 630 631 # Update flags. 632 633 self.in_function = in_function 634 self.in_conditional = in_conditional 635 636 # Define the function using the appropriate name. 637 638 self.set_definition(name, "<function>") 639 640 # Where a function is set conditionally, assign the name. 641 642 if original_name: 643 self.process_assignment_for_function(original_name, name) 644 645 def process_global_node(self, n): 646 647 """ 648 Process the given "global" node 'n'. 649 """ 650 651 path = self.get_namespace_path() 652 653 if path != self.name: 654 self.scope_globals[path].update(n.names) 655 656 def process_if_node(self, n): 657 658 """ 659 Process the given "if" node 'n'. 660 """ 661 662 tracker = self.trackers[-1] 663 tracker.new_branchpoint() 664 665 for test, body in n.tests: 666 self.process_structure_node(test) 667 668 tracker.new_branch() 669 670 in_conditional = self.in_conditional 671 self.in_conditional = True 672 self.process_structure_node(body) 673 self.in_conditional = in_conditional 674 675 tracker.shelve_branch() 676 677 # Maintain a branch for the else clause. 678 679 tracker.new_branch() 680 if n.else_: 681 self.process_structure_node(n.else_) 682 tracker.shelve_branch() 683 684 tracker.merge_branches() 685 686 def process_import_node(self, n): 687 688 "Process the given import node 'n'." 689 690 path = self.get_namespace_path() 691 692 # Load the mentioned module. 693 694 for name, alias in n.names: 695 if name == self.name: 696 raise InspectError("Cannot import the current module.", path, n) 697 698 self.set_module(alias or name.split(".")[-1], name) 699 self.queue_module(name, True) 700 701 def process_invocation_node(self, n): 702 703 "Process the given invocation node 'n'." 704 705 path = self.get_namespace_path() 706 707 self.allocate_arguments(path, n.args) 708 709 try: 710 # Process the expression, obtaining any identified reference. 711 712 in_invocation = self.in_invocation 713 self.in_invocation = True 714 name_ref = self.process_structure_node(n.node) 715 self.in_invocation = in_invocation 716 717 # Process the arguments. 718 719 for arg in n.args: 720 self.process_structure_node(arg) 721 722 # Detect class invocations. 723 724 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 725 return InstanceRef(name_ref.reference().instance_of()) 726 727 elif isinstance(name_ref, NameRef): 728 return InvocationRef(name_ref) 729 730 return None 731 732 finally: 733 self.deallocate_arguments(path, n.args) 734 735 def process_lambda_node(self, n): 736 737 "Process the given lambda node 'n'." 738 739 name = self.get_lambda_name() 740 self.process_function_node(n, name) 741 742 origin = self.get_object_path(name) 743 return ResolvedNameRef(name, Reference("<function>", origin)) 744 745 def process_logical_node(self, n): 746 747 "Process the given operator node 'n'." 748 749 self.process_operator_chain(n.nodes, self.process_structure_node) 750 751 def process_name_node(self, n): 752 753 "Process the given name node 'n'." 754 755 path = self.get_namespace_path() 756 757 # Special names. 758 759 if n.name.startswith("$"): 760 value = self.get_special(n.name) 761 if value: 762 return value 763 764 # Special case for operator functions introduced through code 765 # transformations. 766 767 if n.name.startswith("$op"): 768 769 # Obtain the location of the actual function defined in the operator 770 # package. 771 772 op = n.name[len("$op"):] 773 774 # Attempt to get a reference. 775 776 ref = self.import_name_from_module(op, "operator") 777 self.add_deferred(ref) 778 779 # Record the imported name and provide the resolved name reference. 780 781 value = ResolvedNameRef(n.name, ref) 782 self.set_special(n.name, value) 783 return value 784 785 # Test for self usage, which is only allowed in methods. 786 787 if n.name == "self" and not (self.in_function and self.in_class): 788 raise InspectError("Use of self is only allowed in methods.", path, n) 789 790 # Record usage of the name. 791 792 self.record_name(n.name) 793 794 # Search for unknown names in non-function scopes immediately. 795 # External names in functions are resolved later. 796 797 ref = self.find_name(n.name) 798 if ref: 799 return ResolvedNameRef(n.name, ref) 800 801 # Explicitly-declared global names. 802 803 elif self.in_function and n.name in self.scope_globals[path]: 804 return NameRef(n.name) 805 806 # Examine other names. 807 808 else: 809 tracker = self.trackers[-1] 810 811 # Check local names. 812 813 branches = tracker.tracking_name(n.name) 814 815 # Local name. 816 817 if branches: 818 self.record_branches_for_access(branches, n.name, None) 819 access_number = self.record_access_details(n.name, None, False) 820 return LocalNameRef(n.name, access_number) 821 822 # Possible global or built-in name. 823 824 else: 825 return NameRef(n.name) 826 827 def process_operator_chain(self, nodes, fn): 828 829 """ 830 Process the given chain of 'nodes', applying 'fn' to each node or item. 831 Each node starts a new conditional region, effectively making a deeply- 832 nested collection of if-like statements. 833 """ 834 835 tracker = self.trackers[-1] 836 837 for item in nodes: 838 tracker.new_branchpoint() 839 tracker.new_branch() 840 fn(item) 841 842 for item in nodes[:-1]: 843 tracker.shelve_branch() 844 tracker.new_branch() 845 tracker.shelve_branch() 846 tracker.merge_branches() 847 848 tracker.shelve_branch() 849 tracker.merge_branches() 850 851 def process_try_node(self, n): 852 853 """ 854 Process the given "try...except" node 'n'. 855 """ 856 857 tracker = self.trackers[-1] 858 tracker.new_branchpoint() 859 860 self.process_structure_node(n.body) 861 862 for name, var, handler in n.handlers: 863 if name is not None: 864 self.process_structure_node(name) 865 866 # Any abandoned branches from the body can now be resumed in a new 867 # branch. 868 869 tracker.resume_abandoned_branches() 870 871 # Establish the local for the handler. 872 873 if var is not None: 874 self.process_structure_node(var) 875 if handler is not None: 876 self.process_structure_node(handler) 877 878 tracker.shelve_branch() 879 880 # The else clause maintains the usage from the body but without the 881 # abandoned branches since they would never lead to the else clause 882 # being executed. 883 884 if n.else_: 885 tracker.new_branch() 886 self.process_structure_node(n.else_) 887 tracker.shelve_branch() 888 889 # Without an else clause, a null branch propagates the successful 890 # outcome. 891 892 else: 893 tracker.new_branch() 894 tracker.shelve_branch() 895 896 tracker.merge_branches() 897 898 def process_try_finally_node(self, n): 899 900 """ 901 Process the given "try...finally" node 'n'. 902 """ 903 904 tracker = self.trackers[-1] 905 self.process_structure_node(n.body) 906 907 # Any abandoned branches from the body can now be resumed. 908 909 branches = tracker.resume_all_abandoned_branches() 910 self.process_structure_node(n.final) 911 912 # At the end of the finally clause, abandoned branches are discarded. 913 914 tracker.restore_active_branches(branches) 915 916 def process_while_node(self, n): 917 918 "Process the given while node 'n'." 919 920 tracker = self.trackers[-1] 921 tracker.new_branchpoint(loop_node=True) 922 923 # Evaluate any test or iterator outside the loop. 924 925 self.process_structure_node(n.test) 926 927 # Propagate attribute usage to branches. 928 929 tracker.new_branch(loop_node=True) 930 931 # Enter the loop. 932 933 in_conditional = self.in_conditional 934 self.in_conditional = True 935 self.process_structure_node(n.body) 936 self.in_conditional = in_conditional 937 938 # Continuing branches are resumed before any test. 939 940 tracker.resume_continuing_branches() 941 942 # Evaluate any continuation test within the body. 943 944 self.process_structure_node(n.test) 945 946 tracker.shelve_branch(loop_node=True) 947 948 # Support the non-looping condition. 949 950 tracker.new_branch() 951 tracker.shelve_branch() 952 953 tracker.merge_branches() 954 955 # Evaluate any else clause outside branches. 956 957 if n.else_: 958 self.process_structure_node(n.else_) 959 960 # Connect broken branches to the code after any loop. 961 962 tracker.resume_broken_branches() 963 964 # Branch tracking methods. 965 966 def start_tracking(self, names): 967 968 """ 969 Start tracking attribute usage for names in the current namespace, 970 immediately registering the given 'names'. 971 """ 972 973 path = self.get_namespace_path() 974 parent = self.trackers[-1] 975 tracker = BranchTracker() 976 self.trackers.append(tracker) 977 978 # Record the given names established as new branches. 979 980 tracker.assign_names(names) 981 982 def assign_name(self, name, name_ref): 983 984 "Assign to 'name' the given 'name_ref' in the current namespace." 985 986 name = self.get_name_for_tracking(name) 987 self.trackers[-1].assign_names([name], [name_ref]) 988 989 def stop_tracking(self): 990 991 """ 992 Stop tracking attribute usage, recording computed usage for the current 993 namespace. 994 """ 995 996 path = self.get_namespace_path() 997 tracker = self.trackers.pop() 998 self.record_assignments_for_access(tracker) 999 1000 self.attr_usage[path] = tracker.get_all_usage() 1001 self.name_initialisers[path] = tracker.get_all_values() 1002 1003 def start_tracking_in_module(self): 1004 1005 "Start tracking attribute usage in the module." 1006 1007 tracker = BranchTracker() 1008 self.trackers.append(tracker) 1009 1010 def stop_tracking_in_module(self): 1011 1012 "Stop tracking attribute usage in the module." 1013 1014 tracker = self.trackers[0] 1015 self.record_assignments_for_access(tracker) 1016 self.attr_usage[self.name] = tracker.get_all_usage() 1017 self.name_initialisers[self.name] = tracker.get_all_values() 1018 1019 def record_assignments_for_access(self, tracker): 1020 1021 """ 1022 For the current path, use the given 'tracker' to record assignment 1023 version information for attribute accesses. 1024 """ 1025 1026 path = self.get_path_for_access() 1027 1028 if not self.attr_accessor_branches.has_key(path): 1029 return 1030 1031 init_item(self.attr_accessors, path, dict) 1032 attr_accessors = self.attr_accessors[path] 1033 1034 # Obtain the branches applying during each access. 1035 1036 for access, all_branches in self.attr_accessor_branches[path].items(): 1037 name, attrnames = access 1038 init_item(attr_accessors, access, list) 1039 1040 # Obtain the assignments applying to each branch. 1041 1042 for branches in all_branches: 1043 positions = tracker.get_assignment_positions_for_branches(name, branches) 1044 1045 # Detect missing name information. 1046 1047 if None in positions: 1048 globals = self.global_attr_accesses.get(path) 1049 accesses = globals and globals.get(name) 1050 if not accesses: 1051 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1052 self.get_namespace_path(), name) 1053 positions.remove(None) 1054 1055 attr_accessors[access].append(positions) 1056 1057 def record_branches_for_access(self, branches, name, attrnames): 1058 1059 """ 1060 Record the given 'branches' for an access involving the given 'name' and 1061 'attrnames'. 1062 """ 1063 1064 access = name, attrnames 1065 path = self.get_path_for_access() 1066 1067 init_item(self.attr_accessor_branches, path, dict) 1068 attr_accessor_branches = self.attr_accessor_branches[path] 1069 1070 init_item(attr_accessor_branches, access, list) 1071 attr_accessor_branches[access].append(branches) 1072 1073 def record_access_details(self, name, attrnames, assignment): 1074 1075 """ 1076 For the given 'name' and 'attrnames', record an access indicating 1077 whether 'assignment' is occurring. 1078 1079 These details correspond to accesses otherwise recorded by the attribute 1080 accessor and attribute access dictionaries. 1081 """ 1082 1083 access = name, attrnames 1084 path = self.get_path_for_access() 1085 1086 init_item(self.attr_access_modifiers, path, dict) 1087 init_item(self.attr_access_modifiers[path], access, list) 1088 1089 access_number = len(self.attr_access_modifiers[path][access]) 1090 self.attr_access_modifiers[path][access].append(assignment and "A" or self.in_invocation and "I" or "_") 1091 return access_number 1092 1093 def record_global_access_details(self, name, attrnames): 1094 1095 """ 1096 Record details of a global access via the given 'name' involving the 1097 indicated 'attrnames'. 1098 """ 1099 1100 path = self.get_namespace_path() 1101 1102 init_item(self.global_attr_accesses, path, dict) 1103 init_item(self.global_attr_accesses[path], name, set) 1104 self.global_attr_accesses[path][name].add(attrnames) 1105 1106 # Namespace modification. 1107 1108 def record_name(self, name): 1109 1110 "Record the use of 'name' in a namespace." 1111 1112 path = self.get_namespace_path() 1113 init_item(self.names_used, path, set) 1114 self.names_used[path].add(name) 1115 1116 def set_module(self, name, module_name): 1117 1118 """ 1119 Set a module in the current namespace using the given 'name' associated 1120 with the corresponding 'module_name'. 1121 """ 1122 1123 if name: 1124 self.set_general_local(name, Reference("<module>", module_name)) 1125 1126 def set_definition(self, name, kind): 1127 1128 """ 1129 Set the definition having the given 'name' and 'kind'. 1130 1131 Definitions are set in the static namespace hierarchy, but they can also 1132 be recorded for function locals. 1133 """ 1134 1135 if self.is_global(name): 1136 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1137 self.get_namespace_path(), name) 1138 1139 path = self.get_object_path(name) 1140 self.set_object(path, kind) 1141 1142 ref = self.get_object(path) 1143 if ref.get_kind() == "<var>": 1144 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1145 self.get_namespace_path(), name) 1146 1147 if not self.is_global(name) and self.in_function: 1148 self.set_function_local(name, ref) 1149 1150 def set_function_local(self, name, ref=None): 1151 1152 "Set the local with the given 'name' and optional 'ref'." 1153 1154 locals = self.function_locals[self.get_namespace_path()] 1155 multiple = not ref or locals.has_key(name) and locals[name] != ref 1156 locals[name] = multiple and Reference("<var>") or ref 1157 1158 def assign_general_local(self, name, name_ref): 1159 1160 """ 1161 Set for 'name' the given 'name_ref', recording the name for attribute 1162 usage tracking. 1163 """ 1164 1165 self.set_general_local(name, name_ref) 1166 self.assign_name(name, name_ref) 1167 1168 def set_general_local(self, name, value=None): 1169 1170 """ 1171 Set the 'name' with optional 'value' in any kind of local namespace, 1172 where the 'value' should be a reference if specified. 1173 """ 1174 1175 init_value = self.get_initialising_value(value) 1176 1177 # Module global names. 1178 1179 if self.is_global(name): 1180 path = self.get_global_path(name) 1181 self.set_object(path, init_value) 1182 1183 # Function local names. 1184 1185 elif self.in_function: 1186 path = self.get_object_path(name) 1187 self.set_function_local(name, init_value) 1188 1189 # Other namespaces (classes). 1190 1191 else: 1192 path = self.get_object_path(name) 1193 self.set_name(name, init_value) 1194 1195 def set_name(self, name, ref=None): 1196 1197 "Attach the 'name' with optional 'ref' to the current namespace." 1198 1199 self.set_object(self.get_object_path(name), ref) 1200 1201 def set_instance_attr(self, name, ref=None): 1202 1203 """ 1204 Add an instance attribute of the given 'name' to the current class, 1205 using the optional 'ref'. 1206 """ 1207 1208 init_item(self.instance_attrs, self.in_class, set) 1209 self.instance_attrs[self.in_class].add(name) 1210 1211 if ref: 1212 init_item(self.instance_attr_constants, self.in_class, dict) 1213 self.instance_attr_constants[self.in_class][name] = ref 1214 1215 def get_initialising_value(self, value): 1216 1217 "Return a suitable initialiser reference for 'value'." 1218 1219 # Includes LiteralSequenceRef, ResolvedNameRef... 1220 1221 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1222 return value.reference() 1223 1224 # In general, invocations do not produce known results. However, the 1225 # name initialisers are resolved once a module has been inspected. 1226 1227 elif isinstance(value, InvocationRef): 1228 return value.reference() 1229 1230 else: 1231 return value 1232 1233 # Static, program-relative naming. 1234 1235 def find_name(self, name): 1236 1237 """ 1238 Return the qualified name for the given 'name' used in the current 1239 non-function namespace. 1240 """ 1241 1242 path = self.get_namespace_path() 1243 ref = None 1244 1245 if not self.in_function and name not in predefined_constants: 1246 if self.in_class: 1247 ref = self.get_object(self.get_object_path(name)) 1248 if not ref: 1249 ref = self.get_global_or_builtin(name) 1250 1251 return ref 1252 1253 def get_class(self, node): 1254 1255 """ 1256 Use the given 'node' to obtain the identity of a class. Return a 1257 reference for the class. Unresolved dependencies are permitted and must 1258 be resolved later. 1259 """ 1260 1261 ref = self._get_class(node) 1262 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1263 1264 def _get_class(self, node): 1265 1266 """ 1267 Use the given 'node' to find a class definition. Return a reference to 1268 the class. 1269 """ 1270 1271 if isinstance(node, compiler.ast.Getattr): 1272 1273 # Obtain the identity of the access target. 1274 1275 ref = self._get_class(node.expr) 1276 1277 # Where the target is a class or module, obtain the identity of the 1278 # attribute. 1279 1280 if ref.has_kind(["<function>", "<var>"]): 1281 return None 1282 else: 1283 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1284 return self.get_object(attrname) 1285 1286 # Names can be module-level or built-in. 1287 1288 elif isinstance(node, compiler.ast.Name): 1289 1290 # Record usage of the name and attempt to identify it. 1291 1292 self.record_name(node.name) 1293 return self.find_name(node.name) 1294 else: 1295 return None 1296 1297 def get_constant(self, name, value): 1298 1299 "Return a constant reference for the given type 'name' and 'value'." 1300 1301 ref = self.get_builtin_class(name) 1302 return self.get_constant_reference(ref, value) 1303 1304 def get_literal_instance(self, n, name): 1305 1306 "For node 'n', return a reference to an instance of 'name'." 1307 1308 # Get a reference to the built-in class. 1309 1310 ref = self.get_builtin_class(name) 1311 1312 # Obtain the details of the literal itself. 1313 # An alias to the type is generated for sequences. 1314 1315 if name in ("dict", "list", "tuple"): 1316 self.set_special_literal(name, ref) 1317 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1318 1319 # Constant values are independently recorded. 1320 1321 else: 1322 return self.get_constant_reference(ref, n.value) 1323 1324 # Special names. 1325 1326 def get_special(self, name): 1327 1328 "Return any stored value for the given special 'name'." 1329 1330 return self.special.get(name) 1331 1332 def set_special(self, name, value): 1333 1334 """ 1335 Set a special 'name' that merely tracks the use of an implicit object 1336 'value'. 1337 """ 1338 1339 self.special[name] = value 1340 1341 def set_special_literal(self, name, ref): 1342 1343 """ 1344 Set a special name for the literal type 'name' having type 'ref'. Such 1345 special names provide a way of referring to literal object types. 1346 """ 1347 1348 literal_name = "$L%s" % name 1349 value = ResolvedNameRef(literal_name, ref) 1350 self.set_special(literal_name, value) 1351 1352 # Functions and invocations. 1353 1354 def set_invocation_usage(self): 1355 1356 """ 1357 Discard the current invocation storage figures, retaining the maximum 1358 values. 1359 """ 1360 1361 for path, (current, maximum) in self.function_targets.items(): 1362 self.importer.function_targets[path] = self.function_targets[path] = maximum 1363 1364 for path, (current, maximum) in self.function_arguments.items(): 1365 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1366 1367 def allocate_arguments(self, path, args): 1368 1369 """ 1370 Allocate temporary argument storage using current and maximum 1371 requirements for the given 'path' and 'args'. 1372 """ 1373 1374 init_item(self.function_targets, path, lambda: [0, 0]) 1375 t = self.function_targets[path] 1376 t[0] += 1 1377 t[1] = max(t[0], t[1]) 1378 1379 init_item(self.function_arguments, path, lambda: [0, 0]) 1380 t = self.function_arguments[path] 1381 t[0] += len(args) + 1 1382 t[1] = max(t[0], t[1]) 1383 1384 def deallocate_arguments(self, path, args): 1385 1386 "Deallocate temporary argument storage for the given 'path' and 'args'." 1387 1388 self.function_targets[path][0] -= 1 1389 self.function_arguments[path][0] -= len(args) + 1 1390 1391 # vim: tabstop=4 expandtab shiftwidth=4