1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 47 # Accesses to global attributes. 48 49 self.global_attr_accesses = {} 50 51 # Usage tracking. 52 53 self.trackers = [] 54 self.attr_accessor_branches = {} 55 56 def __repr__(self): 57 return "InspectedModule(%r, %r)" % (self.name, self.importer) 58 59 # Principal methods. 60 61 def parse(self, filename): 62 63 "Parse the file having the given 'filename'." 64 65 self.parse_file(filename) 66 67 # Inspect the module. 68 69 self.start_tracking_in_module() 70 71 # Detect and record imports and globals declared in the module. 72 73 self.assign_general_local("__name__", self.get_constant("string", self.name)) 74 self.assign_general_local("__file__", self.get_constant("string", filename)) 75 self.process_structure(self.astnode) 76 77 # Set the class of the module after the definition has occurred. 78 79 ref = self.get_builtin("object") 80 self.set_name("__class__", ref) 81 82 # Get module-level attribute usage details. 83 84 self.stop_tracking_in_module() 85 86 # Collect external name references. 87 88 self.collect_names() 89 90 def complete(self): 91 92 "Complete the module inspection." 93 94 # Resolve names not definitively mapped to objects. 95 96 self.resolve() 97 98 # Define the invocation requirements in each namespace. 99 100 self.set_invocation_usage() 101 102 # Propagate to the importer information needed in subsequent activities. 103 104 self.propagate() 105 106 # Accessory methods. 107 108 def collect_names(self): 109 110 "Collect the names used by each scope." 111 112 for path in self.names_used.keys(): 113 self.collect_names_for_path(path) 114 115 def collect_names_for_path(self, path): 116 117 """ 118 Collect the names used by the given 'path'. These are propagated to the 119 importer in advance of any dependency resolution. 120 """ 121 122 names = self.names_used.get(path) 123 if not names: 124 return 125 126 in_function = self.function_locals.has_key(path) 127 128 for name in names: 129 if in_function and name in self.function_locals[path]: 130 continue 131 132 key = "%s.%s" % (path, name) 133 134 # Find predefined constant names before anything else. 135 136 if name in predefined_constants: 137 ref = self.get_builtin(name) 138 self.set_name_reference(key, ref) 139 continue 140 141 # Find local definitions (within dynamic namespaces). 142 143 ref = self.get_resolved_object(key) 144 if ref: 145 self.set_name_reference(key, ref) 146 continue 147 148 # Find global. 149 150 ref = self.get_global(name) 151 if ref: 152 self.set_name_reference(key, ref) 153 continue 154 155 # Find presumed built-in definitions. 156 157 ref = self.get_builtin(name) 158 self.set_name_reference(key, ref) 159 160 def set_name_reference(self, path, ref): 161 162 "Map the given name 'path' to 'ref'." 163 164 self.importer.all_name_references[path] = self.name_references[path] = ref 165 166 # Module structure traversal. 167 168 def process_structure_node(self, n): 169 170 "Process the individual node 'n'." 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 self.process_assignment_node(n, None) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 self.process_attribute_access(n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Print statements. 282 283 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 284 self.process_print_node(n) 285 286 # Invocations. 287 288 elif isinstance(n, compiler.ast.CallFunc): 289 return self.process_invocation_node(n) 290 291 # Constant usage. 292 293 elif isinstance(n, compiler.ast.Const): 294 typename = n.value.__class__.__name__ 295 return self.get_literal_instance(n, typename == "str" and "string" or typename) 296 297 elif isinstance(n, compiler.ast.Dict): 298 return self.get_literal_instance(n, "dict") 299 300 elif isinstance(n, compiler.ast.List): 301 return self.get_literal_instance(n, "list") 302 303 elif isinstance(n, compiler.ast.Tuple): 304 return self.get_literal_instance(n, "tuple") 305 306 # Unsupported nodes. 307 308 elif isinstance(n, compiler.ast.GenExpr): 309 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 310 311 elif isinstance(n, compiler.ast.IfExp): 312 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 313 314 elif isinstance(n, compiler.ast.ListComp): 315 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 316 317 # All other nodes are processed depth-first. 318 319 else: 320 self.process_structure(n) 321 322 # By default, no expression details are returned. 323 324 return None 325 326 # Specific node handling. 327 328 def process_assignment_node(self, n, expr): 329 330 "Process the individual node 'n' to be assigned the contents of 'expr'." 331 332 # Names and attributes are assigned the entire expression. 333 334 if isinstance(n, compiler.ast.AssName): 335 if n.name == "self": 336 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 337 338 name_ref = expr and self.process_structure_node(expr) 339 340 # Name assignments populate either function namespaces or the 341 # general namespace hierarchy. 342 343 self.assign_general_local(n.name, name_ref) 344 345 # Record usage of the name. 346 347 self.record_name(n.name) 348 349 elif isinstance(n, compiler.ast.AssAttr): 350 if expr: 351 expr = self.process_structure_node(expr) 352 353 in_assignment = self.in_assignment 354 self.in_assignment = expr 355 self.process_attribute_access(n) 356 self.in_assignment = in_assignment 357 358 # Lists and tuples are matched against the expression and their 359 # items assigned to expression items. 360 361 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 362 self.process_assignment_node_items(n, expr) 363 364 # Slices and subscripts are permitted within assignment nodes. 365 366 elif isinstance(n, compiler.ast.Slice): 367 self.process_slice_node(n, expr) 368 369 elif isinstance(n, compiler.ast.Subscript): 370 self.process_subscript_node(n, expr) 371 372 def process_attribute_access(self, n): 373 374 "Process the given attribute access node 'n'." 375 376 # Obtain any completed chain and return the reference to it. 377 378 name_ref = self.process_attribute_chain(n) 379 380 if self.have_access_expression(n): 381 return name_ref 382 383 # Where the start of the chain of attributes has been reached, determine 384 # the complete access. 385 386 # Given a non-access node, this chain can be handled in its entirety, 387 # either being name-based and thus an access rooted on a name, or being 388 # based on some other node and thus an anonymous access of some kind. 389 390 path = self.get_namespace_path() 391 392 # Start with the the full attribute chain. 393 394 remaining = self.attrs 395 attrnames = ".".join(remaining) 396 397 # If the accessor cannot be identified, or where attributes 398 # remain in an attribute chain, record the anonymous accesses. 399 400 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 401 402 init_item(self.attr_accesses, path, set) 403 self.attr_accesses[path].add(attrnames) 404 405 self.record_access_details(None, attrnames, self.in_assignment, 406 self.in_invocation) 407 del self.attrs[0] 408 return 409 410 # Name-based accesses will handle the first attribute in a 411 # chain. 412 413 else: 414 attrname = remaining[0] 415 416 # Attribute assignments are used to identify instance attributes. 417 418 if isinstance(n, compiler.ast.AssAttr) and \ 419 self.in_class and self.in_function and n.expr.name == "self": 420 421 self.set_instance_attr(attrname) 422 423 # Record attribute usage using any name local to this namespace, 424 # if assigned in the namespace, or using an external name 425 # (presently just globals within classes). 426 427 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 428 tracker = self.trackers[-1] 429 430 immediate_access = len(self.attrs) == 1 431 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 432 433 # Record global-based chains for subsequent resolution. 434 435 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 436 not self.in_function 437 438 if is_global: 439 self.record_global_access_details(name, attrnames) 440 441 # Make sure the name is being tracked: global names will not 442 # already be initialised in a branch and must be added 443 # explicitly. 444 445 if not tracker.have_name(name): 446 tracker.assign_names([name]) 447 if self.in_function: 448 self.scope_globals[path].add(name) 449 450 # Record attribute usage in the tracker, and record the branch 451 # information for the access. 452 453 branches = tracker.use_attribute(name, attrname, self.in_invocation, assignment) 454 455 if not branches: 456 raise InspectError("Name %s is accessed using %s before an assignment." % ( 457 name, attrname), path, n) 458 459 self.record_branches_for_access(branches, name, attrnames) 460 access_number = self.record_access_details(name, attrnames, 461 self.in_assignment, self.in_invocation) 462 463 del self.attrs[0] 464 return AccessRef(name, attrnames, access_number) 465 466 def process_class_node(self, n): 467 468 "Process the given class node 'n'." 469 470 path = self.get_namespace_path() 471 472 # To avoid notions of class "versions" where the same definition 473 # might be parameterised with different state and be referenced 474 # elsewhere (as base classes, for example), classes in functions or 475 # conditions are forbidden. 476 477 if self.in_function or self.in_conditional: 478 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 479 path, n.name) 480 return 481 482 # Resolve base classes. 483 484 bases = [] 485 486 for base in n.bases: 487 base_class = self.get_class(base) 488 489 if not base_class: 490 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 491 path, n.name, base) 492 return 493 else: 494 bases.append(base_class) 495 496 # Record bases for the class and retain the class name. 497 # Note that the function class does not inherit from the object class. 498 499 class_name = self.get_object_path(n.name) 500 501 if not bases and class_name != "__builtins__.core.object" and \ 502 class_name != "__builtins__.core.function": 503 504 ref = self.get_object("__builtins__.object") 505 bases.append(ref) 506 507 self.importer.classes[class_name] = self.classes[class_name] = bases 508 self.importer.subclasses[class_name] = set() 509 self.scope_globals[class_name] = set() 510 511 # Set the definition before entering the namespace rather than 512 # afterwards because methods may reference it. In normal Python, 513 # a class is not accessible until the definition is complete, but 514 # methods can generally reference it since upon being called the 515 # class will already exist. 516 517 self.set_definition(n.name, "<class>") 518 519 in_class = self.in_class 520 self.in_class = class_name 521 self.set_instance_attr("__class__", Reference("<class>", class_name)) 522 self.enter_namespace(n.name) 523 524 # Do not provide the special instantiator attributes on the function 525 # class. Function instances provide these attributes. 526 527 if class_name != "__builtins__.core.function": 528 self.set_name("__fn__") # special instantiator attribute 529 self.set_name("__args__") # special instantiator attribute 530 531 self.assign_general_local("__name__", self.get_constant("string", class_name)) 532 self.process_structure_node(n.code) 533 self.exit_namespace() 534 self.in_class = in_class 535 536 def process_from_node(self, n): 537 538 "Process the given node 'n', importing from another module." 539 540 path = self.get_namespace_path() 541 542 module_name, names = self.get_module_name(n) 543 if module_name == self.name: 544 raise InspectError("Cannot import from the current module.", path, n) 545 546 self.queue_module(module_name) 547 548 # Attempt to obtain the referenced objects. 549 550 for name, alias in n.names: 551 if name == "*": 552 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 553 554 # Explicit names. 555 556 ref = self.import_name_from_module(name, module_name) 557 value = ResolvedNameRef(alias or name, ref) 558 self.set_general_local(alias or name, value) 559 560 def process_function_node(self, n, name): 561 562 """ 563 Process the given function or lambda node 'n' with the given 'name'. 564 """ 565 566 is_lambda = isinstance(n, compiler.ast.Lambda) 567 568 # Where a function is declared conditionally, use a separate name for 569 # the definition, and assign the definition to the stated name. 570 571 if (self.in_conditional or self.in_function) and not is_lambda: 572 original_name = name 573 name = self.get_lambda_name() 574 else: 575 original_name = None 576 577 # Initialise argument and local records. 578 579 function_name = self.get_object_path(name) 580 argnames = get_argnames(n.argnames) 581 is_method = self.in_class and not self.in_function 582 583 # Remove explicit "self" from method parameters. 584 585 if is_method and argnames and argnames[0] == "self": 586 del argnames[0] 587 588 # Copy and propagate the parameters. 589 590 self.importer.function_parameters[function_name] = \ 591 self.function_parameters[function_name] = argnames[:] 592 593 # Define all arguments/parameters in the local namespace. 594 595 locals = \ 596 self.importer.function_locals[function_name] = \ 597 self.function_locals[function_name] = {} 598 599 # Insert "self" into method locals. 600 601 if is_method: 602 argnames.insert(0, "self") 603 604 # Define "self" in terms of the class if in a method. 605 # This does not diminish the need for type-narrowing in the deducer. 606 607 if argnames: 608 if self.in_class and not self.in_function and argnames[0] == "self": 609 locals[argnames[0]] = Reference("<instance>", self.in_class) 610 else: 611 locals[argnames[0]] = Reference("<var>") 612 613 for argname in argnames[1:]: 614 locals[argname] = Reference("<var>") 615 616 globals = self.scope_globals[function_name] = set() 617 618 # Process the defaults. 619 620 defaults = self.importer.function_defaults[function_name] = \ 621 self.function_defaults[function_name] = [] 622 623 for argname, default in compiler.ast.get_defaults(n): 624 if default: 625 626 # Obtain any reference for the default. 627 628 name_ref = self.process_structure_node(default) 629 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 630 631 # Reset conditional tracking to focus on the function contents. 632 633 in_conditional = self.in_conditional 634 self.in_conditional = False 635 636 in_function = self.in_function 637 self.in_function = function_name 638 639 self.enter_namespace(name) 640 641 # Track attribute usage within the namespace. 642 643 path = self.get_namespace_path() 644 645 self.start_tracking(locals) 646 self.process_structure_node(n.code) 647 self.stop_tracking() 648 649 # Exit to the parent. 650 651 self.exit_namespace() 652 653 # Update flags. 654 655 self.in_function = in_function 656 self.in_conditional = in_conditional 657 658 # Define the function using the appropriate name. 659 660 self.set_definition(name, "<function>") 661 662 # Where a function is set conditionally, assign the name. 663 664 if original_name: 665 self.process_assignment_for_function(original_name, name) 666 667 def process_global_node(self, n): 668 669 """ 670 Process the given "global" node 'n'. 671 """ 672 673 path = self.get_namespace_path() 674 675 if path != self.name: 676 self.scope_globals[path].update(n.names) 677 678 def process_if_node(self, n): 679 680 """ 681 Process the given "if" node 'n'. 682 """ 683 684 tracker = self.trackers[-1] 685 tracker.new_branchpoint() 686 687 for test, body in n.tests: 688 self.process_structure_node(test) 689 690 tracker.new_branch() 691 692 in_conditional = self.in_conditional 693 self.in_conditional = True 694 self.process_structure_node(body) 695 self.in_conditional = in_conditional 696 697 tracker.shelve_branch() 698 699 # Maintain a branch for the else clause. 700 701 tracker.new_branch() 702 if n.else_: 703 self.process_structure_node(n.else_) 704 tracker.shelve_branch() 705 706 tracker.merge_branches() 707 708 def process_import_node(self, n): 709 710 "Process the given import node 'n'." 711 712 path = self.get_namespace_path() 713 714 # Load the mentioned module. 715 716 for name, alias in n.names: 717 if name == self.name: 718 raise InspectError("Cannot import the current module.", path, n) 719 720 self.set_module(alias or name.split(".")[-1], name) 721 self.queue_module(name, True) 722 723 def process_invocation_node(self, n): 724 725 "Process the given invocation node 'n'." 726 727 path = self.get_namespace_path() 728 729 self.allocate_arguments(path, n.args) 730 731 try: 732 # Communicate to the invocation target expression that it forms the 733 # target of an invocation, potentially affecting attribute accesses. 734 735 in_invocation = self.in_invocation 736 self.in_invocation = True 737 738 # Process the expression, obtaining any identified reference. 739 740 name_ref = self.process_structure_node(n.node) 741 self.in_invocation = in_invocation 742 743 # Process the arguments. 744 745 for arg in n.args: 746 self.process_structure_node(arg) 747 748 # Detect class invocations. 749 750 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 751 return InstanceRef(name_ref.reference().instance_of()) 752 753 elif isinstance(name_ref, NameRef): 754 return InvocationRef(name_ref) 755 756 return None 757 758 finally: 759 self.deallocate_arguments(path, n.args) 760 761 def process_lambda_node(self, n): 762 763 "Process the given lambda node 'n'." 764 765 name = self.get_lambda_name() 766 self.process_function_node(n, name) 767 768 origin = self.get_object_path(name) 769 return ResolvedNameRef(name, Reference("<function>", origin)) 770 771 def process_logical_node(self, n): 772 773 "Process the given operator node 'n'." 774 775 self.process_operator_chain(n.nodes, self.process_structure_node) 776 777 def process_name_node(self, n): 778 779 "Process the given name node 'n'." 780 781 path = self.get_namespace_path() 782 783 # Special names that have already been identified. 784 785 if n.name.startswith("$"): 786 value = self.get_special(n.name) 787 if value: 788 return value 789 790 # Special case for operator functions introduced through code 791 # transformations. 792 793 if n.name.startswith("$op"): 794 795 # Obtain the location of the actual function defined in the operator 796 # package. 797 798 op = n.name[len("$op"):] 799 800 # Attempt to get a reference. 801 802 ref = self.import_name_from_module(op, "operator") 803 804 # Record the imported name and provide the resolved name reference. 805 806 value = ResolvedNameRef(n.name, ref) 807 self.set_special(n.name, value) 808 return value 809 810 # Special case for print operations. 811 812 elif n.name.startswith("$print"): 813 814 # Attempt to get a reference. 815 816 ref = self.get_builtin("print_") 817 818 # Record the imported name and provide the resolved name reference. 819 820 value = ResolvedNameRef(n.name, ref) 821 self.set_special(n.name, value) 822 return value 823 824 # Test for self usage, which is only allowed in methods. 825 826 if n.name == "self" and not (self.in_function and self.in_class): 827 raise InspectError("Use of self is only allowed in methods.", path, n) 828 829 # Record usage of the name. 830 831 self.record_name(n.name) 832 833 # Search for unknown names in non-function scopes immediately. 834 # External names in functions are resolved later. 835 836 ref = self.find_name(n.name) 837 if ref: 838 return ResolvedNameRef(n.name, ref) 839 840 # Explicitly-declared global names. 841 842 elif self.in_function and n.name in self.scope_globals[path]: 843 return NameRef(n.name) 844 845 # Examine other names. 846 847 else: 848 tracker = self.trackers[-1] 849 850 # Check local names. 851 852 branches = tracker.tracking_name(n.name) 853 854 # Local name. 855 856 if branches: 857 self.record_branches_for_access(branches, n.name, None) 858 access_number = self.record_access_details(n.name, None, False, False) 859 return LocalNameRef(n.name, access_number) 860 861 # Possible global or built-in name. 862 863 else: 864 return NameRef(n.name) 865 866 def process_operator_chain(self, nodes, fn): 867 868 """ 869 Process the given chain of 'nodes', applying 'fn' to each node or item. 870 Each node starts a new conditional region, effectively making a deeply- 871 nested collection of if-like statements. 872 """ 873 874 tracker = self.trackers[-1] 875 876 for item in nodes: 877 tracker.new_branchpoint() 878 tracker.new_branch() 879 fn(item) 880 881 for item in nodes[:-1]: 882 tracker.shelve_branch() 883 tracker.new_branch() 884 tracker.shelve_branch() 885 tracker.merge_branches() 886 887 tracker.shelve_branch() 888 tracker.merge_branches() 889 890 def process_try_node(self, n): 891 892 """ 893 Process the given "try...except" node 'n'. 894 """ 895 896 tracker = self.trackers[-1] 897 tracker.new_branchpoint() 898 899 self.process_structure_node(n.body) 900 901 for name, var, handler in n.handlers: 902 if name is not None: 903 self.process_structure_node(name) 904 905 # Any abandoned branches from the body can now be resumed in a new 906 # branch. 907 908 tracker.resume_abandoned_branches() 909 910 # Establish the local for the handler. 911 912 if var is not None: 913 self.process_structure_node(var) 914 if handler is not None: 915 self.process_structure_node(handler) 916 917 tracker.shelve_branch() 918 919 # The else clause maintains the usage from the body but without the 920 # abandoned branches since they would never lead to the else clause 921 # being executed. 922 923 if n.else_: 924 tracker.new_branch() 925 self.process_structure_node(n.else_) 926 tracker.shelve_branch() 927 928 # Without an else clause, a null branch propagates the successful 929 # outcome. 930 931 else: 932 tracker.new_branch() 933 tracker.shelve_branch() 934 935 tracker.merge_branches() 936 937 def process_try_finally_node(self, n): 938 939 """ 940 Process the given "try...finally" node 'n'. 941 """ 942 943 tracker = self.trackers[-1] 944 self.process_structure_node(n.body) 945 946 # Any abandoned branches from the body can now be resumed. 947 948 branches = tracker.resume_all_abandoned_branches() 949 self.process_structure_node(n.final) 950 951 # At the end of the finally clause, abandoned branches are discarded. 952 953 tracker.restore_active_branches(branches) 954 955 def process_while_node(self, n): 956 957 "Process the given while node 'n'." 958 959 tracker = self.trackers[-1] 960 tracker.new_branchpoint(loop_node=True) 961 962 # Evaluate any test or iterator outside the loop. 963 964 self.process_structure_node(n.test) 965 966 # Propagate attribute usage to branches. 967 968 tracker.new_branch(loop_node=True) 969 970 # Enter the loop. 971 972 in_conditional = self.in_conditional 973 self.in_conditional = True 974 self.process_structure_node(n.body) 975 self.in_conditional = in_conditional 976 977 # Continuing branches are resumed before any test. 978 979 tracker.resume_continuing_branches() 980 981 # Evaluate any continuation test within the body. 982 983 self.process_structure_node(n.test) 984 985 tracker.shelve_branch(loop_node=True) 986 987 # Support the non-looping condition. 988 989 tracker.new_branch() 990 tracker.shelve_branch() 991 992 tracker.merge_branches() 993 994 # Evaluate any else clause outside branches. 995 996 if n.else_: 997 self.process_structure_node(n.else_) 998 999 # Connect broken branches to the code after any loop. 1000 1001 tracker.resume_broken_branches() 1002 1003 # Branch tracking methods. 1004 1005 def start_tracking(self, names): 1006 1007 """ 1008 Start tracking attribute usage for names in the current namespace, 1009 immediately registering the given 'names'. 1010 """ 1011 1012 path = self.get_namespace_path() 1013 parent = self.trackers[-1] 1014 tracker = BranchTracker() 1015 self.trackers.append(tracker) 1016 1017 # Record the given names established as new branches. 1018 1019 tracker.assign_names(names) 1020 1021 def assign_name(self, name, name_ref): 1022 1023 "Assign to 'name' the given 'name_ref' in the current namespace." 1024 1025 name = self.get_name_for_tracking(name) 1026 self.trackers[-1].assign_names([name], [name_ref]) 1027 1028 def stop_tracking(self): 1029 1030 """ 1031 Stop tracking attribute usage, recording computed usage for the current 1032 namespace. 1033 """ 1034 1035 path = self.get_namespace_path() 1036 tracker = self.trackers.pop() 1037 self.record_assignments_for_access(tracker) 1038 1039 self.attr_usage[path] = tracker.get_all_usage() 1040 self.name_initialisers[path] = tracker.get_all_values() 1041 1042 def start_tracking_in_module(self): 1043 1044 "Start tracking attribute usage in the module." 1045 1046 tracker = BranchTracker() 1047 self.trackers.append(tracker) 1048 1049 def stop_tracking_in_module(self): 1050 1051 "Stop tracking attribute usage in the module." 1052 1053 tracker = self.trackers[0] 1054 self.record_assignments_for_access(tracker) 1055 self.attr_usage[self.name] = tracker.get_all_usage() 1056 self.name_initialisers[self.name] = tracker.get_all_values() 1057 1058 def record_assignments_for_access(self, tracker): 1059 1060 """ 1061 For the current path, use the given 'tracker' to record assignment 1062 version information for attribute accesses. 1063 """ 1064 1065 path = self.get_path_for_access() 1066 1067 if not self.attr_accessor_branches.has_key(path): 1068 return 1069 1070 init_item(self.attr_accessors, path, dict) 1071 attr_accessors = self.attr_accessors[path] 1072 1073 # Obtain the branches applying during each access. 1074 1075 for access, all_branches in self.attr_accessor_branches[path].items(): 1076 name, attrnames = access 1077 init_item(attr_accessors, access, list) 1078 1079 # Obtain the assignments applying to each branch. 1080 1081 for branches in all_branches: 1082 positions = tracker.get_assignment_positions_for_branches(name, branches) 1083 1084 # Detect missing name information. 1085 1086 if None in positions: 1087 globals = self.global_attr_accesses.get(path) 1088 accesses = globals and globals.get(name) 1089 if not accesses: 1090 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1091 self.get_namespace_path(), name) 1092 positions.remove(None) 1093 1094 attr_accessors[access].append(positions) 1095 1096 def record_branches_for_access(self, branches, name, attrnames): 1097 1098 """ 1099 Record the given 'branches' for an access involving the given 'name' and 1100 'attrnames'. 1101 """ 1102 1103 access = name, attrnames 1104 path = self.get_path_for_access() 1105 1106 init_item(self.attr_accessor_branches, path, dict) 1107 attr_accessor_branches = self.attr_accessor_branches[path] 1108 1109 init_item(attr_accessor_branches, access, list) 1110 attr_accessor_branches[access].append(branches) 1111 1112 def record_access_details(self, name, attrnames, assignment, invocation): 1113 1114 """ 1115 For the given 'name' and 'attrnames', record an access indicating 1116 whether 'assignment' is occurring. 1117 1118 These details correspond to accesses otherwise recorded by the attribute 1119 accessor and attribute access dictionaries. 1120 """ 1121 1122 access = name, attrnames 1123 path = self.get_path_for_access() 1124 1125 init_item(self.attr_access_modifiers, path, dict) 1126 init_item(self.attr_access_modifiers[path], access, list) 1127 1128 access_number = len(self.attr_access_modifiers[path][access]) 1129 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1130 return access_number 1131 1132 def record_global_access_details(self, name, attrnames): 1133 1134 """ 1135 Record details of a global access via the given 'name' involving the 1136 indicated 'attrnames'. 1137 """ 1138 1139 path = self.get_namespace_path() 1140 1141 init_item(self.global_attr_accesses, path, dict) 1142 init_item(self.global_attr_accesses[path], name, set) 1143 self.global_attr_accesses[path][name].add(attrnames) 1144 1145 # Namespace modification. 1146 1147 def record_name(self, name): 1148 1149 "Record the use of 'name' in a namespace." 1150 1151 path = self.get_namespace_path() 1152 init_item(self.names_used, path, set) 1153 self.names_used[path].add(name) 1154 1155 def set_module(self, name, module_name): 1156 1157 """ 1158 Set a module in the current namespace using the given 'name' associated 1159 with the corresponding 'module_name'. 1160 """ 1161 1162 if name: 1163 self.set_general_local(name, Reference("<module>", module_name)) 1164 1165 def set_definition(self, name, kind): 1166 1167 """ 1168 Set the definition having the given 'name' and 'kind'. 1169 1170 Definitions are set in the static namespace hierarchy, but they can also 1171 be recorded for function locals. 1172 """ 1173 1174 if self.is_global(name): 1175 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1176 self.get_namespace_path(), name) 1177 1178 path = self.get_object_path(name) 1179 self.set_object(path, kind) 1180 1181 ref = self.get_object(path) 1182 if ref.get_kind() == "<var>": 1183 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1184 self.get_namespace_path(), name) 1185 1186 if not self.is_global(name) and self.in_function: 1187 self.set_function_local(name, ref) 1188 1189 def set_function_local(self, name, ref=None): 1190 1191 "Set the local with the given 'name' and optional 'ref'." 1192 1193 locals = self.function_locals[self.get_namespace_path()] 1194 multiple = not ref or locals.has_key(name) and locals[name] != ref 1195 locals[name] = multiple and Reference("<var>") or ref 1196 1197 def assign_general_local(self, name, name_ref): 1198 1199 """ 1200 Set for 'name' the given 'name_ref', recording the name for attribute 1201 usage tracking. 1202 """ 1203 1204 self.set_general_local(name, name_ref) 1205 self.assign_name(name, name_ref) 1206 1207 def set_general_local(self, name, value=None): 1208 1209 """ 1210 Set the 'name' with optional 'value' in any kind of local namespace, 1211 where the 'value' should be a reference if specified. 1212 """ 1213 1214 init_value = self.get_initialising_value(value) 1215 1216 # Module global names. 1217 1218 if self.is_global(name): 1219 path = self.get_global_path(name) 1220 self.set_object(path, init_value) 1221 1222 # Function local names. 1223 1224 elif self.in_function: 1225 path = self.get_object_path(name) 1226 self.set_function_local(name, init_value) 1227 1228 # Other namespaces (classes). 1229 1230 else: 1231 path = self.get_object_path(name) 1232 self.set_name(name, init_value) 1233 1234 def set_name(self, name, ref=None): 1235 1236 "Attach the 'name' with optional 'ref' to the current namespace." 1237 1238 self.set_object(self.get_object_path(name), ref) 1239 1240 def set_instance_attr(self, name, ref=None): 1241 1242 """ 1243 Add an instance attribute of the given 'name' to the current class, 1244 using the optional 'ref'. 1245 """ 1246 1247 init_item(self.instance_attrs, self.in_class, set) 1248 self.instance_attrs[self.in_class].add(name) 1249 1250 if ref: 1251 init_item(self.instance_attr_constants, self.in_class, dict) 1252 self.instance_attr_constants[self.in_class][name] = ref 1253 1254 def get_initialising_value(self, value): 1255 1256 "Return a suitable initialiser reference for 'value'." 1257 1258 # Includes LiteralSequenceRef, ResolvedNameRef... 1259 1260 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1261 return value.reference() 1262 1263 # In general, invocations do not produce known results. However, the 1264 # name initialisers are resolved once a module has been inspected. 1265 1266 elif isinstance(value, InvocationRef): 1267 return value.reference() 1268 1269 else: 1270 return value 1271 1272 # Static, program-relative naming. 1273 1274 def find_name(self, name): 1275 1276 """ 1277 Return the qualified name for the given 'name' used in the current 1278 non-function namespace. 1279 """ 1280 1281 path = self.get_namespace_path() 1282 ref = None 1283 1284 if not self.in_function and name not in predefined_constants: 1285 if self.in_class: 1286 ref = self.get_object(self.get_object_path(name), False) 1287 if not ref: 1288 ref = self.get_global_or_builtin(name) 1289 1290 return ref 1291 1292 def get_class(self, node): 1293 1294 """ 1295 Use the given 'node' to obtain the identity of a class. Return a 1296 reference for the class. Unresolved dependencies are permitted and must 1297 be resolved later. 1298 """ 1299 1300 ref = self._get_class(node) 1301 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1302 1303 def _get_class(self, node): 1304 1305 """ 1306 Use the given 'node' to find a class definition. Return a reference to 1307 the class. 1308 """ 1309 1310 if isinstance(node, compiler.ast.Getattr): 1311 1312 # Obtain the identity of the access target. 1313 1314 ref = self._get_class(node.expr) 1315 1316 # Where the target is a class or module, obtain the identity of the 1317 # attribute. 1318 1319 if ref.has_kind(["<function>", "<var>"]): 1320 return None 1321 else: 1322 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1323 return self.get_object(attrname) 1324 1325 # Names can be module-level or built-in. 1326 1327 elif isinstance(node, compiler.ast.Name): 1328 1329 # Record usage of the name and attempt to identify it. 1330 1331 self.record_name(node.name) 1332 return self.find_name(node.name) 1333 else: 1334 return None 1335 1336 def get_constant(self, name, value): 1337 1338 "Return a constant reference for the given type 'name' and 'value'." 1339 1340 ref = self.get_builtin_class(name) 1341 return self.get_constant_reference(ref, value) 1342 1343 def get_literal_instance(self, n, name): 1344 1345 "For node 'n', return a reference to an instance of 'name'." 1346 1347 # Get a reference to the built-in class. 1348 1349 ref = self.get_builtin_class(name) 1350 1351 # Obtain the details of the literal itself. 1352 # An alias to the type is generated for sequences. 1353 1354 if name in ("dict", "list", "tuple"): 1355 self.set_special_literal(name, ref) 1356 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1357 1358 # Constant values are independently recorded. 1359 1360 else: 1361 return self.get_constant_reference(ref, n.value) 1362 1363 # Special names. 1364 1365 def get_special(self, name): 1366 1367 "Return any stored value for the given special 'name'." 1368 1369 return self.special.get(name) 1370 1371 def set_special(self, name, value): 1372 1373 """ 1374 Set a special 'name' that merely tracks the use of an implicit object 1375 'value'. 1376 """ 1377 1378 self.special[name] = value 1379 1380 def set_special_literal(self, name, ref): 1381 1382 """ 1383 Set a special name for the literal type 'name' having type 'ref'. Such 1384 special names provide a way of referring to literal object types. 1385 """ 1386 1387 literal_name = "$L%s" % name 1388 value = ResolvedNameRef(literal_name, ref) 1389 self.set_special(literal_name, value) 1390 1391 # Functions and invocations. 1392 1393 def set_invocation_usage(self): 1394 1395 """ 1396 Discard the current invocation storage figures, retaining the maximum 1397 values. 1398 """ 1399 1400 for path, (current, maximum) in self.function_targets.items(): 1401 self.importer.function_targets[path] = self.function_targets[path] = maximum 1402 1403 for path, (current, maximum) in self.function_arguments.items(): 1404 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1405 1406 def allocate_arguments(self, path, args): 1407 1408 """ 1409 Allocate temporary argument storage using current and maximum 1410 requirements for the given 'path' and 'args'. 1411 """ 1412 1413 init_item(self.function_targets, path, lambda: [0, 0]) 1414 t = self.function_targets[path] 1415 t[0] += 1 1416 t[1] = max(t[0], t[1]) 1417 1418 init_item(self.function_arguments, path, lambda: [0, 0]) 1419 t = self.function_arguments[path] 1420 t[0] += len(args) + 1 1421 t[1] = max(t[0], t[1]) 1422 1423 def deallocate_arguments(self, path, args): 1424 1425 "Deallocate temporary argument storage for the given 'path' and 'args'." 1426 1427 self.function_targets[path][0] -= 1 1428 self.function_arguments[path][0] -= len(args) + 1 1429 1430 # vim: tabstop=4 expandtab shiftwidth=4