1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 self.global_attr_accesses = {} 47 48 # Usage tracking. 49 50 self.trackers = [] 51 self.attr_accessor_branches = {} 52 53 def __repr__(self): 54 return "InspectedModule(%r, %r)" % (self.name, self.importer) 55 56 # Principal methods. 57 58 def parse(self, filename): 59 60 "Parse the file having the given 'filename'." 61 62 self.parse_file(filename) 63 64 # Inspect the module. 65 66 self.start_tracking_in_module() 67 68 # Detect and record imports and globals declared in the module. 69 70 self.assign_general_local("__name__", self.get_constant("str", self.name)) 71 self.assign_general_local("__file__", self.get_constant("str", filename)) 72 self.process_structure(self.astnode) 73 74 # Set the class of the module after the definition has occurred. 75 76 ref = self.get_builtin("object") 77 self.set_name("__class__", ref) 78 79 # Get module-level attribute usage details. 80 81 self.stop_tracking_in_module() 82 83 # Collect external name references. 84 85 self.collect_names() 86 87 def complete(self): 88 89 "Complete the module inspection." 90 91 # Resolve names not definitively mapped to objects. 92 93 self.resolve() 94 95 # Define the invocation requirements in each namespace. 96 97 self.set_invocation_usage() 98 99 # Propagate to the importer information needed in subsequent activities. 100 101 self.propagate() 102 103 # Accessory methods. 104 105 def collect_names(self): 106 107 "Collect the names used by each scope." 108 109 for path in self.names_used.keys(): 110 self.collect_names_for_path(path) 111 112 def collect_names_for_path(self, path): 113 114 """ 115 Collect the names used by the given 'path'. These are propagated to the 116 importer in advance of any dependency resolution. 117 """ 118 119 names = self.names_used.get(path) 120 if not names: 121 return 122 123 in_function = self.function_locals.has_key(path) 124 125 for name in names: 126 if name in predefined_constants or in_function and name in self.function_locals[path]: 127 continue 128 129 # Find local definitions (within dynamic namespaces). 130 131 key = "%s.%s" % (path, name) 132 ref = self.get_resolved_object(key) 133 if ref: 134 self.set_name_reference(key, ref) 135 continue 136 137 # Find global or known built-in definitions. 138 139 ref = self.get_resolved_global_or_builtin(name) 140 if ref: 141 self.set_name_reference(key, ref) 142 continue 143 144 # Find presumed built-in definitions. 145 146 ref = self.get_builtin(name) 147 self.set_name_reference(key, ref) 148 149 def set_name_reference(self, path, ref): 150 151 "Map the given name 'path' to 'ref'." 152 153 self.importer.all_name_references[path] = self.name_references[path] = ref 154 155 def get_resolved_global_or_builtin(self, name): 156 157 "Return the resolved global or built-in object with the given 'name'." 158 159 # In some circumstances, the name is neither global nor recognised by 160 # the importer. It is then assumed to be a general built-in. 161 162 return self.get_global(name) or \ 163 self.importer.get_object("__builtins__.%s" % name) 164 165 # Module structure traversal. 166 167 def process_structure_node(self, n): 168 169 "Process the individual node 'n'." 170 171 # Module global detection. 172 173 if isinstance(n, compiler.ast.Global): 174 self.process_global_node(n) 175 176 # Module import declarations. 177 178 elif isinstance(n, compiler.ast.From): 179 self.process_from_node(n) 180 181 elif isinstance(n, compiler.ast.Import): 182 self.process_import_node(n) 183 184 # Nodes using operator module functions. 185 186 elif isinstance(n, compiler.ast.Operator): 187 return self.process_operator_node(n) 188 189 elif isinstance(n, compiler.ast.AugAssign): 190 self.process_augassign_node(n) 191 192 elif isinstance(n, compiler.ast.Compare): 193 return self.process_compare_node(n) 194 195 elif isinstance(n, compiler.ast.Slice): 196 return self.process_slice_node(n) 197 198 elif isinstance(n, compiler.ast.Sliceobj): 199 return self.process_sliceobj_node(n) 200 201 elif isinstance(n, compiler.ast.Subscript): 202 return self.process_subscript_node(n) 203 204 # Namespaces within modules. 205 206 elif isinstance(n, compiler.ast.Class): 207 self.process_class_node(n) 208 209 elif isinstance(n, compiler.ast.Function): 210 self.process_function_node(n, n.name) 211 212 elif isinstance(n, compiler.ast.Lambda): 213 return self.process_lambda_node(n) 214 215 # Assignments. 216 217 elif isinstance(n, compiler.ast.Assign): 218 219 # Handle each assignment node. 220 221 for node in n.nodes: 222 self.process_assignment_node(node, n.expr) 223 224 # Assignments within non-Assign nodes. 225 226 elif isinstance(n, compiler.ast.AssName): 227 self.process_assignment_node(n, None) 228 229 elif isinstance(n, compiler.ast.AssAttr): 230 self.process_attribute_access(n) 231 232 # Accesses. 233 234 elif isinstance(n, compiler.ast.Getattr): 235 return self.process_attribute_access(n) 236 237 # Name recording for later testing. 238 239 elif isinstance(n, compiler.ast.Name): 240 return self.process_name_node(n) 241 242 # Conditional statement tracking. 243 244 elif isinstance(n, compiler.ast.For): 245 self.process_for_node(n) 246 247 elif isinstance(n, compiler.ast.While): 248 self.process_while_node(n) 249 250 elif isinstance(n, compiler.ast.If): 251 self.process_if_node(n) 252 253 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 254 return self.process_logical_node(n) 255 256 # Exception control-flow tracking. 257 258 elif isinstance(n, compiler.ast.TryExcept): 259 self.process_try_node(n) 260 261 elif isinstance(n, compiler.ast.TryFinally): 262 self.process_try_finally_node(n) 263 264 # Control-flow modification statements. 265 266 elif isinstance(n, compiler.ast.Break): 267 self.trackers[-1].suspend_broken_branch() 268 269 elif isinstance(n, compiler.ast.Continue): 270 self.trackers[-1].suspend_continuing_branch() 271 272 elif isinstance(n, compiler.ast.Raise): 273 self.process_structure(n) 274 self.trackers[-1].abandon_branch() 275 276 elif isinstance(n, compiler.ast.Return): 277 self.process_structure(n) 278 self.trackers[-1].abandon_returning_branch() 279 280 # Invocations. 281 282 elif isinstance(n, compiler.ast.CallFunc): 283 return self.process_invocation_node(n) 284 285 # Constant usage. 286 287 elif isinstance(n, compiler.ast.Const): 288 return self.get_literal_instance(n, n.value.__class__.__name__) 289 290 elif isinstance(n, compiler.ast.Dict): 291 return self.get_literal_instance(n, "dict") 292 293 elif isinstance(n, compiler.ast.List): 294 return self.get_literal_instance(n, "list") 295 296 elif isinstance(n, compiler.ast.Tuple): 297 return self.get_literal_instance(n, "tuple") 298 299 # Unsupported nodes. 300 301 elif isinstance(n, compiler.ast.GenExpr): 302 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 303 304 elif isinstance(n, compiler.ast.IfExp): 305 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 306 307 elif isinstance(n, compiler.ast.ListComp): 308 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 309 310 # All other nodes are processed depth-first. 311 312 else: 313 self.process_structure(n) 314 315 # By default, no expression details are returned. 316 317 return None 318 319 # Specific node handling. 320 321 def process_assignment_node(self, n, expr): 322 323 "Process the individual node 'n' to be assigned the contents of 'expr'." 324 325 # Names and attributes are assigned the entire expression. 326 327 if isinstance(n, compiler.ast.AssName): 328 if n.name == "self": 329 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 330 331 name_ref = expr and self.process_structure_node(expr) 332 333 # Name assignments populate either function namespaces or the 334 # general namespace hierarchy. 335 336 self.assign_general_local(n.name, name_ref) 337 338 # Record usage of the name. 339 340 self.record_name(n.name) 341 342 elif isinstance(n, compiler.ast.AssAttr): 343 if expr: self.process_structure_node(expr) 344 self.process_attribute_access(n) 345 346 # Lists and tuples are matched against the expression and their 347 # items assigned to expression items. 348 349 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 350 self.process_assignment_node_items(n, expr) 351 352 # Slices and subscripts are permitted within assignment nodes. 353 354 elif isinstance(n, compiler.ast.Slice): 355 self.process_slice_node(n, expr) 356 357 elif isinstance(n, compiler.ast.Subscript): 358 self.process_subscript_node(n, expr) 359 360 def process_attribute_access(self, n): 361 362 "Process the given attribute access node 'n'." 363 364 # Obtain any completed chain and return the reference to it. 365 366 name_ref = self.process_attribute_chain(n) 367 if self.have_access_expression(n): 368 return name_ref 369 370 # Where the start of the chain of attributes has been reached, determine 371 # the complete access. 372 373 # Given a non-access node, this chain can be handled in its entirety, 374 # either being name-based and thus an access rooted on a name, or being 375 # based on some other node and thus an anonymous access of some kind. 376 377 path = self.get_namespace_path() 378 379 # Start with the the full attribute chain. 380 381 remaining = self.attrs 382 attrnames = ".".join(remaining) 383 384 # If the accessor cannot be identified, or where attributes 385 # remain in an attribute chain, record the anonymous accesses. 386 387 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 388 389 assignment = isinstance(n, compiler.ast.AssAttr) 390 391 init_item(self.attr_accesses, path, set) 392 self.attr_accesses[path].add(attrnames) 393 394 self.record_access_details(None, attrnames, assignment) 395 del self.attrs[0] 396 return 397 398 # Name-based accesses will handle the first attribute in a 399 # chain. 400 401 else: 402 attrname = remaining[0] 403 404 # Attribute assignments are used to identify instance attributes. 405 406 if isinstance(n, compiler.ast.AssAttr) and \ 407 self.in_class and self.in_function and n.expr.name == "self": 408 409 self.set_instance_attr(attrname) 410 411 # Record attribute usage using any name local to this namespace, 412 # if assigned in the namespace, or using an external name 413 # (presently just globals within classes). 414 415 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 416 tracker = self.trackers[-1] 417 418 immediate_access = len(self.attrs) == 1 419 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 420 421 del self.attrs[0] 422 423 # Record global-based chains for subsequent resolution. 424 425 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 426 not self.in_function 427 428 if is_global: 429 self.record_global_access_details(name, attrnames) 430 431 # Make sure the name is being tracked: global names will not 432 # already be initialised in a branch and must be added 433 # explicitly. 434 435 if not tracker.have_name(name): 436 tracker.assign_names([name]) 437 if self.in_function: 438 self.scope_globals[path].add(name) 439 440 # Record attribute usage in the tracker, and record the branch 441 # information for the access. 442 443 branches = tracker.use_attribute(name, attrname) 444 445 if not branches: 446 print >>sys.stderr, "In %s, name %s is accessed using %s before an assignment." % ( 447 path, name, attrname) 448 branches = tracker.use_attribute(name, attrname) 449 450 self.record_branches_for_access(branches, name, attrnames) 451 access_number = self.record_access_details(name, attrnames, assignment) 452 return AccessRef(name, attrnames, access_number) 453 454 def process_class_node(self, n): 455 456 "Process the given class node 'n'." 457 458 path = self.get_namespace_path() 459 460 # To avoid notions of class "versions" where the same definition 461 # might be parameterised with different state and be referenced 462 # elsewhere (as base classes, for example), classes in functions or 463 # conditions are forbidden. 464 465 if self.in_function or self.in_conditional: 466 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 467 path, n.name) 468 return 469 470 # Resolve base classes. 471 472 bases = [] 473 474 for base in n.bases: 475 base_class = self.get_class(base) 476 477 if not base_class: 478 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 479 path, n.name, base) 480 return 481 else: 482 bases.append(base_class) 483 484 # Record bases for the class and retain the class name. 485 486 class_name = self.get_object_path(n.name) 487 488 if not bases and class_name != "__builtins__.core.object": 489 ref = self.get_object("__builtins__.object") 490 bases.append(ref) 491 492 self.importer.classes[class_name] = self.classes[class_name] = bases 493 self.importer.subclasses[class_name] = set() 494 self.scope_globals[class_name] = set() 495 496 # Set the definition before entering the namespace rather than 497 # afterwards because methods may reference it. In normal Python, 498 # a class is not accessible until the definition is complete, but 499 # methods can generally reference it since upon being called the 500 # class will already exist. 501 502 self.set_definition(n.name, "<class>") 503 504 in_class = self.in_class 505 self.in_class = class_name 506 self.set_instance_attr("__class__", Reference("<class>", class_name)) 507 self.enter_namespace(n.name) 508 self.set_name("__fn__") # special instantiator attribute 509 self.set_name("__args__") # special instantiator attribute 510 self.assign_general_local("__name__", self.get_constant("str", class_name)) 511 self.process_structure_node(n.code) 512 self.exit_namespace() 513 self.in_class = in_class 514 515 def process_from_node(self, n): 516 517 "Process the given node 'n', importing from another module." 518 519 path = self.get_namespace_path() 520 521 module_name, names = self.get_module_name(n) 522 if module_name == self.name: 523 raise InspectError("Cannot import from the current module.", path, n) 524 525 self.queue_module(module_name) 526 527 # Attempt to obtain the referenced objects. 528 529 for name, alias in n.names: 530 if name == "*": 531 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 532 533 # Explicit names. 534 535 ref = self.import_name_from_module(name, module_name) 536 value = ResolvedNameRef(alias or name, ref) 537 self.set_general_local(alias or name, value) 538 539 def process_function_node(self, n, name): 540 541 """ 542 Process the given function or lambda node 'n' with the given 'name'. 543 """ 544 545 is_lambda = isinstance(n, compiler.ast.Lambda) 546 547 # Where a function is declared conditionally, use a separate name for 548 # the definition, and assign the definition to the stated name. 549 550 if (self.in_conditional or self.in_function) and not is_lambda: 551 original_name = name 552 name = self.get_lambda_name() 553 else: 554 original_name = None 555 556 # Initialise argument and local records. 557 558 function_name = self.get_object_path(name) 559 argnames = get_argnames(n.argnames) 560 is_method = self.in_class and not self.in_function 561 562 # Remove explicit "self" from method parameters. 563 564 if is_method and argnames and argnames[0] == "self": 565 del argnames[0] 566 567 # Copy and propagate the parameters. 568 569 self.importer.function_parameters[function_name] = \ 570 self.function_parameters[function_name] = argnames[:] 571 572 # Define all arguments/parameters in the local namespace. 573 574 locals = self.function_locals[function_name] = {} 575 576 # Insert "self" into method locals. 577 578 if is_method: 579 argnames.insert(0, "self") 580 581 # Define "self" in terms of the class if in a method. 582 # This does not diminish the need for type-narrowing in the deducer. 583 584 if argnames: 585 if self.in_class and not self.in_function and argnames[0] == "self": 586 locals[argnames[0]] = Reference("<instance>", self.in_class) 587 else: 588 locals[argnames[0]] = Reference("<var>") 589 590 for argname in argnames[1:]: 591 locals[argname] = Reference("<var>") 592 593 globals = self.scope_globals[function_name] = set() 594 595 # Process the defaults. 596 597 defaults = self.importer.function_defaults[function_name] = \ 598 self.function_defaults[function_name] = [] 599 600 for argname, default in compiler.ast.get_defaults(n): 601 if default: 602 603 # Obtain any reference for the default. 604 605 name_ref = self.process_structure_node(default) 606 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 607 608 # Reset conditional tracking to focus on the function contents. 609 610 in_conditional = self.in_conditional 611 self.in_conditional = False 612 613 in_function = self.in_function 614 self.in_function = function_name 615 616 self.enter_namespace(name) 617 618 # Track attribute usage within the namespace. 619 620 path = self.get_namespace_path() 621 622 self.start_tracking(locals) 623 self.process_structure_node(n.code) 624 self.stop_tracking() 625 626 # Exit to the parent. 627 628 self.exit_namespace() 629 630 # Update flags. 631 632 self.in_function = in_function 633 self.in_conditional = in_conditional 634 635 # Define the function using the appropriate name. 636 637 self.set_definition(name, "<function>") 638 639 # Where a function is set conditionally, assign the name. 640 641 if original_name: 642 self.process_assignment_for_function(original_name, name) 643 644 def process_global_node(self, n): 645 646 """ 647 Process the given "global" node 'n'. 648 """ 649 650 path = self.get_namespace_path() 651 652 if path != self.name: 653 self.scope_globals[path].update(n.names) 654 655 def process_if_node(self, n): 656 657 """ 658 Process the given "if" node 'n'. 659 """ 660 661 tracker = self.trackers[-1] 662 tracker.new_branchpoint() 663 664 for test, body in n.tests: 665 self.process_structure_node(test) 666 667 tracker.new_branch() 668 669 in_conditional = self.in_conditional 670 self.in_conditional = True 671 self.process_structure_node(body) 672 self.in_conditional = in_conditional 673 674 tracker.shelve_branch() 675 676 # Maintain a branch for the else clause. 677 678 tracker.new_branch() 679 if n.else_: 680 self.process_structure_node(n.else_) 681 tracker.shelve_branch() 682 683 tracker.merge_branches() 684 685 def process_import_node(self, n): 686 687 "Process the given import node 'n'." 688 689 path = self.get_namespace_path() 690 691 # Load the mentioned module. 692 693 for name, alias in n.names: 694 if name == self.name: 695 raise InspectError("Cannot import the current module.", path, n) 696 697 self.set_module(alias or name.split(".")[-1], name) 698 self.queue_module(name, True) 699 700 def process_invocation_node(self, n): 701 702 "Process the given invocation node 'n'." 703 704 path = self.get_namespace_path() 705 706 self.allocate_arguments(path, n.args) 707 708 try: 709 # Process the expression, obtaining any identified reference. 710 711 name_ref = self.process_structure_node(n.node) 712 713 # Process the arguments. 714 715 for arg in n.args: 716 self.process_structure_node(arg) 717 718 # Detect class invocations. 719 720 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 721 return InstanceRef(name_ref.reference().instance_of()) 722 723 elif isinstance(name_ref, NameRef): 724 return InvocationRef(name_ref) 725 726 return None 727 728 finally: 729 self.deallocate_arguments(path, n.args) 730 731 def process_lambda_node(self, n): 732 733 "Process the given lambda node 'n'." 734 735 name = self.get_lambda_name() 736 self.process_function_node(n, name) 737 738 origin = self.get_object_path(name) 739 return ResolvedNameRef(name, Reference("<function>", origin)) 740 741 def process_logical_node(self, n): 742 743 "Process the given operator node 'n'." 744 745 self.process_operator_chain(n.nodes, self.process_structure_node) 746 747 def process_name_node(self, n): 748 749 "Process the given name node 'n'." 750 751 path = self.get_namespace_path() 752 753 # Special names. 754 755 if n.name.startswith("$"): 756 value = self.get_special(n.name) 757 if value: 758 return value 759 760 # Special case for operator functions introduced through code 761 # transformations. 762 763 if n.name.startswith("$op"): 764 765 # Obtain the location of the actual function defined in the operator 766 # package. 767 768 op = n.name[len("$op"):] 769 770 # Attempt to get a reference. 771 772 ref = self.import_name_from_module(op, "operator") 773 self.add_deferred(ref) 774 775 # Record the imported name and provide the resolved name reference. 776 777 value = ResolvedNameRef(n.name, ref) 778 self.set_special(n.name, value) 779 return value 780 781 # Test for self usage, which is only allowed in methods. 782 783 if n.name == "self" and not (self.in_function and self.in_class): 784 raise InspectError("Use of self is only allowed in methods.", path, n) 785 786 # Record usage of the name. 787 788 self.record_name(n.name) 789 790 # Search for unknown names in non-function scopes immediately. 791 # External names in functions are resolved later. 792 793 ref = self.find_name(n.name) 794 if ref: 795 return ResolvedNameRef(n.name, ref) 796 797 # Explicitly-declared global names. 798 799 elif self.in_function and n.name in self.scope_globals[path]: 800 return NameRef(n.name) 801 802 # Examine other names. 803 804 else: 805 tracker = self.trackers[-1] 806 807 # Check local names. 808 809 branches = tracker.tracking_name(n.name) 810 811 # Local name. 812 813 if branches: 814 self.record_branches_for_access(branches, n.name, None) 815 access_number = self.record_access_details(n.name, None, False) 816 return LocalNameRef(n.name, access_number) 817 818 # Possible global or built-in name. 819 820 else: 821 return NameRef(n.name) 822 823 def process_operator_chain(self, nodes, fn): 824 825 """ 826 Process the given chain of 'nodes', applying 'fn' to each node or item. 827 Each node starts a new conditional region, effectively making a deeply- 828 nested collection of if-like statements. 829 """ 830 831 tracker = self.trackers[-1] 832 833 for item in nodes: 834 tracker.new_branchpoint() 835 tracker.new_branch() 836 fn(item) 837 838 for item in nodes[:-1]: 839 tracker.shelve_branch() 840 tracker.new_branch() 841 tracker.shelve_branch() 842 tracker.merge_branches() 843 844 tracker.shelve_branch() 845 tracker.merge_branches() 846 847 def process_try_node(self, n): 848 849 """ 850 Process the given "try...except" node 'n'. 851 """ 852 853 tracker = self.trackers[-1] 854 tracker.new_branchpoint() 855 856 self.process_structure_node(n.body) 857 858 for name, var, handler in n.handlers: 859 if name is not None: 860 self.process_structure_node(name) 861 862 # Any abandoned branches from the body can now be resumed in a new 863 # branch. 864 865 tracker.resume_abandoned_branches() 866 867 # Establish the local for the handler. 868 869 if var is not None: 870 self.process_structure_node(var) 871 if handler is not None: 872 self.process_structure_node(handler) 873 874 tracker.shelve_branch() 875 876 # The else clause maintains the usage from the body but without the 877 # abandoned branches since they would never lead to the else clause 878 # being executed. 879 880 if n.else_: 881 tracker.new_branch() 882 self.process_structure_node(n.else_) 883 tracker.shelve_branch() 884 885 # Without an else clause, a null branch propagates the successful 886 # outcome. 887 888 else: 889 tracker.new_branch() 890 tracker.shelve_branch() 891 892 tracker.merge_branches() 893 894 def process_try_finally_node(self, n): 895 896 """ 897 Process the given "try...finally" node 'n'. 898 """ 899 900 tracker = self.trackers[-1] 901 self.process_structure_node(n.body) 902 903 # Any abandoned branches from the body can now be resumed. 904 905 branches = tracker.resume_all_abandoned_branches() 906 self.process_structure_node(n.final) 907 908 # At the end of the finally clause, abandoned branches are discarded. 909 910 tracker.restore_active_branches(branches) 911 912 def process_while_node(self, n): 913 914 "Process the given while node 'n'." 915 916 tracker = self.trackers[-1] 917 tracker.new_branchpoint(loop_node=True) 918 919 # Evaluate any test or iterator outside the loop. 920 921 self.process_structure_node(n.test) 922 923 # Propagate attribute usage to branches. 924 925 tracker.new_branch(loop_node=True) 926 927 # Enter the loop. 928 929 in_conditional = self.in_conditional 930 self.in_conditional = True 931 self.process_structure_node(n.body) 932 self.in_conditional = in_conditional 933 934 # Continuing branches are resumed before any test. 935 936 tracker.resume_continuing_branches() 937 938 # Evaluate any continuation test within the body. 939 940 self.process_structure_node(n.test) 941 942 tracker.shelve_branch(loop_node=True) 943 944 # Support the non-looping condition. 945 946 tracker.new_branch() 947 tracker.shelve_branch() 948 949 tracker.merge_branches() 950 951 # Evaluate any else clause outside branches. 952 953 if n.else_: 954 self.process_structure_node(n.else_) 955 956 # Connect broken branches to the code after any loop. 957 958 tracker.resume_broken_branches() 959 960 # Branch tracking methods. 961 962 def start_tracking(self, names): 963 964 """ 965 Start tracking attribute usage for names in the current namespace, 966 immediately registering the given 'names'. 967 """ 968 969 path = self.get_namespace_path() 970 parent = self.trackers[-1] 971 tracker = BranchTracker() 972 self.trackers.append(tracker) 973 974 # Record the given names established as new branches. 975 976 tracker.assign_names(names) 977 978 def assign_name(self, name, name_ref): 979 980 "Assign to 'name' the given 'name_ref' in the current namespace." 981 982 name = self.get_name_for_tracking(name) 983 self.trackers[-1].assign_names([name], [name_ref]) 984 985 def stop_tracking(self): 986 987 """ 988 Stop tracking attribute usage, recording computed usage for the current 989 namespace. 990 """ 991 992 path = self.get_namespace_path() 993 tracker = self.trackers.pop() 994 self.record_assignments_for_access(tracker) 995 996 self.attr_usage[path] = tracker.get_all_usage() 997 self.name_initialisers[path] = tracker.get_all_values() 998 999 def start_tracking_in_module(self): 1000 1001 "Start tracking attribute usage in the module." 1002 1003 tracker = BranchTracker() 1004 self.trackers.append(tracker) 1005 1006 def stop_tracking_in_module(self): 1007 1008 "Stop tracking attribute usage in the module." 1009 1010 tracker = self.trackers[0] 1011 self.record_assignments_for_access(tracker) 1012 self.attr_usage[self.name] = tracker.get_all_usage() 1013 self.name_initialisers[self.name] = tracker.get_all_values() 1014 1015 def record_assignments_for_access(self, tracker): 1016 1017 """ 1018 For the current path, use the given 'tracker' to record assignment 1019 version information for attribute accesses. 1020 """ 1021 1022 path = self.get_path_for_access() 1023 1024 if not self.attr_accessor_branches.has_key(path): 1025 return 1026 1027 init_item(self.attr_accessors, path, dict) 1028 attr_accessors = self.attr_accessors[path] 1029 1030 # Obtain the branches applying during each access. 1031 1032 for access, all_branches in self.attr_accessor_branches[path].items(): 1033 name, attrnames = access 1034 init_item(attr_accessors, access, list) 1035 1036 # Obtain the assignments applying to each branch. 1037 1038 for branches in all_branches: 1039 positions = tracker.get_assignment_positions_for_branches(name, branches) 1040 1041 # Detect missing name information. 1042 1043 if None in positions: 1044 globals = self.global_attr_accesses.get(path) 1045 accesses = globals and globals.get(name) 1046 if not accesses: 1047 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1048 self.get_namespace_path(), name) 1049 positions.remove(None) 1050 1051 attr_accessors[access].append(positions) 1052 1053 def record_branches_for_access(self, branches, name, attrnames): 1054 1055 """ 1056 Record the given 'branches' for an access involving the given 'name' and 1057 'attrnames'. 1058 """ 1059 1060 access = name, attrnames 1061 path = self.get_path_for_access() 1062 1063 init_item(self.attr_accessor_branches, path, dict) 1064 attr_accessor_branches = self.attr_accessor_branches[path] 1065 1066 init_item(attr_accessor_branches, access, list) 1067 attr_accessor_branches[access].append(branches) 1068 1069 def record_access_details(self, name, attrnames, assignment): 1070 1071 """ 1072 For the given 'name' and 'attrnames', record an access indicating 1073 whether 'assignment' is occurring. 1074 1075 These details correspond to accesses otherwise recorded by the attribute 1076 accessor and attribute access dictionaries. 1077 """ 1078 1079 access = name, attrnames 1080 path = self.get_path_for_access() 1081 1082 init_item(self.attr_access_modifiers, path, dict) 1083 init_item(self.attr_access_modifiers[path], access, list) 1084 1085 access_number = len(self.attr_access_modifiers[path][access]) 1086 self.attr_access_modifiers[path][access].append(assignment) 1087 return access_number 1088 1089 def record_global_access_details(self, name, attrnames): 1090 1091 """ 1092 Record details of a global access via the given 'name' involving the 1093 indicated 'attrnames'. 1094 """ 1095 1096 path = self.get_namespace_path() 1097 1098 init_item(self.global_attr_accesses, path, dict) 1099 init_item(self.global_attr_accesses[path], name, set) 1100 self.global_attr_accesses[path][name].add(attrnames) 1101 1102 # Namespace modification. 1103 1104 def record_name(self, name): 1105 1106 "Record the use of 'name' in a namespace." 1107 1108 path = self.get_namespace_path() 1109 init_item(self.names_used, path, set) 1110 self.names_used[path].add(name) 1111 1112 def set_module(self, name, module_name): 1113 1114 """ 1115 Set a module in the current namespace using the given 'name' associated 1116 with the corresponding 'module_name'. 1117 """ 1118 1119 if name: 1120 self.set_general_local(name, Reference("<module>", module_name)) 1121 1122 def set_definition(self, name, kind): 1123 1124 """ 1125 Set the definition having the given 'name' and 'kind'. 1126 1127 Definitions are set in the static namespace hierarchy, but they can also 1128 be recorded for function locals. 1129 """ 1130 1131 if self.is_global(name): 1132 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1133 self.get_namespace_path(), name) 1134 1135 path = self.get_object_path(name) 1136 self.set_object(path, kind) 1137 1138 ref = self.get_object(path) 1139 if ref.get_kind() == "<var>": 1140 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1141 self.get_namespace_path(), name) 1142 1143 if not self.is_global(name) and self.in_function: 1144 self.set_function_local(name, ref) 1145 1146 def set_function_local(self, name, ref=None): 1147 1148 "Set the local with the given 'name' and optional 'ref'." 1149 1150 locals = self.function_locals[self.get_namespace_path()] 1151 multiple = not ref or locals.has_key(name) and locals[name] != ref 1152 locals[name] = multiple and Reference("<var>") or ref 1153 1154 def assign_general_local(self, name, name_ref): 1155 1156 """ 1157 Set for 'name' the given 'name_ref', recording the name for attribute 1158 usage tracking. 1159 """ 1160 1161 self.set_general_local(name, name_ref) 1162 self.assign_name(name, name_ref) 1163 1164 def set_general_local(self, name, value=None): 1165 1166 """ 1167 Set the 'name' with optional 'value' in any kind of local namespace, 1168 where the 'value' should be a reference if specified. 1169 """ 1170 1171 init_value = self.get_initialising_value(value) 1172 1173 # Module global names. 1174 1175 if self.is_global(name): 1176 path = self.get_global_path(name) 1177 self.set_object(path, init_value) 1178 1179 # Function local names. 1180 1181 elif self.in_function: 1182 path = self.get_object_path(name) 1183 self.set_function_local(name, init_value) 1184 1185 # Other namespaces (classes). 1186 1187 else: 1188 path = self.get_object_path(name) 1189 self.set_name(name, init_value) 1190 1191 def set_name(self, name, ref=None): 1192 1193 "Attach the 'name' with optional 'ref' to the current namespace." 1194 1195 self.set_object(self.get_object_path(name), ref) 1196 1197 def set_instance_attr(self, name, ref=None): 1198 1199 """ 1200 Add an instance attribute of the given 'name' to the current class, 1201 using the optional 'ref'. 1202 """ 1203 1204 init_item(self.instance_attrs, self.in_class, set) 1205 self.instance_attrs[self.in_class].add(name) 1206 1207 if ref: 1208 init_item(self.instance_attr_constants, self.in_class, dict) 1209 self.instance_attr_constants[self.in_class][name] = ref 1210 1211 def get_initialising_value(self, value): 1212 1213 "Return a suitable initialiser reference for 'value'." 1214 1215 # Includes LiteralSequenceRef, ResolvedNameRef... 1216 1217 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1218 return value.reference() 1219 1220 # In general, invocations do not produce known results. However, the 1221 # name initialisers are resolved once a module has been inspected. 1222 1223 elif isinstance(value, InvocationRef): 1224 return value.reference() 1225 1226 else: 1227 return value 1228 1229 # Static, program-relative naming. 1230 1231 def find_name(self, name): 1232 1233 """ 1234 Return the qualified name for the given 'name' used in the current 1235 non-function namespace. 1236 """ 1237 1238 path = self.get_namespace_path() 1239 ref = None 1240 1241 if not self.in_function and name not in predefined_constants: 1242 if self.in_class: 1243 ref = self.get_object(self.get_object_path(name)) 1244 if not ref: 1245 ref = self.get_global_or_builtin(name) 1246 1247 return ref 1248 1249 def get_class(self, node): 1250 1251 """ 1252 Use the given 'node' to obtain the identity of a class. Return a 1253 reference for the class. Unresolved dependencies are permitted and must 1254 be resolved later. 1255 """ 1256 1257 ref = self._get_class(node) 1258 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1259 1260 def _get_class(self, node): 1261 1262 """ 1263 Use the given 'node' to find a class definition. Return a reference to 1264 the class. 1265 """ 1266 1267 if isinstance(node, compiler.ast.Getattr): 1268 1269 # Obtain the identity of the access target. 1270 1271 ref = self._get_class(node.expr) 1272 1273 # Where the target is a class or module, obtain the identity of the 1274 # attribute. 1275 1276 if ref.has_kind(["<function>", "<var>"]): 1277 return None 1278 else: 1279 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1280 return self.get_object(attrname) 1281 1282 # Names can be module-level or built-in. 1283 1284 elif isinstance(node, compiler.ast.Name): 1285 1286 # Record usage of the name and attempt to identify it. 1287 1288 self.record_name(node.name) 1289 return self.find_name(node.name) 1290 else: 1291 return None 1292 1293 def get_constant(self, name, value): 1294 1295 "Return a constant reference for the given type 'name' and 'value'." 1296 1297 ref = self.get_builtin_class(name) 1298 return self.get_constant_reference(ref, value) 1299 1300 def get_literal_instance(self, n, name): 1301 1302 "For node 'n', return a reference to an instance of 'name'." 1303 1304 # Get a reference to the built-in class. 1305 1306 ref = self.get_builtin_class(name) 1307 1308 # Obtain the details of the literal itself. 1309 # An alias to the type is generated for sequences. 1310 1311 if name in ("dict", "list", "tuple"): 1312 self.set_special_literal(name, ref) 1313 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1314 1315 # Constant values are independently recorded. 1316 1317 else: 1318 return self.get_constant_reference(ref, n.value) 1319 1320 # Special names. 1321 1322 def get_special(self, name): 1323 1324 "Return any stored value for the given special 'name'." 1325 1326 return self.special.get(name) 1327 1328 def set_special(self, name, value): 1329 1330 """ 1331 Set a special 'name' that merely tracks the use of an implicit object 1332 'value'. 1333 """ 1334 1335 self.special[name] = value 1336 1337 def set_special_literal(self, name, ref): 1338 1339 """ 1340 Set a special name for the literal type 'name' having type 'ref'. Such 1341 special names provide a way of referring to literal object types. 1342 """ 1343 1344 literal_name = "$L%s" % name 1345 value = ResolvedNameRef(literal_name, ref) 1346 self.set_special(literal_name, value) 1347 1348 # Functions and invocations. 1349 1350 def set_invocation_usage(self): 1351 1352 """ 1353 Discard the current invocation storage figures, retaining the maximum 1354 values. 1355 """ 1356 1357 for path, (current, maximum) in self.function_targets.items(): 1358 self.importer.function_targets[path] = self.function_targets[path] = maximum 1359 1360 for path, (current, maximum) in self.function_arguments.items(): 1361 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1362 1363 def allocate_arguments(self, path, args): 1364 1365 """ 1366 Allocate temporary argument storage using current and maximum 1367 requirements for the given 'path' and 'args'. 1368 """ 1369 1370 init_item(self.function_targets, path, lambda: [0, 0]) 1371 t = self.function_targets[path] 1372 t[0] += 1 1373 t[1] = max(t[0], t[1]) 1374 1375 init_item(self.function_arguments, path, lambda: [0, 0]) 1376 t = self.function_arguments[path] 1377 t[0] += len(args) + 1 1378 t[1] = max(t[0], t[1]) 1379 1380 def deallocate_arguments(self, path, args): 1381 1382 "Deallocate temporary argument storage for the given 'path' and 'args'." 1383 1384 self.function_targets[path][0] -= 1 1385 self.function_arguments[path][0] -= len(args) + 1 1386 1387 # vim: tabstop=4 expandtab shiftwidth=4