1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 self.global_attr_accesses = {} 47 48 # Usage tracking. 49 50 self.trackers = [] 51 self.attr_accessor_branches = {} 52 53 def __repr__(self): 54 return "InspectedModule(%r, %r)" % (self.name, self.importer) 55 56 # Principal methods. 57 58 def parse(self, filename): 59 60 "Parse the file having the given 'filename'." 61 62 self.parse_file(filename) 63 64 # Inspect the module. 65 66 self.start_tracking_in_module() 67 68 # Detect and record imports and globals declared in the module. 69 70 self.assign_general_local("__name__", self.get_constant("str", self.name)) 71 self.assign_general_local("__file__", self.get_constant("str", filename)) 72 self.process_structure(self.astnode) 73 74 # Set the class of the module after the definition has occurred. 75 76 ref = self.get_builtin("object") 77 self.set_name("__class__", ref) 78 79 # Get module-level attribute usage details. 80 81 self.stop_tracking_in_module() 82 83 # Collect external name references. 84 85 self.collect_names() 86 87 def complete(self): 88 89 "Complete the module inspection." 90 91 # Resolve names not definitively mapped to objects. 92 93 self.resolve() 94 95 # Define the invocation requirements in each namespace. 96 97 self.set_invocation_usage() 98 99 # Propagate to the importer information needed in subsequent activities. 100 101 self.propagate() 102 103 # Accessory methods. 104 105 def collect_names(self): 106 107 "Collect the names used by each scope." 108 109 for path in self.names_used.keys(): 110 self.collect_names_for_path(path) 111 112 def collect_names_for_path(self, path): 113 114 """ 115 Collect the names used by the given 'path'. These are propagated to the 116 importer in advance of any dependency resolution. 117 """ 118 119 names = self.names_used.get(path) 120 if not names: 121 return 122 123 in_function = self.function_locals.has_key(path) 124 125 for name in names: 126 if name in predefined_constants or in_function and name in self.function_locals[path]: 127 continue 128 129 # Find local definitions (within dynamic namespaces). 130 131 key = "%s.%s" % (path, name) 132 ref = self.get_resolved_object(key) 133 if ref: 134 self.set_name_reference(key, ref) 135 continue 136 137 # Find global or known built-in definitions. 138 139 ref = self.get_resolved_global_or_builtin(name) 140 if ref: 141 self.set_name_reference(key, ref) 142 continue 143 144 # Find presumed built-in definitions. 145 146 ref = self.get_builtin(name) 147 self.set_name_reference(key, ref) 148 149 def set_name_reference(self, path, ref): 150 151 "Map the given name 'path' to 'ref'." 152 153 self.importer.all_name_references[path] = self.name_references[path] = ref 154 155 def get_resolved_global_or_builtin(self, name): 156 157 "Return the resolved global or built-in object with the given 'name'." 158 159 # In some circumstances, the name is neither global nor recognised by 160 # the importer. It is then assumed to be a general built-in. 161 162 return self.get_global(name) or \ 163 self.importer.get_object("__builtins__.%s" % name) 164 165 # Module structure traversal. 166 167 def process_structure_node(self, n): 168 169 "Process the individual node 'n'." 170 171 # Module global detection. 172 173 if isinstance(n, compiler.ast.Global): 174 self.process_global_node(n) 175 176 # Module import declarations. 177 178 elif isinstance(n, compiler.ast.From): 179 self.process_from_node(n) 180 181 elif isinstance(n, compiler.ast.Import): 182 self.process_import_node(n) 183 184 # Nodes using operator module functions. 185 186 elif isinstance(n, compiler.ast.Operator): 187 return self.process_operator_node(n) 188 189 elif isinstance(n, compiler.ast.AugAssign): 190 self.process_augassign_node(n) 191 192 elif isinstance(n, compiler.ast.Compare): 193 return self.process_compare_node(n) 194 195 elif isinstance(n, compiler.ast.Slice): 196 return self.process_slice_node(n) 197 198 elif isinstance(n, compiler.ast.Sliceobj): 199 return self.process_sliceobj_node(n) 200 201 elif isinstance(n, compiler.ast.Subscript): 202 return self.process_subscript_node(n) 203 204 # Namespaces within modules. 205 206 elif isinstance(n, compiler.ast.Class): 207 self.process_class_node(n) 208 209 elif isinstance(n, compiler.ast.Function): 210 self.process_function_node(n, n.name) 211 212 elif isinstance(n, compiler.ast.Lambda): 213 return self.process_lambda_node(n) 214 215 # Assignments. 216 217 elif isinstance(n, compiler.ast.Assign): 218 219 # Handle each assignment node. 220 221 for node in n.nodes: 222 self.process_assignment_node(node, n.expr) 223 224 # Assignments within non-Assign nodes. 225 226 elif isinstance(n, compiler.ast.AssName): 227 self.process_assignment_node(n, None) 228 229 elif isinstance(n, compiler.ast.AssAttr): 230 self.process_attribute_access(n) 231 232 # Accesses. 233 234 elif isinstance(n, compiler.ast.Getattr): 235 return self.process_attribute_access(n) 236 237 # Name recording for later testing. 238 239 elif isinstance(n, compiler.ast.Name): 240 return self.process_name_node(n) 241 242 # Conditional statement tracking. 243 244 elif isinstance(n, compiler.ast.For): 245 self.process_for_node(n) 246 247 elif isinstance(n, compiler.ast.While): 248 self.process_while_node(n) 249 250 elif isinstance(n, compiler.ast.If): 251 self.process_if_node(n) 252 253 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 254 return self.process_logical_node(n) 255 256 # Exception control-flow tracking. 257 258 elif isinstance(n, compiler.ast.TryExcept): 259 self.process_try_node(n) 260 261 elif isinstance(n, compiler.ast.TryFinally): 262 self.process_try_finally_node(n) 263 264 # Control-flow modification statements. 265 266 elif isinstance(n, compiler.ast.Break): 267 self.trackers[-1].suspend_broken_branch() 268 269 elif isinstance(n, compiler.ast.Continue): 270 self.trackers[-1].suspend_continuing_branch() 271 272 elif isinstance(n, compiler.ast.Raise): 273 self.process_structure(n) 274 self.trackers[-1].abandon_branch() 275 276 elif isinstance(n, compiler.ast.Return): 277 self.process_structure(n) 278 self.trackers[-1].abandon_returning_branch() 279 280 # Invocations. 281 282 elif isinstance(n, compiler.ast.CallFunc): 283 return self.process_invocation_node(n) 284 285 # Constant usage. 286 287 elif isinstance(n, compiler.ast.Const): 288 return self.get_literal_instance(n, n.value.__class__.__name__) 289 290 elif isinstance(n, compiler.ast.Dict): 291 return self.get_literal_instance(n, "dict") 292 293 elif isinstance(n, compiler.ast.List): 294 return self.get_literal_instance(n, "list") 295 296 elif isinstance(n, compiler.ast.Tuple): 297 return self.get_literal_instance(n, "tuple") 298 299 # Unsupported nodes. 300 301 elif isinstance(n, compiler.ast.GenExpr): 302 raise InspectError("Generator expressions are not supported.", self.get_namespace_path(), n) 303 304 elif isinstance(n, compiler.ast.IfExp): 305 raise InspectError("If-else expressions are not supported.", self.get_namespace_path(), n) 306 307 elif isinstance(n, compiler.ast.ListComp): 308 raise InspectError("List comprehensions are not supported.", self.get_namespace_path(), n) 309 310 # All other nodes are processed depth-first. 311 312 else: 313 self.process_structure(n) 314 315 # By default, no expression details are returned. 316 317 return None 318 319 # Specific node handling. 320 321 def process_assignment_node(self, n, expr): 322 323 "Process the individual node 'n' to be assigned the contents of 'expr'." 324 325 # Names and attributes are assigned the entire expression. 326 327 if isinstance(n, compiler.ast.AssName): 328 329 name_ref = expr and self.process_structure_node(expr) 330 331 # Name assignments populate either function namespaces or the 332 # general namespace hierarchy. 333 334 self.assign_general_local(n.name, name_ref) 335 336 # Record usage of the name. 337 338 self.record_name(n.name) 339 340 elif isinstance(n, compiler.ast.AssAttr): 341 if expr: self.process_structure_node(expr) 342 self.process_attribute_access(n) 343 344 # Lists and tuples are matched against the expression and their 345 # items assigned to expression items. 346 347 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 348 self.process_assignment_node_items(n, expr) 349 350 # Slices and subscripts are permitted within assignment nodes. 351 352 elif isinstance(n, compiler.ast.Slice): 353 self.process_slice_node(n, expr) 354 355 elif isinstance(n, compiler.ast.Subscript): 356 self.process_subscript_node(n, expr) 357 358 def process_attribute_access(self, n): 359 360 "Process the given attribute access node 'n'." 361 362 # Obtain any completed chain and return the reference to it. 363 364 name_ref = self.process_attribute_chain(n) 365 if self.have_access_expression(n): 366 return name_ref 367 368 # Where the start of the chain of attributes has been reached, determine 369 # the complete access. 370 371 # Given a non-access node, this chain can be handled in its entirety, 372 # either being name-based and thus an access rooted on a name, or being 373 # based on some other node and thus an anonymous access of some kind. 374 375 path = self.get_namespace_path() 376 377 # Start with the the full attribute chain. 378 379 remaining = self.attrs 380 attrnames = ".".join(remaining) 381 382 # If the accessor cannot be identified, or where attributes 383 # remain in an attribute chain, record the anonymous accesses. 384 385 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 386 387 assignment = isinstance(n, compiler.ast.AssAttr) 388 389 init_item(self.attr_accesses, path, set) 390 self.attr_accesses[path].add(attrnames) 391 392 self.record_access_details(None, attrnames, assignment) 393 del self.attrs[0] 394 return 395 396 # Name-based accesses will handle the first attribute in a 397 # chain. 398 399 else: 400 attrname = remaining[0] 401 402 # Attribute assignments are used to identify instance attributes. 403 404 if isinstance(n, compiler.ast.AssAttr) and \ 405 self.in_class and self.in_function and n.expr.name == "self": 406 407 self.set_instance_attr(attrname) 408 409 # Record attribute usage using any name local to this namespace, 410 # if assigned in the namespace, or using an external name 411 # (presently just globals within classes). 412 413 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 414 tracker = self.trackers[-1] 415 416 immediate_access = len(self.attrs) == 1 417 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 418 419 del self.attrs[0] 420 421 # Record global-based chains for subsequent resolution. 422 423 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 424 not self.in_function 425 426 if is_global: 427 self.record_global_access_details(name, attrnames) 428 429 # Make sure the name is being tracked: global names will not 430 # already be initialised in a branch and must be added 431 # explicitly. 432 433 if not tracker.have_name(name): 434 tracker.assign_names([name]) 435 if self.in_function: 436 self.scope_globals[path].add(name) 437 438 # Record attribute usage in the tracker, and record the branch 439 # information for the access. 440 441 branches = tracker.use_attribute(name, attrname) 442 443 if not branches: 444 print >>sys.stderr, "In %s, name %s is accessed using %s before an assignment." % ( 445 path, name, attrname) 446 branches = tracker.use_attribute(name, attrname) 447 448 self.record_branches_for_access(branches, name, attrnames) 449 access_number = self.record_access_details(name, attrnames, assignment) 450 return AccessRef(name, attrnames, access_number) 451 452 def process_class_node(self, n): 453 454 "Process the given class node 'n'." 455 456 path = self.get_namespace_path() 457 458 # To avoid notions of class "versions" where the same definition 459 # might be parameterised with different state and be referenced 460 # elsewhere (as base classes, for example), classes in functions or 461 # conditions are forbidden. 462 463 if self.in_function or self.in_conditional: 464 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 465 path, n.name) 466 return 467 468 # Resolve base classes. 469 470 bases = [] 471 472 for base in n.bases: 473 base_class = self.get_class(base) 474 475 if not base_class: 476 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 477 path, n.name, base) 478 return 479 else: 480 bases.append(base_class) 481 482 # Record bases for the class and retain the class name. 483 484 class_name = self.get_object_path(n.name) 485 486 if not bases and class_name != "__builtins__.core.object": 487 ref = self.get_object("__builtins__.object") 488 bases.append(ref) 489 490 self.importer.classes[class_name] = self.classes[class_name] = bases 491 self.importer.subclasses[class_name] = set() 492 self.scope_globals[class_name] = set() 493 494 # Set the definition before entering the namespace rather than 495 # afterwards because methods may reference it. In normal Python, 496 # a class is not accessible until the definition is complete, but 497 # methods can generally reference it since upon being called the 498 # class will already exist. 499 500 self.set_definition(n.name, "<class>") 501 502 in_class = self.in_class 503 self.in_class = class_name 504 self.set_instance_attr("__class__", Reference("<class>", class_name)) 505 self.enter_namespace(n.name) 506 self.set_name("__fn__") # special instantiator attribute 507 self.set_name("__args__") # special instantiator attribute 508 self.assign_general_local("__name__", self.get_constant("str", class_name)) 509 self.process_structure_node(n.code) 510 self.exit_namespace() 511 self.in_class = in_class 512 513 def process_from_node(self, n): 514 515 "Process the given node 'n', importing from another module." 516 517 path = self.get_namespace_path() 518 519 module_name, names = self.get_module_name(n) 520 if module_name == self.name: 521 raise InspectError("Cannot import from the current module.", path, n) 522 523 self.queue_module(module_name) 524 525 # Attempt to obtain the referenced objects. 526 527 for name, alias in n.names: 528 if name == "*": 529 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 530 531 # Explicit names. 532 533 ref = self.import_name_from_module(name, module_name) 534 value = ResolvedNameRef(alias or name, ref) 535 self.set_general_local(alias or name, value) 536 537 def process_function_node(self, n, name): 538 539 """ 540 Process the given function or lambda node 'n' with the given 'name'. 541 """ 542 543 is_lambda = isinstance(n, compiler.ast.Lambda) 544 545 # Where a function is declared conditionally, use a separate name for 546 # the definition, and assign the definition to the stated name. 547 548 if (self.in_conditional or self.in_function) and not is_lambda: 549 original_name = name 550 name = self.get_lambda_name() 551 else: 552 original_name = None 553 554 # Initialise argument and local records. 555 556 function_name = self.get_object_path(name) 557 argnames = get_argnames(n.argnames) 558 is_method = self.in_class and not self.in_function 559 560 # Remove explicit "self" from method parameters. 561 562 if is_method and argnames and argnames[0] == "self": 563 del argnames[0] 564 565 # Copy and propagate the parameters. 566 567 self.importer.function_parameters[function_name] = \ 568 self.function_parameters[function_name] = argnames[:] 569 570 # Define all arguments/parameters in the local namespace. 571 572 locals = self.function_locals[function_name] = {} 573 574 # Insert "self" into method locals. 575 576 if is_method: 577 argnames.insert(0, "self") 578 579 # Define "self" in terms of the class if in a method. 580 # This does not diminish the need for type-narrowing in the deducer. 581 582 if argnames: 583 if self.in_class and not self.in_function and argnames[0] == "self": 584 locals[argnames[0]] = Reference("<instance>", self.in_class) 585 else: 586 locals[argnames[0]] = Reference("<var>") 587 588 for argname in argnames[1:]: 589 locals[argname] = Reference("<var>") 590 591 globals = self.scope_globals[function_name] = set() 592 593 # Process the defaults. 594 595 defaults = self.importer.function_defaults[function_name] = \ 596 self.function_defaults[function_name] = [] 597 598 for argname, default in compiler.ast.get_defaults(n): 599 if default: 600 601 # Obtain any reference for the default. 602 603 name_ref = self.process_structure_node(default) 604 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 605 606 # Reset conditional tracking to focus on the function contents. 607 608 in_conditional = self.in_conditional 609 self.in_conditional = False 610 611 in_function = self.in_function 612 self.in_function = function_name 613 614 self.enter_namespace(name) 615 616 # Track attribute usage within the namespace. 617 618 path = self.get_namespace_path() 619 620 self.start_tracking(locals) 621 self.process_structure_node(n.code) 622 self.stop_tracking() 623 624 # Exit to the parent. 625 626 self.exit_namespace() 627 628 # Update flags. 629 630 self.in_function = in_function 631 self.in_conditional = in_conditional 632 633 # Define the function using the appropriate name. 634 635 self.set_definition(name, "<function>") 636 637 # Where a function is set conditionally, assign the name. 638 639 if original_name: 640 self.process_assignment_for_function(original_name, name) 641 642 def process_global_node(self, n): 643 644 """ 645 Process the given "global" node 'n'. 646 """ 647 648 path = self.get_namespace_path() 649 650 if path != self.name: 651 self.scope_globals[path].update(n.names) 652 653 def process_if_node(self, n): 654 655 """ 656 Process the given "if" node 'n'. 657 """ 658 659 tracker = self.trackers[-1] 660 tracker.new_branchpoint() 661 662 for test, body in n.tests: 663 self.process_structure_node(test) 664 665 tracker.new_branch() 666 667 in_conditional = self.in_conditional 668 self.in_conditional = True 669 self.process_structure_node(body) 670 self.in_conditional = in_conditional 671 672 tracker.shelve_branch() 673 674 # Maintain a branch for the else clause. 675 676 tracker.new_branch() 677 if n.else_: 678 self.process_structure_node(n.else_) 679 tracker.shelve_branch() 680 681 tracker.merge_branches() 682 683 def process_import_node(self, n): 684 685 "Process the given import node 'n'." 686 687 path = self.get_namespace_path() 688 689 # Load the mentioned module. 690 691 for name, alias in n.names: 692 if name == self.name: 693 raise InspectError("Cannot import the current module.", path, n) 694 if not alias and len(n.names) > 1: 695 raise InspectError("Imported modules must be aliased unless a simple module is imported.", path, n) 696 697 self.set_module(alias or name.split(".")[-1], name) 698 self.queue_module(name, True) 699 700 def process_invocation_node(self, n): 701 702 "Process the given invocation node 'n'." 703 704 path = self.get_namespace_path() 705 706 self.allocate_arguments(path, n.args) 707 708 try: 709 # Process the expression, obtaining any identified reference. 710 711 name_ref = self.process_structure_node(n.node) 712 713 # Process the arguments. 714 715 for arg in n.args: 716 self.process_structure_node(arg) 717 718 # Detect class invocations. 719 720 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 721 return InstanceRef(name_ref.reference().instance_of()) 722 723 elif isinstance(name_ref, NameRef): 724 return InvocationRef(name_ref) 725 726 return None 727 728 finally: 729 self.deallocate_arguments(path, n.args) 730 731 def process_lambda_node(self, n): 732 733 "Process the given lambda node 'n'." 734 735 name = self.get_lambda_name() 736 self.process_function_node(n, name) 737 738 origin = self.get_object_path(name) 739 return ResolvedNameRef(name, Reference("<function>", origin)) 740 741 def process_logical_node(self, n): 742 743 "Process the given operator node 'n'." 744 745 self.process_operator_chain(n.nodes, self.process_structure_node) 746 747 def process_name_node(self, n): 748 749 "Process the given name node 'n'." 750 751 path = self.get_namespace_path() 752 753 # Special names. 754 755 if n.name.startswith("$"): 756 value = self.get_special(n.name) 757 if value: 758 return value 759 760 # Special case for operator functions introduced through code 761 # transformations. 762 763 if n.name.startswith("$op"): 764 765 # Obtain the location of the actual function defined in the operator 766 # package. 767 768 op = n.name[len("$op"):] 769 770 # Attempt to get a reference. 771 772 ref = self.import_name_from_module(op, "operator") 773 self.add_deferred(ref) 774 775 # Record the imported name and provide the resolved name reference. 776 777 value = ResolvedNameRef(n.name, ref) 778 self.set_special(n.name, value) 779 return value 780 781 # Record usage of the name. 782 783 self.record_name(n.name) 784 785 # Search for unknown names in non-function scopes immediately. 786 # External names in functions are resolved later. 787 788 ref = self.find_name(n.name) 789 if ref: 790 return ResolvedNameRef(n.name, ref) 791 792 # Explicitly-declared global names. 793 794 elif self.in_function and n.name in self.scope_globals[path]: 795 return NameRef(n.name) 796 797 # Examine other names. 798 799 else: 800 tracker = self.trackers[-1] 801 802 # Check local names. 803 804 branches = tracker.tracking_name(n.name) 805 806 # Local name. 807 808 if branches: 809 self.record_branches_for_access(branches, n.name, None) 810 access_number = self.record_access_details(n.name, None, False) 811 return LocalNameRef(n.name, access_number) 812 813 # Possible global or built-in name. 814 815 else: 816 return NameRef(n.name) 817 818 def process_operator_chain(self, nodes, fn): 819 820 """ 821 Process the given chain of 'nodes', applying 'fn' to each node or item. 822 Each node starts a new conditional region, effectively making a deeply- 823 nested collection of if-like statements. 824 """ 825 826 tracker = self.trackers[-1] 827 828 for item in nodes: 829 tracker.new_branchpoint() 830 tracker.new_branch() 831 fn(item) 832 833 for item in nodes[:-1]: 834 tracker.shelve_branch() 835 tracker.new_branch() 836 tracker.shelve_branch() 837 tracker.merge_branches() 838 839 tracker.shelve_branch() 840 tracker.merge_branches() 841 842 def process_try_node(self, n): 843 844 """ 845 Process the given "try...except" node 'n'. 846 """ 847 848 tracker = self.trackers[-1] 849 tracker.new_branchpoint() 850 851 self.process_structure_node(n.body) 852 853 for name, var, handler in n.handlers: 854 if name is not None: 855 self.process_structure_node(name) 856 857 # Any abandoned branches from the body can now be resumed in a new 858 # branch. 859 860 tracker.resume_abandoned_branches() 861 862 # Establish the local for the handler. 863 864 if var is not None: 865 self.process_structure_node(var) 866 if handler is not None: 867 self.process_structure_node(handler) 868 869 tracker.shelve_branch() 870 871 # The else clause maintains the usage from the body but without the 872 # abandoned branches since they would never lead to the else clause 873 # being executed. 874 875 if n.else_: 876 tracker.new_branch() 877 self.process_structure_node(n.else_) 878 tracker.shelve_branch() 879 880 # Without an else clause, a null branch propagates the successful 881 # outcome. 882 883 else: 884 tracker.new_branch() 885 tracker.shelve_branch() 886 887 tracker.merge_branches() 888 889 def process_try_finally_node(self, n): 890 891 """ 892 Process the given "try...finally" node 'n'. 893 """ 894 895 tracker = self.trackers[-1] 896 self.process_structure_node(n.body) 897 898 # Any abandoned branches from the body can now be resumed. 899 900 branches = tracker.resume_all_abandoned_branches() 901 self.process_structure_node(n.final) 902 903 # At the end of the finally clause, abandoned branches are discarded. 904 905 tracker.restore_active_branches(branches) 906 907 def process_while_node(self, n): 908 909 "Process the given while node 'n'." 910 911 tracker = self.trackers[-1] 912 tracker.new_branchpoint(loop_node=True) 913 914 # Evaluate any test or iterator outside the loop. 915 916 self.process_structure_node(n.test) 917 918 # Propagate attribute usage to branches. 919 920 tracker.new_branch(loop_node=True) 921 922 # Enter the loop. 923 924 in_conditional = self.in_conditional 925 self.in_conditional = True 926 self.process_structure_node(n.body) 927 self.in_conditional = in_conditional 928 929 # Continuing branches are resumed before any test. 930 931 tracker.resume_continuing_branches() 932 933 # Evaluate any continuation test within the body. 934 935 self.process_structure_node(n.test) 936 937 tracker.shelve_branch(loop_node=True) 938 939 # Support the non-looping condition. 940 941 tracker.new_branch() 942 tracker.shelve_branch() 943 944 tracker.merge_branches() 945 946 # Evaluate any else clause outside branches. 947 948 if n.else_: 949 self.process_structure_node(n.else_) 950 951 # Connect broken branches to the code after any loop. 952 953 tracker.resume_broken_branches() 954 955 # Branch tracking methods. 956 957 def start_tracking(self, names): 958 959 """ 960 Start tracking attribute usage for names in the current namespace, 961 immediately registering the given 'names'. 962 """ 963 964 path = self.get_namespace_path() 965 parent = self.trackers[-1] 966 tracker = BranchTracker() 967 self.trackers.append(tracker) 968 969 # Record the given names established as new branches. 970 971 tracker.assign_names(names) 972 973 def assign_name(self, name, name_ref): 974 975 "Assign to 'name' the given 'name_ref' in the current namespace." 976 977 name = self.get_name_for_tracking(name) 978 self.trackers[-1].assign_names([name], [name_ref]) 979 980 def stop_tracking(self): 981 982 """ 983 Stop tracking attribute usage, recording computed usage for the current 984 namespace. 985 """ 986 987 path = self.get_namespace_path() 988 tracker = self.trackers.pop() 989 self.record_assignments_for_access(tracker) 990 991 self.attr_usage[path] = tracker.get_all_usage() 992 self.name_initialisers[path] = tracker.get_all_values() 993 994 def start_tracking_in_module(self): 995 996 "Start tracking attribute usage in the module." 997 998 tracker = BranchTracker() 999 self.trackers.append(tracker) 1000 1001 def stop_tracking_in_module(self): 1002 1003 "Stop tracking attribute usage in the module." 1004 1005 tracker = self.trackers[0] 1006 self.record_assignments_for_access(tracker) 1007 self.attr_usage[self.name] = tracker.get_all_usage() 1008 self.name_initialisers[self.name] = tracker.get_all_values() 1009 1010 def record_assignments_for_access(self, tracker): 1011 1012 """ 1013 For the current path, use the given 'tracker' to record assignment 1014 version information for attribute accesses. 1015 """ 1016 1017 path = self.get_path_for_access() 1018 1019 if not self.attr_accessor_branches.has_key(path): 1020 return 1021 1022 init_item(self.attr_accessors, path, dict) 1023 attr_accessors = self.attr_accessors[path] 1024 1025 # Obtain the branches applying during each access. 1026 1027 for access, all_branches in self.attr_accessor_branches[path].items(): 1028 name, attrnames = access 1029 init_item(attr_accessors, access, list) 1030 1031 # Obtain the assignments applying to each branch. 1032 1033 for branches in all_branches: 1034 positions = tracker.get_assignment_positions_for_branches(name, branches) 1035 1036 # Detect missing name information. 1037 1038 if None in positions: 1039 globals = self.global_attr_accesses.get(path) 1040 accesses = globals and globals.get(name) 1041 if not accesses: 1042 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1043 self.get_namespace_path(), name) 1044 positions.remove(None) 1045 1046 attr_accessors[access].append(positions) 1047 1048 def record_branches_for_access(self, branches, name, attrnames): 1049 1050 """ 1051 Record the given 'branches' for an access involving the given 'name' and 1052 'attrnames'. 1053 """ 1054 1055 access = name, attrnames 1056 path = self.get_path_for_access() 1057 1058 init_item(self.attr_accessor_branches, path, dict) 1059 attr_accessor_branches = self.attr_accessor_branches[path] 1060 1061 init_item(attr_accessor_branches, access, list) 1062 attr_accessor_branches[access].append(branches) 1063 1064 def record_access_details(self, name, attrnames, assignment): 1065 1066 """ 1067 For the given 'name' and 'attrnames', record an access indicating 1068 whether 'assignment' is occurring. 1069 1070 These details correspond to accesses otherwise recorded by the attribute 1071 accessor and attribute access dictionaries. 1072 """ 1073 1074 access = name, attrnames 1075 path = self.get_path_for_access() 1076 1077 init_item(self.attr_access_modifiers, path, dict) 1078 init_item(self.attr_access_modifiers[path], access, list) 1079 1080 access_number = len(self.attr_access_modifiers[path][access]) 1081 self.attr_access_modifiers[path][access].append(assignment) 1082 return access_number 1083 1084 def record_global_access_details(self, name, attrnames): 1085 1086 """ 1087 Record details of a global access via the given 'name' involving the 1088 indicated 'attrnames'. 1089 """ 1090 1091 path = self.get_namespace_path() 1092 1093 init_item(self.global_attr_accesses, path, dict) 1094 init_item(self.global_attr_accesses[path], name, set) 1095 self.global_attr_accesses[path][name].add(attrnames) 1096 1097 # Namespace modification. 1098 1099 def record_name(self, name): 1100 1101 "Record the use of 'name' in a namespace." 1102 1103 path = self.get_namespace_path() 1104 init_item(self.names_used, path, set) 1105 self.names_used[path].add(name) 1106 1107 def set_module(self, name, module_name): 1108 1109 """ 1110 Set a module in the current namespace using the given 'name' associated 1111 with the corresponding 'module_name'. 1112 """ 1113 1114 if name: 1115 self.set_general_local(name, Reference("<module>", module_name)) 1116 1117 def set_definition(self, name, kind): 1118 1119 """ 1120 Set the definition having the given 'name' and 'kind'. 1121 1122 Definitions are set in the static namespace hierarchy, but they can also 1123 be recorded for function locals. 1124 """ 1125 1126 if self.is_global(name): 1127 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1128 self.get_namespace_path(), name) 1129 1130 path = self.get_object_path(name) 1131 self.set_object(path, kind) 1132 1133 ref = self.get_object(path) 1134 if ref.get_kind() == "<var>": 1135 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1136 self.get_namespace_path(), name) 1137 1138 if not self.is_global(name) and self.in_function: 1139 self.set_function_local(name, ref) 1140 1141 def set_function_local(self, name, ref=None): 1142 1143 "Set the local with the given 'name' and optional 'ref'." 1144 1145 locals = self.function_locals[self.get_namespace_path()] 1146 multiple = not ref or locals.has_key(name) and locals[name] != ref 1147 locals[name] = multiple and Reference("<var>") or ref 1148 1149 def assign_general_local(self, name, name_ref): 1150 1151 """ 1152 Set for 'name' the given 'name_ref', recording the name for attribute 1153 usage tracking. 1154 """ 1155 1156 self.set_general_local(name, name_ref) 1157 self.assign_name(name, name_ref) 1158 1159 def set_general_local(self, name, value=None): 1160 1161 """ 1162 Set the 'name' with optional 'value' in any kind of local namespace, 1163 where the 'value' should be a reference if specified. 1164 """ 1165 1166 init_value = self.get_initialising_value(value) 1167 1168 # Module global names. 1169 1170 if self.is_global(name): 1171 path = self.get_global_path(name) 1172 self.set_object(path, init_value) 1173 1174 # Function local names. 1175 1176 elif self.in_function: 1177 path = self.get_object_path(name) 1178 self.set_function_local(name, init_value) 1179 1180 # Other namespaces (classes). 1181 1182 else: 1183 path = self.get_object_path(name) 1184 self.set_name(name, init_value) 1185 1186 def set_name(self, name, ref=None): 1187 1188 "Attach the 'name' with optional 'ref' to the current namespace." 1189 1190 self.set_object(self.get_object_path(name), ref) 1191 1192 def set_instance_attr(self, name, ref=None): 1193 1194 """ 1195 Add an instance attribute of the given 'name' to the current class, 1196 using the optional 'ref'. 1197 """ 1198 1199 init_item(self.instance_attrs, self.in_class, set) 1200 self.instance_attrs[self.in_class].add(name) 1201 1202 if ref: 1203 init_item(self.instance_attr_constants, self.in_class, dict) 1204 self.instance_attr_constants[self.in_class][name] = ref 1205 1206 def get_initialising_value(self, value): 1207 1208 "Return a suitable initialiser reference for 'value'." 1209 1210 # Includes LiteralSequenceRef, ResolvedNameRef... 1211 1212 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1213 return value.reference() 1214 1215 # In general, invocations do not produce known results. However, the 1216 # name initialisers are resolved once a module has been inspected. 1217 1218 elif isinstance(value, InvocationRef): 1219 return value.reference() 1220 1221 else: 1222 return value 1223 1224 # Static, program-relative naming. 1225 1226 def find_name(self, name): 1227 1228 """ 1229 Return the qualified name for the given 'name' used in the current 1230 non-function namespace. 1231 """ 1232 1233 path = self.get_namespace_path() 1234 ref = None 1235 1236 if not self.in_function and name not in predefined_constants: 1237 if self.in_class: 1238 ref = self.get_object(self.get_object_path(name)) 1239 if not ref: 1240 ref = self.get_global_or_builtin(name) 1241 1242 return ref 1243 1244 def get_class(self, node): 1245 1246 """ 1247 Use the given 'node' to obtain the identity of a class. Return a 1248 reference for the class. Unresolved dependencies are permitted and must 1249 be resolved later. 1250 """ 1251 1252 ref = self._get_class(node) 1253 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1254 1255 def _get_class(self, node): 1256 1257 """ 1258 Use the given 'node' to find a class definition. Return a reference to 1259 the class. 1260 """ 1261 1262 if isinstance(node, compiler.ast.Getattr): 1263 1264 # Obtain the identity of the access target. 1265 1266 ref = self._get_class(node.expr) 1267 1268 # Where the target is a class or module, obtain the identity of the 1269 # attribute. 1270 1271 if ref.has_kind(["<function>", "<var>"]): 1272 return None 1273 else: 1274 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1275 return self.get_object(attrname) 1276 1277 # Names can be module-level or built-in. 1278 1279 elif isinstance(node, compiler.ast.Name): 1280 1281 # Record usage of the name and attempt to identify it. 1282 1283 self.record_name(node.name) 1284 return self.get_global_or_builtin(node.name) 1285 else: 1286 return None 1287 1288 def get_constant(self, name, value): 1289 1290 "Return a constant reference for the given type 'name' and 'value'." 1291 1292 ref = self.get_builtin_class(name) 1293 return self.get_constant_reference(ref, value) 1294 1295 def get_literal_instance(self, n, name): 1296 1297 "For node 'n', return a reference to an instance of 'name'." 1298 1299 # Get a reference to the built-in class. 1300 1301 ref = self.get_builtin_class(name) 1302 1303 # Obtain the details of the literal itself. 1304 # An alias to the type is generated for sequences. 1305 1306 if name in ("dict", "list", "tuple"): 1307 self.set_special_literal(name, ref) 1308 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1309 1310 # Constant values are independently recorded. 1311 1312 else: 1313 return self.get_constant_reference(ref, n.value) 1314 1315 # Special names. 1316 1317 def get_special(self, name): 1318 1319 "Return any stored value for the given special 'name'." 1320 1321 return self.special.get(name) 1322 1323 def set_special(self, name, value): 1324 1325 """ 1326 Set a special 'name' that merely tracks the use of an implicit object 1327 'value'. 1328 """ 1329 1330 self.special[name] = value 1331 1332 def set_special_literal(self, name, ref): 1333 1334 """ 1335 Set a special name for the literal type 'name' having type 'ref'. Such 1336 special names provide a way of referring to literal object types. 1337 """ 1338 1339 literal_name = "$L%s" % name 1340 value = ResolvedNameRef(literal_name, ref) 1341 self.set_special(literal_name, value) 1342 1343 # Functions and invocations. 1344 1345 def set_invocation_usage(self): 1346 1347 """ 1348 Discard the current invocation storage figures, retaining the maximum 1349 values. 1350 """ 1351 1352 for path, (current, maximum) in self.function_targets.items(): 1353 self.importer.function_targets[path] = self.function_targets[path] = maximum 1354 1355 for path, (current, maximum) in self.function_arguments.items(): 1356 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1357 1358 def allocate_arguments(self, path, args): 1359 1360 """ 1361 Allocate temporary argument storage using current and maximum 1362 requirements for the given 'path' and 'args'. 1363 """ 1364 1365 init_item(self.function_targets, path, lambda: [0, 0]) 1366 t = self.function_targets[path] 1367 t[0] += 1 1368 t[1] = max(t[0], t[1]) 1369 1370 init_item(self.function_arguments, path, lambda: [0, 0]) 1371 t = self.function_arguments[path] 1372 t[0] += len(args) + 1 1373 t[1] = max(t[0], t[1]) 1374 1375 def deallocate_arguments(self, path, args): 1376 1377 "Deallocate temporary argument storage for the given 'path' and 'args'." 1378 1379 self.function_targets[path][0] -= 1 1380 self.function_arguments[path][0] -= len(args) + 1 1381 1382 # vim: tabstop=4 expandtab shiftwidth=4