1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, get_builtin_type, init_item, \ 25 predefined_constants 26 from modules import BasicModule, CacheWritingModule, InspectionNaming 27 from errors import InspectError 28 from referencing import Reference 29 from resolving import NameResolving 30 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 31 LocalNameRef, NameRef, ResolvedNameRef, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__mname__", self.get_constant("string", self.name).reference()) 81 self.set_name("__file__", self.get_constant("string", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("string", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Define the invocation requirements in each namespace. 105 106 self.set_invocation_usage() 107 108 # Propagate to the importer information needed in subsequent activities. 109 110 self.propagate() 111 112 # Accessory methods. 113 114 def collect_names(self): 115 116 "Collect the names used by each scope." 117 118 for path in self.names_used.keys(): 119 self.collect_names_for_path(path) 120 121 def collect_names_for_path(self, path): 122 123 """ 124 Collect the names used by the given 'path'. These are propagated to the 125 importer in advance of any dependency resolution. 126 """ 127 128 names = self.names_used.get(path) 129 if not names: 130 return 131 132 in_function = self.function_locals.has_key(path) 133 134 for name in names: 135 if in_function and name in self.function_locals[path]: 136 continue 137 138 key = "%s.%s" % (path, name) 139 140 # Find predefined constant names before anything else. 141 142 if name in predefined_constants: 143 ref = self.get_builtin(name) 144 self.set_name_reference(key, ref) 145 continue 146 147 # Find local definitions (within dynamic namespaces). 148 149 ref = self.get_resolved_object(key) 150 if ref: 151 self.set_name_reference(key, ref) 152 continue 153 154 # Find global. 155 156 ref = self.get_global(name) 157 if ref: 158 self.set_name_reference(key, ref) 159 continue 160 161 # Find presumed built-in definitions. 162 163 ref = self.get_builtin(name) 164 self.set_name_reference(key, ref) 165 166 def set_name_reference(self, path, ref): 167 168 "Map the given name 'path' to 'ref'." 169 170 self.importer.all_name_references[path] = self.name_references[path] = ref 171 172 # Module structure traversal. 173 174 def process_structure_node(self, n): 175 176 "Process the individual node 'n'." 177 178 path = self.get_namespace_path() 179 180 # Module global detection. 181 182 if isinstance(n, compiler.ast.Global): 183 self.process_global_node(n) 184 185 # Module import declarations. 186 187 elif isinstance(n, compiler.ast.From): 188 self.process_from_node(n) 189 190 elif isinstance(n, compiler.ast.Import): 191 self.process_import_node(n) 192 193 # Nodes using operator module functions. 194 195 elif isinstance(n, compiler.ast.Operator): 196 return self.process_operator_node(n) 197 198 elif isinstance(n, compiler.ast.AugAssign): 199 self.process_augassign_node(n) 200 201 elif isinstance(n, compiler.ast.Compare): 202 return self.process_compare_node(n) 203 204 elif isinstance(n, compiler.ast.Slice): 205 return self.process_slice_node(n) 206 207 elif isinstance(n, compiler.ast.Sliceobj): 208 return self.process_sliceobj_node(n) 209 210 elif isinstance(n, compiler.ast.Subscript): 211 return self.process_subscript_node(n) 212 213 # Namespaces within modules. 214 215 elif isinstance(n, compiler.ast.Class): 216 self.process_class_node(n) 217 218 elif isinstance(n, compiler.ast.Function): 219 self.process_function_node(n, n.name) 220 221 elif isinstance(n, compiler.ast.Lambda): 222 return self.process_lambda_node(n) 223 224 # Assignments. 225 226 elif isinstance(n, compiler.ast.Assign): 227 228 # Handle each assignment node. 229 230 for node in n.nodes: 231 self.process_assignment_node(node, n.expr) 232 233 # Assignments within non-Assign nodes. 234 235 elif isinstance(n, compiler.ast.AssName): 236 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 237 238 elif isinstance(n, compiler.ast.AssAttr): 239 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 240 241 # Accesses. 242 243 elif isinstance(n, compiler.ast.Getattr): 244 return self.process_attribute_access(n) 245 246 # Name recording for later testing. 247 248 elif isinstance(n, compiler.ast.Name): 249 return self.process_name_node(n) 250 251 # Conditional statement tracking. 252 253 elif isinstance(n, compiler.ast.For): 254 self.process_for_node(n) 255 256 elif isinstance(n, compiler.ast.While): 257 self.process_while_node(n) 258 259 elif isinstance(n, compiler.ast.If): 260 self.process_if_node(n) 261 262 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 263 return self.process_logical_node(n) 264 265 # Exception control-flow tracking. 266 267 elif isinstance(n, compiler.ast.TryExcept): 268 self.process_try_node(n) 269 270 elif isinstance(n, compiler.ast.TryFinally): 271 self.process_try_finally_node(n) 272 273 # Control-flow modification statements. 274 275 elif isinstance(n, compiler.ast.Break): 276 self.trackers[-1].suspend_broken_branch() 277 278 elif isinstance(n, compiler.ast.Continue): 279 self.trackers[-1].suspend_continuing_branch() 280 281 elif isinstance(n, compiler.ast.Raise): 282 self.process_structure(n) 283 self.trackers[-1].abandon_branch() 284 285 elif isinstance(n, compiler.ast.Return): 286 self.process_structure(n) 287 self.trackers[-1].abandon_returning_branch() 288 289 # Print statements. 290 291 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 292 self.process_print_node(n) 293 294 # Invocations. 295 296 elif isinstance(n, compiler.ast.CallFunc): 297 return self.process_invocation_node(n) 298 299 # Constant usage. 300 301 elif isinstance(n, compiler.ast.Const): 302 return self.get_literal_instance(n) 303 304 elif isinstance(n, compiler.ast.Dict): 305 return self.get_literal_instance(n, "dict") 306 307 elif isinstance(n, compiler.ast.List): 308 return self.get_literal_instance(n, "list") 309 310 elif isinstance(n, compiler.ast.Tuple): 311 return self.get_literal_instance(n, "tuple") 312 313 # Unsupported nodes. 314 315 elif isinstance(n, compiler.ast.GenExpr): 316 raise InspectError("Generator expressions are not supported.", path, n) 317 318 elif isinstance(n, compiler.ast.IfExp): 319 raise InspectError("If-else expressions are not supported.", path, n) 320 321 elif isinstance(n, compiler.ast.ListComp): 322 raise InspectError("List comprehensions are not supported.", path, n) 323 324 # All other nodes are processed depth-first. 325 326 else: 327 self.process_structure(n) 328 329 # By default, no expression details are returned. 330 331 return None 332 333 # Specific node handling. 334 335 def process_assignment_node(self, n, expr): 336 337 "Process the individual node 'n' to be assigned the contents of 'expr'." 338 339 # Names and attributes are assigned the entire expression. 340 341 if isinstance(n, compiler.ast.AssName): 342 if n.name == "self": 343 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 344 345 name_ref = expr and self.process_structure_node(expr) 346 347 # Name assignments populate either function namespaces or the 348 # general namespace hierarchy. 349 350 self.assign_general_local(n.name, name_ref) 351 352 # Record usage of the name. 353 354 self.record_name(n.name) 355 356 elif isinstance(n, compiler.ast.AssAttr): 357 if expr: 358 expr = self.process_structure_node(expr) 359 360 in_assignment = self.in_assignment 361 self.in_assignment = True 362 self.process_attribute_access(n) 363 self.in_assignment = in_assignment 364 365 # Lists and tuples are matched against the expression and their 366 # items assigned to expression items. 367 368 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 369 self.process_assignment_node_items(n, expr) 370 371 # Slices and subscripts are permitted within assignment nodes. 372 373 elif isinstance(n, compiler.ast.Slice): 374 self.process_slice_node(n, expr) 375 376 elif isinstance(n, compiler.ast.Subscript): 377 self.process_subscript_node(n, expr) 378 379 def process_attribute_access(self, n): 380 381 "Process the given attribute access node 'n'." 382 383 # Obtain any completed chain and return the reference to it. 384 385 name_ref = self.process_attribute_chain(n) 386 387 if self.have_access_expression(n): 388 return name_ref 389 390 # Where the start of the chain of attributes has been reached, determine 391 # the complete access. 392 393 # Given a non-access node, this chain can be handled in its entirety, 394 # either being name-based and thus an access rooted on a name, or being 395 # based on some other node and thus an anonymous access of some kind. 396 397 path = self.get_namespace_path() 398 399 # Start with the the full attribute chain. 400 401 remaining = self.attrs 402 attrnames = ".".join(remaining) 403 404 # If the accessor cannot be identified, or where attributes 405 # remain in an attribute chain, record the anonymous accesses. 406 407 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 408 409 init_item(self.attr_accesses, path, set) 410 self.attr_accesses[path].add(attrnames) 411 412 self.record_access_details(None, attrnames, self.in_assignment, 413 self.in_invocation) 414 del self.attrs[0] 415 return 416 417 # Name-based accesses will handle the first attribute in a 418 # chain. 419 420 else: 421 attrname = remaining[0] 422 423 # Attribute assignments are used to identify instance attributes. 424 425 if isinstance(n, compiler.ast.AssAttr) and \ 426 self.in_class and self.in_function and n.expr.name == "self": 427 428 self.set_instance_attr(attrname) 429 430 # Record attribute usage using any name local to this namespace, 431 # if assigned in the namespace, or using an external name 432 # (presently just globals within classes). 433 434 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 435 tracker = self.trackers[-1] 436 437 immediate_access = len(self.attrs) == 1 438 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 439 440 # Record global-based chains for subsequent resolution. 441 442 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 443 not self.in_function 444 445 if is_global: 446 self.record_global_access_details(name, attrnames) 447 448 # Make sure the name is being tracked: global names will not 449 # already be initialised in a branch and must be added 450 # explicitly. 451 452 if not tracker.have_name(name): 453 tracker.assign_names([name]) 454 if self.in_function: 455 self.scope_globals[path].add(name) 456 457 # Record attribute usage in the tracker, and record the branch 458 # information for the access. 459 460 branches = tracker.use_attribute(name, attrname, self.in_invocation, assignment) 461 462 if not branches: 463 raise InspectError("Name %s is accessed using %s before an assignment." % ( 464 name, attrname), path, n) 465 466 self.record_branches_for_access(branches, name, attrnames) 467 access_number = self.record_access_details(name, attrnames, 468 self.in_assignment, self.in_invocation) 469 470 del self.attrs[0] 471 return AccessRef(name, attrnames, access_number) 472 473 def process_class_node(self, n): 474 475 "Process the given class node 'n'." 476 477 path = self.get_namespace_path() 478 479 # To avoid notions of class "versions" where the same definition 480 # might be parameterised with different state and be referenced 481 # elsewhere (as base classes, for example), classes in functions or 482 # conditions are forbidden. 483 484 if self.in_function or self.in_conditional: 485 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 486 path, n.name) 487 return 488 489 # Resolve base classes. 490 491 bases = [] 492 493 for base in n.bases: 494 base_class = self.get_class(base) 495 496 if not base_class: 497 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 498 path, n.name, base) 499 return 500 else: 501 bases.append(base_class) 502 503 # Detect conflicting definitions. Such definitions cause conflicts in 504 # the storage of namespace-related information. 505 506 class_name = self.get_object_path(n.name) 507 ref = self.get_object(class_name) 508 509 if ref.static(): 510 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 511 512 # Record bases for the class and retain the class name. 513 # Note that the function class does not inherit from the object class. 514 515 if not bases and class_name != "__builtins__.core.object" and \ 516 class_name != "__builtins__.core.function": 517 518 ref = self.get_object("__builtins__.object") 519 bases.append(ref) 520 521 self.importer.classes[class_name] = self.classes[class_name] = bases 522 self.importer.subclasses[class_name] = set() 523 self.scope_globals[class_name] = set() 524 525 # Set the definition before entering the namespace rather than 526 # afterwards because methods may reference it. In normal Python, 527 # a class is not accessible until the definition is complete, but 528 # methods can generally reference it since upon being called the 529 # class will already exist. 530 531 self.set_definition(n.name, "<class>") 532 533 in_class = self.in_class 534 self.in_class = class_name 535 self.set_instance_attr("__class__", Reference("<class>", class_name)) 536 self.enter_namespace(n.name) 537 538 # Do not provide the special instantiator attributes on the function 539 # class. Function instances provide these attributes. 540 541 if class_name != "__builtins__.core.function": 542 self.set_name("__fn__") # special instantiator attribute 543 self.set_name("__args__") # special instantiator attribute 544 545 self.set_name("__name__", self.get_constant("string", class_name).reference()) 546 547 self.process_structure_node(n.code) 548 self.exit_namespace() 549 self.in_class = in_class 550 551 def process_from_node(self, n): 552 553 "Process the given node 'n', importing from another module." 554 555 path = self.get_namespace_path() 556 557 module_name, names = self.get_module_name(n) 558 if module_name == self.name: 559 raise InspectError("Cannot import from the current module.", path, n) 560 561 self.queue_module(module_name) 562 563 # Attempt to obtain the referenced objects. 564 565 for name, alias in n.names: 566 if name == "*": 567 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 568 569 # Explicit names. 570 571 ref = self.import_name_from_module(name, module_name) 572 value = ResolvedNameRef(alias or name, ref) 573 self.set_general_local(alias or name, value) 574 575 def process_function_node(self, n, name): 576 577 """ 578 Process the given function or lambda node 'n' with the given 'name'. 579 """ 580 581 is_lambda = isinstance(n, compiler.ast.Lambda) 582 583 # Where a function is declared conditionally, use a separate name for 584 # the definition, and assign the definition to the stated name. 585 586 if (self.in_conditional or self.in_function) and not is_lambda: 587 original_name = name 588 name = self.get_lambda_name() 589 else: 590 original_name = None 591 592 # Detect conflicting definitions. Such definitions cause conflicts in 593 # the storage of namespace-related information. 594 595 function_name = self.get_object_path(name) 596 ref = self.get_object(function_name) 597 598 if ref.static(): 599 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 600 601 # Initialise argument and local records. 602 603 argnames = get_argnames(n.argnames) 604 is_method = self.in_class and not self.in_function 605 606 # Remove explicit "self" from method parameters. 607 608 if is_method and argnames and argnames[0] == "self": 609 del argnames[0] 610 611 # Copy and propagate the parameters. 612 613 self.importer.function_parameters[function_name] = \ 614 self.function_parameters[function_name] = argnames[:] 615 616 # Define all arguments/parameters in the local namespace. 617 618 locals = \ 619 self.importer.function_locals[function_name] = \ 620 self.function_locals[function_name] = {} 621 622 # Insert "self" into method locals. 623 624 if is_method: 625 argnames.insert(0, "self") 626 627 # Define "self" in terms of the class if in a method. 628 # This does not diminish the need for type-narrowing in the deducer. 629 630 if argnames: 631 if self.in_class and not self.in_function and argnames[0] == "self": 632 locals[argnames[0]] = Reference("<instance>", self.in_class) 633 else: 634 locals[argnames[0]] = Reference("<var>") 635 636 for argname in argnames[1:]: 637 locals[argname] = Reference("<var>") 638 639 globals = self.scope_globals[function_name] = set() 640 641 # Process the defaults. 642 643 defaults = self.importer.function_defaults[function_name] = \ 644 self.function_defaults[function_name] = [] 645 646 for argname, default in compiler.ast.get_defaults(n): 647 if default: 648 649 # Obtain any reference for the default. 650 651 name_ref = self.process_structure_node(default) 652 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 653 654 # Reset conditional tracking to focus on the function contents. 655 656 in_conditional = self.in_conditional 657 self.in_conditional = False 658 659 in_function = self.in_function 660 self.in_function = function_name 661 662 self.enter_namespace(name) 663 664 # Define a name attribute value for the function instance. 665 666 ref = self.get_builtin_class("string") 667 self.reserve_constant(function_name, function_name, ref.get_origin()) 668 669 # Track attribute usage within the namespace. 670 671 path = self.get_namespace_path() 672 673 self.start_tracking(locals) 674 self.process_structure_node(n.code) 675 self.stop_tracking() 676 677 # Exit to the parent. 678 679 self.exit_namespace() 680 681 # Update flags. 682 683 self.in_function = in_function 684 self.in_conditional = in_conditional 685 686 # Define the function using the appropriate name. 687 688 self.set_definition(name, "<function>") 689 690 # Where a function is set conditionally, assign the name. 691 692 if original_name: 693 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 694 695 def process_global_node(self, n): 696 697 """ 698 Process the given "global" node 'n'. 699 """ 700 701 path = self.get_namespace_path() 702 703 if path != self.name: 704 self.scope_globals[path].update(n.names) 705 706 def process_if_node(self, n): 707 708 """ 709 Process the given "if" node 'n'. 710 """ 711 712 tracker = self.trackers[-1] 713 tracker.new_branchpoint() 714 715 for test, body in n.tests: 716 self.process_structure_node(test) 717 718 tracker.new_branch() 719 720 in_conditional = self.in_conditional 721 self.in_conditional = True 722 self.process_structure_node(body) 723 self.in_conditional = in_conditional 724 725 tracker.shelve_branch() 726 727 # Maintain a branch for the else clause. 728 729 tracker.new_branch() 730 if n.else_: 731 self.process_structure_node(n.else_) 732 tracker.shelve_branch() 733 734 tracker.merge_branches() 735 736 def process_import_node(self, n): 737 738 "Process the given import node 'n'." 739 740 path = self.get_namespace_path() 741 742 # Load the mentioned module. 743 744 for name, alias in n.names: 745 if name == self.name: 746 raise InspectError("Cannot import the current module.", path, n) 747 748 self.set_module(alias or name.split(".")[-1], name) 749 self.queue_module(name, True) 750 751 def process_invocation_node(self, n): 752 753 "Process the given invocation node 'n'." 754 755 path = self.get_namespace_path() 756 757 self.allocate_arguments(path, n.args) 758 759 try: 760 # Communicate to the invocation target expression that it forms the 761 # target of an invocation, potentially affecting attribute accesses. 762 763 in_invocation = self.in_invocation 764 self.in_invocation = True 765 766 # Process the expression, obtaining any identified reference. 767 768 name_ref = self.process_structure_node(n.node) 769 self.in_invocation = False 770 771 # Process the arguments. 772 773 for arg in n.args: 774 self.process_structure_node(arg) 775 776 self.in_invocation = in_invocation 777 778 # Detect class invocations. 779 780 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 781 return InstanceRef(name_ref.reference().instance_of()) 782 783 elif isinstance(name_ref, NameRef): 784 return InvocationRef(name_ref) 785 786 # Provide a general reference to indicate that something is produced 787 # by the invocation, useful for retaining assignment expression 788 # details. 789 790 return VariableRef() 791 792 finally: 793 self.deallocate_arguments(path, n.args) 794 795 def process_lambda_node(self, n): 796 797 "Process the given lambda node 'n'." 798 799 name = self.get_lambda_name() 800 self.process_function_node(n, name) 801 802 origin = self.get_object_path(name) 803 804 if self.function_defaults.get(origin): 805 return None 806 else: 807 return ResolvedNameRef(name, Reference("<function>", origin)) 808 809 def process_logical_node(self, n): 810 811 "Process the given operator node 'n'." 812 813 self.process_operator_chain(n.nodes, self.process_structure_node) 814 815 def process_name_node(self, n): 816 817 "Process the given name node 'n'." 818 819 path = self.get_namespace_path() 820 821 # Special names that have already been identified. 822 823 if n.name.startswith("$"): 824 value = self.get_special(n.name) 825 if value: 826 return value 827 828 # Special case for operator functions introduced through code 829 # transformations. 830 831 if n.name.startswith("$op"): 832 833 # Obtain the location of the actual function defined in the operator 834 # package. 835 836 op = n.name[len("$op"):] 837 838 # Attempt to get a reference. 839 840 ref = self.import_name_from_module(op, "operator") 841 842 # Record the imported name and provide the resolved name reference. 843 844 value = ResolvedNameRef(n.name, ref) 845 self.set_special(n.name, value) 846 return value 847 848 # Special case for print operations. 849 850 elif n.name.startswith("$print"): 851 852 # Attempt to get a reference. 853 854 ref = self.get_builtin("print_") 855 856 # Record the imported name and provide the resolved name reference. 857 858 value = ResolvedNameRef(n.name, ref) 859 self.set_special(n.name, value) 860 return value 861 862 # Test for self usage, which is only allowed in methods. 863 864 if n.name == "self" and not (self.in_function and self.in_class): 865 raise InspectError("Use of self is only allowed in methods.", path, n) 866 867 # Record usage of the name. 868 869 self.record_name(n.name) 870 871 # Search for unknown names in non-function scopes immediately. 872 # External names in functions are resolved later. 873 874 ref = self.find_name(n.name) 875 if ref: 876 return ResolvedNameRef(n.name, ref) 877 878 # Explicitly-declared global names. 879 880 elif self.in_function and n.name in self.scope_globals[path]: 881 return NameRef(n.name) 882 883 # Examine other names. 884 885 else: 886 tracker = self.trackers[-1] 887 888 # Check local names. 889 890 branches = tracker.tracking_name(n.name) 891 892 # Local name. 893 894 if branches: 895 self.record_branches_for_access(branches, n.name, None) 896 access_number = self.record_access_details(n.name, None, False, False) 897 return LocalNameRef(n.name, access_number) 898 899 # Possible global or built-in name. 900 901 else: 902 return NameRef(n.name) 903 904 def process_operator_chain(self, nodes, fn): 905 906 """ 907 Process the given chain of 'nodes', applying 'fn' to each node or item. 908 Each node starts a new conditional region, effectively making a deeply- 909 nested collection of if-like statements. 910 """ 911 912 tracker = self.trackers[-1] 913 914 for item in nodes: 915 tracker.new_branchpoint() 916 tracker.new_branch() 917 fn(item) 918 919 for item in nodes[:-1]: 920 tracker.shelve_branch() 921 tracker.new_branch() 922 tracker.shelve_branch() 923 tracker.merge_branches() 924 925 tracker.shelve_branch() 926 tracker.merge_branches() 927 928 def process_try_node(self, n): 929 930 """ 931 Process the given "try...except" node 'n'. 932 """ 933 934 tracker = self.trackers[-1] 935 tracker.new_branchpoint() 936 937 self.process_structure_node(n.body) 938 939 for name, var, handler in n.handlers: 940 if name is not None: 941 self.process_structure_node(name) 942 943 # Any abandoned branches from the body can now be resumed in a new 944 # branch. 945 946 tracker.resume_abandoned_branches() 947 948 # Establish the local for the handler. 949 950 if var is not None: 951 self.process_assignment_node(var, None) 952 if handler is not None: 953 self.process_structure_node(handler) 954 955 tracker.shelve_branch() 956 957 # The else clause maintains the usage from the body but without the 958 # abandoned branches since they would never lead to the else clause 959 # being executed. 960 961 if n.else_: 962 tracker.new_branch() 963 self.process_structure_node(n.else_) 964 tracker.shelve_branch() 965 966 # Without an else clause, a null branch propagates the successful 967 # outcome. 968 969 else: 970 tracker.new_branch() 971 tracker.shelve_branch() 972 973 tracker.merge_branches() 974 975 def process_try_finally_node(self, n): 976 977 """ 978 Process the given "try...finally" node 'n'. 979 """ 980 981 tracker = self.trackers[-1] 982 self.process_structure_node(n.body) 983 984 # Any abandoned branches from the body can now be resumed. 985 986 branches = tracker.resume_all_abandoned_branches() 987 self.process_structure_node(n.final) 988 989 # At the end of the finally clause, abandoned branches are discarded. 990 991 tracker.restore_active_branches(branches) 992 993 def process_while_node(self, n): 994 995 "Process the given while node 'n'." 996 997 tracker = self.trackers[-1] 998 tracker.new_branchpoint(loop_node=True) 999 1000 # Evaluate any test or iterator outside the loop. 1001 1002 self.process_structure_node(n.test) 1003 1004 # Propagate attribute usage to branches. 1005 1006 tracker.new_branch(loop_node=True) 1007 1008 # Enter the loop. 1009 1010 in_conditional = self.in_conditional 1011 self.in_conditional = True 1012 self.process_structure_node(n.body) 1013 self.in_conditional = in_conditional 1014 1015 # Continuing branches are resumed before any test. 1016 1017 tracker.resume_continuing_branches() 1018 1019 # Evaluate any continuation test within the body. 1020 1021 self.process_structure_node(n.test) 1022 1023 tracker.shelve_branch(loop_node=True) 1024 1025 # Support the non-looping condition. 1026 1027 tracker.new_branch() 1028 tracker.shelve_branch() 1029 1030 tracker.merge_branches() 1031 1032 # Evaluate any else clause outside branches. 1033 1034 if n.else_: 1035 self.process_structure_node(n.else_) 1036 1037 # Connect broken branches to the code after any loop. 1038 1039 tracker.resume_broken_branches() 1040 1041 # Branch tracking methods. 1042 1043 def start_tracking(self, names): 1044 1045 """ 1046 Start tracking attribute usage for names in the current namespace, 1047 immediately registering the given 'names'. 1048 """ 1049 1050 path = self.get_namespace_path() 1051 parent = self.trackers[-1] 1052 tracker = BranchTracker() 1053 self.trackers.append(tracker) 1054 1055 # Record the given names established as new branches. 1056 1057 tracker.assign_names(names) 1058 1059 def assign_name(self, name, name_ref): 1060 1061 "Assign to 'name' the given 'name_ref' in the current namespace." 1062 1063 name = self.get_name_for_tracking(name) 1064 self.trackers[-1].assign_names([name], [name_ref]) 1065 1066 def stop_tracking(self): 1067 1068 """ 1069 Stop tracking attribute usage, recording computed usage for the current 1070 namespace. 1071 """ 1072 1073 path = self.get_namespace_path() 1074 tracker = self.trackers.pop() 1075 self.record_assignments_for_access(tracker) 1076 1077 self.attr_usage[path] = tracker.get_all_usage() 1078 self.name_initialisers[path] = tracker.get_all_values() 1079 1080 def start_tracking_in_module(self): 1081 1082 "Start tracking attribute usage in the module." 1083 1084 tracker = BranchTracker() 1085 self.trackers.append(tracker) 1086 1087 def stop_tracking_in_module(self): 1088 1089 "Stop tracking attribute usage in the module." 1090 1091 tracker = self.trackers[0] 1092 self.record_assignments_for_access(tracker) 1093 self.attr_usage[self.name] = tracker.get_all_usage() 1094 self.name_initialisers[self.name] = tracker.get_all_values() 1095 1096 def record_assignments_for_access(self, tracker): 1097 1098 """ 1099 For the current path, use the given 'tracker' to record assignment 1100 version information for attribute accesses. 1101 """ 1102 1103 path = self.get_path_for_access() 1104 1105 if not self.attr_accessor_branches.has_key(path): 1106 return 1107 1108 init_item(self.attr_accessors, path, dict) 1109 attr_accessors = self.attr_accessors[path] 1110 1111 # Obtain the branches applying during each access. 1112 1113 for access, all_branches in self.attr_accessor_branches[path].items(): 1114 name, attrnames = access 1115 init_item(attr_accessors, access, list) 1116 1117 # Obtain the assignments applying to each branch. 1118 1119 for branches in all_branches: 1120 positions = tracker.get_assignment_positions_for_branches(name, branches) 1121 1122 # Detect missing name information. 1123 1124 if None in positions: 1125 globals = self.global_attr_accesses.get(path) 1126 accesses = globals and globals.get(name) 1127 if not accesses: 1128 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1129 self.get_namespace_path(), name) 1130 positions.remove(None) 1131 1132 attr_accessors[access].append(positions) 1133 1134 def record_branches_for_access(self, branches, name, attrnames): 1135 1136 """ 1137 Record the given 'branches' for an access involving the given 'name' and 1138 'attrnames'. 1139 """ 1140 1141 access = name, attrnames 1142 path = self.get_path_for_access() 1143 1144 init_item(self.attr_accessor_branches, path, dict) 1145 attr_accessor_branches = self.attr_accessor_branches[path] 1146 1147 init_item(attr_accessor_branches, access, list) 1148 attr_accessor_branches[access].append(branches) 1149 1150 def record_access_details(self, name, attrnames, assignment, invocation): 1151 1152 """ 1153 For the given 'name' and 'attrnames', record an access indicating 1154 whether 'assignment' is occurring. 1155 1156 These details correspond to accesses otherwise recorded by the attribute 1157 accessor and attribute access dictionaries. 1158 """ 1159 1160 access = name, attrnames 1161 path = self.get_path_for_access() 1162 1163 init_item(self.attr_access_modifiers, path, dict) 1164 init_item(self.attr_access_modifiers[path], access, list) 1165 1166 access_number = len(self.attr_access_modifiers[path][access]) 1167 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1168 return access_number 1169 1170 def record_global_access_details(self, name, attrnames): 1171 1172 """ 1173 Record details of a global access via the given 'name' involving the 1174 indicated 'attrnames'. 1175 """ 1176 1177 path = self.get_namespace_path() 1178 1179 init_item(self.global_attr_accesses, path, dict) 1180 init_item(self.global_attr_accesses[path], name, set) 1181 self.global_attr_accesses[path][name].add(attrnames) 1182 1183 # Namespace modification. 1184 1185 def record_name(self, name): 1186 1187 "Record the use of 'name' in a namespace." 1188 1189 path = self.get_namespace_path() 1190 init_item(self.names_used, path, set) 1191 self.names_used[path].add(name) 1192 1193 def set_module(self, name, module_name): 1194 1195 """ 1196 Set a module in the current namespace using the given 'name' associated 1197 with the corresponding 'module_name'. 1198 """ 1199 1200 if name: 1201 self.set_general_local(name, Reference("<module>", module_name)) 1202 1203 def set_definition(self, name, kind): 1204 1205 """ 1206 Set the definition having the given 'name' and 'kind'. 1207 1208 Definitions are set in the static namespace hierarchy, but they can also 1209 be recorded for function locals. 1210 """ 1211 1212 if self.is_global(name): 1213 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1214 self.get_namespace_path(), name) 1215 1216 path = self.get_object_path(name) 1217 self.set_object(path, kind) 1218 1219 ref = self.get_object(path) 1220 if ref.get_kind() == "<var>": 1221 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1222 self.get_namespace_path(), name) 1223 1224 if not self.is_global(name) and self.in_function: 1225 self.set_function_local(name, ref) 1226 1227 def set_function_local(self, name, ref=None): 1228 1229 "Set the local with the given 'name' and optional 'ref'." 1230 1231 locals = self.function_locals[self.get_namespace_path()] 1232 multiple = not ref or locals.has_key(name) and locals[name] != ref 1233 locals[name] = multiple and Reference("<var>") or ref 1234 1235 def assign_general_local(self, name, name_ref): 1236 1237 """ 1238 Set for 'name' the given 'name_ref', recording the name for attribute 1239 usage tracking. 1240 """ 1241 1242 self.set_general_local(name, name_ref) 1243 self.assign_name(name, name_ref) 1244 1245 def set_general_local(self, name, value=None): 1246 1247 """ 1248 Set the 'name' with optional 'value' in any kind of local namespace, 1249 where the 'value' should be a reference if specified. 1250 """ 1251 1252 init_value = self.get_initialising_value(value) 1253 1254 # Module global names. 1255 1256 if self.is_global(name): 1257 path = self.get_global_path(name) 1258 self.set_object(path, init_value) 1259 1260 # Function local names. 1261 1262 elif self.in_function: 1263 path = self.get_object_path(name) 1264 self.set_function_local(name, init_value) 1265 1266 # Other namespaces (classes). 1267 1268 else: 1269 path = self.get_object_path(name) 1270 self.set_name(name, init_value) 1271 1272 def set_name(self, name, ref=None): 1273 1274 "Attach the 'name' with optional 'ref' to the current namespace." 1275 1276 self.set_object(self.get_object_path(name), ref) 1277 1278 def set_instance_attr(self, name, ref=None): 1279 1280 """ 1281 Add an instance attribute of the given 'name' to the current class, 1282 using the optional 'ref'. 1283 """ 1284 1285 self._set_instance_attr(self.in_class, name, ref) 1286 1287 def _set_instance_attr(self, path, name, ref=None): 1288 1289 init_item(self.instance_attrs, path, set) 1290 self.instance_attrs[path].add(name) 1291 1292 if ref: 1293 init_item(self.instance_attr_constants, path, dict) 1294 self.instance_attr_constants[path][name] = ref 1295 1296 def get_initialising_value(self, value): 1297 1298 "Return a suitable initialiser reference for 'value'." 1299 1300 # Includes LiteralSequenceRef, ResolvedNameRef... 1301 1302 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1303 return value.reference() 1304 1305 # In general, invocations do not produce known results. However, the 1306 # name initialisers are resolved once a module has been inspected. 1307 1308 elif isinstance(value, InvocationRef): 1309 return value.reference() 1310 1311 # Variable references are unknown results. 1312 1313 elif isinstance(value, VariableRef): 1314 return value.reference() 1315 1316 else: 1317 return value 1318 1319 # Static, program-relative naming. 1320 1321 def find_name(self, name): 1322 1323 """ 1324 Return the qualified name for the given 'name' used in the current 1325 non-function namespace. 1326 """ 1327 1328 path = self.get_namespace_path() 1329 ref = None 1330 1331 if not self.in_function and name not in predefined_constants: 1332 if self.in_class: 1333 ref = self.get_object(self.get_object_path(name), False) 1334 if not ref: 1335 ref = self.get_global_or_builtin(name) 1336 1337 return ref 1338 1339 def get_class(self, node): 1340 1341 """ 1342 Use the given 'node' to obtain the identity of a class. Return a 1343 reference for the class. Unresolved dependencies are permitted and must 1344 be resolved later. 1345 """ 1346 1347 ref = self._get_class(node) 1348 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1349 1350 def _get_class(self, node): 1351 1352 """ 1353 Use the given 'node' to find a class definition. Return a reference to 1354 the class. 1355 """ 1356 1357 if isinstance(node, compiler.ast.Getattr): 1358 1359 # Obtain the identity of the access target. 1360 1361 ref = self._get_class(node.expr) 1362 1363 # Where the target is a class or module, obtain the identity of the 1364 # attribute. 1365 1366 if ref.has_kind(["<function>", "<var>"]): 1367 return None 1368 else: 1369 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1370 return self.get_object(attrname) 1371 1372 # Names can be module-level or built-in. 1373 1374 elif isinstance(node, compiler.ast.Name): 1375 1376 # Record usage of the name and attempt to identify it. 1377 1378 self.record_name(node.name) 1379 return self.find_name(node.name) 1380 else: 1381 return None 1382 1383 def get_constant(self, name, value): 1384 1385 "Return a constant reference for the given type 'name' and 'value'." 1386 1387 ref = self.get_builtin_class(name) 1388 return self.get_constant_reference(ref, value) 1389 1390 def get_literal_instance(self, n, name=None): 1391 1392 """ 1393 For node 'n', return a reference to an instance of 'name', or if 'name' 1394 is not specified, deduce the type from the value. 1395 """ 1396 1397 # Handle stray None constants (Sliceobj seems to produce them). 1398 1399 if name == "NoneType": 1400 return self.process_name_node(compiler.ast.Name("None")) 1401 1402 # Obtain the details of the literal itself. 1403 # An alias to the type is generated for sequences. 1404 1405 if name in ("dict", "list", "tuple"): 1406 ref = self.get_builtin_class(name) 1407 self.set_special_literal(name, ref) 1408 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1409 1410 # Constant values are independently recorded. 1411 1412 else: 1413 value, typename, encoding = self.get_constant_value(n.value, n.literal) 1414 name = get_builtin_type(typename) 1415 ref = self.get_builtin_class(name) 1416 return self.get_constant_reference(ref, value, encoding) 1417 1418 # Special names. 1419 1420 def get_special(self, name): 1421 1422 "Return any stored value for the given special 'name'." 1423 1424 return self.special.get(name) 1425 1426 def set_special(self, name, value): 1427 1428 """ 1429 Set a special 'name' that merely tracks the use of an implicit object 1430 'value'. 1431 """ 1432 1433 self.special[name] = value 1434 1435 def set_special_literal(self, name, ref): 1436 1437 """ 1438 Set a special name for the literal type 'name' having type 'ref'. Such 1439 special names provide a way of referring to literal object types. 1440 """ 1441 1442 literal_name = "$L%s" % name 1443 value = ResolvedNameRef(literal_name, ref) 1444 self.set_special(literal_name, value) 1445 1446 # Functions and invocations. 1447 1448 def set_invocation_usage(self): 1449 1450 """ 1451 Discard the current invocation storage figures, retaining the maximum 1452 values. 1453 """ 1454 1455 for path, (current, maximum) in self.function_targets.items(): 1456 self.importer.function_targets[path] = self.function_targets[path] = maximum 1457 1458 for path, (current, maximum) in self.function_arguments.items(): 1459 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1460 1461 def allocate_arguments(self, path, args): 1462 1463 """ 1464 Allocate temporary argument storage using current and maximum 1465 requirements for the given 'path' and 'args'. 1466 """ 1467 1468 # Class and module initialisation is ultimately combined. 1469 1470 if not self.in_function: 1471 path = self.name 1472 1473 init_item(self.function_targets, path, lambda: [0, 0]) 1474 t = self.function_targets[path] 1475 t[0] += 1 1476 t[1] = max(t[0], t[1]) 1477 1478 init_item(self.function_arguments, path, lambda: [0, 0]) 1479 t = self.function_arguments[path] 1480 t[0] += len(args) + 1 1481 t[1] = max(t[0], t[1]) 1482 1483 def deallocate_arguments(self, path, args): 1484 1485 "Deallocate temporary argument storage for the given 'path' and 'args'." 1486 1487 # Class and module initialisation is ultimately combined. 1488 1489 if not self.in_function: 1490 path = self.name 1491 1492 self.function_targets[path][0] -= 1 1493 self.function_arguments[path][0] -= len(args) + 1 1494 1495 # vim: tabstop=4 expandtab shiftwidth=4