1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef, VariableRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 47 # Accesses to global attributes. 48 49 self.global_attr_accesses = {} 50 51 # Usage tracking. 52 53 self.trackers = [] 54 self.attr_accessor_branches = {} 55 56 def __repr__(self): 57 return "InspectedModule(%r, %r)" % (self.name, self.importer) 58 59 # Principal methods. 60 61 def parse(self, filename): 62 63 "Parse the file having the given 'filename'." 64 65 self.parse_file(filename) 66 67 # Inspect the module. 68 69 self.start_tracking_in_module() 70 71 # Detect and record imports and globals declared in the module. 72 73 self.process_structure(self.astnode) 74 75 # Set the class of the module after the definition has occurred. 76 77 ref = self.get_builtin("module") 78 self.set_name("__class__", ref) 79 self.set_name("__name__", self.get_constant("string", self.name).reference()) 80 self.set_name("__file__", self.get_constant("string", filename).reference()) 81 82 # Reserve a constant for the encoding. 83 84 if self.encoding: 85 self.get_constant("string", self.encoding) 86 87 # Get module-level attribute usage details. 88 89 self.stop_tracking_in_module() 90 91 # Collect external name references. 92 93 self.collect_names() 94 95 def complete(self): 96 97 "Complete the module inspection." 98 99 # Resolve names not definitively mapped to objects. 100 101 self.resolve() 102 103 # Define the invocation requirements in each namespace. 104 105 self.set_invocation_usage() 106 107 # Propagate to the importer information needed in subsequent activities. 108 109 self.propagate() 110 111 # Accessory methods. 112 113 def collect_names(self): 114 115 "Collect the names used by each scope." 116 117 for path in self.names_used.keys(): 118 self.collect_names_for_path(path) 119 120 def collect_names_for_path(self, path): 121 122 """ 123 Collect the names used by the given 'path'. These are propagated to the 124 importer in advance of any dependency resolution. 125 """ 126 127 names = self.names_used.get(path) 128 if not names: 129 return 130 131 in_function = self.function_locals.has_key(path) 132 133 for name in names: 134 if in_function and name in self.function_locals[path]: 135 continue 136 137 key = "%s.%s" % (path, name) 138 139 # Find local definitions (within dynamic namespaces). 140 141 ref = self.get_resolved_object(key) 142 if ref: 143 self.set_name_reference(key, ref) 144 continue 145 146 # Find global. 147 148 ref = self.get_global(name) 149 if ref: 150 self.set_name_reference(key, ref) 151 continue 152 153 # Find presumed built-in definitions. 154 155 ref = self.get_builtin(name) 156 self.set_name_reference(key, ref) 157 158 def set_name_reference(self, path, ref): 159 160 "Map the given name 'path' to 'ref'." 161 162 self.importer.all_name_references[path] = self.name_references[path] = ref 163 164 # Module structure traversal. 165 166 def process_structure_node(self, n): 167 168 "Process the individual node 'n'." 169 170 path = self.get_namespace_path() 171 172 # Module global detection. 173 174 if isinstance(n, compiler.ast.Global): 175 self.process_global_node(n) 176 177 # Module import declarations. 178 179 elif isinstance(n, compiler.ast.From): 180 self.process_from_node(n) 181 182 elif isinstance(n, compiler.ast.Import): 183 self.process_import_node(n) 184 185 # Nodes using operator module functions. 186 187 elif isinstance(n, compiler.ast.Operator): 188 return self.process_operator_node(n) 189 190 elif isinstance(n, compiler.ast.AugAssign): 191 self.process_augassign_node(n) 192 193 elif isinstance(n, compiler.ast.Compare): 194 return self.process_compare_node(n) 195 196 elif isinstance(n, compiler.ast.Slice): 197 return self.process_slice_node(n) 198 199 elif isinstance(n, compiler.ast.Sliceobj): 200 return self.process_sliceobj_node(n) 201 202 elif isinstance(n, compiler.ast.Subscript): 203 return self.process_subscript_node(n) 204 205 # Namespaces within modules. 206 207 elif isinstance(n, compiler.ast.Class): 208 self.process_class_node(n) 209 210 elif isinstance(n, compiler.ast.Function): 211 self.process_function_node(n, n.name) 212 213 elif isinstance(n, compiler.ast.Lambda): 214 return self.process_lambda_node(n) 215 216 # Assignments. 217 218 elif isinstance(n, compiler.ast.Assign): 219 220 # Handle each assignment node. 221 222 for node in n.nodes: 223 self.process_assignment_node(node, n.expr) 224 225 # Assignments within non-Assign nodes. 226 227 elif isinstance(n, compiler.ast.AssName): 228 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 229 230 elif isinstance(n, compiler.ast.AssAttr): 231 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 232 233 # Accesses. 234 235 elif isinstance(n, compiler.ast.Getattr): 236 return self.process_attribute_access(n) 237 238 # Name recording for later testing. 239 240 elif isinstance(n, compiler.ast.Name): 241 return self.process_name_node(n) 242 243 # Conditional statement tracking. 244 245 elif isinstance(n, compiler.ast.For): 246 self.process_for_node(n) 247 248 elif isinstance(n, compiler.ast.While): 249 self.process_while_node(n) 250 251 elif isinstance(n, compiler.ast.If): 252 self.process_if_node(n) 253 254 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 255 return self.process_logical_node(n) 256 257 # Exception control-flow tracking. 258 259 elif isinstance(n, compiler.ast.TryExcept): 260 self.process_try_node(n) 261 262 elif isinstance(n, compiler.ast.TryFinally): 263 self.process_try_finally_node(n) 264 265 # Control-flow modification statements. 266 267 elif isinstance(n, compiler.ast.Break): 268 self.trackers[-1].suspend_broken_branch() 269 270 elif isinstance(n, compiler.ast.Continue): 271 self.trackers[-1].suspend_continuing_branch() 272 273 elif isinstance(n, compiler.ast.Raise): 274 self.process_structure(n) 275 self.trackers[-1].abandon_branch() 276 277 elif isinstance(n, compiler.ast.Return): 278 self.process_structure(n) 279 self.trackers[-1].abandon_returning_branch() 280 281 # Print statements. 282 283 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 284 self.process_print_node(n) 285 286 # Invocations. 287 288 elif isinstance(n, compiler.ast.CallFunc): 289 return self.process_invocation_node(n) 290 291 # Constant usage. 292 293 elif isinstance(n, compiler.ast.Const): 294 return self.get_literal_instance(n) 295 296 elif isinstance(n, compiler.ast.Dict): 297 return self.get_literal_instance(n, "dict") 298 299 elif isinstance(n, compiler.ast.List): 300 return self.get_literal_instance(n, "list") 301 302 elif isinstance(n, compiler.ast.Tuple): 303 return self.get_literal_instance(n, "tuple") 304 305 # All other nodes are processed depth-first. 306 307 else: 308 self.process_structure(n) 309 310 # By default, no expression details are returned. 311 312 return None 313 314 # Specific node handling. 315 316 def process_assignment_node(self, n, expr): 317 318 "Process the individual node 'n' to be assigned the contents of 'expr'." 319 320 # Names and attributes are assigned the entire expression. 321 322 if isinstance(n, compiler.ast.AssName): 323 if n.name == "self": 324 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 325 326 name_ref = expr and self.process_structure_node(expr) 327 328 # Name assignments populate either function namespaces or the 329 # general namespace hierarchy. 330 331 self.assign_general_local(n.name, name_ref) 332 333 # Record usage of the name. 334 335 self.record_name(n.name) 336 337 elif isinstance(n, compiler.ast.AssAttr): 338 if expr: 339 expr = self.process_structure_node(expr) 340 341 in_assignment = self.in_assignment 342 self.in_assignment = True 343 self.process_attribute_access(n) 344 self.in_assignment = in_assignment 345 346 # Lists and tuples are matched against the expression and their 347 # items assigned to expression items. 348 349 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 350 self.process_assignment_node_items(n, expr) 351 352 # Slices and subscripts are permitted within assignment nodes. 353 354 elif isinstance(n, compiler.ast.Slice): 355 self.process_slice_node(n, expr) 356 357 elif isinstance(n, compiler.ast.Subscript): 358 self.process_subscript_node(n, expr) 359 360 def process_attribute_access(self, n): 361 362 "Process the given attribute access node 'n'." 363 364 # Obtain any completed chain and return the reference to it. 365 366 name_ref = self.process_attribute_chain(n) 367 368 if self.have_access_expression(n): 369 return name_ref 370 371 # Where the start of the chain of attributes has been reached, determine 372 # the complete access. 373 374 # Given a non-access node, this chain can be handled in its entirety, 375 # either being name-based and thus an access rooted on a name, or being 376 # based on some other node and thus an anonymous access of some kind. 377 378 path = self.get_namespace_path() 379 380 # Start with the the full attribute chain. 381 382 remaining = self.attrs 383 attrnames = ".".join(remaining) 384 385 # If the accessor cannot be identified, or where attributes 386 # remain in an attribute chain, record the anonymous accesses. 387 388 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 389 390 init_item(self.attr_accesses, path, set) 391 self.attr_accesses[path].add(attrnames) 392 393 self.record_access_details(None, attrnames, self.in_assignment, 394 self.in_invocation) 395 del self.attrs[0] 396 return 397 398 # Name-based accesses will handle the first attribute in a 399 # chain. 400 401 else: 402 attrname = remaining[0] 403 404 # Attribute assignments are used to identify instance attributes. 405 406 if isinstance(n, compiler.ast.AssAttr) and \ 407 self.in_class and self.in_function and n.expr.name == "self": 408 409 self.set_instance_attr(attrname) 410 411 # Record attribute usage using any name local to this namespace, 412 # if assigned in the namespace, or using an external name 413 # (presently just globals within classes). 414 415 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 416 tracker = self.trackers[-1] 417 418 immediate_access = len(self.attrs) == 1 419 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 420 421 # Record global-based chains for subsequent resolution. 422 423 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 424 not self.in_function 425 426 if is_global: 427 self.record_global_access_details(name, attrnames) 428 429 # Make sure the name is being tracked: global names will not 430 # already be initialised in a branch and must be added 431 # explicitly. 432 433 if not tracker.have_name(name): 434 tracker.assign_names([name]) 435 if self.in_function: 436 self.scope_globals[path].add(name) 437 438 # Record attribute usage in the tracker, and record the branch 439 # information for the access. 440 441 branches = tracker.use_attribute(name, attrname, self.in_invocation, assignment) 442 443 if not branches: 444 raise InspectError("Name %s is accessed using %s before an assignment." % ( 445 name, attrname), path, n) 446 447 self.record_branches_for_access(branches, name, attrnames) 448 access_number = self.record_access_details(name, attrnames, 449 self.in_assignment, self.in_invocation) 450 451 del self.attrs[0] 452 return AccessRef(name, attrnames, access_number) 453 454 def process_class_node(self, n): 455 456 "Process the given class node 'n'." 457 458 path = self.get_namespace_path() 459 460 # To avoid notions of class "versions" where the same definition 461 # might be parameterised with different state and be referenced 462 # elsewhere (as base classes, for example), classes in functions or 463 # conditions are forbidden. 464 465 if self.in_function or self.in_conditional: 466 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 467 path, n.name) 468 return 469 470 # Resolve base classes. 471 472 bases = [] 473 474 for base in n.bases: 475 base_class = self.get_class(base) 476 477 if not base_class: 478 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 479 path, n.name, base) 480 return 481 else: 482 bases.append(base_class) 483 484 # Detect conflicting definitions. Such definitions cause conflicts in 485 # the storage of namespace-related information. 486 487 class_name = self.get_object_path(n.name) 488 ref = self.get_object(class_name, defer=False) 489 490 if ref and ref.static(): 491 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 492 493 # Record bases for the class and retain the class name. 494 # Note that the function class does not inherit from the object class. 495 496 if not bases and class_name != "__builtins__.core.object" and \ 497 class_name != "__builtins__.core.function": 498 499 ref = self.get_object("__builtins__.object") 500 bases.append(ref) 501 502 self.importer.classes[class_name] = self.classes[class_name] = bases 503 self.importer.subclasses[class_name] = set() 504 self.scope_globals[class_name] = set() 505 506 # Set the definition before entering the namespace rather than 507 # afterwards because methods may reference it. In normal Python, 508 # a class is not accessible until the definition is complete, but 509 # methods can generally reference it since upon being called the 510 # class will already exist. 511 512 self.set_definition(n.name, "<class>") 513 514 in_class = self.in_class 515 self.in_class = class_name 516 self.set_instance_attr("__class__", Reference("<class>", class_name)) 517 self.enter_namespace(n.name) 518 519 # Do not provide the special instantiator attributes on the function 520 # class. Function instances provide these attributes. 521 522 if class_name != "__builtins__.core.function": 523 self.set_name("__fn__") # special instantiator attribute 524 self.set_name("__args__") # special instantiator attribute 525 526 # Provide leafname and parent attributes. 527 528 parent, leafname = class_name.rsplit(".", 1) 529 self.set_name("__name__", self.get_constant("string", leafname).reference()) 530 self.set_name("__parent__") 531 532 self.process_structure_node(n.code) 533 self.exit_namespace() 534 self.in_class = in_class 535 536 def process_from_node(self, n): 537 538 "Process the given node 'n', importing from another module." 539 540 path = self.get_namespace_path() 541 542 module_name, names = self.get_module_name(n) 543 if module_name == self.name: 544 raise InspectError("Cannot import from the current module.", path, n) 545 546 self.queue_module(module_name) 547 548 # Attempt to obtain the referenced objects. 549 550 for name, alias in n.names: 551 if name == "*": 552 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 553 554 # Explicit names. 555 556 ref = self.import_name_from_module(name, module_name) 557 value = ResolvedNameRef(alias or name, ref) 558 self.set_general_local(alias or name, value) 559 560 def process_function_node(self, n, name): 561 562 """ 563 Process the given function or lambda node 'n' with the given 'name'. 564 """ 565 566 is_lambda = isinstance(n, compiler.ast.Lambda) 567 568 # Where a function is declared conditionally, use a separate name for 569 # the definition, and assign the definition to the stated name. 570 571 if (self.in_conditional or self.in_function) and not is_lambda: 572 original_name = name 573 name = self.get_lambda_name() 574 else: 575 original_name = None 576 577 # Detect conflicting definitions. Such definitions cause conflicts in 578 # the storage of namespace-related information. 579 580 function_name = self.get_object_path(name) 581 ref = self.get_object(function_name, defer=False) 582 583 if ref and ref.static(): 584 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 585 586 # Initialise argument and local records. 587 588 argnames = get_argnames(n.argnames) 589 is_method = self.in_class and not self.in_function 590 591 # Remove explicit "self" from method parameters. 592 593 if is_method and argnames and argnames[0] == "self": 594 del argnames[0] 595 596 # Copy and propagate the parameters. 597 598 self.importer.function_parameters[function_name] = \ 599 self.function_parameters[function_name] = argnames[:] 600 601 # Define all arguments/parameters in the local namespace. 602 603 locals = \ 604 self.importer.function_locals[function_name] = \ 605 self.function_locals[function_name] = {} 606 607 # Insert "self" into method locals. 608 609 if is_method: 610 argnames.insert(0, "self") 611 612 # Define "self" in terms of the class if in a method. 613 # This does not diminish the need for type-narrowing in the deducer. 614 615 if argnames: 616 if self.in_class and not self.in_function and argnames[0] == "self": 617 locals[argnames[0]] = Reference("<instance>", self.in_class) 618 else: 619 locals[argnames[0]] = Reference("<var>") 620 621 for argname in argnames[1:]: 622 locals[argname] = Reference("<var>") 623 624 globals = self.scope_globals[function_name] = set() 625 626 # Process the defaults. 627 628 defaults = self.importer.function_defaults[function_name] = \ 629 self.function_defaults[function_name] = [] 630 631 for argname, default in compiler.ast.get_defaults(n): 632 if default: 633 634 # Obtain any reference for the default. 635 636 name_ref = self.process_structure_node(default) 637 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 638 639 # Reset conditional tracking to focus on the function contents. 640 641 in_conditional = self.in_conditional 642 self.in_conditional = False 643 644 in_function = self.in_function 645 self.in_function = function_name 646 647 self.enter_namespace(name) 648 649 # Define a leafname attribute value for the function instance. 650 651 ref = self.get_builtin_class("string") 652 self.reserve_constant(function_name, name, ref.get_origin()) 653 654 # Track attribute usage within the namespace. 655 656 path = self.get_namespace_path() 657 658 self.start_tracking(locals) 659 self.process_structure_node(n.code) 660 self.stop_tracking() 661 662 # Exit to the parent. 663 664 self.exit_namespace() 665 666 # Update flags. 667 668 self.in_function = in_function 669 self.in_conditional = in_conditional 670 671 # Define the function using the appropriate name. 672 673 self.set_definition(name, "<function>") 674 675 # Where a function is set conditionally, assign the name. 676 677 if original_name: 678 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 679 680 def process_global_node(self, n): 681 682 """ 683 Process the given "global" node 'n'. 684 """ 685 686 path = self.get_namespace_path() 687 688 if path != self.name: 689 self.scope_globals[path].update(n.names) 690 691 def process_if_node(self, n): 692 693 """ 694 Process the given "if" node 'n'. 695 """ 696 697 tracker = self.trackers[-1] 698 tracker.new_branchpoint() 699 700 for test, body in n.tests: 701 self.process_structure_node(test) 702 703 tracker.new_branch() 704 705 in_conditional = self.in_conditional 706 self.in_conditional = True 707 self.process_structure_node(body) 708 self.in_conditional = in_conditional 709 710 tracker.shelve_branch() 711 712 # Maintain a branch for the else clause. 713 714 tracker.new_branch() 715 if n.else_: 716 self.process_structure_node(n.else_) 717 tracker.shelve_branch() 718 719 tracker.merge_branches() 720 721 def process_import_node(self, n): 722 723 "Process the given import node 'n'." 724 725 path = self.get_namespace_path() 726 727 # Load the mentioned module. 728 729 for name, alias in n.names: 730 if name == self.name: 731 raise InspectError("Cannot import the current module.", path, n) 732 733 self.set_module(alias or name.split(".")[-1], name) 734 self.queue_module(name, True) 735 736 def process_invocation_node(self, n): 737 738 "Process the given invocation node 'n'." 739 740 path = self.get_namespace_path() 741 742 self.allocate_arguments(path, n.args) 743 744 try: 745 # Communicate to the invocation target expression that it forms the 746 # target of an invocation, potentially affecting attribute accesses. 747 748 in_invocation = self.in_invocation 749 self.in_invocation = True 750 751 # Process the expression, obtaining any identified reference. 752 753 name_ref = self.process_structure_node(n.node) 754 self.in_invocation = False 755 756 # Process the arguments. 757 758 for arg in n.args: 759 self.process_structure_node(arg) 760 761 self.in_invocation = in_invocation 762 763 # Detect class invocations. 764 765 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 766 return InstanceRef(name_ref.reference().instance_of()) 767 768 elif isinstance(name_ref, NameRef): 769 return InvocationRef(name_ref) 770 771 # Provide a general reference to indicate that something is produced 772 # by the invocation, useful for retaining assignment expression 773 # details. 774 775 return VariableRef() 776 777 finally: 778 self.deallocate_arguments(path, n.args) 779 780 def process_lambda_node(self, n): 781 782 "Process the given lambda node 'n'." 783 784 name = self.get_lambda_name() 785 self.process_function_node(n, name) 786 787 origin = self.get_object_path(name) 788 789 if self.function_defaults.get(origin): 790 return None 791 else: 792 return ResolvedNameRef(name, Reference("<function>", origin)) 793 794 def process_logical_node(self, n): 795 796 "Process the given operator node 'n'." 797 798 self.process_operator_chain(n.nodes, self.process_structure_node) 799 800 def process_name_node(self, n): 801 802 "Process the given name node 'n'." 803 804 path = self.get_namespace_path() 805 806 # Find predefined constant names before anything else. 807 808 if n.name in predefined_constants: 809 ref = self.get_builtin(n.name) 810 value = ResolvedNameRef(n.name, ref) 811 return value 812 813 # Special names that have already been identified. 814 815 if n.name.startswith("$"): 816 value = self.get_special(n.name) 817 if value: 818 return value 819 820 # Special case for operator functions introduced through code 821 # transformations. 822 823 if n.name.startswith("$op"): 824 825 # Obtain the location of the actual function defined in the operator 826 # package. 827 828 op = n.name[len("$op"):] 829 830 # Attempt to get a reference. 831 832 ref = self.import_name_from_module(op, "operator") 833 834 # Record the imported name and provide the resolved name reference. 835 836 value = ResolvedNameRef(n.name, ref) 837 self.set_special(n.name, value) 838 return value 839 840 # Special case for print operations. 841 842 elif n.name.startswith("$print"): 843 844 # Attempt to get a reference. 845 846 ref = self.get_builtin("print_") 847 848 # Record the imported name and provide the resolved name reference. 849 850 value = ResolvedNameRef(n.name, ref) 851 self.set_special(n.name, value) 852 return value 853 854 # Test for self usage, which is only allowed in methods. 855 856 if n.name == "self" and not (self.in_function and self.in_class): 857 raise InspectError("Use of self is only allowed in methods.", path, n) 858 859 # Record usage of the name. 860 861 self.record_name(n.name) 862 863 # Search for unknown names in non-function scopes immediately. 864 # External names in functions are resolved later. 865 866 ref = self.find_name(n.name) 867 if ref: 868 return ResolvedNameRef(n.name, ref) 869 870 # Explicitly-declared global names. 871 872 elif self.in_function and n.name in self.scope_globals[path]: 873 return NameRef(n.name) 874 875 # Examine other names. 876 877 else: 878 tracker = self.trackers[-1] 879 880 # Check local names. 881 882 branches = tracker.tracking_name(n.name) 883 884 # Local name. 885 886 if branches: 887 self.record_branches_for_access(branches, n.name, None) 888 access_number = self.record_access_details(n.name, None, False, False) 889 return LocalNameRef(n.name, access_number) 890 891 # Possible global or built-in name. 892 893 else: 894 return NameRef(n.name) 895 896 def process_operator_chain(self, nodes, fn): 897 898 """ 899 Process the given chain of 'nodes', applying 'fn' to each node or item. 900 Each node starts a new conditional region, effectively making a deeply- 901 nested collection of if-like statements. 902 """ 903 904 tracker = self.trackers[-1] 905 906 for item in nodes: 907 tracker.new_branchpoint() 908 tracker.new_branch() 909 fn(item) 910 911 for item in nodes[:-1]: 912 tracker.shelve_branch() 913 tracker.new_branch() 914 tracker.shelve_branch() 915 tracker.merge_branches() 916 917 tracker.shelve_branch() 918 tracker.merge_branches() 919 920 def process_try_node(self, n): 921 922 """ 923 Process the given "try...except" node 'n'. 924 """ 925 926 self.record_exception_handler() 927 928 tracker = self.trackers[-1] 929 tracker.new_branchpoint() 930 931 self.process_structure_node(n.body) 932 933 for name, var, handler in n.handlers: 934 if name is not None: 935 self.process_structure_node(name) 936 937 # Any abandoned branches from the body can now be resumed in a new 938 # branch. 939 940 tracker.resume_abandoned_branches() 941 942 # Establish the local for the handler. 943 944 if var is not None: 945 self.process_assignment_node(var, None) 946 if handler is not None: 947 self.process_structure_node(handler) 948 949 tracker.shelve_branch() 950 951 # The else clause maintains the usage from the body but without the 952 # abandoned branches since they would never lead to the else clause 953 # being executed. 954 955 if n.else_: 956 tracker.new_branch() 957 self.process_structure_node(n.else_) 958 tracker.shelve_branch() 959 960 # Without an else clause, a null branch propagates the successful 961 # outcome. 962 963 else: 964 tracker.new_branch() 965 tracker.shelve_branch() 966 967 tracker.merge_branches() 968 969 def process_try_finally_node(self, n): 970 971 """ 972 Process the given "try...finally" node 'n'. 973 """ 974 975 self.record_exception_handler() 976 977 tracker = self.trackers[-1] 978 self.process_structure_node(n.body) 979 980 # Any abandoned branches from the body can now be resumed. 981 982 branches = tracker.resume_all_abandoned_branches() 983 self.process_structure_node(n.final) 984 985 # At the end of the finally clause, abandoned branches are discarded. 986 987 tracker.restore_active_branches(branches) 988 989 def process_while_node(self, n): 990 991 "Process the given while node 'n'." 992 993 tracker = self.trackers[-1] 994 tracker.new_branchpoint(loop_node=True) 995 996 # Evaluate any test or iterator outside the loop. 997 998 self.process_structure_node(n.test) 999 1000 # Propagate attribute usage to branches. 1001 1002 tracker.new_branch(loop_node=True) 1003 1004 # Enter the loop. 1005 1006 in_conditional = self.in_conditional 1007 self.in_conditional = True 1008 self.process_structure_node(n.body) 1009 self.in_conditional = in_conditional 1010 1011 # Continuing branches are resumed before any test. 1012 1013 tracker.resume_continuing_branches() 1014 1015 # Evaluate any continuation test within the body. 1016 1017 self.process_structure_node(n.test) 1018 1019 tracker.shelve_branch(loop_node=True) 1020 1021 # Support the non-looping condition. 1022 1023 tracker.new_branch() 1024 tracker.shelve_branch() 1025 1026 tracker.merge_branches() 1027 1028 # Evaluate any else clause outside branches. 1029 1030 if n.else_: 1031 self.process_structure_node(n.else_) 1032 1033 # Connect broken branches to the code after any loop. 1034 1035 tracker.resume_broken_branches() 1036 1037 # Branch tracking methods. 1038 1039 def start_tracking(self, names): 1040 1041 """ 1042 Start tracking attribute usage for names in the current namespace, 1043 immediately registering the given 'names'. 1044 """ 1045 1046 path = self.get_namespace_path() 1047 parent = self.trackers[-1] 1048 tracker = BranchTracker() 1049 self.trackers.append(tracker) 1050 1051 # Record the given names established as new branches. 1052 1053 tracker.assign_names(names) 1054 1055 def assign_name(self, name, name_ref): 1056 1057 "Assign to 'name' the given 'name_ref' in the current namespace." 1058 1059 name = self.get_name_for_tracking(name) 1060 self.trackers[-1].assign_names([name], [name_ref]) 1061 1062 def stop_tracking(self): 1063 1064 """ 1065 Stop tracking attribute usage, recording computed usage for the current 1066 namespace. 1067 """ 1068 1069 path = self.get_namespace_path() 1070 tracker = self.trackers.pop() 1071 self.record_assignments_for_access(tracker) 1072 1073 self.attr_usage[path] = tracker.get_all_usage() 1074 self.name_initialisers[path] = tracker.get_all_values() 1075 1076 def start_tracking_in_module(self): 1077 1078 "Start tracking attribute usage in the module." 1079 1080 tracker = BranchTracker() 1081 self.trackers.append(tracker) 1082 1083 def stop_tracking_in_module(self): 1084 1085 "Stop tracking attribute usage in the module." 1086 1087 tracker = self.trackers[0] 1088 self.record_assignments_for_access(tracker) 1089 self.attr_usage[self.name] = tracker.get_all_usage() 1090 self.name_initialisers[self.name] = tracker.get_all_values() 1091 1092 def record_assignments_for_access(self, tracker): 1093 1094 """ 1095 For the current path, use the given 'tracker' to record assignment 1096 version information for attribute accesses. 1097 """ 1098 1099 path = self.get_path_for_access() 1100 1101 if not self.attr_accessor_branches.has_key(path): 1102 return 1103 1104 init_item(self.attr_accessors, path, dict) 1105 attr_accessors = self.attr_accessors[path] 1106 1107 # Obtain the branches applying during each access. 1108 1109 for access, all_branches in self.attr_accessor_branches[path].items(): 1110 name, attrnames = access 1111 init_item(attr_accessors, access, list) 1112 1113 # Obtain the assignments applying to each branch. 1114 1115 for branches in all_branches: 1116 positions = tracker.get_assignment_positions_for_branches(name, branches) 1117 1118 # Detect missing name information. 1119 1120 if None in positions: 1121 globals = self.global_attr_accesses.get(path) 1122 accesses = globals and globals.get(name) 1123 if not accesses: 1124 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1125 self.get_namespace_path(), name) 1126 positions.remove(None) 1127 1128 attr_accessors[access].append(positions) 1129 1130 def record_branches_for_access(self, branches, name, attrnames): 1131 1132 """ 1133 Record the given 'branches' for an access involving the given 'name' and 1134 'attrnames'. 1135 """ 1136 1137 access = name, attrnames 1138 path = self.get_path_for_access() 1139 1140 init_item(self.attr_accessor_branches, path, dict) 1141 attr_accessor_branches = self.attr_accessor_branches[path] 1142 1143 init_item(attr_accessor_branches, access, list) 1144 attr_accessor_branches[access].append(branches) 1145 1146 def record_access_details(self, name, attrnames, assignment, invocation): 1147 1148 """ 1149 For the given 'name' and 'attrnames', record an access indicating 1150 whether 'assignment' is occurring. 1151 1152 These details correspond to accesses otherwise recorded by the attribute 1153 accessor and attribute access dictionaries. 1154 """ 1155 1156 access = name, attrnames 1157 path = self.get_path_for_access() 1158 1159 init_item(self.attr_access_modifiers, path, dict) 1160 init_item(self.attr_access_modifiers[path], access, list) 1161 1162 access_number = len(self.attr_access_modifiers[path][access]) 1163 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1164 return access_number 1165 1166 def record_global_access_details(self, name, attrnames): 1167 1168 """ 1169 Record details of a global access via the given 'name' involving the 1170 indicated 'attrnames'. 1171 """ 1172 1173 path = self.get_namespace_path() 1174 1175 init_item(self.global_attr_accesses, path, dict) 1176 init_item(self.global_attr_accesses[path], name, set) 1177 self.global_attr_accesses[path][name].add(attrnames) 1178 1179 # Namespace modification. 1180 1181 def record_name(self, name): 1182 1183 "Record the use of 'name' in a namespace." 1184 1185 path = self.get_namespace_path() 1186 init_item(self.names_used, path, set) 1187 self.names_used[path].add(name) 1188 1189 def set_module(self, name, module_name): 1190 1191 """ 1192 Set a module in the current namespace using the given 'name' associated 1193 with the corresponding 'module_name'. 1194 """ 1195 1196 if name: 1197 self.set_general_local(name, Reference("<module>", module_name)) 1198 1199 def set_definition(self, name, kind): 1200 1201 """ 1202 Set the definition having the given 'name' and 'kind'. 1203 1204 Definitions are set in the static namespace hierarchy, but they can also 1205 be recorded for function locals. 1206 """ 1207 1208 if self.is_global(name): 1209 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1210 self.get_namespace_path(), name) 1211 1212 path = self.get_object_path(name) 1213 self.set_object(path, kind) 1214 1215 ref = self.get_object(path) 1216 if ref.get_kind() == "<var>": 1217 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1218 self.get_namespace_path(), name) 1219 1220 if not self.is_global(name) and self.in_function: 1221 self.set_function_local(name, ref) 1222 1223 def set_function_local(self, name, ref=None): 1224 1225 "Set the local with the given 'name' and optional 'ref'." 1226 1227 locals = self.function_locals[self.get_namespace_path()] 1228 multiple = not ref or locals.has_key(name) and locals[name] != ref 1229 locals[name] = multiple and Reference("<var>") or ref 1230 1231 def assign_general_local(self, name, name_ref): 1232 1233 """ 1234 Set for 'name' the given 'name_ref', recording the name for attribute 1235 usage tracking. 1236 """ 1237 1238 self.set_general_local(name, name_ref) 1239 self.assign_name(name, name_ref) 1240 1241 def set_general_local(self, name, value=None): 1242 1243 """ 1244 Set the 'name' with optional 'value' in any kind of local namespace, 1245 where the 'value' should be a reference if specified. 1246 """ 1247 1248 init_value = self.get_initialising_value(value) 1249 1250 # Module global names. 1251 1252 if self.is_global(name): 1253 path = self.get_global_path(name) 1254 self.set_object(path, init_value) 1255 1256 # Function local names. 1257 1258 elif self.in_function: 1259 path = self.get_object_path(name) 1260 self.set_function_local(name, init_value) 1261 1262 # Other namespaces (classes). 1263 1264 else: 1265 path = self.get_object_path(name) 1266 self.set_name(name, init_value) 1267 1268 def set_name(self, name, ref=None): 1269 1270 "Attach the 'name' with optional 'ref' to the current namespace." 1271 1272 self.set_object(self.get_object_path(name), ref) 1273 1274 def set_instance_attr(self, name, ref=None): 1275 1276 """ 1277 Add an instance attribute of the given 'name' to the current class, 1278 using the optional 'ref'. 1279 """ 1280 1281 self._set_instance_attr(self.in_class, name, ref) 1282 1283 def _set_instance_attr(self, path, name, ref=None): 1284 1285 init_item(self.instance_attrs, path, set) 1286 self.instance_attrs[path].add(name) 1287 1288 if ref: 1289 init_item(self.instance_attr_constants, path, dict) 1290 self.instance_attr_constants[path][name] = ref 1291 1292 def get_initialising_value(self, value): 1293 1294 "Return a suitable initialiser reference for 'value'." 1295 1296 # Includes LiteralSequenceRef, ResolvedNameRef... 1297 1298 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1299 return value.reference() 1300 1301 # In general, invocations do not produce known results. However, the 1302 # name initialisers are resolved once a module has been inspected. 1303 1304 elif isinstance(value, InvocationRef): 1305 return value.reference() 1306 1307 # Variable references are unknown results. 1308 1309 elif isinstance(value, VariableRef): 1310 return value.reference() 1311 1312 else: 1313 return value 1314 1315 # Static, program-relative naming. 1316 1317 def find_name(self, name): 1318 1319 """ 1320 Return the qualified name for the given 'name' used in the current 1321 non-function namespace. 1322 """ 1323 1324 path = self.get_namespace_path() 1325 ref = None 1326 1327 if not self.in_function and name not in predefined_constants: 1328 if self.in_class: 1329 ref = self.get_object(self.get_object_path(name), False) 1330 if not ref: 1331 ref = self.get_global_or_builtin(name) 1332 1333 return ref 1334 1335 def get_class(self, node): 1336 1337 """ 1338 Use the given 'node' to obtain the identity of a class. Return a 1339 reference for the class. Unresolved dependencies are permitted and must 1340 be resolved later. 1341 """ 1342 1343 ref = self._get_class(node) 1344 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1345 1346 def _get_class(self, node): 1347 1348 """ 1349 Use the given 'node' to find a class definition. Return a reference to 1350 the class. 1351 """ 1352 1353 if isinstance(node, compiler.ast.Getattr): 1354 1355 # Obtain the identity of the access target. 1356 1357 ref = self._get_class(node.expr) 1358 1359 # Where the target is a class or module, obtain the identity of the 1360 # attribute. 1361 1362 if ref.has_kind(["<function>", "<var>"]): 1363 return None 1364 else: 1365 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1366 return self.get_object(attrname) 1367 1368 # Names can be module-level or built-in. 1369 1370 elif isinstance(node, compiler.ast.Name): 1371 1372 # Record usage of the name and attempt to identify it. 1373 1374 self.record_name(node.name) 1375 return self.find_name(node.name) 1376 else: 1377 return None 1378 1379 def get_constant(self, name, value): 1380 1381 "Return a constant reference for the given type 'name' and 'value'." 1382 1383 ref = self.get_builtin_class(name) 1384 return self.get_constant_reference(ref, value) 1385 1386 def get_literal_instance(self, n, name=None): 1387 1388 """ 1389 For node 'n', return a reference to an instance of 'name', or if 'name' 1390 is not specified, deduce the type from the value. 1391 """ 1392 1393 # Handle stray None constants (Sliceobj seems to produce them). 1394 1395 if name == "NoneType": 1396 return self.process_name_node(compiler.ast.Name("None")) 1397 1398 # Obtain the details of the literal itself. 1399 # An alias to the type is generated for sequences. 1400 1401 if name in ("dict", "list", "tuple"): 1402 ref = self.get_builtin_class(name) 1403 self.set_special_literal(name, ref) 1404 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1405 1406 # Constant values are independently recorded. 1407 1408 else: 1409 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1410 ref = self.get_builtin_class(typename) 1411 return self.get_constant_reference(ref, value, encoding) 1412 1413 # Special names. 1414 1415 def get_special(self, name): 1416 1417 "Return any stored value for the given special 'name'." 1418 1419 value = self.special.get(name) 1420 if value: 1421 ref, paths = value 1422 else: 1423 ref = None 1424 return ref 1425 1426 def set_special(self, name, value): 1427 1428 """ 1429 Set a special 'name' that merely tracks the use of an implicit object 1430 'value'. 1431 """ 1432 1433 if not self.special.has_key(name): 1434 paths = set() 1435 self.special[name] = value, paths 1436 else: 1437 _ref, paths = self.special[name] 1438 1439 paths.add(self.get_namespace_path()) 1440 1441 def set_special_literal(self, name, ref): 1442 1443 """ 1444 Set a special name for the literal type 'name' having type 'ref'. Such 1445 special names provide a way of referring to literal object types. 1446 """ 1447 1448 literal_name = "$L%s" % name 1449 value = ResolvedNameRef(literal_name, ref) 1450 self.set_special(literal_name, value) 1451 1452 # Functions and invocations. 1453 1454 def set_invocation_usage(self): 1455 1456 """ 1457 Discard the current invocation storage figures, retaining the maximum 1458 values. 1459 """ 1460 1461 for path, (current, maximum) in self.function_targets.items(): 1462 self.importer.function_targets[path] = self.function_targets[path] = maximum 1463 1464 for path, (current, maximum) in self.function_arguments.items(): 1465 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 1466 1467 def allocate_arguments(self, path, args): 1468 1469 """ 1470 Allocate temporary argument storage using current and maximum 1471 requirements for the given 'path' and 'args'. 1472 """ 1473 1474 # Class and module initialisation is ultimately combined. 1475 1476 if not self.in_function: 1477 path = self.name 1478 1479 init_item(self.function_targets, path, lambda: [0, 0]) 1480 t = self.function_targets[path] 1481 t[0] += 1 1482 t[1] = max(t[0], t[1]) 1483 1484 init_item(self.function_arguments, path, lambda: [0, 0]) 1485 t = self.function_arguments[path] 1486 t[0] += len(args) + 1 1487 t[1] = max(t[0], t[1]) 1488 1489 def deallocate_arguments(self, path, args): 1490 1491 "Deallocate temporary argument storage for the given 'path' and 'args'." 1492 1493 # Class and module initialisation is ultimately combined. 1494 1495 if not self.in_function: 1496 path = self.name 1497 1498 self.function_targets[path][0] -= 1 1499 self.function_arguments[path][0] -= len(args) + 1 1500 1501 # Exceptions. 1502 1503 def record_exception_handler(self): 1504 1505 "Record the current namespace as employing an exception handler." 1506 1507 self.exception_namespaces.add(self.get_namespace_path()) 1508 1509 # vim: tabstop=4 expandtab shiftwidth=4