1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, NameRef, ResolvedNameRef, VariableRef 31 import compiler 32 import sys 33 34 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 35 36 "A module inspector." 37 38 def __init__(self, name, importer): 39 40 "Initialise the module with basic details." 41 42 BasicModule.__init__(self, name, importer) 43 44 self.in_class = False 45 self.in_conditional = False 46 47 # Accesses to global attributes. 48 49 self.global_attr_accesses = {} 50 51 # Usage tracking. 52 53 self.trackers = [] 54 self.attr_accessor_branches = {} 55 56 def __repr__(self): 57 return "InspectedModule(%r, %r)" % (self.name, self.importer) 58 59 # Principal methods. 60 61 def parse(self, filename): 62 63 "Parse the file having the given 'filename'." 64 65 self.parse_file(filename) 66 67 # Inspect the module. 68 69 self.start_tracking_in_module() 70 71 # Detect and record imports and globals declared in the module. 72 73 self.process_structure(self.astnode) 74 75 # Set the class of the module after the definition has occurred. 76 77 ref = self.get_builtin("module") 78 self.set_name("__class__", ref) 79 self.set_name("__name__", self.get_constant("string", self.name).reference()) 80 self.set_name("__file__", self.get_constant("string", filename).reference()) 81 82 # Reserve a constant for the encoding. 83 84 if self.encoding: 85 self.get_constant("string", self.encoding) 86 87 # Get module-level attribute usage details. 88 89 self.stop_tracking_in_module() 90 91 # Collect external name references. 92 93 self.collect_names() 94 95 def complete(self): 96 97 "Complete the module inspection." 98 99 # Resolve names not definitively mapped to objects. 100 101 self.resolve() 102 103 # Propagate to the importer information needed in subsequent activities. 104 105 self.propagate() 106 107 # Accessory methods. 108 109 def collect_names(self): 110 111 "Collect the names used by each scope." 112 113 for path in self.names_used.keys(): 114 self.collect_names_for_path(path) 115 116 def collect_names_for_path(self, path): 117 118 """ 119 Collect the names used by the given 'path'. These are propagated to the 120 importer in advance of any dependency resolution. 121 """ 122 123 names = self.names_used.get(path) 124 if not names: 125 return 126 127 in_function = self.function_locals.has_key(path) 128 129 for name in names: 130 if in_function and name in self.function_locals[path]: 131 continue 132 133 key = "%s.%s" % (path, name) 134 135 # Find local definitions. 136 137 ref = self.get_resolved_object(key, True) 138 if ref: 139 self.set_name_reference(key, ref) 140 continue 141 142 # Find global. 143 144 ref = self.get_global(name) 145 if ref: 146 self.set_name_reference(key, ref) 147 continue 148 149 # Find presumed built-in definitions. 150 151 ref = self.get_builtin(name) 152 self.set_name_reference(key, ref) 153 154 def set_name_reference(self, path, ref): 155 156 "Map the given name 'path' to 'ref'." 157 158 self.importer.all_name_references[path] = self.name_references[path] = ref 159 160 # Module structure traversal. 161 162 def process_structure_node(self, n): 163 164 "Process the individual node 'n'." 165 166 path = self.get_namespace_path() 167 168 # Module global detection. 169 170 if isinstance(n, compiler.ast.Global): 171 self.process_global_node(n) 172 173 # Module import declarations. 174 175 elif isinstance(n, compiler.ast.From): 176 self.process_from_node(n) 177 178 elif isinstance(n, compiler.ast.Import): 179 self.process_import_node(n) 180 181 # Nodes using operator module functions. 182 183 elif isinstance(n, compiler.ast.Operator): 184 return self.process_operator_node(n) 185 186 elif isinstance(n, compiler.ast.AugAssign): 187 self.process_augassign_node(n) 188 189 elif isinstance(n, compiler.ast.Compare): 190 return self.process_compare_node(n) 191 192 elif isinstance(n, compiler.ast.Slice): 193 return self.process_slice_node(n) 194 195 elif isinstance(n, compiler.ast.Sliceobj): 196 return self.process_sliceobj_node(n) 197 198 elif isinstance(n, compiler.ast.Subscript): 199 return self.process_subscript_node(n) 200 201 # Namespaces within modules. 202 203 elif isinstance(n, compiler.ast.Class): 204 self.process_class_node(n) 205 206 elif isinstance(n, compiler.ast.Function): 207 self.process_function_node(n, n.name) 208 209 elif isinstance(n, compiler.ast.Lambda): 210 return self.process_lambda_node(n) 211 212 # Assignments. 213 214 elif isinstance(n, compiler.ast.Assign): 215 216 # Handle each assignment node. 217 218 for node in n.nodes: 219 self.process_assignment_node(node, n.expr) 220 221 # Assignments within non-Assign nodes. 222 223 elif isinstance(n, compiler.ast.AssName): 224 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 225 226 elif isinstance(n, compiler.ast.AssAttr): 227 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 228 229 # Accesses. 230 231 elif isinstance(n, compiler.ast.Getattr): 232 return self.process_attribute_access(n) 233 234 # Name recording for later testing. 235 236 elif isinstance(n, compiler.ast.Name): 237 return self.process_name_node(n) 238 239 # Conditional statement tracking. 240 241 elif isinstance(n, compiler.ast.For): 242 self.process_for_node(n) 243 244 elif isinstance(n, compiler.ast.While): 245 self.process_while_node(n) 246 247 elif isinstance(n, compiler.ast.If): 248 self.process_if_node(n) 249 250 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 251 return self.process_logical_node(n) 252 253 # Exception control-flow tracking. 254 255 elif isinstance(n, compiler.ast.TryExcept): 256 self.process_try_node(n) 257 258 elif isinstance(n, compiler.ast.TryFinally): 259 self.process_try_finally_node(n) 260 261 # Control-flow modification statements. 262 263 elif isinstance(n, compiler.ast.Break): 264 self.trackers[-1].suspend_broken_branch() 265 266 elif isinstance(n, compiler.ast.Continue): 267 self.trackers[-1].suspend_continuing_branch() 268 269 elif isinstance(n, compiler.ast.Raise): 270 self.process_structure(n) 271 self.trackers[-1].abandon_branch() 272 273 elif isinstance(n, compiler.ast.Return): 274 self.record_return_value(self.process_structure_node(n.value)) 275 self.trackers[-1].abandon_returning_branch() 276 277 # Print statements. 278 279 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 280 self.process_print_node(n) 281 282 # Invocations. 283 284 elif isinstance(n, compiler.ast.CallFunc): 285 return self.process_invocation_node(n) 286 287 # Constant usage. 288 289 elif isinstance(n, compiler.ast.Const): 290 return self.get_literal_instance(n) 291 292 elif isinstance(n, compiler.ast.Dict): 293 return self.get_literal_instance(n, "dict") 294 295 elif isinstance(n, compiler.ast.List): 296 return self.get_literal_instance(n, "list") 297 298 elif isinstance(n, compiler.ast.Tuple): 299 return self.get_literal_instance(n, "tuple") 300 301 # All other nodes are processed depth-first. 302 303 else: 304 self.process_structure(n) 305 306 # By default, no expression details are returned. 307 308 return None 309 310 # Specific node handling. 311 312 def process_assignment_node(self, n, expr): 313 314 "Process the individual node 'n' to be assigned the contents of 'expr'." 315 316 # Names and attributes are assigned the entire expression. 317 318 if isinstance(n, compiler.ast.AssName): 319 if n.name == "self": 320 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 321 322 name_ref = expr and self.process_structure_node(expr) 323 324 # Name assignments populate either function namespaces or the 325 # general namespace hierarchy. 326 327 self.assign_general_local(n.name, name_ref) 328 329 # Record usage of the name. 330 331 self.record_name(n.name) 332 333 elif isinstance(n, compiler.ast.AssAttr): 334 if expr: 335 expr = self.process_structure_node(expr) 336 337 in_assignment = self.in_assignment 338 self.in_assignment = True 339 self.process_attribute_access(n) 340 self.in_assignment = in_assignment 341 342 # Lists and tuples are matched against the expression and their 343 # items assigned to expression items. 344 345 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 346 self.process_assignment_node_items(n, expr) 347 348 # Slices and subscripts are permitted within assignment nodes. 349 350 elif isinstance(n, compiler.ast.Slice): 351 self.process_slice_node(n, expr) 352 353 elif isinstance(n, compiler.ast.Subscript): 354 self.process_subscript_node(n, expr) 355 356 def process_attribute_access(self, n): 357 358 "Process the given attribute access node 'n'." 359 360 # Obtain any completed chain and return the reference to it. 361 362 name_ref = self.process_attribute_chain(n) 363 364 if self.have_access_expression(n): 365 return name_ref 366 367 # Where the start of the chain of attributes has been reached, determine 368 # the complete access. 369 370 # Given a non-access node, this chain can be handled in its entirety, 371 # either being name-based and thus an access rooted on a name, or being 372 # based on some other node and thus an anonymous access of some kind. 373 374 path = self.get_namespace_path() 375 376 # Start with the the full attribute chain. 377 378 remaining = self.attrs 379 attrnames = ".".join(remaining) 380 381 # If the accessor cannot be identified, or where attributes 382 # remain in an attribute chain, record the anonymous accesses. 383 384 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 385 386 init_item(self.attr_accesses, path, set) 387 self.attr_accesses[path].add(attrnames) 388 389 self.record_access_details(None, attrnames, self.in_assignment, 390 self.in_invocation) 391 del self.attrs[0] 392 return 393 394 # Name-based accesses will handle the first attribute in a 395 # chain. 396 397 else: 398 attrname = remaining[0] 399 400 # Attribute assignments are used to identify instance attributes. 401 402 if isinstance(n, compiler.ast.AssAttr) and \ 403 self.in_class and self.in_function and n.expr.name == "self": 404 405 self.set_instance_attr(attrname) 406 407 # Record attribute usage using any name local to this namespace, 408 # if assigned in the namespace, or using an external name 409 # (presently just globals within classes). 410 411 name = self.get_name_for_tracking(name_ref.name, name_ref) 412 tracker = self.trackers[-1] 413 414 immediate_access = len(self.attrs) == 1 415 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 416 417 # Record global-based chains for subsequent resolution. 418 419 if name_ref.is_global_name(): 420 self.record_global_access_details(name, attrnames) 421 422 # Make sure the name is being tracked: global names will not 423 # already be initialised in a branch and must be added 424 # explicitly. 425 426 if not tracker.have_name(name): 427 tracker.assign_names([name]) 428 if self.in_function: 429 self.scope_globals[path].add(name) 430 431 # Record attribute usage in the tracker, and record the branch 432 # information for the access. 433 434 branches = tracker.use_attribute(name, attrname, 435 self.in_invocation is not None, assignment) 436 437 if not branches: 438 raise InspectError("Name %s is accessed using %s before an assignment." % ( 439 name, attrname), path, n) 440 441 self.record_branches_for_access(branches, name, attrnames) 442 access_number = self.record_access_details(name, attrnames, 443 self.in_assignment, self.in_invocation) 444 445 del self.attrs[0] 446 return AccessRef(name, attrnames, access_number) 447 448 def process_class_node(self, n): 449 450 "Process the given class node 'n'." 451 452 path = self.get_namespace_path() 453 454 # To avoid notions of class "versions" where the same definition 455 # might be parameterised with different state and be referenced 456 # elsewhere (as base classes, for example), classes in functions or 457 # conditions are forbidden. 458 459 if self.in_function or self.in_conditional: 460 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 461 path, n.name) 462 return 463 464 # Resolve base classes. 465 466 bases = [] 467 468 for base in n.bases: 469 base_class = self.get_class(base) 470 471 if not base_class: 472 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 473 path, n.name, base) 474 return 475 else: 476 bases.append(base_class) 477 478 # Detect conflicting definitions. Such definitions cause conflicts in 479 # the storage of namespace-related information. 480 481 class_name = self.get_object_path(n.name) 482 ref = self.get_object(class_name, defer=False) 483 484 if ref and ref.static(): 485 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 486 487 # Record bases for the class and retain the class name. 488 # Note that the function class does not inherit from the object class. 489 490 if not bases and class_name != "__builtins__.core.object" and \ 491 class_name != "__builtins__.core.function": 492 493 ref = self.get_object("__builtins__.object") 494 bases.append(ref) 495 496 self.importer.classes[class_name] = self.classes[class_name] = bases 497 self.importer.subclasses[class_name] = set() 498 self.scope_globals[class_name] = set() 499 500 # Set the definition before entering the namespace rather than 501 # afterwards because methods may reference it. In normal Python, 502 # a class is not accessible until the definition is complete, but 503 # methods can generally reference it since upon being called the 504 # class will already exist. 505 506 self.set_definition(n.name, "<class>") 507 508 in_class = self.in_class 509 self.in_class = class_name 510 self.set_instance_attr("__class__", Reference("<class>", class_name)) 511 self.enter_namespace(n.name) 512 513 # Do not provide the special instantiator attributes on the function 514 # class. Function instances provide these attributes. 515 516 if class_name != "__builtins__.core.function": 517 518 self.set_name("__fn__") # special instantiator attribute 519 self.set_name("__args__") # special instantiator attribute 520 521 # Provide leafname, parent and context attributes. 522 523 parent, leafname = class_name.rsplit(".", 1) 524 self.set_name("__name__", self.get_constant("string", leafname).reference()) 525 526 if class_name != "__builtins__.core.function": 527 self.set_name("__parent__") 528 529 self.process_structure_node(n.code) 530 self.exit_namespace() 531 self.in_class = in_class 532 533 def process_from_node(self, n): 534 535 "Process the given node 'n', importing from another module." 536 537 path = self.get_namespace_path() 538 539 module_name, names = self.get_module_name(n) 540 if module_name == self.name: 541 raise InspectError("Cannot import from the current module.", path, n) 542 543 self.queue_module(module_name) 544 545 # Attempt to obtain the referenced objects. 546 547 for name, alias in n.names: 548 if name == "*": 549 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 550 551 # Explicit names. 552 553 ref = self.import_name_from_module(name, module_name) 554 value = ResolvedNameRef(alias or name, ref) 555 self.set_general_local(alias or name, value) 556 557 def process_function_node(self, n, name): 558 559 """ 560 Process the given function or lambda node 'n' with the given 'name'. 561 """ 562 563 is_lambda = isinstance(n, compiler.ast.Lambda) 564 565 # Where a function is declared conditionally, use a separate name for 566 # the definition, and assign the definition to the stated name. 567 568 if (self.in_conditional or self.in_function) and not is_lambda: 569 original_name = name 570 name = self.get_lambda_name() 571 else: 572 original_name = None 573 574 # Detect conflicting definitions. Such definitions cause conflicts in 575 # the storage of namespace-related information. 576 577 function_name = self.get_object_path(name) 578 ref = self.get_object(function_name, defer=False) 579 580 if ref and ref.static(): 581 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 582 583 # Initialise argument and local records. 584 585 argnames = get_argnames(n.argnames) 586 is_method = self.in_class and not self.in_function 587 588 # Remove explicit "self" from method parameters. 589 590 if is_method and argnames and argnames[0] == "self": 591 del argnames[0] 592 593 # Copy and propagate the parameters. 594 595 self.importer.function_parameters[function_name] = \ 596 self.function_parameters[function_name] = argnames[:] 597 598 # Define all arguments/parameters in the local namespace. 599 600 locals = \ 601 self.importer.function_locals[function_name] = \ 602 self.function_locals[function_name] = {} 603 604 # Insert "self" into method locals. 605 606 if is_method: 607 argnames.insert(0, "self") 608 609 # Define "self" in terms of the class if in a method. 610 # This does not diminish the need for type-narrowing in the deducer. 611 612 if argnames: 613 if self.in_class and not self.in_function and argnames[0] == "self": 614 locals[argnames[0]] = Reference("<instance>", self.in_class) 615 else: 616 locals[argnames[0]] = Reference("<var>") 617 618 for argname in argnames[1:]: 619 locals[argname] = Reference("<var>") 620 621 globals = self.scope_globals[function_name] = set() 622 623 # Process the defaults. 624 625 defaults = self.importer.function_defaults[function_name] = \ 626 self.function_defaults[function_name] = [] 627 628 for argname, default in compiler.ast.get_defaults(n): 629 if default: 630 631 # Obtain any reference for the default. 632 633 name_ref = self.process_structure_node(default) 634 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 635 636 # Reset conditional tracking to focus on the function contents. 637 638 in_conditional = self.in_conditional 639 self.in_conditional = False 640 641 in_function = self.in_function 642 self.in_function = function_name 643 644 self.enter_namespace(name) 645 646 # Define a leafname attribute value for the function instance. 647 648 ref = self.get_builtin_class("string") 649 self.reserve_constant(function_name, name, ref.get_origin()) 650 651 # Track attribute usage within the namespace. 652 653 path = self.get_namespace_path() 654 655 self.start_tracking(locals) 656 self.process_structure_node(n.code) 657 returns_value = self.stop_tracking() 658 659 # Record any null result. 660 661 is_initialiser = is_method and name == "__init__" 662 663 if not returns_value and not is_initialiser: 664 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 665 666 # Exit to the parent. 667 668 self.exit_namespace() 669 670 # Update flags. 671 672 self.in_function = in_function 673 self.in_conditional = in_conditional 674 675 # Define the function using the appropriate name. 676 677 self.set_definition(name, "<function>") 678 679 # Where a function is set conditionally, assign the name. 680 681 if original_name: 682 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 683 684 def process_global_node(self, n): 685 686 """ 687 Process the given "global" node 'n'. 688 """ 689 690 path = self.get_namespace_path() 691 692 if path != self.name: 693 self.scope_globals[path].update(n.names) 694 695 def process_if_node(self, n): 696 697 """ 698 Process the given "if" node 'n'. 699 """ 700 701 tracker = self.trackers[-1] 702 tracker.new_branchpoint() 703 704 for test, body in n.tests: 705 self.process_structure_node(test) 706 707 tracker.new_branch() 708 709 in_conditional = self.in_conditional 710 self.in_conditional = True 711 self.process_structure_node(body) 712 self.in_conditional = in_conditional 713 714 tracker.shelve_branch() 715 716 # Maintain a branch for the else clause. 717 718 tracker.new_branch() 719 if n.else_: 720 self.process_structure_node(n.else_) 721 tracker.shelve_branch() 722 723 tracker.merge_branches() 724 725 def process_import_node(self, n): 726 727 "Process the given import node 'n'." 728 729 path = self.get_namespace_path() 730 731 # Load the mentioned module. 732 733 for name, alias in n.names: 734 if name == self.name: 735 raise InspectError("Cannot import the current module.", path, n) 736 737 self.set_module(alias or name.split(".")[-1], name) 738 self.queue_module(name, True) 739 740 def process_invocation_node(self, n): 741 742 "Process the given invocation node 'n'." 743 744 path = self.get_namespace_path() 745 746 in_invocation = self.in_invocation 747 self.in_invocation = None 748 749 # Process the arguments. 750 751 keywords = set() 752 753 for arg in n.args: 754 self.process_structure_node(arg) 755 if isinstance(arg, compiler.ast.Keyword): 756 keywords.add(arg.name) 757 758 keywords = list(keywords) 759 keywords.sort() 760 761 # Communicate to the invocation target expression that it forms the 762 # target of an invocation, potentially affecting attribute accesses. 763 764 self.in_invocation = len(n.args), keywords 765 766 # Process the expression, obtaining any identified reference. 767 768 name_ref = self.process_structure_node(n.node) 769 self.in_invocation = in_invocation 770 771 # Detect class invocations. 772 773 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 774 return InstanceRef(name_ref.reference().instance_of()) 775 776 elif isinstance(name_ref, (NameRef, AccessRef)): 777 return InvocationRef(name_ref) 778 779 # Provide a general reference to indicate that something is produced 780 # by the invocation, useful for retaining assignment expression 781 # details. 782 783 return VariableRef() 784 785 def process_lambda_node(self, n): 786 787 "Process the given lambda node 'n'." 788 789 name = self.get_lambda_name() 790 self.process_function_node(n, name) 791 792 origin = self.get_object_path(name) 793 794 if self.function_defaults.get(origin): 795 return None 796 else: 797 return ResolvedNameRef(name, Reference("<function>", origin)) 798 799 def process_logical_node(self, n): 800 801 "Process the given operator node 'n'." 802 803 self.process_operator_chain(n.nodes, self.process_structure_node) 804 805 def process_name_node(self, n): 806 807 "Process the given name node 'n'." 808 809 path = self.get_namespace_path() 810 811 # Find predefined constant names before anything else. 812 813 if n.name in predefined_constants: 814 ref = self.get_builtin(n.name) 815 value = ResolvedNameRef(n.name, ref) 816 return value 817 818 # Special names that have already been identified. 819 820 if n.name.startswith("$"): 821 value = self.get_special(n.name) 822 if value: 823 return value 824 825 # Special case for operator functions introduced through code 826 # transformations. 827 828 if n.name.startswith("$op"): 829 830 # Obtain the location of the actual function defined in the operator 831 # package. 832 833 op = n.name[len("$op"):] 834 835 # Attempt to get a reference. 836 837 ref = self.import_name_from_module(op, "operator") 838 839 # Record the imported name and provide the resolved name reference. 840 841 value = ResolvedNameRef(n.name, ref) 842 self.set_special(n.name, value) 843 return value 844 845 # Special case for print operations. 846 847 elif n.name.startswith("$print"): 848 849 # Attempt to get a reference. 850 851 ref = self.get_builtin("print_") 852 853 # Record the imported name and provide the resolved name reference. 854 855 value = ResolvedNameRef(n.name, ref) 856 self.set_special(n.name, value) 857 return value 858 859 # Test for self usage, which is only allowed in methods. 860 861 if n.name == "self" and not (self.in_function and self.in_class): 862 raise InspectError("Use of self is only allowed in methods.", path, n) 863 864 # Record usage of the name. 865 866 self.record_name(n.name) 867 868 # Search for unknown names in non-function scopes immediately. 869 # Temporary names should not be re-used between scopes. 870 # External names in functions are resolved later. 871 872 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 873 874 if ref: 875 self.record_name_access(n.name, True) 876 return ResolvedNameRef(n.name, ref, is_global=True) 877 878 # Explicitly-declared global names. 879 880 elif self.in_function and n.name in self.scope_globals[path]: 881 self.record_name_access(n.name, True) 882 return NameRef(n.name, is_global=True) 883 884 # Examine other names. 885 886 else: 887 888 # Check local names. 889 890 access_number = self.record_name_access(n.name) 891 892 # Local name. 893 894 if access_number is not None: 895 return LocalNameRef(n.name, access_number) 896 897 # Possible global or built-in name. 898 899 else: 900 self.record_name_access(n.name, True) 901 return NameRef(n.name, is_global=True) 902 903 def record_name_access(self, name, is_global=False): 904 905 """ 906 Record an access involving 'name' if the name is being tracked, using 907 'is_global' to indicate whether the name is global. 908 """ 909 910 name = self.get_name_for_tracking(name, is_global=is_global) 911 branches = self.trackers[-1].tracking_name(name) 912 if branches: 913 self.record_branches_for_access(branches, name, None) 914 return self.record_access_details(name, None, self.in_assignment, 915 self.in_invocation) 916 return None 917 918 def process_operator_chain(self, nodes, fn): 919 920 """ 921 Process the given chain of 'nodes', applying 'fn' to each node or item. 922 Each node starts a new conditional region, effectively making a deeply- 923 nested collection of if-like statements. 924 """ 925 926 tracker = self.trackers[-1] 927 928 for item in nodes: 929 tracker.new_branchpoint() 930 tracker.new_branch() 931 fn(item) 932 933 for item in nodes[:-1]: 934 tracker.shelve_branch() 935 tracker.new_branch() 936 tracker.shelve_branch() 937 tracker.merge_branches() 938 939 tracker.shelve_branch() 940 tracker.merge_branches() 941 942 def process_try_node(self, n): 943 944 """ 945 Process the given "try...except" node 'n'. 946 """ 947 948 self.record_exception_handler() 949 950 tracker = self.trackers[-1] 951 tracker.new_branchpoint() 952 953 self.process_structure_node(n.body) 954 955 for name, var, handler in n.handlers: 956 if name is not None: 957 self.process_structure_node(name) 958 959 # Any abandoned branches from the body can now be resumed in a new 960 # branch. 961 962 tracker.resume_abandoned_branches() 963 964 # Establish the local for the handler. 965 966 if var is not None: 967 self.process_assignment_node(var, None) 968 if handler is not None: 969 self.process_structure_node(handler) 970 971 tracker.shelve_branch() 972 973 # The else clause maintains the usage from the body but without the 974 # abandoned branches since they would never lead to the else clause 975 # being executed. 976 977 if n.else_: 978 tracker.new_branch() 979 self.process_structure_node(n.else_) 980 tracker.shelve_branch() 981 982 # Without an else clause, a null branch propagates the successful 983 # outcome. 984 985 else: 986 tracker.new_branch() 987 tracker.shelve_branch() 988 989 tracker.merge_branches() 990 991 def process_try_finally_node(self, n): 992 993 """ 994 Process the given "try...finally" node 'n'. 995 """ 996 997 self.record_exception_handler() 998 999 tracker = self.trackers[-1] 1000 self.process_structure_node(n.body) 1001 1002 # Any abandoned branches from the body can now be resumed. 1003 1004 branches = tracker.resume_all_abandoned_branches() 1005 self.process_structure_node(n.final) 1006 1007 # At the end of the finally clause, abandoned branches are discarded. 1008 1009 tracker.restore_active_branches(branches) 1010 1011 def process_while_node(self, n): 1012 1013 "Process the given while node 'n'." 1014 1015 tracker = self.trackers[-1] 1016 tracker.new_branchpoint(loop_node=True) 1017 1018 # Evaluate any test or iterator outside the loop. 1019 1020 self.process_structure_node(n.test) 1021 1022 # Propagate attribute usage to branches. 1023 1024 tracker.new_branch(loop_node=True) 1025 1026 # Enter the loop. 1027 1028 in_conditional = self.in_conditional 1029 self.in_conditional = True 1030 self.process_structure_node(n.body) 1031 self.in_conditional = in_conditional 1032 1033 # Continuing branches are resumed before any test. 1034 1035 tracker.resume_continuing_branches() 1036 1037 # Evaluate any continuation test within the body. 1038 1039 self.process_structure_node(n.test) 1040 1041 tracker.shelve_branch(loop_node=True) 1042 1043 # Support the non-looping condition. 1044 1045 tracker.new_branch() 1046 tracker.shelve_branch() 1047 1048 tracker.merge_branches() 1049 1050 # Evaluate any else clause outside branches. 1051 1052 if n.else_: 1053 self.process_structure_node(n.else_) 1054 1055 # Connect broken branches to the code after any loop. 1056 1057 tracker.resume_broken_branches() 1058 1059 # Branch tracking methods. 1060 1061 def start_tracking(self, names): 1062 1063 """ 1064 Start tracking attribute usage for names in the current namespace, 1065 immediately registering the given 'names'. 1066 """ 1067 1068 path = self.get_namespace_path() 1069 parent = self.trackers[-1] 1070 tracker = BranchTracker() 1071 self.trackers.append(tracker) 1072 1073 # Record the given names established as new branches. 1074 1075 tracker.assign_names(names) 1076 1077 def assign_name(self, name, name_ref): 1078 1079 "Assign to 'name' the given 'name_ref' in the current namespace." 1080 1081 name = self.get_name_for_tracking(name) 1082 self.trackers[-1].assign_names([name], [name_ref]) 1083 1084 def stop_tracking(self): 1085 1086 """ 1087 Stop tracking attribute usage, recording computed usage for the current 1088 namespace. Indicate whether a value is always returned from the 1089 namespace. 1090 """ 1091 1092 path = self.get_namespace_path() 1093 tracker = self.trackers.pop() 1094 self.record_assignments_for_access(tracker) 1095 1096 self.attr_usage[path] = tracker.get_all_usage() 1097 self.name_initialisers[path] = tracker.get_all_values() 1098 1099 return tracker.returns_value() 1100 1101 def start_tracking_in_module(self): 1102 1103 "Start tracking attribute usage in the module." 1104 1105 tracker = BranchTracker() 1106 self.trackers.append(tracker) 1107 1108 def stop_tracking_in_module(self): 1109 1110 "Stop tracking attribute usage in the module." 1111 1112 tracker = self.trackers[0] 1113 self.record_assignments_for_access(tracker) 1114 self.attr_usage[self.name] = tracker.get_all_usage() 1115 self.name_initialisers[self.name] = tracker.get_all_values() 1116 1117 def record_assignments_for_access(self, tracker): 1118 1119 """ 1120 For the current path, use the given 'tracker' to record assignment 1121 version information for attribute accesses. 1122 """ 1123 1124 path = self.get_path_for_access() 1125 1126 if not self.attr_accessor_branches.has_key(path): 1127 return 1128 1129 init_item(self.attr_accessors, path, dict) 1130 attr_accessors = self.attr_accessors[path] 1131 1132 # Obtain the branches applying during each access. 1133 1134 for access, all_branches in self.attr_accessor_branches[path].items(): 1135 name, attrnames = access 1136 init_item(attr_accessors, access, list) 1137 1138 # Obtain the assignments applying to each branch. 1139 1140 for branches in all_branches: 1141 positions = tracker.get_assignment_positions_for_branches(name, branches) 1142 1143 # Detect missing name information. 1144 1145 if None in positions: 1146 globals = self.global_attr_accesses.get(path) 1147 accesses = globals and globals.get(name) 1148 if not accesses: 1149 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1150 self.get_namespace_path(), name) 1151 positions.remove(None) 1152 1153 attr_accessors[access].append(positions) 1154 1155 def record_branches_for_access(self, branches, name, attrnames): 1156 1157 """ 1158 Record the given 'branches' for an access involving the given 'name' and 1159 'attrnames'. 1160 """ 1161 1162 access = name, attrnames 1163 path = self.get_path_for_access() 1164 1165 init_item(self.attr_accessor_branches, path, dict) 1166 attr_accessor_branches = self.attr_accessor_branches[path] 1167 1168 init_item(attr_accessor_branches, access, list) 1169 attr_accessor_branches[access].append(branches) 1170 1171 def record_access_details(self, name, attrnames, assignment, invocation): 1172 1173 """ 1174 For the given 'name' and 'attrnames', record an access indicating 1175 whether an 'assignment' or an 'invocation' is occurring. 1176 1177 These details correspond to accesses otherwise recorded by the attribute 1178 accessor and attribute access dictionaries. 1179 """ 1180 1181 access = name, attrnames 1182 path = self.get_path_for_access() 1183 1184 init_item(self.attr_access_modifiers, path, dict) 1185 init_item(self.attr_access_modifiers[path], access, list) 1186 1187 access_number = len(self.attr_access_modifiers[path][access]) 1188 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1189 return access_number 1190 1191 def record_global_access_details(self, name, attrnames): 1192 1193 """ 1194 Record details of a global access via the given 'name' involving the 1195 indicated 'attrnames'. 1196 """ 1197 1198 path = self.get_namespace_path() 1199 1200 init_item(self.global_attr_accesses, path, dict) 1201 init_item(self.global_attr_accesses[path], name, set) 1202 self.global_attr_accesses[path][name].add(attrnames) 1203 1204 # Namespace modification. 1205 1206 def record_name(self, name): 1207 1208 "Record the use of 'name' in a namespace." 1209 1210 path = self.get_namespace_path() 1211 init_item(self.names_used, path, set) 1212 self.names_used[path].add(name) 1213 1214 def set_module(self, name, module_name): 1215 1216 """ 1217 Set a module in the current namespace using the given 'name' associated 1218 with the corresponding 'module_name'. 1219 """ 1220 1221 if name: 1222 self.set_general_local(name, Reference("<module>", module_name)) 1223 1224 def set_definition(self, name, kind): 1225 1226 """ 1227 Set the definition having the given 'name' and 'kind'. 1228 1229 Definitions are set in the static namespace hierarchy, but they can also 1230 be recorded for function locals. 1231 """ 1232 1233 if self.is_global(name): 1234 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1235 self.get_namespace_path(), name) 1236 1237 path = self.get_object_path(name) 1238 self.set_object(path, kind) 1239 1240 ref = self.get_object(path) 1241 if ref.get_kind() == "<var>": 1242 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1243 self.get_namespace_path(), name) 1244 1245 if not self.is_global(name) and self.in_function: 1246 self.set_function_local(name, ref) 1247 1248 def set_function_local(self, name, ref=None): 1249 1250 "Set the local with the given 'name' and optional 'ref'." 1251 1252 path = self.get_namespace_path() 1253 locals = self.function_locals[path] 1254 used = self.names_used.get(path) 1255 1256 if not locals.has_key(name) and used and name in used: 1257 raise InspectError("Name %s assigned locally but used previously." % name, path) 1258 1259 multiple = not ref or locals.has_key(name) and locals[name] != ref 1260 locals[name] = multiple and Reference("<var>") or ref 1261 1262 def assign_general_local(self, name, name_ref): 1263 1264 """ 1265 Set for 'name' the given 'name_ref', recording the name for attribute 1266 usage tracking. 1267 """ 1268 1269 self.set_general_local(name, name_ref) 1270 self.assign_name(name, name_ref) 1271 1272 def set_general_local(self, name, value=None): 1273 1274 """ 1275 Set the 'name' with optional 'value' in any kind of local namespace, 1276 where the 'value' should be a reference if specified. 1277 """ 1278 1279 init_value = self.get_initialising_value(value) 1280 1281 # Module global names. 1282 1283 if self.is_global(name): 1284 path = self.get_global_path(name) 1285 self.set_object(path, init_value) 1286 1287 # Function local names. 1288 1289 elif self.in_function: 1290 self.set_function_local(name, init_value) 1291 1292 # Other namespaces (classes). 1293 1294 else: 1295 self.set_name(name, init_value) 1296 1297 def set_name(self, name, ref=None): 1298 1299 "Attach the 'name' with optional 'ref' to the current namespace." 1300 1301 self.set_object(self.get_object_path(name), ref) 1302 1303 def set_instance_attr(self, name, ref=None): 1304 1305 """ 1306 Add an instance attribute of the given 'name' to the current class, 1307 using the optional 'ref'. 1308 """ 1309 1310 self._set_instance_attr(self.in_class, name, ref) 1311 1312 def _set_instance_attr(self, path, name, ref=None): 1313 1314 init_item(self.instance_attrs, path, set) 1315 self.instance_attrs[path].add(name) 1316 1317 if ref: 1318 init_item(self.instance_attr_constants, path, dict) 1319 self.instance_attr_constants[path][name] = ref 1320 1321 def get_initialising_value(self, value): 1322 1323 "Return a suitable initialiser reference for 'value'." 1324 1325 # Includes LiteralSequenceRef, ResolvedNameRef... 1326 1327 if isinstance(value, (NameRef, AccessRef, InstanceRef)): 1328 return value.reference() 1329 1330 # In general, invocations do not produce known results. However, the 1331 # name initialisers are resolved once a module has been inspected. 1332 1333 elif isinstance(value, InvocationRef): 1334 return value.reference() 1335 1336 # Variable references are unknown results. 1337 1338 elif isinstance(value, VariableRef): 1339 return value.reference() 1340 1341 else: 1342 return value 1343 1344 # Static, program-relative naming. 1345 1346 def find_name(self, name): 1347 1348 """ 1349 Return the qualified name for the given 'name' used in the current 1350 non-function namespace. 1351 """ 1352 1353 path = self.get_namespace_path() 1354 ref = None 1355 1356 if not self.in_function and name not in predefined_constants: 1357 if self.in_class: 1358 ref = self.get_object(self.get_object_path(name), False) 1359 if not ref: 1360 ref = self.get_global_or_builtin(name) 1361 1362 return ref 1363 1364 def get_class(self, node): 1365 1366 """ 1367 Use the given 'node' to obtain the identity of a class. Return a 1368 reference for the class. Unresolved dependencies are permitted and must 1369 be resolved later. 1370 """ 1371 1372 ref = self._get_class(node) 1373 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1374 1375 def _get_class(self, node): 1376 1377 """ 1378 Use the given 'node' to find a class definition. Return a reference to 1379 the class. 1380 """ 1381 1382 if isinstance(node, compiler.ast.Getattr): 1383 1384 # Obtain the identity of the access target. 1385 1386 ref = self._get_class(node.expr) 1387 1388 # Where the target is a class or module, obtain the identity of the 1389 # attribute. 1390 1391 if ref.has_kind(["<function>", "<var>"]): 1392 return None 1393 else: 1394 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1395 return self.get_object(attrname) 1396 1397 # Names can be module-level or built-in. 1398 1399 elif isinstance(node, compiler.ast.Name): 1400 1401 # Record usage of the name and attempt to identify it. 1402 1403 self.record_name(node.name) 1404 return self.find_name(node.name) 1405 else: 1406 return None 1407 1408 def get_constant(self, name, value): 1409 1410 "Return a constant reference for the given type 'name' and 'value'." 1411 1412 ref = self.get_builtin_class(name) 1413 return self.get_constant_reference(ref, value) 1414 1415 def get_literal_instance(self, n, name=None): 1416 1417 """ 1418 For node 'n', return a reference to an instance of 'name', or if 'name' 1419 is not specified, deduce the type from the value. 1420 """ 1421 1422 # Handle stray None constants (Sliceobj seems to produce them). 1423 1424 if name == "NoneType": 1425 return self.process_name_node(compiler.ast.Name("None")) 1426 1427 # Obtain the details of the literal itself. 1428 # An alias to the type is generated for sequences. 1429 1430 if name in ("dict", "list", "tuple"): 1431 ref = self.get_builtin_class(name) 1432 self.set_special_literal(name, ref) 1433 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1434 1435 # Constant values are independently recorded. 1436 1437 else: 1438 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1439 ref = self.get_builtin_class(typename) 1440 return self.get_constant_reference(ref, value, encoding) 1441 1442 # Special names. 1443 1444 def get_special(self, name): 1445 1446 "Return any stored value for the given special 'name'." 1447 1448 value = self.special.get(name) 1449 if value: 1450 ref, paths = value 1451 else: 1452 ref = None 1453 return ref 1454 1455 def set_special(self, name, value): 1456 1457 """ 1458 Set a special 'name' that merely tracks the use of an implicit object 1459 'value'. 1460 """ 1461 1462 if not self.special.has_key(name): 1463 paths = set() 1464 self.special[name] = value, paths 1465 else: 1466 _ref, paths = self.special[name] 1467 1468 paths.add(self.get_namespace_path()) 1469 1470 def set_special_literal(self, name, ref): 1471 1472 """ 1473 Set a special name for the literal type 'name' having type 'ref'. Such 1474 special names provide a way of referring to literal object types. 1475 """ 1476 1477 literal_name = "$L%s" % name 1478 value = ResolvedNameRef(literal_name, ref) 1479 self.set_special(literal_name, value) 1480 1481 # Exceptions. 1482 1483 def record_exception_handler(self): 1484 1485 "Record the current namespace as employing an exception handler." 1486 1487 self.exception_namespaces.add(self.get_namespace_path()) 1488 1489 # Return values. 1490 1491 def record_return_value(self, expr): 1492 1493 "Record the given return 'expr'." 1494 1495 path = self.get_namespace_path() 1496 l = init_item(self.return_values, path, list) 1497 l.append(expr) 1498 if not self.importer.all_return_values.has_key(path): 1499 self.importer.all_return_values[path] = l 1500 1501 # vim: tabstop=4 expandtab shiftwidth=4