1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("string", self.name).reference()) 81 self.set_name("__file__", self.get_constant("string", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("string", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 # Obtain any completed chain and return the reference to it. 362 363 name_ref = self.process_attribute_chain(n) 364 365 if self.have_access_expression(n): 366 return name_ref 367 368 # Where the start of the chain of attributes has been reached, determine 369 # the complete access. 370 371 # Given a non-access node, this chain can be handled in its entirety, 372 # either being name-based and thus an access rooted on a name, or being 373 # based on some other node and thus an anonymous access of some kind. 374 375 path = self.get_namespace_path() 376 377 # Start with the the full attribute chain. 378 379 remaining = self.attrs 380 attrnames = ".".join(remaining) 381 382 # If the accessor cannot be identified, or where attributes 383 # remain in an attribute chain, record the anonymous accesses. 384 385 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 386 387 init_item(self.attr_accesses, path, set) 388 self.attr_accesses[path].add(attrnames) 389 390 self.record_access_details(None, attrnames, self.in_assignment, 391 self.in_invocation) 392 del self.attrs[0] 393 return 394 395 # Name-based accesses will handle the first attribute in a 396 # chain. 397 398 else: 399 attrname = remaining[0] 400 401 # Attribute assignments are used to identify instance attributes. 402 403 if isinstance(n, compiler.ast.AssAttr) and \ 404 self.in_class and self.in_function and n.expr.name == "self": 405 406 self.set_instance_attr(attrname) 407 408 # Record attribute usage using any name local to this namespace, 409 # if assigned in the namespace, or using an external name 410 # (presently just globals within classes). 411 412 name = self.get_name_for_tracking(name_ref.name, name_ref) 413 tracker = self.trackers[-1] 414 415 immediate_access = len(self.attrs) == 1 416 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 417 418 # Record global-based chains for subsequent resolution. 419 420 if name_ref.is_global_name(): 421 self.record_global_access_details(name, attrnames) 422 423 # Make sure the name is being tracked: global names will not 424 # already be initialised in a branch and must be added 425 # explicitly. 426 427 if not tracker.have_name(name): 428 tracker.assign_names([name]) 429 if self.in_function: 430 self.scope_globals[path].add(name) 431 432 # Record attribute usage in the tracker, and record the branch 433 # information for the access. 434 435 branches = tracker.use_attribute(name, attrname, 436 self.in_invocation is not None, assignment) 437 438 if not branches: 439 raise InspectError("Name %s is accessed using %s before an assignment." % ( 440 name, attrname), path, n) 441 442 self.record_branches_for_access(branches, name, attrnames) 443 access_number = self.record_access_details(name, attrnames, 444 self.in_assignment, self.in_invocation) 445 446 del self.attrs[0] 447 return AccessRef(name, attrnames, access_number) 448 449 def process_class_node(self, n): 450 451 "Process the given class node 'n'." 452 453 path = self.get_namespace_path() 454 455 # To avoid notions of class "versions" where the same definition 456 # might be parameterised with different state and be referenced 457 # elsewhere (as base classes, for example), classes in functions or 458 # conditions are forbidden. 459 460 if self.in_function or self.in_conditional: 461 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 462 path, n.name) 463 return 464 465 # Resolve base classes. 466 467 bases = [] 468 469 for base in n.bases: 470 base_class = self.get_class(base) 471 472 if not base_class: 473 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 474 path, n.name, base) 475 return 476 else: 477 bases.append(base_class) 478 479 # Detect conflicting definitions. Such definitions cause conflicts in 480 # the storage of namespace-related information. 481 482 class_name = self.get_object_path(n.name) 483 ref = self.get_object(class_name, defer=False) 484 485 if ref and ref.static(): 486 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 487 488 # Record bases for the class and retain the class name. 489 # Note that the function class does not inherit from the object class. 490 491 if not bases and class_name != "__builtins__.core.object" and \ 492 class_name != "__builtins__.core.function": 493 494 ref = self.get_object("__builtins__.object") 495 bases.append(ref) 496 497 self.importer.classes[class_name] = self.classes[class_name] = bases 498 self.importer.subclasses[class_name] = set() 499 self.scope_globals[class_name] = set() 500 501 # Set the definition before entering the namespace rather than 502 # afterwards because methods may reference it. In normal Python, 503 # a class is not accessible until the definition is complete, but 504 # methods can generally reference it since upon being called the 505 # class will already exist. 506 507 self.set_definition(n.name, "<class>") 508 509 in_class = self.in_class 510 self.in_class = class_name 511 self.set_instance_attr("__class__", Reference("<class>", class_name)) 512 self.enter_namespace(n.name) 513 514 # Do not provide the special instantiator attributes on the function 515 # class. Function instances provide these attributes. 516 517 if class_name != "__builtins__.core.function": 518 519 self.set_name("__fn__") # special instantiator attribute 520 self.set_name("__args__") # special instantiator attribute 521 522 # Provide leafname, parent and context attributes. 523 524 parent, leafname = class_name.rsplit(".", 1) 525 self.set_name("__name__", self.get_constant("string", leafname).reference()) 526 527 if class_name != "__builtins__.core.function": 528 self.set_name("__parent__") 529 530 self.process_structure_node(n.code) 531 self.exit_namespace() 532 self.in_class = in_class 533 534 def process_from_node(self, n): 535 536 "Process the given node 'n', importing from another module." 537 538 path = self.get_namespace_path() 539 540 module_name, names = self.get_module_name(n) 541 if module_name == self.name: 542 raise InspectError("Cannot import from the current module.", path, n) 543 544 self.queue_module(module_name) 545 546 # Attempt to obtain the referenced objects. 547 548 for name, alias in n.names: 549 if name == "*": 550 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 551 552 # Explicit names. 553 554 ref = self.import_name_from_module(name, module_name) 555 value = ResolvedNameRef(alias or name, ref) 556 self.set_general_local(alias or name, value) 557 558 def process_function_node(self, n, name): 559 560 """ 561 Process the given function or lambda node 'n' with the given 'name'. 562 """ 563 564 is_lambda = isinstance(n, compiler.ast.Lambda) 565 566 # Where a function is declared conditionally, use a separate name for 567 # the definition, and assign the definition to the stated name. 568 569 if (self.in_conditional or self.in_function) and not is_lambda: 570 original_name = name 571 name = self.get_lambda_name() 572 else: 573 original_name = None 574 575 # Detect conflicting definitions. Such definitions cause conflicts in 576 # the storage of namespace-related information. 577 578 function_name = self.get_object_path(name) 579 ref = self.get_object(function_name, defer=False) 580 581 if ref and ref.static(): 582 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 583 584 # Initialise argument and local records. 585 586 argnames = get_argnames(n.argnames) 587 is_method = self.in_class and not self.in_function 588 589 # Remove explicit "self" from method parameters. 590 591 if is_method and argnames and argnames[0] == "self": 592 del argnames[0] 593 594 # Copy and propagate the parameters. 595 596 self.importer.function_parameters[function_name] = \ 597 self.function_parameters[function_name] = argnames[:] 598 599 # Define all arguments/parameters in the local namespace. 600 601 locals = \ 602 self.importer.function_locals[function_name] = \ 603 self.function_locals[function_name] = {} 604 605 # Insert "self" into method locals. 606 607 if is_method: 608 argnames.insert(0, "self") 609 610 # Define "self" in terms of the class if in a method. 611 # This does not diminish the need for type-narrowing in the deducer. 612 613 if argnames: 614 if self.in_class and not self.in_function and argnames[0] == "self": 615 locals[argnames[0]] = Reference("<instance>", self.in_class) 616 else: 617 locals[argnames[0]] = Reference("<var>") 618 619 for argname in argnames[1:]: 620 locals[argname] = Reference("<var>") 621 622 globals = self.scope_globals[function_name] = set() 623 624 # Process the defaults. 625 626 defaults = self.importer.function_defaults[function_name] = \ 627 self.function_defaults[function_name] = [] 628 629 for argname, default in compiler.ast.get_defaults(n): 630 if default: 631 632 # Obtain any reference for the default. 633 634 name_ref = self.process_structure_node(default) 635 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 636 637 # Reset conditional tracking to focus on the function contents. 638 639 in_conditional = self.in_conditional 640 self.in_conditional = False 641 642 in_function = self.in_function 643 self.in_function = function_name 644 645 self.enter_namespace(name) 646 647 # Define a leafname attribute value for the function instance. 648 649 ref = self.get_builtin_class("string") 650 self.reserve_constant(function_name, name, ref.get_origin()) 651 652 # Track attribute usage within the namespace. 653 654 path = self.get_namespace_path() 655 656 self.start_tracking(locals) 657 self.process_structure_node(n.code) 658 returns_value = self.stop_tracking() 659 660 # Record any null result. 661 662 is_initialiser = is_method and name == "__init__" 663 664 if not returns_value and not is_initialiser: 665 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 666 667 # Exit to the parent. 668 669 self.exit_namespace() 670 671 # Update flags. 672 673 self.in_function = in_function 674 self.in_conditional = in_conditional 675 676 # Define the function using the appropriate name. 677 678 self.set_definition(name, "<function>") 679 680 # Where a function is set conditionally, assign the name. 681 682 if original_name: 683 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 684 685 def process_global_node(self, n): 686 687 """ 688 Process the given "global" node 'n'. 689 """ 690 691 path = self.get_namespace_path() 692 693 if path != self.name: 694 self.scope_globals[path].update(n.names) 695 696 def process_if_node(self, n): 697 698 """ 699 Process the given "if" node 'n'. 700 """ 701 702 tracker = self.trackers[-1] 703 tracker.new_branchpoint() 704 705 for test, body in n.tests: 706 self.process_structure_node(test) 707 708 tracker.new_branch() 709 710 in_conditional = self.in_conditional 711 self.in_conditional = True 712 self.process_structure_node(body) 713 self.in_conditional = in_conditional 714 715 tracker.shelve_branch() 716 717 # Maintain a branch for the else clause. 718 719 tracker.new_branch() 720 if n.else_: 721 self.process_structure_node(n.else_) 722 tracker.shelve_branch() 723 724 tracker.merge_branches() 725 726 def process_import_node(self, n): 727 728 "Process the given import node 'n'." 729 730 path = self.get_namespace_path() 731 732 # Load the mentioned module. 733 734 for name, alias in n.names: 735 if name == self.name: 736 raise InspectError("Cannot import the current module.", path, n) 737 738 self.set_module(alias or name.split(".")[-1], name) 739 self.queue_module(name, True) 740 741 def process_invocation_node(self, n): 742 743 "Process the given invocation node 'n'." 744 745 path = self.get_namespace_path() 746 747 in_invocation = self.in_invocation 748 self.in_invocation = None 749 750 # Process the arguments. 751 752 keywords = set() 753 754 for arg in n.args: 755 self.process_structure_node(arg) 756 if isinstance(arg, compiler.ast.Keyword): 757 keywords.add(arg.name) 758 759 keywords = list(keywords) 760 keywords.sort() 761 762 # Communicate to the invocation target expression that it forms the 763 # target of an invocation, potentially affecting attribute accesses. 764 765 self.in_invocation = len(n.args), keywords 766 767 # Process the expression, obtaining any identified reference. 768 769 name_ref = self.process_structure_node(n.node) 770 self.in_invocation = in_invocation 771 772 # Detect class invocations. 773 774 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 775 return InstanceRef(name_ref.reference().instance_of()) 776 777 elif isinstance(name_ref, (NameRef, AccessRef)): 778 return InvocationRef(name_ref) 779 780 # Provide a general reference to indicate that something is produced 781 # by the invocation, useful for retaining assignment expression 782 # details. 783 784 return VariableRef() 785 786 def process_lambda_node(self, n): 787 788 "Process the given lambda node 'n'." 789 790 name = self.get_lambda_name() 791 self.process_function_node(n, name) 792 793 origin = self.get_object_path(name) 794 795 if self.function_defaults.get(origin): 796 return None 797 else: 798 return ResolvedNameRef(name, Reference("<function>", origin)) 799 800 def process_logical_node(self, n): 801 802 "Process the given operator node 'n'." 803 804 return self.process_operator_chain(n.nodes) 805 806 def process_name_node(self, n): 807 808 "Process the given name node 'n'." 809 810 path = self.get_namespace_path() 811 812 # Find predefined constant names before anything else. 813 814 if n.name in predefined_constants: 815 ref = self.get_builtin(n.name) 816 value = ResolvedNameRef(n.name, ref) 817 return value 818 819 # Special names that have already been identified. 820 821 if n.name.startswith("$"): 822 value = self.get_special(n.name) 823 if value: 824 return value 825 826 # Special case for operator functions introduced through code 827 # transformations. 828 829 if n.name.startswith("$op"): 830 831 # Obtain the location of the actual function defined in the operator 832 # package. 833 834 op = n.name[len("$op"):] 835 836 # Attempt to get a reference. 837 838 ref = self.import_name_from_module(op, "operator") 839 840 # Record the imported name and provide the resolved name reference. 841 842 value = ResolvedNameRef(n.name, ref) 843 self.set_special(n.name, value) 844 return value 845 846 # Special case for print operations. 847 848 elif n.name.startswith("$print"): 849 850 # Attempt to get a reference. 851 852 ref = self.get_builtin("print_") 853 854 # Record the imported name and provide the resolved name reference. 855 856 value = ResolvedNameRef(n.name, ref) 857 self.set_special(n.name, value) 858 return value 859 860 # Test for self usage, which is only allowed in methods. 861 862 if n.name == "self" and not (self.in_function and self.in_class): 863 raise InspectError("Use of self is only allowed in methods.", path, n) 864 865 # Record usage of the name. 866 867 self.record_name(n.name) 868 869 # Search for unknown names in non-function scopes immediately. 870 # Temporary names should not be re-used between scopes. 871 # External names in functions are resolved later. 872 873 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 874 875 if ref: 876 self.record_name_access(n.name, True) 877 return ResolvedNameRef(n.name, ref, is_global=True) 878 879 # Explicitly-declared global names. 880 881 elif self.in_function and n.name in self.scope_globals[path]: 882 self.record_name_access(n.name, True) 883 return NameRef(n.name, is_global=True) 884 885 # Examine other names. 886 887 else: 888 889 # Check local names. 890 891 access_number = self.record_name_access(n.name) 892 893 # Local name. 894 895 if access_number is not None: 896 return LocalNameRef(n.name, access_number) 897 898 # Possible global or built-in name. 899 900 else: 901 self.record_name_access(n.name, True) 902 return NameRef(n.name, is_global=True) 903 904 def record_name_access(self, name, is_global=False): 905 906 """ 907 Record an access involving 'name' if the name is being tracked, using 908 'is_global' to indicate whether the name is global. 909 """ 910 911 name = self.get_name_for_tracking(name, is_global=is_global) 912 branches = self.trackers[-1].tracking_name(name) 913 if branches: 914 self.record_branches_for_access(branches, name, None) 915 return self.record_access_details(name, None, self.in_assignment, 916 self.in_invocation) 917 return None 918 919 def process_operator_chain(self, nodes): 920 921 """ 922 Process the given chain of 'nodes', processing each node or item. 923 Each node starts a new conditional region, effectively making a deeply- 924 nested collection of if-like statements. 925 """ 926 927 results = [] 928 929 tracker = self.trackers[-1] 930 931 for item in nodes: 932 tracker.new_branchpoint() 933 tracker.new_branch() 934 result = self.process_structure_node(item) 935 if result: 936 results.append(result) 937 938 for item in nodes[:-1]: 939 tracker.shelve_branch() 940 tracker.new_branch() 941 tracker.shelve_branch() 942 tracker.merge_branches() 943 944 tracker.shelve_branch() 945 tracker.merge_branches() 946 947 return MultipleRef(results) 948 949 def process_try_node(self, n): 950 951 """ 952 Process the given "try...except" node 'n'. 953 """ 954 955 self.record_exception_handler() 956 957 tracker = self.trackers[-1] 958 tracker.new_branchpoint() 959 960 self.process_structure_node(n.body) 961 962 for name, var, handler in n.handlers: 963 if name is not None: 964 self.process_structure_node(name) 965 966 # Any abandoned branches from the body can now be resumed in a new 967 # branch. 968 969 tracker.resume_abandoned_branches() 970 971 # Establish the local for the handler. 972 973 if var is not None: 974 self.process_assignment_node(var, None) 975 if handler is not None: 976 self.process_structure_node(handler) 977 978 tracker.shelve_branch() 979 980 # The else clause maintains the usage from the body but without the 981 # abandoned branches since they would never lead to the else clause 982 # being executed. 983 984 if n.else_: 985 tracker.new_branch() 986 self.process_structure_node(n.else_) 987 tracker.shelve_branch() 988 989 # Without an else clause, a null branch propagates the successful 990 # outcome. 991 992 else: 993 tracker.new_branch() 994 tracker.shelve_branch() 995 996 tracker.merge_branches() 997 998 def process_try_finally_node(self, n): 999 1000 """ 1001 Process the given "try...finally" node 'n'. 1002 """ 1003 1004 self.record_exception_handler() 1005 1006 tracker = self.trackers[-1] 1007 self.process_structure_node(n.body) 1008 1009 # Any abandoned branches from the body can now be resumed. 1010 1011 branches = tracker.resume_all_abandoned_branches() 1012 self.process_structure_node(n.final) 1013 1014 # At the end of the finally clause, abandoned branches are discarded. 1015 1016 tracker.restore_active_branches(branches) 1017 1018 def process_while_node(self, n): 1019 1020 "Process the given while node 'n'." 1021 1022 tracker = self.trackers[-1] 1023 tracker.new_branchpoint(loop_node=True) 1024 1025 # Evaluate any test or iterator outside the loop. 1026 1027 self.process_structure_node(n.test) 1028 1029 # Propagate attribute usage to branches. 1030 1031 tracker.new_branch(loop_node=True) 1032 1033 # Enter the loop. 1034 1035 in_conditional = self.in_conditional 1036 self.in_conditional = True 1037 self.process_structure_node(n.body) 1038 self.in_conditional = in_conditional 1039 1040 # Continuing branches are resumed before any test. 1041 1042 tracker.resume_continuing_branches() 1043 1044 # Evaluate any continuation test within the body. 1045 1046 self.process_structure_node(n.test) 1047 1048 tracker.shelve_branch(loop_node=True) 1049 1050 # Support the non-looping condition. 1051 1052 tracker.new_branch() 1053 tracker.shelve_branch() 1054 1055 tracker.merge_branches() 1056 1057 # Evaluate any else clause outside branches. 1058 1059 if n.else_: 1060 self.process_structure_node(n.else_) 1061 1062 # Connect broken branches to the code after any loop. 1063 1064 tracker.resume_broken_branches() 1065 1066 # Branch tracking methods. 1067 1068 def start_tracking(self, names): 1069 1070 """ 1071 Start tracking attribute usage for names in the current namespace, 1072 immediately registering the given 'names'. 1073 """ 1074 1075 path = self.get_namespace_path() 1076 parent = self.trackers[-1] 1077 tracker = BranchTracker() 1078 self.trackers.append(tracker) 1079 1080 # Record the given names established as new branches. 1081 1082 tracker.assign_names(names) 1083 1084 def assign_name(self, name, name_ref): 1085 1086 "Assign to 'name' the given 'name_ref' in the current namespace." 1087 1088 name = self.get_name_for_tracking(name) 1089 self.trackers[-1].assign_names([name], [name_ref]) 1090 1091 def stop_tracking(self): 1092 1093 """ 1094 Stop tracking attribute usage, recording computed usage for the current 1095 namespace. Indicate whether a value is always returned from the 1096 namespace. 1097 """ 1098 1099 path = self.get_namespace_path() 1100 tracker = self.trackers.pop() 1101 self.record_assignments_for_access(tracker) 1102 1103 self.attr_usage[path] = tracker.get_all_usage() 1104 self.name_initialisers[path] = tracker.get_all_values() 1105 1106 return tracker.returns_value() 1107 1108 def start_tracking_in_module(self): 1109 1110 "Start tracking attribute usage in the module." 1111 1112 tracker = BranchTracker() 1113 self.trackers.append(tracker) 1114 1115 def stop_tracking_in_module(self): 1116 1117 "Stop tracking attribute usage in the module." 1118 1119 tracker = self.trackers[0] 1120 self.record_assignments_for_access(tracker) 1121 self.attr_usage[self.name] = tracker.get_all_usage() 1122 self.name_initialisers[self.name] = tracker.get_all_values() 1123 1124 def record_assignments_for_access(self, tracker): 1125 1126 """ 1127 For the current path, use the given 'tracker' to record assignment 1128 version information for attribute accesses. 1129 """ 1130 1131 path = self.get_path_for_access() 1132 1133 if not self.attr_accessor_branches.has_key(path): 1134 return 1135 1136 init_item(self.attr_accessors, path, dict) 1137 attr_accessors = self.attr_accessors[path] 1138 1139 # Obtain the branches applying during each access. 1140 1141 for access, all_branches in self.attr_accessor_branches[path].items(): 1142 name, attrnames = access 1143 init_item(attr_accessors, access, list) 1144 1145 # Obtain the assignments applying to each branch. 1146 1147 for branches in all_branches: 1148 positions = tracker.get_assignment_positions_for_branches(name, branches) 1149 1150 # Detect missing name information. 1151 1152 if None in positions: 1153 globals = self.global_attr_accesses.get(path) 1154 accesses = globals and globals.get(name) 1155 if not accesses: 1156 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1157 self.get_namespace_path(), name) 1158 positions.remove(None) 1159 1160 attr_accessors[access].append(positions) 1161 1162 def record_branches_for_access(self, branches, name, attrnames): 1163 1164 """ 1165 Record the given 'branches' for an access involving the given 'name' and 1166 'attrnames'. 1167 """ 1168 1169 access = name, attrnames 1170 path = self.get_path_for_access() 1171 1172 init_item(self.attr_accessor_branches, path, dict) 1173 attr_accessor_branches = self.attr_accessor_branches[path] 1174 1175 init_item(attr_accessor_branches, access, list) 1176 attr_accessor_branches[access].append(branches) 1177 1178 def record_access_details(self, name, attrnames, assignment, invocation): 1179 1180 """ 1181 For the given 'name' and 'attrnames', record an access indicating 1182 whether an 'assignment' or an 'invocation' is occurring. 1183 1184 These details correspond to accesses otherwise recorded by the attribute 1185 accessor and attribute access dictionaries. 1186 """ 1187 1188 access = name, attrnames 1189 path = self.get_path_for_access() 1190 1191 init_item(self.attr_access_modifiers, path, dict) 1192 init_item(self.attr_access_modifiers[path], access, list) 1193 1194 access_number = len(self.attr_access_modifiers[path][access]) 1195 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1196 return access_number 1197 1198 def record_global_access_details(self, name, attrnames): 1199 1200 """ 1201 Record details of a global access via the given 'name' involving the 1202 indicated 'attrnames'. 1203 """ 1204 1205 path = self.get_namespace_path() 1206 1207 init_item(self.global_attr_accesses, path, dict) 1208 init_item(self.global_attr_accesses[path], name, set) 1209 self.global_attr_accesses[path][name].add(attrnames) 1210 1211 # Namespace modification. 1212 1213 def record_name(self, name): 1214 1215 "Record the use of 'name' in a namespace." 1216 1217 path = self.get_namespace_path() 1218 init_item(self.names_used, path, set) 1219 self.names_used[path].add(name) 1220 1221 def set_module(self, name, module_name): 1222 1223 """ 1224 Set a module in the current namespace using the given 'name' associated 1225 with the corresponding 'module_name'. 1226 """ 1227 1228 if name: 1229 self.set_general_local(name, Reference("<module>", module_name)) 1230 1231 def set_definition(self, name, kind): 1232 1233 """ 1234 Set the definition having the given 'name' and 'kind'. 1235 1236 Definitions are set in the static namespace hierarchy, but they can also 1237 be recorded for function locals. 1238 """ 1239 1240 if self.is_global(name): 1241 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1242 self.get_namespace_path(), name) 1243 1244 path = self.get_object_path(name) 1245 self.set_object(path, kind) 1246 1247 ref = self.get_object(path) 1248 if ref.get_kind() == "<var>": 1249 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1250 self.get_namespace_path(), name) 1251 1252 if not self.is_global(name) and self.in_function: 1253 self.set_function_local(name, ref) 1254 1255 def set_function_local(self, name, ref=None): 1256 1257 "Set the local with the given 'name' and optional 'ref'." 1258 1259 path = self.get_namespace_path() 1260 locals = self.function_locals[path] 1261 used = self.names_used.get(path) 1262 1263 if not locals.has_key(name) and used and name in used: 1264 raise InspectError("Name %s assigned locally but used previously." % name, path) 1265 1266 multiple = not ref or locals.has_key(name) and locals[name] != ref 1267 locals[name] = multiple and Reference("<var>") or ref 1268 1269 def assign_general_local(self, name, name_ref): 1270 1271 """ 1272 Set for 'name' the given 'name_ref', recording the name for attribute 1273 usage tracking. 1274 """ 1275 1276 self.set_general_local(name, name_ref) 1277 self.assign_name(name, name_ref) 1278 1279 def set_general_local(self, name, value=None): 1280 1281 """ 1282 Set the 'name' with optional 'value' in any kind of local namespace, 1283 where the 'value' should be a reference if specified. 1284 """ 1285 1286 init_value = self.get_initialising_value(value) 1287 1288 # Module global names. 1289 1290 if self.is_global(name): 1291 path = self.get_global_path(name) 1292 self.set_object(path, init_value) 1293 1294 # Function local names. 1295 1296 elif self.in_function: 1297 self.set_function_local(name, init_value) 1298 1299 # Other namespaces (classes). 1300 1301 else: 1302 self.set_name(name, init_value) 1303 1304 def set_name(self, name, ref=None): 1305 1306 "Attach the 'name' with optional 'ref' to the current namespace." 1307 1308 self.set_object(self.get_object_path(name), ref) 1309 1310 def set_instance_attr(self, name, ref=None): 1311 1312 """ 1313 Add an instance attribute of the given 'name' to the current class, 1314 using the optional 'ref'. 1315 """ 1316 1317 self._set_instance_attr(self.in_class, name, ref) 1318 1319 def _set_instance_attr(self, path, name, ref=None): 1320 1321 init_item(self.instance_attrs, path, set) 1322 self.instance_attrs[path].add(name) 1323 1324 if ref: 1325 init_item(self.instance_attr_constants, path, dict) 1326 self.instance_attr_constants[path][name] = ref 1327 1328 def get_initialising_value(self, value): 1329 1330 "Return a suitable initialiser reference for 'value'." 1331 1332 if isinstance(value, Result): 1333 return value.reference() 1334 else: 1335 return value 1336 1337 # Static, program-relative naming. 1338 1339 def find_name(self, name): 1340 1341 """ 1342 Return the qualified name for the given 'name' used in the current 1343 non-function namespace. 1344 """ 1345 1346 path = self.get_namespace_path() 1347 ref = None 1348 1349 if not self.in_function and name not in predefined_constants: 1350 if self.in_class: 1351 ref = self.get_object(self.get_object_path(name), False) 1352 if not ref: 1353 ref = self.get_global_or_builtin(name) 1354 1355 return ref 1356 1357 def get_class(self, node): 1358 1359 """ 1360 Use the given 'node' to obtain the identity of a class. Return a 1361 reference for the class. Unresolved dependencies are permitted and must 1362 be resolved later. 1363 """ 1364 1365 ref = self._get_class(node) 1366 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1367 1368 def _get_class(self, node): 1369 1370 """ 1371 Use the given 'node' to find a class definition. Return a reference to 1372 the class. 1373 """ 1374 1375 if isinstance(node, compiler.ast.Getattr): 1376 1377 # Obtain the identity of the access target. 1378 1379 ref = self._get_class(node.expr) 1380 1381 # Where the target is a class or module, obtain the identity of the 1382 # attribute. 1383 1384 if ref.has_kind(["<function>", "<var>"]): 1385 return None 1386 else: 1387 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1388 return self.get_object(attrname) 1389 1390 # Names can be module-level or built-in. 1391 1392 elif isinstance(node, compiler.ast.Name): 1393 1394 # Record usage of the name and attempt to identify it. 1395 1396 self.record_name(node.name) 1397 return self.find_name(node.name) 1398 else: 1399 return None 1400 1401 def get_constant(self, name, value): 1402 1403 "Return a constant reference for the given type 'name' and 'value'." 1404 1405 ref = self.get_builtin_class(name) 1406 return self.get_constant_reference(ref, value) 1407 1408 def get_literal_instance(self, n, name=None): 1409 1410 """ 1411 For node 'n', return a reference to an instance of 'name', or if 'name' 1412 is not specified, deduce the type from the value. 1413 """ 1414 1415 # Handle stray None constants (Sliceobj seems to produce them). 1416 1417 if name == "NoneType": 1418 return self.process_name_node(compiler.ast.Name("None")) 1419 1420 # Obtain the details of the literal itself. 1421 # An alias to the type is generated for sequences. 1422 1423 if name in ("dict", "list", "tuple"): 1424 ref = self.get_builtin_class(name) 1425 self.set_special_literal(name, ref) 1426 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1427 1428 # Constant values are independently recorded. 1429 1430 else: 1431 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1432 ref = self.get_builtin_class(typename) 1433 return self.get_constant_reference(ref, value, encoding) 1434 1435 # Special names. 1436 1437 def get_special(self, name): 1438 1439 "Return any stored value for the given special 'name'." 1440 1441 value = self.special.get(name) 1442 if value: 1443 ref, paths = value 1444 else: 1445 ref = None 1446 return ref 1447 1448 def set_special(self, name, value): 1449 1450 """ 1451 Set a special 'name' that merely tracks the use of an implicit object 1452 'value'. 1453 """ 1454 1455 if not self.special.has_key(name): 1456 paths = set() 1457 self.special[name] = value, paths 1458 else: 1459 _ref, paths = self.special[name] 1460 1461 paths.add(self.get_namespace_path()) 1462 1463 def set_special_literal(self, name, ref): 1464 1465 """ 1466 Set a special name for the literal type 'name' having type 'ref'. Such 1467 special names provide a way of referring to literal object types. 1468 """ 1469 1470 literal_name = "$L%s" % name 1471 value = ResolvedNameRef(literal_name, ref) 1472 self.set_special(literal_name, value) 1473 1474 # Exceptions. 1475 1476 def record_exception_handler(self): 1477 1478 "Record the current namespace as employing an exception handler." 1479 1480 self.exception_namespaces.add(self.get_namespace_path()) 1481 1482 # Return values. 1483 1484 def record_return_value(self, expr): 1485 1486 "Record the given return 'expr'." 1487 1488 path = self.get_namespace_path() 1489 l = init_item(self.return_values, path, list) 1490 l.append(expr) 1491 if not self.importer.all_return_values.has_key(path): 1492 self.importer.all_return_values[path] = l 1493 1494 # vim: tabstop=4 expandtab shiftwidth=4