1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("string", self.name).reference()) 81 self.set_name("__file__", self.get_constant("string", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("string", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 # Obtain any completed chain and return the reference to it. 362 363 name_ref = self.process_attribute_chain(n) 364 365 if self.have_access_expression(n): 366 return name_ref 367 368 # Where the start of the chain of attributes has been reached, determine 369 # the complete access. 370 371 # Given a non-access node, this chain can be handled in its entirety, 372 # either being name-based and thus an access rooted on a name, or being 373 # based on some other node and thus an anonymous access of some kind. 374 375 path = self.get_namespace_path() 376 377 # Start with the the full attribute chain. 378 379 remaining = self.attrs 380 attrnames = ".".join(remaining) 381 382 # If the accessor cannot be identified, or where attributes 383 # remain in an attribute chain, record the anonymous accesses. 384 385 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 386 387 init_item(self.attr_accesses, path, set) 388 self.attr_accesses[path].add(attrnames) 389 390 self.record_access_details(None, attrnames, self.in_assignment, 391 self.in_invocation) 392 del self.attrs[0] 393 return 394 395 # Name-based accesses will handle the first attribute in a 396 # chain. 397 398 else: 399 attrname = remaining[0] 400 401 # Attribute assignments are used to identify instance attributes. 402 403 if isinstance(n, compiler.ast.AssAttr) and \ 404 self.in_class and self.in_function and n.expr.name == "self": 405 406 self.set_instance_attr(attrname) 407 408 # Record attribute usage using any name local to this namespace, 409 # if assigned in the namespace, or using an external name 410 # (presently just globals within classes). 411 412 name = self.get_name_for_tracking(name_ref.name, name_ref) 413 tracker = self.trackers[-1] 414 415 immediate_access = len(self.attrs) == 1 416 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 417 418 # Record global-based chains for subsequent resolution. 419 420 if name_ref.is_global_name(): 421 self.record_global_access_details(name, attrnames) 422 423 # Make sure the name is being tracked: global names will not 424 # already be initialised in a branch and must be added 425 # explicitly. 426 427 if not tracker.have_name(name): 428 tracker.assign_names([name]) 429 if self.in_function: 430 self.scope_globals[path].add(name) 431 432 # Record attribute usage in the tracker, and record the branch 433 # information for the access. 434 435 branches = tracker.use_attribute(name, attrname, 436 self.in_invocation is not None, assignment) 437 438 if not branches: 439 raise InspectError("Name %s is accessed using %s before an assignment." % ( 440 name, attrname), path, n) 441 442 self.record_branches_for_access(branches, name, attrnames) 443 access_number = self.record_access_details(name, attrnames, 444 self.in_assignment, self.in_invocation) 445 446 del self.attrs[0] 447 return AccessRef(name, attrnames, access_number) 448 449 def process_class_node(self, n): 450 451 "Process the given class node 'n'." 452 453 path = self.get_namespace_path() 454 455 # To avoid notions of class "versions" where the same definition 456 # might be parameterised with different state and be referenced 457 # elsewhere (as base classes, for example), classes in functions or 458 # conditions are forbidden. 459 460 if self.in_function or self.in_conditional: 461 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 462 path, n.name) 463 return 464 465 # Resolve base classes. 466 467 bases = [] 468 469 for base in n.bases: 470 base_class = self.get_class(base) 471 472 if not base_class: 473 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 474 path, n.name, base) 475 return 476 else: 477 bases.append(base_class) 478 479 # Detect conflicting definitions. Such definitions cause conflicts in 480 # the storage of namespace-related information. 481 482 class_name = self.get_object_path(n.name) 483 ref = self.get_object(class_name, defer=False) 484 485 if ref and ref.static(): 486 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 487 488 # Record bases for the class and retain the class name. 489 # Note that the function class does not inherit from the object class. 490 491 if not bases and class_name != "__builtins__.core.object" and \ 492 class_name != "__builtins__.core.function": 493 494 ref = self.get_object("__builtins__.object") 495 bases.append(ref) 496 497 self.importer.classes[class_name] = self.classes[class_name] = bases 498 self.importer.subclasses[class_name] = set() 499 self.scope_globals[class_name] = set() 500 501 # Set the definition before entering the namespace rather than 502 # afterwards because methods may reference it. In normal Python, 503 # a class is not accessible until the definition is complete, but 504 # methods can generally reference it since upon being called the 505 # class will already exist. 506 507 self.set_definition(n.name, "<class>") 508 509 in_class = self.in_class 510 self.in_class = class_name 511 self.set_instance_attr("__class__", Reference("<class>", class_name)) 512 self.enter_namespace(n.name) 513 514 # Do not provide the special instantiator attributes on the function 515 # class. Function instances provide these attributes. 516 517 if class_name != "__builtins__.core.function": 518 519 self.set_name("__fn__") # special instantiator attribute 520 self.set_name("__args__") # special instantiator attribute 521 522 # Provide leafname, parent and context attributes. 523 524 parent, leafname = class_name.rsplit(".", 1) 525 self.set_name("__name__", self.get_constant("string", leafname).reference()) 526 527 if class_name != "__builtins__.core.function": 528 self.set_name("__parent__") 529 530 self.process_structure_node(n.code) 531 self.exit_namespace() 532 self.in_class = in_class 533 534 def process_from_node(self, n): 535 536 "Process the given node 'n', importing from another module." 537 538 path = self.get_namespace_path() 539 540 module_name, names = self.get_module_name(n) 541 if module_name == self.name: 542 raise InspectError("Cannot import from the current module.", path, n) 543 544 self.queue_module(module_name) 545 546 # Attempt to obtain the referenced objects. 547 548 for name, alias in n.names: 549 if name == "*": 550 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 551 552 # Explicit names. 553 554 ref = self.import_name_from_module(name, module_name) 555 value = ResolvedNameRef(alias or name, ref) 556 self.set_general_local(alias or name, value) 557 558 def process_function_node(self, n, name): 559 560 """ 561 Process the given function or lambda node 'n' with the given 'name'. 562 """ 563 564 is_lambda = isinstance(n, compiler.ast.Lambda) 565 566 # Where a function is declared conditionally, use a separate name for 567 # the definition, and assign the definition to the stated name. 568 569 if (self.in_conditional or self.in_function) and not is_lambda: 570 original_name = name 571 name = self.get_lambda_name() 572 else: 573 original_name = None 574 575 # Detect conflicting definitions. Such definitions cause conflicts in 576 # the storage of namespace-related information. 577 578 function_name = self.get_object_path(name) 579 ref = self.get_object(function_name, defer=False) 580 581 if ref and ref.static(): 582 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 583 584 # Initialise argument and local records. 585 586 argnames = get_argnames(n.argnames) 587 is_method = self.in_class and not self.in_function 588 589 # Remove explicit "self" from method parameters. 590 591 if is_method and argnames and argnames[0] == "self": 592 del argnames[0] 593 594 # Convert .name entries in the parameters, provided this is a method. 595 596 l = [] 597 attr_initialisers = [] 598 599 for argname in argnames: 600 if argname[0] == ".": 601 if not is_method: 602 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 603 604 argname = argname[1:] 605 attr_initialisers.append(argname) 606 607 if argname in l: 608 raise InspectError("Argument name %s is repeated." % argname, function_name, n) 609 610 l.append(argname) 611 612 argnames = l 613 614 # Copy and propagate the parameters. 615 616 self.importer.function_parameters[function_name] = \ 617 self.function_parameters[function_name] = argnames[:] 618 619 self.importer.function_attr_initialisers[function_name] = \ 620 self.function_attr_initialisers[function_name] = attr_initialisers 621 622 # Define all arguments/parameters in the local namespace. 623 624 locals = \ 625 self.importer.function_locals[function_name] = \ 626 self.function_locals[function_name] = {} 627 628 # Insert "self" into method locals. 629 630 if is_method: 631 argnames.insert(0, "self") 632 633 # Define "self" in terms of the class if in a method. 634 # This does not diminish the need for type-narrowing in the deducer. 635 636 if argnames: 637 if self.in_class and not self.in_function and argnames[0] == "self": 638 locals[argnames[0]] = Reference("<instance>", self.in_class) 639 else: 640 locals[argnames[0]] = Reference("<var>") 641 642 for argname in argnames[1:]: 643 locals[argname] = Reference("<var>") 644 645 globals = self.scope_globals[function_name] = set() 646 647 # Process the defaults. 648 649 defaults = self.importer.function_defaults[function_name] = \ 650 self.function_defaults[function_name] = [] 651 652 for argname, default in compiler.ast.get_defaults(n): 653 if default: 654 655 # Obtain any reference for the default. 656 657 name_ref = self.process_structure_node(default) 658 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 659 660 # Reset conditional tracking to focus on the function contents. 661 662 in_conditional = self.in_conditional 663 self.in_conditional = False 664 665 in_function = self.in_function 666 self.in_function = function_name 667 668 self.enter_namespace(name) 669 670 # Define a leafname attribute value for the function instance. 671 672 ref = self.get_builtin_class("string") 673 self.reserve_constant(function_name, name, ref.get_origin()) 674 675 # Track attribute usage within the namespace. 676 677 path = self.get_namespace_path() 678 self.start_tracking(locals) 679 680 # Establish attributes for .name entries, provided this is a method. 681 682 for argname in attr_initialisers: 683 self.process_assignment_node( 684 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 685 compiler.ast.Name(argname)) 686 687 self.process_structure_node(n.code) 688 returns_value = self.stop_tracking() 689 690 # Record any null result. 691 692 is_initialiser = is_method and name == "__init__" 693 694 if not returns_value and not is_initialiser: 695 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 696 697 # Exit to the parent. 698 699 self.exit_namespace() 700 701 # Update flags. 702 703 self.in_function = in_function 704 self.in_conditional = in_conditional 705 706 # Define the function using the appropriate name. 707 708 self.set_definition(name, "<function>") 709 710 # Where a function is set conditionally, assign the name. 711 712 if original_name: 713 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 714 715 def process_global_node(self, n): 716 717 """ 718 Process the given "global" node 'n'. 719 """ 720 721 path = self.get_namespace_path() 722 723 if path != self.name: 724 self.scope_globals[path].update(n.names) 725 726 def process_if_node(self, n): 727 728 """ 729 Process the given "if" node 'n'. 730 """ 731 732 tracker = self.trackers[-1] 733 tracker.new_branchpoint() 734 735 for test, body in n.tests: 736 self.process_structure_node(test) 737 738 tracker.new_branch() 739 740 in_conditional = self.in_conditional 741 self.in_conditional = True 742 self.process_structure_node(body) 743 self.in_conditional = in_conditional 744 745 tracker.shelve_branch() 746 747 # Maintain a branch for the else clause. 748 749 tracker.new_branch() 750 if n.else_: 751 self.process_structure_node(n.else_) 752 tracker.shelve_branch() 753 754 tracker.merge_branches() 755 756 def process_import_node(self, n): 757 758 "Process the given import node 'n'." 759 760 path = self.get_namespace_path() 761 762 # Load the mentioned module. 763 764 for name, alias in n.names: 765 if name == self.name: 766 raise InspectError("Cannot import the current module.", path, n) 767 768 self.set_module(alias or name.split(".")[-1], name) 769 self.queue_module(name, True) 770 771 def process_invocation_node(self, n): 772 773 "Process the given invocation node 'n'." 774 775 path = self.get_namespace_path() 776 777 in_invocation = self.in_invocation 778 self.in_invocation = None 779 780 # Process the arguments. 781 782 keywords = set() 783 784 for arg in n.args: 785 self.process_structure_node(arg) 786 if isinstance(arg, compiler.ast.Keyword): 787 keywords.add(arg.name) 788 789 keywords = list(keywords) 790 keywords.sort() 791 792 # Communicate to the invocation target expression that it forms the 793 # target of an invocation, potentially affecting attribute accesses. 794 795 self.in_invocation = len(n.args), keywords 796 797 # Process the expression, obtaining any identified reference. 798 799 name_ref = self.process_structure_node(n.node) 800 self.in_invocation = in_invocation 801 802 # Detect class invocations. 803 804 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 805 return InstanceRef(name_ref.reference().instance_of()) 806 807 elif isinstance(name_ref, (NameRef, AccessRef)): 808 return InvocationRef(name_ref) 809 810 # Provide a general reference to indicate that something is produced 811 # by the invocation, useful for retaining assignment expression 812 # details. 813 814 return VariableRef() 815 816 def process_lambda_node(self, n): 817 818 "Process the given lambda node 'n'." 819 820 name = self.get_lambda_name() 821 self.process_function_node(n, name) 822 823 origin = self.get_object_path(name) 824 825 if self.function_defaults.get(origin): 826 return None 827 else: 828 return ResolvedNameRef(name, Reference("<function>", origin)) 829 830 def process_logical_node(self, n): 831 832 "Process the given operator node 'n'." 833 834 return self.process_operator_chain(n.nodes) 835 836 def process_name_node(self, n): 837 838 "Process the given name node 'n'." 839 840 path = self.get_namespace_path() 841 842 # Find predefined constant names before anything else. 843 844 if n.name in predefined_constants: 845 ref = self.get_builtin(n.name) 846 value = ResolvedNameRef(n.name, ref) 847 return value 848 849 # Special names that have already been identified. 850 851 if n.name.startswith("$"): 852 value = self.get_special(n.name) 853 if value: 854 return value 855 856 # Special case for operator functions introduced through code 857 # transformations. 858 859 if n.name.startswith("$op"): 860 861 # Obtain the location of the actual function defined in the operator 862 # package. 863 864 op = n.name[len("$op"):] 865 866 # Attempt to get a reference. 867 868 ref = self.import_name_from_module(op, "operator") 869 870 # Record the imported name and provide the resolved name reference. 871 872 value = ResolvedNameRef(n.name, ref) 873 self.set_special(n.name, value) 874 return value 875 876 # Special case for sequence length testing. 877 878 elif n.name.startswith("$seq"): 879 op = n.name[len("$seq"):] 880 ref = self.import_name_from_module(op, "__builtins__.sequence") 881 value = ResolvedNameRef(n.name, ref) 882 self.set_special(n.name, value) 883 return value 884 885 # Special case for print operations. 886 887 elif n.name.startswith("$print"): 888 889 # Attempt to get a reference. 890 891 ref = self.get_builtin("print_") 892 893 # Record the imported name and provide the resolved name reference. 894 895 value = ResolvedNameRef(n.name, ref) 896 self.set_special(n.name, value) 897 return value 898 899 # Test for self usage, which is only allowed in methods. 900 901 if n.name == "self" and not (self.in_function and self.in_class): 902 raise InspectError("Use of self is only allowed in methods.", path, n) 903 904 # Record usage of the name. 905 906 self.record_name(n.name) 907 908 # Search for unknown names in non-function scopes immediately. 909 # Temporary names should not be re-used between scopes. 910 # External names in functions are resolved later. 911 912 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 913 914 if ref: 915 self.record_name_access(n.name, True) 916 return ResolvedNameRef(n.name, ref, is_global=True) 917 918 # Explicitly-declared global names. 919 920 elif self.in_function and n.name in self.scope_globals[path]: 921 self.record_name_access(n.name, True) 922 return NameRef(n.name, is_global=True) 923 924 # Examine other names. 925 926 else: 927 928 # Check local names. 929 930 access_number = self.record_name_access(n.name) 931 932 # Local name. 933 934 if access_number is not None: 935 return LocalNameRef(n.name, access_number) 936 937 # Possible global or built-in name. 938 939 else: 940 self.record_name_access(n.name, True) 941 return NameRef(n.name, is_global=True) 942 943 def record_name_access(self, name, is_global=False): 944 945 """ 946 Record an access involving 'name' if the name is being tracked, using 947 'is_global' to indicate whether the name is global. 948 """ 949 950 name = self.get_name_for_tracking(name, is_global=is_global) 951 branches = self.trackers[-1].tracking_name(name) 952 if branches: 953 self.record_branches_for_access(branches, name, None) 954 return self.record_access_details(name, None, self.in_assignment, 955 self.in_invocation) 956 return None 957 958 def process_operator_chain(self, nodes): 959 960 """ 961 Process the given chain of 'nodes', processing each node or item. 962 Each node starts a new conditional region, effectively making a deeply- 963 nested collection of if-like statements. 964 """ 965 966 results = [] 967 968 tracker = self.trackers[-1] 969 970 for item in nodes: 971 tracker.new_branchpoint() 972 tracker.new_branch() 973 result = self.process_structure_node(item) 974 if result: 975 results.append(result) 976 977 for item in nodes[:-1]: 978 tracker.shelve_branch() 979 tracker.new_branch() 980 tracker.shelve_branch() 981 tracker.merge_branches() 982 983 tracker.shelve_branch() 984 tracker.merge_branches() 985 986 return MultipleRef(results) 987 988 def process_try_node(self, n): 989 990 """ 991 Process the given "try...except" node 'n'. 992 """ 993 994 self.record_exception_handler() 995 996 tracker = self.trackers[-1] 997 tracker.new_branchpoint() 998 999 self.process_structure_node(n.body) 1000 1001 for name, var, handler in n.handlers: 1002 if name is not None: 1003 self.process_structure_node(name) 1004 1005 # Any abandoned branches from the body can now be resumed in a new 1006 # branch. 1007 1008 tracker.resume_abandoned_branches() 1009 1010 # Establish the local for the handler. 1011 1012 if var is not None: 1013 self.process_assignment_node(var, None) 1014 if handler is not None: 1015 self.process_structure_node(handler) 1016 1017 tracker.shelve_branch() 1018 1019 # The else clause maintains the usage from the body but without the 1020 # abandoned branches since they would never lead to the else clause 1021 # being executed. 1022 1023 if n.else_: 1024 tracker.new_branch() 1025 self.process_structure_node(n.else_) 1026 tracker.shelve_branch() 1027 1028 # Without an else clause, a null branch propagates the successful 1029 # outcome. 1030 1031 else: 1032 tracker.new_branch() 1033 tracker.shelve_branch() 1034 1035 tracker.merge_branches() 1036 1037 def process_try_finally_node(self, n): 1038 1039 """ 1040 Process the given "try...finally" node 'n'. 1041 """ 1042 1043 self.record_exception_handler() 1044 1045 tracker = self.trackers[-1] 1046 self.process_structure_node(n.body) 1047 1048 # Any abandoned branches from the body can now be resumed. 1049 1050 branches = tracker.resume_all_abandoned_branches() 1051 self.process_structure_node(n.final) 1052 1053 # At the end of the finally clause, abandoned branches are discarded. 1054 1055 tracker.restore_active_branches(branches) 1056 1057 def process_while_node(self, n): 1058 1059 "Process the given while node 'n'." 1060 1061 tracker = self.trackers[-1] 1062 tracker.new_branchpoint(loop_node=True) 1063 1064 # Evaluate any test or iterator outside the loop. 1065 1066 self.process_structure_node(n.test) 1067 1068 # Propagate attribute usage to branches. 1069 1070 tracker.new_branch(loop_node=True) 1071 1072 # Enter the loop. 1073 1074 in_conditional = self.in_conditional 1075 self.in_conditional = True 1076 self.process_structure_node(n.body) 1077 self.in_conditional = in_conditional 1078 1079 # Continuing branches are resumed before any test. 1080 1081 tracker.resume_continuing_branches() 1082 1083 # Evaluate any continuation test within the body. 1084 1085 self.process_structure_node(n.test) 1086 1087 tracker.shelve_branch(loop_node=True) 1088 1089 # Support the non-looping condition. 1090 1091 tracker.new_branch() 1092 tracker.shelve_branch() 1093 1094 tracker.merge_branches() 1095 1096 # Evaluate any else clause outside branches. 1097 1098 if n.else_: 1099 self.process_structure_node(n.else_) 1100 1101 # Connect broken branches to the code after any loop. 1102 1103 tracker.resume_broken_branches() 1104 1105 # Branch tracking methods. 1106 1107 def start_tracking(self, names): 1108 1109 """ 1110 Start tracking attribute usage for names in the current namespace, 1111 immediately registering the given 'names'. 1112 """ 1113 1114 path = self.get_namespace_path() 1115 parent = self.trackers[-1] 1116 tracker = BranchTracker() 1117 self.trackers.append(tracker) 1118 1119 # Record the given names established as new branches. 1120 1121 tracker.assign_names(names) 1122 1123 def assign_name(self, name, name_ref): 1124 1125 "Assign to 'name' the given 'name_ref' in the current namespace." 1126 1127 name = self.get_name_for_tracking(name) 1128 self.trackers[-1].assign_names([name], [name_ref]) 1129 1130 def stop_tracking(self): 1131 1132 """ 1133 Stop tracking attribute usage, recording computed usage for the current 1134 namespace. Indicate whether a value is always returned from the 1135 namespace. 1136 """ 1137 1138 path = self.get_namespace_path() 1139 tracker = self.trackers.pop() 1140 self.record_assignments_for_access(tracker) 1141 1142 self.attr_usage[path] = tracker.get_all_usage() 1143 self.name_initialisers[path] = tracker.get_all_values() 1144 1145 return tracker.returns_value() 1146 1147 def start_tracking_in_module(self): 1148 1149 "Start tracking attribute usage in the module." 1150 1151 tracker = BranchTracker() 1152 self.trackers.append(tracker) 1153 1154 def stop_tracking_in_module(self): 1155 1156 "Stop tracking attribute usage in the module." 1157 1158 tracker = self.trackers[0] 1159 self.record_assignments_for_access(tracker) 1160 self.attr_usage[self.name] = tracker.get_all_usage() 1161 self.name_initialisers[self.name] = tracker.get_all_values() 1162 1163 def record_assignments_for_access(self, tracker): 1164 1165 """ 1166 For the current path, use the given 'tracker' to record assignment 1167 version information for attribute accesses. 1168 """ 1169 1170 path = self.get_path_for_access() 1171 1172 if not self.attr_accessor_branches.has_key(path): 1173 return 1174 1175 init_item(self.attr_accessors, path, dict) 1176 attr_accessors = self.attr_accessors[path] 1177 1178 # Obtain the branches applying during each access. 1179 1180 for access, all_branches in self.attr_accessor_branches[path].items(): 1181 name, attrnames = access 1182 init_item(attr_accessors, access, list) 1183 1184 # Obtain the assignments applying to each branch. 1185 1186 for branches in all_branches: 1187 positions = tracker.get_assignment_positions_for_branches(name, branches) 1188 1189 # Detect missing name information. 1190 1191 if None in positions: 1192 globals = self.global_attr_accesses.get(path) 1193 accesses = globals and globals.get(name) 1194 if not accesses: 1195 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1196 self.get_namespace_path(), name) 1197 positions.remove(None) 1198 1199 attr_accessors[access].append(positions) 1200 1201 def record_branches_for_access(self, branches, name, attrnames): 1202 1203 """ 1204 Record the given 'branches' for an access involving the given 'name' and 1205 'attrnames'. 1206 """ 1207 1208 access = name, attrnames 1209 path = self.get_path_for_access() 1210 1211 init_item(self.attr_accessor_branches, path, dict) 1212 attr_accessor_branches = self.attr_accessor_branches[path] 1213 1214 init_item(attr_accessor_branches, access, list) 1215 attr_accessor_branches[access].append(branches) 1216 1217 def record_access_details(self, name, attrnames, assignment, invocation): 1218 1219 """ 1220 For the given 'name' and 'attrnames', record an access indicating 1221 whether an 'assignment' or an 'invocation' is occurring. 1222 1223 These details correspond to accesses otherwise recorded by the attribute 1224 accessor and attribute access dictionaries. 1225 """ 1226 1227 access = name, attrnames 1228 path = self.get_path_for_access() 1229 1230 init_item(self.attr_access_modifiers, path, dict) 1231 init_item(self.attr_access_modifiers[path], access, list) 1232 1233 access_number = len(self.attr_access_modifiers[path][access]) 1234 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1235 return access_number 1236 1237 def record_global_access_details(self, name, attrnames): 1238 1239 """ 1240 Record details of a global access via the given 'name' involving the 1241 indicated 'attrnames'. 1242 """ 1243 1244 path = self.get_namespace_path() 1245 1246 init_item(self.global_attr_accesses, path, dict) 1247 init_item(self.global_attr_accesses[path], name, set) 1248 self.global_attr_accesses[path][name].add(attrnames) 1249 1250 # Namespace modification. 1251 1252 def record_name(self, name): 1253 1254 "Record the use of 'name' in a namespace." 1255 1256 path = self.get_namespace_path() 1257 init_item(self.names_used, path, set) 1258 self.names_used[path].add(name) 1259 1260 def set_module(self, name, module_name): 1261 1262 """ 1263 Set a module in the current namespace using the given 'name' associated 1264 with the corresponding 'module_name'. 1265 """ 1266 1267 if name: 1268 self.set_general_local(name, Reference("<module>", module_name)) 1269 1270 def set_definition(self, name, kind): 1271 1272 """ 1273 Set the definition having the given 'name' and 'kind'. 1274 1275 Definitions are set in the static namespace hierarchy, but they can also 1276 be recorded for function locals. 1277 """ 1278 1279 if self.is_global(name): 1280 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1281 self.get_namespace_path(), name) 1282 1283 path = self.get_object_path(name) 1284 self.set_object(path, kind) 1285 1286 ref = self.get_object(path) 1287 if ref.get_kind() == "<var>": 1288 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1289 self.get_namespace_path(), name) 1290 1291 if not self.is_global(name) and self.in_function: 1292 self.set_function_local(name, ref) 1293 1294 def set_function_local(self, name, ref=None): 1295 1296 "Set the local with the given 'name' and optional 'ref'." 1297 1298 path = self.get_namespace_path() 1299 locals = self.function_locals[path] 1300 used = self.names_used.get(path) 1301 1302 if not locals.has_key(name) and used and name in used: 1303 raise InspectError("Name %s assigned locally but used previously." % name, path) 1304 1305 multiple = not ref or locals.has_key(name) and locals[name] != ref 1306 locals[name] = multiple and Reference("<var>") or ref 1307 1308 def assign_general_local(self, name, name_ref): 1309 1310 """ 1311 Set for 'name' the given 'name_ref', recording the name for attribute 1312 usage tracking. 1313 """ 1314 1315 self.set_general_local(name, name_ref) 1316 self.assign_name(name, name_ref) 1317 1318 def set_general_local(self, name, value=None): 1319 1320 """ 1321 Set the 'name' with optional 'value' in any kind of local namespace, 1322 where the 'value' should be a reference if specified. 1323 """ 1324 1325 init_value = self.get_initialising_value(value) 1326 1327 # Module global names. 1328 1329 if self.is_global(name): 1330 path = self.get_global_path(name) 1331 self.set_object(path, init_value) 1332 1333 # Function local names. 1334 1335 elif self.in_function: 1336 self.set_function_local(name, init_value) 1337 1338 # Other namespaces (classes). 1339 1340 else: 1341 self.set_name(name, init_value) 1342 1343 def set_name(self, name, ref=None): 1344 1345 "Attach the 'name' with optional 'ref' to the current namespace." 1346 1347 self.set_object(self.get_object_path(name), ref) 1348 1349 def set_instance_attr(self, name, ref=None): 1350 1351 """ 1352 Add an instance attribute of the given 'name' to the current class, 1353 using the optional 'ref'. 1354 """ 1355 1356 self._set_instance_attr(self.in_class, name, ref) 1357 1358 def _set_instance_attr(self, path, name, ref=None): 1359 1360 init_item(self.instance_attrs, path, set) 1361 self.instance_attrs[path].add(name) 1362 1363 if ref: 1364 init_item(self.instance_attr_constants, path, dict) 1365 self.instance_attr_constants[path][name] = ref 1366 1367 def get_initialising_value(self, value): 1368 1369 "Return a suitable initialiser reference for 'value'." 1370 1371 if isinstance(value, Result): 1372 return value.reference() 1373 else: 1374 return value 1375 1376 # Static, program-relative naming. 1377 1378 def find_name(self, name): 1379 1380 """ 1381 Return the qualified name for the given 'name' used in the current 1382 non-function namespace. 1383 """ 1384 1385 path = self.get_namespace_path() 1386 ref = None 1387 1388 if not self.in_function and name not in predefined_constants: 1389 if self.in_class: 1390 ref = self.get_object(self.get_object_path(name), False) 1391 if not ref: 1392 ref = self.get_global_or_builtin(name) 1393 1394 return ref 1395 1396 def get_class(self, node): 1397 1398 """ 1399 Use the given 'node' to obtain the identity of a class. Return a 1400 reference for the class. Unresolved dependencies are permitted and must 1401 be resolved later. 1402 """ 1403 1404 ref = self._get_class(node) 1405 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1406 1407 def _get_class(self, node): 1408 1409 """ 1410 Use the given 'node' to find a class definition. Return a reference to 1411 the class. 1412 """ 1413 1414 if isinstance(node, compiler.ast.Getattr): 1415 1416 # Obtain the identity of the access target. 1417 1418 ref = self._get_class(node.expr) 1419 1420 # Where the target is a class or module, obtain the identity of the 1421 # attribute. 1422 1423 if ref.has_kind(["<function>", "<var>"]): 1424 return None 1425 else: 1426 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1427 return self.get_object(attrname) 1428 1429 # Names can be module-level or built-in. 1430 1431 elif isinstance(node, compiler.ast.Name): 1432 1433 # Record usage of the name and attempt to identify it. 1434 1435 self.record_name(node.name) 1436 return self.find_name(node.name) 1437 else: 1438 return None 1439 1440 def get_constant(self, name, value): 1441 1442 "Return a constant reference for the given type 'name' and 'value'." 1443 1444 ref = self.get_builtin_class(name) 1445 return self.get_constant_reference(ref, value) 1446 1447 def get_literal_instance(self, n, name=None): 1448 1449 """ 1450 For node 'n', return a reference to an instance of 'name', or if 'name' 1451 is not specified, deduce the type from the value. 1452 """ 1453 1454 # Handle stray None constants (Sliceobj seems to produce them). 1455 1456 if name == "NoneType": 1457 return self.process_name_node(compiler.ast.Name("None")) 1458 1459 # Obtain the details of the literal itself. 1460 # An alias to the type is generated for sequences. 1461 1462 if name in ("dict", "list", "tuple"): 1463 ref = self.get_builtin_class(name) 1464 self.set_special_literal(name, ref) 1465 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1466 1467 # Constant values are independently recorded. 1468 1469 else: 1470 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1471 ref = self.get_builtin_class(typename) 1472 return self.get_constant_reference(ref, value, encoding) 1473 1474 # Special names. 1475 1476 def get_special(self, name): 1477 1478 "Return any stored value for the given special 'name'." 1479 1480 value = self.special.get(name) 1481 if value: 1482 ref, paths = value 1483 else: 1484 ref = None 1485 return ref 1486 1487 def set_special(self, name, value): 1488 1489 """ 1490 Set a special 'name' that merely tracks the use of an implicit object 1491 'value'. 1492 """ 1493 1494 if not self.special.has_key(name): 1495 paths = set() 1496 self.special[name] = value, paths 1497 else: 1498 _ref, paths = self.special[name] 1499 1500 paths.add(self.get_namespace_path()) 1501 1502 def set_special_literal(self, name, ref): 1503 1504 """ 1505 Set a special name for the literal type 'name' having type 'ref'. Such 1506 special names provide a way of referring to literal object types. 1507 """ 1508 1509 literal_name = "$L%s" % name 1510 value = ResolvedNameRef(literal_name, ref) 1511 self.set_special(literal_name, value) 1512 1513 # Exceptions. 1514 1515 def record_exception_handler(self): 1516 1517 "Record the current namespace as employing an exception handler." 1518 1519 self.exception_namespaces.add(self.get_namespace_path()) 1520 1521 # Return values. 1522 1523 def record_return_value(self, expr): 1524 1525 "Record the given return 'expr'." 1526 1527 path = self.get_namespace_path() 1528 l = init_item(self.return_values, path, list) 1529 l.append(expr) 1530 if not self.importer.all_return_values.has_key(path): 1531 self.importer.all_return_values[path] = l 1532 1533 # vim: tabstop=4 expandtab shiftwidth=4