1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("string", self.name).reference()) 81 self.set_name("__file__", self.get_constant("string", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("string", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 # Obtain any completed chain and return the reference to it. 362 363 name_ref = self.process_attribute_chain(n) 364 365 if self.have_access_expression(n): 366 return name_ref 367 368 # Where the start of the chain of attributes has been reached, determine 369 # the complete access. 370 371 # Given a non-access node, this chain can be handled in its entirety, 372 # either being name-based and thus an access rooted on a name, or being 373 # based on some other node and thus an anonymous access of some kind. 374 375 path = self.get_namespace_path() 376 377 # Start with the the full attribute chain. 378 379 remaining = self.attrs 380 attrnames = ".".join(remaining) 381 382 # If the accessor cannot be identified, or where attributes 383 # remain in an attribute chain, record the anonymous accesses. 384 385 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 386 387 init_item(self.attr_accesses, path, set) 388 self.attr_accesses[path].add(attrnames) 389 390 self.record_access_details(None, attrnames, self.in_assignment, 391 self.in_invocation) 392 del self.attrs[0] 393 return 394 395 # Name-based accesses will handle the first attribute in a 396 # chain. 397 398 else: 399 attrname = remaining[0] 400 401 # Attribute assignments are used to identify instance attributes. 402 403 if isinstance(n, compiler.ast.AssAttr) and \ 404 self.in_class and self.in_function and n.expr.name == "self": 405 406 self.set_instance_attr(attrname) 407 408 # Record attribute usage using any name local to this namespace, 409 # if assigned in the namespace, or using an external name 410 # (presently just globals within classes). 411 412 name = self.get_name_for_tracking(name_ref.name, name_ref) 413 tracker = self.trackers[-1] 414 415 immediate_access = len(self.attrs) == 1 416 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 417 418 # Record global-based chains for subsequent resolution. 419 420 if name_ref.is_global_name(): 421 self.record_global_access_details(name, attrnames) 422 423 # Make sure the name is being tracked: global names will not 424 # already be initialised in a branch and must be added 425 # explicitly. 426 427 if not tracker.have_name(name): 428 tracker.assign_names([name]) 429 if self.in_function: 430 self.scope_globals[path].add(name) 431 432 # Record attribute usage in the tracker, and record the branch 433 # information for the access. 434 435 branches = tracker.use_attribute(name, attrname, 436 self.in_invocation is not None, assignment) 437 438 if not branches: 439 raise InspectError("Name %s is accessed using %s before an assignment." % ( 440 name, attrname), path, n) 441 442 self.record_branches_for_access(branches, name, attrnames) 443 access_number = self.record_access_details(name, attrnames, 444 self.in_assignment, self.in_invocation) 445 446 del self.attrs[0] 447 return AccessRef(name, attrnames, access_number) 448 449 def process_class_node(self, n): 450 451 "Process the given class node 'n'." 452 453 path = self.get_namespace_path() 454 455 # To avoid notions of class "versions" where the same definition 456 # might be parameterised with different state and be referenced 457 # elsewhere (as base classes, for example), classes in functions or 458 # conditions are forbidden. 459 460 if self.in_function or self.in_conditional: 461 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 462 path, n.name) 463 return 464 465 # Resolve base classes. 466 467 bases = [] 468 469 for base in n.bases: 470 base_class = self.get_class(base) 471 472 if not base_class: 473 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 474 path, n.name, base) 475 return 476 else: 477 bases.append(base_class) 478 479 # Detect conflicting definitions. Such definitions cause conflicts in 480 # the storage of namespace-related information. 481 482 class_name = self.get_object_path(n.name) 483 ref = self.get_object(class_name, defer=False) 484 485 if ref and ref.static(): 486 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 487 488 # Record bases for the class and retain the class name. 489 # Note that the function class does not inherit from the object class. 490 491 if not bases and class_name != "__builtins__.core.object" and \ 492 class_name != "__builtins__.core.function": 493 494 ref = self.get_object("__builtins__.object") 495 bases.append(ref) 496 497 self.importer.classes[class_name] = self.classes[class_name] = bases 498 self.importer.subclasses[class_name] = set() 499 self.scope_globals[class_name] = set() 500 501 # Set the definition before entering the namespace rather than 502 # afterwards because methods may reference it. In normal Python, 503 # a class is not accessible until the definition is complete, but 504 # methods can generally reference it since upon being called the 505 # class will already exist. 506 507 self.set_definition(n.name, "<class>") 508 509 in_class = self.in_class 510 self.in_class = class_name 511 self.set_instance_attr("__class__", Reference("<class>", class_name)) 512 self.enter_namespace(n.name) 513 514 # Do not provide the special instantiator attributes on the function 515 # class. Function instances provide these attributes. 516 517 if class_name != "__builtins__.core.function": 518 519 self.set_name("__fn__") # special instantiator attribute 520 self.set_name("__args__") # special instantiator attribute 521 522 # Provide leafname, parent and context attributes. 523 524 parent, leafname = class_name.rsplit(".", 1) 525 self.set_name("__name__", self.get_constant("string", leafname).reference()) 526 527 if class_name != "__builtins__.core.function": 528 self.set_name("__parent__") 529 530 self.process_structure_node(n.code) 531 self.exit_namespace() 532 self.in_class = in_class 533 534 def process_from_node(self, n): 535 536 "Process the given node 'n', importing from another module." 537 538 path = self.get_namespace_path() 539 540 module_name, names = self.get_module_name(n) 541 if module_name == self.name: 542 raise InspectError("Cannot import from the current module.", path, n) 543 544 self.queue_module(module_name) 545 546 # Attempt to obtain the referenced objects. 547 548 for name, alias in n.names: 549 if name == "*": 550 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 551 552 # Explicit names. 553 554 ref = self.import_name_from_module(name, module_name) 555 value = ResolvedNameRef(alias or name, ref) 556 self.set_general_local(alias or name, value) 557 558 def process_function_node(self, n, name): 559 560 """ 561 Process the given function or lambda node 'n' with the given 'name'. 562 """ 563 564 is_lambda = isinstance(n, compiler.ast.Lambda) 565 566 # Where a function is declared conditionally, use a separate name for 567 # the definition, and assign the definition to the stated name. 568 569 if (self.in_conditional or self.in_function) and not is_lambda: 570 original_name = name 571 name = self.get_lambda_name() 572 else: 573 original_name = None 574 575 # Detect conflicting definitions. Such definitions cause conflicts in 576 # the storage of namespace-related information. 577 578 function_name = self.get_object_path(name) 579 ref = self.get_object(function_name, defer=False) 580 581 if ref and ref.static(): 582 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 583 584 # Initialise argument and local records. 585 586 argnames = get_argnames(n.argnames) 587 is_method = self.in_class and not self.in_function 588 589 # Remove explicit "self" from method parameters. 590 591 if is_method and argnames and argnames[0] == "self": 592 del argnames[0] 593 594 # Convert .name entries in the parameters, provided this is a method. 595 596 l = [] 597 attr_initialisers = [] 598 599 for argname in argnames: 600 if argname[0] == ".": 601 if not is_method: 602 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 603 604 argname = argname[1:] 605 attr_initialisers.append(argname) 606 607 if argname in l: 608 raise InspectError("Argument name %s is repeated." % argname, function_name, n) 609 610 l.append(argname) 611 612 argnames = l 613 614 # Copy and propagate the parameters. 615 616 self.importer.function_parameters[function_name] = \ 617 self.function_parameters[function_name] = argnames[:] 618 619 self.importer.function_attr_initialisers[function_name] = \ 620 self.function_attr_initialisers[function_name] = attr_initialisers 621 622 # Define all arguments/parameters in the local namespace. 623 624 locals = \ 625 self.importer.function_locals[function_name] = \ 626 self.function_locals[function_name] = {} 627 628 # Insert "self" into method locals. 629 630 if is_method: 631 argnames.insert(0, "self") 632 633 # Define "self" in terms of the class if in a method. 634 # This does not diminish the need for type-narrowing in the deducer. 635 636 if argnames: 637 if self.in_class and not self.in_function and argnames[0] == "self": 638 locals[argnames[0]] = Reference("<instance>", self.in_class) 639 else: 640 locals[argnames[0]] = Reference("<var>") 641 642 for argname in argnames[1:]: 643 locals[argname] = Reference("<var>") 644 645 globals = self.scope_globals[function_name] = set() 646 647 # Process the defaults. 648 649 defaults = self.importer.function_defaults[function_name] = \ 650 self.function_defaults[function_name] = [] 651 652 for argname, default in compiler.ast.get_defaults(n): 653 if default: 654 655 # Obtain any reference for the default. 656 657 name_ref = self.process_structure_node(default) 658 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 659 660 # Reset conditional tracking to focus on the function contents. 661 662 in_conditional = self.in_conditional 663 self.in_conditional = False 664 665 in_function = self.in_function 666 self.in_function = function_name 667 668 self.enter_namespace(name) 669 670 # Define a leafname attribute value for the function instance. 671 672 ref = self.get_builtin_class("string") 673 self.reserve_constant(function_name, name, ref.get_origin()) 674 675 # Track attribute usage within the namespace. 676 677 path = self.get_namespace_path() 678 self.start_tracking(locals) 679 680 # Establish attributes for .name entries, provided this is a method. 681 682 for argname in attr_initialisers: 683 self.process_assignment_node( 684 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 685 compiler.ast.Name(argname)) 686 687 self.process_structure_node(n.code) 688 returns_value = self.stop_tracking() 689 690 # Record any null result. 691 692 is_initialiser = is_method and name == "__init__" 693 694 if not returns_value and not is_initialiser: 695 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 696 697 # Exit to the parent. 698 699 self.exit_namespace() 700 701 # Update flags. 702 703 self.in_function = in_function 704 self.in_conditional = in_conditional 705 706 # Define the function using the appropriate name. 707 708 self.set_definition(name, "<function>") 709 710 # Where a function is set conditionally, assign the name. 711 712 if original_name: 713 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 714 715 def process_global_node(self, n): 716 717 """ 718 Process the given "global" node 'n'. 719 """ 720 721 path = self.get_namespace_path() 722 723 if path != self.name: 724 self.scope_globals[path].update(n.names) 725 726 def process_if_node(self, n): 727 728 """ 729 Process the given "if" node 'n'. 730 """ 731 732 tracker = self.trackers[-1] 733 tracker.new_branchpoint() 734 735 for test, body in n.tests: 736 self.process_structure_node(test) 737 738 tracker.new_branch() 739 740 in_conditional = self.in_conditional 741 self.in_conditional = True 742 self.process_structure_node(body) 743 self.in_conditional = in_conditional 744 745 tracker.shelve_branch() 746 747 # Maintain a branch for the else clause. 748 749 tracker.new_branch() 750 if n.else_: 751 self.process_structure_node(n.else_) 752 tracker.shelve_branch() 753 754 tracker.merge_branches() 755 756 def process_import_node(self, n): 757 758 "Process the given import node 'n'." 759 760 path = self.get_namespace_path() 761 762 # Load the mentioned module. 763 764 for name, alias in n.names: 765 if name == self.name: 766 raise InspectError("Cannot import the current module.", path, n) 767 768 self.set_module(alias or name.split(".")[-1], name) 769 self.queue_module(name, True) 770 771 def process_invocation_node(self, n): 772 773 "Process the given invocation node 'n'." 774 775 path = self.get_namespace_path() 776 777 in_invocation = self.in_invocation 778 self.in_invocation = None 779 780 # Process the arguments. 781 782 keywords = set() 783 784 for arg in n.args: 785 self.process_structure_node(arg) 786 if isinstance(arg, compiler.ast.Keyword): 787 keywords.add(arg.name) 788 789 keywords = list(keywords) 790 keywords.sort() 791 792 # Communicate to the invocation target expression that it forms the 793 # target of an invocation, potentially affecting attribute accesses. 794 795 self.in_invocation = len(n.args), keywords 796 797 # Process the expression, obtaining any identified reference. 798 799 name_ref = self.process_structure_node(n.node) 800 self.in_invocation = in_invocation 801 802 # Detect class invocations. 803 804 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 805 return InstanceRef(name_ref.reference().instance_of()) 806 807 elif isinstance(name_ref, (NameRef, AccessRef)): 808 return InvocationRef(name_ref) 809 810 # Provide a general reference to indicate that something is produced 811 # by the invocation, useful for retaining assignment expression 812 # details. 813 814 return VariableRef() 815 816 def process_lambda_node(self, n): 817 818 "Process the given lambda node 'n'." 819 820 name = self.get_lambda_name() 821 self.process_function_node(n, name) 822 823 origin = self.get_object_path(name) 824 825 if self.function_defaults.get(origin): 826 return None 827 else: 828 return ResolvedNameRef(name, Reference("<function>", origin)) 829 830 def process_logical_node(self, n): 831 832 "Process the given operator node 'n'." 833 834 return self.process_operator_chain(n.nodes) 835 836 def process_name_node(self, n): 837 838 "Process the given name node 'n'." 839 840 path = self.get_namespace_path() 841 842 # Find predefined constant names before anything else. 843 844 if n.name in predefined_constants: 845 ref = self.get_builtin(n.name) 846 value = ResolvedNameRef(n.name, ref) 847 return value 848 849 # Special names that have already been identified. 850 851 if n.name.startswith("$"): 852 value = self.get_special(n.name) 853 if value: 854 return value 855 856 # Special case for operator functions introduced through code 857 # transformations. 858 859 if n.name.startswith("$op"): 860 861 # Obtain the location of the actual function defined in the operator 862 # package. 863 864 op = n.name[len("$op"):] 865 866 # Attempt to get a reference. 867 868 ref = self.import_name_from_module(op, "operator") 869 870 # Record the imported name and provide the resolved name reference. 871 872 value = ResolvedNameRef(n.name, ref) 873 self.set_special(n.name, value) 874 return value 875 876 # Special case for print operations. 877 878 elif n.name.startswith("$print"): 879 880 # Attempt to get a reference. 881 882 ref = self.get_builtin("print_") 883 884 # Record the imported name and provide the resolved name reference. 885 886 value = ResolvedNameRef(n.name, ref) 887 self.set_special(n.name, value) 888 return value 889 890 # Test for self usage, which is only allowed in methods. 891 892 if n.name == "self" and not (self.in_function and self.in_class): 893 raise InspectError("Use of self is only allowed in methods.", path, n) 894 895 # Record usage of the name. 896 897 self.record_name(n.name) 898 899 # Search for unknown names in non-function scopes immediately. 900 # Temporary names should not be re-used between scopes. 901 # External names in functions are resolved later. 902 903 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 904 905 if ref: 906 self.record_name_access(n.name, True) 907 return ResolvedNameRef(n.name, ref, is_global=True) 908 909 # Explicitly-declared global names. 910 911 elif self.in_function and n.name in self.scope_globals[path]: 912 self.record_name_access(n.name, True) 913 return NameRef(n.name, is_global=True) 914 915 # Examine other names. 916 917 else: 918 919 # Check local names. 920 921 access_number = self.record_name_access(n.name) 922 923 # Local name. 924 925 if access_number is not None: 926 return LocalNameRef(n.name, access_number) 927 928 # Possible global or built-in name. 929 930 else: 931 self.record_name_access(n.name, True) 932 return NameRef(n.name, is_global=True) 933 934 def record_name_access(self, name, is_global=False): 935 936 """ 937 Record an access involving 'name' if the name is being tracked, using 938 'is_global' to indicate whether the name is global. 939 """ 940 941 name = self.get_name_for_tracking(name, is_global=is_global) 942 branches = self.trackers[-1].tracking_name(name) 943 if branches: 944 self.record_branches_for_access(branches, name, None) 945 return self.record_access_details(name, None, self.in_assignment, 946 self.in_invocation) 947 return None 948 949 def process_operator_chain(self, nodes): 950 951 """ 952 Process the given chain of 'nodes', processing each node or item. 953 Each node starts a new conditional region, effectively making a deeply- 954 nested collection of if-like statements. 955 """ 956 957 results = [] 958 959 tracker = self.trackers[-1] 960 961 for item in nodes: 962 tracker.new_branchpoint() 963 tracker.new_branch() 964 result = self.process_structure_node(item) 965 if result: 966 results.append(result) 967 968 for item in nodes[:-1]: 969 tracker.shelve_branch() 970 tracker.new_branch() 971 tracker.shelve_branch() 972 tracker.merge_branches() 973 974 tracker.shelve_branch() 975 tracker.merge_branches() 976 977 return MultipleRef(results) 978 979 def process_try_node(self, n): 980 981 """ 982 Process the given "try...except" node 'n'. 983 """ 984 985 self.record_exception_handler() 986 987 tracker = self.trackers[-1] 988 tracker.new_branchpoint() 989 990 self.process_structure_node(n.body) 991 992 for name, var, handler in n.handlers: 993 if name is not None: 994 self.process_structure_node(name) 995 996 # Any abandoned branches from the body can now be resumed in a new 997 # branch. 998 999 tracker.resume_abandoned_branches() 1000 1001 # Establish the local for the handler. 1002 1003 if var is not None: 1004 self.process_assignment_node(var, None) 1005 if handler is not None: 1006 self.process_structure_node(handler) 1007 1008 tracker.shelve_branch() 1009 1010 # The else clause maintains the usage from the body but without the 1011 # abandoned branches since they would never lead to the else clause 1012 # being executed. 1013 1014 if n.else_: 1015 tracker.new_branch() 1016 self.process_structure_node(n.else_) 1017 tracker.shelve_branch() 1018 1019 # Without an else clause, a null branch propagates the successful 1020 # outcome. 1021 1022 else: 1023 tracker.new_branch() 1024 tracker.shelve_branch() 1025 1026 tracker.merge_branches() 1027 1028 def process_try_finally_node(self, n): 1029 1030 """ 1031 Process the given "try...finally" node 'n'. 1032 """ 1033 1034 self.record_exception_handler() 1035 1036 tracker = self.trackers[-1] 1037 self.process_structure_node(n.body) 1038 1039 # Any abandoned branches from the body can now be resumed. 1040 1041 branches = tracker.resume_all_abandoned_branches() 1042 self.process_structure_node(n.final) 1043 1044 # At the end of the finally clause, abandoned branches are discarded. 1045 1046 tracker.restore_active_branches(branches) 1047 1048 def process_while_node(self, n): 1049 1050 "Process the given while node 'n'." 1051 1052 tracker = self.trackers[-1] 1053 tracker.new_branchpoint(loop_node=True) 1054 1055 # Evaluate any test or iterator outside the loop. 1056 1057 self.process_structure_node(n.test) 1058 1059 # Propagate attribute usage to branches. 1060 1061 tracker.new_branch(loop_node=True) 1062 1063 # Enter the loop. 1064 1065 in_conditional = self.in_conditional 1066 self.in_conditional = True 1067 self.process_structure_node(n.body) 1068 self.in_conditional = in_conditional 1069 1070 # Continuing branches are resumed before any test. 1071 1072 tracker.resume_continuing_branches() 1073 1074 # Evaluate any continuation test within the body. 1075 1076 self.process_structure_node(n.test) 1077 1078 tracker.shelve_branch(loop_node=True) 1079 1080 # Support the non-looping condition. 1081 1082 tracker.new_branch() 1083 tracker.shelve_branch() 1084 1085 tracker.merge_branches() 1086 1087 # Evaluate any else clause outside branches. 1088 1089 if n.else_: 1090 self.process_structure_node(n.else_) 1091 1092 # Connect broken branches to the code after any loop. 1093 1094 tracker.resume_broken_branches() 1095 1096 # Branch tracking methods. 1097 1098 def start_tracking(self, names): 1099 1100 """ 1101 Start tracking attribute usage for names in the current namespace, 1102 immediately registering the given 'names'. 1103 """ 1104 1105 path = self.get_namespace_path() 1106 parent = self.trackers[-1] 1107 tracker = BranchTracker() 1108 self.trackers.append(tracker) 1109 1110 # Record the given names established as new branches. 1111 1112 tracker.assign_names(names) 1113 1114 def assign_name(self, name, name_ref): 1115 1116 "Assign to 'name' the given 'name_ref' in the current namespace." 1117 1118 name = self.get_name_for_tracking(name) 1119 self.trackers[-1].assign_names([name], [name_ref]) 1120 1121 def stop_tracking(self): 1122 1123 """ 1124 Stop tracking attribute usage, recording computed usage for the current 1125 namespace. Indicate whether a value is always returned from the 1126 namespace. 1127 """ 1128 1129 path = self.get_namespace_path() 1130 tracker = self.trackers.pop() 1131 self.record_assignments_for_access(tracker) 1132 1133 self.attr_usage[path] = tracker.get_all_usage() 1134 self.name_initialisers[path] = tracker.get_all_values() 1135 1136 return tracker.returns_value() 1137 1138 def start_tracking_in_module(self): 1139 1140 "Start tracking attribute usage in the module." 1141 1142 tracker = BranchTracker() 1143 self.trackers.append(tracker) 1144 1145 def stop_tracking_in_module(self): 1146 1147 "Stop tracking attribute usage in the module." 1148 1149 tracker = self.trackers[0] 1150 self.record_assignments_for_access(tracker) 1151 self.attr_usage[self.name] = tracker.get_all_usage() 1152 self.name_initialisers[self.name] = tracker.get_all_values() 1153 1154 def record_assignments_for_access(self, tracker): 1155 1156 """ 1157 For the current path, use the given 'tracker' to record assignment 1158 version information for attribute accesses. 1159 """ 1160 1161 path = self.get_path_for_access() 1162 1163 if not self.attr_accessor_branches.has_key(path): 1164 return 1165 1166 init_item(self.attr_accessors, path, dict) 1167 attr_accessors = self.attr_accessors[path] 1168 1169 # Obtain the branches applying during each access. 1170 1171 for access, all_branches in self.attr_accessor_branches[path].items(): 1172 name, attrnames = access 1173 init_item(attr_accessors, access, list) 1174 1175 # Obtain the assignments applying to each branch. 1176 1177 for branches in all_branches: 1178 positions = tracker.get_assignment_positions_for_branches(name, branches) 1179 1180 # Detect missing name information. 1181 1182 if None in positions: 1183 globals = self.global_attr_accesses.get(path) 1184 accesses = globals and globals.get(name) 1185 if not accesses: 1186 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1187 self.get_namespace_path(), name) 1188 positions.remove(None) 1189 1190 attr_accessors[access].append(positions) 1191 1192 def record_branches_for_access(self, branches, name, attrnames): 1193 1194 """ 1195 Record the given 'branches' for an access involving the given 'name' and 1196 'attrnames'. 1197 """ 1198 1199 access = name, attrnames 1200 path = self.get_path_for_access() 1201 1202 init_item(self.attr_accessor_branches, path, dict) 1203 attr_accessor_branches = self.attr_accessor_branches[path] 1204 1205 init_item(attr_accessor_branches, access, list) 1206 attr_accessor_branches[access].append(branches) 1207 1208 def record_access_details(self, name, attrnames, assignment, invocation): 1209 1210 """ 1211 For the given 'name' and 'attrnames', record an access indicating 1212 whether an 'assignment' or an 'invocation' is occurring. 1213 1214 These details correspond to accesses otherwise recorded by the attribute 1215 accessor and attribute access dictionaries. 1216 """ 1217 1218 access = name, attrnames 1219 path = self.get_path_for_access() 1220 1221 init_item(self.attr_access_modifiers, path, dict) 1222 init_item(self.attr_access_modifiers[path], access, list) 1223 1224 access_number = len(self.attr_access_modifiers[path][access]) 1225 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1226 return access_number 1227 1228 def record_global_access_details(self, name, attrnames): 1229 1230 """ 1231 Record details of a global access via the given 'name' involving the 1232 indicated 'attrnames'. 1233 """ 1234 1235 path = self.get_namespace_path() 1236 1237 init_item(self.global_attr_accesses, path, dict) 1238 init_item(self.global_attr_accesses[path], name, set) 1239 self.global_attr_accesses[path][name].add(attrnames) 1240 1241 # Namespace modification. 1242 1243 def record_name(self, name): 1244 1245 "Record the use of 'name' in a namespace." 1246 1247 path = self.get_namespace_path() 1248 init_item(self.names_used, path, set) 1249 self.names_used[path].add(name) 1250 1251 def set_module(self, name, module_name): 1252 1253 """ 1254 Set a module in the current namespace using the given 'name' associated 1255 with the corresponding 'module_name'. 1256 """ 1257 1258 if name: 1259 self.set_general_local(name, Reference("<module>", module_name)) 1260 1261 def set_definition(self, name, kind): 1262 1263 """ 1264 Set the definition having the given 'name' and 'kind'. 1265 1266 Definitions are set in the static namespace hierarchy, but they can also 1267 be recorded for function locals. 1268 """ 1269 1270 if self.is_global(name): 1271 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1272 self.get_namespace_path(), name) 1273 1274 path = self.get_object_path(name) 1275 self.set_object(path, kind) 1276 1277 ref = self.get_object(path) 1278 if ref.get_kind() == "<var>": 1279 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1280 self.get_namespace_path(), name) 1281 1282 if not self.is_global(name) and self.in_function: 1283 self.set_function_local(name, ref) 1284 1285 def set_function_local(self, name, ref=None): 1286 1287 "Set the local with the given 'name' and optional 'ref'." 1288 1289 path = self.get_namespace_path() 1290 locals = self.function_locals[path] 1291 used = self.names_used.get(path) 1292 1293 if not locals.has_key(name) and used and name in used: 1294 raise InspectError("Name %s assigned locally but used previously." % name, path) 1295 1296 multiple = not ref or locals.has_key(name) and locals[name] != ref 1297 locals[name] = multiple and Reference("<var>") or ref 1298 1299 def assign_general_local(self, name, name_ref): 1300 1301 """ 1302 Set for 'name' the given 'name_ref', recording the name for attribute 1303 usage tracking. 1304 """ 1305 1306 self.set_general_local(name, name_ref) 1307 self.assign_name(name, name_ref) 1308 1309 def set_general_local(self, name, value=None): 1310 1311 """ 1312 Set the 'name' with optional 'value' in any kind of local namespace, 1313 where the 'value' should be a reference if specified. 1314 """ 1315 1316 init_value = self.get_initialising_value(value) 1317 1318 # Module global names. 1319 1320 if self.is_global(name): 1321 path = self.get_global_path(name) 1322 self.set_object(path, init_value) 1323 1324 # Function local names. 1325 1326 elif self.in_function: 1327 self.set_function_local(name, init_value) 1328 1329 # Other namespaces (classes). 1330 1331 else: 1332 self.set_name(name, init_value) 1333 1334 def set_name(self, name, ref=None): 1335 1336 "Attach the 'name' with optional 'ref' to the current namespace." 1337 1338 self.set_object(self.get_object_path(name), ref) 1339 1340 def set_instance_attr(self, name, ref=None): 1341 1342 """ 1343 Add an instance attribute of the given 'name' to the current class, 1344 using the optional 'ref'. 1345 """ 1346 1347 self._set_instance_attr(self.in_class, name, ref) 1348 1349 def _set_instance_attr(self, path, name, ref=None): 1350 1351 init_item(self.instance_attrs, path, set) 1352 self.instance_attrs[path].add(name) 1353 1354 if ref: 1355 init_item(self.instance_attr_constants, path, dict) 1356 self.instance_attr_constants[path][name] = ref 1357 1358 def get_initialising_value(self, value): 1359 1360 "Return a suitable initialiser reference for 'value'." 1361 1362 if isinstance(value, Result): 1363 return value.reference() 1364 else: 1365 return value 1366 1367 # Static, program-relative naming. 1368 1369 def find_name(self, name): 1370 1371 """ 1372 Return the qualified name for the given 'name' used in the current 1373 non-function namespace. 1374 """ 1375 1376 path = self.get_namespace_path() 1377 ref = None 1378 1379 if not self.in_function and name not in predefined_constants: 1380 if self.in_class: 1381 ref = self.get_object(self.get_object_path(name), False) 1382 if not ref: 1383 ref = self.get_global_or_builtin(name) 1384 1385 return ref 1386 1387 def get_class(self, node): 1388 1389 """ 1390 Use the given 'node' to obtain the identity of a class. Return a 1391 reference for the class. Unresolved dependencies are permitted and must 1392 be resolved later. 1393 """ 1394 1395 ref = self._get_class(node) 1396 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1397 1398 def _get_class(self, node): 1399 1400 """ 1401 Use the given 'node' to find a class definition. Return a reference to 1402 the class. 1403 """ 1404 1405 if isinstance(node, compiler.ast.Getattr): 1406 1407 # Obtain the identity of the access target. 1408 1409 ref = self._get_class(node.expr) 1410 1411 # Where the target is a class or module, obtain the identity of the 1412 # attribute. 1413 1414 if ref.has_kind(["<function>", "<var>"]): 1415 return None 1416 else: 1417 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1418 return self.get_object(attrname) 1419 1420 # Names can be module-level or built-in. 1421 1422 elif isinstance(node, compiler.ast.Name): 1423 1424 # Record usage of the name and attempt to identify it. 1425 1426 self.record_name(node.name) 1427 return self.find_name(node.name) 1428 else: 1429 return None 1430 1431 def get_constant(self, name, value): 1432 1433 "Return a constant reference for the given type 'name' and 'value'." 1434 1435 ref = self.get_builtin_class(name) 1436 return self.get_constant_reference(ref, value) 1437 1438 def get_literal_instance(self, n, name=None): 1439 1440 """ 1441 For node 'n', return a reference to an instance of 'name', or if 'name' 1442 is not specified, deduce the type from the value. 1443 """ 1444 1445 # Handle stray None constants (Sliceobj seems to produce them). 1446 1447 if name == "NoneType": 1448 return self.process_name_node(compiler.ast.Name("None")) 1449 1450 # Obtain the details of the literal itself. 1451 # An alias to the type is generated for sequences. 1452 1453 if name in ("dict", "list", "tuple"): 1454 ref = self.get_builtin_class(name) 1455 self.set_special_literal(name, ref) 1456 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1457 1458 # Constant values are independently recorded. 1459 1460 else: 1461 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1462 ref = self.get_builtin_class(typename) 1463 return self.get_constant_reference(ref, value, encoding) 1464 1465 # Special names. 1466 1467 def get_special(self, name): 1468 1469 "Return any stored value for the given special 'name'." 1470 1471 value = self.special.get(name) 1472 if value: 1473 ref, paths = value 1474 else: 1475 ref = None 1476 return ref 1477 1478 def set_special(self, name, value): 1479 1480 """ 1481 Set a special 'name' that merely tracks the use of an implicit object 1482 'value'. 1483 """ 1484 1485 if not self.special.has_key(name): 1486 paths = set() 1487 self.special[name] = value, paths 1488 else: 1489 _ref, paths = self.special[name] 1490 1491 paths.add(self.get_namespace_path()) 1492 1493 def set_special_literal(self, name, ref): 1494 1495 """ 1496 Set a special name for the literal type 'name' having type 'ref'. Such 1497 special names provide a way of referring to literal object types. 1498 """ 1499 1500 literal_name = "$L%s" % name 1501 value = ResolvedNameRef(literal_name, ref) 1502 self.set_special(literal_name, value) 1503 1504 # Exceptions. 1505 1506 def record_exception_handler(self): 1507 1508 "Record the current namespace as employing an exception handler." 1509 1510 self.exception_namespaces.add(self.get_namespace_path()) 1511 1512 # Return values. 1513 1514 def record_return_value(self, expr): 1515 1516 "Record the given return 'expr'." 1517 1518 path = self.get_namespace_path() 1519 l = init_item(self.return_values, path, list) 1520 l.append(expr) 1521 if not self.importer.all_return_values.has_key(path): 1522 self.importer.all_return_values[path] = l 1523 1524 # vim: tabstop=4 expandtab shiftwidth=4