1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007-2019, 2021 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 """ 21 22 from branching import BranchTracker 23 from common import CommonModule, get_argnames, init_item, \ 24 predefined_constants, privileged_attributes 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("str", self.name).reference()) 81 self.set_name("__file__", self.get_constant("str", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("str", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 path = self.get_namespace_path() 362 363 # Test for access to special privileged attributes. 364 365 if isinstance(n, compiler.ast.Getattr) and \ 366 n.attrname in privileged_attributes and not n.privileged: 367 368 raise InspectError("Attribute %s is accessed by an unprivileged operation." % 369 n.attrname, path, n) 370 371 # Obtain any completed chain and return the reference to it. 372 373 name_ref = self.process_attribute_chain(n) 374 375 if self.have_access_expression(n): 376 return name_ref 377 378 # Where the start of the chain of attributes has been reached, determine 379 # the complete access. 380 381 # Given a non-access node, this chain can be handled in its entirety, 382 # either being name-based and thus an access rooted on a name, or being 383 # based on some other node and thus an anonymous access of some kind. 384 385 # Start with the the full attribute chain. 386 387 remaining = self.attrs 388 attrnames = ".".join(remaining) 389 390 # If the accessor cannot be identified, or where attributes 391 # remain in an attribute chain, record the anonymous accesses. 392 393 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 394 395 init_item(self.attr_accesses, path, set) 396 self.attr_accesses[path].add(attrnames) 397 398 self.record_access_details(None, attrnames, self.in_assignment, 399 self.in_invocation) 400 del self.attrs[0] 401 return 402 403 # Name-based accesses will handle the first attribute in a 404 # chain. 405 406 else: 407 attrname = remaining[0] 408 409 # Attribute assignments are used to identify instance attributes. 410 411 if isinstance(n, compiler.ast.AssAttr) and \ 412 self.in_class and self.in_function and n.expr.name == "self": 413 414 self.set_instance_attr(attrname) 415 416 # Record attribute usage using any name local to this namespace, 417 # if assigned in the namespace, or using an external name 418 # (presently just globals within classes). 419 420 name = self.get_name_for_tracking(name_ref.name, name_ref) 421 tracker = self.trackers[-1] 422 423 immediate_access = len(self.attrs) == 1 424 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 425 426 # Record global-based chains for subsequent resolution. 427 428 if name_ref.is_global_name(): 429 self.record_global_access_details(name, attrnames) 430 431 # Make sure the name is being tracked: global names will not 432 # already be initialised in a branch and must be added 433 # explicitly. 434 435 if not tracker.have_name(name): 436 tracker.assign_names([name]) 437 if self.in_function: 438 self.scope_globals[path].add(name) 439 440 # Record attribute usage in the tracker, and record the branch 441 # information for the access. 442 443 branches = tracker.use_attribute(name, attrname, 444 self.in_invocation is not None, assignment) 445 446 if not branches: 447 raise InspectError("Name %s is accessed using %s before an assignment." % ( 448 name, attrname), path, n) 449 450 self.record_branches_for_access(branches, name, attrnames) 451 access_number = self.record_access_details(name, attrnames, 452 self.in_assignment, self.in_invocation) 453 454 del self.attrs[0] 455 return AccessRef(name, attrnames, access_number) 456 457 def process_class_node(self, n): 458 459 "Process the given class node 'n'." 460 461 path = self.get_namespace_path() 462 463 # To avoid notions of class "versions" where the same definition 464 # might be parameterised with different state and be referenced 465 # elsewhere (as base classes, for example), classes in functions or 466 # conditions are forbidden. 467 468 if self.in_function or self.in_conditional: 469 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 470 path, n.name) 471 return 472 473 # Resolve base classes. 474 475 bases = [] 476 477 for base in n.bases: 478 base_class = self.get_class(base) 479 480 if not base_class: 481 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 482 path, n.name, base) 483 return 484 else: 485 bases.append(base_class) 486 487 # Detect conflicting definitions. Such definitions cause conflicts in 488 # the storage of namespace-related information. 489 490 class_name = self.get_object_path(n.name) 491 ref = self.get_object(class_name, defer=False) 492 493 if ref and ref.static(): 494 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 495 496 # Record bases for the class and retain the class name. 497 # Note that the function class does not inherit from the object class. 498 499 if not bases and class_name != "__builtins__.core.object" and \ 500 class_name != "__builtins__.core.function": 501 502 ref = self.get_object("__builtins__.object") 503 bases.append(ref) 504 505 self.importer.classes[class_name] = self.classes[class_name] = bases 506 self.importer.subclasses[class_name] = set() 507 self.scope_globals[class_name] = set() 508 509 # Set the definition before entering the namespace rather than 510 # afterwards because methods may reference it. In normal Python, 511 # a class is not accessible until the definition is complete, but 512 # methods can generally reference it since upon being called the 513 # class will already exist. 514 515 self.set_definition(n.name, "<class>") 516 517 in_class = self.in_class 518 self.in_class = class_name 519 self.set_instance_attr("__class__", Reference("<class>", class_name)) 520 self.enter_namespace(n.name) 521 522 # Do not provide the special instantiator attributes on the function 523 # class. Function instances provide these attributes. 524 525 if class_name != "__builtins__.core.function": 526 527 self.set_name("__fn__") # special instantiator attribute 528 self.set_name("__args__") # special instantiator attribute 529 530 # Provide leafname, parent and context attributes. 531 532 parent, leafname = class_name.rsplit(".", 1) 533 self.set_name("__name__", self.get_constant("str", leafname).reference()) 534 535 if class_name != "__builtins__.core.function": 536 self.set_name("__parent__") 537 538 self.process_structure_node(n.code) 539 self.exit_namespace() 540 self.in_class = in_class 541 542 def process_from_node(self, n): 543 544 "Process the given node 'n', importing from another module." 545 546 path = self.get_namespace_path() 547 548 module_name, names = self.get_module_name(n) 549 if module_name == self.name: 550 raise InspectError("Cannot import from the current module.", path, n) 551 552 self.queue_module(module_name) 553 554 # Attempt to obtain the referenced objects. 555 556 for name, alias in n.names: 557 if name == "*": 558 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 559 560 # Explicit names. 561 562 ref = self.import_name_from_module(name, module_name) 563 value = ResolvedNameRef(alias or name, ref) 564 self.set_general_local(alias or name, value) 565 566 def process_function_node(self, n, name): 567 568 """ 569 Process the given function or lambda node 'n' with the given 'name'. 570 """ 571 572 is_lambda = isinstance(n, compiler.ast.Lambda) 573 574 # Where a function is declared conditionally, use a separate name for 575 # the definition, and assign the definition to the stated name. 576 577 if (self.in_conditional or self.in_function) and not is_lambda: 578 original_name = name 579 name = self.get_lambda_name() 580 else: 581 original_name = None 582 583 # Detect conflicting definitions. Such definitions cause conflicts in 584 # the storage of namespace-related information. 585 586 function_name = self.get_object_path(name) 587 ref = self.get_object(function_name, defer=False) 588 589 if ref and ref.static(): 590 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 591 592 # Initialise argument and local records. 593 594 argnames = get_argnames(n.argnames) 595 is_method = self.in_class and not self.in_function 596 597 # Remove explicit "self" from method parameters. 598 599 if is_method and argnames and argnames[0] == "self": 600 del argnames[0] 601 602 # Convert .name entries in the parameters, provided this is a method. 603 604 l = [] 605 attr_initialisers = [] 606 607 for argname in argnames: 608 if argname[0] == ".": 609 if not is_method: 610 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 611 612 argname = argname[1:] 613 attr_initialisers.append(argname) 614 615 if argname in l: 616 raise InspectError("Argument name %s is repeated." % argname, function_name, n) 617 618 l.append(argname) 619 620 argnames = l 621 622 # Copy and propagate the parameters. 623 624 self.importer.function_parameters[function_name] = \ 625 self.function_parameters[function_name] = argnames[:] 626 627 self.importer.function_attr_initialisers[function_name] = \ 628 self.function_attr_initialisers[function_name] = attr_initialisers 629 630 # Define all arguments/parameters in the local namespace. 631 632 locals = \ 633 self.importer.function_locals[function_name] = \ 634 self.function_locals[function_name] = {} 635 636 # Insert "self" into method locals. 637 638 if is_method: 639 argnames.insert(0, "self") 640 641 # Define "self" in terms of the class if in a method. 642 # This does not diminish the need for type-narrowing in the deducer. 643 644 if argnames: 645 if self.in_class and not self.in_function and argnames[0] == "self": 646 locals[argnames[0]] = Reference("<instance>", self.in_class) 647 else: 648 locals[argnames[0]] = Reference("<var>") 649 650 for argname in argnames[1:]: 651 locals[argname] = Reference("<var>") 652 653 globals = self.scope_globals[function_name] = set() 654 655 # Process the defaults. 656 657 defaults = self.importer.function_defaults[function_name] = \ 658 self.function_defaults[function_name] = [] 659 660 for argname, default in compiler.ast.get_defaults(n): 661 if argname[0] == ".": 662 argname = argname[1:] 663 664 if default: 665 666 # Obtain any reference for the default. 667 668 name_ref = self.process_structure_node(default) 669 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 670 671 # Reset conditional tracking to focus on the function contents. 672 673 in_conditional = self.in_conditional 674 self.in_conditional = False 675 676 in_function = self.in_function 677 self.in_function = function_name 678 679 self.enter_namespace(name) 680 681 # Define a leafname attribute value for the function instance. 682 683 ref = self.get_builtin_class("str") 684 self.reserve_constant(function_name, name, ref.get_origin()) 685 686 # Track attribute usage within the namespace. 687 688 path = self.get_namespace_path() 689 self.start_tracking(locals) 690 691 # Establish attributes for .name entries, provided this is a method. 692 693 for argname in attr_initialisers: 694 self.process_assignment_node( 695 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 696 compiler.ast.Name(argname)) 697 698 self.process_structure_node(n.code) 699 returns_value = self.stop_tracking() 700 701 # Record any null result. 702 703 is_initialiser = is_method and name == "__init__" 704 705 if not returns_value and not is_initialiser: 706 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 707 708 # Exit to the parent. 709 710 self.exit_namespace() 711 712 # Update flags. 713 714 self.in_function = in_function 715 self.in_conditional = in_conditional 716 717 # Define the function using the appropriate name. 718 719 self.set_definition(name, "<function>") 720 721 # Where a function is set conditionally, assign the name. 722 723 if original_name: 724 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 725 726 def process_global_node(self, n): 727 728 """ 729 Process the given "global" node 'n'. 730 """ 731 732 path = self.get_namespace_path() 733 734 if path != self.name: 735 self.scope_globals[path].update(n.names) 736 737 def process_if_node(self, n): 738 739 """ 740 Process the given "if" node 'n'. 741 """ 742 743 tracker = self.trackers[-1] 744 tracker.new_branchpoint() 745 746 for test, body in n.tests: 747 self.process_structure_node(test) 748 749 tracker.new_branch() 750 751 in_conditional = self.in_conditional 752 self.in_conditional = True 753 self.process_structure_node(body) 754 self.in_conditional = in_conditional 755 756 tracker.shelve_branch() 757 758 # Maintain a branch for the else clause. 759 760 tracker.new_branch() 761 if n.else_: 762 self.process_structure_node(n.else_) 763 tracker.shelve_branch() 764 765 tracker.merge_branches() 766 767 def process_import_node(self, n): 768 769 "Process the given import node 'n'." 770 771 path = self.get_namespace_path() 772 773 # Load the mentioned module. 774 775 for name, alias in n.names: 776 if name == self.name: 777 raise InspectError("Cannot import the current module.", path, n) 778 779 self.set_module(alias or name.split(".")[-1], name) 780 self.queue_module(name, True) 781 782 def process_invocation_node(self, n): 783 784 "Process the given invocation node 'n'." 785 786 path = self.get_namespace_path() 787 788 in_invocation = self.in_invocation 789 self.in_invocation = None 790 791 # Process the arguments. 792 793 keywords = set() 794 795 for arg in n.args: 796 self.process_structure_node(arg) 797 if isinstance(arg, compiler.ast.Keyword): 798 keywords.add(arg.name) 799 800 keywords = list(keywords) 801 keywords.sort() 802 803 # Communicate to the invocation target expression that it forms the 804 # target of an invocation, potentially affecting attribute accesses. 805 806 self.in_invocation = len(n.args), keywords 807 808 # Process the expression, obtaining any identified reference. 809 810 name_ref = self.process_structure_node(n.node) 811 self.in_invocation = in_invocation 812 813 # Detect class invocations. 814 815 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 816 return InstanceRef(name_ref.reference().instance_of()) 817 818 elif isinstance(name_ref, (NameRef, AccessRef)): 819 return InvocationRef(name_ref) 820 821 # Provide a general reference to indicate that something is produced 822 # by the invocation, useful for retaining assignment expression 823 # details. 824 825 return VariableRef() 826 827 def process_lambda_node(self, n): 828 829 "Process the given lambda node 'n'." 830 831 name = self.get_lambda_name() 832 self.process_function_node(n, name) 833 834 origin = self.get_object_path(name) 835 836 if self.function_defaults.get(origin): 837 return None 838 else: 839 return ResolvedNameRef(name, Reference("<function>", origin)) 840 841 def process_logical_node(self, n): 842 843 "Process the given operator node 'n'." 844 845 return self.process_operator_chain(n.nodes) 846 847 def process_name_node(self, n): 848 849 "Process the given name node 'n'." 850 851 path = self.get_namespace_path() 852 853 # Find predefined constant names before anything else. 854 855 if n.name in predefined_constants: 856 ref = self.get_builtin(n.name) 857 value = ResolvedNameRef(n.name, ref) 858 return value 859 860 # Special names that have already been identified. 861 862 if n.name.startswith("$"): 863 value = self.get_special(n.name) 864 if value: 865 return value 866 867 # Special case for operator functions introduced through code 868 # transformations. 869 870 if n.name.startswith("$op"): 871 872 # Obtain the location of the actual function defined in the operator 873 # package. 874 875 op = n.name[len("$op"):] 876 877 # Attempt to get a reference. 878 879 ref = self.import_name_from_module(op, "operator") 880 881 # Record the imported name and provide the resolved name reference. 882 883 value = ResolvedNameRef(n.name, ref) 884 self.set_special(n.name, value) 885 return value 886 887 # Special case for sequence length testing. 888 889 elif n.name.startswith("$seq"): 890 op = n.name[len("$seq"):] 891 ref = self.import_name_from_module(op, "__builtins__.sequence") 892 value = ResolvedNameRef(n.name, ref) 893 self.set_special(n.name, value) 894 return value 895 896 # Special case for print operations. 897 898 elif n.name.startswith("$print"): 899 900 # Attempt to get a reference. 901 902 ref = self.get_builtin("print_") 903 904 # Record the imported name and provide the resolved name reference. 905 906 value = ResolvedNameRef(n.name, ref) 907 self.set_special(n.name, value) 908 return value 909 910 # Test for self usage, which is only allowed in methods. 911 912 if n.name == "self" and not (self.in_function and self.in_class): 913 raise InspectError("Use of self is only allowed in methods.", path, n) 914 915 # Record usage of the name. 916 917 self.record_name(n.name) 918 919 # Search for unknown names in non-function scopes immediately. 920 # Temporary names should not be re-used between scopes. 921 # External names in functions are resolved later. 922 923 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 924 925 if ref: 926 self.record_name_access(n.name, True) 927 return ResolvedNameRef(n.name, ref, is_global=True) 928 929 # Explicitly-declared global names. 930 931 elif self.in_function and n.name in self.scope_globals[path]: 932 self.record_name_access(n.name, True) 933 return NameRef(n.name, is_global=True) 934 935 # Examine other names. 936 937 else: 938 939 # Check local names. 940 941 access_number = self.record_name_access(n.name) 942 943 # Local name. 944 945 if access_number is not None: 946 return LocalNameRef(n.name, access_number) 947 948 # Possible global or built-in name. 949 950 else: 951 self.record_name_access(n.name, True) 952 return NameRef(n.name, is_global=True) 953 954 def record_name_access(self, name, is_global=False): 955 956 """ 957 Record an access involving 'name' if the name is being tracked, using 958 'is_global' to indicate whether the name is global. 959 """ 960 961 name = self.get_name_for_tracking(name, is_global=is_global) 962 branches = self.trackers[-1].tracking_name(name) 963 if branches: 964 self.record_branches_for_access(branches, name, None) 965 return self.record_access_details(name, None, self.in_assignment, 966 self.in_invocation) 967 return None 968 969 def process_operator_chain(self, nodes): 970 971 """ 972 Process the given chain of 'nodes', processing each node or item. 973 Each node starts a new conditional region, effectively making a deeply- 974 nested collection of if-like statements. 975 """ 976 977 results = [] 978 979 tracker = self.trackers[-1] 980 981 for item in nodes: 982 tracker.new_branchpoint() 983 tracker.new_branch() 984 result = self.process_structure_node(item) 985 if result: 986 results.append(result) 987 988 for item in nodes[:-1]: 989 tracker.shelve_branch() 990 tracker.new_branch() 991 tracker.shelve_branch() 992 tracker.merge_branches() 993 994 tracker.shelve_branch() 995 tracker.merge_branches() 996 997 return MultipleRef(results) 998 999 def process_try_node(self, n): 1000 1001 """ 1002 Process the given "try...except" node 'n'. 1003 """ 1004 1005 self.record_exception_handler() 1006 1007 tracker = self.trackers[-1] 1008 tracker.new_branchpoint() 1009 1010 self.process_structure_node(n.body) 1011 1012 for name, var, handler in n.handlers: 1013 if name is not None: 1014 self.process_structure_node(name) 1015 1016 # Any abandoned branches from the body can now be resumed in a new 1017 # branch. 1018 1019 tracker.resume_abandoned_branches() 1020 1021 # Establish the local for the handler. 1022 1023 if var is not None: 1024 self.process_assignment_node(var, None) 1025 if handler is not None: 1026 self.process_structure_node(handler) 1027 1028 tracker.shelve_branch() 1029 1030 # The else clause maintains the usage from the body but without the 1031 # abandoned branches since they would never lead to the else clause 1032 # being executed. 1033 1034 if n.else_: 1035 tracker.new_branch() 1036 self.process_structure_node(n.else_) 1037 tracker.shelve_branch() 1038 1039 # Without an else clause, a null branch propagates the successful 1040 # outcome. 1041 1042 else: 1043 tracker.new_branch() 1044 tracker.shelve_branch() 1045 1046 tracker.merge_branches() 1047 1048 def process_try_finally_node(self, n): 1049 1050 """ 1051 Process the given "try...finally" node 'n'. 1052 """ 1053 1054 self.record_exception_handler() 1055 1056 tracker = self.trackers[-1] 1057 self.process_structure_node(n.body) 1058 1059 # Any abandoned branches from the body can now be resumed. 1060 1061 branches = tracker.resume_all_abandoned_branches() 1062 self.process_structure_node(n.final) 1063 1064 # At the end of the finally clause, abandoned branches are discarded. 1065 1066 tracker.restore_active_branches(branches) 1067 1068 def process_while_node(self, n): 1069 1070 "Process the given while node 'n'." 1071 1072 tracker = self.trackers[-1] 1073 tracker.new_branchpoint(loop_node=True) 1074 1075 # Evaluate any test or iterator outside the loop. 1076 1077 self.process_structure_node(n.test) 1078 1079 # Propagate attribute usage to branches. 1080 1081 tracker.new_branch(loop_node=True) 1082 1083 # Enter the loop. 1084 1085 in_conditional = self.in_conditional 1086 self.in_conditional = True 1087 self.process_structure_node(n.body) 1088 self.in_conditional = in_conditional 1089 1090 # Continuing branches are resumed before any test. 1091 1092 tracker.resume_continuing_branches() 1093 1094 # Evaluate any continuation test within the body. 1095 1096 self.process_structure_node(n.test) 1097 1098 tracker.shelve_branch(loop_node=True) 1099 1100 # Support the non-looping condition. 1101 1102 tracker.new_branch() 1103 tracker.shelve_branch() 1104 1105 tracker.merge_branches() 1106 1107 # Evaluate any else clause outside branches. 1108 1109 if n.else_: 1110 self.process_structure_node(n.else_) 1111 1112 # Connect broken branches to the code after any loop. 1113 1114 tracker.resume_broken_branches() 1115 1116 # Branch tracking methods. 1117 1118 def start_tracking(self, names): 1119 1120 """ 1121 Start tracking attribute usage for names in the current namespace, 1122 immediately registering the given 'names'. 1123 """ 1124 1125 path = self.get_namespace_path() 1126 parent = self.trackers[-1] 1127 tracker = BranchTracker() 1128 self.trackers.append(tracker) 1129 1130 # Record the given names established as new branches. 1131 1132 tracker.assign_names(names) 1133 1134 def assign_name(self, name, name_ref): 1135 1136 "Assign to 'name' the given 'name_ref' in the current namespace." 1137 1138 name = self.get_name_for_tracking(name) 1139 self.trackers[-1].assign_names([name], [name_ref]) 1140 1141 def stop_tracking(self): 1142 1143 """ 1144 Stop tracking attribute usage, recording computed usage for the current 1145 namespace. Indicate whether a value is always returned from the 1146 namespace. 1147 """ 1148 1149 path = self.get_namespace_path() 1150 tracker = self.trackers.pop() 1151 self.record_assignments_for_access(tracker) 1152 1153 self.attr_usage[path] = tracker.get_all_usage() 1154 self.name_initialisers[path] = tracker.get_all_values() 1155 1156 return tracker.returns_value() 1157 1158 def start_tracking_in_module(self): 1159 1160 "Start tracking attribute usage in the module." 1161 1162 tracker = BranchTracker() 1163 self.trackers.append(tracker) 1164 1165 def stop_tracking_in_module(self): 1166 1167 "Stop tracking attribute usage in the module." 1168 1169 tracker = self.trackers[0] 1170 self.record_assignments_for_access(tracker) 1171 self.attr_usage[self.name] = tracker.get_all_usage() 1172 self.name_initialisers[self.name] = tracker.get_all_values() 1173 1174 def record_assignments_for_access(self, tracker): 1175 1176 """ 1177 For the current path, use the given 'tracker' to record assignment 1178 version information for attribute accesses. 1179 """ 1180 1181 path = self.get_path_for_access() 1182 1183 if not self.attr_accessor_branches.has_key(path): 1184 return 1185 1186 init_item(self.attr_accessors, path, dict) 1187 attr_accessors = self.attr_accessors[path] 1188 1189 # Obtain the branches applying during each access. 1190 1191 for access, all_branches in self.attr_accessor_branches[path].items(): 1192 name, attrnames = access 1193 init_item(attr_accessors, access, list) 1194 1195 # Obtain the assignments applying to each branch. 1196 1197 for branches in all_branches: 1198 positions = tracker.get_assignment_positions_for_branches(name, branches) 1199 1200 # Detect missing name information. 1201 1202 if None in positions: 1203 globals = self.global_attr_accesses.get(path) 1204 accesses = globals and globals.get(name) 1205 if not accesses: 1206 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1207 self.get_namespace_path(), name) 1208 positions.remove(None) 1209 1210 attr_accessors[access].append(positions) 1211 1212 def record_branches_for_access(self, branches, name, attrnames): 1213 1214 """ 1215 Record the given 'branches' for an access involving the given 'name' and 1216 'attrnames'. 1217 """ 1218 1219 access = name, attrnames 1220 path = self.get_path_for_access() 1221 1222 init_item(self.attr_accessor_branches, path, dict) 1223 attr_accessor_branches = self.attr_accessor_branches[path] 1224 1225 init_item(attr_accessor_branches, access, list) 1226 attr_accessor_branches[access].append(branches) 1227 1228 def record_access_details(self, name, attrnames, assignment, invocation): 1229 1230 """ 1231 For the given 'name' and 'attrnames', record an access indicating 1232 whether an 'assignment' or an 'invocation' is occurring. 1233 1234 These details correspond to accesses otherwise recorded by the attribute 1235 accessor and attribute access dictionaries. 1236 """ 1237 1238 access = name, attrnames 1239 path = self.get_path_for_access() 1240 1241 init_item(self.attr_access_modifiers, path, dict) 1242 init_item(self.attr_access_modifiers[path], access, list) 1243 1244 access_number = len(self.attr_access_modifiers[path][access]) 1245 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1246 return access_number 1247 1248 def record_global_access_details(self, name, attrnames): 1249 1250 """ 1251 Record details of a global access via the given 'name' involving the 1252 indicated 'attrnames'. 1253 """ 1254 1255 path = self.get_namespace_path() 1256 1257 init_item(self.global_attr_accesses, path, dict) 1258 init_item(self.global_attr_accesses[path], name, set) 1259 self.global_attr_accesses[path][name].add(attrnames) 1260 1261 # Namespace modification. 1262 1263 def record_name(self, name): 1264 1265 "Record the use of 'name' in a namespace." 1266 1267 path = self.get_namespace_path() 1268 init_item(self.names_used, path, set) 1269 self.names_used[path].add(name) 1270 1271 def set_module(self, name, module_name): 1272 1273 """ 1274 Set a module in the current namespace using the given 'name' associated 1275 with the corresponding 'module_name'. 1276 """ 1277 1278 if name: 1279 self.set_general_local(name, Reference("<module>", module_name)) 1280 1281 def set_definition(self, name, kind): 1282 1283 """ 1284 Set the definition having the given 'name' and 'kind'. 1285 1286 Definitions are set in the static namespace hierarchy, but they can also 1287 be recorded for function locals. 1288 """ 1289 1290 if self.is_global(name): 1291 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1292 self.get_namespace_path(), name) 1293 1294 path = self.get_object_path(name) 1295 self.set_object(path, kind) 1296 1297 ref = self.get_object(path) 1298 if ref.get_kind() == "<var>": 1299 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1300 self.get_namespace_path(), name) 1301 1302 if not self.is_global(name) and self.in_function: 1303 self.set_function_local(name, ref) 1304 1305 def set_function_local(self, name, ref=None): 1306 1307 "Set the local with the given 'name' and optional 'ref'." 1308 1309 path = self.get_namespace_path() 1310 locals = self.function_locals[path] 1311 used = self.names_used.get(path) 1312 1313 if not locals.has_key(name) and used and name in used: 1314 raise InspectError("Name %s assigned locally but used previously." % name, path) 1315 1316 multiple = not ref or locals.has_key(name) and locals[name] != ref 1317 locals[name] = multiple and Reference("<var>") or ref 1318 1319 def assign_general_local(self, name, name_ref): 1320 1321 """ 1322 Set for 'name' the given 'name_ref', recording the name for attribute 1323 usage tracking. 1324 """ 1325 1326 self.set_general_local(name, name_ref) 1327 self.assign_name(name, name_ref) 1328 1329 def set_general_local(self, name, value=None): 1330 1331 """ 1332 Set the 'name' with optional 'value' in any kind of local namespace, 1333 where the 'value' should be a reference if specified. 1334 """ 1335 1336 init_value = self.get_initialising_value(value) 1337 1338 # Module global names. 1339 1340 if self.is_global(name): 1341 path = self.get_global_path(name) 1342 self.set_object(path, init_value) 1343 1344 # Function local names. 1345 1346 elif self.in_function: 1347 self.set_function_local(name, init_value) 1348 1349 # Other namespaces (classes). 1350 1351 else: 1352 self.set_name(name, init_value) 1353 1354 def set_name(self, name, ref=None): 1355 1356 "Attach the 'name' with optional 'ref' to the current namespace." 1357 1358 self.set_object(self.get_object_path(name), ref) 1359 1360 def set_instance_attr(self, name, ref=None): 1361 1362 """ 1363 Add an instance attribute of the given 'name' to the current class, 1364 using the optional 'ref'. 1365 """ 1366 1367 self._set_instance_attr(self.in_class, name, ref) 1368 1369 def _set_instance_attr(self, path, name, ref=None): 1370 1371 init_item(self.instance_attrs, path, set) 1372 self.instance_attrs[path].add(name) 1373 1374 if ref: 1375 init_item(self.instance_attr_constants, path, dict) 1376 self.instance_attr_constants[path][name] = ref 1377 1378 def get_initialising_value(self, value): 1379 1380 "Return a suitable initialiser reference for 'value'." 1381 1382 if isinstance(value, Result): 1383 return value.reference() 1384 else: 1385 return value 1386 1387 # Static, program-relative naming. 1388 1389 def find_name(self, name): 1390 1391 """ 1392 Return the qualified name for the given 'name' used in the current 1393 non-function namespace. 1394 """ 1395 1396 path = self.get_namespace_path() 1397 ref = None 1398 1399 if not self.in_function and name not in predefined_constants: 1400 if self.in_class: 1401 ref = self.get_object(self.get_object_path(name), False) 1402 if not ref: 1403 ref = self.get_global_or_builtin(name) 1404 1405 return ref 1406 1407 def get_class(self, node): 1408 1409 """ 1410 Use the given 'node' to obtain the identity of a class. Return a 1411 reference for the class. Unresolved dependencies are permitted and must 1412 be resolved later. 1413 """ 1414 1415 ref = self._get_class(node) 1416 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1417 1418 def _get_class(self, node): 1419 1420 """ 1421 Use the given 'node' to find a class definition. Return a reference to 1422 the class. 1423 """ 1424 1425 if isinstance(node, compiler.ast.Getattr): 1426 1427 # Obtain the identity of the access target. 1428 1429 ref = self._get_class(node.expr) 1430 1431 # Where the target is a class or module, obtain the identity of the 1432 # attribute. 1433 1434 if ref.has_kind(["<function>", "<var>"]): 1435 return None 1436 else: 1437 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1438 return self.get_object(attrname) 1439 1440 # Names can be module-level or built-in. 1441 1442 elif isinstance(node, compiler.ast.Name): 1443 1444 # Record usage of the name and attempt to identify it. 1445 1446 self.record_name(node.name) 1447 return self.find_name(node.name) 1448 else: 1449 return None 1450 1451 def get_constant(self, name, value): 1452 1453 "Return a constant reference for the given type 'name' and 'value'." 1454 1455 ref = self.get_builtin_class(name) 1456 return self.get_constant_reference(ref, value) 1457 1458 def get_literal_instance(self, n, name=None): 1459 1460 """ 1461 For node 'n', return a reference to an instance of 'name', or if 'name' 1462 is not specified, deduce the type from the value. 1463 """ 1464 1465 # Handle stray None constants (Sliceobj seems to produce them). 1466 1467 if name == "NoneType": 1468 return self.process_name_node(compiler.ast.Name("None")) 1469 1470 # Obtain the details of the literal itself. 1471 # An alias to the type is generated for sequences. 1472 1473 if name in ("dict", "list", "tuple"): 1474 ref = self.get_builtin_class(name) 1475 self.set_special_literal(name, ref) 1476 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1477 1478 # Constant values are independently recorded. 1479 1480 else: 1481 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1482 ref = self.get_builtin_class(typename) 1483 return self.get_constant_reference(ref, value, encoding) 1484 1485 # Special names. 1486 1487 def get_special(self, name): 1488 1489 "Return any stored value for the given special 'name'." 1490 1491 value = self.special.get(name) 1492 if value: 1493 ref, paths = value 1494 else: 1495 ref = None 1496 return ref 1497 1498 def set_special(self, name, value): 1499 1500 """ 1501 Set a special 'name' that merely tracks the use of an implicit object 1502 'value'. 1503 """ 1504 1505 if not self.special.has_key(name): 1506 paths = set() 1507 self.special[name] = value, paths 1508 else: 1509 _ref, paths = self.special[name] 1510 1511 paths.add(self.get_namespace_path()) 1512 1513 def set_special_literal(self, name, ref): 1514 1515 """ 1516 Set a special name for the literal type 'name' having type 'ref'. Such 1517 special names provide a way of referring to literal object types. 1518 """ 1519 1520 literal_name = "$L%s" % name 1521 value = ResolvedNameRef(literal_name, ref) 1522 self.set_special(literal_name, value) 1523 1524 # Exceptions. 1525 1526 def record_exception_handler(self): 1527 1528 "Record the current namespace as employing an exception handler." 1529 1530 self.exception_namespaces.add(self.get_namespace_path()) 1531 1532 # Return values. 1533 1534 def record_return_value(self, expr): 1535 1536 "Record the given return 'expr'." 1537 1538 path = self.get_namespace_path() 1539 l = init_item(self.return_values, path, list) 1540 l.append(expr) 1541 if not self.importer.all_return_values.has_key(path): 1542 self.importer.all_return_values[path] = l 1543 1544 # vim: tabstop=4 expandtab shiftwidth=4