1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 if delete_all or not self.importer.uses_name(name): 202 del obj[name] 203 if attr.assignments == 1: 204 value = attr.get_value() 205 206 # Delete any unambiguous attribute value. 207 208 if value is not obj and value in self.all_objects: 209 self.all_objects.remove(value) 210 211 # Delete class contents. 212 213 if isinstance(value, Class): 214 self.vacuum_object(value, 1) 215 216 def finalise(self): 217 218 "Finalise the module." 219 220 for obj in self.all_objects: 221 if isinstance(obj, (Class, Function)): 222 obj.finalise_attributes() 223 224 def add_object(self, obj, any_scope=0): 225 226 """ 227 Record 'obj' if non-local or if the optional 'any_scope' is set to a 228 true value. 229 """ 230 231 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 232 self.all_objects.add(obj) 233 234 # Optimisation tests. 235 236 def should_optimise_unused_objects(self): 237 return "unused_objects" in self.optimisations 238 239 # Namespace methods. 240 241 def store(self, name, obj): 242 243 "Record attribute or local 'name', storing 'obj'." 244 245 if not self.namespaces: 246 self.set(name, obj, not self.in_loop) 247 else: 248 self.namespaces[-1].set(name, obj, not self.in_loop) 249 250 def store_lambda(self, obj): 251 252 "Store a lambda function 'obj'." 253 254 self.add_object(obj) 255 256 def store_module_attr(self, name, module): 257 258 """ 259 Record module attribute 'name' in the given 'module' using the current 260 expression. 261 """ 262 263 module.set(name, self.expr, 0) 264 265 def store_class_attr(self, name): 266 267 """ 268 Record class attribute 'name' in the current class using the current 269 expression. 270 """ 271 272 if self.in_method and self.namespaces[-2].has_key(name): 273 self.namespaces[-2].set(name, self.expr, 0) 274 return 1 275 276 return 0 277 278 def store_instance_attr(self, name): 279 280 "Record instance attribute 'name' in the current class." 281 282 if self.in_method: 283 284 # Current namespace is the function. 285 # Previous namespace is the class. 286 287 self.namespaces[-2].add_instance_attribute(name) 288 289 def get_namespace(self): 290 291 "Return the parent (or most recent) namespace currently exposed." 292 293 return (self.namespaces[-1:] or [self])[0] 294 295 def use_name(self, name): 296 297 "Use the given 'name' within the current namespace/unit." 298 299 unit = self.get_namespace() 300 self.importer.use_name(name, unit.name) 301 302 def new_branchpoint(self): 303 self.get_namespace()._new_branchpoint() 304 305 def new_branch(self): 306 self.get_namespace()._new_branch() 307 308 def shelve_branch(self): 309 self.get_namespace()._shelve_branch() 310 311 def merge_branches(self): 312 self.get_namespace()._merge_branches() 313 314 def reset_attributes(self, name): 315 self.get_namespace()._reset_attributes(name) 316 317 def reset_all_attributes(self): 318 self.get_namespace()._reset_all_attributes() 319 320 def use_attribute(self, attr, attrname): 321 return self.get_namespace()._use_attribute(attr, attrname) 322 323 # Visitor methods. 324 325 def default(self, node, *args): 326 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 327 328 def dispatch(self, node, *args): 329 return ASTVisitor.dispatch(self, node, *args) 330 331 def NOP(self, node): 332 for n in node.getChildNodes(): 333 self.dispatch(n) 334 return None 335 336 def OP(self, node): 337 for n in node.getChildNodes(): 338 self.dispatch(n) 339 return Instance() 340 341 def _visitUnary(self, node): 342 343 "Accounting method for the unary operator 'node'." 344 345 method = unary_methods[node.__class__.__name__] 346 self.use_name(method) 347 return self.OP(node) 348 349 def _visitBinary(self, node): 350 351 "Accounting method for the binary operator 'node'." 352 353 left_method, right_method = binary_methods[node.__class__.__name__] 354 self.use_name(left_method) 355 self.use_name(right_method) 356 return self.OP(node) 357 358 def _visitFunction(self, node, name): 359 360 """ 361 Return a function object for the function defined by 'node' with the 362 given 'name'. If a lambda expression is being visited, 'name' should be 363 None. 364 """ 365 366 # Define the function object. 367 368 function = Function( 369 name, 370 self.get_namespace(), 371 node.argnames, 372 node.defaults, 373 (node.flags & 4 != 0), 374 (node.flags & 8 != 0), 375 self, 376 node 377 ) 378 379 self.add_object(function, any_scope=1) 380 381 # Make a back reference from the node for code generation. 382 383 node.unit = function 384 385 # Process the defaults. 386 387 for n in node.defaults: 388 self.expr = self.dispatch(n) 389 function.store_default(self.expr) 390 391 self.functions.append((node, self.namespaces + [function])) 392 393 if name is not None: 394 self.store(name, function) 395 else: 396 self.store_lambda(function) 397 398 # Where defaults exist, an instance needs creating. Thus, it makes 399 # no sense to return a reference to the function here, since the 400 # recipient will not be referencing the function itself. 401 402 if node.defaults: 403 return Instance() # indicates no known target 404 405 return function 406 407 def _visitFunctionBody(self, node, namespaces): 408 409 "Enter the function." 410 411 # Current namespace is the function. 412 # Previous namespace is the class. 413 414 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 415 if namespaces[-1].name == "__init__": 416 self.in_init = 1 417 self.in_method = 1 418 419 self.namespaces = namespaces 420 self.dispatch(node.code) 421 422 self.in_init = 0 423 self.in_method = 0 424 425 # Specific handler methods. 426 427 visitAdd = _visitBinary 428 429 visitAnd = OP 430 431 visitAssert = NOP 432 433 def visitAssign(self, node): 434 self.expr = self.dispatch(node.expr) 435 for n in node.nodes: 436 self.dispatch(n) 437 return None 438 439 def visitAssAttr(self, node): 440 expr = self.dispatch(node.expr) 441 442 # Record the attribute on the presumed target. 443 444 if isinstance(expr, Attr): 445 if expr.name == "self": 446 if not self.store_class_attr(node.attrname): 447 self.store_instance_attr(node.attrname) 448 elif isinstance(expr.get_value(), Module): 449 self.store_module_attr(node.attrname, expr.get_value()) 450 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 451 452 # Note usage of the attribute. 453 454 if expr.parent is self.get_namespace(): 455 node._attrnames = self.use_attribute(expr, node.attrname) 456 457 return None 458 459 def visitAssList(self, node): 460 461 # Declare names which will be used by generated code. 462 463 self.use_name("__getitem__") 464 465 # Process the assignment. 466 467 for i, n in enumerate(node.nodes): 468 self.dispatch(n) 469 self.importer.make_constant(i) # for __getitem__(i) at run-time 470 return None 471 472 def visitAssName(self, node): 473 if node.flags == "OP_DELETE": 474 raise InspectError(self.full_name(), node, "Deletion of attribute %r is not supported." % node.name) 475 476 self.store(node.name, self.expr) 477 self.reset_attributes(node.name) 478 self.use_name(node.name) 479 return None 480 481 visitAssTuple = visitAssList 482 483 def visitAugAssign(self, node): 484 485 # Accounting. 486 487 aug_method, (left_method, right_method) = augassign_methods[node.op] 488 self.use_name(aug_method) 489 self.use_name(left_method) 490 self.use_name(right_method) 491 492 # Process the assignment. 493 494 self.expr = self.dispatch(node.expr) 495 496 # NOTE: Similar to micropython.ast handler code. 497 # NOTE: Slices and subscripts not supported. 498 499 if isinstance(node.node, compiler.ast.Name): 500 self.visitAssName(node.node) 501 elif isinstance(node.node, compiler.ast.Getattr): 502 self.visitAssAttr(node.node) 503 else: 504 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 505 506 return None 507 508 visitBackquote = OP 509 510 visitBitand = _visitBinary 511 512 visitBitor = _visitBinary 513 514 visitBitxor = _visitBinary 515 516 def visitBreak(self, node): 517 self.reset_all_attributes() 518 519 visitCallFunc = OP 520 521 def visitClass(self, node): 522 523 """ 524 Register the class at the given 'node' subject to the restrictions 525 mentioned in the module docstring. 526 """ 527 528 if self.namespaces: 529 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 530 return None 531 else: 532 cls = Class(node.name, self.get_namespace(), self, node) 533 534 # Visit the base class expressions, attempting to find concrete 535 # definitions of classes. 536 537 for base in node.bases: 538 expr = self.dispatch(base) 539 if isinstance(expr, Attr): 540 if expr.assignments != 1: 541 raise InspectError(self.full_name(), node, 542 "Base class %r for %r is not constant." % (base, cls.full_name())) 543 else: 544 cls.add_base(expr.get_value()) 545 else: # if expr is None: 546 raise InspectError(self.full_name(), node, 547 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 548 549 # NOTE: Potentially dubious measure to permit __init__ availability. 550 # If no bases exist, adopt the 'object' class. 551 552 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 553 expr = self.dispatch(compiler.ast.Name("object")) 554 cls.add_base(expr.get_value()) 555 556 # Make a back reference from the node for code generation. 557 558 node.unit = cls 559 560 # Make an entry for the class. 561 562 self.store(node.name, cls) 563 self.add_object(cls) 564 565 # Process the class body. 566 567 self.namespaces.append(cls) 568 self.dispatch(node.code) 569 self.namespaces.pop() 570 571 return cls 572 573 def visitCompare(self, node): 574 575 # Accounting. 576 # NOTE: Replicates some code in micropython.ast.visitCompare. 577 578 for op in node.ops: 579 op_name, next_node = op 580 methods = comparison_methods[op_name] 581 if methods is not None: 582 self.use_name(methods[0]) 583 self.use_name(methods[1]) 584 elif op_name.endswith("in"): 585 self.use_name("__contains__") 586 587 return self.OP(node) 588 589 def visitConst(self, node): 590 591 # Register the constant, if necessary, returning the resulting object. 592 593 self.use_name(self.importer.get_constant_type_name(node.value)) 594 return self.importer.make_constant(node.value) 595 596 def visitContinue(self, node): 597 self.reset_all_attributes() 598 599 visitDecorators = NOP 600 601 visitDict = OP 602 603 visitDiscard = NOP 604 605 visitDiv = _visitBinary 606 607 visitEllipsis = NOP 608 609 visitExec = NOP 610 611 visitExpression = OP 612 613 visitFloorDiv = _visitBinary 614 615 def visitFor(self, node): 616 self.new_branchpoint() 617 618 # Declare names which will be used by generated code. 619 620 self.use_name("__iter__") 621 self.use_name("next") 622 623 self.dispatch(node.assign) 624 self.dispatch(node.list) 625 626 # Enter the loop. 627 # Propagate attribute usage to branches. 628 629 self.in_loop = 1 630 self.new_branch() 631 self.dispatch(node.body) 632 self.shelve_branch() 633 self.in_loop = 0 634 635 # Maintain a branch for the else clause or the current retained usage 636 # where execution avoids the conditional clauses. 637 638 self.new_branch() 639 if node.else_ is not None: 640 self.dispatch(node.else_) 641 self.shelve_branch() 642 643 self.merge_branches() 644 return None 645 646 def visitFrom(self, node): 647 module = self.importer.load(node.modname, 1) 648 649 #if module is None: 650 # print "Warning:", node.modname, "not imported." 651 652 for name, alias in node.names: 653 if name != "*": 654 if module is not None and module.namespace.has_key(name): 655 attr = module[name] 656 self.store(alias or name, attr) 657 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 658 self.importer.load(attr.get_value().name) 659 660 # Support the import of names from missing modules. 661 662 else: 663 self.store(alias or name, UnresolvedName(name, node.modname, self)) 664 else: 665 if module is not None: 666 for n in module.namespace.keys(): 667 attr = module[n] 668 self.store(n, attr) 669 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 670 self.importer.load(attr.get_value().name) 671 672 return None 673 674 def visitFunction(self, node): 675 return self._visitFunction(node, node.name) 676 677 visitGenExpr = OP 678 679 visitGenExprFor = NOP 680 681 visitGenExprIf = NOP 682 683 visitGenExprInner = NOP 684 685 def visitGetattr(self, node): 686 expr = self.dispatch(node.expr) 687 attrname = node.attrname 688 689 # Attempt to identify the nature of the attribute. 690 691 if isinstance(expr, Attr): 692 value = expr.get_value() 693 if isinstance(value, (Class, Module)): 694 attr = value.namespace.get(attrname) 695 elif isinstance(value, UnresolvedName): 696 attr = UnresolvedName(attrname, value.full_name(), self) 697 else: 698 attr = None 699 700 # Note usage of the attribute. 701 702 if expr.parent is self.get_namespace(): 703 node._attrnames = self.use_attribute(expr, attrname) 704 705 elif self.builtins is not None: 706 attr = self.builtins.get(attrname) 707 else: 708 attr = UnresolvedName(attrname, value.full_name(), self) 709 710 # Accounting. 711 712 self.use_name(attrname) 713 714 return attr 715 716 def visitGlobal(self, node): 717 if self.namespaces: 718 for name in node.names: 719 ns = self.namespaces[-1] 720 if not ns.make_global(name): 721 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 722 723 # The name is recorded in an earlier process. 724 725 def visitIf(self, node): 726 self.new_branchpoint() 727 728 # Propagate attribute usage to branches. 729 730 for test, body in node.tests: 731 self.dispatch(test) 732 733 self.new_branch() 734 self.dispatch(body) 735 self.shelve_branch() 736 737 # Maintain a branch for the else clause or the current retained usage 738 # where execution avoids the conditional clauses. 739 740 self.new_branch() 741 if node.else_ is not None: 742 self.dispatch(node.else_) 743 self.shelve_branch() 744 745 self.merge_branches() 746 return None 747 748 visitIfExp = NOP 749 750 def visitImport(self, node): 751 for name, alias in node.names: 752 if alias is not None: 753 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 754 self.store(alias, module) 755 else: 756 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 757 self.store(name.split(".")[0], module) 758 759 return None 760 761 visitInvert = _visitUnary 762 763 def visitKeyword(self, node): 764 self.dispatch(node.expr) 765 self.importer.make_constant(node.name) 766 self.keyword_names.add(node.name) 767 return None 768 769 def visitLambda(self, node): 770 self.use_name(None) # lambda functions have no names but are assumed to be invoked 771 return self._visitFunction(node, None) 772 773 visitLeftShift = _visitBinary 774 775 visitList = OP 776 777 visitListComp = OP 778 779 visitListCompFor = NOP 780 781 visitListCompIf = NOP 782 783 visitMod = _visitBinary 784 785 def visitModule(self, node): 786 787 # Make a back reference from the node for code generation. 788 789 node.unit = self 790 return self.dispatch(node.node) 791 792 visitMul = _visitBinary 793 794 def visitName(self, node): 795 name = node.name 796 797 if self.importer.predefined_constants.has_key(name): 798 attr = self.importer.get_predefined_constant(name) 799 elif self.namespaces and self.namespaces[-1].has_key(name): 800 attr = self.namespaces[-1][name] 801 elif self.has_key(name): 802 attr = self[name] 803 elif self.builtins is not None and self.builtins.has_key(name): 804 attr = self.builtins[name] 805 else: 806 attr = None 807 808 self.use_name(name) 809 return attr 810 811 visitNot = OP 812 813 visitOr = OP 814 815 visitPass = NOP 816 817 visitPower = _visitBinary 818 819 visitPrint = NOP 820 821 visitPrintnl = NOP 822 823 visitRaise = NOP 824 825 visitReturn = NOP 826 827 visitRightShift = _visitBinary 828 829 visitSlice = OP 830 831 visitSliceobj = OP 832 833 def visitStmt(self, node): 834 for n in node.nodes: 835 self.dispatch(n) 836 return None 837 838 visitSub = _visitBinary 839 840 def visitSubscript(self, node): 841 self.use_name("__getitem__") 842 self.OP(node) 843 844 def visitTryExcept(self, node): 845 self.dispatch(node.body) 846 847 self.new_branchpoint() 848 849 for name, var, n in node.handlers: 850 self.new_branch() 851 852 # Establish the local for the handler. 853 854 if var is not None: 855 self.dispatch(var) 856 if n is not None: 857 self.dispatch(n) 858 859 self.shelve_branch() 860 861 if node.else_ is not None: 862 self.new_branch() 863 self.dispatch(node.else_) 864 self.shelve_branch() 865 866 self.merge_branches() 867 return None 868 869 visitTryFinally = NOP 870 871 visitTuple = OP 872 873 visitUnaryAdd = _visitUnary 874 875 visitUnarySub = _visitUnary 876 877 def visitWhile(self, node): 878 self.new_branchpoint() 879 880 # Propagate attribute usage to branches. 881 882 self.in_loop = 1 883 self.dispatch(node.test) 884 self.new_branch() 885 self.dispatch(node.body) 886 self.shelve_branch() 887 self.in_loop = 0 888 889 # Maintain a branch for the else clause or the current retained usage 890 # where execution avoids the conditional clauses. 891 892 self.new_branch() 893 if node.else_ is not None: 894 self.dispatch(node.else_) 895 self.shelve_branch() 896 897 self.merge_branches() 898 return None 899 900 visitWith = NOP 901 902 visitYield = NOP 903 904 # vim: tabstop=4 expandtab shiftwidth=4