1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 118 def parse(self, filename): 119 120 "Parse the file having the given 'filename'." 121 122 module = compiler.parseFile(filename) 123 self.process(module) 124 125 def process(self, module): 126 127 "Process the given 'module'." 128 129 self.astnode = self.module = module 130 processed = self.dispatch(module) 131 if self.has_key("__all__"): 132 all = self["__all__"] 133 if isinstance(all, compiler.ast.List): 134 for n in all.nodes: 135 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 136 return processed 137 138 def vacuum(self): 139 140 """ 141 Vacuum the module namespace, removing unreferenced objects and unused 142 names. 143 """ 144 145 for name, value in self.items(): 146 147 if self.should_optimise_unused_objects(): 148 149 # Remove entries for unreferenced objects. 150 # This, due to the nature of the referenced attribute, assumes 151 # that only explicitly mentioned classes and functions are 152 # employed in the final program. 153 154 if isinstance(value, Attr): 155 156 # Only remove entries for classes and functions, not methods. 157 158 for attr_value in value.get_values(): 159 if (isinstance(attr_value, Function) and not attr_value.is_method() or 160 isinstance(attr_value, Class)) and not attr_value.referenced: 161 pass 162 else: 163 break 164 else: 165 del self[name] 166 167 # Complain about globals not initialised at the module level. 168 169 if isinstance(value, Global): 170 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 171 172 # Remove unreferenced objects. 173 174 if self.should_optimise_unused_objects(): 175 176 all_objects = list(self.all_objects) 177 178 for obj in all_objects: 179 180 # Only remove entries for classes and functions, not methods. 181 182 if (isinstance(obj, Function) and not obj.is_method() or 183 isinstance(obj, Class)) and not obj.referenced: 184 185 self.all_objects.remove(obj) 186 187 # Remove unused entries from classes plus associated methods. 188 189 if isinstance(obj, Class): 190 for name, attr in obj.class_attributes().items(): 191 192 # Methods can only be deleted if they are the only 193 # assigned object to the class and are unreferenced. 194 195 if name not in self.importer.names_used and \ 196 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 197 attr.get_value().is_method() and not attr.get_value().referenced: 198 199 self.all_objects.remove(attr.get_value()) 200 del obj[name] 201 202 def finalise(self): 203 204 "Finalise the module." 205 206 for obj in self.all_objects: 207 if isinstance(obj, (Class, Function)): 208 obj.finalise_attributes() 209 210 def add_object(self, obj, any_scope=0): 211 212 """ 213 Record 'obj' if non-local or if the optional 'any_scope' is set to a 214 true value. 215 """ 216 217 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 218 self.all_objects.add(obj) 219 220 # Optimisation tests. 221 222 def should_optimise_unused_objects(self): 223 return "unused_objects" in self.optimisations 224 225 # Namespace methods. 226 227 def store(self, name, obj): 228 229 "Record attribute or local 'name', storing 'obj'." 230 231 if not self.namespaces: 232 self.set(name, obj, not self.in_loop) 233 else: 234 self.namespaces[-1].set(name, obj, not self.in_loop) 235 236 def store_lambda(self, obj): 237 238 "Store a lambda function 'obj'." 239 240 self.add_object(obj) 241 242 def store_module_attr(self, name, module): 243 244 """ 245 Record module attribute 'name' in the given 'module' using the current 246 expression. 247 """ 248 249 module.set(name, self.expr, 0) 250 251 def store_class_attr(self, name): 252 253 """ 254 Record class attribute 'name' in the current class using the current 255 expression. 256 """ 257 258 if self.in_method and self.namespaces[-2].has_key(name): 259 self.namespaces[-2].set(name, self.expr, 0) 260 return 1 261 262 return 0 263 264 def store_instance_attr(self, name): 265 266 "Record instance attribute 'name' in the current class." 267 268 if self.in_method: 269 270 # Current namespace is the function. 271 # Previous namespace is the class. 272 273 self.namespaces[-2].add_instance_attribute(name) 274 275 def get_parent(self): 276 277 "Return the parent (or most recent) namespace currently exposed." 278 279 return (self.namespaces[-1:] or [self])[0] 280 281 # Visitor methods. 282 283 def default(self, node, *args): 284 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 285 286 def dispatch(self, node, *args): 287 return ASTVisitor.dispatch(self, node, *args) 288 289 def NOP(self, node): 290 for n in node.getChildNodes(): 291 self.dispatch(n) 292 return None 293 294 def OP(self, node): 295 for n in node.getChildNodes(): 296 self.dispatch(n) 297 return Instance() 298 299 def _visitUnary(self, node): 300 301 "Accounting method for the unary operator 'node'." 302 303 method = unary_methods[node.__class__.__name__] 304 self.importer.use_name(method) 305 return self.OP(node) 306 307 def _visitBinary(self, node): 308 309 "Accounting method for the binary operator 'node'." 310 311 left_method, right_method = binary_methods[node.__class__.__name__] 312 self.importer.use_name(left_method) 313 self.importer.use_name(right_method) 314 return self.OP(node) 315 316 def _visitFunction(self, node, name): 317 318 """ 319 Return a function object for the function defined by 'node' with the 320 given 'name'. If a lambda expression is being visited, 'name' should be 321 None. 322 """ 323 324 # Define the function object. 325 326 function = Function( 327 name, 328 self.get_parent(), 329 node.argnames, 330 node.defaults, 331 (node.flags & 4 != 0), 332 (node.flags & 8 != 0), 333 self, 334 node 335 ) 336 337 self.add_object(function, any_scope=1) 338 339 # Make a back reference from the node for code generation. 340 341 node.unit = function 342 343 # Process the defaults. 344 345 for n in node.defaults: 346 self.expr = self.dispatch(n) 347 function.store_default(self.expr) 348 349 # Enter the function. 350 351 self.namespaces.append(function) 352 353 # Current namespace is the function. 354 # Previous namespace is the class. 355 356 if len(self.namespaces) > 1 and isinstance(self.namespaces[-2], Class): 357 if name == "__init__": 358 self.in_init = 1 359 self.in_method = 1 360 361 self.dispatch(node.code) 362 self.in_init = 0 363 self.in_method = 0 364 self.namespaces.pop() 365 366 if name is not None: 367 self.store(name, function) 368 else: 369 self.store_lambda(function) 370 371 # Lambda functions are always assumed to be referenced. This is 372 # because other means of discovering the referencing of objects rely 373 # on the original names inherent in the definition of those objects. 374 375 function.set_referenced() 376 377 # Where defaults exist, an instance needs creating. Thus, it makes 378 # no sense to return a reference to the function here, since the 379 # recipient will not be referencing the function itself. 380 381 if node.defaults: 382 return Instance() # indicates no known target 383 384 return function 385 386 # Specific handler methods. 387 388 visitAdd = _visitBinary 389 390 visitAnd = OP 391 392 visitAssert = NOP 393 394 def visitAssign(self, node): 395 self.expr = self.dispatch(node.expr) 396 for n in node.nodes: 397 self.dispatch(n) 398 return None 399 400 def visitAssAttr(self, node): 401 expr = self.dispatch(node.expr) 402 if isinstance(expr, Attr): 403 if expr.name == "self": 404 if not self.store_class_attr(node.attrname): 405 self.store_instance_attr(node.attrname) 406 elif isinstance(expr.get_value(), Module): 407 self.store_module_attr(node.attrname, expr.get_value()) 408 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 409 return None 410 411 def visitAssList(self, node): 412 413 # Declare names which will be used by generated code. 414 415 self.importer.use_name("__getitem__") 416 417 # Process the assignment. 418 419 for i, n in enumerate(node.nodes): 420 self.dispatch(n) 421 self.importer.make_constant(i) # for __getitem__(i) at run-time 422 return None 423 424 def visitAssName(self, node): 425 self.store(node.name, self.expr) 426 return None 427 428 visitAssTuple = visitAssList 429 430 def visitAugAssign(self, node): 431 432 # Accounting. 433 434 aug_method, (left_method, right_method) = augassign_methods[node.op] 435 self.importer.use_name(aug_method) 436 self.importer.use_name(left_method) 437 self.importer.use_name(right_method) 438 return self.OP(node) 439 440 visitBackquote = OP 441 442 visitBitand = _visitBinary 443 444 visitBitor = _visitBinary 445 446 visitBitxor = _visitBinary 447 448 visitBreak = NOP 449 450 visitCallFunc = OP 451 452 def visitClass(self, node): 453 454 """ 455 Register the class at the given 'node' subject to the restrictions 456 mentioned in the module docstring. 457 """ 458 459 if self.namespaces: 460 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 461 return None 462 else: 463 cls = Class(node.name, self.get_parent(), self, node) 464 465 # Visit the base class expressions, attempting to find concrete 466 # definitions of classes. 467 468 for base in node.bases: 469 expr = self.dispatch(base) 470 if isinstance(expr, Attr): 471 if expr.assignments != 1: 472 raise InspectError(self.full_name(), node, 473 "Base class %r for %r is not constant." % (base, cls.full_name())) 474 else: 475 cls.add_base(expr.get_value()) 476 else: # if expr is None: 477 raise InspectError(self.full_name(), node, 478 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 479 480 # NOTE: Potentially dubious measure to permit __init__ availability. 481 # If no bases exist, adopt the 'object' class. 482 483 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 484 expr = self.dispatch(compiler.ast.Name("object")) 485 cls.add_base(expr.get_value()) 486 487 # Make a back reference from the node for code generation. 488 489 node.unit = cls 490 491 # Make an entry for the class. 492 493 self.store(node.name, cls) 494 self.add_object(cls) 495 496 # Process the class body. 497 498 self.namespaces.append(cls) 499 self.dispatch(node.code) 500 self.namespaces.pop() 501 502 return cls 503 504 def visitCompare(self, node): 505 506 # Accounting. 507 # NOTE: Replicates some code in micropython.ast.visitCompare. 508 509 for op in node.ops: 510 op_name, next_node = op 511 methods = comparison_methods[op_name] 512 if methods is not None: 513 self.importer.use_name(methods[0]) 514 self.importer.use_name(methods[1]) 515 elif op_name.endswith("in"): 516 self.importer.use_name("__contains__") 517 518 return self.OP(node) 519 520 def visitConst(self, node): 521 522 # Register the constant, if necessary, returning the resulting object. 523 524 return self.importer.make_constant(node.value) 525 526 visitContinue = NOP 527 528 visitDecorators = NOP 529 530 visitDict = OP 531 532 visitDiscard = NOP 533 534 visitDiv = _visitBinary 535 536 visitEllipsis = NOP 537 538 visitExec = NOP 539 540 visitExpression = OP 541 542 visitFloorDiv = _visitBinary 543 544 def visitFor(self, node): 545 546 # Declare names which will be used by generated code. 547 548 self.importer.use_name("__iter__") 549 self.importer.use_name("next") 550 551 # Enter the loop. 552 553 self.in_loop = 1 554 self.NOP(node) 555 self.in_loop = 0 556 return None 557 558 def visitFrom(self, node): 559 module = self.importer.load(node.modname, 1) 560 561 if module is not None: 562 module.set_referenced() 563 564 #if module is None: 565 # print "Warning:", node.modname, "not imported." 566 567 for name, alias in node.names: 568 if name != "*": 569 if module is not None and module.namespace.has_key(name): 570 attr = module[name] 571 self.store(alias or name, attr) 572 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 573 self.importer.load(attr.get_value().name) 574 575 # Support the import of names from missing modules. 576 577 else: 578 self.store(alias or name, UnresolvedName(name, node.modname, self)) 579 else: 580 if module is not None: 581 for n in module.namespace.keys(): 582 attr = module[n] 583 self.store(n, attr) 584 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 585 self.importer.load(attr.get_value().name) 586 587 return None 588 589 def visitFunction(self, node): 590 return self._visitFunction(node, node.name) 591 592 visitGenExpr = OP 593 594 visitGenExprFor = NOP 595 596 visitGenExprIf = NOP 597 598 visitGenExprInner = NOP 599 600 def visitGetattr(self, node): 601 expr = self.dispatch(node.expr) 602 attrname = node.attrname 603 604 if isinstance(expr, Attr): 605 value = expr.get_value() 606 if isinstance(value, (Class, Module)): 607 attr = value.namespace.get(attrname) 608 elif isinstance(value, UnresolvedName): 609 attr = UnresolvedName(attrname, value.full_name(), self) 610 else: 611 attr = None 612 elif self.builtins is not None: 613 attr = self.builtins.get(attrname) 614 else: 615 attr = UnresolvedName(attrname, value.full_name(), self) 616 617 # Accounting. 618 619 if attr is not None: 620 attr.set_referenced() 621 622 self.importer.use_name(attrname) 623 624 return attr 625 626 def visitGlobal(self, node): 627 if self.namespaces: 628 for name in node.names: 629 ns = self.namespaces[-1] 630 if not ns.make_global(name): 631 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 632 633 # Record a global entry for the name in the module. 634 635 if not self.has_key(name): 636 self[name] = Global() 637 638 def visitIf(self, node): 639 for test, body in node.tests: 640 self.dispatch(test) 641 self.dispatch(body) 642 if node.else_ is not None: 643 self.dispatch(node.else_) 644 return None 645 646 visitIfExp = NOP 647 648 def visitImport(self, node): 649 for name, alias in node.names: 650 if alias is not None: 651 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 652 self.store(alias, module) 653 else: 654 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 655 self.store(name.split(".")[0], module) 656 module.set_referenced() 657 658 return None 659 660 visitInvert = _visitUnary 661 662 def visitKeyword(self, node): 663 self.dispatch(node.expr) 664 self.importer.make_constant(node.name) 665 self.keyword_names.add(node.name) 666 return None 667 668 def visitLambda(self, node): 669 return self._visitFunction(node, None) 670 671 visitLeftShift = _visitBinary 672 673 visitList = OP 674 675 visitListComp = OP 676 677 visitListCompFor = NOP 678 679 visitListCompIf = NOP 680 681 visitMod = _visitBinary 682 683 def visitModule(self, node): 684 685 # Make a back reference from the node for code generation. 686 687 node.unit = self 688 return self.dispatch(node.node) 689 690 visitMul = _visitBinary 691 692 def visitName(self, node): 693 name = node.name 694 695 if self.importer.predefined_constants.has_key(name): 696 attr = self.importer.get_predefined_constant(name) 697 elif self.namespaces and self.namespaces[-1].has_key(name): 698 attr = self.namespaces[-1][name] 699 elif self.has_key(name): 700 attr = self[name] 701 elif self.builtins is not None and self.builtins.has_key(name): 702 attr = self.builtins[name] 703 else: 704 attr = None 705 706 # Accounting. 707 708 if attr is not None: 709 attr.set_referenced() 710 711 self.importer.use_name(name) 712 713 return attr 714 715 visitNot = OP 716 717 visitOr = OP 718 719 visitPass = NOP 720 721 visitPower = _visitBinary 722 723 visitPrint = NOP 724 725 visitPrintnl = NOP 726 727 visitRaise = NOP 728 729 visitReturn = NOP 730 731 visitRightShift = _visitBinary 732 733 visitSlice = OP 734 735 visitSliceobj = OP 736 737 def visitStmt(self, node): 738 for n in node.nodes: 739 self.dispatch(n) 740 return None 741 742 visitSub = _visitBinary 743 744 def visitSubscript(self, node): 745 self.importer.use_name("__getitem__") 746 self.OP(node) 747 748 def visitTryExcept(self, node): 749 self.dispatch(node.body) 750 for name, var, n in node.handlers: 751 752 # Establish the local for the handler. 753 754 if var is not None: 755 self.dispatch(var) 756 if n is not None: 757 self.dispatch(n) 758 if node.else_ is not None: 759 self.dispatch(node.else_) 760 return None 761 762 visitTryFinally = NOP 763 764 visitTuple = OP 765 766 visitUnaryAdd = _visitUnary 767 768 visitUnarySub = _visitUnary 769 770 def visitWhile(self, node): 771 self.in_loop = 1 772 self.NOP(node) 773 self.in_loop = 0 774 return None 775 776 visitWith = NOP 777 778 visitYield = NOP 779 780 class Global: 781 782 """ 783 A reference to an object assigned to a global from outside the module 784 top-level. 785 """ 786 787 pass 788 789 # vim: tabstop=4 expandtab shiftwidth=4