1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009, 2010 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def finalise(self): 88 89 "Finalise the program." 90 91 # Need the tables to finalise. 92 93 objtable = self.get_object_table() 94 self.get_parameter_table() 95 96 self.importer.vacuum(objtable) 97 self.importer.finalise() 98 99 # Now remove unneeded things from the tables. 100 101 self.get_object_table(reset=1) 102 self.get_parameter_table(reset=1) 103 104 def get_image(self, with_builtins=0): 105 106 """ 107 Return the program image including built-in objects if 'with_builtins' 108 is specified and set to a true value. 109 """ 110 111 if self.code is not None: 112 return self.code 113 114 # Optimise and regenerate the object table. 115 116 self.finalise() 117 self.code = [] 118 119 # Append constants to the image. 120 121 for const in self.importer.constants(): 122 self.code.append(const) 123 124 last_module = self.importer.modules_ordered[-1] 125 126 for module in self.importer.modules_ordered: 127 suppress_builtins = not with_builtins and module.name == "__builtins__" 128 129 # Position the module in the image and make a translation. 130 131 trans = micropython.ast.Translation(module, self) 132 133 # Add header details. 134 135 self.code.append(module) 136 137 # Append module attributes to the image. 138 139 attributes = module.module_attributes() 140 self.code += module.attributes_as_list() 141 142 # Append classes and functions to the image. 143 144 for obj in module.all_objects: 145 if isinstance(obj, micropython.inspect.Class): 146 147 # Add header details. 148 149 self.code.append(obj) 150 151 # Append class attributes to the image. 152 153 attributes = obj.class_attributes() 154 self.code += obj.attributes_as_list() 155 156 # Omit built-in function code where requested. 157 158 if suppress_builtins and obj.astnode.doc is None: 159 continue 160 161 # Generate the instantiator/initialiser. 162 # Append the function code to the image. 163 164 instantiator = obj.get_instantiator() 165 code = trans.get_instantiator_code(obj) 166 self.code += code 167 168 # Class-level code is generated separately at the module 169 # level, and the code location is set within the code 170 # generation process for the module. 171 172 elif isinstance(obj, micropython.inspect.Function): 173 174 # Add header details. 175 176 self.code.append(obj) 177 178 # Append any default values to the image. 179 # Only do this for functions which are not dynamic. 180 181 if not obj.is_dynamic(): 182 self.code += obj.default_attrs 183 184 # Omit built-in function code where requested. 185 186 if suppress_builtins and obj.astnode.doc is None: 187 pass 188 189 # Append the function code to the image. 190 191 else: 192 code = trans.get_code(obj) 193 self.code += code 194 195 # Omit built-in module code where requested. 196 197 if suppress_builtins: 198 pass 199 200 # Append the module top-level code to the image. 201 202 else: 203 code = trans.get_module_code() 204 self.code += code 205 206 return self.code 207 208 def get_raw_image(self, architecture=None, with_builtins=0): 209 210 "Return the raw image representation of the program." 211 212 architecture = architecture or micropython.rsvp 213 214 self.get_image(with_builtins) 215 216 objtable = self.get_object_table() 217 paramtable = self.get_parameter_table() 218 219 # Position the objects. 220 221 pos = 0 222 223 for item in self.code: 224 arch_item = architecture.get_object(item) 225 226 # Get the raw version for the architecture. 227 228 if arch_item is not None: 229 pos = arch_item.set_location(pos, with_builtins) 230 else: 231 pos += 1 232 233 # Generate the raw code. 234 235 self.raw_code = [] 236 237 for item in self.code: 238 arch_item = architecture.get_object(item) 239 240 # Get the raw version for the architecture. 241 242 if arch_item is not None: 243 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 244 arch_item.finalise_location(with_builtins) 245 else: 246 self.raw_code.append(item) 247 248 # Fix the module locations. 249 250 for module in self.importer.modules_ordered: 251 252 if not with_builtins and module.name == "__builtins__": 253 continue 254 255 module.code_location = module.blocks[0].location 256 257 self.code_location = self.importer.modules["__main__"].code_location 258 return self.raw_code 259 260 def get_object_table(self, reset=0): 261 262 "Return a table with details of attributes for classes and modules." 263 264 if self.objtable is None or reset: 265 266 t = self.objtable = micropython.table.ObjectTable() 267 for module in self.importer.get_modules(): 268 t.add(module.full_name(), module.module_attributes()) 269 270 # Add class and instance attributes for all classes, together 271 # with descendant information. 272 273 for obj in module.all_objects: 274 if isinstance(obj, micropython.inspect.Class): 275 276 # Prevent ambiguous classes. 277 278 full_name = obj.full_name() 279 name = obj.name 280 281 if module.has_key(name) and module[name].defines_ambiguous_class(): 282 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 283 284 # Define a table entry for the class. 285 286 attributes = {full_name : obj} 287 attributes.update(obj.all_attributes()) 288 attributes.update(obj.all_descendants()) 289 t.add(full_name, attributes) 290 291 return self.objtable 292 293 def get_parameter_table(self, reset=0): 294 295 "Return a table with details of parameters for functions and methods." 296 297 # Need the object table to get at class details. 298 299 if self.paramtable is None or reset: 300 t = self.paramtable = micropython.table.ParameterTable() 301 302 # Visit each module, getting function and method details. 303 304 for module in self.importer.get_modules(): 305 for obj in module.all_objects: 306 if isinstance(obj, micropython.inspect.Function): 307 t.add(obj.full_name(), obj.parameters()) 308 309 # Classes are callable, too. 310 # Take details of the appropriate __init__ method to make an 311 # entry for an instantiation function for the class. 312 313 elif isinstance(obj, micropython.inspect.Class): 314 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 315 316 # Filter out all parameter table entries not referenced by keyword 317 # arguments. 318 319 keyword_names = set() 320 321 for module in self.importer.get_modules(): 322 keyword_names.update(module.keyword_names) 323 324 for function_name, parameters in t.table.items(): 325 for name in parameters.keys(): 326 if name in keyword_names: 327 break 328 else: 329 del t.table[function_name] 330 331 return self.paramtable 332 333 class Importer: 334 335 "An import machine, searching for and loading modules." 336 337 predefined_constants = { 338 "None" : None, 339 "True" : True, 340 "False" : False, 341 #"Ellipsis" : Ellipsis, 342 "NotImplemented" : NotImplemented 343 } 344 345 names_always_used = [ 346 "bool", "__call__", "__bool__" 347 ] 348 349 def __init__(self, path=None, verbose=0, optimisations=None): 350 351 """ 352 Initialise the importer with the given search 'path' - a list of 353 directories to search for Python modules. 354 355 The optional 'verbose' parameter causes output concerning the activities 356 of the object to be produced if set to a true value (not the default). 357 358 The optional 'optimisations' cause certain techniques to be used in 359 reducing program size and improving program efficiency. 360 """ 361 362 self.path = path or [os.getcwd()] 363 self.verbose = verbose 364 self.optimisations = optimisations or set() 365 366 self.modules = {} 367 self.modules_ordered = [] 368 self.loading = set() 369 370 # Constant records. 371 372 self.constant_values = {} 373 self.constant_list = None # cache for constants 374 self.init_predefined_constants() 375 376 # Attribute usage. 377 378 self.attributes_used = set() 379 self.name_references = {} 380 self.specific_name_references = {} 381 self.attribute_users_visited = set() 382 self.attributes_to_visit = {} 383 384 # Status information. 385 386 self.vacuumed = 0 387 self.finalised = 0 388 389 def get_modules(self): 390 391 "Return all modules known to the importer." 392 393 return self.modules.values() 394 395 def get_module(self, name): 396 397 "Return the module with the given 'name'." 398 399 return self.modules[name] 400 401 # General maintenance. 402 403 def vacuum(self, objtable): 404 405 "Tidy up the modules." 406 407 if self.vacuumed: 408 return 409 410 # Complete the list of attribute names used in the program. 411 412 self.collect_attributes(objtable) 413 414 for name, module in self.modules.items(): 415 if module.loaded: 416 module.vacuum() 417 else: 418 del self.modules[name] 419 420 self.vacuumed = 1 421 422 def finalise(self): 423 424 "Finalise the program (which should have been vacuumed first)." 425 426 if self.finalised: 427 return 428 429 # Reset any previously compiled information. 430 431 for module in self.get_modules(): 432 module.unfinalise() 433 434 # Prepare module information again. 435 436 for module in self.get_modules(): 437 module.finalise() 438 439 self.finalised = 1 440 441 # Name accounting. 442 443 def use_name(self, name, from_name): 444 445 """ 446 Register the given 'name' as being used in the program from within an 447 object with the specified 'from_name'. 448 """ 449 450 if not self.name_references.has_key(from_name): 451 self.name_references[from_name] = set() 452 self.name_references[from_name].add((name,)) 453 454 def use_names(self, names, from_name): 455 456 """ 457 Register the given 'names' as being used in the program from within an 458 object with the specified 'from_name'. 459 """ 460 461 if not self.name_references.has_key(from_name): 462 self.name_references[from_name] = set() 463 self.name_references[from_name].add(names) 464 465 def use_specific_name(self, objname, attrname, from_name): 466 467 """ 468 Register the given 'objname' (for an object) whose 'attrname' is being 469 used in the program from within an object with the specified 470 'from_name'. 471 """ 472 473 if not self.specific_name_references.has_key(from_name): 474 self.specific_name_references[from_name] = set() 475 self.specific_name_references[from_name].add((objname, attrname)) 476 477 # Name accounting products. 478 479 def uses_attribute(self, objname, name): 480 481 """ 482 Return whether the attribute of the object with the given 'objname' 483 having the given 'name' is used as an attribute in the program. 484 """ 485 486 return (objname + "." + name) in self.attributes_used 487 488 def use_attribute(self, objname, name): 489 490 """ 491 Indicate that in the object with the given 'objname', the attribute of 492 the given 'name' is used. 493 """ 494 495 self.attributes_used.add(objname + "." + name) 496 497 def use_object(self, objname): 498 499 "Indicate that the object with the given 'objname' is used." 500 501 self.attributes_used.add(objname) 502 503 def collect_attributes(self, objtable): 504 505 "Collect attribute references for the entire program." 506 507 # Include names which may not be explicitly used in programs. 508 # NOTE: Potentially declare these when inspecting. 509 510 for attrname in self.names_always_used: 511 for objname in objtable.all_possible_objects([attrname]): 512 513 # Record attributes of objects for potential visiting. 514 515 self.add_attribute_to_visit(objname, attrname) 516 517 # Start with the "root" modules, finding referenced objects. 518 519 self._collect_attributes("__builtins__", objtable) 520 self._collect_attributes("__main__", objtable) 521 522 def add_attribute_to_visit(self, objname, attrname): 523 524 """ 525 Queue an attribute of the object with the given 'objname', having the 526 given 'attrname', to the list for potential visiting if the specified 527 object is actually referenced. 528 """ 529 530 if not self.attributes_to_visit.has_key(objname): 531 self.attributes_to_visit[objname] = set() 532 self.attributes_to_visit[objname].add(attrname) 533 534 def _collect_attributes(self, from_name, objtable): 535 536 """ 537 Given an object called 'from_name', find all names referenced from such 538 an object according to the register of names. 539 """ 540 541 if from_name in self.attribute_users_visited: 542 return 543 544 self.attribute_users_visited.add(from_name) 545 546 # Get name references and find possible objects which support such 547 # combinations of attribute names. 548 549 for names in self.name_references.get(from_name, []): 550 objnames = objtable.all_possible_objects(names) 551 if not objnames: 552 print "Warning: usage in %r finds no object supporting all attributes %r" % (from_name, names) 553 objnames = objtable.any_possible_objects(names) 554 if not objnames: 555 print "Warning: usage in %r finds no object supporting any attributes %r" % (from_name, names) 556 557 # For each suggested object, consider each attribute given by the 558 # names. 559 560 for objname in objnames: 561 for name in names: 562 563 # Visit attributes of objects known to be used. 564 565 if objname in self.attributes_used: 566 self.use_attribute(objname, name) 567 self._collect_attributes(objname + "." + name, objtable) 568 569 # Record attributes of other objects for potential visiting. 570 571 else: 572 self.add_attribute_to_visit(objname, name) 573 574 # Get specific name references and visit the referenced objects. 575 576 for objname, attrname in self.specific_name_references.get(from_name, []): 577 self.use_attribute(objname, attrname) 578 self._collect_attributes(objname + "." + attrname, objtable) 579 580 # Where the object has an __init__ attribute, assume that it is an 581 # initialiser which is called at some point, and collect attributes used 582 # in this initialiser. 583 584 if "__init__" in objtable.table.get(from_name, []): 585 self.use_attribute(from_name, "__init__") 586 self._collect_attributes(from_name + ".__init__", objtable) 587 588 # Visit attributes on objects not previously visited. 589 590 attributes_to_visit = self.attributes_to_visit.get(from_name, []) 591 592 if attributes_to_visit: 593 del self.attributes_to_visit[from_name] 594 595 for attrname in attributes_to_visit: 596 self.use_attribute(from_name, attrname) 597 self._collect_attributes(from_name + "." + attrname, objtable) 598 599 # Constant accounting. 600 601 def init_predefined_constants(self): 602 603 "Ensure the predefined constants." 604 605 for name, value in self.predefined_constants.items(): 606 self.make_constant(value) 607 608 def get_predefined_constant(self, name): 609 610 "Return the predefined constant for the given 'name'." 611 612 return self.make_constant(self.predefined_constants[name]) 613 614 def get_constant(self, value): 615 616 "Return a constant for the given 'value'." 617 618 const = micropython.data.Const(value) 619 return self.constant_values[const] 620 621 def get_constant_type_name(self, value): 622 return value.__class__.__name__ 623 624 def make_constant(self, value): 625 626 "Make and return a constant for the given 'value'." 627 628 # Ensure the presence of the constant's type. 629 630 name = self.get_constant_type_name(value) 631 if self.modules.has_key("__builtins__"): 632 attr = self.modules["__builtins__"].get(name) 633 634 # Make a constant object and return it. 635 636 const = micropython.data.Const(value) 637 if not self.constant_values.has_key(const): 638 self.constant_values[const] = const 639 return self.constant_values[const] 640 641 def constants(self): 642 643 "Return a list of constants." 644 645 if self.constant_list is None: 646 self.constant_list = list(self.constant_values.values()) 647 648 return self.constant_list 649 650 # Import methods. 651 652 def find_in_path(self, name): 653 654 """ 655 Find the given module 'name' in the search path, returning None where no 656 such module could be found, or a 2-tuple from the 'find' method 657 otherwise. 658 """ 659 660 for d in self.path: 661 m = self.find(d, name) 662 if m: return m 663 return None 664 665 def find(self, d, name): 666 667 """ 668 In the directory 'd', find the given module 'name', where 'name' can 669 either refer to a single file module or to a package. Return None if the 670 'name' cannot be associated with either a file or a package directory, 671 or a 2-tuple from '_find_package' or '_find_module' otherwise. 672 """ 673 674 m = self._find_package(d, name) 675 if m: return m 676 m = self._find_module(d, name) 677 if m: return m 678 return None 679 680 def _find_module(self, d, name): 681 682 """ 683 In the directory 'd', find the given module 'name', returning None where 684 no suitable file exists in the directory, or a 2-tuple consisting of 685 None (indicating that no package directory is involved) and a filename 686 indicating the location of the module. 687 """ 688 689 name_py = name + os.extsep + "py" 690 filename = self._find_file(d, name_py) 691 if filename: 692 return None, filename 693 return None 694 695 def _find_package(self, d, name): 696 697 """ 698 In the directory 'd', find the given package 'name', returning None 699 where no suitable package directory exists, or a 2-tuple consisting of 700 a directory (indicating the location of the package directory itself) 701 and a filename indicating the location of the __init__.py module which 702 declares the package's top-level contents. 703 """ 704 705 filename = self._find_file(d, name) 706 if filename: 707 init_py = "__init__" + os.path.extsep + "py" 708 init_py_filename = self._find_file(filename, init_py) 709 if init_py_filename: 710 return filename, init_py_filename 711 return None 712 713 def _find_file(self, d, filename): 714 715 """ 716 Return the filename obtained when searching the directory 'd' for the 717 given 'filename', or None if no actual file exists for the filename. 718 """ 719 720 filename = os.path.join(d, filename) 721 if os.path.exists(filename): 722 return filename 723 else: 724 return None 725 726 def load(self, name, return_leaf=0): 727 728 """ 729 Load the module or package with the given 'name'. Return an object 730 referencing the loaded module or package, or None if no such module or 731 package exists. 732 """ 733 734 if self.modules.has_key(name) and self.modules[name].loaded: 735 #print "Cached (%s)" % name 736 return self.modules[name] 737 if self.verbose: 738 print "Loading", name 739 740 # Split the name into path components, and try to find the uppermost in 741 # the search path. 742 743 path = name.split(".") 744 m = self.find_in_path(path[0]) 745 if not m: 746 if self.verbose: 747 print "Not found (%s)" % path[0] 748 return None # NOTE: Import error. 749 d, filename = m 750 751 # Either acquire a reference to an already-imported module, or load the 752 # module from a file. 753 754 top = module = self.load_from_file(filename, path[0]) 755 756 # For hierarchical names, traverse each path component... 757 758 if len(path) > 1: 759 if not d: 760 if self.verbose: 761 print "No package (%s)" % filename 762 return None # NOTE: Import error (package not found). 763 else: 764 self.add_submodules(d, module) 765 766 path_so_far = path[:1] 767 for p in path[1:]: 768 path_so_far.append(p) 769 770 # Find the package or module concerned. 771 772 m = self.find(d, p) 773 if not m: 774 if self.verbose: 775 print "Not found (%s)" % p 776 return None # NOTE: Import error. 777 d, filename = m 778 module_name = ".".join(path_so_far) 779 780 # Either reference an imported module or load one from a file. 781 782 submodule = self.load_from_file(filename, module_name) 783 784 if d: 785 self.add_submodules(d, module) 786 787 # Store the submodule within its parent module. 788 789 module.set_module(p, submodule) 790 module = submodule 791 792 # Return either the deepest or the uppermost module. 793 794 if return_leaf: 795 return module 796 else: 797 return top 798 799 def load_from_file(self, name, module_name=None): 800 801 """ 802 Load the module with the given 'name' (which may be a full module path). 803 """ 804 805 if module_name is None: 806 module_name = "__main__" 807 808 module = self.add_module(module_name) 809 if not module.loaded and module not in self.loading: 810 self.loading.add(module) 811 #print "Parsing", name 812 module.parse(name) 813 #print "Done", name 814 self.loading.remove(module) 815 module.loaded = 1 816 817 # Record the module. 818 819 self.use_object(module.full_name()) 820 #print "Loaded", module_name, "with namespace", module.namespace.keys() 821 return module 822 823 def add_module(self, module_name): 824 825 """ 826 Return the module with the given 'module_name', adding a new module 827 object if one does not already exist. 828 """ 829 830 if not self.modules.has_key(module_name): 831 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 832 self.modules_ordered.append(module) 833 else: 834 module = self.modules[module_name] 835 return module 836 837 def add_submodules(self, pathname, module): 838 839 """ 840 Work around insufficient __all__ declarations and examine the directory 841 with the given 'pathname', adding submodules to the given 'module'. 842 """ 843 844 for filename in os.listdir(pathname): 845 submodule, ext = os.path.splitext(filename) 846 if ext not in ("", ".py"): 847 continue 848 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 849 850 # vim: tabstop=4 expandtab shiftwidth=4