1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009, 2010 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def get_image(self, with_builtins=0): 88 89 """ 90 Return the program image including built-in objects if 'with_builtins' 91 is specified and set to a true value. 92 """ 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 100 # Optimise and regenerate the object table. 101 102 self.importer.vacuum(objtable) 103 self.importer.finalise() 104 105 objtable = self.get_object_table(reset=1) 106 paramtable = self.get_parameter_table(reset=1) 107 108 self.code = [] 109 110 # Append constants to the image. 111 112 for const in self.importer.constants(): 113 self.code.append(const) 114 115 last_module = self.importer.modules_ordered[-1] 116 117 for module in self.importer.modules_ordered: 118 suppress_builtins = not with_builtins and module.name == "__builtins__" 119 120 # Position the module in the image and make a translation. 121 122 trans = micropython.ast.Translation(module, self) 123 124 # Add header details. 125 126 self.code.append(module) 127 128 # Append module attributes to the image. 129 130 attributes = module.module_attributes() 131 self.code += module.attributes_as_list() 132 133 # Append classes and functions to the image. 134 135 for obj in module.all_objects: 136 if isinstance(obj, micropython.inspect.Class): 137 138 # Add header details. 139 140 self.code.append(obj) 141 142 # Append class attributes to the image. 143 144 attributes = obj.class_attributes() 145 self.code += obj.attributes_as_list() 146 147 # Omit built-in function code where requested. 148 149 if suppress_builtins and obj.astnode.doc is None: 150 continue 151 152 # Generate the instantiator/initialiser. 153 # Append the function code to the image. 154 155 instantiator = obj.get_instantiator() 156 code = trans.get_instantiator_code(obj) 157 self.code += code 158 159 # Class-level code is generated separately at the module 160 # level, and the code location is set within the code 161 # generation process for the module. 162 163 elif isinstance(obj, micropython.inspect.Function): 164 165 # Add header details. 166 167 self.code.append(obj) 168 169 # Append any default values to the image. 170 # Only do this for functions which are not dynamic. 171 172 if not obj.is_dynamic(): 173 self.code += obj.default_attrs 174 175 # Omit built-in function code where requested. 176 177 if suppress_builtins and obj.astnode.doc is None: 178 pass 179 180 # Append the function code to the image. 181 182 else: 183 code = trans.get_code(obj) 184 self.code += code 185 186 # Omit built-in module code where requested. 187 188 if suppress_builtins: 189 pass 190 191 # Append the module top-level code to the image. 192 193 else: 194 code = trans.get_module_code() 195 self.code += code 196 197 return self.code 198 199 def get_raw_image(self, architecture=None, with_builtins=0): 200 201 "Return the raw image representation of the program." 202 203 architecture = architecture or micropython.rsvp 204 205 self.get_image(with_builtins) 206 207 objtable = self.get_object_table() 208 paramtable = self.get_parameter_table() 209 210 # Position the objects. 211 212 pos = 0 213 214 for item in self.code: 215 arch_item = architecture.get_object(item) 216 217 # Get the raw version for the architecture. 218 219 if arch_item is not None: 220 pos = arch_item.set_location(pos, with_builtins) 221 else: 222 pos += 1 223 224 # Generate the raw code. 225 226 self.raw_code = [] 227 228 for item in self.code: 229 arch_item = architecture.get_object(item) 230 231 # Get the raw version for the architecture. 232 233 if arch_item is not None: 234 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 235 arch_item.finalise_location(with_builtins) 236 else: 237 self.raw_code.append(item) 238 239 # Fix the module locations. 240 241 for module in self.importer.modules_ordered: 242 243 if not with_builtins and module.name == "__builtins__": 244 continue 245 246 module.code_location = module.blocks[0].location 247 248 self.code_location = self.importer.modules["__main__"].code_location 249 return self.raw_code 250 251 def get_object_table(self, reset=0): 252 253 "Return a table with details of attributes for classes and modules." 254 255 if self.objtable is None or reset: 256 257 t = self.objtable = micropython.table.ObjectTable() 258 for module in self.importer.get_modules(): 259 t.add(module.full_name(), module.module_attributes()) 260 261 # Add class and instance attributes for all classes, together 262 # with descendant information. 263 264 for obj in module.all_objects: 265 if isinstance(obj, micropython.inspect.Class): 266 267 # Prevent ambiguous classes. 268 269 full_name = obj.full_name() 270 name = obj.name 271 272 if module.has_key(name) and module[name].defines_ambiguous_class(): 273 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 274 275 # Define a table entry for the class. 276 277 attributes = {full_name : obj} 278 attributes.update(obj.all_attributes()) 279 attributes.update(obj.all_descendants()) 280 t.add(full_name, attributes) 281 282 return self.objtable 283 284 def get_parameter_table(self, reset=0): 285 286 "Return a table with details of parameters for functions and methods." 287 288 # Need the object table to get at class details. 289 290 if self.paramtable is None or reset: 291 t = self.paramtable = micropython.table.ParameterTable() 292 293 # Visit each module, getting function and method details. 294 295 for module in self.importer.get_modules(): 296 for obj in module.all_objects: 297 if isinstance(obj, micropython.inspect.Function): 298 t.add(obj.full_name(), obj.parameters()) 299 300 # Classes are callable, too. 301 # Take details of the appropriate __init__ method to make an 302 # entry for an instantiation function for the class. 303 304 elif isinstance(obj, micropython.inspect.Class): 305 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 306 307 # Filter out all parameter table entries not referenced by keyword 308 # arguments. 309 310 keyword_names = set() 311 312 for module in self.importer.get_modules(): 313 keyword_names.update(module.keyword_names) 314 315 for function_name, parameters in t.table.items(): 316 for name in parameters.keys(): 317 if name in keyword_names: 318 break 319 else: 320 del t.table[function_name] 321 322 return self.paramtable 323 324 class Importer: 325 326 "An import machine, searching for and loading modules." 327 328 predefined_constants = { 329 "None" : None, 330 "True" : True, 331 "False" : False, 332 #"Ellipsis" : Ellipsis, 333 "NotImplemented" : NotImplemented 334 } 335 336 names_always_used = [ 337 "bool", "__call__", "__bool__" 338 ] 339 340 def __init__(self, path=None, verbose=0, optimisations=None): 341 342 """ 343 Initialise the importer with the given search 'path' - a list of 344 directories to search for Python modules. 345 346 The optional 'verbose' parameter causes output concerning the activities 347 of the object to be produced if set to a true value (not the default). 348 349 The optional 'optimisations' cause certain techniques to be used in 350 reducing program size and improving program efficiency. 351 """ 352 353 self.path = path or [os.getcwd()] 354 self.verbose = verbose 355 self.optimisations = optimisations or set() 356 357 self.modules = {} 358 self.modules_ordered = [] 359 self.loading = set() 360 361 # Constant records. 362 363 self.constant_values = {} 364 self.constant_list = None # cache for constants 365 self.init_predefined_constants() 366 367 # Attribute usage. 368 369 self.attributes_used = set() 370 self.name_references = {} 371 self.specific_name_references = {} 372 self.attribute_users_visited = set() 373 self.attributes_to_visit = {} 374 375 # Status information. 376 377 self.vacuumed = 0 378 self.finalised = 0 379 380 def get_modules(self): 381 382 "Return all modules known to the importer." 383 384 return self.modules.values() 385 386 def get_module(self, name): 387 388 "Return the module with the given 'name'." 389 390 return self.modules[name] 391 392 # General maintenance. 393 394 def vacuum(self, objtable): 395 396 "Tidy up the modules." 397 398 if self.vacuumed: 399 return 400 401 # Complete the list of attribute names used in the program. 402 403 self.collect_attributes(objtable) 404 405 for name, module in self.modules.items(): 406 if module.loaded: 407 module.vacuum() 408 else: 409 del self.modules[name] 410 411 self.vacuumed = 1 412 413 def finalise(self): 414 415 "Finalise the program (which should have been vacuumed first)." 416 417 if self.finalised: 418 return 419 420 # Reset any previously compiled information. 421 422 for module in self.get_modules(): 423 module.unfinalise() 424 425 # Prepare module information again. 426 427 for module in self.get_modules(): 428 module.finalise() 429 430 self.finalised = 1 431 432 # Name accounting. 433 434 def use_name(self, name, from_name): 435 436 """ 437 Register the given 'name' as being used in the program from within an 438 object with the specified 'from_name'. 439 """ 440 441 if not self.name_references.has_key(from_name): 442 self.name_references[from_name] = set() 443 self.name_references[from_name].add((name,)) 444 445 def use_names(self, names, from_name): 446 447 """ 448 Register the given 'names' as being used in the program from within an 449 object with the specified 'from_name'. 450 """ 451 452 if not self.name_references.has_key(from_name): 453 self.name_references[from_name] = set() 454 self.name_references[from_name].add(names) 455 456 def use_specific_name(self, objname, attrname, from_name): 457 458 """ 459 Register the given 'objname' (for an object) whose 'attrname' is being 460 used in the program from within an object with the specified 461 'from_name'. 462 """ 463 464 if not self.specific_name_references.has_key(from_name): 465 self.specific_name_references[from_name] = set() 466 self.specific_name_references[from_name].add((objname, attrname)) 467 468 # Name accounting products. 469 470 def uses_attribute(self, objname, name): 471 472 """ 473 Return whether the attribute of the object with the given 'objname' 474 having the given 'name' is used as an attribute in the program. 475 """ 476 477 return (objname + "." + name) in self.attributes_used 478 479 def use_attribute(self, objname, name): 480 481 """ 482 Indicate that in the object with the given 'objname', the attribute of 483 the given 'name' is used. 484 """ 485 486 self.attributes_used.add(objname + "." + name) 487 488 def use_object(self, objname): 489 490 "Indicate that the object with the given 'objname' is used." 491 492 self.attributes_used.add(objname) 493 494 def collect_attributes(self, objtable): 495 496 "Collect attribute references for the entire program." 497 498 # Include names which may not be explicitly used in programs. 499 # NOTE: Potentially declare these when inspecting. 500 501 for attrname in self.names_always_used: 502 for objname in objtable.all_possible_objects([attrname]): 503 504 # Record attributes of objects for potential visiting. 505 506 self.add_attribute_to_visit(objname, attrname) 507 508 # Start with the "root" modules, finding referenced objects. 509 510 self._collect_attributes("__builtins__", objtable) 511 self._collect_attributes("__main__", objtable) 512 513 def add_attribute_to_visit(self, objname, attrname): 514 515 """ 516 Queue an attribute of the object with the given 'objname', having the 517 given 'attrname', to the list for potential visiting if the specified 518 object is actually referenced. 519 """ 520 521 if not self.attributes_to_visit.has_key(objname): 522 self.attributes_to_visit[objname] = set() 523 self.attributes_to_visit[objname].add(attrname) 524 525 def _collect_attributes(self, from_name, objtable): 526 527 """ 528 Given an object called 'from_name', find all names referenced from such 529 an object according to the register of names. 530 """ 531 532 if from_name in self.attribute_users_visited: 533 return 534 535 self.attribute_users_visited.add(from_name) 536 537 # Get name references and find possible objects which support such 538 # combinations of attribute names. 539 540 for names in self.name_references.get(from_name, []): 541 objnames = objtable.all_possible_objects(names) 542 if not objnames: 543 print "Warning: usage in %r finds no object supporting all attributes %r" % (from_name, names) 544 objnames = objtable.any_possible_objects(names) 545 if not objnames: 546 print "Warning: usage in %r finds no object supporting any attributes %r" % (from_name, names) 547 548 # For each suggested object, consider each attribute given by the 549 # names. 550 551 for objname in objnames: 552 for name in names: 553 554 # Visit attributes of objects known to be used. 555 556 if objname in self.attributes_used: 557 self.use_attribute(objname, name) 558 self._collect_attributes(objname + "." + name, objtable) 559 560 # Record attributes of other objects for potential visiting. 561 562 else: 563 self.add_attribute_to_visit(objname, name) 564 565 # Get specific name references and visit the referenced objects. 566 567 for objname, attrname in self.specific_name_references.get(from_name, []): 568 self.use_attribute(objname, attrname) 569 self._collect_attributes(objname + "." + attrname, objtable) 570 571 # Where the object has an __init__ attribute, assume that it is an 572 # initialiser which is called at some point, and collect attributes used 573 # in this initialiser. 574 575 if "__init__" in objtable.table.get(from_name, []): 576 self.use_attribute(from_name, "__init__") 577 self._collect_attributes(from_name + ".__init__", objtable) 578 579 # Visit attributes on objects not previously visited. 580 581 attributes_to_visit = self.attributes_to_visit.get(from_name, []) 582 583 if attributes_to_visit: 584 del self.attributes_to_visit[from_name] 585 586 for attrname in attributes_to_visit: 587 self.use_attribute(from_name, attrname) 588 self._collect_attributes(from_name + "." + attrname, objtable) 589 590 # Constant accounting. 591 592 def init_predefined_constants(self): 593 594 "Ensure the predefined constants." 595 596 for name, value in self.predefined_constants.items(): 597 self.make_constant(value) 598 599 def get_predefined_constant(self, name): 600 601 "Return the predefined constant for the given 'name'." 602 603 return self.make_constant(self.predefined_constants[name]) 604 605 def get_constant(self, value): 606 607 "Return a constant for the given 'value'." 608 609 const = micropython.data.Const(value) 610 return self.constant_values[const] 611 612 def get_constant_type_name(self, value): 613 return value.__class__.__name__ 614 615 def make_constant(self, value): 616 617 "Make and return a constant for the given 'value'." 618 619 # Ensure the presence of the constant's type. 620 621 name = self.get_constant_type_name(value) 622 if self.modules.has_key("__builtins__"): 623 attr = self.modules["__builtins__"].get(name) 624 625 # Make a constant object and return it. 626 627 const = micropython.data.Const(value) 628 if not self.constant_values.has_key(const): 629 self.constant_values[const] = const 630 return self.constant_values[const] 631 632 def constants(self): 633 634 "Return a list of constants." 635 636 if self.constant_list is None: 637 self.constant_list = list(self.constant_values.values()) 638 639 return self.constant_list 640 641 # Import methods. 642 643 def find_in_path(self, name): 644 645 """ 646 Find the given module 'name' in the search path, returning None where no 647 such module could be found, or a 2-tuple from the 'find' method 648 otherwise. 649 """ 650 651 for d in self.path: 652 m = self.find(d, name) 653 if m: return m 654 return None 655 656 def find(self, d, name): 657 658 """ 659 In the directory 'd', find the given module 'name', where 'name' can 660 either refer to a single file module or to a package. Return None if the 661 'name' cannot be associated with either a file or a package directory, 662 or a 2-tuple from '_find_package' or '_find_module' otherwise. 663 """ 664 665 m = self._find_package(d, name) 666 if m: return m 667 m = self._find_module(d, name) 668 if m: return m 669 return None 670 671 def _find_module(self, d, name): 672 673 """ 674 In the directory 'd', find the given module 'name', returning None where 675 no suitable file exists in the directory, or a 2-tuple consisting of 676 None (indicating that no package directory is involved) and a filename 677 indicating the location of the module. 678 """ 679 680 name_py = name + os.extsep + "py" 681 filename = self._find_file(d, name_py) 682 if filename: 683 return None, filename 684 return None 685 686 def _find_package(self, d, name): 687 688 """ 689 In the directory 'd', find the given package 'name', returning None 690 where no suitable package directory exists, or a 2-tuple consisting of 691 a directory (indicating the location of the package directory itself) 692 and a filename indicating the location of the __init__.py module which 693 declares the package's top-level contents. 694 """ 695 696 filename = self._find_file(d, name) 697 if filename: 698 init_py = "__init__" + os.path.extsep + "py" 699 init_py_filename = self._find_file(filename, init_py) 700 if init_py_filename: 701 return filename, init_py_filename 702 return None 703 704 def _find_file(self, d, filename): 705 706 """ 707 Return the filename obtained when searching the directory 'd' for the 708 given 'filename', or None if no actual file exists for the filename. 709 """ 710 711 filename = os.path.join(d, filename) 712 if os.path.exists(filename): 713 return filename 714 else: 715 return None 716 717 def load(self, name, return_leaf=0): 718 719 """ 720 Load the module or package with the given 'name'. Return an object 721 referencing the loaded module or package, or None if no such module or 722 package exists. 723 """ 724 725 if self.modules.has_key(name) and self.modules[name].loaded: 726 #print "Cached (%s)" % name 727 return self.modules[name] 728 if self.verbose: 729 print "Loading", name 730 731 # Split the name into path components, and try to find the uppermost in 732 # the search path. 733 734 path = name.split(".") 735 m = self.find_in_path(path[0]) 736 if not m: 737 if self.verbose: 738 print "Not found (%s)" % path[0] 739 return None # NOTE: Import error. 740 d, filename = m 741 742 # Either acquire a reference to an already-imported module, or load the 743 # module from a file. 744 745 top = module = self.load_from_file(filename, path[0]) 746 747 # For hierarchical names, traverse each path component... 748 749 if len(path) > 1: 750 if not d: 751 if self.verbose: 752 print "No package (%s)" % filename 753 return None # NOTE: Import error (package not found). 754 else: 755 self.add_submodules(d, module) 756 757 path_so_far = path[:1] 758 for p in path[1:]: 759 path_so_far.append(p) 760 761 # Find the package or module concerned. 762 763 m = self.find(d, p) 764 if not m: 765 if self.verbose: 766 print "Not found (%s)" % p 767 return None # NOTE: Import error. 768 d, filename = m 769 module_name = ".".join(path_so_far) 770 771 # Either reference an imported module or load one from a file. 772 773 submodule = self.load_from_file(filename, module_name) 774 775 if d: 776 self.add_submodules(d, module) 777 778 # Store the submodule within its parent module. 779 780 module.set_module(p, submodule) 781 module = submodule 782 783 # Return either the deepest or the uppermost module. 784 785 if return_leaf: 786 return module 787 else: 788 return top 789 790 def load_from_file(self, name, module_name=None): 791 792 """ 793 Load the module with the given 'name' (which may be a full module path). 794 """ 795 796 if module_name is None: 797 module_name = "__main__" 798 799 module = self.add_module(module_name) 800 if not module.loaded and module not in self.loading: 801 self.loading.add(module) 802 #print "Parsing", name 803 module.parse(name) 804 #print "Done", name 805 self.loading.remove(module) 806 module.loaded = 1 807 808 # Record the module. 809 810 self.use_object(module.full_name()) 811 #print "Loaded", module_name, "with namespace", module.namespace.keys() 812 return module 813 814 def add_module(self, module_name): 815 816 """ 817 Return the module with the given 'module_name', adding a new module 818 object if one does not already exist. 819 """ 820 821 if not self.modules.has_key(module_name): 822 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 823 self.modules_ordered.append(module) 824 else: 825 module = self.modules[module_name] 826 return module 827 828 def add_submodules(self, pathname, module): 829 830 """ 831 Work around insufficient __all__ declarations and examine the directory 832 with the given 'pathname', adding submodules to the given 'module'. 833 """ 834 835 for filename in os.listdir(pathname): 836 submodule, ext = os.path.splitext(filename) 837 if ext not in ("", ".py"): 838 continue 839 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 840 841 # vim: tabstop=4 expandtab shiftwidth=4