1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 112 # Position the module in the image and make a translation. 113 114 trans = micropython.ast.Translation(module, self) 115 116 # Add header details. 117 118 self.code.append(module) 119 pos += 1 120 121 # Append module attributes to the image. 122 123 attributes = module.module_attributes() 124 self.code += module.attributes_as_list() 125 pos += len(attributes.keys()) 126 127 # Append classes and functions to the image. 128 129 for obj in module.all_objects: 130 if isinstance(obj, micropython.inspect.Class): 131 132 # Add header details. 133 134 self.code.append(obj) 135 pos += 1 136 137 # Append class attributes to the image. 138 139 attributes = obj.class_attributes() 140 self.code += obj.attributes_as_list() 141 pos += len(attributes.keys()) 142 143 # Omit built-in function code where requested. 144 145 if not with_builtins and module.name == "__builtins__": 146 continue 147 148 # Generate the instantiator/initialiser. 149 # Append the function code to the image. 150 151 instantiator = obj.get_instantiator() 152 code = trans.get_instantiator_code(obj) 153 self.code += code 154 pos += len(code) 155 156 # Class-level code is generated separately at the module 157 # level, and the code location is set within the code 158 # generation process for the module. 159 160 elif isinstance(obj, micropython.inspect.Function): 161 162 # Add header details. 163 164 self.code.append(obj) 165 pos += 1 166 167 # Append any default values to the image. 168 # Only do this for named functions (not lambdas). 169 170 if obj.name is not None: 171 self.code += obj.default_attrs 172 pos += len(obj.default_attrs) 173 174 # Omit built-in function code where requested. 175 176 if not with_builtins and module.name == "__builtins__": 177 pass 178 179 # Append the function code to the image. 180 181 else: 182 code = trans.get_code(obj) 183 self.code += code 184 pos += len(code) 185 186 # Omit built-in module code where requested. 187 188 if not with_builtins and module.name == "__builtins__": 189 pass 190 191 # Append the module top-level code to the image. 192 193 else: 194 code = trans.get_module_code(final=(module is last_module)) 195 self.code += code 196 pos += len(code) 197 198 return self.code 199 200 def get_raw_image(self, with_builtins=0): 201 202 "Return the raw image representation of the program." 203 204 self.get_image(with_builtins) 205 206 objtable = self.get_object_table() 207 paramtable = self.get_parameter_table() 208 209 # Position the objects. 210 211 pos = 0 212 for item in self.code: 213 214 # Blocks are positioned leaving space for their expansion. 215 216 if isinstance(item, Block): 217 item.location = pos 218 pos += len(item.code) 219 220 # Other multi-location objects. 221 222 elif isinstance(item, ( 223 micropython.data.Class, micropython.data.Const, 224 micropython.data.Function, micropython.data.Module 225 )): 226 227 if isinstance(item, micropython.data.Class): 228 229 # Append a template of an instance for use when 230 # instantiating classes. 231 232 item.instance_template_location = pos 233 pos += 1 234 235 # Record the location of the object. 236 237 item.location = pos 238 pos += 1 239 240 # Code and details are associated with certain objects. 241 242 if isinstance(item, micropython.data.Function): 243 244 # Set the code location only where the code has been 245 # generated. 246 247 if not with_builtins and item.module.name == "__builtins__": 248 item.code_location = item.full_name() 249 250 # Skip any defaults. 251 252 else: 253 item.code_location = pos + len(item.defaults) 254 255 elif isinstance(item, micropython.data.Const): 256 pos += len(item.raw_data()) 257 258 else: 259 pos += 1 260 261 # Generate the raw code. 262 263 self.raw_code = [] 264 265 for item in self.code: 266 267 if isinstance(item, micropython.data.Attr): 268 self.raw_code += item.as_raw(objtable, paramtable) 269 270 elif isinstance(item, Block): 271 assert item.location == len(self.raw_code) 272 self.raw_code += item.as_raw(objtable, paramtable) 273 274 # Using classcode, attrcode, codeaddr, codedetails, instance. 275 276 elif isinstance(item, micropython.data.Class): 277 assert item.instance_template_location == len(self.raw_code) 278 self.raw_code += item.as_raw(objtable, paramtable, with_builtins or item.module.name != "__builtins__") 279 assert item.location == len(self.raw_code) - 1 280 281 elif isinstance(item, micropython.data.Const): 282 assert item.location == len(self.raw_code) 283 self.raw_code += item.as_raw(objtable, paramtable) 284 285 elif isinstance(item, micropython.data.Function): 286 assert item.location == len(self.raw_code) 287 self.raw_code += item.as_raw(objtable, paramtable) 288 289 # Check the code location only where the code has been generated. 290 291 assert (not with_builtins and item.module.name == "__builtins__") or \ 292 item.code_location == len(self.raw_code) + len(item.defaults) 293 294 elif isinstance(item, micropython.data.Module): 295 assert item.location == len(self.raw_code) 296 self.raw_code += item.as_raw(objtable, paramtable) 297 298 else: 299 self.raw_code.append(item) 300 301 # Fix the module locations. 302 303 for module in self.importer.modules_ordered: 304 305 if not with_builtins and module.name == "__builtins__": 306 continue 307 308 module.code_location = module.blocks[0].location 309 310 self.code_location = self.importer.modules["__main__"].code_location 311 return self.raw_code 312 313 def get_object_table(self): 314 315 "Return a table with details of attributes for classes and modules." 316 317 if self.objtable is None: 318 self.importer.vacuum() 319 320 t = self.objtable = micropython.table.ObjectTable() 321 for module in self.importer.get_modules(): 322 t.add(module.full_name(), module.module_attributes()) 323 324 # Add class and instance attributes for all classes, together 325 # with descendant information. 326 327 for obj in module.all_objects: 328 if isinstance(obj, micropython.inspect.Class): 329 attributes = {obj.full_name() : obj} 330 attributes.update(obj.all_attributes()) 331 attributes.update(obj.all_descendants()) 332 t.add(obj.full_name(), attributes) 333 334 return self.objtable 335 336 def get_parameter_table(self): 337 338 "Return a table with details of parameters for functions and methods." 339 340 # Need the object table to get at class details. 341 342 objtable = self.get_object_table() 343 344 if self.paramtable is None: 345 t = self.paramtable = micropython.table.ParameterTable() 346 347 # Visit each module, getting function and method details. 348 349 for module in self.importer.get_modules(): 350 for obj in module.all_objects: 351 if isinstance(obj, micropython.inspect.Function): 352 t.add(obj.full_name(), obj.parameters()) 353 354 # Classes are callable, too. 355 # Take details of the appropriate __init__ method to make an 356 # entry for an instantiation function for the class. 357 358 elif isinstance(obj, micropython.inspect.Class): 359 t.add(obj.full_name(), obj.get_instantiator().parameters()) 360 361 # Filter out all parameter table entries not referenced by keyword 362 # arguments. 363 364 keyword_names = set() 365 366 for module in self.importer.get_modules(): 367 keyword_names.update(module.keyword_names) 368 369 for function_name, parameters in t.table.items(): 370 for name in parameters.keys(): 371 if name in keyword_names: 372 break 373 else: 374 del t.table[function_name] 375 376 return self.paramtable 377 378 class Importer: 379 380 "An import machine, searching for and loading modules." 381 382 predefined_constants = { 383 "None" : None, 384 "True" : True, 385 "False" : False, 386 #"Ellipsis" : Ellipsis, 387 "NotImplemented" : NotImplemented 388 } 389 390 def __init__(self, path=None, verbose=0, optimisations=None): 391 392 """ 393 Initialise the importer with the given search 'path' - a list of 394 directories to search for Python modules. 395 396 The optional 'verbose' parameter causes output concerning the activities 397 of the object to be produced if set to a true value (not the default). 398 399 The optional 'optimisations' cause certain techniques to be used in 400 reducing program size and improving program efficiency. 401 """ 402 403 self.path = path or [os.getcwd()] 404 self.verbose = verbose 405 self.optimisations = optimisations or set() 406 407 self.modules = {} 408 self.modules_ordered = [] 409 self.loading = set() 410 411 # Constant records. 412 413 self.constant_values = {} 414 self.constant_list = None # cache for constants 415 self.init_predefined_constants() 416 417 # Name records (used to track actual use of names). 418 # Include names which may not be explicitly used in programs. 419 # NOTE: Potentially declare these when inspecting. 420 421 self.names_used = set(["__init__", "__call__", "__bool__"]) 422 423 # Status information. 424 425 self.vacuumed = 0 426 self.finalised = 0 427 428 def get_modules(self): 429 430 "Return all modules known to the importer." 431 432 return self.modules.values() 433 434 # General maintenance. 435 436 def vacuum(self): 437 438 "Tidy up the modules." 439 440 if self.vacuumed: 441 return 442 443 for name, module in self.modules.items(): 444 if module.loaded: 445 module.vacuum() 446 else: 447 del self.modules[name] 448 449 self.vacuumed = 1 450 451 def finalise(self): 452 453 "Finalise the program." 454 455 if self.finalised: 456 return 457 458 self.vacuum() 459 460 for module in self.get_modules(): 461 module.finalise() 462 463 self.finalised = 1 464 465 # Name accounting. 466 467 def use_name(self, name): 468 469 "Register the given 'name' as being used in the program." 470 471 self.names_used.add(name) 472 473 # Constant accounting. 474 475 def init_predefined_constants(self): 476 477 "Ensure the predefined constants." 478 479 for name, value in self.predefined_constants.items(): 480 self.make_constant(value) 481 482 def get_predefined_constant(self, name): 483 484 "Return the predefined constant for the given 'name'." 485 486 return self.make_constant(self.predefined_constants[name]) 487 488 def get_constant(self, value): 489 490 "Return a constant for the given 'value'." 491 492 const = micropython.data.Const(value) 493 return self.constant_values[const] 494 495 def make_constant(self, value): 496 497 "Make and return a constant for the given 'value'." 498 499 const = micropython.data.Const(value) 500 if not self.constant_values.has_key(const): 501 self.constant_values[const] = const 502 return self.constant_values[const] 503 504 def constants(self): 505 506 "Return a list of constants." 507 508 if self.constant_list is None: 509 self.constant_list = list(self.constant_values.values()) 510 511 return self.constant_list 512 513 # Import methods. 514 515 def find_in_path(self, name): 516 517 """ 518 Find the given module 'name' in the search path, returning None where no 519 such module could be found, or a 2-tuple from the 'find' method 520 otherwise. 521 """ 522 523 for d in self.path: 524 m = self.find(d, name) 525 if m: return m 526 return None 527 528 def find(self, d, name): 529 530 """ 531 In the directory 'd', find the given module 'name', where 'name' can 532 either refer to a single file module or to a package. Return None if the 533 'name' cannot be associated with either a file or a package directory, 534 or a 2-tuple from '_find_package' or '_find_module' otherwise. 535 """ 536 537 m = self._find_package(d, name) 538 if m: return m 539 m = self._find_module(d, name) 540 if m: return m 541 return None 542 543 def _find_module(self, d, name): 544 545 """ 546 In the directory 'd', find the given module 'name', returning None where 547 no suitable file exists in the directory, or a 2-tuple consisting of 548 None (indicating that no package directory is involved) and a filename 549 indicating the location of the module. 550 """ 551 552 name_py = name + os.extsep + "py" 553 filename = self._find_file(d, name_py) 554 if filename: 555 return None, filename 556 return None 557 558 def _find_package(self, d, name): 559 560 """ 561 In the directory 'd', find the given package 'name', returning None 562 where no suitable package directory exists, or a 2-tuple consisting of 563 a directory (indicating the location of the package directory itself) 564 and a filename indicating the location of the __init__.py module which 565 declares the package's top-level contents. 566 """ 567 568 filename = self._find_file(d, name) 569 if filename: 570 init_py = "__init__" + os.path.extsep + "py" 571 init_py_filename = self._find_file(filename, init_py) 572 if init_py_filename: 573 return filename, init_py_filename 574 return None 575 576 def _find_file(self, d, filename): 577 578 """ 579 Return the filename obtained when searching the directory 'd' for the 580 given 'filename', or None if no actual file exists for the filename. 581 """ 582 583 filename = os.path.join(d, filename) 584 if os.path.exists(filename): 585 return filename 586 else: 587 return None 588 589 def load(self, name, return_leaf=0): 590 591 """ 592 Load the module or package with the given 'name'. Return an object 593 referencing the loaded module or package, or None if no such module or 594 package exists. 595 """ 596 597 if self.modules.has_key(name) and self.modules[name].loaded: 598 #print "Cached (%s)" % name 599 return self.modules[name] 600 if self.verbose: 601 print "Loading", name 602 603 # Split the name into path components, and try to find the uppermost in 604 # the search path. 605 606 path = name.split(".") 607 m = self.find_in_path(path[0]) 608 if not m: 609 if self.verbose: 610 print "Not found (%s)" % path[0] 611 return None # NOTE: Import error. 612 d, filename = m 613 614 # Either acquire a reference to an already-imported module, or load the 615 # module from a file. 616 617 top = module = self.load_from_file(filename, path[0]) 618 619 # For hierarchical names, traverse each path component... 620 621 if len(path) > 1: 622 if not d: 623 if self.verbose: 624 print "No package (%s)" % filename 625 return None # NOTE: Import error (package not found). 626 else: 627 self.add_submodules(d, module) 628 629 path_so_far = path[:1] 630 for p in path[1:]: 631 path_so_far.append(p) 632 633 # Find the package or module concerned. 634 635 m = self.find(d, p) 636 if not m: 637 if self.verbose: 638 print "Not found (%s)" % p 639 return None # NOTE: Import error. 640 d, filename = m 641 module_name = ".".join(path_so_far) 642 643 # Either reference an imported module or load one from a file. 644 645 submodule = self.load_from_file(filename, module_name) 646 647 if d: 648 self.add_submodules(d, module) 649 650 # Store the submodule within its parent module. 651 652 module.set_module(p, submodule) 653 module = submodule 654 655 # Return either the deepest or the uppermost module. 656 657 if return_leaf: 658 return module 659 else: 660 return top 661 662 def load_from_file(self, name, module_name=None): 663 664 """ 665 Load the module with the given 'name' (which may be a full module path). 666 """ 667 668 if module_name is None: 669 module_name = "__main__" 670 671 module = self.add_module(module_name) 672 if not module.loaded and module not in self.loading: 673 self.loading.add(module) 674 #print "Parsing", name 675 module.parse(name) 676 #print "Done", name 677 self.loading.remove(module) 678 module.loaded = 1 679 680 # Record the module. 681 682 #print "Loaded", module_name, "with namespace", module.namespace.keys() 683 return module 684 685 def add_module(self, module_name): 686 687 """ 688 Return the module with the given 'module_name', adding a new module 689 object if one does not already exist. 690 """ 691 692 if not self.modules.has_key(module_name): 693 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 694 self.modules_ordered.append(module) 695 else: 696 module = self.modules[module_name] 697 return module 698 699 def add_submodules(self, pathname, module): 700 701 """ 702 Work around insufficient __all__ declarations and examine the directory 703 with the given 'pathname', adding submodules to the given 'module'. 704 """ 705 706 for filename in os.listdir(pathname): 707 submodule, ext = os.path.splitext(filename) 708 if ext not in ("", ".py"): 709 continue 710 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 711 712 # vim: tabstop=4 expandtab shiftwidth=4