#!/usr/bin/env python  """ The micropython package for processing Python source code. The code originates from the simplify package but has had various details related to that package removed.  Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk>  This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.  You should have received a copy of the GNU General Public License along with this program.  If not, see <http://www.gnu.org/licenses/>.  --------  To use this module, an importer should be constructed. Here, the standard path for module searching is employed:  importer = Importer(sys.path)  To generate programs, the above importer should be supplied in the initialisation of a program instance, and then various methods are called:  program = Program(importer) image = program.get_raw_image()  Such importer and program objects are the most convenient mechanism through which the functionality of the micropython package may be accessed. """  from micropython.common import ObjectSet, ProcessingError, TableError, \     TableGenerationError import micropython.ast import micropython.data import micropython.opt import micropython.inspect import micropython.table import os  try:     set except NameError:     from sets import Set as set  class Program:      "This class supports the generation of a program image."      supported_optimisations = micropython.opt.Optimiser.supported_optimisations      def __init__(self, importer, optimisations=None):          """         Initialise the program representation with an 'importer' which is able         to locate and load Python modules.          The optional 'optimisations' cause certain techniques to be used in         reducing program size and improving program efficiency.         """          self.importer = importer         self.optimisations = optimisations or set()          # Remember the tables once generated.          self.objtable = None         self.paramtable = None          # Main program information.          self.code = None         self.code_location = None      def get_importer(self):         return self.importer      # Access to finalised program information.      def finalise(self):          "Finalise the program."          # Need the tables to finalise.          objtable = self.get_object_table()         self.get_parameter_table()          self.importer.vacuum(objtable)         self.importer.finalise()          # Now remove unneeded things from the tables.          self.get_object_table(reset=1)         self.get_parameter_table(reset=1)      def get_image(self, with_builtins=0):          """         Return the program image including built-in objects if 'with_builtins'         is specified and set to a true value.         """          if self.code is not None:             return self.code          # Optimise and regenerate the object table.          self.finalise()         self.code = []          # Append constants to the image.          for const in self.importer.constants():             self.code.append(const)          last_module = self.importer.modules_ordered[-1]          for module in self.importer.modules_ordered:             suppress_builtins = not with_builtins and module.name == "__builtins__"              # Position the module in the image and make a translation.              trans = micropython.ast.Translation(module, self)              # Add header details.              self.code.append(module)              # Append module attributes to the image.              attributes = module.module_attributes()             self.code += module.attributes_as_list()              # Append classes and functions to the image.              for obj in module.all_objects:                 if isinstance(obj, micropython.data.Class):                      # Add header details.                      self.code.append(obj)                      # Append class attributes to the image.                      attributes = obj.class_attributes()                     self.code += obj.attributes_as_list()                      # Omit built-in function code where requested.                      if suppress_builtins and obj.astnode.doc is None:                         continue                      # Generate the instantiator/initialiser.                     # Append the function code to the image.                      code = trans.get_instantiator_code(obj)                     self.code += code                      # Class-level code is generated separately at the module                     # level, and the code location is set within the code                     # generation process for the module.                  elif isinstance(obj, micropython.data.Function):                      # Add header details.                      self.code.append(obj)                      # Append any default values to the image.                     # Only do this for functions which are not dynamic.                      if not obj.is_dynamic():                         self.code += obj.default_attrs                      # Omit built-in function code where requested.                      if suppress_builtins and obj.astnode.doc is None:                         pass                      # Append the function code to the image.                      else:                         code = trans.get_code(obj)                         self.code += code              # Omit built-in module code where requested.              if suppress_builtins:                 pass              # Append the module top-level code to the image.              else:                 code = trans.get_module_code()                 self.code += code          return self.code      def get_raw_image(self, architecture=None, with_builtins=0):          "Return the raw image representation of the program."          architecture = architecture or micropython.rsvp          self.get_image(with_builtins)          objtable = self.get_object_table()         paramtable = self.get_parameter_table()          # Position the objects.          pos = 0          for item in self.code:             arch_item = architecture.get_object(item)              # Get the raw version for the architecture.              if arch_item is not None:                 pos = arch_item.set_location(pos, objtable, with_builtins)             else:                 pos += 1          # Generate the raw code.          self.raw_code = []          for item in self.code:             arch_item = architecture.get_object(item)              # Get the raw version for the architecture.              if arch_item is not None:                 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins)                 arch_item.finalise_location(with_builtins)             else:                 self.raw_code.append(item)          # Fix the module locations.          for module in self.importer.modules_ordered:              if not with_builtins and module.name == "__builtins__":                 continue              module.code_location = module.blocks[0].location          self.code_location = self.importer.modules["__main__"].code_location         return self.raw_code      def get_object_table(self, reset=0):          "Return a table with details of attributes for classes and modules."          if self.objtable is None or reset:              t = self.objtable = micropython.table.ObjectTable()             for module in self.importer.get_modules():                  # Add module attributes and module identity information.                  full_name = module.full_name()                 attributes = {full_name : module}                 attributes.update(module.module_attributes())                 t.add(full_name, attributes)                  # Add class and instance attributes for all classes, together                 # with descendant information.                  for obj in module.all_objects:                     if isinstance(obj, micropython.data.Class):                          # Prevent ambiguous classes.                          full_name = obj.full_name()                          #name = obj.name                         #if module.has_key(name) and module[name].defines_ambiguous_class():                         #    raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name())                          # Define a table entry for the class.                          attributes = {full_name : obj}                         attributes.update(obj.all_attributes())                         attributes.update(obj.all_descendants())                         t.add(full_name, attributes)          return self.objtable      def get_parameter_table(self, reset=0):          "Return a table with details of parameters for functions and methods."          # Need the object table to get at class details.          if self.paramtable is None or reset:             t = self.paramtable = micropython.table.ParameterTable()              # Visit each module, getting function and method details.              for module in self.importer.get_modules():                 for obj in module.all_objects:                     if isinstance(obj, micropython.data.Function):                         t.add(obj.full_name(), obj.parameters())                      # Classes are callable, too.                     # Take details of the appropriate __init__ method to make an                     # entry for an instantiation function for the class.                      elif isinstance(obj, micropython.data.Class):                         t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters())              # Filter out all parameter table entries not referenced by keyword             # arguments.              keyword_names = set()              for module in self.importer.get_modules():                 keyword_names.update(module.keyword_names)              for function_name, parameters in t.table.items():                 for name in parameters.keys():                     if name in keyword_names:                         break                 else:                     del t.table[function_name]          return self.paramtable  class Importer:      "An import machine, searching for and loading modules."      predefined_constants = {         "None" : None,         "True" : True,         "False" : False,         #"Ellipsis" : Ellipsis,         "NotImplemented" : NotImplemented         }      names_always_used = [         "bool", "__call__", "__bool__"         ]      def __init__(self, path=None, verbose=0, optimisations=None):          """         Initialise the importer with the given search 'path' - a list of         directories to search for Python modules.          The optional 'verbose' parameter causes output concerning the activities         of the object to be produced if set to a true value (not the default).          The optional 'optimisations' cause certain techniques to be used in         reducing program size and improving program efficiency.         """          self.path = path or [os.getcwd()]         self.verbose = verbose         self.optimisations = optimisations or set()          self.modules = {}         self.modules_ordered = []         self.loading = set()          # Constant records.          self.constant_values = {}         self.constant_list = None # cache for constants         self.init_predefined_constants()          # Attribute usage.          self.attributes_used = set()         self.name_references = {}         self.specific_name_references = {}          # Attribute coverage calculated during collection.          self.inferred_name_references = {}          # Attribute coverage status during collection.          self.attribute_users_visited = set()         self.attributes_to_visit = {}          # Status information.          self.vacuumed = 0         self.finalised = 0      def get_modules(self):          "Return all modules known to the importer."          return self.modules.values()      def get_module(self, name):          "Return the module with the given 'name'."          return self.modules[name]      # General maintenance.      def vacuum(self, objtable):          "Tidy up the modules."          if self.vacuumed:             return          # Complete the list of attribute names used in the program.          self.collect_attributes(objtable)          for name, module in self.modules.items():             if module.loaded:                 module.vacuum()             else:                 del self.modules[name]          self.vacuumed = 1      def finalise(self):          "Finalise the program (which should have been vacuumed first)."          if self.finalised:             return          # Reset any previously compiled information.          for module in self.get_modules():             module.unfinalise()          # Prepare module information again.          for module in self.get_modules():             module.finalise()          self.finalised = 1      # Name accounting.      def use_name(self, name, from_name, value=None):          """         Register the given 'name' as being used in the program from within an         object with the specified 'from_name'. If the optional 'value' is given,         note an assignment.         """          if not self.name_references.has_key(from_name):             self.name_references[from_name] = set()          attrnames = ObjectSet([name])         usage = (attrnames,)         self.name_references[from_name].add((None, None, usage))      def use_names(self, user, name, usage, from_name):          """         For the given attribute 'user' (which may be None if no specific user is         given), register for the given 'name' the given attribute 'usage'         (combinations of attribute names), noting the scope of this usage as         being the program object with the specified 'from_name'.         """          if not self.name_references.has_key(from_name):             self.name_references[from_name] = set()          self.name_references[from_name].add((user, name, usage))      def use_specific_name(self, objname, attrname, from_name):          """         Register the given 'objname' (for an object) whose 'attrname' is being         used in the program from within an object with the specified         'from_name'.         """          if not self.specific_name_references.has_key(from_name):             self.specific_name_references[from_name] = set()         self.specific_name_references[from_name].add((objname, attrname))      # Name accounting products.      def uses_attribute(self, objname, name):          """         Return whether the attribute of the object with the given 'objname'         having the given 'name' is used as an attribute in the program.         """          return (objname + "." + name) in self.attributes_used      def use_attribute(self, objname, name):          """         Indicate that in the object with the given 'objname', the attribute of         the given 'name' is used.         """          self.attributes_used.add(objname + "." + name)      def use_object(self, objname):          "Indicate that the object with the given 'objname' is used."          self.attributes_used.add(objname)      def collect_attributes(self, objtable):          "Collect attribute references for the entire program."          # Include names which may not be explicitly used in programs.         # NOTE: Potentially declare these when inspecting.          for attrname in self.names_always_used:             for objname in objtable.all_possible_objects([attrname]):                  # Record attributes of objects for potential visiting.                  self.add_attribute_to_visit(objname, attrname)          # Visit all modules, since some may employ initialisation code which has         # some kind of side-effect.          for name in self.modules.keys():             self._collect_attributes(name, objtable)      def add_attribute_to_visit(self, objname, attrname):          """         Queue an attribute of the object with the given 'objname', having the         given 'attrname', to the list for potential visiting if the specified         object is actually referenced.         """          if not self.attributes_to_visit.has_key(objname):             self.attributes_to_visit[objname] = set()         self.attributes_to_visit[objname].add(attrname)      def _collect_attributes_from(self, from_name, objname, attrname, objtable):          """         Record the association between 'from_name' and the attribute of         'objname' with the given 'attrname'. Then collect attributes for the         referenced attribute using 'objtable'.         """          if not self.inferred_name_references.has_key(from_name):             self.inferred_name_references[from_name] = set()          self.inferred_name_references[from_name].add((objname, attrname))         self._collect_attributes(objname + "." + attrname, objtable)      def _collect_attributes(self, from_name, objtable):          """         Given an object called 'from_name', find all names referenced from such         an object according to the register of names, using 'objtable' to infer         types.         """          if from_name in self.attribute_users_visited:             return          self.attribute_users_visited.add(from_name)          # The getattr function is a special case: it can potentially reference         # any known attribute. Since accessor attributes must be known         # constants, the intersection of known constants and attributes is used         # to build a set of objects that might be referenced by getattr.          if from_name == "__builtins__.getattr":             all_attributes = set(objtable.attribute_names())             all_string_constants = set([const.get_value() for const in self.constants() if const.value_type_name() == "__builtins__.str"])             all_attribute_constants = all_attributes.intersection(all_string_constants)              # Get the types supporting each attribute and visit the referenced             # objects.              all_objtypes = set()              for attrname in all_attribute_constants:                 objtypes = objtable.any_possible_objects_plus_status([attrname])                 all_objtypes.update(objtypes)              # Attribute assignment does not take place, so an empty list of             # values is given.              self._collect_attributes_for_types(from_name, objtable, all_objtypes,                 [{attrname : []} for attrname in all_attribute_constants])          # Get name references and find possible objects which support such         # combinations of attribute names.          for user, name, usage in self.name_references.get(from_name, []):              # Using all attribute names for a particular name, attempt to get             # specific object types.              all_objtypes = set()              for attrnames in usage:                 objtypes = objtable.all_possible_objects_plus_status(attrnames)                 if not objtypes:                     print "Warning: usage in %r for %r finds no object supporting all attributes %r" % (from_name, name, attrnames)                     objtypes = objtable.any_possible_objects_plus_status(attrnames)                     if not objtypes:                         print "Warning: usage in %r for %r finds no object supporting any attributes %r" % (from_name, name, attrnames)                  all_objtypes.update(objtypes)              # Record the object types for generating guards.              if user is not None:                 if not hasattr(user, "_attrtypes"):                     user._attrtypes = {}                  user._attrtypes[name] = all_objtypes                 self._collect_attributes_for_types(from_name, objtable, all_objtypes, usage)          # Get specific name references and visit the referenced objects.          for objname, attrname in self.specific_name_references.get(from_name, []):             self.use_attribute(objname, attrname)             self._collect_attributes_from(from_name, objname, attrname, objtable)          # Where the object has an __init__ attribute, assume that it is an         # initialiser which is called at some point, and collect attributes used         # in this initialiser.          if "__init__" in objtable.table.get(from_name, []):             self.use_attribute(from_name, "__init__")             self._collect_attributes_from(from_name, from_name, "__init__", objtable)          # Visit attributes on this object that were queued in case of the object         # being referenced.          attributes_to_visit = self.attributes_to_visit.get(from_name, [])          if attributes_to_visit:             del self.attributes_to_visit[from_name]              for attrname in attributes_to_visit:                 self.use_attribute(from_name, attrname)                 self._collect_attributes_from(from_name, from_name, attrname, objtable)      def _collect_attributes_for_types(self, from_name, objtable, objtypes, usage):          """         For the unit known as 'from_name' and using the 'objtable' to validate         each attribute, identify and attempt to visit attributes found for each         of the suggested object types given by 'objtypes' and the 'usage'         provided.         """          for objname, is_static in objtypes:             for attrnames in usage:                 for attrname, attrvalues in attrnames.items():                      # Test for the presence of an attribute on the suggested                     # object type.                      try:                         attr = objtable.access(objname, attrname)                     except TableError:                         #print "Warning: object type %r does not support attribute %r" % (objname, attrname)                         continue                      # Get the real identity of the attribute in order to                     # properly collect usage from it.                      parent = attr.parent                     if isinstance(parent, micropython.data.Instance):                         parentname = objname                     else:                         parentname = parent.full_name()                      # Test for assignment.                      if attrvalues:                         for attrvalue in attrvalues:                             parent.set(attrname, attrvalue, 0)                      # Visit attributes of objects known to be used.                      if parentname in self.attributes_used:                         self.use_attribute(parentname, attrname)                         self._collect_attributes_from(from_name, parentname, attrname, objtable)                      # Record attributes of other objects for potential visiting.                      else:                         self.add_attribute_to_visit(parentname, attrname)      # Constant accounting.      def init_predefined_constants(self):          "Ensure the predefined constants."          for name, value in self.predefined_constants.items():             self.make_constant(value)      def get_predefined_constant(self, name):          "Return the predefined constant for the given 'name'."          return self.make_constant(self.predefined_constants[name])      def get_constant(self, value):          "Return a constant for the given 'value'."          const = micropython.data.Const(value)         return self.constant_values[const]      def get_constant_type_name(self, value):          "Return the type name for the given constant 'value'."          return value.__class__.__name__      def make_constant(self, value):          "Make and return a constant for the given 'value'."          # Make a constant object and return it.          const = micropython.data.Const(value)         if not self.constant_values.has_key(const):             self.constant_values[const] = const         return self.constant_values[const]      def constants(self):          "Return a list of constants."          if self.constant_list is None:             self.constant_list = list(self.constant_values.values())          return self.constant_list      # Import methods.      def find_in_path(self, name):          """         Find the given module 'name' in the search path, returning None where no         such module could be found, or a 2-tuple from the 'find' method         otherwise.         """          for d in self.path:             m = self.find(d, name)             if m: return m         return None      def find(self, d, name):          """         In the directory 'd', find the given module 'name', where 'name' can         either refer to a single file module or to a package. Return None if the         'name' cannot be associated with either a file or a package directory,         or a 2-tuple from '_find_package' or '_find_module' otherwise.         """          m = self._find_package(d, name)         if m: return m         m = self._find_module(d, name)         if m: return m         return None      def _find_module(self, d, name):          """         In the directory 'd', find the given module 'name', returning None where         no suitable file exists in the directory, or a 2-tuple consisting of         None (indicating that no package directory is involved) and a filename         indicating the location of the module.         """          name_py = name + os.extsep + "py"         filename = self._find_file(d, name_py)         if filename:             return None, filename         return None      def _find_package(self, d, name):          """         In the directory 'd', find the given package 'name', returning None         where no suitable package directory exists, or a 2-tuple consisting of         a directory (indicating the location of the package directory itself)         and a filename indicating the location of the __init__.py module which         declares the package's top-level contents.         """          filename = self._find_file(d, name)         if filename:             init_py = "__init__" + os.path.extsep + "py"             init_py_filename = self._find_file(filename, init_py)             if init_py_filename:                 return filename, init_py_filename         return None      def _find_file(self, d, filename):          """         Return the filename obtained when searching the directory 'd' for the         given 'filename', or None if no actual file exists for the filename.         """          filename = os.path.join(d, filename)         if os.path.exists(filename):             return filename         else:             return None      def load(self, name, return_leaf=0):          """         Load the module or package with the given 'name'. Return an object         referencing the loaded module or package, or None if no such module or         package exists.         """          if return_leaf:             name_for_return = name         else:             name_for_return = name.split(".")[0]          if self.modules.has_key(name) and self.modules[name].loaded:             #print "Cached (%s)" % name             return self.modules[name_for_return]          if self.verbose:             print "Loading", name          # Split the name into path components, and try to find the uppermost in         # the search path.          path = name.split(".")         m = self.find_in_path(path[0])         if not m:             if self.verbose:                 print "Not found (%s)" % path[0]             return None # NOTE: Import error.         d, filename = m          # Either acquire a reference to an already-imported module, or load the         # module from a file.          top = module = self.load_from_file(filename, path[0])          # For hierarchical names, traverse each path component...          if len(path) > 1:             if not d:                 if self.verbose:                     print "No package (%s)" % filename                 return None # NOTE: Import error (package not found).             else:                 self.add_submodules(d, module)              path_so_far = path[:1]             for p in path[1:]:                 path_so_far.append(p)                  # Find the package or module concerned.                  m = self.find(d, p)                 if not m:                     if self.verbose:                         print "Not found (%s)" % p                     return None # NOTE: Import error.                 d, filename = m                 module_name = ".".join(path_so_far)                  # Either reference an imported module or load one from a file.                  submodule = self.load_from_file(filename, module_name)                  if d:                     self.add_submodules(d, module)                  # Store the submodule within its parent module.                  module.set_module(p, submodule)                 module = submodule          # Return either the deepest or the uppermost module.          if return_leaf:             return module         else:             return top      def load_from_file(self, name, module_name=None):          """         Load the module with the given 'name' (which may be a full module path).         """          if module_name is None:             module_name = "__main__"          module = self.add_module(module_name)         if not module.loaded and module not in self.loading:             self.loading.add(module)             #print "Parsing", name             module.parse(name)             #print "Done", name             self.loading.remove(module)             module.loaded = 1          # Record the module.          self.use_object(module.full_name())         #print "Loaded", module_name, "with namespace", module.namespace.keys()         return module      def add_module(self, module_name):          """         Return the module with the given 'module_name', adding a new module         object if one does not already exist.         """          if not self.modules.has_key(module_name):             self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self)             self.modules_ordered.append(module)         else:             module = self.modules[module_name]         return module      def add_submodules(self, pathname, module):          """         Work around insufficient __all__ declarations and examine the directory         with the given 'pathname', adding submodules to the given 'module'.         """          for filename in os.listdir(pathname):             submodule, ext = os.path.splitext(filename)             if ext not in ("", ".py"):                 continue             module.set_module(submodule, self.add_module(module.name + "." + submodule))  # vim: tabstop=4 expandtab shiftwidth=4