#!/usr/bin/env python  """ Parsing of textual content.  Copyright (C) 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>  This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.  You should have received a copy of the GNU General Public License along with this program.  If not, see <http://www.gnu.org/licenses/>. """  from imiptools.filesys import fix_permissions from os.path import isfile import codecs import re  def have_table(obj, filename):      "Return whether 'obj' is a table using the given 'filename'."      return hasattr(obj, "get_filename") and obj.get_filename() == filename  class FileTable:      "A file-based data table."      def __init__(self, filename, mutable=True,                  in_defaults=None, out_defaults=None,                  in_converter=None, out_converter=None,                  tab_separated=True, headers=False):          """         Open the table from the file having the given 'filename'. If 'mutable'         is given as a true value (as is the default), the table can be modified.          The 'in_defaults' is a list of (index, value) tuples indicating the         default value where a column either does not exist or provides an empty         value. The 'out_defaults' is a corresponding list used to serialise         missing and empty values.          The 'in_converter' is a callable accepting a tuple of values and         returning an object. The corresponding 'out_converter' accepts an object         and returns a tuple of values.          If 'tab_separated' is specified and is a false value, line parsing using         the imiptools.text.parse_line function will be performed instead of         splitting each line of the file using tab characters as separators.          If 'headers' is specified and is not false, the first line in the table         will provide header value information.         """          self.filename = filename         self.mutable = mutable         self.in_defaults = in_defaults         self.out_defaults = out_defaults         self.in_converter = in_converter         self.out_converter = out_converter         self.tab_separated = tab_separated          # Obtain the items. In subsequent implementations, the items could be         # retrieved dynamically.          items = []          if isfile(filename):             for item in get_table(filename, in_defaults, tab_separated):                 if self.in_converter:                     item = self.in_converter(item)                 items.append(item)          # Obtain header values and separate them from the rest of the data.          self.table = items[headers and 1 or 0:]         self.header_values = headers and items and items[0] or []         self.headers = headers      def get_filename(self):         return self.filename      def get_header_values(self):         return self.header_values      def set_header_values(self, values):         self.header_values = values      def close(self):          "Write any modifications and close the table."          if self.mutable:             f = codecs.open(self.filename, "wb", encoding="utf-8")             try:                 sep = self.tab_separated and "\t" or " "                  # Include any headers in the output.                  if self.headers:                     self.table.insert(0, self.header_values)                  for item in self.table:                     if self.out_converter:                         item = self.out_converter(item)                      # Insert defaults for empty columns.                      if self.out_defaults:                         item = set_defaults(list(item), self.out_defaults)                      # Separate the columns and write to the file.                      print >>f, sep.join(item)                  # Remove the headers from the items in case the table is                 # accessed again.                  if self.headers:                     del self.table[0]              finally:                 f.close()                 fix_permissions(self.filename)      # General collection methods.      def __nonzero__(self):         return bool(self.table)      # List emulation methods.      def __iadd__(self, other):         for value in other:             self.append(value)         return self      def __iter__(self):         return iter(self.table)      def __len__(self):         return len(self.table)      def __delitem__(self, i):         del self.table[i]      def __delslice__(self, start, end):         del self.table[start:end]      def __getitem__(self, i):         return self.table[i]      def __getslice__(self, start, end):         return self.table[start:end]      def __setitem__(self, i, value):         self.table[i] = value      def __setslice__(self, start, end, values):         self.table[start:end] = values      def append(self, value):         self.table.append(value)      def insert(self, i, value):         self.table.insert(i, value)      def remove(self, value):         self.table.remove(value)      # Dictionary emulation methods (even though this is not a mapping).      def clear(self):         del self.table[:]      # Additional modification methods.      def replaceall(self, values):         self.table[:] = values  class FileTableDict(FileTable):      "A file-based table acting as a dictionary."      def __init__(self, filename, mutable=True,                  in_defaults=None, out_defaults=None,                  in_converter=None, out_converter=None,                  tab_separated=True, headers=False):          FileTable.__init__(self, filename, mutable, in_defaults, out_defaults,                            in_converter, out_converter, tab_separated, headers)         self.mapping = dict(self.table)      def close(self):         self.table = self.mapping.items()         FileTable.close(self)      # General collection methods.      def __nonzero__(self):         return bool(self.mapping)      # List emulation methods.      def __iter__(self):         return iter(self.mapping)      def __len__(self):         return len(self.mapping)      def append(self, value):         key, value = value         self.mapping[key] = value      def insert(self, i, value):         self.append(value)      def remove(self, value):         key, value = value         del self.mapping[key]      # Unimplemented methods.      def __delslice__(self, start, end):         raise NotImplementedError, "__delslice__"      def __getslice__(self, start, end):         raise NotImplementedError, "__getslice__"      def __setslice__(self, start, end, values):         raise NotImplementedError, "__setslice__"      # Dictionary emulation methods.      def clear(self):         self.mapping.clear()      def get(self, i, default=None):         return self.mapping.get(i, default)      def keys(self):         return self.mapping.keys()      def items(self):         return self.mapping.items()      def update(self, other):         self.mapping.update(other)      def values(self):         return self.mapping.values()      def __delitem__(self, i):         del self.mapping[i]      def __getitem__(self, i):         return self.mapping[i]      def __setitem__(self, i, value):         if self.mutable:             self.mapping[i] = value      # Additional modification methods.      def replaceall(self, values):         self.mapping = {}         self.mapping.update(dict(values))      def updateall(self, mapping):         self.mapping = {}         self.mapping.update(mapping)  def first(t):     return t[0]  def tuplevalue(v):     return (v,)  class FileTableSingle(FileTable):      "A file-based table providing single value items."      def __iter__(self):         return iter(self[:])      def __getitem__(self, i):         return self.table[i][0]      def __getslice__(self, start, end):         return map(first, self.table[start:end])      def __setitem__(self, i, value):         self.table[i] = [(value,)]      def __setslice__(self, start, end, values):         self.table[start:end] = map(tuplevalue, values)      def append(self, value):         self.table.append((value,))      def insert(self, i, value):         self.table.insert(i, (value,))      def remove(self, value):         self.table.remove((value,))      # Additional modification methods.      def replaceall(self, values):         self.table[:] = map(tuplevalue, values)    # Parsing of tabular files.  def set_defaults(t, empty_defaults):      """     In the list 't', replace values that are empty or absent with defaults     provided by the 'empty_defaults' collection whose entries are of the form     (index, value).     """      for i, default in empty_defaults:         if i >= len(t):             t += [None] * (i - len(t) + 1)         if not t[i]:             t[i] = default     return t  def get_table(filename, empty_defaults=None, tab_separated=True):      """     From the file having the given 'filename', return a list of tuples     representing the file's contents.      The 'empty_defaults' is a list of (index, value) tuples indicating the     default value where a column either does not exist or provides an empty     value.      If 'tab_separated' is specified and is a false value, line parsing using     the imiptools.text.parse_line function will be performed instead of     splitting each line of the file using tab characters as separators.     """      f = codecs.open(filename, "rb", encoding="utf-8")     try:         return get_table_from_stream(f, empty_defaults, tab_separated)     finally:         f.close()  def get_table_from_stream(f, empty_defaults=None, tab_separated=True):      """     Return a list of tuples representing the contents of the stream 'f'.      The 'empty_defaults' is a list of (index, value) tuples indicating the     default value where a column either does not exist or provides an empty     value.      If 'tab_separated' is specified and is a false value, line parsing using     the imiptools.text.parse_line function will be performed instead of     splitting each line of the file using tab characters as separators.     """      l = []      for line in f.readlines():         line = line.strip(" \r\n")          if tab_separated:             t = line.split("\t")         else:             t = parse_line(line)          if empty_defaults:             t = set_defaults(t, empty_defaults)         l.append(tuple(t))      return l    # Parsing of lines to obtain functions and arguments.  line_pattern_str = (                    r"(?:"                    r"(?:'(.*?)')" # single-quoted text                    r"|"                    r'(?:"(.*?)")' # double-quoted text                    r"|"                    r"([^\s]+)"    # non-whitespace characters                    r")+"                    r"(?:\s+|$)"   # optional trailing whitespace before line end                    )  line_pattern = re.compile(line_pattern_str)  def parse_line(text):      """     Parse the given 'text', returning a list of words separated by whitespace in     the input, where whitespace may occur inside words if quoted using single or     double quotes.      Hello world     -> ['Hello', 'world']     Hello ' world'  -> ['Hello', ' world']     Hello' 'world   -> ["'Hello'", "'world']     """      parts = []      # Match the components of each part.      for match in line_pattern.finditer(text):          # Combine the components by traversing the matching groups.          parts.append(reduce(lambda a, b: (a or "") + (b or ""), match.groups()))      return parts  # vim: tabstop=4 expandtab shiftwidth=4