1.1 --- a/iixr/files.py Thu Feb 03 01:26:35 2011 +0100
1.2 +++ b/iixr/files.py Mon Feb 07 02:05:38 2011 +0100
1.3 @@ -18,7 +18,7 @@
1.4 with this program. If not, see <http://www.gnu.org/licenses/>.
1.5 """
1.6
1.7 -from iixr.data import vint, vint_to_array, vint_from_array
1.8 +from iixr.data import *
1.9 from array import array
1.10 import zlib
1.11
1.12 @@ -30,7 +30,8 @@
1.13
1.14 def __init__(self, f):
1.15 self.f = f
1.16 - self.data = array('B')
1.17 + self.data = array('B') # master buffer
1.18 + self.record = array('B') # record buffer
1.19 self.reset()
1.20
1.21 def reset(self):
1.22 @@ -47,29 +48,11 @@
1.23 self.f.seek(0)
1.24 self.reset()
1.25
1.26 - def flush(self):
1.27 - if self.f is not None:
1.28 - self.data.tofile(self.f)
1.29 - self.data = array('B')
1.30 -
1.31 def close(self):
1.32 if self.f is not None:
1.33 - self.data.tofile(self.f)
1.34 self.f.close()
1.35 self.f = None
1.36
1.37 - def get_value_size(self, value):
1.38 - if isinstance(value, (list, tuple)):
1.39 - return len(value)
1.40 - else:
1.41 - return 0
1.42 -
1.43 - def get_initial_value(self, size):
1.44 - if size:
1.45 - return [0] * size
1.46 - else:
1.47 - return 0
1.48 -
1.49 class FileWriter(File):
1.50
1.51 "Writing basic data types to files."
1.52 @@ -77,18 +60,26 @@
1.53 def tell(self):
1.54 return self.f.tell() + len(self.data)
1.55
1.56 + def begin_record(self):
1.57 + pass
1.58 +
1.59 + def end_record(self):
1.60 + vint_to_array(len(self.record), self.data)
1.61 + self.data += self.record
1.62 + self.record = array('B')
1.63 +
1.64 def write_number(self, number):
1.65
1.66 "Write 'number' to the file using a variable length encoding."
1.67
1.68 - vint_to_array(number, self.data)
1.69 + vint_to_array(number, self.record)
1.70
1.71 def write_numbers(self, numbers):
1.72
1.73 "Write 'numbers' to the file using a variable length encoding."
1.74
1.75 for number in numbers:
1.76 - vint_to_array(number, self.data)
1.77 + vint_to_array(number, self.record)
1.78
1.79 def write_string(self, s, compress=0):
1.80
1.81 @@ -121,120 +112,122 @@
1.82 # Write the length of the data before the data itself.
1.83
1.84 length = len(s)
1.85 - self.data.fromstring("".join([flag, vint(length), s]))
1.86 + self.record.fromstring("".join([flag, vint(length), s]))
1.87 +
1.88 + def write_sequence_value(self, value):
1.89 + sequence_to_array(value, self.record)
1.90 +
1.91 + def write_sequence_values(self, values):
1.92 + vint_to_array(len(values), self.record)
1.93 + for value in values:
1.94 + self.write_sequence_value(value)
1.95
1.96 - def write_sequence(self, value, last, size, monotonic=1):
1.97 - if size:
1.98 - emit_delta = 1
1.99 - for v, l in map(None, value, last)[:size]:
1.100 - if v is None:
1.101 - v = l
1.102 - if monotonic or emit_delta:
1.103 - v_out = v - l
1.104 - if emit_delta and v_out != 0:
1.105 - emit_delta = 0
1.106 - else:
1.107 - v_out = v + 1
1.108 - vint_to_array(v_out, self.data)
1.109 - else:
1.110 - vint_to_array(value - last, self.data)
1.111 + def write_delta_sequence(self, values):
1.112 + convert_sequence(values, get_subtractor(values[0]))
1.113 + self.write_sequence_values(values)
1.114 +
1.115 + def write_monotonic_sequence(self, values):
1.116 + convert_sequence(values, get_monotonic_subtractor(values[0]))
1.117 + self.write_sequence_values(values)
1.118
1.119 - return value
1.120 + def flush(self):
1.121 + if self.f is not None:
1.122 + self.data.tofile(self.f)
1.123 + self.data = array('B')
1.124 +
1.125 + def close(self):
1.126 + self.flush()
1.127 + File.close(self)
1.128
1.129 class FileReader(File):
1.130
1.131 "Reading basic data types from files."
1.132
1.133 - def read_number(self):
1.134 + def begin_record(self):
1.135 + size = self.read_number_from_file()
1.136 + self.record.fromfile(self.f, size)
1.137 + self.start = 0
1.138 +
1.139 + def end_record(self):
1.140 + self.record = array('B')
1.141 +
1.142 + def read_number_from_file(self):
1.143
1.144 "Read a number from the file."
1.145
1.146 # Read each byte, adding it to the number.
1.147
1.148 f = self.f
1.149 - a = self.data
1.150 + a = array('B')
1.151 fromfile = a.fromfile
1.152
1.153 - try:
1.154 - fromfile(f, 1)
1.155 - csd = a[-1]
1.156 - if csd < 128:
1.157 - return csd
1.158 - else:
1.159 - while csd & 128:
1.160 - fromfile(f, 1)
1.161 - csd = a[-1]
1.162 - return vint_from_array(self.data)
1.163 - finally:
1.164 - self.data = array('B')
1.165 + fromfile(f, 1)
1.166 + csd = a[-1]
1.167 + if csd < 128:
1.168 + return csd
1.169 + else:
1.170 + while csd & 128:
1.171 + fromfile(f, 1)
1.172 + csd = a[-1]
1.173 + return vint_from_array(a)
1.174 +
1.175 + def read_number(self):
1.176 +
1.177 + "Read a number from the current record."
1.178 +
1.179 + n, self.start = vint_from_array_start(self.record, self.start)
1.180 + return n
1.181
1.182 def read_string(self, decompress=0):
1.183
1.184 """
1.185 - Read a string from the file, decompressing the stored data if
1.186 + Read a string from the current record, decompressing the stored data if
1.187 'decompress' is set to a true value.
1.188 """
1.189
1.190 # Decompress the data if requested.
1.191
1.192 if decompress:
1.193 - flag = self.f.read(1)
1.194 + flag = chr(self.record[self.start])
1.195 + self.start += 1
1.196 else:
1.197 flag = "-"
1.198
1.199 length = self.read_number()
1.200 + start = self.start
1.201 + self.start += length
1.202 + s = self.record[start:self.start].tostring()
1.203
1.204 - try:
1.205 - self.data.fromfile(self.f, length)
1.206 - s = self.data.tostring()
1.207 -
1.208 - # Perform decompression if applicable.
1.209 + # Perform decompression if applicable.
1.210
1.211 - if flag == "z":
1.212 - s = zlib.decompress(s)
1.213 + if flag == "z":
1.214 + s = zlib.decompress(s)
1.215
1.216 - # Convert strings to Unicode objects.
1.217 + # Convert strings to Unicode objects.
1.218
1.219 - return unicode(s, "utf-8")
1.220 + return unicode(s, "utf-8")
1.221
1.222 - finally:
1.223 - self.data = array('B')
1.224 + def read_sequence_value(self):
1.225 + value, self.start = sequence_from_array(self.record, self.start)
1.226 + return value
1.227
1.228 - def read_sequence(self, last, size, monotonic=1):
1.229 - if size:
1.230 - value = []
1.231 - if monotonic:
1.232 - for v in last:
1.233 - v_in = self.read_number()
1.234 - value.append(v + v_in)
1.235 - else:
1.236 - i = 0
1.237 - n = len(last)
1.238 - value = list(last)
1.239 -
1.240 - # Traverse a copy of the last value.
1.241 -
1.242 - while i < n:
1.243 - v_in = self.read_number()
1.244 + def read_sequences(self):
1.245 + values = []
1.246 + length = self.read_number()
1.247 + i = 0
1.248 + while i < length:
1.249 + values.append(self.read_sequence_value())
1.250 + i += 1
1.251 + return values
1.252
1.253 - # While zeros are read, retain the last value elements.
1.254 - # Otherwise, add the delta...
1.255 -
1.256 - if v_in != 0:
1.257 - value[i] += v_in
1.258 - i += 1
1.259 -
1.260 - # Then set absolute values for the remaining elements.
1.261 + def read_delta_sequence(self):
1.262 + values = self.read_sequences()
1.263 + convert_sequence(values, get_adder(values[0]))
1.264 + return values
1.265
1.266 - while i < n:
1.267 - value[i] = self.read_number() - 1
1.268 - i += 1
1.269 - break
1.270 -
1.271 - i += 1
1.272 -
1.273 - return tuple(value)
1.274 - else:
1.275 - return last + self.read_number()
1.276 + def read_monotonic_sequence(self):
1.277 + values = self.read_sequences()
1.278 + convert_sequence(values, get_monotonic_adder(values[0]))
1.279 + return values
1.280
1.281 # vim: tabstop=4 expandtab shiftwidth=4