1 #!/usr/bin/env python 2 3 """ 4 Generic file access. 5 6 Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 15 PARTICULAR PURPOSE. See the GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program. If not, see <http://www.gnu.org/licenses/>. 19 """ 20 21 from iixr.data import vint, vint_to_array, vint_from_array 22 from array import array 23 import zlib 24 25 # Constants. 26 27 class File: 28 29 "A basic file abstraction." 30 31 def __init__(self, f): 32 self.f = f 33 self.reset() 34 35 def reset(self): 36 37 "To be used to reset the state of the reader or writer between records." 38 39 pass 40 41 def seek(self, offset): 42 self.f.seek(offset) 43 self.reset() 44 45 def rewind(self): 46 self.f.seek(0) 47 self.reset() 48 49 def close(self): 50 if self.f is not None: 51 self.f.close() 52 self.f = None 53 54 def get_value_size(self, value): 55 if isinstance(value, (list, tuple)): 56 return len(value) 57 else: 58 return 0 59 60 def get_initial_value(self, size): 61 if size: 62 return [0] * size 63 else: 64 return 0 65 66 class FileWriter(File): 67 68 "Writing basic data types to files." 69 70 def __init__(self, f): 71 File.__init__(self, f) 72 73 def write_number(self, number): 74 75 "Write 'number' to the file using a variable length encoding." 76 77 self.f.write(vint(number)) 78 79 def write_numbers(self, numbers): 80 81 "Write 'numbers' to the file using a variable length encoding." 82 83 output = array('B') 84 for number in numbers: 85 vint_to_array(number, output) 86 output.tofile(self.f) 87 88 def write_string(self, s, compress=0): 89 90 """ 91 Write 's' to the file, recording its length and compressing the string 92 if 'compress' is set to a true value. 93 """ 94 95 # Convert Unicode objects to strings. 96 97 if isinstance(s, unicode): 98 s = s.encode("utf-8") 99 100 # Compress the string if requested. 101 102 if compress: 103 cs = zlib.compress(s) 104 105 # Take any shorter than the original. 106 107 if len(cs) < len(s): 108 flag = "z" 109 s = cs 110 else: 111 flag = "-" 112 113 else: 114 flag = "" 115 116 # Write the length of the data before the data itself. 117 118 length = len(s) 119 self.f.write("".join([flag, vint(length), s])) 120 121 def write_sequence(self, output, value, last, size, monotonic=1): 122 if size: 123 emit_delta = 1 124 for v, l in map(None, value, last)[:size]: 125 if v is None: 126 v = l 127 if monotonic or emit_delta: 128 v_out = v - l 129 if emit_delta and v_out != 0: 130 emit_delta = 0 131 else: 132 v_out = v + 1 133 vint_to_array(v_out, output) 134 else: 135 vint_to_array(value - last, output) 136 137 return value 138 139 class FileReader(File): 140 141 "Reading basic data types from files." 142 143 def __init__(self, f): 144 File.__init__(self, f) 145 146 def read_number(self): 147 148 "Read a number from the file." 149 150 # Read each byte, adding it to the number. 151 152 a = array('B') 153 fromfile = a.fromfile 154 f = self.f 155 156 fromfile(f, 1) 157 csd = a[-1] 158 if csd < 128: 159 return csd 160 else: 161 while csd & 128: 162 fromfile(f, 1) 163 csd = a[-1] 164 return vint_from_array(a) 165 166 def read_string(self, decompress=0): 167 168 """ 169 Read a string from the file, decompressing the stored data if 170 'decompress' is set to a true value. 171 """ 172 173 read = self.f.read 174 175 # Decompress the data if requested. 176 177 if decompress: 178 flag = read(1) 179 else: 180 flag = "-" 181 182 length = self.read_number() 183 s = read(length) 184 185 # Perform decompression if applicable. 186 187 if flag == "z": 188 s = zlib.decompress(s) 189 190 # Convert strings to Unicode objects. 191 192 return unicode(s, "utf-8") 193 194 def read_sequence(self, last, size, monotonic=1): 195 if size: 196 value = [] 197 if monotonic: 198 for v in last: 199 v_in = self.read_number() 200 value.append(v + v_in) 201 else: 202 i = 0 203 n = len(last) 204 value = list(last) 205 206 # Traverse a copy of the last value. 207 208 while i < n: 209 v_in = self.read_number() 210 211 # While zeros are read, retain the last value elements. 212 # Otherwise, add the delta... 213 214 if v_in != 0: 215 value[i] += v_in 216 i += 1 217 218 # Then set absolute values for the remaining elements. 219 220 while i < n: 221 value[i] = self.read_number() - 1 222 i += 1 223 break 224 225 i += 1 226 227 return tuple(value) 228 else: 229 return last + self.read_number() 230 231 # vim: tabstop=4 expandtab shiftwidth=4