1 #!/usr/bin/env python 2 3 """ 4 Generic file access. 5 6 Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 15 PARTICULAR PURPOSE. See the GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program. If not, see <http://www.gnu.org/licenses/>. 19 """ 20 21 from iixr.data import vint, vint_to_array, vint_from_array 22 from array import array 23 import zlib 24 25 # Constants. 26 27 class File: 28 29 "A basic file abstraction." 30 31 def __init__(self, f): 32 self.f = f 33 self.data = array('B') 34 self.reset() 35 36 def reset(self): 37 38 "To be used to reset the state of the reader or writer between records." 39 40 pass 41 42 def seek(self, offset): 43 self.f.seek(offset) 44 self.reset() 45 46 def rewind(self): 47 self.f.seek(0) 48 self.reset() 49 50 def flush(self): 51 if self.f is not None: 52 self.data.tofile(self.f) 53 self.data = array('B') 54 55 def close(self): 56 if self.f is not None: 57 self.data.tofile(self.f) 58 self.f.close() 59 self.f = None 60 61 def get_value_size(self, value): 62 if isinstance(value, (list, tuple)): 63 return len(value) 64 else: 65 return 0 66 67 def get_initial_value(self, size): 68 if size: 69 return [0] * size 70 else: 71 return 0 72 73 class FileWriter(File): 74 75 "Writing basic data types to files." 76 77 def tell(self): 78 return self.f.tell() + len(self.data) 79 80 def write_number(self, number): 81 82 "Write 'number' to the file using a variable length encoding." 83 84 vint_to_array(number, self.data) 85 86 def write_numbers(self, numbers): 87 88 "Write 'numbers' to the file using a variable length encoding." 89 90 for number in numbers: 91 vint_to_array(number, self.data) 92 93 def write_string(self, s, compress=0): 94 95 """ 96 Write 's' to the file, recording its length and compressing the string 97 if 'compress' is set to a true value. 98 """ 99 100 # Convert Unicode objects to strings. 101 102 if isinstance(s, unicode): 103 s = s.encode("utf-8") 104 105 # Compress the string if requested. 106 107 if compress: 108 cs = zlib.compress(s) 109 110 # Take any shorter than the original. 111 112 if len(cs) < len(s): 113 flag = "z" 114 s = cs 115 else: 116 flag = "-" 117 118 else: 119 flag = "" 120 121 # Write the length of the data before the data itself. 122 123 length = len(s) 124 self.data.fromstring("".join([flag, vint(length), s])) 125 126 def write_sequence(self, value, last, size, monotonic=1): 127 if size: 128 emit_delta = 1 129 for v, l in map(None, value, last)[:size]: 130 if v is None: 131 v = l 132 if monotonic or emit_delta: 133 v_out = v - l 134 if emit_delta and v_out != 0: 135 emit_delta = 0 136 else: 137 v_out = v + 1 138 vint_to_array(v_out, self.data) 139 else: 140 vint_to_array(value - last, self.data) 141 142 return value 143 144 class FileReader(File): 145 146 "Reading basic data types from files." 147 148 def read_number(self): 149 150 "Read a number from the file." 151 152 # Read each byte, adding it to the number. 153 154 f = self.f 155 a = self.data 156 fromfile = a.fromfile 157 158 try: 159 fromfile(f, 1) 160 csd = a[-1] 161 if csd < 128: 162 return csd 163 else: 164 while csd & 128: 165 fromfile(f, 1) 166 csd = a[-1] 167 return vint_from_array(self.data) 168 finally: 169 self.data = array('B') 170 171 def read_string(self, decompress=0): 172 173 """ 174 Read a string from the file, decompressing the stored data if 175 'decompress' is set to a true value. 176 """ 177 178 # Decompress the data if requested. 179 180 if decompress: 181 flag = self.f.read(1) 182 else: 183 flag = "-" 184 185 length = self.read_number() 186 187 try: 188 self.data.fromfile(self.f, length) 189 s = self.data.tostring() 190 191 # Perform decompression if applicable. 192 193 if flag == "z": 194 s = zlib.decompress(s) 195 196 # Convert strings to Unicode objects. 197 198 return unicode(s, "utf-8") 199 200 finally: 201 self.data = array('B') 202 203 def read_sequence(self, last, size, monotonic=1): 204 if size: 205 value = [] 206 if monotonic: 207 for v in last: 208 v_in = self.read_number() 209 value.append(v + v_in) 210 else: 211 i = 0 212 n = len(last) 213 value = list(last) 214 215 # Traverse a copy of the last value. 216 217 while i < n: 218 v_in = self.read_number() 219 220 # While zeros are read, retain the last value elements. 221 # Otherwise, add the delta... 222 223 if v_in != 0: 224 value[i] += v_in 225 i += 1 226 227 # Then set absolute values for the remaining elements. 228 229 while i < n: 230 value[i] = self.read_number() - 1 231 i += 1 232 break 233 234 i += 1 235 236 return tuple(value) 237 else: 238 return last + self.read_number() 239 240 # vim: tabstop=4 expandtab shiftwidth=4