1 #!/usr/bin/env python 2 3 """ 4 Generic file access. 5 6 Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 15 PARTICULAR PURPOSE. See the GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program. If not, see <http://www.gnu.org/licenses/>. 19 """ 20 21 from iixr.data import * 22 from array import array 23 import zlib 24 25 # Classes. 26 27 class File: 28 29 "A basic file abstraction." 30 31 def __init__(self, f): 32 self.f = f 33 self.record = array('B') # record buffer 34 self.data_start = None 35 36 def begin(self): 37 38 """ 39 Initialise file-wide parameters. In writers, this method may require 40 parameters to be specified. In readers, the parameters may be read from 41 the file. 42 """ 43 44 self.data_start = 0 45 46 def tell(self): 47 # NOTE: Will not be accurate within the current record. 48 return self.f.tell() 49 50 def seek(self, offset): 51 self.f.seek(offset) 52 53 def rewind(self): 54 self.seek(self.data_start) 55 56 def close(self): 57 if self.f is not None: 58 self.f.close() 59 self.f = None 60 61 class FileWriter(File): 62 63 "Writing basic data types to files." 64 65 def begin_record(self): 66 pass 67 68 def end_record(self): 69 if self.record: 70 self.f.write(vint(len(self.record))) 71 self.record.tofile(self.f) 72 self.record = array('B') 73 74 def write_remaining(self, a): 75 76 "Write remaining data from the raw array 'a'." 77 78 self.record += a 79 80 def write_byte(self, b): 81 82 "Write the given byte 'b'." 83 84 self.record.append(b) 85 86 def write_number(self, number): 87 88 "Write 'number' to the file using a variable length encoding." 89 90 vint_to_array(number, self.record) 91 92 def write_numbers(self, numbers): 93 94 "Write 'numbers' to the file using a variable length encoding." 95 96 for number in numbers: 97 vint_to_array(number, self.record) 98 99 def write_string(self, s, compress=0): 100 101 """ 102 Write 's' to the file, recording its length and compressing the string 103 if 'compress' is set to a true value. 104 """ 105 106 # Convert Unicode objects to strings. 107 108 if isinstance(s, unicode): 109 s = s.encode("utf-8") 110 111 # Compress the string if requested. 112 113 if compress: 114 cs = zlib.compress(s) 115 116 # Take any shorter than the original. 117 118 if len(cs) < len(s): 119 flag = "z" 120 s = cs 121 else: 122 flag = "-" 123 124 else: 125 flag = "" 126 127 # Write the length of the data before the data itself. 128 129 length = len(s) 130 self.record.fromstring("".join([flag, vint(length), s])) 131 132 def write_sequence_value(self, value, size): 133 sequence_to_array(value, size, self.record) 134 135 def write_sequence_values(self, values, size): 136 vint_to_array(len(values), self.record) 137 for value in values: 138 self.write_sequence_value(value, size) 139 140 def write_delta_sequence(self, values, size): 141 self.write_sequence_values( 142 convert_sequence(values, get_subtractor(size), 1), 143 size) 144 145 def write_monotonic_sequence(self, values, size): 146 self.write_sequence_values( 147 convert_sequence(values, get_monotonic_subtractor(size), 1), 148 size) 149 150 def close(self): 151 self.end_record() 152 File.close(self) 153 154 class FileReader(File): 155 156 "Reading basic data types from files." 157 158 def __init__(self, f): 159 File.__init__(self, f) 160 self.begin() 161 162 def begin_record(self): 163 self.start = 0 164 self.record = array('B') 165 try: 166 size = self.read_number_from_file() 167 self.record.fromfile(self.f, size) 168 except EOFError: 169 pass 170 171 def end_record(self): 172 pass 173 174 def read_remaining(self): 175 176 "Read remaining data as a raw array." 177 178 return self.record[self.start:] 179 180 def read_byte(self): 181 182 "Read a byte from the record." 183 184 b = self.record[self.start] 185 self.start += 1 186 return b 187 188 def read_number_from_file(self): 189 190 "Read a number from the file." 191 192 # Read each byte, adding it to the number. 193 194 a = array('B') 195 a.fromfile(self.f, 1) 196 csd = a[-1] 197 if csd < 128: 198 return csd 199 else: 200 while csd & 128: 201 a.fromfile(self.f, 1) 202 csd = a[-1] 203 return vint_from_array(a) 204 205 def read_number(self): 206 207 "Read a number from the current record." 208 209 n, self.start = vint_from_array_start(self.record, self.start) 210 return n 211 212 def read_numbers(self, n): 213 l = [] 214 i = 0 215 while i < n: 216 l.append(self.read_number()) 217 i += 1 218 return l 219 220 def read_string(self, decompress=0): 221 222 """ 223 Read a string from the current record, decompressing the stored data if 224 'decompress' is set to a true value. 225 """ 226 227 # Decompress the data if requested. 228 229 if decompress: 230 flag = chr(self.record[self.start]) 231 self.start += 1 232 else: 233 flag = "-" 234 235 length = self.read_number() 236 start = self.start 237 self.start += length 238 s = self.record[start:self.start].tostring() 239 240 # Perform decompression if applicable. 241 242 if flag == "z": 243 s = zlib.decompress(s) 244 245 # Convert strings to Unicode objects. 246 247 return unicode(s, "utf-8") 248 249 def read_sequence_value(self, size): 250 value, self.start = sequence_from_array(self.record, size, self.start) 251 return value 252 253 def read_sequences(self, size): 254 values = [] 255 length = self.read_number() 256 i = 0 257 while i < length: 258 values.append(self.read_sequence_value(size)) 259 i += 1 260 return values 261 262 def read_delta_sequence(self, size): 263 return convert_sequence(self.read_sequences(size), get_adder(size), 0) 264 265 def read_monotonic_sequence(self, size): 266 return convert_sequence(self.read_sequences(size), get_monotonic_adder(size), 0) 267 268 # vim: tabstop=4 expandtab shiftwidth=4