1 #!/usr/bin/env python 2 3 """ 4 Generic file access. 5 6 Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 15 PARTICULAR PURPOSE. See the GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program. If not, see <http://www.gnu.org/licenses/>. 19 """ 20 21 from iixr.data import * 22 from array import array 23 import zlib 24 25 # Constants. 26 27 class File: 28 29 "A basic file abstraction." 30 31 def __init__(self, f): 32 self.f = f 33 self.data = array('B') # master buffer 34 self.record = array('B') # record buffer 35 self.reset() 36 37 def reset(self): 38 39 "To be used to reset the state of the reader or writer between records." 40 41 pass 42 43 def seek(self, offset): 44 self.f.seek(offset) 45 self.reset() 46 47 def rewind(self): 48 self.f.seek(0) 49 self.reset() 50 51 def close(self): 52 if self.f is not None: 53 self.f.close() 54 self.f = None 55 56 class FileWriter(File): 57 58 "Writing basic data types to files." 59 60 def tell(self): 61 return self.f.tell() + len(self.data) 62 63 def begin_record(self): 64 pass 65 66 def end_record(self): 67 vint_to_array(len(self.record), self.data) 68 self.data += self.record 69 self.record = array('B') 70 71 def write_number(self, number): 72 73 "Write 'number' to the file using a variable length encoding." 74 75 vint_to_array(number, self.record) 76 77 def write_numbers(self, numbers): 78 79 "Write 'numbers' to the file using a variable length encoding." 80 81 for number in numbers: 82 vint_to_array(number, self.record) 83 84 def write_string(self, s, compress=0): 85 86 """ 87 Write 's' to the file, recording its length and compressing the string 88 if 'compress' is set to a true value. 89 """ 90 91 # Convert Unicode objects to strings. 92 93 if isinstance(s, unicode): 94 s = s.encode("utf-8") 95 96 # Compress the string if requested. 97 98 if compress: 99 cs = zlib.compress(s) 100 101 # Take any shorter than the original. 102 103 if len(cs) < len(s): 104 flag = "z" 105 s = cs 106 else: 107 flag = "-" 108 109 else: 110 flag = "" 111 112 # Write the length of the data before the data itself. 113 114 length = len(s) 115 self.record.fromstring("".join([flag, vint(length), s])) 116 117 def write_sequence_value(self, value): 118 sequence_to_array(value, self.record) 119 120 def write_sequence_values(self, values): 121 vint_to_array(len(values), self.record) 122 for value in values: 123 self.write_sequence_value(value) 124 125 def write_delta_sequence(self, values): 126 convert_sequence(values, get_subtractor(values[0])) 127 self.write_sequence_values(values) 128 129 def write_monotonic_sequence(self, values): 130 convert_sequence(values, get_monotonic_subtractor(values[0])) 131 self.write_sequence_values(values) 132 133 def flush(self): 134 if self.f is not None: 135 self.data.tofile(self.f) 136 self.data = array('B') 137 138 def close(self): 139 self.flush() 140 File.close(self) 141 142 class FileReader(File): 143 144 "Reading basic data types from files." 145 146 def begin_record(self): 147 size = self.read_number_from_file() 148 self.record.fromfile(self.f, size) 149 self.start = 0 150 151 def end_record(self): 152 self.record = array('B') 153 154 def read_number_from_file(self): 155 156 "Read a number from the file." 157 158 # Read each byte, adding it to the number. 159 160 f = self.f 161 a = array('B') 162 fromfile = a.fromfile 163 164 fromfile(f, 1) 165 csd = a[-1] 166 if csd < 128: 167 return csd 168 else: 169 while csd & 128: 170 fromfile(f, 1) 171 csd = a[-1] 172 return vint_from_array(a) 173 174 def read_number(self): 175 176 "Read a number from the current record." 177 178 n, self.start = vint_from_array_start(self.record, self.start) 179 return n 180 181 def read_string(self, decompress=0): 182 183 """ 184 Read a string from the current record, decompressing the stored data if 185 'decompress' is set to a true value. 186 """ 187 188 # Decompress the data if requested. 189 190 if decompress: 191 flag = chr(self.record[self.start]) 192 self.start += 1 193 else: 194 flag = "-" 195 196 length = self.read_number() 197 start = self.start 198 self.start += length 199 s = self.record[start:self.start].tostring() 200 201 # Perform decompression if applicable. 202 203 if flag == "z": 204 s = zlib.decompress(s) 205 206 # Convert strings to Unicode objects. 207 208 return unicode(s, "utf-8") 209 210 def read_sequence_value(self): 211 value, self.start = sequence_from_array(self.record, self.start) 212 return value 213 214 def read_sequences(self): 215 values = [] 216 length = self.read_number() 217 i = 0 218 while i < length: 219 values.append(self.read_sequence_value()) 220 i += 1 221 return values 222 223 def read_delta_sequence(self): 224 values = self.read_sequences() 225 convert_sequence(values, get_adder(values[0])) 226 return values 227 228 def read_monotonic_sequence(self): 229 values = self.read_sequences() 230 convert_sequence(values, get_monotonic_adder(values[0])) 231 return values 232 233 # vim: tabstop=4 expandtab shiftwidth=4