paul@44 | 1 | #!/usr/bin/env python |
paul@44 | 2 | |
paul@44 | 3 | """ |
paul@44 | 4 | Specific classes for storing term information. |
paul@44 | 5 | |
paul@89 | 6 | Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> |
paul@44 | 7 | |
paul@44 | 8 | This program is free software; you can redistribute it and/or modify it under |
paul@44 | 9 | the terms of the GNU General Public License as published by the Free Software |
paul@44 | 10 | Foundation; either version 3 of the License, or (at your option) any later |
paul@44 | 11 | version. |
paul@44 | 12 | |
paul@44 | 13 | This program is distributed in the hope that it will be useful, but WITHOUT ANY |
paul@44 | 14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A |
paul@44 | 15 | PARTICULAR PURPOSE. See the GNU General Public License for more details. |
paul@44 | 16 | |
paul@44 | 17 | You should have received a copy of the GNU General Public License along |
paul@44 | 18 | with this program. If not, see <http://www.gnu.org/licenses/>. |
paul@44 | 19 | """ |
paul@44 | 20 | |
paul@96 | 21 | from iixr.data import * |
paul@44 | 22 | from iixr.files import * |
paul@97 | 23 | from itermerge import itermerge |
paul@44 | 24 | from os.path import commonprefix # to find common string prefixes |
paul@97 | 25 | from bisect import bisect_right, insort_right |
paul@97 | 26 | import operator |
paul@44 | 27 | |
paul@44 | 28 | class TermWriter(FileWriter): |
paul@44 | 29 | |
paul@44 | 30 | "Writing term information to files." |
paul@44 | 31 | |
paul@96 | 32 | def begin(self, docnum_size, position_size): |
paul@96 | 33 | |
paul@96 | 34 | "Begin writing to the file." |
paul@96 | 35 | |
paul@96 | 36 | self.write_numbers((docnum_size, position_size)) |
paul@90 | 37 | self.end_record() |
paul@96 | 38 | |
paul@96 | 39 | self.data_start = self.tell() |
paul@96 | 40 | self.docnum_size = docnum_size |
paul@96 | 41 | self.position_size = position_size |
paul@96 | 42 | self.subtractor = get_subtractor(docnum_size) |
paul@44 | 43 | self.last_term = "" |
paul@44 | 44 | |
paul@96 | 45 | def write_terms(self, terms): |
paul@44 | 46 | |
paul@44 | 47 | """ |
paul@96 | 48 | Write the 'terms' to the term information file, with each term's details |
paul@96 | 49 | stored in a separate record. |
paul@44 | 50 | """ |
paul@44 | 51 | |
paul@96 | 52 | if hasattr(terms, "items"): |
paul@96 | 53 | terms = terms.items() |
paul@96 | 54 | terms.sort() |
paul@96 | 55 | |
paul@96 | 56 | for term, doc_positions in terms: |
paul@96 | 57 | if not doc_positions: |
paul@96 | 58 | continue |
paul@96 | 59 | |
paul@96 | 60 | if hasattr(doc_positions, "items"): |
paul@96 | 61 | doc_positions = doc_positions.items() |
paul@96 | 62 | |
paul@96 | 63 | docnum, positions = doc_positions[0] |
paul@96 | 64 | |
paul@96 | 65 | if not positions: |
paul@96 | 66 | continue |
paul@96 | 67 | |
paul@96 | 68 | # Start the writing, if appropriate. |
paul@96 | 69 | |
paul@96 | 70 | if self.data_start is None: |
paul@96 | 71 | self.begin(sizeof(docnum), sizeof(positions[0])) |
paul@96 | 72 | |
paul@96 | 73 | # Write each term and document positions. |
paul@96 | 74 | |
paul@96 | 75 | self.write_term(term, doc_positions) |
paul@96 | 76 | self.end_record() |
paul@96 | 77 | |
paul@96 | 78 | # Methods requiring an open record. |
paul@96 | 79 | |
paul@96 | 80 | def write_term(self, term, doc_positions): |
paul@96 | 81 | |
paul@96 | 82 | """ |
paul@96 | 83 | Write the given 'term', its document frequency (number of documents in |
paul@96 | 84 | which it appears), and 'doc_positions' to the term information file. |
paul@96 | 85 | """ |
paul@96 | 86 | |
paul@96 | 87 | self.write_term_only(term) |
paul@96 | 88 | |
paul@96 | 89 | # Write the document frequency and the term positions. |
paul@96 | 90 | |
paul@96 | 91 | self.write_positions(doc_positions) |
paul@96 | 92 | |
paul@96 | 93 | def write_term_plus_remaining(self, term, data): |
paul@96 | 94 | |
paul@96 | 95 | "Write the given 'term' and the document position 'data'." |
paul@96 | 96 | |
paul@96 | 97 | self.write_term_only(term) |
paul@96 | 98 | self.write_remaining(data) |
paul@96 | 99 | |
paul@96 | 100 | def write_term_only(self, term): |
paul@96 | 101 | |
paul@96 | 102 | "Write only the given 'term'." |
paul@96 | 103 | |
paul@75 | 104 | if term <= self.last_term: |
paul@75 | 105 | raise ValueError, "Term %r precedes the previous term %r." % (term, self.last_term) |
paul@75 | 106 | |
paul@44 | 107 | # Write the prefix length and term suffix. |
paul@44 | 108 | |
paul@44 | 109 | common = len(commonprefix([self.last_term, term])) |
paul@44 | 110 | suffix = term[common:] |
paul@44 | 111 | |
paul@44 | 112 | self.write_number(common) |
paul@44 | 113 | self.write_string(suffix) |
paul@44 | 114 | |
paul@96 | 115 | self.last_term = term |
paul@96 | 116 | |
paul@96 | 117 | def write_positions(self, doc_positions): |
paul@96 | 118 | |
paul@96 | 119 | "Write the given 'doc_positions' to the file." |
paul@96 | 120 | |
paul@96 | 121 | # Make sure that the positions are sorted. |
paul@96 | 122 | |
paul@96 | 123 | doc_positions.sort() |
paul@96 | 124 | |
paul@44 | 125 | # Write the document frequency. |
paul@44 | 126 | |
paul@96 | 127 | self.write_number(len(doc_positions)) |
paul@96 | 128 | |
paul@96 | 129 | last_docnum = None |
paul@96 | 130 | |
paul@96 | 131 | for docnum, positions in doc_positions: |
paul@96 | 132 | |
paul@96 | 133 | # Store the first document number as it is. |
paul@96 | 134 | |
paul@96 | 135 | if last_docnum is None: |
paul@96 | 136 | docnum_seq = docnum |
paul@96 | 137 | |
paul@96 | 138 | # Reject out-of-order documents. |
paul@96 | 139 | |
paul@96 | 140 | elif docnum < last_docnum: |
paul@96 | 141 | raise ValueError, "Document number %r is less than previous number %r." % (docnum, last_docnum) |
paul@44 | 142 | |
paul@96 | 143 | # Calculate an ongoing delta. |
paul@96 | 144 | |
paul@96 | 145 | else: |
paul@96 | 146 | docnum_seq = self.subtractor(docnum, last_docnum) |
paul@96 | 147 | |
paul@96 | 148 | # Write the document number and positions. |
paul@96 | 149 | |
paul@96 | 150 | self.write_sequence_value(docnum_seq, self.docnum_size) |
paul@96 | 151 | self.write_monotonic_sequence(positions, self.position_size) |
paul@96 | 152 | |
paul@96 | 153 | last_docnum = docnum |
paul@96 | 154 | |
paul@96 | 155 | # Write a terminating byte to indicate that no more document pages |
paul@96 | 156 | # exist. |
paul@96 | 157 | |
paul@96 | 158 | self.write_byte(0) |
paul@44 | 159 | |
paul@44 | 160 | class TermReader(FileReader): |
paul@44 | 161 | |
paul@44 | 162 | "Reading term information from files." |
paul@44 | 163 | |
paul@96 | 164 | def begin(self): |
paul@96 | 165 | |
paul@96 | 166 | "Begin reading from the file." |
paul@96 | 167 | |
paul@96 | 168 | self.begin_record() |
paul@96 | 169 | try: |
paul@96 | 170 | self.docnum_size, self.position_size = self.read_numbers(2) |
paul@96 | 171 | except EOFError: |
paul@96 | 172 | self.docnum_size, self.position_size = 0, 0 # NOTE: No positions! |
paul@96 | 173 | |
paul@96 | 174 | self.data_start = self.tell() |
paul@96 | 175 | self.adder = get_adder(self.docnum_size) |
paul@44 | 176 | self.last_term = "" |
paul@96 | 177 | |
paul@96 | 178 | def get_sizes(self): |
paul@96 | 179 | return self.docnum_size, self.position_size |
paul@96 | 180 | |
paul@96 | 181 | # Methods requiring an open record. |
paul@44 | 182 | |
paul@44 | 183 | def read_term(self): |
paul@44 | 184 | |
paul@96 | 185 | "Read a term and its document positions from the term information file." |
paul@96 | 186 | |
paul@96 | 187 | # Read the term. |
paul@96 | 188 | |
paul@96 | 189 | self.read_term_only() |
paul@96 | 190 | |
paul@96 | 191 | # Read the document frequency and the term positions. |
paul@96 | 192 | |
paul@96 | 193 | positions = self.read_positions() |
paul@96 | 194 | |
paul@96 | 195 | return self.last_term, positions |
paul@96 | 196 | |
paul@96 | 197 | def read_term_plus_remaining(self): |
paul@96 | 198 | |
paul@44 | 199 | """ |
paul@96 | 200 | Read a term and the unprocessed document position data. |
paul@44 | 201 | """ |
paul@44 | 202 | |
paul@96 | 203 | self.read_term_only() |
paul@96 | 204 | return self.last_term, self.read_remaining() |
paul@96 | 205 | |
paul@96 | 206 | def read_term_only(self): |
paul@96 | 207 | |
paul@96 | 208 | "Read a term only." |
paul@96 | 209 | |
paul@44 | 210 | # Read the prefix length and term suffix. |
paul@44 | 211 | |
paul@44 | 212 | common = self.read_number() |
paul@44 | 213 | suffix = self.read_string() |
paul@44 | 214 | |
paul@44 | 215 | self.last_term = self.last_term[:common] + suffix |
paul@96 | 216 | return self.last_term |
paul@44 | 217 | |
paul@96 | 218 | def read_positions(self): |
paul@44 | 219 | |
paul@96 | 220 | "Read document positions from the term information file." |
paul@44 | 221 | |
paul@96 | 222 | doc_positions = [] |
paul@44 | 223 | |
paul@96 | 224 | while 1: |
paul@44 | 225 | |
paul@96 | 226 | # Read the document frequency. |
paul@44 | 227 | |
paul@96 | 228 | npositions = self.read_number() |
paul@91 | 229 | |
paul@96 | 230 | last_docnum = None |
paul@96 | 231 | i = 0 |
paul@96 | 232 | while i < npositions: |
paul@44 | 233 | |
paul@96 | 234 | # Read the document number. |
paul@44 | 235 | |
paul@96 | 236 | docnum = self.read_sequence_value(self.docnum_size) |
paul@96 | 237 | if last_docnum is not None: |
paul@96 | 238 | docnum = self.adder(docnum, last_docnum) |
paul@44 | 239 | |
paul@96 | 240 | # Read the positions. |
paul@44 | 241 | |
paul@96 | 242 | positions = self.read_monotonic_sequence(self.position_size) |
paul@96 | 243 | doc_positions.append((docnum, positions)) |
paul@44 | 244 | |
paul@96 | 245 | last_docnum = docnum |
paul@96 | 246 | i += 1 |
paul@44 | 247 | |
paul@96 | 248 | # Read a terminating byte to discover whether more document pages |
paul@96 | 249 | # exist. |
paul@44 | 250 | |
paul@96 | 251 | if not self.read_byte(): |
paul@96 | 252 | break |
paul@44 | 253 | |
paul@96 | 254 | return doc_positions |
paul@81 | 255 | |
paul@97 | 256 | # Indexes covering the information files. |
paul@97 | 257 | |
paul@97 | 258 | class TermIndexWriter(FileWriter): |
paul@97 | 259 | |
paul@97 | 260 | "Writing term index information to files." |
paul@97 | 261 | |
paul@97 | 262 | def begin(self): |
paul@97 | 263 | |
paul@97 | 264 | "Begin writing to the file." |
paul@97 | 265 | |
paul@97 | 266 | self.data_start = self.tell() |
paul@97 | 267 | self.last_term = "" |
paul@97 | 268 | self.last_offset = 0 |
paul@97 | 269 | |
paul@97 | 270 | def write_term(self, term, offset): |
paul@97 | 271 | |
paul@97 | 272 | "Write the given 'term' and 'offset'." |
paul@97 | 273 | |
paul@97 | 274 | if term <= self.last_term: |
paul@97 | 275 | raise ValueError, "Term %r precedes the previous term %r." % (term, self.last_term) |
paul@97 | 276 | |
paul@97 | 277 | # Write the prefix length and term suffix. |
paul@97 | 278 | |
paul@97 | 279 | common = len(commonprefix([self.last_term, term])) |
paul@97 | 280 | suffix = term[common:] |
paul@97 | 281 | |
paul@97 | 282 | self.write_number(common) |
paul@97 | 283 | self.write_string(suffix) |
paul@97 | 284 | |
paul@97 | 285 | # Write the offset delta. |
paul@97 | 286 | |
paul@97 | 287 | self.write_number(offset - self.last_offset) |
paul@97 | 288 | |
paul@97 | 289 | self.last_term = term |
paul@97 | 290 | self.last_offset = offset |
paul@97 | 291 | |
paul@97 | 292 | class TermIndexReader(FileReader): |
paul@58 | 293 | |
paul@97 | 294 | "Reading term index information to files." |
paul@97 | 295 | |
paul@97 | 296 | def begin(self): |
paul@97 | 297 | |
paul@97 | 298 | "Begin reading from the file." |
paul@97 | 299 | |
paul@97 | 300 | self.data_start = self.tell() |
paul@97 | 301 | self.last_term = "" |
paul@97 | 302 | self.last_offset = 0 |
paul@97 | 303 | |
paul@97 | 304 | def read_term(self): |
paul@97 | 305 | |
paul@97 | 306 | "Read a term and an offset from the file." |
paul@97 | 307 | |
paul@97 | 308 | # Read the prefix length and term suffix. |
paul@97 | 309 | |
paul@97 | 310 | common = self.read_number() |
paul@97 | 311 | suffix = self.read_string() |
paul@97 | 312 | |
paul@97 | 313 | self.last_term = self.last_term[:common] + suffix |
paul@97 | 314 | |
paul@97 | 315 | # Read the offset delta. |
paul@97 | 316 | |
paul@97 | 317 | self.last_offset += self.read_number() |
paul@97 | 318 | return self.last_term, self.last_offset |
paul@97 | 319 | |
paul@97 | 320 | # Iterator support classes. |
paul@97 | 321 | |
paul@97 | 322 | class Iterator: |
paul@97 | 323 | |
paul@97 | 324 | "Common iterator support." |
paul@97 | 325 | |
paul@99 | 326 | def __init__(self): |
paul@99 | 327 | |
paul@99 | 328 | "Cache the last term and positions." |
paul@99 | 329 | |
paul@99 | 330 | self.last_term_returned = None |
paul@99 | 331 | self.last_positions_returned = None |
paul@99 | 332 | |
paul@97 | 333 | def go_to_term(self, term): |
paul@99 | 334 | if term == self.last_term_returned: |
paul@99 | 335 | return self.last_term_returned, self.last_positions_returned |
paul@99 | 336 | |
paul@97 | 337 | t, dp = self.next() |
paul@97 | 338 | while t < term: |
paul@97 | 339 | t, dp = self.next() |
paul@99 | 340 | self.last_term_returned, self.last_positions_returned = t, dp |
paul@97 | 341 | return t, dp |
paul@44 | 342 | |
paul@44 | 343 | def __iter__(self): |
paul@44 | 344 | return self |
paul@44 | 345 | |
paul@99 | 346 | def next(self): |
paul@99 | 347 | self.last_term_returned, self.last_positions_returned = t = self._next() |
paul@99 | 348 | return t |
paul@99 | 349 | |
paul@97 | 350 | # External reading classes. |
paul@97 | 351 | |
paul@97 | 352 | class TermIterator(TermReader, Iterator): |
paul@97 | 353 | |
paul@97 | 354 | "An iterator over terms and positions read from a file." |
paul@97 | 355 | |
paul@99 | 356 | def __init__(self, f): |
paul@99 | 357 | TermReader.__init__(self, f) |
paul@99 | 358 | Iterator.__init__(self) |
paul@99 | 359 | |
paul@99 | 360 | def _next(self): |
paul@44 | 361 | try: |
paul@96 | 362 | self.begin_record() |
paul@44 | 363 | return self.read_term() |
paul@44 | 364 | except EOFError: |
paul@44 | 365 | raise StopIteration |
paul@44 | 366 | |
paul@97 | 367 | class TermDataIterator(TermReader, Iterator): |
paul@90 | 368 | |
paul@96 | 369 | "An iterator over terms and unprocessed document positions data." |
paul@90 | 370 | |
paul@99 | 371 | def __init__(self, f): |
paul@99 | 372 | TermReader.__init__(self, f) |
paul@99 | 373 | Iterator.__init__(self) |
paul@72 | 374 | |
paul@99 | 375 | def _next(self): |
paul@44 | 376 | try: |
paul@96 | 377 | self.begin_record() |
paul@96 | 378 | return self.read_term_plus_remaining() |
paul@44 | 379 | except EOFError: |
paul@96 | 380 | raise StopIteration |
paul@44 | 381 | |
paul@97 | 382 | class TermIndexIterator(TermIndexReader): |
paul@97 | 383 | |
paul@97 | 384 | "An iterator over terms and offsets read from a file." |
paul@97 | 385 | |
paul@97 | 386 | def __iter__(self): |
paul@97 | 387 | return self |
paul@97 | 388 | |
paul@97 | 389 | def next(self): |
paul@97 | 390 | try: |
paul@97 | 391 | self.begin_record() |
paul@97 | 392 | return self.read_term() |
paul@97 | 393 | except EOFError: |
paul@97 | 394 | raise StopIteration |
paul@97 | 395 | |
paul@99 | 396 | class CombinedIterator(Iterator): |
paul@97 | 397 | |
paul@97 | 398 | "An iterator providing index and information file access." |
paul@97 | 399 | |
paul@97 | 400 | def __init__(self, reader, index_reader): |
paul@99 | 401 | Iterator.__init__(self) |
paul@97 | 402 | self.reader = reader |
paul@97 | 403 | self.index_reader = index_reader |
paul@97 | 404 | self.records = list(index_reader) |
paul@98 | 405 | self.terms = [t for t, dp in self.records] |
paul@98 | 406 | |
paul@97 | 407 | def go_to_term(self, term): |
paul@97 | 408 | |
paul@98 | 409 | """ |
paul@98 | 410 | Return the 'term' and positions or nearest following term and positions. |
paul@98 | 411 | """ |
paul@98 | 412 | |
paul@99 | 413 | if self.last_term_returned == term: |
paul@99 | 414 | return self.last_term_returned, self.last_positions_returned |
paul@98 | 415 | |
paul@97 | 416 | # Get the record providing a term less than or equal to the requested |
paul@97 | 417 | # term, getting the first entry if no such records exist. |
paul@97 | 418 | |
paul@98 | 419 | after = bisect_right(self.terms, term) |
paul@98 | 420 | before = max(0, after - 1) |
paul@98 | 421 | |
paul@98 | 422 | t, offset = self.records[before] |
paul@98 | 423 | terms_after = self.terms[after:] |
paul@97 | 424 | |
paul@97 | 425 | # Seek to the corresponding record in the information file. |
paul@98 | 426 | # Only do this if the term is more quickly reached by seeking. |
paul@97 | 427 | |
paul@99 | 428 | if term <= t or self.last_term_returned is None or term <= self.last_term_returned or \ |
paul@99 | 429 | self.last_term_returned < t or terms_after and terms_after[0] <= self.last_term_returned: |
paul@98 | 430 | |
paul@98 | 431 | self.reader.seek(offset) |
paul@99 | 432 | self.reader.last_term = t |
paul@97 | 433 | |
paul@97 | 434 | # Where the found term is equal or greater, just read the positions for |
paul@97 | 435 | # the index entry. |
paul@97 | 436 | |
paul@97 | 437 | if t >= term: |
paul@97 | 438 | |
paul@97 | 439 | # Skip the term information, overwrite the reader's state, and get |
paul@97 | 440 | # the positions. |
paul@97 | 441 | |
paul@97 | 442 | self.reader.begin_record() |
paul@97 | 443 | self.reader.read_term_only() |
paul@97 | 444 | self.reader.last_term = t |
paul@97 | 445 | |
paul@99 | 446 | self.last_term_returned, self.last_positions_returned = t, self.reader.read_positions() |
paul@99 | 447 | return self.last_term_returned, self.last_positions_returned |
paul@97 | 448 | |
paul@97 | 449 | # Where the found term is less, use the information file to find the |
paul@97 | 450 | # term or the one after. |
paul@97 | 451 | |
paul@97 | 452 | else: |
paul@97 | 453 | |
paul@97 | 454 | # Overwrite the reader's state, then scan for the term. |
paul@97 | 455 | |
paul@97 | 456 | t, dp = self.reader.next() |
paul@97 | 457 | while t < term: |
paul@97 | 458 | t, dp = self.reader.next() |
paul@97 | 459 | |
paul@99 | 460 | self.last_term_returned, self.last_positions_returned = t, dp |
paul@97 | 461 | return t, dp |
paul@97 | 462 | |
paul@99 | 463 | def _next(self): |
paul@99 | 464 | return self.reader.next() |
paul@97 | 465 | |
paul@97 | 466 | def close(self): |
paul@97 | 467 | if self.reader is not None: |
paul@97 | 468 | self.reader.close() |
paul@97 | 469 | self.reader = None |
paul@97 | 470 | if self.index_reader is not None: |
paul@97 | 471 | self.index_reader.close() |
paul@97 | 472 | self.index_reader = None |
paul@97 | 473 | |
paul@97 | 474 | class MultipleReader(itermerge): |
paul@97 | 475 | |
paul@97 | 476 | "Accessing many term readers at once." |
paul@97 | 477 | |
paul@97 | 478 | def __init__(self, readers, combine=None): |
paul@97 | 479 | |
paul@97 | 480 | """ |
paul@97 | 481 | Initialise a master index reader using underlying 'readers' and a |
paul@97 | 482 | 'combine' function which knows how to combine position information from |
paul@97 | 483 | different sources. |
paul@97 | 484 | """ |
paul@97 | 485 | |
paul@97 | 486 | self.readers = readers |
paul@97 | 487 | self.combine = combine or operator.add |
paul@97 | 488 | |
paul@97 | 489 | # Initialise this object as an iterator over the readers. |
paul@97 | 490 | |
paul@97 | 491 | itermerge.__init__(self, self.readers) |
paul@97 | 492 | self.next_value = None |
paul@97 | 493 | |
paul@97 | 494 | def get_sizes(self): |
paul@97 | 495 | |
paul@97 | 496 | # Readers must have compatible sizes. |
paul@97 | 497 | |
paul@97 | 498 | if self.readers: |
paul@97 | 499 | return self.readers[0].get_sizes() |
paul@97 | 500 | else: |
paul@97 | 501 | return 0, 0 |
paul@97 | 502 | |
paul@97 | 503 | def go_to_term(self, term): |
paul@97 | 504 | self.iters = [] |
paul@97 | 505 | for reader in self.readers: |
paul@97 | 506 | try: |
paul@97 | 507 | insort_right(self.iters, (reader.go_to_term(term), reader.next)) |
paul@97 | 508 | except StopIteration: |
paul@97 | 509 | pass |
paul@97 | 510 | self.next_value = None |
paul@97 | 511 | return self.next() |
paul@97 | 512 | |
paul@97 | 513 | def next(self): |
paul@97 | 514 | if self.next_value is not None: |
paul@97 | 515 | term, positions = self.next_value |
paul@97 | 516 | else: |
paul@97 | 517 | term, positions = itermerge.next(self) |
paul@97 | 518 | |
paul@97 | 519 | # Look at the next item to see if it is has positions for the current |
paul@97 | 520 | # term. |
paul@97 | 521 | |
paul@97 | 522 | try: |
paul@97 | 523 | t, p = itermerge.next(self) |
paul@97 | 524 | while t == term: |
paul@97 | 525 | positions = self.combine(positions, p) |
paul@97 | 526 | t, p = itermerge.next(self) |
paul@97 | 527 | self.next_value = t, p |
paul@97 | 528 | |
paul@97 | 529 | # Where an item could not be fetched, cause future requests to fail. |
paul@97 | 530 | |
paul@97 | 531 | except StopIteration: |
paul@97 | 532 | self.next_value = None |
paul@97 | 533 | |
paul@97 | 534 | return term, positions |
paul@97 | 535 | |
paul@97 | 536 | def close(self): |
paul@97 | 537 | for reader in self.readers: |
paul@97 | 538 | reader.close() |
paul@97 | 539 | self.readers = [] |
paul@97 | 540 | |
paul@44 | 541 | # vim: tabstop=4 expandtab shiftwidth=4 |