paul@44 | 1 | #!/usr/bin/env python |
paul@44 | 2 | |
paul@44 | 3 | """ |
paul@44 | 4 | File access. |
paul@44 | 5 | |
paul@96 | 6 | Copyright (C) 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> |
paul@44 | 7 | |
paul@44 | 8 | This program is free software; you can redistribute it and/or modify it under |
paul@44 | 9 | the terms of the GNU General Public License as published by the Free Software |
paul@44 | 10 | Foundation; either version 3 of the License, or (at your option) any later |
paul@44 | 11 | version. |
paul@44 | 12 | |
paul@44 | 13 | This program is distributed in the hope that it will be useful, but WITHOUT ANY |
paul@44 | 14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A |
paul@44 | 15 | PARTICULAR PURPOSE. See the GNU General Public License for more details. |
paul@44 | 16 | |
paul@44 | 17 | You should have received a copy of the GNU General Public License along |
paul@44 | 18 | with this program. If not, see <http://www.gnu.org/licenses/>. |
paul@44 | 19 | """ |
paul@44 | 20 | |
paul@44 | 21 | from iixr.terms import * |
paul@76 | 22 | from os import listdir, remove, rename # partition manipulation |
paul@76 | 23 | from shutil import copy # index updating |
paul@44 | 24 | from os.path import join |
paul@44 | 25 | |
paul@76 | 26 | try: |
paul@76 | 27 | set |
paul@76 | 28 | except NameError: |
paul@76 | 29 | from sets import Set as set |
paul@76 | 30 | |
paul@44 | 31 | # Constants. |
paul@44 | 32 | |
paul@97 | 33 | TERM_FILENAMES = "terms", "term-index" |
paul@44 | 34 | |
paul@44 | 35 | # Utility functions. |
paul@44 | 36 | |
paul@76 | 37 | def get_partitions(pathname, prefix): |
paul@76 | 38 | |
paul@76 | 39 | """ |
paul@76 | 40 | Return a set of partition names for partitions residing at the given |
paul@76 | 41 | 'pathname' having the given 'prefix'. |
paul@76 | 42 | """ |
paul@76 | 43 | |
paul@76 | 44 | prefix_length = len(prefix) |
paul@76 | 45 | |
paul@76 | 46 | partitions = set() |
paul@76 | 47 | for filename in listdir(pathname): |
paul@76 | 48 | if filename.startswith(prefix): |
paul@96 | 49 | partition = filename[prefix_length:] |
paul@96 | 50 | if partition.isdigit(): |
paul@96 | 51 | partitions.add(int(partition)) |
paul@76 | 52 | return partitions |
paul@76 | 53 | |
paul@76 | 54 | def get_term_partitions(pathname): |
paul@76 | 55 | |
paul@76 | 56 | """ |
paul@76 | 57 | Return a set of term partition identifiers for partitions residing at the |
paul@76 | 58 | given 'pathname'. |
paul@76 | 59 | """ |
paul@76 | 60 | |
paul@76 | 61 | return get_partitions(pathname, "terms-") |
paul@76 | 62 | |
paul@96 | 63 | def get_next_partition(partitions): |
paul@96 | 64 | return max(partitions or [-1]) + 1 |
paul@96 | 65 | |
paul@97 | 66 | def get_writer(pathname, name, partition, cls): |
paul@97 | 67 | f = open(join(pathname, "%s-%s" % (name, partition)), "wb") |
paul@97 | 68 | return cls(f) |
paul@97 | 69 | |
paul@96 | 70 | def get_term_writer(pathname, partition): |
paul@76 | 71 | |
paul@76 | 72 | """ |
paul@96 | 73 | Return a term writer using files under the given 'pathname' labelled |
paul@96 | 74 | according to the given 'partition'. |
paul@44 | 75 | """ |
paul@44 | 76 | |
paul@97 | 77 | return get_writer(pathname, "terms", partition, TermWriter) |
paul@97 | 78 | |
paul@97 | 79 | def get_term_index_writer(pathname, partition): |
paul@97 | 80 | |
paul@97 | 81 | """ |
paul@97 | 82 | Return a term index writer using files under the given 'pathname' labelled |
paul@97 | 83 | according to the given 'partition'. |
paul@97 | 84 | """ |
paul@97 | 85 | |
paul@97 | 86 | return get_writer(pathname, "term-index", partition, TermIndexWriter) |
paul@44 | 87 | |
paul@96 | 88 | def get_reader(pathname, name, partition, cls): |
paul@96 | 89 | f = open(join(pathname, "%s-%s" % (name, partition)), "rb") |
paul@96 | 90 | return cls(f) |
paul@44 | 91 | |
paul@44 | 92 | def get_term_reader(pathname, partition): |
paul@44 | 93 | |
paul@44 | 94 | """ |
paul@96 | 95 | Return a term reader using files under the given 'pathname' labelled |
paul@96 | 96 | according to the given 'partition'. |
paul@96 | 97 | """ |
paul@96 | 98 | |
paul@96 | 99 | return get_reader(pathname, "terms", partition, TermIterator) |
paul@96 | 100 | |
paul@96 | 101 | def get_term_data_reader(pathname, partition): |
paul@96 | 102 | |
paul@96 | 103 | """ |
paul@96 | 104 | Return a term plus data reader using files under the given 'pathname' |
paul@44 | 105 | labelled according to the given 'partition'. |
paul@44 | 106 | """ |
paul@44 | 107 | |
paul@96 | 108 | return get_reader(pathname, "terms", partition, TermDataIterator) |
paul@44 | 109 | |
paul@97 | 110 | def get_term_index_reader(pathname, partition): |
paul@97 | 111 | |
paul@97 | 112 | """ |
paul@97 | 113 | Return a term index reader using files under the given 'pathname' labelled |
paul@97 | 114 | according to the given 'partition'. |
paul@97 | 115 | """ |
paul@97 | 116 | |
paul@97 | 117 | return get_reader(pathname, "term-index", partition, TermIndexIterator) |
paul@97 | 118 | |
paul@97 | 119 | def get_combined_term_reader(pathname, partition): |
paul@97 | 120 | |
paul@97 | 121 | """ |
paul@97 | 122 | Return a combined term reader using files under the given 'pathname' |
paul@97 | 123 | labelled according to the given 'partition'. |
paul@97 | 124 | """ |
paul@97 | 125 | |
paul@97 | 126 | return CombinedIterator(get_term_reader(pathname, partition), get_term_index_reader(pathname, partition)) |
paul@97 | 127 | |
paul@58 | 128 | # Renaming. |
paul@58 | 129 | |
paul@44 | 130 | def rename_files(pathname, names, from_partition, to_partition): |
paul@44 | 131 | for name in names: |
paul@97 | 132 | try: |
paul@97 | 133 | rename(join(pathname, "%s-%s" % (name, from_partition)), join(pathname, "%s-%s" % (name, to_partition))) |
paul@97 | 134 | except OSError: |
paul@97 | 135 | pass |
paul@44 | 136 | |
paul@44 | 137 | def rename_term_files(pathname, from_partition, to_partition): |
paul@44 | 138 | rename_files(pathname, TERM_FILENAMES, from_partition, to_partition) |
paul@44 | 139 | |
paul@58 | 140 | # Removal/deletion. |
paul@58 | 141 | |
paul@44 | 142 | def remove_files(pathname, names, partition): |
paul@44 | 143 | for name in names: |
paul@97 | 144 | try: |
paul@97 | 145 | remove(join(pathname, "%s-%s" % (name, partition))) |
paul@97 | 146 | except OSError: |
paul@97 | 147 | pass |
paul@44 | 148 | |
paul@44 | 149 | def remove_term_files(pathname, partition): |
paul@44 | 150 | remove_files(pathname, TERM_FILENAMES, partition) |
paul@44 | 151 | |
paul@58 | 152 | # Copying. |
paul@58 | 153 | |
paul@58 | 154 | def copy_files(source, names, partition, destination, suffix): |
paul@58 | 155 | for name in names: |
paul@58 | 156 | filename = "%s-%s" % (name, partition) |
paul@97 | 157 | try: |
paul@97 | 158 | copy(join(source, filename), join(destination, filename + suffix)) |
paul@97 | 159 | except OSError: |
paul@97 | 160 | pass |
paul@58 | 161 | |
paul@58 | 162 | def copy_term_files(source, partition, destination, suffix): |
paul@58 | 163 | copy_files(source, TERM_FILENAMES, partition, destination, suffix) |
paul@58 | 164 | |
paul@44 | 165 | # vim: tabstop=4 expandtab shiftwidth=4 |