paulb@498 | 1 | #!/usr/bin/env python |
paulb@498 | 2 | |
paulb@498 | 3 | """ |
paulb@498 | 4 | Directory repositories for WebStack. |
paulb@498 | 5 | |
paul@773 | 6 | Copyright (C) 2005, 2006, 2007, 2009 Paul Boddie <paul@boddie.org.uk> |
paulb@498 | 7 | |
paulb@498 | 8 | This library is free software; you can redistribute it and/or |
paulb@498 | 9 | modify it under the terms of the GNU Lesser General Public |
paulb@498 | 10 | License as published by the Free Software Foundation; either |
paulb@498 | 11 | version 2.1 of the License, or (at your option) any later version. |
paulb@498 | 12 | |
paulb@498 | 13 | This library is distributed in the hope that it will be useful, |
paulb@498 | 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
paulb@498 | 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
paulb@498 | 16 | Lesser General Public License for more details. |
paulb@498 | 17 | |
paulb@498 | 18 | You should have received a copy of the GNU Lesser General Public |
paulb@498 | 19 | License along with this library; if not, write to the Free Software |
paulb@498 | 20 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA |
paulb@498 | 21 | """ |
paulb@498 | 22 | |
paulb@498 | 23 | import os |
paulb@535 | 24 | import time |
paulb@638 | 25 | import glob |
paulb@498 | 26 | |
paulb@498 | 27 | class DirectoryRepository: |
paulb@498 | 28 | |
paulb@498 | 29 | "A directory repository providing session-like access to files." |
paulb@498 | 30 | |
paulb@520 | 31 | new_filename = "__new__" |
paulb@520 | 32 | |
paulb@511 | 33 | def __init__(self, path, fsencoding=None, delay=1): |
paulb@498 | 34 | |
paulb@498 | 35 | """ |
paulb@498 | 36 | Initialise the repository using the given 'path' to indicate the |
paulb@498 | 37 | location of the repository. If no such location exists in the filesystem |
paulb@498 | 38 | an attempt will be made to create the directory. |
paulb@498 | 39 | |
paulb@498 | 40 | The optional 'fsencoding' parameter can be used to assert a particular |
paulb@498 | 41 | character encoding used by the filesystem to represent filenames. By |
paulb@498 | 42 | default, the default encoding is detected (or Unicode objects are used |
paulb@498 | 43 | if appropriate). |
paulb@511 | 44 | |
paulb@511 | 45 | The optional 'delay' argument specifies the time in seconds between each |
paulb@511 | 46 | poll of an opened repository file when that file is found to be locked |
paulb@511 | 47 | for editing. |
paulb@498 | 48 | """ |
paulb@498 | 49 | |
paulb@520 | 50 | # Convert the path to an absolute path. |
paulb@520 | 51 | |
paulb@520 | 52 | self.path = os.path.abspath(path) |
paulb@520 | 53 | self.fsencoding = fsencoding |
paulb@520 | 54 | self.delay = delay |
paulb@520 | 55 | |
paulb@520 | 56 | # Create a directory and initialise it with a special file. |
paulb@520 | 57 | |
paulb@498 | 58 | if not os.path.exists(path): |
paul@765 | 59 | os.mkdir(path) |
paulb@642 | 60 | self.create_resource(self.full_path(self.new_filename)) |
paulb@498 | 61 | |
paulb@498 | 62 | # Guess the filesystem encoding. |
paulb@498 | 63 | |
paulb@498 | 64 | if fsencoding is None: |
paulb@498 | 65 | if os.path.supports_unicode_filenames: |
paulb@498 | 66 | self.fsencoding = None |
paulb@498 | 67 | else: |
paulb@498 | 68 | import locale |
paulb@566 | 69 | self.fsencoding = locale.getdefaultlocale()[1] or 'ascii' |
paulb@498 | 70 | |
paulb@498 | 71 | # Or override any guesses. |
paulb@498 | 72 | |
paulb@498 | 73 | else: |
paulb@566 | 74 | self.fsencoding = fsencoding or 'ascii' |
paulb@498 | 75 | |
paul@773 | 76 | def rollback(self): |
paul@773 | 77 | |
paul@773 | 78 | "A method used by the StoreSelector." |
paul@773 | 79 | |
paul@773 | 80 | pass |
paul@773 | 81 | |
paulb@498 | 82 | def _convert_name(self, name): |
paulb@520 | 83 | |
paulb@520 | 84 | "Convert the given 'name' to a plain string in the filesystem encoding." |
paulb@520 | 85 | |
paulb@498 | 86 | if self.fsencoding: |
paulb@498 | 87 | return name.encode(self.fsencoding) |
paulb@498 | 88 | else: |
paulb@498 | 89 | return name |
paulb@498 | 90 | |
paulb@498 | 91 | def _convert_fsname(self, name): |
paulb@520 | 92 | |
paulb@520 | 93 | """ |
paulb@520 | 94 | Convert the given 'name' as a plain string in the filesystem encoding to |
paulb@520 | 95 | a Unicode object. |
paulb@520 | 96 | """ |
paulb@520 | 97 | |
paulb@498 | 98 | if self.fsencoding: |
paulb@498 | 99 | return unicode(name, self.fsencoding) |
paulb@498 | 100 | else: |
paulb@498 | 101 | return name |
paulb@498 | 102 | |
paulb@498 | 103 | def keys(self): |
paulb@520 | 104 | |
paulb@520 | 105 | "Return the names of the files stored in the repository." |
paulb@520 | 106 | |
paulb@511 | 107 | # NOTE: Special names converted using a simple rule. |
paulb@511 | 108 | l = [] |
paulb@511 | 109 | for name in os.listdir(self.path): |
paulb@511 | 110 | if name.endswith(".edit"): |
paulb@526 | 111 | name = name[:-5] |
paulb@526 | 112 | if name != self.new_filename: |
paulb@511 | 113 | l.append(name) |
paulb@511 | 114 | return map(self._convert_fsname, l) |
paulb@498 | 115 | |
paulb@520 | 116 | def full_path(self, key, edit=0): |
paulb@520 | 117 | |
paulb@520 | 118 | """ |
paulb@520 | 119 | Return the full path to the 'key' in the filesystem. If the optional |
paulb@520 | 120 | 'edit' parameter is set to a true value, the returned path will refer to |
paulb@520 | 121 | the editable version of the file. |
paulb@520 | 122 | """ |
paulb@520 | 123 | |
paulb@520 | 124 | # NOTE: Special names converted using a simple rule. |
paulb@513 | 125 | path = os.path.abspath(os.path.join(self.path, self._convert_name(key))) |
paulb@520 | 126 | if edit: |
paulb@520 | 127 | path = path + ".edit" |
paulb@513 | 128 | if not path.startswith(self.path): |
paulb@513 | 129 | raise ValueError, key |
paulb@513 | 130 | else: |
paulb@513 | 131 | return path |
paulb@498 | 132 | |
paulb@520 | 133 | def edit_path(self, key): |
paulb@520 | 134 | |
paulb@520 | 135 | """ |
paulb@520 | 136 | Return the full path to the 'key' in the filesystem provided that the |
paulb@520 | 137 | file associated with the 'key' is locked for editing. |
paulb@520 | 138 | """ |
paulb@520 | 139 | |
paulb@520 | 140 | return self.full_path(key, edit=1) |
paulb@520 | 141 | |
paulb@503 | 142 | def has_key(self, key): |
paulb@520 | 143 | |
paulb@520 | 144 | """ |
paulb@520 | 145 | Return whether a file with the name specified by 'key is stored in the |
paulb@520 | 146 | repository. |
paulb@520 | 147 | """ |
paulb@520 | 148 | |
paulb@503 | 149 | return key in self.keys() |
paulb@503 | 150 | |
paulb@498 | 151 | # NOTE: Methods very similar to Helpers.Session.Wrapper. |
paulb@498 | 152 | |
paulb@498 | 153 | def items(self): |
paulb@520 | 154 | |
paulb@520 | 155 | """ |
paulb@520 | 156 | Return a list of (name, value) tuples for the files stored in the |
paulb@520 | 157 | repository. |
paulb@520 | 158 | """ |
paulb@520 | 159 | |
paulb@498 | 160 | results = [] |
paulb@498 | 161 | for key in self.keys(): |
paulb@498 | 162 | results.append((key, self[key])) |
paulb@498 | 163 | return results |
paulb@498 | 164 | |
paulb@498 | 165 | def values(self): |
paulb@520 | 166 | |
paulb@520 | 167 | "Return the contents of the files stored in the repository." |
paulb@520 | 168 | |
paulb@498 | 169 | results = [] |
paulb@498 | 170 | for key in self.keys(): |
paulb@498 | 171 | results.append(self[key]) |
paulb@498 | 172 | return results |
paulb@498 | 173 | |
paulb@520 | 174 | def lock(self, key, create=0, opener=None): |
paulb@513 | 175 | |
paulb@513 | 176 | """ |
paulb@513 | 177 | Lock the file associated with the given 'key'. If the optional 'create' |
paulb@513 | 178 | parameter is set to a true value (unlike the default), the file will be |
paulb@513 | 179 | created if it did not already exist; otherwise, a KeyError will be |
paulb@513 | 180 | raised. |
paulb@513 | 181 | |
paulb@520 | 182 | If the optional 'opener' parameter is specified, it will be used to |
paulb@520 | 183 | create any new file in the case where 'create' is set to a true value; |
paulb@520 | 184 | otherwise, the standard 'open' function will be used to create the file. |
paulb@520 | 185 | |
paulb@520 | 186 | Return the full path to the editable file. |
paulb@513 | 187 | """ |
paulb@513 | 188 | |
paulb@520 | 189 | path = self.full_path(key) |
paulb@520 | 190 | edit_path = self.edit_path(key) |
paulb@520 | 191 | |
paulb@520 | 192 | # Attempt to lock the file by renaming it. |
paulb@520 | 193 | # NOTE: This assumes that renaming is an atomic operation. |
paulb@520 | 194 | |
paulb@520 | 195 | if os.path.exists(path) or os.path.exists(edit_path): |
paulb@511 | 196 | while 1: |
paulb@511 | 197 | try: |
paulb@520 | 198 | os.rename(path, edit_path) |
paulb@511 | 199 | except OSError: |
paulb@511 | 200 | time.sleep(self.delay) |
paulb@511 | 201 | else: |
paulb@511 | 202 | break |
paulb@520 | 203 | |
paulb@520 | 204 | # Where a file does not exist, attempt to create a new file. |
paulb@520 | 205 | # Since file creation is probably not atomic, we use the renaming of a |
paulb@520 | 206 | # special file in an attempt to impose some kind of atomic "bottleneck". |
paulb@520 | 207 | |
paulb@513 | 208 | elif create: |
paulb@523 | 209 | |
paulb@523 | 210 | # NOTE: Avoid failure case where no __new__ file exists for some |
paulb@523 | 211 | # NOTE: reason. |
paulb@523 | 212 | |
paulb@523 | 213 | try: |
paulb@523 | 214 | self.lock(self.new_filename) |
paulb@523 | 215 | except KeyError: |
paulb@642 | 216 | self.create_resource(self.edit_path(self.new_filename)) |
paulb@523 | 217 | |
paulb@520 | 218 | try: |
paulb@642 | 219 | self.create_data(edit_path, opener) |
paulb@520 | 220 | finally: |
paulb@520 | 221 | self.unlock(self.new_filename) |
paulb@523 | 222 | |
paulb@523 | 223 | # Where no creation is requested, raise an exception. |
paulb@523 | 224 | |
paulb@511 | 225 | else: |
paulb@513 | 226 | raise KeyError, key |
paulb@520 | 227 | |
paulb@520 | 228 | return edit_path |
paulb@513 | 229 | |
paulb@513 | 230 | def unlock(self, key): |
paulb@513 | 231 | |
paulb@513 | 232 | """ |
paulb@513 | 233 | Unlock the file associated with the given 'key'. |
paulb@513 | 234 | |
paulb@513 | 235 | Important note: this method should be used in a finally clause in order |
paulb@513 | 236 | to avoid files being locked and never being unlocked by the same process |
paulb@513 | 237 | because an unhandled exception was raised. |
paulb@513 | 238 | """ |
paulb@513 | 239 | |
paulb@520 | 240 | path = self.full_path(key) |
paulb@520 | 241 | edit_path = self.edit_path(key) |
paul@772 | 242 | # NOTE: Add fsync-related stuff here due to ext4 and other filesystems? |
paulb@520 | 243 | os.rename(edit_path, path) |
paulb@498 | 244 | |
paulb@498 | 245 | def __delitem__(self, key): |
paulb@520 | 246 | |
paulb@520 | 247 | "Remove the file associated with the given 'key'." |
paulb@520 | 248 | |
paulb@645 | 249 | edit_path = self.lock(key) |
paulb@645 | 250 | self.remove_resource(edit_path) |
paulb@511 | 251 | |
paulb@511 | 252 | def __getitem__(self, key): |
paulb@520 | 253 | |
paulb@520 | 254 | "Return the contents of the file associated with the given 'key'." |
paulb@520 | 255 | |
paulb@520 | 256 | edit_path = self.lock(key, create=0) |
paulb@511 | 257 | try: |
paulb@642 | 258 | f = self.open_resource(edit_path, "rb") |
paulb@511 | 259 | s = "" |
paulb@511 | 260 | try: |
paulb@511 | 261 | s = f.read() |
paulb@511 | 262 | finally: |
paulb@511 | 263 | f.close() |
paulb@511 | 264 | finally: |
paulb@513 | 265 | self.unlock(key) |
paulb@511 | 266 | |
paulb@511 | 267 | return s |
paulb@498 | 268 | |
paulb@498 | 269 | def __setitem__(self, key, value): |
paulb@520 | 270 | |
paulb@520 | 271 | """ |
paulb@520 | 272 | Set the contents of the file associated with the given 'key' using the |
paulb@520 | 273 | given 'value'. |
paulb@520 | 274 | """ |
paulb@520 | 275 | |
paulb@520 | 276 | edit_path = self.lock(key, create=1) |
paulb@498 | 277 | try: |
paulb@642 | 278 | f = self.open_resource(edit_path, "wb") |
paulb@511 | 279 | try: |
paulb@511 | 280 | f.write(value) |
paulb@511 | 281 | finally: |
paulb@511 | 282 | f.close() |
paulb@498 | 283 | finally: |
paulb@513 | 284 | self.unlock(key) |
paulb@498 | 285 | |
paulb@642 | 286 | def create_resource(self, filename): |
paulb@642 | 287 | f = open(filename, "wb") |
paulb@642 | 288 | f.close() |
paulb@642 | 289 | |
paulb@642 | 290 | def create_data(self, edit_path, opener): |
paulb@642 | 291 | if opener is None: |
paulb@642 | 292 | f = open(edit_path, "wb") |
paulb@642 | 293 | f.close() |
paulb@642 | 294 | else: |
paulb@642 | 295 | f = opener(edit_path) |
paulb@642 | 296 | f.close() |
paulb@642 | 297 | |
paulb@642 | 298 | def remove_resource(self, edit_path): |
paulb@642 | 299 | os.remove(edit_path) |
paulb@642 | 300 | |
paulb@642 | 301 | def open_resource(self, edit_path, mode): |
paulb@642 | 302 | return open(edit_path, mode) |
paulb@642 | 303 | |
paulb@642 | 304 | class SessionDirectoryRepository(DirectoryRepository): |
paulb@642 | 305 | |
paulb@642 | 306 | def create_resource(self, filename): |
paulb@642 | 307 | os.mkdir(filename) |
paulb@642 | 308 | |
paulb@642 | 309 | def create_data(self, edit_path, opener): |
paulb@642 | 310 | os.mkdir(edit_path) |
paulb@642 | 311 | if opener is None: |
paulb@642 | 312 | f = open(os.path.join(edit_path, "data"), "wb") |
paulb@642 | 313 | f.close() |
paulb@642 | 314 | else: |
paulb@642 | 315 | f = opener(os.path.join(edit_path, "data")) |
paulb@642 | 316 | f.close() |
paulb@642 | 317 | |
paulb@642 | 318 | def remove_resource(self, edit_path): |
paulb@642 | 319 | for filename in glob.glob(os.path.join(edit_path, "*")): |
paulb@642 | 320 | os.remove(filename) |
paulb@642 | 321 | os.rmdir(edit_path) |
paulb@642 | 322 | |
paulb@642 | 323 | def open_resource(self, edit_path, mode): |
paulb@642 | 324 | return open(os.path.join(edit_path, "data"), mode) |
paulb@642 | 325 | |
paulb@498 | 326 | # vim: tabstop=4 expandtab shiftwidth=4 |