MoinSupport

Changeset

111:ed584d9fa250
2014-01-29 Paul Boddie raw files shortlog changelog graph Moved URL access in the acquisition of resources to a separate function.
MoinRemoteSupport.py (file)
     1.1 --- a/MoinRemoteSupport.py	Mon Jan 27 21:59:37 2014 +0100
     1.2 +++ b/MoinRemoteSupport.py	Wed Jan 29 12:42:55 2014 +0100
     1.3 @@ -2,7 +2,7 @@
     1.4  """
     1.5      MoinMoin - MoinRemoteSupport library
     1.6  
     1.7 -    @copyright: 2011, 2012, 2013 by Paul Boddie <paul@boddie.org.uk>
     1.8 +    @copyright: 2011, 2012, 2013, 2014 by Paul Boddie <paul@boddie.org.uk>
     1.9      @license: GNU GPL (v2 or later), see COPYING.txt for details.
    1.10  """
    1.11  
    1.12 @@ -11,7 +11,7 @@
    1.13  from MoinMoin import caching
    1.14  import urllib2, time
    1.15  
    1.16 -def getCachedResource(request, url, arena, scope, max_cache_age):
    1.17 +def getCachedResource(request, url, arena, scope, max_cache_age, reader=None):
    1.18  
    1.19      """
    1.20      Using the given 'request', return the resource data for the given 'url',
    1.21 @@ -19,6 +19,10 @@
    1.22      has already been downloaded. The 'max_cache_age' indicates the length in
    1.23      seconds that a cache entry remains valid.
    1.24  
    1.25 +    If the optional 'reader' object is given, it will be used to access the
    1.26 +    'url' and write the downloaded data to a cache entry. Otherwise, a standard
    1.27 +    URL reader will be used.
    1.28 +
    1.29      If the resource cannot be downloaded and cached, None is returned.
    1.30      Otherwise, the form of the data is as follows:
    1.31  
    1.32 @@ -29,6 +33,8 @@
    1.33      content-body
    1.34      """
    1.35  
    1.36 +    reader = reader or urlreader
    1.37 +
    1.38      # See if the URL is cached.
    1.39  
    1.40      cache_key = cache.key(request, content=url)
    1.41 @@ -50,18 +56,9 @@
    1.42          cache_entry.open(mode="w")
    1.43  
    1.44          try:
    1.45 -            f = urllib2.urlopen(url)
    1.46 -            try:
    1.47 -                cache_entry.write(url + "\n")
    1.48 -                cache_entry.write((f.headers.get("content-type") or "") + "\n")
    1.49 -                for key, value in f.headers.items():
    1.50 -                    if key.lower() != "content-type":
    1.51 -                        cache_entry.write("%s: %s\n" % (key, value))
    1.52 -                cache_entry.write("\n")
    1.53 -                cache_entry.write(f.read())
    1.54 -            finally:
    1.55 -                cache_entry.close()
    1.56 -                f.close()
    1.57 +            # Read from the source and write to the cache.
    1.58 +
    1.59 +            reader(url, cache_entry)
    1.60  
    1.61          # In case of an exception, return None.
    1.62  
    1.63 @@ -78,6 +75,23 @@
    1.64      finally:
    1.65          cache_entry.close()
    1.66  
    1.67 +def urlreader(url, cache_entry):
    1.68 +
    1.69 +    "Retrieve data from the given 'url', writing it to the 'cache_entry'."
    1.70 +
    1.71 +    f = urllib2.urlopen(url)
    1.72 +    try:
    1.73 +        cache_entry.write(url + "\n")
    1.74 +        cache_entry.write((f.headers.get("content-type") or "") + "\n")
    1.75 +        for key, value in f.headers.items():
    1.76 +            if key.lower() != "content-type":
    1.77 +                cache_entry.write("%s: %s\n" % (key, value))
    1.78 +        cache_entry.write("\n")
    1.79 +        cache_entry.write(f.read())
    1.80 +    finally:
    1.81 +        cache_entry.close()
    1.82 +        f.close()
    1.83 +
    1.84  def getCachedResourceMetadata(f):
    1.85  
    1.86      "Return a metadata dictionary for the given resource file-like object 'f'."