Login

Non-pickling locmem (in-process memory) cache backend

Author:
akaihola
Posted:
March 18, 2011
Language:
Python
Version:
Not specified
Tags:
cache pickle backend locmem memory process
Score:
0 (after 0 ratings)

You can use this cache backend to cache data in-process and avoid the overhead of pickling. Make absolutely sure you don't modify any data you've stored to or retrieved from the cache. Make deep copies instead if necessary.

The backend is basically identical to Django's stock locmem cache (as of r15852 - after 1.3rc1) with pickling removed. It has been tested with that specific Django revision, so basically it's >=1.3 compatible.

See Django ticket #6124 for some background information.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
"Thread-safe in-memory cache backend which doesn't use pickling."

# Example usage as a secondary cache:
# 
# ## settings.py
# CACHES = {'default': {<your default cache settings>},
#           'inprocess': {'BACKEND': 'locmemnopickle.LocMemNoPickleCache'
#                         'KEY_PREFIX': 'my-prefix-',
#                         'TIMEOUT': 30}}
#
# ## myapp/views.py
# from django.core.cache import get_cache
# def myview(request):
#     cache = get_cache('inprocess')
#     key = generate_key(request)
#     data = cache.get(key)
#     if data is None:
#         data = generate_data(request)
#         cache.set(key, data)
#     return generate_response(data)

import time

from django.core.cache.backends.base import BaseCache
from django.utils.synch import RWLock

# Global in-memory store of cache data. Keyed by name, to provide
# multiple named local memory caches.
_caches = {}
_expire_info = {}
_locks = {}

class LocMemNoPickleCache(BaseCache):
    def __init__(self, name, params):
        BaseCache.__init__(self, params)
        global _caches, _expire_info, _locks
        self._cache = _caches.setdefault(name, {})
        self._expire_info = _expire_info.setdefault(name, {})
        self._lock = _locks.setdefault(name, RWLock())

    def add(self, key, value, timeout=None, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._lock.writer_enters()
        try:
            exp = self._expire_info.get(key)
            if exp is None or exp <= time.time():
                self._set(key, value, timeout)
                return True
            return False
        finally:
            self._lock.writer_leaves()

    def get(self, key, default=None, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._lock.reader_enters()
        try:
            exp = self._expire_info.get(key)
            if exp is None:
                return default
            elif exp > time.time():
                return self._cache[key]
        finally:
            self._lock.reader_leaves()
        self._lock.writer_enters()
        try:
            try:
                del self._cache[key]
                del self._expire_info[key]
            except KeyError:
                pass
            return default
        finally:
            self._lock.writer_leaves()

    def _set(self, key, value, timeout=None):
        if len(self._cache) >= self._max_entries:
            self._cull()
        if timeout is None:
            timeout = self.default_timeout
        self._cache[key] = value
        self._expire_info[key] = time.time() + timeout

    def set(self, key, value, timeout=None, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._lock.writer_enters()
        # Python 2.4 doesn't allow combined try-except-finally blocks.
        try:
            self._set(key, value, timeout)
        finally:
            self._lock.writer_leaves()

    def has_key(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._lock.reader_enters()
        try:
            exp = self._expire_info.get(key)
            if exp is None:
                return False
            elif exp > time.time():
                return True
        finally:
            self._lock.reader_leaves()

        self._lock.writer_enters()
        try:
            try:
                del self._cache[key]
                del self._expire_info[key]
            except KeyError:
                pass
            return False
        finally:
            self._lock.writer_leaves()

    def _cull(self):
        if self._cull_frequency == 0:
            self.clear()
        else:
            doomed = [k for (i, k) in enumerate(self._cache) if i % self._cull_frequency == 0]
            for k in doomed:
                self._delete(k)

    def _delete(self, key):
        try:
            del self._cache[key]
        except KeyError:
            pass
        try:
            del self._expire_info[key]
        except KeyError:
            pass

    def delete(self, key, version=None):
        key = self.make_key(key, version=version)
        self.validate_key(key)
        self._lock.writer_enters()
        try:
            self._delete(key)
        finally:
            self._lock.writer_leaves()

    def clear(self):
        self._cache.clear()
        self._expire_info.clear()

# For backwards compatibility
class CacheClass(LocMemNoPickleCache):
    pass

More like this

  1. Clear Django Cache by girasquid 6 years, 9 months ago
  2. Function decorator for caching function results in local memory by bikeshedder 2 years, 9 months ago
  3. JSON instead of pickle for memcached by diverman 3 years, 5 months ago
  4. Cached lookup model mixin by isagalaev 7 years, 5 months ago
  5. Improved Pickled Object Field by taavi223 5 years, 10 months ago

Comments

Please login first before commenting.