Login

CompressedTextField for Django 1.0+

Author:
bwhittington
Posted:
May 7, 2009
Language:
Python
Version:
1.0
Tags:
text model field compressed gzip
Score:
0 (after 0 ratings)

This snippet updates http://www.djangosnippets.org/snippets/383/ for Django 1.0+, and adds support for sqlite3. Original snippet text: A CompressedTextField to transparently save data gzipped in the database and uncompress at retrieval.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
from django.db import models
from django.utils.text import compress_string
from django.db.models.signals import post_init

def uncompress_string(s):
    '''helper function to reverse django.utils.text.compress_string'''
    import cStringIO, gzip
    try:
        zbuf = cStringIO.StringIO(s)
        zfile = gzip.GzipFile(fileobj=zbuf)
        ret = zfile.read()
        zfile.close()
    except:
        ret = s
    return ret


class CompressedTextField(models.TextField):
    '''transparently compress data before hitting the db and uncompress after fetching'''

    def get_db_prep_save(self, value):
        if value is not None:
            value = compress_string(value) 
        return models.TextField.get_db_prep_save(self, value)
 
    def _get_val_from_obj(self, obj):
        if obj:
            return uncompress_string(getattr(obj, self.attname))
        else:
            return self.get_default() 
    
    def post_init(self, instance=None, **kwargs):
        value = self._get_val_from_obj(instance)
        if value:
            setattr(instance, self.attname, value)

    def contribute_to_class(self, cls, name):
        super(CompressedTextField, self).contribute_to_class(cls, name)
        post_init.connect(self.post_init, sender=cls)
    
    def get_internal_type(self):
        return "TextField"
                
    def db_type(self):
        from django.conf import settings
        db_types = {'mysql':'longblob','sqlite3':'blob'}
        try:
            return db_types[settings.DATABASE_ENGINE]
        except KeyError:
            raise Exception, '%s currently works only with: %s'%(self.__class__.__name__,','.join(db_types.keys()))

More like this

  1. CompressedTextField for Django 1.4+ by devhulu 2 years, 5 months ago
  2. DebugFooter middleware for django 1.0 and higher by ngadimin 5 years, 8 months ago
  3. Use email addresses for user name for django 1.0+ by jasongreen 5 years, 7 months ago
  4. Unobtrusive comment moderation, updated for Django 1.0 by shimonrura 6 years, 6 months ago
  5. CompressedTextField by arne 7 years, 11 months ago

Comments

s29 (on November 9, 2010):

Works great!

Small hack for Django 1.3 (old DATABASE_ settings are gone) --

def db_type(self):
    db_types = {'django.db.backends.mysql':'longblob', 'django.db.backends.sqlite3':'blob'}
    try:
        return db_types[settings.DATABASES['default']['ENGINE']]

#

s29 (on January 12, 2012):

Another tweak, to transparently support unicode/utf8 data (zlib.crc32 needs plain binary) --

def get_prep_value(self, value):
    if value is not None:
        if isinstance(value, unicode):
            value = value.encode('utf8')
        value = compress_string(value) 
    return value

def _get_val_from_obj(self, obj):
    if obj:
        value = uncompress_string(getattr(obj, self.attname))
        try:
            value = value.decode('utf8')
        except UnicodeDecodeError:
            pass
        return value
    else:
        return self.get_default()

#

Please login first before commenting.