Login

CompressedTextField for Django 1.4+

Author:
devhulu
Posted:
February 15, 2013
Language:
Python
Version:
1.4
Tags:
text model field compressed gzip south
Score:
1 (after 1 ratings)

This snippet updates http://www.djangosnippets.org/snippets/383/ and http://www.djangosnippets.org/snippets/1495/ for Django 1.4+, and adds support for sqlite3 and south. Original snippet text: A CompressedTextField to transparently save data gzipped in the database and uncompress at retrieval.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# -*- coding: utf-8 -*-
# http://www.djangosnippets.org/snippets/383/
# Source: http://djangosnippets.org/snippets/1495/

from django.db import models
from django.utils.text import compress_string
from django.db.models.signals import post_init

def uncompress_string(s):
    """helper function to reverse django.utils.text.compress_string"""
    import cStringIO, gzip
    try:
        val = s.encode('utf').decode('base64')
        zbuf = cStringIO.StringIO(val)
        zfile = gzip.GzipFile(fileobj=zbuf)
        ret = zfile.read()
        zfile.close()
    except:
        ret = s
    return ret


class CompressedTextField(models.TextField):
    """transparently compress data before hitting the db and uncompress after fetching"""

    def get_db_prep_save(self, value, connection):
        if value is not None:
            if isinstance(value, unicode):
                value = value.encode('utf8')
            value = compress_string(value)
            value = value.encode('base64').decode('utf8')
        return models.TextField.get_db_prep_save(self, value, connection=connection)

    def _get_val_from_obj(self, obj):
        if obj:
            value = uncompress_string(getattr(obj, self.attname))
            if value is not None:
                try:
                    value = value.decode('utf8')
                except UnicodeDecodeError:
                    pass
                return value
            else:
                return self.get_default()
        else:
            return self.get_default()

    def post_init(self, instance=None, **kwargs):
        value = self._get_val_from_obj(instance)
        if value:
            setattr(instance, self.attname, value)

    def contribute_to_class(self, cls, name):
        super(CompressedTextField, self).contribute_to_class(cls, name)
        post_init.connect(self.post_init, sender=cls)

    def get_internal_type(self):
        return "TextField"

    def db_type(self):
        from django.conf import settings
        db_types = {'django.db.backends.mysql':'longblob', 'django.db.backends.sqlite3':'blob'}
        try:
            return db_types[settings.DATABASES['default']['ENGINE']]
        except KeyError:
            raise Exception, '%s currently works only with: %s'%(self.__class__.__name__,','.join(db_types.keys()))

    def south_field_triple(self):
        """Returns a suitable description of this field for South."""
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = "django.db.models.fields.TextField"
        args, kwargs = introspector(self)
        # That's our definition!
        return (field_class, args, kwargs)

More like this

  1. CompressedTextField for Django 1.0+ by bwhittington 5 years, 9 months ago
  2. Drupal password hasher for migration by dgrtwo 2 years, 10 months ago
  3. South introspection rules for TimeZoneFields by serkan 3 years, 11 months ago
  4. Loading initial data per model at table creation (useful with migrations) by GigiusB 7 months, 3 weeks ago
  5. Unit Test Profiling for Django 1.3/1.4 by hoffmaje 2 years, 10 months ago

Comments

druska (on March 4, 2013):

Remember to add connection to db_type: def db_type(self, connection):

#

Please login first before commenting.