Login

Django model objects and querysets dehydration/hydration

Author:
Kronuz
Posted:
October 28, 2011
Language:
Python
Version:
1.3
Score:
0 (after 0 ratings)

Dehydrates objects that can be dictionaries, lists or tuples containing django model objects or django querysets. For each of those, it creates a smaller/dehydrated version of it for saving in cache or pickling. The reverse operation is also provided so dehydrated objects can also be re-hydrated.

Example:

>>> import pickle

>>> users = list(User.objects.all()[:20])

>>> print users

[<User: Indiana Jones>, <User: Bilbo Baggins>, ...]

>>> pickled_users = pickle.dumps(users)

>>> print len(pickled_users)

17546

>>> dehydrated_users = dehydrate(users)

>>> pickled_dehydrated_users = pickle.dumps(dehydrated_users)

>>> rehydrated_users = hydrate(pickle.loads(pickled_dehydrated_users))

>>> print rehydrated_users

[<User: Indiana Jones>, <User: Bilbo Baggins>, ...]

>>> print len(pickled_dehydrated_users)

1471
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
# Django model objects and querysets dehydration/hydration

# Dehydrates objects that can be dictionaries, lists or tuples containing django
# model objects or django querysets. For each of those, it creates a
# smaller/dehydrated version of it for saving in cache or pickling. The reverse
# operation is also provided so dehydrated objects can also be re-hydrated.

# Example:
# >>> import pickle
# >>> users = list(User.objects.all()[:20])
# >>> print users
# [<User: Indiana Jones>, <User: Bilbo Baggins>, <User: Lara Croft>, <User: Angus MacGyver>, <User: Luke Skywalker>, <User: Obi-Wan Kenobi>, ...]
# >>> pickled_users = pickle.dumps(users)
# >>> print len(pickled_users)
# 17546
# >>> dehydrated_users = dehydrate(users)
# >>> pickled_dehydrated_users = pickle.dumps(dehydrated_users)
# >>> rehydrated_users = hydrate(pickle.loads(pickled_dehydrated_users))
# >>> print rehydrated_users
# [<User: Indiana Jones>, <User: Bilbo Baggins>, <User: Lara Croft>, <User: Angus MacGyver>, <User: Luke Skywalker>, <User: Obi-Wan Kenobi>, ...]
# >>> print len(pickled_dehydrated_users)
# 1471

import sys
from django.db import models
from django.db.models.query import QuerySet
from django.contrib.contenttypes.models import ContentType

def _walk(obj, fnct, raise_exc=False, delayed=None, maxlevels=10, level=0, context=None):
    if context is None:
        context = {}
    if not obj:
        return obj
    if maxlevels and level >= maxlevels:
        return obj
    objid = id(obj)
    if objid in context:
        return obj
    typ = type(obj)
    if typ is str:
        return obj
    if issubclass(typ, dict):
        if objid in context:
            return obj
        context[objid] = True
        ret = {}
        for k, v in obj.items():
            ret[k] = _walk(v, fnct, raise_exc, delayed, maxlevels, level + 1, context)
        del context[objid]
        return ret
    if issubclass(typ, list) or issubclass(typ, tuple) or issubclass(typ, set):
        context[objid] = True
        ret = []
        for o in obj:
            ret.append(_walk(o, fnct, raise_exc, delayed, maxlevels, level + 1, context))
        del context[objid]
        if issubclass(typ, tuple):
            return tuple(ret)
        if issubclass(typ, set):
            return set(ret)
        return ret
    return fnct(obj, raise_exc, delayed)

class Dehydrated(object):
    def __init__(self, obj):
        self._obj = obj

    def __repr__(self):
        try:
            u = unicode(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return str(u'<%s: %s>' % (self.__class__.__name__, u))

    def __str__(self):
        if hasattr(self, '__unicode__'):
            return unicode(self).encode('utf-8')
        return '%s object' % (self.__class__.__name__,)

    def __getstate__(self):
        ret = self.__dict__.copy()
        ret.pop('_obj', None)
        return ret

class DehydratedModel(Dehydrated):
    def __init__(self, obj):
        super(DehydratedModel, self).__init__(obj)
        self.app_label, self.object_name = obj._meta.app_label, obj._meta.object_name
        self.pk_attrname = obj._meta.pk.attname
        setattr(self, self.pk_attrname, getattr(obj, '_pk', obj.pk))
        if self.pk:
            fields = [f.name for f in obj._meta.fields]
            self.data = dict((k, v) for k, v in obj.__dict__.items() if not k.startswith('_') and not k.endswith('_id') and k not in fields)
        else:
            self.data = dict((k, v) for k, v in obj.__dict__.items() if not k.startswith('_'))

    def __str__(self):
        if hasattr(self, '__unicode__'):
            return unicode(self).encode('utf-8')
        return '%s.%s.%s' % (self.app_label, self.object_name, self.pk)

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.pk == other.pk

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.pk)

    def _get_pk_val(self, meta=None):
        return getattr(self, self.pk_attrname)

    def _set_pk_val(self, value):
        return setattr(self, self.pk_attrname, value)

    pk = property(_get_pk_val, _set_pk_val)

    @staticmethod
    def delayed_hydrate(raise_exc, delayed):
        for Model, objects in delayed.items():
            for _obj in Model._default_manager.filter(pk__in=objects.keys()):
                obj, dobjs = objects[_obj.pk]
                obj.__dict__.update(_obj.__dict__)
                for dobj in dobjs:
                    dobj._obj = obj
            for pk, objs in objects.items():
                if not objs[0].pk:
                    msg = "%s matching query does not exist. Cannot hydrate model instance %s.%s.%s." % (
                        Model._meta.object_name,
                        Model._meta.app_label,
                        Model._meta.object_name,
                        pk,
                    )
                    if raise_exc:
                        raise Model.DoesNotExist(msg)
                    sys.stderr.write("%s\n" % msg)

    def hydrate(self, raise_exc=False, delayed=None):
        if hasattr(self, '_obj'):
            obj = self._obj
        else:
            content_type = ContentType.objects.get_by_natural_key(self.app_label, self.object_name.lower())
            Model = content_type.model_class()
            if self.pk:
                if delayed is not None:
                    delayed.setdefault(Model, {})
                    if self.pk in delayed[Model]:
                        obj, dobjs = delayed[Model][self.pk]
                        dobjs.append(self)
                    else:
                        obj = Model()
                        obj._pk = self.pk
                        delayed[Model][self.pk] = (obj, [self])
                else:
                    obj = Model._default_manager.get(pk=self.pk)
                    self._obj = obj
            else:
                obj = Model()
                self._obj = obj
            if hasattr(self, 'data'):
                obj.__dict__.update(self.data)
        return obj

class DehydratedQuerySet(Dehydrated):
    def __init__(self, qs):
        super(DehydratedQuerySet, self).__init__(qs)
        self.app_label, self.object_name = qs.model._meta.app_label, qs.model._meta.object_name
        self.query = qs.query

    @staticmethod
    def delayed_hydrate(raise_exc, delayed):
        pass

    def hydrate(self, raise_exc=False, delayed=None):
        if not hasattr(self, '_obj'):
            content_type = ContentType.objects.get_by_natural_key(self.app_label, self.object_name.lower())
            Model = content_type.model_class()
            qs = Model._default_manager.all()
            qs.query = self.query
            self._obj = qs
        return self._obj

def _dehydrate(obj, raise_exc, delayed):
    typ = type(obj)
    if issubclass(typ, models.Model):
        return DehydratedModel(obj)
    if issubclass(typ, QuerySet):
        return DehydratedQuerySet(obj)
    return obj

def _hydrate(obj, raise_exc, delayed):
    typ = type(obj)
    if issubclass(typ, DehydratedModel):
        if delayed is not None:
            delayed.setdefault(DehydratedModel, {})
            delayed = delayed[DehydratedModel]
        return obj.hydrate(raise_exc, delayed)
    if issubclass(typ, DehydratedQuerySet):
        if delayed is not None:
            delayed.setdefault(DehydratedQuerySet, {})
            delayed = delayed[DehydratedQuerySet]
        return obj.hydrate(raise_exc, delayed)
    return obj

def dehydrate(obj, raise_exc=False):
    """
    Dehydrates objects containing django model objects and querysets.
    """
    return _walk(obj, _dehydrate, raise_exc)

def hydrate(obj, raise_exc=False):
    """
    Hydrates objects containing dehydrated django model objects and querysets.
    """
    delayed = {}
    ret = _walk(obj, _hydrate, raise_exc, delayed)
    for klass, _delayed in delayed.items():
        klass.delayed_hydrate(raise_exc, _delayed)
    return ret

More like this

  1. Template tag - list punctuation for a list of items by shapiromatron 2 months, 2 weeks ago
  2. JSONRequestMiddleware adds a .json() method to your HttpRequests by cdcarter 2 months, 3 weeks ago
  3. Serializer factory with Django Rest Framework by julio 9 months, 2 weeks ago
  4. Image compression before saving the new model / work with JPG, PNG by Schleidens 10 months, 1 week ago
  5. Help text hyperlinks by sa2812 11 months ago

Comments

Please login first before commenting.