Pickled Object Field

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
# --------------------------------------- fields.py  --------------------------------------- #

from django.db import models

try:
	import cPickle as pickle
except ImportError:
	import pickle

class PickledObject(str):
	"""A subclass of string so it can be told whether a string is
	   a pickled object or not (if the object is an instance of this class
	   then it must [well, should] be a pickled one)."""
	pass

class PickledObjectField(models.Field):
	__metaclass__ = models.SubfieldBase
	
	def to_python(self, value):
		if isinstance(value, PickledObject):
			# If the value is a definite pickle; and an error is raised in de-pickling
			# it should be allowed to propogate.
			return pickle.loads(str(value))
		else:
			try:
				return pickle.loads(str(value))
			except:
				# If an error was raised, just return the plain value
				return value
	
	def get_db_prep_save(self, value):
		if value is not None and not isinstance(value, PickledObject):
			value = PickledObject(pickle.dumps(value))
		return value
	
	def get_internal_type(self): 
		return 'TextField'
	
	def get_db_prep_lookup(self, lookup_type, value):
		if lookup_type == 'exact':
			value = self.get_db_prep_save(value)
			return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
		elif lookup_type == 'in':
			value = [self.get_db_prep_save(v) for v in value]
			return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
		else:
			raise TypeError('Lookup type %s is not supported.' % lookup_type)


# --------------------------------------- tests.py  --------------------------------------- #

# -*- coding: utf-8 -*-
"""Unit testing for this module."""

from django.test import TestCase
from django.db import models
from fields import PickledObjectField

class TestingModel(models.Model):
	pickle_field = PickledObjectField()

class TestCustomDataType(str):
	pass

class PickledObjectFieldTests(TestCase):
	def setUp(self):
		self.testing_data = (
			{1:1, 2:4, 3:6, 4:8, 5:10},
			'Hello World',
			(1, 2, 3, 4, 5),
			[1, 2, 3, 4, 5],
			TestCustomDataType('Hello World'),
		)
		return super(PickledObjectFieldTests, self).setUp()
	
	def testDataIntegriry(self):
		"""Tests that data remains the same when saved to and fetched from the database."""
		for value in self.testing_data:
			model_test = TestingModel(pickle_field=value)
			model_test.save()
			model_test = TestingModel.objects.get(id__exact=model_test.id)
			self.assertEquals(value, model_test.pickle_field)
			model_test.delete()
	
	def testLookups(self):
		"""Tests that lookups can be performed on data once stored in the database."""
		for value in self.testing_data:
			model_test = TestingModel(pickle_field=value)
			model_test.save()
			self.assertEquals(value, TestingModel.objects.get(pickle_field__exact=value).pickle_field)
			model_test.delete()

Comments

yaxu (on December 20, 2007):

I'm having problems with this... 

class PickleTest(models.Model):
    name = models.CharField(max_length=200)
    pickleme = PickledObjectField()

It works fine until I save an object and load it back: 

>>> foo = PickleTest(name = "henry", pickleme = "hello world")
>>> foo.pickleme
'hello world'
>>> foo.save()
>>> foo.pickleme
'hello world'
>>> foo.id
1L
>>> foo = PickleTest.objects.filter(id = 1)
>>> foo[0].pickleme
u"S'hello world'\np1\n."

... as you can see, once it's gone via the database I get the pickle instead of the object.

I'm using the svn version of django.

#

yaxu (on December 20, 2007):

This works, but not for string / unicode objects. 

from django.db import models

# via http://www.djangosnippets.org/snippets/513/

import codecs

try:
    import cPickle as pickle
except ImportError:
    import pickle

class PickledObject(str):
    """A subclass of string so it can be told whether a string is
    a pickled object or not (if the object is an instance of this class
    then it must [well, should] be a pickled one)."""
    pass

class PickledObjectField(models.TextField):
    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        if isinstance(value, unicode):
            enc = codecs.getencoder('utf8')
            return pickle.loads(enc(value)[0])
        elif isinstance(value, str):
            return pickle.loads(value)
        return value

    def get_db_prep_save(self, value):
        if value is not None and not isinstance(value, PickledObject):
            value = PickledObject(pickle.dumps(value))
            return value

    def get_internal_type(self): 
        return 'TextField'

    def get_db_prep_lookup(self, lookup_type, value):
        if lookup_type == 'exact':
            value = self.get_db_prep_save(value)
            return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
        elif lookup_type == 'in':
            value = [self.get_db_prep_save(v) for v in value]
            return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
        else:
            raise TypeError('Lookup type %s is not supported.' % lookup_type)

#

nikolay (on December 22, 2007):

Not working :o(

b.save()

Traceback (most recent call last): File "", line 1, in File "/home/niksite/lib/site-python/django/db/models/base.py", line 261, in save ','.join(placeholders)), db_values) File "/home/niksite/lib/site-python/django/db/backends/util.py", line 18, in execute return self.cursor.execute(sql, params) psycopg2.ProgrammingError: can't adapt

#

obeattie (on December 26, 2007):

yaxu: I'll take a look at this problem now and see what I can come up with. I probably ought to write some unit tests while I'm at it, too…

nikolay: I don't use PostgreSQL (I use MySQL), but I'll have to install it and give it a go.

#

obeattie (on December 27, 2007):

Right, I've revised this code, and added some unit tests. (Please notice that these are two separate files now!)

yaxu: Your issue should be fixed.

nikolay: Haven't tested yours yet, though it's on my to-do list!

#

obeattie (on December 27, 2007):

Oh, and the unit tests do pass over here!

#

(Forgotten your password?)

You may use Markdown syntax here, but raw HTML will be removed.