Django snippets: Pickled Object Field

Pickled Object Field

# --------------------------------------- fields.py  --------------------------------------- #

from django.db import models

try:
	import cPickle as pickle
except ImportError:
	import pickle

class PickledObject(str):
	"""A subclass of string so it can be told whether a string is
	   a pickled object or not (if the object is an instance of this class
	   then it must [well, should] be a pickled one)."""
	pass

class PickledObjectField(models.Field):
	__metaclass__ = models.SubfieldBase
	
	def to_python(self, value):
		if isinstance(value, PickledObject):
			# If the value is a definite pickle; and an error is raised in de-pickling
			# it should be allowed to propogate.
			return pickle.loads(str(value))
		else:
			try:
				return pickle.loads(str(value))
			except:
				# If an error was raised, just return the plain value
				return value
	
	def get_db_prep_save(self, value):
		if value is not None and not isinstance(value, PickledObject):
			value = PickledObject(pickle.dumps(value))
		return value
	
	def get_internal_type(self): 
		return 'TextField'
	
	def get_db_prep_lookup(self, lookup_type, value):
		if lookup_type == 'exact':
			value = self.get_db_prep_save(value)
			return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
		elif lookup_type == 'in':
			value = [self.get_db_prep_save(v) for v in value]
			return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
		else:
			raise TypeError('Lookup type %s is not supported.' % lookup_type)


# --------------------------------------- tests.py  --------------------------------------- #

# -*- coding: utf-8 -*-
"""Unit testing for this module."""

from django.test import TestCase
from django.db import models
from fields import PickledObjectField

class TestingModel(models.Model):
	pickle_field = PickledObjectField()

class TestCustomDataType(str):
	pass

class PickledObjectFieldTests(TestCase):
	def setUp(self):
		self.testing_data = (
			{1:1, 2:4, 3:6, 4:8, 5:10},
			'Hello World',
			(1, 2, 3, 4, 5),
			[1, 2, 3, 4, 5],
			TestCustomDataType('Hello World'),
		)
		return super(PickledObjectFieldTests, self).setUp()
	
	def testDataIntegriry(self):
		"""Tests that data remains the same when saved to and fetched from the database."""
		for value in self.testing_data:
			model_test = TestingModel(pickle_field=value)
			model_test.save()
			model_test = TestingModel.objects.get(id__exact=model_test.id)
			self.assertEquals(value, model_test.pickle_field)
			model_test.delete()
	
	def testLookups(self):
		"""Tests that lookups can be performed on data once stored in the database."""
		for value in self.testing_data:
			model_test = TestingModel(pickle_field=value)
			model_test.save()
			self.assertEquals(value, TestingModel.objects.get(pickle_field__exact=value).pickle_field)
			model_test.delete()

Incredibly useful for storing just about anything in the database (provided it is Pickle-able, of course) when there isn't a 'proper' field for the job:

A field which can store any pickleable object in the database. It is database-agnostic, and should work with any database backend you can throw at it.

Pass in any object and it will be automagically converted behind the scenes, and you never have to manually pickle or unpickle anything. Also works fine when querying.

Please note that this is supposed to be two files, one fields.py and one tests.py (if you don't care about the unit tests, just use fields.py)

Author:: obeattie
Posted:: December 16, 2007
Language:: Python
Tags:: database db field model object orm pickle pickled
Score:: 2 (after 2 ratings)

Tools

Comments

yaxu (on December 20, 2007):

I'm having problems with this...

class PickleTest(models.Model):
    name = models.CharField(max_length=200)
    pickleme = PickledObjectField()

It works fine until I save an object and load it back:

>>> foo = PickleTest(name = "henry", pickleme = "hello world")
>>> foo.pickleme
'hello world'
>>> foo.save()
>>> foo.pickleme
'hello world'
>>> foo.id
1L
>>> foo = PickleTest.objects.filter(id = 1)
>>> foo[0].pickleme
u"S'hello world'\np1\n."

... as you can see, once it's gone via the database I get the pickle instead of the object.

I'm using the svn version of django.

yaxu (on December 20, 2007):

This works, but not for string / unicode objects.

from django.db import models

# via http://www.djangosnippets.org/snippets/513/

import codecs

try:
    import cPickle as pickle
except ImportError:
    import pickle

class PickledObject(str):
    """A subclass of string so it can be told whether a string is
    a pickled object or not (if the object is an instance of this class
    then it must [well, should] be a pickled one)."""
    pass

class PickledObjectField(models.TextField):
    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        if isinstance(value, unicode):
            enc = codecs.getencoder('utf8')
            return pickle.loads(enc(value)[0])
        elif isinstance(value, str):
            return pickle.loads(value)
        return value

    def get_db_prep_save(self, value):
        if value is not None and not isinstance(value, PickledObject):
            value = PickledObject(pickle.dumps(value))
            return value

    def get_internal_type(self): 
        return 'TextField'

    def get_db_prep_lookup(self, lookup_type, value):
        if lookup_type == 'exact':
            value = self.get_db_prep_save(value)
            return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
        elif lookup_type == 'in':
            value = [self.get_db_prep_save(v) for v in value]
            return super(PickledObjectField, self).get_db_prep_lookup(lookup_type, value)
        else:
            raise TypeError('Lookup type %s is not supported.' % lookup_type)

nikolay (on December 22, 2007):

Not working :o(

b.save()

Traceback (most recent call last): File "", line 1, in File "/home/niksite/lib/site-python/django/db/models/base.py", line 261, in save ','.join(placeholders)), db_values) File "/home/niksite/lib/site-python/django/db/backends/util.py", line 18, in execute return self.cursor.execute(sql, params) psycopg2.ProgrammingError: can't adapt

obeattie (on December 26, 2007):

yaxu: I'll take a look at this problem now and see what I can come up with. I probably ought to write some unit tests while I'm at it, too…

nikolay: I don't use PostgreSQL (I use MySQL), but I'll have to install it and give it a go.

obeattie (on December 27, 2007):

Right, I've revised this code, and added some unit tests. (Please notice that these are two separate files now!)

yaxu: Your issue should be fixed.

nikolay: Haven't tested yours yet, though it's on my to-do list!

obeattie (on December 27, 2007):

Oh, and the unit tests do pass over here!

justquick (on January 27, 2009):

Very nice snippet, I use it a lot. I have one suggestion which is to use the highest protocol available for any given system when dumping objects to pickles. This can be accomplished by using pickle.HIGHEST_PROTOCOL when dumping. For this case, line #33 should look something like

value = PickledObject(pickle.dumps(value,pickle.HIGHEST_PROTOCOL))

In my testing this boosted performance 28 percent. Thanks again!

samsutch (on February 17, 2009):

PostgreSQL Users

To get this to work with PostgreSQL you need to register PickledObject with it's marshaler:

psycopg2.extensions.register_adapter(PickledObject, psycopg2.extensions.QuotedString)

Admin Site Users

Mark your field editable=False to retain data while editing the object with admin.

rodrigoc (on June 24, 2009):

I am having trouble pickling, and then saving to the DB, django models with Unicode text in them.

I created a model that holds each object's state at a certain datetime, like so:

from fields import PickledObjectField

class UndoRecord(models.Model): user = models.ForeignKey(User) pickle = PickledObjectField() date = models.DateTimeField(auto_now_add=True)

and then in my views, whenever a user changes something, I save an UndoRecord like so:

undo = UndoRecord(user=request.user, pickle=obj) undo.save()

This works fine, except when the object that I am saving has non-ASCII characters in it, which case I get a lovely:

DjangoUnicodeDecodeError: 'utf8' codec can't decode bytes in position 372-373: invalid data. You passed in "crbml_core.modelsnMusiconp1n (tRp2n(dp3nS'foto'np4nccopy_regn_reconstructornp5n cdjango.db.models.fields.filesnImageFieldFilenp6ncbuiltinnobjectnp7nNtRp8n (dp9nS'_committed'np10nI01nsS'_file'np11nNsS'name'np12nVuploads/musicos/235470296_54ae5af3e9_o.jpg np13nsS'closed'np14nI00nsbsS'user_id'np15nNsS'visible'np16nI1nsS'influencias'np17nVFaith No Morenp18nsS'nombre'np19nVxd1xedxfaxf6np20nsS'id'np21nL2L nsb." ()

I have been trying to fix this using the approach show here: http://www.mail-archive.com/django-users@googlegroups.com/msg67883.html, but no such luck.

Any help is much appreciated.

Django snippets

Pickled Object Field

Tools

Comments

PostgreSQL Users

Admin Site Users

Snippets

About