From fbcecf5737714206d43bb50af2cce88e074c462c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:31:06 -0500 Subject: [PATCH 01/22] git ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0205d62 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +.DS_Store From f265731fc91f800ec2a4d19d0f6b3192927d4605 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:31:38 -0500 Subject: [PATCH 02/22] appengine key and ancestor support --- db/compiler.py | 81 +++++++++++++++++------- db/creation.py | 4 +- fields.py | 69 ++++++++++++++++++++ models.py | 66 +++++++++++++++++++ tests/__init__.py | 1 + tests/keys.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 356 insertions(+), 23 deletions(-) create mode 100644 fields.py create mode 100644 tests/keys.py diff --git a/db/compiler.py b/db/compiler.py index f76fcfa..11c0f4f 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -1,6 +1,8 @@ from .db_settings import get_model_indexes from .utils import commit_locked from .expressions import ExpressionEvaluator +from ..fields import GAEKeyField +from ..models import GAEKey, GAEAncestorKey import datetime import sys @@ -73,6 +75,7 @@ def __init__(self, compiler, fields): self.pk_filters = None self.excluded_pks = () self.has_negated_exact_filter = False + self.ancestor_key = None self.ordering = () self.gae_ordering = [] pks_only = False @@ -173,6 +176,14 @@ def add_filter(self, column, lookup_type, negated, db_type, value): if column == self.query.get_meta().pk.column: column = '__key__' db_table = self.query.get_meta().db_table + + if lookup_type == 'exact' and isinstance(value, GAEAncestorKey): + if negated: + raise DatabaseError("You can't negate an ancestor operation.") + if self.ancestor_key is not None: + raise DatabaseError("You can't use more than one ancestor operation.") + self.ancestor_key = value.key() + return if lookup_type in ('exact', 'in'): # Optimization: batch-get by key if self.pk_filters is not None: @@ -319,6 +330,8 @@ def _make_entity(self, entity): def _build_query(self): for query in self.gae_query: query.Order(*self.gae_ordering) + if self.ancestor_key: + query.Ancestor(self.ancestor_key) if len(self.gae_query) > 1: return MultiQuery(self.gae_query, self.gae_ordering) return self.gae_query[0] @@ -391,24 +404,24 @@ def convert_value_from_db(self, db_type, value): # contain non unicode strings, nevertheless work with unicode ones) value = value.decode('utf-8') elif isinstance(value, Key): - # for now we do not support KeyFields thus a Key has to be the own - # primary key - # TODO: GAE: support parents via GAEKeyField - assert value.parent() is None, "Parents are not yet supported!" - if db_type == 'integer': - if value.id() is None: - raise DatabaseError('Wrong type for Key. Expected integer, found' - 'None') - else: - value = value.id() - elif db_type == 'text': - if value.name() is None: - raise DatabaseError('Wrong type for Key. Expected string, found' - 'None') - else: - value = value.name() + if db_type == 'gae_key': + value = GAEKey(real_key=value) else: - raise DatabaseError("%s fields cannot be keys on GAE" % db_type) + assert value.parent() is None, "Use GAEKeyField to enable parent keys!" + if db_type == 'integer': + if value.id() is None: + raise DatabaseError('Wrong type for Key. Expected integer, found' + 'None') + else: + value = value.id() + elif db_type == 'text': + if value.name() is None: + raise DatabaseError('Wrong type for Key. Expected string, found' + 'None') + else: + value = value.name() + else: + raise DatabaseError("%s fields cannot be keys on GAE" % db_type) elif db_type == 'date' and isinstance(value, datetime.datetime): value = value.date() elif db_type == 'time' and isinstance(value, datetime.datetime): @@ -435,7 +448,10 @@ def convert_value_for_db(self, db_type, value): value = Blob(pickle.dumps(value)) if db_type == 'gae_key': - return value + if isinstance(value, GAEKey) and value.has_real_key(): + return value.real_key + else: + return value elif db_type == 'longtext': # long text fields cannot be indexed on GAE so use GAE's database # type Text @@ -465,21 +481,37 @@ def insert(self, data, return_id=False): kwds = {'unindexed_properties': unindexed_cols} for column, value in data.items(): if column == opts.pk.column: - if isinstance(value, basestring): + if isinstance(value, GAEKey): + if value.parent_key and value.parent_key.has_real_key(): + kwds['parent'] = value.parent_key.real_key + if isinstance(value.id_or_name, basestring): + kwds['name'] = value.id_or_name + elif value.id_or_name is not None: + kwds['id'] = value.id_or_name + elif isinstance(value, Key): + kwds['parent'] = value.parent() + if value.name(): + kwds['name'] = value.name() + elif value.id(): + kwds['id'] = value.id() + elif isinstance(value, basestring): kwds['name'] = value else: kwds['id'] = value elif isinstance(value, (tuple, list)) and not len(value): - # gae does not store emty lists (and even does not allow passing empty + # gae does not store empty lists (and even does not allow passing empty # lists to Entity.update) so skip them continue else: gae_data[column] = value - entity = Entity(self.query.get_meta().db_table, **kwds) + entity = Entity(opts.db_table, **kwds) entity.update(gae_data) key = Put(entity) - return key.id_or_name() + + if not isinstance(opts.pk, GAEKeyField): + key = key.id_or_name() + return key class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): def execute_sql(self, result_type=MULTI): @@ -556,6 +588,11 @@ def to_datetime(value): value.second, value.microsecond) def create_key(db_table, value): + if isinstance(value, GAEKey): + parent = None + if value.parent_key is not None: + parent = value.parent.real_key + return Key.from_path(db_table, value.id_or_name, parent=parent) if isinstance(value, (int, long)) and value < 1: return None return Key.from_path(db_table, value) diff --git a/db/creation.py b/db/creation.py index 912a52e..42f4f0f 100644 --- a/db/creation.py +++ b/db/creation.py @@ -15,12 +15,14 @@ def __mod__(self, field): return self.internal_type def get_data_types(): - # TODO: Add GAEKeyField and a corresponding db_type string_types = ('text', 'longtext') data_types = NonrelDatabaseCreation.data_types.copy() for name, field_type in data_types.items(): if field_type in string_types: data_types[name] = StringType(field_type) + + data_types['GAEKeyField'] = 'gae_key' + return data_types class DatabaseCreation(NonrelDatabaseCreation): diff --git a/fields.py b/fields.py new file mode 100644 index 0000000..2131a8d --- /dev/null +++ b/fields.py @@ -0,0 +1,69 @@ +from django.db import models +from google.appengine.api.datastore import Key +from .models import GAEKey, GAEAncestorKey + +class GAEKeyField(models.Field): + description = "A field for Google AppEngine Key objects" + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + assert kwargs.get('primary_key', False) is True, "%ss must have primary_key=True." % self.__class__.__name__ + kwargs['null'] = True + kwargs['blank'] = True + self.parent_key_attname = kwargs.pop('parent_key_name', None) + + super(GAEKeyField, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name): + assert not cls._meta.has_auto_field, "A model can't have more than one auto field." + super(GAEKeyField, self).contribute_to_class(cls, name) + cls._meta.has_auto_field = True + cls._meta.auto_field = self + + if self.parent_key_attname is not None: + def get_parent_key(instance, instance_type=None): + if instance is None: + return self + return instance.__dict__.get(self.parent_key_attname) + + def set_parent_key(instance, value): + if instance is None: + raise AttributeError("Attribute must be accessed via instance") + + if not isinstance(value, GAEKey): + raise ValueError("parent must be a GAEKey") + + instance.__dict__[self.parent_key_attname] = value + + setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + + def to_python(self, value): + if value is None: + return None + if isinstance(value, GAEKey): + return value + if isinstance(value, Key): + return GAEKey(real_key=value) + if isinstance(value, basestring): + return GAEKey(real_key=Key(encoded=value)) + return GAEKey(id_or_name=value) + + def get_prep_value(self, value): + if value is None: + return None + if not isinstance(value, (GAEKey, GAEAncestorKey)): + raise ValueError('must by type GAEKey or GAEAncestorKey, not <%s>' % type(value)) + return value + + def formfield(self, **kwargs): + return None + + def pre_save(self, model_instance, add): + if add and self.parent_key_attname is not None: + parent_key = getattr(model_instance, self.parent_key_attname) + if parent_key is not None: + key = GAEKey(parent_key=parent_key) + setattr(model_instance, self.attname, key) + return key + + return super(GAEKeyField, self).pre_save(model_instance, add) diff --git a/models.py b/models.py index e69de29..788445b 100644 --- a/models.py +++ b/models.py @@ -0,0 +1,66 @@ +from django.db import models +from google.appengine.api.datastore import Key + +# TODO: look for better exceptions to raise + +class GAEAncestorKey(object): + def __init__(self, key): + if not isinstance(key, Key): + raise ValueError('key must be of type Key') + + self._key = key + + def key(self): + return self._key + +class GAEKey(object): + def __init__(self, id_or_name=None, parent_key=None, real_key=None): + self._id_or_name = id_or_name + self._parent_key = parent_key + self._real_key = None + + if real_key is not None: + if id_or_name is not None or parent_key is not None: + raise ValueError("You can't set both a real_key and an id_or_name or parent_key") + + self._real_key = real_key + if real_key.parent(): + self._parent_key = GAEKey(real_key=real_key.parent()) + self._id_or_name = real_key.id_or_name() + + def _get_id_or_name(self): + return self._id_or_name + id_or_name = property(_get_id_or_name) + + def _get_parent_key(self): + return self._parent_key + parent_key = property(_get_parent_key) + + def _get_real_key(self): + if self._real_key is None: + raise ValueError("Incomplete key, please save the entity first.") + return self._real_key + real_key = property(_get_real_key) + + def has_real_key(self): + return self._real_key is not None + + def as_ancestor(self): + return GAEAncestorKey(self._get_real_key()) + + def __cmp__(self, other): + if not isinstance(other, GAEKey): + return 1 + if self._real_key is None or other._real_key is None: + raise ValueError("You can't compare unsaved keys.") + + return cmp(self._real_key, other._real_key) + + def __hash__(self): + if self._real_key is None: + raise ValueError("You can't hash an unsaved key.") + + return hash(self._real_key) + + def __str__(self): + return str(self._real_key) diff --git a/tests/__init__.py b/tests/__init__.py index c5f867d..cb4e8ab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,3 +6,4 @@ from .not_return_sets import NonReturnSetsTest from .decimals import DecimalTest from .transactions import TransactionTest +from .keys import KeysTest diff --git a/tests/keys.py b/tests/keys.py new file mode 100644 index 0000000..e852393 --- /dev/null +++ b/tests/keys.py @@ -0,0 +1,158 @@ +from django.db import models +from django.test import TestCase +from django.db.utils import DatabaseError + +from google.appengine.api.datastore import Key + +from ..fields import GAEKeyField +from ..models import GAEKey + +class ParentModel(models.Model): + key = GAEKeyField(primary_key=True) + +class NonGAEParentModel(models.Model): + id = models.AutoField(primary_key=True) + +class ChildModel(models.Model): + key = GAEKeyField(primary_key=True, parent_key_name='parent_key') + +class AnotherChildModel(models.Model): + key = GAEKeyField(primary_key=True, parent_key_name='also_parent_key') + +class ForeignKeyModel(models.Model): + id = models.AutoField(primary_key=True) + relation = models.ForeignKey(ParentModel) + +class KeysTest(TestCase): + def testGAEKeySave(self): + model = ParentModel() + model.save() + + self.assertIsNotNone(model.pk) + + def testUnsavedParent(self): + parent = ParentModel() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) + + def testNonGAEParent(self): + parent = NonGAEParentModel() + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) + + def testParentChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + child = ChildModel(parent_key=parent.pk) + orig_child_pk = child.pk + child.save() + + self.assertNotEquals(parent.pk, orig_parent_pk) + self.assertNotEquals(child.pk, orig_child_pk) + self.assertEquals(child.pk.parent_key, parent.pk) + self.assertEquals(child.pk.parent_key.real_key, parent.pk.real_key) + + def testAncestorFilterQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + def testAncestorGetQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + result = ChildModel.objects.get(pk=parent.pk.as_ancestor()) + + self.assertEquals(result.pk, child.pk) + + def testEmptyAncestorQuery(self): + parent = ParentModel() + parent.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(0, len(results)) + + def testEmptyAncestorQueryWithUnsavedChild(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(0, len(results)) + + def testUnsavedAncestorQuery(self): + parent = ParentModel() + + with self.assertRaises(AttributeError): + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + def testDifferentChildrenAncestorQuery(self): + parent = ParentModel() + parent.save() + + child1 = ChildModel(parent_key=parent.pk) + child1.save() + child2 = AnotherChildModel(also_parent_key=parent.pk) + child2.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child1.pk) + + results = list(AnotherChildModel.objects.filter(pk=parent.pk.as_ancestor())) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child2.pk) + + def testDifferentParentsAncestorQuery(self): + parent1 = ParentModel() + parent1.save() + + child1 = ChildModel(parent_key=parent1.pk) + child1.save() + + parent2 = ParentModel() + parent2.save() + + child2 = ChildModel(parent_key=parent2.pk) + child2.save() + + results = list(ChildModel.objects.filter(pk=parent1.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child1.pk) + + results = list(ChildModel.objects.filter(pk=parent2.pk.as_ancestor())) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child2.pk) + + def testForeignKeyWithGAEKey(self): + parent = ParentModel() + parent.save() + + fkm = ForeignKeyModel() + fkm.relation = parent + fkm.save() + + results = list(ForeignKeyModel.objects.filter(relation=parent)) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, fkm.pk) + From decfc123debbb6376016adeecfe1d9eda1d6806c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:37:11 -0500 Subject: [PATCH 03/22] fixes for non-primary GAEKeys --- db/compiler.py | 3 ++- fields.py | 56 +++++++++++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index 11c0f4f..bdc7a3c 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -511,6 +511,7 @@ def insert(self, data, return_id=False): if not isinstance(opts.pk, GAEKeyField): key = key.id_or_name() + return key class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): @@ -591,7 +592,7 @@ def create_key(db_table, value): if isinstance(value, GAEKey): parent = None if value.parent_key is not None: - parent = value.parent.real_key + parent = value.parent_key.real_key return Key.from_path(db_table, value.id_or_name, parent=parent) if isinstance(value, (int, long)) and value < 1: return None diff --git a/fields.py b/fields.py index 2131a8d..1408777 100644 --- a/fields.py +++ b/fields.py @@ -1,5 +1,5 @@ from django.db import models -from google.appengine.api.datastore import Key +from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey class GAEKeyField(models.Field): @@ -7,35 +7,39 @@ class GAEKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - assert kwargs.get('primary_key', False) is True, "%ss must have primary_key=True." % self.__class__.__name__ kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) + if self.parent_key_attname is not None and kwargs.get('primary_key', None) is None: + raise ValueError("Primary key must be true to set parent_key_name") + super(GAEKeyField, self).__init__(*args, **kwargs) def contribute_to_class(self, cls, name): - assert not cls._meta.has_auto_field, "A model can't have more than one auto field." - super(GAEKeyField, self).contribute_to_class(cls, name) - cls._meta.has_auto_field = True - cls._meta.auto_field = self + if self.primary_key: + assert not cls._meta.has_auto_field, "A model can't have more than one auto field." + cls._meta.has_auto_field = True + cls._meta.auto_field = self - if self.parent_key_attname is not None: - def get_parent_key(instance, instance_type=None): - if instance is None: - return self - return instance.__dict__.get(self.parent_key_attname) + if self.parent_key_attname is not None: + def get_parent_key(instance, instance_type=None): + if instance is None: + return self + return instance.__dict__.get(self.parent_key_attname) - def set_parent_key(instance, value): - if instance is None: - raise AttributeError("Attribute must be accessed via instance") + def set_parent_key(instance, value): + if instance is None: + raise AttributeError("Attribute must be accessed via instance") - if not isinstance(value, GAEKey): - raise ValueError("parent must be a GAEKey") + if not isinstance(value, GAEKey): + raise ValueError("parent must be a GAEKey") - instance.__dict__[self.parent_key_attname] = value + instance.__dict__[self.parent_key_attname] = value - setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + + super(GAEKeyField, self).contribute_to_class(cls, name) def to_python(self, value): if value is None: @@ -45,14 +49,24 @@ def to_python(self, value): if isinstance(value, Key): return GAEKey(real_key=value) if isinstance(value, basestring): - return GAEKey(real_key=Key(encoded=value)) - return GAEKey(id_or_name=value) + try: + return GAEKey(real_key=Key(encoded=value)) + except datastore_errors.BadKeyError: + pass + raise ValueError("this value is not allowed %s" % value) def get_prep_value(self, value): if value is None: return None + if isinstance(value, Key): + return GAEKey(real_key=value) + if isinstance(value, basestring): + try: + return GAEKey(real_key=Key(encoded=value)) + except datastore_errors.BadKeyError: + raise ValueError("this value is not allowed %s" % value) if not isinstance(value, (GAEKey, GAEAncestorKey)): - raise ValueError('must by type GAEKey or GAEAncestorKey, not <%s>' % type(value)) + raise ValueError('Must by type GAEKey, GAEAncestorKey, basestring. Not <%s>' % type(value)) return value def formfield(self, **kwargs): From 87a9f855d94b5b9dcd3e12bcd1068641b079bc1e Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:56:16 -0500 Subject: [PATCH 04/22] use id_or_name as string value --- models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models.py b/models.py index 788445b..91a00fd 100644 --- a/models.py +++ b/models.py @@ -38,7 +38,7 @@ def _get_parent_key(self): def _get_real_key(self): if self._real_key is None: - raise ValueError("Incomplete key, please save the entity first.") + raise AttributeError("Incomplete key, please save the entity first.") return self._real_key real_key = property(_get_real_key) @@ -63,4 +63,4 @@ def __hash__(self): return hash(self._real_key) def __str__(self): - return str(self._real_key) + return str(self.id_or_name) From fd234e61812b45415fc4380ba5030498195f02c6 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:57:26 -0500 Subject: [PATCH 05/22] better value conversion to match str from GAEKey --- fields.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/fields.py b/fields.py index 1408777..8c711e4 100644 --- a/fields.py +++ b/fields.py @@ -1,3 +1,4 @@ +from django.core.exceptions import ValidationError from django.db import models from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey @@ -52,22 +53,16 @@ def to_python(self, value): try: return GAEKey(real_key=Key(encoded=value)) except datastore_errors.BadKeyError: - pass - raise ValueError("this value is not allowed %s" % value) + return GAEKey(real_key=Key.from_path(self.model._meta.db_table, long(value))) + if isinstance(value, (int, long)): + return GAEKey(real_key=Key.from_path(self.model._meta.db_table, value)) + + raise ValidationError("GAEKeyField does not accept %s" % type(value)) def get_prep_value(self, value): - if value is None: - return None - if isinstance(value, Key): - return GAEKey(real_key=value) - if isinstance(value, basestring): - try: - return GAEKey(real_key=Key(encoded=value)) - except datastore_errors.BadKeyError: - raise ValueError("this value is not allowed %s" % value) - if not isinstance(value, (GAEKey, GAEAncestorKey)): - raise ValueError('Must by type GAEKey, GAEAncestorKey, basestring. Not <%s>' % type(value)) - return value + if isinstance(value, GAEAncestorKey): + return value + return self.to_python(value) def formfield(self, **kwargs): return None From 150c915a4defdae2ba90f2581bda9d2e63aeb424 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:57:48 -0500 Subject: [PATCH 06/22] test cases for different kinds of pk representations --- tests/keys.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/keys.py b/tests/keys.py index e852393..4e6bd06 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -147,12 +147,36 @@ def testDifferentParentsAncestorQuery(self): def testForeignKeyWithGAEKey(self): parent = ParentModel() parent.save() - + fkm = ForeignKeyModel() fkm.relation = parent fkm.save() - + results = list(ForeignKeyModel.objects.filter(relation=parent)) self.assertEquals(1, len(results)) self.assertEquals(results[0].pk, fkm.pk) + def testPrimaryKeyQuery(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=parent.pk) + + self.assertEquals(parent.pk, db_parent.pk) + + def testPrimaryKeyQueryStringKey(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=str(parent.pk)) + + self.assertEquals(parent.pk, db_parent.pk) + + def testPrimaryKeyQueryIntKey(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=int(str(parent.pk))) + + self.assertEquals(parent.pk, db_parent.pk) + \ No newline at end of file From 29a1c620b7305678f415ad4ce87eb1ea3c41f268 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Tue, 6 Dec 2011 11:31:36 -0500 Subject: [PATCH 07/22] fix cmp for GAEKey --- models.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/models.py b/models.py index 91a00fd..f0fb719 100644 --- a/models.py +++ b/models.py @@ -1,8 +1,6 @@ from django.db import models from google.appengine.api.datastore import Key -# TODO: look for better exceptions to raise - class GAEAncestorKey(object): def __init__(self, key): if not isinstance(key, Key): @@ -51,10 +49,21 @@ def as_ancestor(self): def __cmp__(self, other): if not isinstance(other, GAEKey): return 1 - if self._real_key is None or other._real_key is None: - raise ValueError("You can't compare unsaved keys.") - - return cmp(self._real_key, other._real_key) + + if self._real_key is not None and other._real_key is not None: + return cmp(self._real_key, other._real_key) + + if self._id_or_name is None or other._id_or_name is None: + raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) + + result = 0 + if self._parent_key is not None: + result = cmp(self._parent_key, other._parent_key) + + if result == 0: + result = cmp(self._id_or_name, other._id_or_name) + + return result def __hash__(self): if self._real_key is None: From 091b12ca010794e194e5959ceb3694458f00819f Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 29 Dec 2011 13:22:54 -0500 Subject: [PATCH 08/22] use methods instead of properties for GAEKey fields --- db/compiler.py | 20 ++++++++++---------- models.py | 14 ++++++-------- tests/keys.py | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index bdc7a3c..5c0934c 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -449,7 +449,7 @@ def convert_value_for_db(self, db_type, value): if db_type == 'gae_key': if isinstance(value, GAEKey) and value.has_real_key(): - return value.real_key + return value.real_key() else: return value elif db_type == 'longtext': @@ -482,12 +482,12 @@ def insert(self, data, return_id=False): for column, value in data.items(): if column == opts.pk.column: if isinstance(value, GAEKey): - if value.parent_key and value.parent_key.has_real_key(): - kwds['parent'] = value.parent_key.real_key - if isinstance(value.id_or_name, basestring): - kwds['name'] = value.id_or_name - elif value.id_or_name is not None: - kwds['id'] = value.id_or_name + if value.parent_key() and value.parent_key().has_real_key(): + kwds['parent'] = value.parent_key().real_key() + if isinstance(value.id_or_name(), basestring): + kwds['name'] = value.id_or_name() + elif value.id_or_name() is not None: + kwds['id'] = value.id_or_name() elif isinstance(value, Key): kwds['parent'] = value.parent() if value.name(): @@ -591,9 +591,9 @@ def to_datetime(value): def create_key(db_table, value): if isinstance(value, GAEKey): parent = None - if value.parent_key is not None: - parent = value.parent_key.real_key - return Key.from_path(db_table, value.id_or_name, parent=parent) + if value.parent_key() is not None: + parent = value.parent_key().real_key() + return Key.from_path(db_table, value.id_or_name(), parent=parent) if isinstance(value, (int, long)) and value < 1: return None return Key.from_path(db_table, value) diff --git a/models.py b/models.py index f0fb719..b3852e9 100644 --- a/models.py +++ b/models.py @@ -8,6 +8,7 @@ def __init__(self, key): self._key = key + @property def key(self): return self._key @@ -26,25 +27,22 @@ def __init__(self, id_or_name=None, parent_key=None, real_key=None): self._parent_key = GAEKey(real_key=real_key.parent()) self._id_or_name = real_key.id_or_name() - def _get_id_or_name(self): + def id_or_name(self): return self._id_or_name - id_or_name = property(_get_id_or_name) - def _get_parent_key(self): + def parent_key(self): return self._parent_key - parent_key = property(_get_parent_key) - def _get_real_key(self): + def real_key(self): if self._real_key is None: raise AttributeError("Incomplete key, please save the entity first.") return self._real_key - real_key = property(_get_real_key) def has_real_key(self): return self._real_key is not None def as_ancestor(self): - return GAEAncestorKey(self._get_real_key()) + return GAEAncestorKey(self.real_key()) def __cmp__(self, other): if not isinstance(other, GAEKey): @@ -72,4 +70,4 @@ def __hash__(self): return hash(self._real_key) def __str__(self): - return str(self.id_or_name) + return str(self._id_or_name) diff --git a/tests/keys.py b/tests/keys.py index 4e6bd06..f6be381 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -55,7 +55,7 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) self.assertEquals(child.pk.parent_key, parent.pk) - self.assertEquals(child.pk.parent_key.real_key, parent.pk.real_key) + self.assertEquals(child.pk.parent_key.real_key(), parent.pk.real_key()) def testAncestorFilterQuery(self): parent = ParentModel() From 902c3638a23f275a3e665315797b03e87ac55854 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 29 Dec 2011 16:18:50 -0500 Subject: [PATCH 09/22] remove missed @property, update unit tests --- models.py | 1 - tests/keys.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/models.py b/models.py index b3852e9..32a9da7 100644 --- a/models.py +++ b/models.py @@ -8,7 +8,6 @@ def __init__(self, key): self._key = key - @property def key(self): return self._key diff --git a/tests/keys.py b/tests/keys.py index f6be381..01b76f8 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -54,8 +54,8 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) - self.assertEquals(child.pk.parent_key, parent.pk) - self.assertEquals(child.pk.parent_key.real_key(), parent.pk.real_key()) + self.assertEquals(child.pk.parent_key(), parent.pk) + self.assertEquals(child.pk.parent_key().real_key(), parent.pk.real_key()) def testAncestorFilterQuery(self): parent = ParentModel() From 6aa66b7857e1ca71645142486512ade270335a6d Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Wed, 25 Jan 2012 16:53:30 -0500 Subject: [PATCH 10/22] use real_key for serialization --- fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fields.py b/fields.py index 8c711e4..f81691c 100644 --- a/fields.py +++ b/fields.py @@ -1,5 +1,6 @@ from django.core.exceptions import ValidationError from django.db import models +from django.utils.encoding import smart_unicode from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey @@ -76,3 +77,6 @@ def pre_save(self, model_instance, add): return key return super(GAEKeyField, self).pre_save(model_instance, add) + + def value_to_string(self, obj): + return smart_unicode(self._get_val_from_obj(obj).real_key()) From 216aa48c7686cc5bc07254b5834f09def1f30c77 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 18:11:58 -0400 Subject: [PATCH 11/22] modify SqlInsertCompiler to handle bulk inserts --- db/compiler.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index 7d9a249..5bd03a5 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -453,30 +453,36 @@ def convert_value_for_db(self, db_type, value): class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler): @safe_call - def insert(self, data, return_id=False): - gae_data = {} + def insert(self, docs, return_id=False): opts = self.query.get_meta() unindexed_fields = get_model_indexes(self.query.model)['unindexed'] unindexed_cols = [opts.get_field(name).column for name in unindexed_fields] - kwds = {'unindexed_properties': unindexed_cols} - for column, value in data.items(): - if column == opts.pk.column: - if isinstance(value, basestring): - kwds['name'] = value - else: - kwds['id'] = value - elif isinstance(value, (tuple, list)) and not len(value): - # gae does not store emty lists (and even does not allow passing empty - # lists to Entity.update) so skip them - continue - else: - gae_data[column] = value - entity = Entity(self.query.get_meta().db_table, **kwds) - entity.update(gae_data) - key = Put(entity) - return key.id_or_name() + entity_list = [] + for data in docs: + gae_data = {} + kwds = {'unindexed_properties': unindexed_cols} + for column, value in data.items(): + if column == opts.pk.column: + if isinstance(value, basestring): + kwds['name'] = value + else: + kwds['id'] = value + elif isinstance(value, (tuple, list)) and not len(value): + # gae does not store emty lists (and even does not allow passing empty + # lists to Entity.update) so skip them + continue + else: + gae_data[column] = value + + entity = Entity(opts.db_table, **kwds) + entity.update(gae_data) + entity_list.append(entity) + keys = Put(entity_list) + if not isinstance(keys, list): + keys = [keys] + return keys[0].id_or_name() class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): def execute_sql(self, result_type=MULTI): From 7df916edbf39a99309a1b2ac6aaa95c5fd871530 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 18:12:07 -0400 Subject: [PATCH 12/22] remove usage of XMLField --- tests/field_db_conversion.py | 6 +++--- tests/testmodels.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/field_db_conversion.py b/tests/field_db_conversion.py index e8a50d5..882322a 100644 --- a/tests/field_db_conversion.py +++ b/tests/field_db_conversion.py @@ -18,7 +18,7 @@ def test_db_conversion(self): comma_seperated_integer="5,4,3,2", ip_address='194.167.1.1', slug='you slugy slut :)', url='http://www.scholardocs.com', long_text=1000*'A', - indexed_text='hello', xml=2000*'B', + indexed_text='hello', integer=-400, small_integer=-4, positiv_integer=400, positiv_small_integer=4) entity.save() @@ -29,7 +29,7 @@ def test_db_conversion(self): entity.pk)) for name, gae_db_type in [('long_text', Text), - ('indexed_text', unicode), ('xml', Text), + ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('time', datetime.datetime), @@ -47,7 +47,7 @@ def test_db_conversion(self): # right types entity = FieldsWithoutOptionsModel.objects.get() for name, expected_type in [('long_text', unicode), - ('indexed_text', unicode), ('xml', unicode), + ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('datetime', datetime.datetime), diff --git a/tests/testmodels.py b/tests/testmodels.py index 125a30f..4999969 100644 --- a/tests/testmodels.py +++ b/tests/testmodels.py @@ -28,7 +28,6 @@ class FieldsWithoutOptionsModel(models.Model): # file_path = models.FilePathField() long_text = models.TextField() indexed_text = models.TextField() - xml = models.XMLField() integer = models.IntegerField() small_integer = models.SmallIntegerField() positiv_integer = models.PositiveIntegerField() @@ -63,7 +62,6 @@ class FieldsWithOptionsModel(models.Model): # file = FileField() # file_path = FilePathField() long_text = models.TextField(default=1000*'A') - xml = models.XMLField(default=2000*'B') integer = models.IntegerField(default=100) small_integer = models.SmallIntegerField(default=-5) positiv_integer = models.PositiveIntegerField(default=80) From 82080679a2309b7b1c4214b0af7aa44e750fc220 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 19:29:02 -0400 Subject: [PATCH 13/22] repr method for GAEKey --- models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/models.py b/models.py index 32a9da7..b1fe7e2 100644 --- a/models.py +++ b/models.py @@ -46,20 +46,20 @@ def as_ancestor(self): def __cmp__(self, other): if not isinstance(other, GAEKey): return 1 - + if self._real_key is not None and other._real_key is not None: return cmp(self._real_key, other._real_key) - + if self._id_or_name is None or other._id_or_name is None: raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) result = 0 if self._parent_key is not None: result = cmp(self._parent_key, other._parent_key) - + if result == 0: result = cmp(self._id_or_name, other._id_or_name) - + return result def __hash__(self): @@ -70,3 +70,6 @@ def __hash__(self): def __str__(self): return str(self._id_or_name) + + def __repr__(self): + return "%s(id_or_name=%r, parent_key=%r, real_key=%r)" % (self.__class__, self._id_or_name, self._parent_key, self._real_key) From 2fc14a5cfe2d8ea1c732337e07f9580e4b50eb1c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 19:32:03 -0400 Subject: [PATCH 14/22] do not encode cursor if it doesnt exist --- db/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/utils.py b/db/utils.py index 76085c3..5376d5c 100644 --- a/db/utils.py +++ b/db/utils.py @@ -16,7 +16,9 @@ def get_cursor(queryset): # Evaluate QuerySet len(queryset) cursor = getattr(queryset.query, '_gae_cursor', None) - return Cursor.to_websafe_string(cursor) + if cursor: + return Cursor.to_websafe_string(cursor) + return None def set_cursor(queryset, start=None, end=None): queryset = queryset.all() From b23cc69028f529bac25266e3f292d01d5a609072 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 22:01:58 -0400 Subject: [PATCH 15/22] do not allow null primary keys, check for empty strings when creating keys --- fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fields.py b/fields.py index f81691c..de981ed 100644 --- a/fields.py +++ b/fields.py @@ -9,7 +9,6 @@ class GAEKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) @@ -23,7 +22,7 @@ def contribute_to_class(self, cls, name): assert not cls._meta.has_auto_field, "A model can't have more than one auto field." cls._meta.has_auto_field = True cls._meta.auto_field = self - + if self.parent_key_attname is not None: def get_parent_key(instance, instance_type=None): if instance is None: @@ -46,6 +45,8 @@ def set_parent_key(instance, value): def to_python(self, value): if value is None: return None + if isinstance(value, basestring) and len(value) == 0: + return None if isinstance(value, GAEKey): return value if isinstance(value, Key): @@ -62,7 +63,7 @@ def to_python(self, value): def get_prep_value(self, value): if isinstance(value, GAEAncestorKey): - return value + return value return self.to_python(value) def formfield(self, **kwargs): From 6a10a9d3dcdb47db7e7808e2eecd26e82b02df2d Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 23 Mar 2012 15:50:40 -0400 Subject: [PATCH 16/22] rewrite ancestor queries for type-conversion-refactor --- db/base.py | 3 +- db/compiler.py | 11 ++-- db/utils.py | 18 +++++-- fields.py | 67 +++++++++++++----------- models.py | 75 -------------------------- tests/__init__.py | 4 +- tests/keys.py | 131 +++++++++++++++++++++------------------------- 7 files changed, 118 insertions(+), 191 deletions(-) diff --git a/db/base.py b/db/base.py index 24d04af..631bf36 100644 --- a/db/base.py +++ b/db/base.py @@ -155,7 +155,8 @@ def _value_for_db(self, value, field, field_kind, db_type, lookup): if db_type == 'key': # value = self._value_for_db_key(value, field_kind) try: - value = key_from_path(field.model._meta.db_table, value) + if not isinstance(value, Key): + value = key_from_path(field.model._meta.db_table, value) except (BadArgumentError, BadValueError,): raise DatabaseError("Only strings and positive integers " "may be used as keys on GAE.") diff --git a/db/compiler.py b/db/compiler.py index c880f4d..5c7eb5d 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -22,8 +22,6 @@ from .db_settings import get_model_indexes from .expressions import ExpressionEvaluator -from ..fields import GAEKeyField -from ..models import GAEKey, GAEAncestorKey from .utils import commit_locked @@ -193,12 +191,12 @@ def add_filter(self, field, lookup_type, negated, value): # Optimization: batch-get by key; this is only suitable for # primary keys, not for anything that uses the key type. if field.primary_key and lookup_type in ('exact', 'in'): - if lookup_type == 'exact' and isinstance(value, GAEAncestorKey): + if lookup_type == 'exact' and isinstance(value, AncestorKey): if negated: - raise DatabaseError("You can't negate an ancestor operation.") + raise DatabaseError("You can't negate an ancestor operator.") if self.ancestor_key is not None: - raise DatabaseError("You can't use more than one ancestor operation.") - self.ancestor_key = value.key() + raise DatabaseError("You can't use more than one ancestor operator.") + self.ancestor_key = value.key return if self.included_pks is not None: @@ -383,6 +381,7 @@ def insert(self, data, return_id=False): if value is not None: kwds['id'] = value.id() kwds['name'] = value.name() + kwds['parent'] = value.parent() # GAE does not store empty lists (and even does not allow # passing empty lists to Entity.update) so skip them. diff --git a/db/utils.py b/db/utils.py index 068715d..e324697 100644 --- a/db/utils.py +++ b/db/utils.py @@ -1,4 +1,6 @@ +from google.appengine.api.datastore import Key from google.appengine.datastore.datastore_query import Cursor + from django.db import models, DEFAULT_DB_ALIAS try: @@ -20,9 +22,7 @@ def get_cursor(queryset): # Evaluate QuerySet. len(queryset) cursor = getattr(queryset.query, '_gae_cursor', None) - if cursor: - return Cursor.to_websafe_string(cursor) - return None + return Cursor.to_websafe_string(cursor) if cursor else None def set_cursor(queryset, start=None, end=None): @@ -59,3 +59,15 @@ def _commit_locked(*args, **kw): if callable(func_or_using): return inner_commit_locked(func_or_using, DEFAULT_DB_ALIAS) return lambda func: inner_commit_locked(func, func_or_using) + +class AncestorKey(object): + def __init__(self, key): + self.key = key + +def as_ancestor(key): + if key is None: + raise ValueError("key must not be None") + return AncestorKey(key) + +def make_key(model, id_or_name, parent=None): + return Key.from_path(model._meta.db_table, id_or_name, parent=parent) diff --git a/fields.py b/fields.py index de981ed..e013482 100644 --- a/fields.py +++ b/fields.py @@ -1,21 +1,27 @@ from django.core.exceptions import ValidationError from django.db import models from django.utils.encoding import smart_unicode + +from djangoappengine.db.utils import AncestorKey + from google.appengine.api.datastore import Key, datastore_errors -from .models import GAEKey, GAEAncestorKey -class GAEKeyField(models.Field): - description = "A field for Google AppEngine Key objects" +import logging + +class DbKeyField(models.Field): + description = "A field for native database key objects" __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): + kwargs['null'] = True kwargs['blank'] = True + self.parent_key_attname = kwargs.pop('parent_key_name', None) if self.parent_key_attname is not None and kwargs.get('primary_key', None) is None: - raise ValueError("Primary key must be true to set parent_key_name") + raise ValueError("Primary key must be true to use parent_key_name") - super(GAEKeyField, self).__init__(*args, **kwargs) + super(DbKeyField, self).__init__(*args, **kwargs) def contribute_to_class(self, cls, name): if self.primary_key: @@ -27,57 +33,54 @@ def contribute_to_class(self, cls, name): def get_parent_key(instance, instance_type=None): if instance is None: return self + return instance.__dict__.get(self.parent_key_attname) def set_parent_key(instance, value): if instance is None: raise AttributeError("Attribute must be accessed via instance") - if not isinstance(value, GAEKey): - raise ValueError("parent must be a GAEKey") + if not isinstance(value, Key): + raise ValueError("'%s' must be a Key" % self.parent_key_attname) instance.__dict__[self.parent_key_attname] = value setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) - super(GAEKeyField, self).contribute_to_class(cls, name) + super(DbKeyField, self).contribute_to_class(cls, name) def to_python(self, value): if value is None: return None - if isinstance(value, basestring) and len(value) == 0: - return None - if isinstance(value, GAEKey): - return value if isinstance(value, Key): - return GAEKey(real_key=value) + return value if isinstance(value, basestring): try: - return GAEKey(real_key=Key(encoded=value)) + return Key(encoded=value) except datastore_errors.BadKeyError: - return GAEKey(real_key=Key.from_path(self.model._meta.db_table, long(value))) + return Key.from_path(self.model._meta.db_table, long(value)) if isinstance(value, (int, long)): - return GAEKey(real_key=Key.from_path(self.model._meta.db_table, value)) + return Key.from_path(self.model._meta.db_table, value) - raise ValidationError("GAEKeyField does not accept %s" % type(value)) + raise ValidationError("DbKeyField does not accept %s" % type(value)) - def get_prep_value(self, value): - if isinstance(value, GAEAncestorKey): - return value - return self.to_python(value) + def pre_save(self, model_instance, add): + value = super(DbKeyField, self).pre_save(model_instance, add) - def formfield(self, **kwargs): - return None + if add and value is None and self.parent_key_attname is not None and hasattr(model_instance, self.parent_key_attname): + stashed_parent = getattr(model_instance, self.parent_key_attname) + value = Key.from_path(self.model._meta.db_table, 0, parent=stashed_parent) - def pre_save(self, model_instance, add): - if add and self.parent_key_attname is not None: - parent_key = getattr(model_instance, self.parent_key_attname) - if parent_key is not None: - key = GAEKey(parent_key=parent_key) - setattr(model_instance, self.attname, key) - return key + return value - return super(GAEKeyField, self).pre_save(model_instance, add) + def get_prep_lookup(self, lookup_type, value): + if not isinstance(value, (Key, AncestorKey)): + raise ValueError(u"'%s' only accepts Key or ancestor objects, not %s" % (self.name, type(value))) + + return value + + def formfield(self, **kwargs): + return None def value_to_string(self, obj): - return smart_unicode(self._get_val_from_obj(obj).real_key()) + return smart_unicode(self._get_val_from_obj(obj)) diff --git a/models.py b/models.py index b1fe7e2..e69de29 100644 --- a/models.py +++ b/models.py @@ -1,75 +0,0 @@ -from django.db import models -from google.appengine.api.datastore import Key - -class GAEAncestorKey(object): - def __init__(self, key): - if not isinstance(key, Key): - raise ValueError('key must be of type Key') - - self._key = key - - def key(self): - return self._key - -class GAEKey(object): - def __init__(self, id_or_name=None, parent_key=None, real_key=None): - self._id_or_name = id_or_name - self._parent_key = parent_key - self._real_key = None - - if real_key is not None: - if id_or_name is not None or parent_key is not None: - raise ValueError("You can't set both a real_key and an id_or_name or parent_key") - - self._real_key = real_key - if real_key.parent(): - self._parent_key = GAEKey(real_key=real_key.parent()) - self._id_or_name = real_key.id_or_name() - - def id_or_name(self): - return self._id_or_name - - def parent_key(self): - return self._parent_key - - def real_key(self): - if self._real_key is None: - raise AttributeError("Incomplete key, please save the entity first.") - return self._real_key - - def has_real_key(self): - return self._real_key is not None - - def as_ancestor(self): - return GAEAncestorKey(self.real_key()) - - def __cmp__(self, other): - if not isinstance(other, GAEKey): - return 1 - - if self._real_key is not None and other._real_key is not None: - return cmp(self._real_key, other._real_key) - - if self._id_or_name is None or other._id_or_name is None: - raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) - - result = 0 - if self._parent_key is not None: - result = cmp(self._parent_key, other._parent_key) - - if result == 0: - result = cmp(self._id_or_name, other._id_or_name) - - return result - - def __hash__(self): - if self._real_key is None: - raise ValueError("You can't hash an unsaved key.") - - return hash(self._real_key) - - def __str__(self): - return str(self._id_or_name) - - def __repr__(self): - return "%s(id_or_name=%r, parent_key=%r, real_key=%r)" % (self.__class__, self._id_or_name, self._parent_key, self._real_key) diff --git a/tests/__init__.py b/tests/__init__.py index 9851772..7da57e2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,9 +1,9 @@ from .backend import BackendTest +from .decimals import DecimalTest from .field_db_conversion import FieldDBConversionTest from .field_options import FieldOptionsTest from .filter import FilterTest -from .keys import KeysTest +from .keys import KeysTest, DbKeyFieldTest, AncestorQueryTest, ParentKeyTest from .not_return_sets import NonReturnSetsTest from .order import OrderTest from .transactions import TransactionTest -from .keys import AncestorKeysTest, KeysTest diff --git a/tests/keys.py b/tests/keys.py index 517ce37..4603435 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -6,12 +6,12 @@ from django.test import TestCase from django.utils import unittest +from djangoappengine.fields import DbKeyField +from djangoappengine.db.utils import as_ancestor from djangotoolbox.fields import ListField from google.appengine.api.datastore import Key -from ..fields import GAEKeyField -from ..models import GAEKey class AutoKey(models.Model): @@ -295,41 +295,49 @@ def test_key_kind(self): class ParentModel(models.Model): - key = GAEKeyField(primary_key=True) + key = DbKeyField(primary_key=True) -class NonGAEParentModel(models.Model): +class NonDbKeyParentModel(models.Model): id = models.AutoField(primary_key=True) class ChildModel(models.Model): - key = GAEKeyField(primary_key=True, parent_key_name='parent_key') + key = DbKeyField(primary_key=True, parent_key_name='parent_key') class AnotherChildModel(models.Model): - key = GAEKeyField(primary_key=True, parent_key_name='also_parent_key') + key = DbKeyField(primary_key=True, parent_key_name='parent_key') class ForeignKeyModel(models.Model): id = models.AutoField(primary_key=True) relation = models.ForeignKey(ParentModel) -class AncestorKeysTest(TestCase): - def testGAEKeySave(self): +class DbKeyFieldTest(TestCase): + def testDbKeySave(self): model = ParentModel() model.save() self.assertIsNotNone(model.pk) - def testUnsavedParent(self): + def testForeignKeyWithGAEKey(self): parent = ParentModel() + parent.save() - with self.assertRaises(ValueError): - child = ChildModel(parent_key=parent.pk) + fkm = ForeignKeyModel() + fkm.relation = parent + fkm.save() - def testNonGAEParent(self): - parent = NonGAEParentModel() + results = list(ForeignKeyModel.objects.filter(relation=parent)) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, fkm.pk) + + def testPrimaryKeyQuery(self): + parent = ParentModel() parent.save() - with self.assertRaises(ValueError): - child = ChildModel(parent_key=parent.pk) + db_parent = ParentModel.objects.get(pk=parent.pk) + self.assertEquals(parent.pk, db_parent.pk) + +class ParentKeyTest(TestCase): def testParentChildSave(self): parent = ParentModel() orig_parent_pk = parent.pk @@ -341,9 +349,24 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) - self.assertEquals(child.pk.parent_key(), parent.pk) - self.assertEquals(child.pk.parent_key().real_key(), parent.pk.real_key()) + self.assertEquals(child.pk.parent(), parent.pk) + + def testParentModelChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent) + + def testNonDbKeyParent(self): + parent = NonDbKeyParentModel() + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) +class AncestorQueryTest(TestCase): def testAncestorFilterQuery(self): parent = ParentModel() parent.save() @@ -351,7 +374,7 @@ def testAncestorFilterQuery(self): child = ChildModel(parent_key=parent.pk) child.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(1, len(results)) self.assertEquals(results[0].pk, child.pk) @@ -363,7 +386,7 @@ def testAncestorGetQuery(self): child = ChildModel(parent_key=parent.pk) child.save() - result = ChildModel.objects.get(pk=parent.pk.as_ancestor()) + result = ChildModel.objects.get(pk=as_ancestor(parent.pk)) self.assertEquals(result.pk, child.pk) @@ -371,7 +394,7 @@ def testEmptyAncestorQuery(self): parent = ParentModel() parent.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(0, len(results)) @@ -381,15 +404,15 @@ def testEmptyAncestorQueryWithUnsavedChild(self): child = ChildModel(parent_key=parent.pk) - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(0, len(results)) def testUnsavedAncestorQuery(self): parent = ParentModel() - with self.assertRaises(AttributeError): - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + with self.assertRaises(ValueError): + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) def testDifferentChildrenAncestorQuery(self): parent = ParentModel() @@ -397,17 +420,17 @@ def testDifferentChildrenAncestorQuery(self): child1 = ChildModel(parent_key=parent.pk) child1.save() - child2 = AnotherChildModel(also_parent_key=parent.pk) + child2 = AnotherChildModel(parent_key=parent.pk) child2.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child1.pk) + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) - results = list(AnotherChildModel.objects.filter(pk=parent.pk.as_ancestor())) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child2.pk) + results2 = list(AnotherChildModel.objects.filter(pk=as_ancestor(parent.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) def testDifferentParentsAncestorQuery(self): parent1 = ParentModel() @@ -422,47 +445,11 @@ def testDifferentParentsAncestorQuery(self): child2 = ChildModel(parent_key=parent2.pk) child2.save() - results = list(ChildModel.objects.filter(pk=parent1.pk.as_ancestor())) - - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child1.pk) + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent1.pk))) - results = list(ChildModel.objects.filter(pk=parent2.pk.as_ancestor())) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child2.pk) + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) - def testForeignKeyWithGAEKey(self): - parent = ParentModel() - parent.save() - - fkm = ForeignKeyModel() - fkm.relation = parent - fkm.save() - - results = list(ForeignKeyModel.objects.filter(relation=parent)) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, fkm.pk) - - def testPrimaryKeyQuery(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=parent.pk) - - self.assertEquals(parent.pk, db_parent.pk) - - def testPrimaryKeyQueryStringKey(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=str(parent.pk)) - - self.assertEquals(parent.pk, db_parent.pk) - - def testPrimaryKeyQueryIntKey(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=int(str(parent.pk))) - - self.assertEquals(parent.pk, db_parent.pk) + results2 = list(ChildModel.objects.filter(pk=as_ancestor(parent2.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) From 7407854cebd3a90d00fa868e2f7c87ac89d952b9 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 19 Apr 2012 21:45:26 -0400 Subject: [PATCH 17/22] DBKeyField is a special case for conversion --- db/creation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/creation.py b/db/creation.py index 4848190..c1749b4 100644 --- a/db/creation.py +++ b/db/creation.py @@ -32,6 +32,13 @@ def db_type(self, field): field is to be indexed, and the "text" db_type (db.Text) if it's registered as unindexed. """ + from djangoappengine.fields import DbKeyField + + # DBKeyField reads/stores db.Key objects directly + # so its treated as a special case + if isinstance(field, DbKeyField): + return field.db_type(connection=self.connection) + if self.connection.settings_dict.get('STORE_RELATIONS_AS_DB_KEYS'): if field.primary_key or field.rel is not None: return 'key' From b77c18306973391eb9aba99935ae70494e0dc4cd Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:10:15 -0400 Subject: [PATCH 18/22] add missing import --- db/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/compiler.py b/db/compiler.py index 0bb5e0b..0ce7d23 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -22,7 +22,7 @@ from .db_settings import get_model_indexes from .expressions import ExpressionEvaluator -from .utils import commit_locked +from .utils import AncestorKey, commit_locked # Valid query types (a dictionary is used for speedy lookups). From 9be9b5f634e075be8a72adc5036bc21c5ef37e99 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:16:40 -0400 Subject: [PATCH 19/22] dbkeyfield should not be forced to nullable --- fields.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fields.py b/fields.py index e013482..54197d5 100644 --- a/fields.py +++ b/fields.py @@ -13,7 +13,6 @@ class DbKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) @@ -55,6 +54,9 @@ def to_python(self, value): if isinstance(value, Key): return value if isinstance(value, basestring): + if len(value) == 0: + return None + try: return Key(encoded=value) except datastore_errors.BadKeyError: @@ -64,6 +66,11 @@ def to_python(self, value): raise ValidationError("DbKeyField does not accept %s" % type(value)) + def get_prep_value(self, value): + if isinstance(value, AncestorKey): + return value + return self.to_python(value) + def pre_save(self, model_instance, add): value = super(DbKeyField, self).pre_save(model_instance, add) @@ -73,12 +80,6 @@ def pre_save(self, model_instance, add): return value - def get_prep_lookup(self, lookup_type, value): - if not isinstance(value, (Key, AncestorKey)): - raise ValueError(u"'%s' only accepts Key or ancestor objects, not %s" % (self.name, type(value))) - - return value - def formfield(self, **kwargs): return None From 4c6ebdf6cb80fa8c670b69ef4d48901290d96ad3 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:17:20 -0400 Subject: [PATCH 20/22] added make_key function to simplify created DbKeys from models --- db/utils.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/db/utils.py b/db/utils.py index e324697..f7540be 100644 --- a/db/utils.py +++ b/db/utils.py @@ -64,10 +64,30 @@ class AncestorKey(object): def __init__(self, key): self.key = key -def as_ancestor(key): - if key is None: - raise ValueError("key must not be None") - return AncestorKey(key) +def as_ancestor(key_or_model): + if key_or_model is None: + raise ValueError("key_or_model must not be None") -def make_key(model, id_or_name, parent=None): - return Key.from_path(model._meta.db_table, id_or_name, parent=parent) + if isinstance(key_or_model, models.Model): + key_or_model = Key.from_path(key_or_model._meta.db_table, key_or_model.pk) + + return AncestorKey(key_or_model) + +def make_key(*args, **kwargs): + parent = kwargs.pop('parent', None) + + if kwargs: + raise AssertionError('Excess keyword arguments; received %s' % kwargs) + + if not args or len(args) % 2: + raise AssertionError('A non-zero even number of positional arguments is required; received %s' % args) + + if isinstance(parent, models.Model): + parent = Key.from_path(parent._meta.db_table, parent.pk) + + converted_args = [] + for i in xrange(0, len(args), 2): + model, id_or_name = args[i:i+2] + converted_args.extend((model._meta.db_table, id_or_name)) + + return Key.from_path(*converted_args, parent=parent) From 13c8b64749a84043707b55b11cd948442eb2b6bf Mon Sep 17 00:00:00 2001 From: User Date: Wed, 23 May 2012 17:52:12 -0400 Subject: [PATCH 21/22] Merge branch 'feature/ancestor-query-1.4' of git://github.com/django-nonrel/djangoappengine into develop Conflicts: db/compiler.py --- db/creation.py | 9 +++++---- tests/__init__.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/db/creation.py b/db/creation.py index c1749b4..7a269dc 100644 --- a/db/creation.py +++ b/db/creation.py @@ -77,7 +77,8 @@ def _create_test_db(self, *args, **kw): stub_manager.activate_test_stubs() def _destroy_test_db(self, *args, **kw): - if self._had_test_stubs: - stub_manager.deactivate_test_stubs() - stub_manager.setup_stubs(self.connection) - del self._had_test_stubs + if hasattr(self, '_had_test_stubs'): + if self._had_test_stubs: + stub_manager.deactivate_test_stubs() + stub_manager.setup_stubs(self.connection) + del self._had_test_stubs diff --git a/tests/__init__.py b/tests/__init__.py index 7da57e2..fb3bf09 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,5 @@ from .backend import BackendTest -from .decimals import DecimalTest +#from .decimals import DecimalTest from .field_db_conversion import FieldDBConversionTest from .field_options import FieldOptionsTest from .filter import FilterTest From a417ac0cbc2284a1c6a59da3fc6663040af94354 Mon Sep 17 00:00:00 2001 From: User Date: Fri, 25 May 2012 17:45:38 -0400 Subject: [PATCH 22/22] Add some syntactic sugar classes for doing ancestor queries. --- db/models/__init__.py | 0 db/models/manager.py | 14 +++++++++++++ db/models/query.py | 17 ++++++++++++++++ test.py | 32 +++++++++++++++++++++++++++++ tests/__init__.py | 1 + tests/ancestor.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 111 insertions(+) create mode 100644 db/models/__init__.py create mode 100644 db/models/manager.py create mode 100644 db/models/query.py create mode 100644 test.py create mode 100644 tests/ancestor.py diff --git a/db/models/__init__.py b/db/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/models/manager.py b/db/models/manager.py new file mode 100644 index 0000000..5b5e732 --- /dev/null +++ b/db/models/manager.py @@ -0,0 +1,14 @@ +from django.db.models import Manager as _baseManager +from djangoappengine.db.utils import as_ancestor +from djangoappengine.db.models.query import QuerySet + +class Manager(_baseManager): + + def get_query_set(self): + """Returns a new QuerySet object. Subclasses can override this method + to easily customize the behavior of the Manager. + """ + return QuerySet(self.model, using=self._db) + + def ancestor(self, ancestor): + return self.get_query_set().ancestor(ancestor) \ No newline at end of file diff --git a/db/models/query.py b/db/models/query.py new file mode 100644 index 0000000..d81c661 --- /dev/null +++ b/db/models/query.py @@ -0,0 +1,17 @@ +from django.db.models.query import QuerySet as _baseQuerySet +from djangoappengine.db.utils import as_ancestor + +class QuerySet(_baseQuerySet): + def ancestor(self, ancestor): + """ + Returns a new QuerySet instance with the args ANDed to the existing + set. + """ + return self._filter_or_exclude(False, pk=as_ancestor(ancestor)) + +class EmptyQuerySet(QuerySet): + def ancestor(self, *args, **kwargs): + """ + Always returns EmptyQuerySet. + """ + return self \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..ae8a773 --- /dev/null +++ b/test.py @@ -0,0 +1,32 @@ +from django.test import TestCase + +from google.appengine.datastore import datastore_stub_util + +from db.stubs import stub_manager + +class GAETestCase(TestCase): + def _pre_setup(self): + """Performs any pre-test setup. + * Set the dev_appserver consistency state. + """ + super(GAETestCase,self)._pre_setup() + + if hasattr(self, 'consistency_probability'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + self._orig_policy = datastore._consistency_policy + + datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=self.consistency_probability)) + + + def _post_teardown(self): + """ Performs any post-test things. This includes: + + * Putting back the original ROOT_URLCONF if it was changed. + * Force closing the connection, so that the next test gets + a clean cursor. + """ + if hasattr(self, '_orig_policy'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + datastore.SetConsistencyPolicy(self._orig_policy) + + super(GAETestCase,self)._post_teardown() diff --git a/tests/__init__.py b/tests/__init__.py index fb3bf09..75b8a05 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,3 +7,4 @@ from .not_return_sets import NonReturnSetsTest from .order import OrderTest from .transactions import TransactionTest +from .ancestor import AncestorTest diff --git a/tests/ancestor.py b/tests/ancestor.py new file mode 100644 index 0000000..0f4b815 --- /dev/null +++ b/tests/ancestor.py @@ -0,0 +1,47 @@ +from django.test import TestCase +from django.utils import unittest +from django.db import models + +from djangoappengine.fields import DbKeyField + +from djangoappengine.db.models.manager import Manager + +#from djangotoolbox.fields import ListField +#from google.appengine.api.datastore import Key + +class ParentFoo(models.Model): + key = DbKeyField(primary_key=True) + foo = models.IntegerField() + objects = Manager() + +class ChildFoo(models.Model): + key = DbKeyField(primary_key=True, parent_key_name='parent_key') + foo = models.IntegerField() + objects = Manager() + +class AncestorTest(TestCase): + def test_simple(self): + px = ParentFoo(foo=5) + px.save() + px = ParentFoo(foo=2) + px.save() + + parents = ParentFoo.objects.all() + self.assertEqual(2, parents.count()) + + parents = ParentFoo.objects.filter(foo=2) + self.assertEqual(1, parents.count()) + + child = ChildFoo(foo=10, parent_key=px.pk) + orig_child_pk = child.pk + child.save() + + results = list(ChildFoo.objects.ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + results = list(ChildFoo.objects.all().ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) \ No newline at end of file