From 42f3c9c72eb3a9f96c14375d668b5b2bd9712e38 Mon Sep 17 00:00:00 2001 From: stillinbeta Date: Sat, 22 Oct 2011 01:37:01 -0400 Subject: [PATCH 1/5] Allow dot notation indices of embedded documents --- django_mongodb_engine/creation.py | 10 +++++++++- tests/mongodb/models.py | 8 ++++++-- tests/mongodb/tests.py | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/django_mongodb_engine/creation.py b/django_mongodb_engine/creation.py index 5413a7bf..a72b8c59 100644 --- a/django_mongodb_engine/creation.py +++ b/django_mongodb_engine/creation.py @@ -48,13 +48,21 @@ def _handle_newstyle_indexes(self, ensure_index, meta, indexes): assert isinstance(fields, (list, tuple)) indexes.append({'fields': make_index_list(fields), 'unique': True}) + #Allow for embedded document indexes, with proper root field names + def find_field(name): + dot = name.find('.') + if dot == -1: + dot = len(name) + root = meta.get_field(name[0:dot]).column + return root + name[dot:] + for index in indexes: if isinstance(index, dict): kwargs = index.copy() fields = kwargs.pop('fields') else: fields, kwargs = index, {} - fields = [(meta.get_field(name).column, direction) + fields = [(find_field(name), direction) for name, direction in make_index_list(fields)] ensure_index(fields, **kwargs) diff --git a/tests/mongodb/models.py b/tests/mongodb/models.py index ab3e9258..053a25bc 100644 --- a/tests/mongodb/models.py +++ b/tests/mongodb/models.py @@ -1,5 +1,5 @@ from django.db import models -from djangotoolbox.fields import RawField, ListField, EmbeddedModelField +from djangotoolbox.fields import RawField, ListField, EmbeddedModelField, DictField from django_mongodb_engine.fields import GridFSField, GridFSString from query.models import Post @@ -49,6 +49,8 @@ class NewStyleIndexesTestModel(models.Model): f = models.IntegerField(unique=True) geo = models.IntegerField() geo2 = models.IntegerField(db_column='geo') + h = DictField() + i = DictField(db_column='i2') class Meta: unique_together = [('a', 'b'), ('a', 'd')] @@ -59,7 +61,9 @@ class MongoMeta: {'fields': 'a', 'sparse': True}, {'fields': [('b', -1), 'd']}, [('geo', '2d')], - {'fields': [('geo2', '2d'), 'a'], 'min': 42, 'max': 21} + {'fields': [('geo2', '2d'), 'a'], 'min': 42, 'max': 21}, + {'fields': [('h.q',1)]}, + {'fields': [('i.q',1)]} ] class GridFSFieldTestModel(models.Model): diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 1296a7c3..7059797d 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -346,6 +346,8 @@ def test_indexes(self): self.assertHaveIndex([('b2', -1), ('d', 1)]) self.assertHaveIndex([('geo', '2d')]) self.assertHaveIndex([('geo', '2d'), ('a', 1)], min=42, max=21) + self.assertHaveIndex([('h.q', 1)]) + self.assertHaveIndex([('i2.q', 1)]) class GridFSFieldTests(TestCase): def tearDown(self): From 3ccdfaada97ac532719c0815bc2a5934659aa5af Mon Sep 17 00:00:00 2001 From: stillinbeta Date: Sat, 22 Oct 2011 13:29:29 -0400 Subject: [PATCH 2/5] Recursively check db_column names for embedded fields --- django_mongodb_engine/creation.py | 21 +++++++++++++++++---- tests/mongodb/models.py | 13 +++++++++++-- tests/mongodb/tests.py | 2 ++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/django_mongodb_engine/creation.py b/django_mongodb_engine/creation.py index a72b8c59..5a07ae4b 100644 --- a/django_mongodb_engine/creation.py +++ b/django_mongodb_engine/creation.py @@ -44,17 +44,30 @@ def _handle_newstyle_indexes(self, ensure_index, meta, indexes): # Django unique_together indexes indexes = list(indexes) + for fields in getattr(meta, 'unique_together', []): assert isinstance(fields, (list, tuple)) indexes.append({'fields': make_index_list(fields), 'unique': True}) #Allow for embedded document indexes, with proper root field names - def find_field(name): + def find_field(name,meta): dot = name.find('.') if dot == -1: dot = len(name) - root = meta.get_field(name[0:dot]).column - return root + name[dot:] + field = meta.get_field(name[:dot]) + model = None + new_field = field + #Allow arbitrarily deep nesting of ListFields or SetFields + while True: + if hasattr(new_field,'embedded_model'): + model = new_field.embedded_model + break + elif hasattr(new_field, 'item_field'): + new_field = new_field.item_field + else: + return field.column + name[dot:] + + return field.column + '.' + find_field(name[dot+1:], model._meta) for index in indexes: if isinstance(index, dict): @@ -62,7 +75,7 @@ def find_field(name): fields = kwargs.pop('fields') else: fields, kwargs = index, {} - fields = [(find_field(name), direction) + fields = [(find_field(name,meta), direction) for name, direction in make_index_list(fields)] ensure_index(fields, **kwargs) diff --git a/tests/mongodb/models.py b/tests/mongodb/models.py index 053a25bc..51126594 100644 --- a/tests/mongodb/models.py +++ b/tests/mongodb/models.py @@ -40,6 +40,9 @@ class IndexTestModel2(models.Model): class MongoMeta: index_together = ['a', ('b', -1)] +class CustomColumnEmbeddedModel(models.Model): + a = models.IntegerField(db_column='a2') + class NewStyleIndexesTestModel(models.Model): a = models.IntegerField() b = models.IntegerField(db_column='b2') @@ -49,9 +52,12 @@ class NewStyleIndexesTestModel(models.Model): f = models.IntegerField(unique=True) geo = models.IntegerField() geo2 = models.IntegerField(db_column='geo') + # Embedded Keys h = DictField() i = DictField(db_column='i2') - + # Embedded Keys in Embedded Models + j = EmbeddedModelField(CustomColumnEmbeddedModel) + k = ListField(EmbeddedModelField(CustomColumnEmbeddedModel)) class Meta: unique_together = [('a', 'b'), ('a', 'd')] @@ -63,7 +69,10 @@ class MongoMeta: [('geo', '2d')], {'fields': [('geo2', '2d'), 'a'], 'min': 42, 'max': 21}, {'fields': [('h.q',1)]}, - {'fields': [('i.q',1)]} + {'fields': [('i.q',1)]}, + {'fields' :[('j.a',1)]}, + {'fields' :[('k.a',1)]}, + ] class GridFSFieldTestModel(models.Model): diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 7059797d..aefad16e 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -348,6 +348,8 @@ def test_indexes(self): self.assertHaveIndex([('geo', '2d'), ('a', 1)], min=42, max=21) self.assertHaveIndex([('h.q', 1)]) self.assertHaveIndex([('i2.q', 1)]) + self.assertHaveIndex([('j.a2',1)]) + self.assertHaveIndex([('k.a2',1)]) class GridFSFieldTests(TestCase): def tearDown(self): From 55f4257f0121e16e3b212f934aaacdf3b8904a93 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 22 Oct 2011 20:55:13 +0200 Subject: [PATCH 3/5] Cleanup --- django_mongodb_engine/creation.py | 34 +++++++++++------------ tests/mongodb/models.py | 45 ++++++++++++++++--------------- tests/mongodb/tests.py | 24 ++++++++--------- 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/django_mongodb_engine/creation.py b/django_mongodb_engine/creation.py index 5a07ae4b..6d322805 100644 --- a/django_mongodb_engine/creation.py +++ b/django_mongodb_engine/creation.py @@ -34,6 +34,8 @@ def ensure_index(*args, **kwargs): self._handle_oldstyle_indexes(ensure_index, meta) def _handle_newstyle_indexes(self, ensure_index, meta, indexes): + from djangotoolbox.fields import AbstractIterableField, EmbeddedModelField + # Django indexes for field in meta.local_fields: if not (field.unique or field.db_index): @@ -49,25 +51,19 @@ def _handle_newstyle_indexes(self, ensure_index, meta, indexes): assert isinstance(fields, (list, tuple)) indexes.append({'fields': make_index_list(fields), 'unique': True}) - #Allow for embedded document indexes, with proper root field names - def find_field(name,meta): - dot = name.find('.') - if dot == -1: - dot = len(name) - field = meta.get_field(name[:dot]) - model = None - new_field = field - #Allow arbitrarily deep nesting of ListFields or SetFields - while True: - if hasattr(new_field,'embedded_model'): - model = new_field.embedded_model + def get_column_name(field): + opts = meta + parts = field.split('.') + for i, part in enumerate(parts): + field = opts.get_field(part) + parts[i] = field.column + if isinstance(field, AbstractIterableField): + field = field.item_field + if isinstance(field, EmbeddedModelField): + opts = field.embedded_model._meta + else: break - elif hasattr(new_field, 'item_field'): - new_field = new_field.item_field - else: - return field.column + name[dot:] - - return field.column + '.' + find_field(name[dot+1:], model._meta) + return '.'.join(parts) for index in indexes: if isinstance(index, dict): @@ -75,7 +71,7 @@ def find_field(name,meta): fields = kwargs.pop('fields') else: fields, kwargs = index, {} - fields = [(find_field(name,meta), direction) + fields = [(get_column_name(name), direction) for name, direction in make_index_list(fields)] ensure_index(fields, **kwargs) diff --git a/tests/mongodb/models.py b/tests/mongodb/models.py index 51126594..8456d1a1 100644 --- a/tests/mongodb/models.py +++ b/tests/mongodb/models.py @@ -44,34 +44,35 @@ class CustomColumnEmbeddedModel(models.Model): a = models.IntegerField(db_column='a2') class NewStyleIndexesTestModel(models.Model): - a = models.IntegerField() - b = models.IntegerField(db_column='b2') - c = models.IntegerField(db_index=True) - d = models.IntegerField() - e = models.IntegerField() - f = models.IntegerField(unique=True) + f1 = models.IntegerField() + f2 = models.IntegerField() + f3 = models.IntegerField() + + db_index = models.IntegerField(db_index=True) + unique = models.IntegerField(unique=True) + custom_column = models.IntegerField(db_column='custom') geo = models.IntegerField() - geo2 = models.IntegerField(db_column='geo') - # Embedded Keys - h = DictField() - i = DictField(db_column='i2') - # Embedded Keys in Embedded Models - j = EmbeddedModelField(CustomColumnEmbeddedModel) - k = ListField(EmbeddedModelField(CustomColumnEmbeddedModel)) + geo_custom_column = models.IntegerField(db_column='geo') + + dict1 = DictField() + dict_custom_column = DictField(db_column='dict_custom') + embedded = EmbeddedModelField(CustomColumnEmbeddedModel) + embedded_list = ListField(EmbeddedModelField(CustomColumnEmbeddedModel)) + class Meta: - unique_together = [('a', 'b'), ('a', 'd')] + unique_together = [('f2', 'custom_column'), ('f2', 'f3')] class MongoMeta: indexes = [ - [('e', -1)], - {'fields': 'a', 'sparse': True}, - {'fields': [('b', -1), 'd']}, + [('f1', -1)], + {'fields': 'f2', 'sparse': True}, + {'fields': [('custom_column', -1), 'f3']}, [('geo', '2d')], - {'fields': [('geo2', '2d'), 'a'], 'min': 42, 'max': 21}, - {'fields': [('h.q',1)]}, - {'fields': [('i.q',1)]}, - {'fields' :[('j.a',1)]}, - {'fields' :[('k.a',1)]}, + {'fields': [('geo_custom_column', '2d'), 'f2'], 'min': 42, 'max': 21}, + {'fields': [('dict1.foo', 1)]}, + {'fields': [('dict_custom_column.foo', 1)]}, + {'fields' :[('embedded.a', 1)]}, + {'fields' :[('embedded_list.a', 1)]}, ] diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index aefad16e..434bf9e7 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -337,19 +337,19 @@ def assertHaveIndex(self, key, **properties): self.assertEqual(info[index_name], dict(default_properties, **properties)) def test_indexes(self): - self.assertHaveIndex([('c', 1)]) - self.assertHaveIndex([('f', 1)], unique=True) - self.assertHaveIndex([('a', 1), ('b2', 1)], unique=True) - self.assertHaveIndex([('a', 1), ('d', 1)], unique=True) - self.assertHaveIndex([('e', -1)]) - self.assertHaveIndex([('a', 1)], sparse=True) - self.assertHaveIndex([('b2', -1), ('d', 1)]) + self.assertHaveIndex([('db_index', 1)]) + self.assertHaveIndex([('unique', 1)], unique=True) + self.assertHaveIndex([('f2', 1), ('custom', 1)], unique=True) + self.assertHaveIndex([('f2', 1), ('f3', 1)], unique=True) + self.assertHaveIndex([('f1', -1)]) + self.assertHaveIndex([('f2', 1)], sparse=True) + self.assertHaveIndex([('custom', -1), ('f3', 1)]) self.assertHaveIndex([('geo', '2d')]) - self.assertHaveIndex([('geo', '2d'), ('a', 1)], min=42, max=21) - self.assertHaveIndex([('h.q', 1)]) - self.assertHaveIndex([('i2.q', 1)]) - self.assertHaveIndex([('j.a2',1)]) - self.assertHaveIndex([('k.a2',1)]) + self.assertHaveIndex([('geo', '2d'), ('f2', 1)], min=42, max=21) + self.assertHaveIndex([('dict1.foo', 1)]) + self.assertHaveIndex([('dict_custom.foo', 1)]) + self.assertHaveIndex([('embedded.a2', 1)]) + self.assertHaveIndex([('embedded_list.a2', 1)]) class GridFSFieldTests(TestCase): def tearDown(self): From dddd3bfc9d48d1aba9b60089480a7133a14cfe14 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 22 Oct 2011 20:57:02 +0200 Subject: [PATCH 4/5] Mention #77 to get this into the Pull Request page From bce8c19bd55054f92b29b590ed49c334efadaaf8 Mon Sep 17 00:00:00 2001 From: stillinbeta Date: Sat, 22 Oct 2011 20:02:06 -0400 Subject: [PATCH 5/5] Worked over creation.py with a pep8 comb --- django_mongodb_engine/creation.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/django_mongodb_engine/creation.py b/django_mongodb_engine/creation.py index 5a07ae4b..e934f669 100644 --- a/django_mongodb_engine/creation.py +++ b/django_mongodb_engine/creation.py @@ -2,12 +2,13 @@ from djangotoolbox.db.base import NonrelDatabaseCreation from .utils import make_index_list + class DatabaseCreation(NonrelDatabaseCreation): data_types = dict(NonrelDatabaseCreation.data_types, **{ 'AutoField': 'objectid', 'ForeignKey': 'objectid', 'OneToOneField': 'objectid', - 'RelatedAutoField' : 'objectid', + 'RelatedAutoField': 'objectid', 'DecimalField': 'float', }) @@ -22,7 +23,8 @@ def sql_indexes_for_model(self, model, termstyle): def ensure_index(*args, **kwargs): if ensure_index.first_index: - print "Installing indices for %s.%s model" % (meta.app_label, meta.object_name) + print "Installing indices for %s.%s model" % (meta.app_label, + meta.object_name) ensure_index.first_index = False return collection.ensure_index(*args, **kwargs) ensure_index.first_index = True @@ -50,7 +52,7 @@ def _handle_newstyle_indexes(self, ensure_index, meta, indexes): indexes.append({'fields': make_index_list(fields), 'unique': True}) #Allow for embedded document indexes, with proper root field names - def find_field(name,meta): + def find_field(name, meta): dot = name.find('.') if dot == -1: dot = len(name) @@ -59,15 +61,15 @@ def find_field(name,meta): new_field = field #Allow arbitrarily deep nesting of ListFields or SetFields while True: - if hasattr(new_field,'embedded_model'): + if hasattr(new_field, 'embedded_model'): model = new_field.embedded_model break elif hasattr(new_field, 'item_field'): new_field = new_field.item_field - else: + else: return field.column + name[dot:] - - return field.column + '.' + find_field(name[dot+1:], model._meta) + + return field.column + '.' + find_field(name[dot + 1:], model._meta) for index in indexes: if isinstance(index, dict): @@ -75,11 +77,10 @@ def find_field(name,meta): fields = kwargs.pop('fields') else: fields, kwargs = index, {} - fields = [(find_field(name,meta), direction) + fields = [(find_field(name, meta), direction) for name, direction in make_index_list(fields)] ensure_index(fields, **kwargs) - def _handle_oldstyle_indexes(self, ensure_index, meta): from warnings import warn warn("'descending_indexes', 'sparse_indexes' and 'index_together' are" @@ -90,7 +91,8 @@ def _handle_oldstyle_indexes(self, ensure_index, meta): # Lets normalize the sparse_index values changing [], set() to () for idx in set(getattr(meta, 'sparse_indexes', ())): - sparse_indexes.append(isinstance(idx, (tuple, set, list)) and tuple(idx) or idx ) + sparse_indexes.append(isinstance(idx, (tuple, set, list)) + and tuple(idx) or idx) # Ordinary indexes for field in meta.local_fields: @@ -103,7 +105,6 @@ def _handle_oldstyle_indexes(self, ensure_index, meta): ensure_index(column, unique=field.unique, sparse=field.name in sparse_indexes) - def create_compound_indexes(indexes, **kwargs): # indexes: (field1, field2, ...) if not indexes: @@ -133,7 +134,8 @@ def create_compound_indexes(indexes, **kwargs): return [] def sql_create_model(self, model, *unused): - """ Creates the collection for model. Mostly used for capped collections. """ + """ Creates the collection for model. + Mostly used for capped collections. """ kwargs = {} for option in ('capped', 'collection_max', 'collection_size'): x = getattr(model._meta, option, None) @@ -168,14 +170,15 @@ def create_test_db(self, verbosity=1, autoclobber=False): self._drop_database(test_database_name) from django.core.management import call_command - call_command('syncdb', verbosity=max(verbosity-1, 0), + call_command('syncdb', verbosity=max(verbosity - 1, 0), interactive=False, database=self.connection.alias) return test_database_name def destroy_test_db(self, old_database_name, verbosity=1): if verbosity >= 1: - print "Destroying test database for alias '%s'..." % self.connection.alias + print ("Destroying test database for alias '%s'..." + % self.connection.alias) test_database_name = self.connection.settings_dict['NAME'] self._drop_database(test_database_name) self.connection.settings_dict['NAME'] = old_database_name