From edf93522bb42df0ed97225846a7862fef9d6ce99 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 14 Dec 2024 15:21:24 -0500 Subject: [PATCH 1/6] update to Django 5.2 --- .evergreen/run-tests.sh | 2 +- .github/workflows/test-python.yml | 2 +- django_mongodb_backend/__init__.py | 2 +- django_mongodb_backend/features.py | 37 +++++++++++++++++++++++ django_mongodb_backend/lookups.py | 7 +++-- django_mongodb_backend/operations.py | 22 -------------- django_mongodb_backend/query_utils.py | 5 +-- tests/model_forms_/test_embedded_model.py | 8 ++--- 8 files changed, 49 insertions(+), 36 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f383a71a..f49a0e9a 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -9,7 +9,7 @@ python -m pip install -U pip pip install -e . # Install django and test dependencies -git clone --branch mongodb-5.1.x https://github.com/mongodb-forks/django django_repo +git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo pushd django_repo/tests/ pip install -e .. pip install -r requirements/py3.txt diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index cc791d55..d471f3f2 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.1.x' + ref: 'mongodb-5.2.x' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index d8529e9d..9ecbaaec 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.1.0b1.dev0" +__version__ = "5.2.0a0" # Check Django compatibility before other imports which may fail if the # wrong version of Django is installed. diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index a286a2cb..3d42cae9 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -88,6 +88,31 @@ class DatabaseFeatures(BaseDatabaseFeatures): # of $setIsSubset must be arrays. Second argument is of type: null" # https://jira.mongodb.org/browse/SERVER-99186 "model_fields_.test_arrayfield.QueryingTests.test_contained_by_subquery", + # Broken by https://github.com/django/django/commit/65ad4ade74dc9208b9d686a451cd6045df0c9c3a + "aggregation.tests.AggregateTestCase.test_even_more_aggregate", + "aggregation.tests.AggregateTestCase.test_grouped_annotation_in_group_by", + "aggregation.tests.AggregateTestCase.test_non_grouped_annotation_not_in_group_by", + "aggregation_regress.tests.AggregationTests.test_aggregate_fexpr", + "aggregation_regress.tests.AggregationTests.test_values_list_annotation_args_ordering", + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_and_aggregate_values_chaining", + "annotations.tests.NonAggregateAnnotationTestCase.test_values_fields_annotations_order", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_and_datetime_annotations", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_list_and_datetime_annotations", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_list_and_annotations", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_field_and_annotation_values", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_two_annotated_values_list", + "queries.tests.Queries1Tests.test_union_values_subquery", + # JSONArray not implemented. + "db_functions.json.test_json_array.JSONArrayTests", + # Some usage of prefetch_related() raises "ColPairs is not supported." + "known_related_objects.tests.ExistingRelatedInstancesTests.test_one_to_one_multi_prefetch_related", + "known_related_objects.tests.ExistingRelatedInstancesTests.test_one_to_one_prefetch_related", + "prefetch_related.tests.DeprecationTests.test_prefetch_one_level_fallback", + "prefetch_related.tests.MultiDbTests.test_using_is_honored_fkey", + "prefetch_related.tests.MultiDbTests.test_using_is_honored_inheritance", + "prefetch_related.tests.NestedPrefetchTests.test_nested_prefetch_is_not_overwritten_by_related_object", + "prefetch_related.tests.NullableTest.test_prefetch_nullable", + "prefetch_related.tests.Ticket19607Tests.test_bug", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { @@ -112,6 +137,7 @@ def django_test_expected_failures(self): # bson.errors.InvalidDocument: cannot encode object: #
-
+
Publisher: -
    +
    • Enter all required values.
    @@ -252,9 +252,9 @@ def test_invalid_field_data(self): maxlength="50" required id="id_title">
    -
    +
    Publisher: -
      +
      • Ensure this value has at most 2 characters (it has 8).
      From c507fbce54c494b06f87dae60a2f89e419fff552 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 17 Dec 2024 20:49:45 -0500 Subject: [PATCH 2/6] use DatabaseFeatures.rounds_to_even --- django_mongodb_backend/features.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 3d42cae9..281f1c98 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -12,6 +12,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): greatest_least_ignores_nulls = True has_json_object_function = False has_native_json_field = True + rounds_to_even = True supports_boolean_expr_in_select_clause = True supports_collation_on_charfield = False supports_column_check_constraints = False @@ -56,8 +57,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): # Pattern lookups that use regexMatch don't work on JSONField: # Unsupported conversion from array to string in $convert "model_fields.test_jsonfield.TestQuerying.test_icontains", - # MongoDB gives ROUND(365, -1)=360 instead of 370 like other databases. - "db_functions.math.test_round.RoundTests.test_integer_with_negative_precision", # Truncating in another timezone doesn't work becauase MongoDB converts # the result back to UTC. "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone", From 99670085981fbd5524b5c758a07b712b51967e3b Mon Sep 17 00:00:00 2001 From: Emanuel Lupi Date: Sat, 21 Dec 2024 00:31:47 -0300 Subject: [PATCH 3/6] Fix columns order in aggregation queries --- django_mongodb_backend/compiler.py | 39 ++++++++++++++++++--------- django_mongodb_backend/expressions.py | 6 ++++- django_mongodb_backend/features.py | 14 ---------- tests/indexes_/test_condition.py | 2 +- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/django_mongodb_backend/compiler.py b/django_mongodb_backend/compiler.py index cf666619..c7c41e2c 100644 --- a/django_mongodb_backend/compiler.py +++ b/django_mongodb_backend/compiler.py @@ -403,12 +403,6 @@ def columns(self): columns = ( self.get_default_columns(select_mask) if self.query.default_cols else self.query.select ) - # Populate QuerySet.select_related() data. - related_columns = [] - if self.query.select_related: - self.get_related_selections(related_columns, select_mask) - if related_columns: - related_columns, _ = zip(*related_columns, strict=True) annotation_idx = 1 @@ -427,11 +421,28 @@ def project_field(column): annotation_idx += 1 return target, column - return ( - tuple(map(project_field, columns)) - + tuple(self.annotations.items()) - + tuple(map(project_field, related_columns)) - ) + selected = [] + if self.query.selected is None: + selected = [ + *(project_field(col) for col in columns), + *self.annotations.items(), + ] + else: + for expression in self.query.selected.values(): + # Reference to an annotation. + if isinstance(expression, str): + alias, expression = expression, self.annotations[expression] + # Reference to a column. + elif isinstance(expression, int): + alias, expression = project_field(columns[expression]) + selected.append((alias, expression)) + # Populate QuerySet.select_related() data. + related_columns = [] + if self.query.select_related: + self.get_related_selections(related_columns, select_mask) + if related_columns: + related_columns, _ = zip(*related_columns, strict=True) + return tuple(selected) + tuple(map(project_field, related_columns)) @cached_property def base_table(self): @@ -478,7 +489,11 @@ def get_combinator_queries(self): # If the columns list is limited, then all combined queries # must have the same columns list. Set the selects defined on # the query on all combined queries, if not already set. - if not compiler_.query.values_select and self.query.values_select: + selected = self.query.selected + if selected is not None and compiler_.query.selected is None: + compiler_.query = compiler_.query.clone() + compiler_.query.set_values(selected) + elif not compiler_.query.values_select and self.query.values_select: compiler_.query = compiler_.query.clone() compiler_.query.set_values( ( diff --git a/django_mongodb_backend/expressions.py b/django_mongodb_backend/expressions.py index 8e8c1815..db3475fc 100644 --- a/django_mongodb_backend/expressions.py +++ b/django_mongodb_backend/expressions.py @@ -150,7 +150,11 @@ def ref(self, compiler, connection): # noqa: ARG001 if isinstance(self.source, Col) and self.source.alias != compiler.collection_name else "" ) - return f"${prefix}{self.refs}" + if hasattr(self, "ordinal"): + refs, _ = compiler.columns[self.ordinal - 1] + else: + refs = self.refs + return f"${prefix}{refs}" def star(self, compiler, connection): # noqa: ARG001 diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 281f1c98..1317a729 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -87,20 +87,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): # of $setIsSubset must be arrays. Second argument is of type: null" # https://jira.mongodb.org/browse/SERVER-99186 "model_fields_.test_arrayfield.QueryingTests.test_contained_by_subquery", - # Broken by https://github.com/django/django/commit/65ad4ade74dc9208b9d686a451cd6045df0c9c3a - "aggregation.tests.AggregateTestCase.test_even_more_aggregate", - "aggregation.tests.AggregateTestCase.test_grouped_annotation_in_group_by", - "aggregation.tests.AggregateTestCase.test_non_grouped_annotation_not_in_group_by", - "aggregation_regress.tests.AggregationTests.test_aggregate_fexpr", - "aggregation_regress.tests.AggregationTests.test_values_list_annotation_args_ordering", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_and_aggregate_values_chaining", - "annotations.tests.NonAggregateAnnotationTestCase.test_values_fields_annotations_order", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_and_datetime_annotations", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_list_and_datetime_annotations", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_multiple_models_with_values_list_and_annotations", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_field_and_annotation_values", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_two_annotated_values_list", - "queries.tests.Queries1Tests.test_union_values_subquery", # JSONArray not implemented. "db_functions.json.test_json_array.JSONArrayTests", # Some usage of prefetch_related() raises "ColPairs is not supported." diff --git a/tests/indexes_/test_condition.py b/tests/indexes_/test_condition.py index 5077dd0b..3c49f1ec 100644 --- a/tests/indexes_/test_condition.py +++ b/tests/indexes_/test_condition.py @@ -78,7 +78,7 @@ def test_composite_index(self): { "$and": [ {"number": {"$gte": 3}}, - {"$or": [{"body": {"$gt": "test1"}}, {"body": {"$in": ["A", "B"]}}]}, + {"$or": [{"body": {"$gt": "test1"}}, {"body": {"$in": ("A", "B")}}]}, ] }, ) From 87e0dd554cd07581ed500879f9991017d9bafd2f Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 3 Jan 2025 09:54:11 -0500 Subject: [PATCH 4/6] adapt SQLDeleteCompiler.execute_sql() for https://github.com/django/django/commit/ddefc3fed1cf1f0d3fab455babbbc009b76e4196 --- django_mongodb_backend/compiler.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/django_mongodb_backend/compiler.py b/django_mongodb_backend/compiler.py index c7c41e2c..66154902 100644 --- a/django_mongodb_backend/compiler.py +++ b/django_mongodb_backend/compiler.py @@ -17,7 +17,6 @@ from django.utils.functional import cached_property from pymongo import ASCENDING, DESCENDING -from .base import Cursor from .query import MongoQuery, wrap_database_errors @@ -705,15 +704,12 @@ def collection_name(self): class SQLDeleteCompiler(compiler.SQLDeleteCompiler, SQLCompiler): def execute_sql(self, result_type=MULTI): - cursor = Cursor() try: query = self.build_query() except EmptyResultSet: - rowcount = 0 + return 0 else: - rowcount = query.delete() - cursor.rowcount = rowcount - return cursor + return query.delete() def check_query(self): super().check_query() From 7931488ca775d5233f383b9ffe292ca019e79fa4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 21 Feb 2025 13:09:39 -0500 Subject: [PATCH 5/6] add ExpressionList.as_mql() for https://github.com/django/django/commit/d99985bbc121749c5a6bb9eb9a4a9099b6a002eb --- django_mongodb_backend/expressions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/django_mongodb_backend/expressions.py b/django_mongodb_backend/expressions.py index db3475fc..e5ef3358 100644 --- a/django_mongodb_backend/expressions.py +++ b/django_mongodb_backend/expressions.py @@ -10,6 +10,7 @@ Col, CombinedExpression, Exists, + ExpressionList, ExpressionWrapper, F, NegatedExpression, @@ -24,6 +25,8 @@ ) from django.db.models.sql import Query +from .query_utils import process_lhs + def case(self, compiler, connection): case_parts = [] @@ -83,6 +86,10 @@ def expression_wrapper(self, compiler, connection): return self.expression.as_mql(compiler, connection) +def expression_list(self, compiler, connection): + return process_lhs(self, compiler, connection) + + def f(self, compiler, connection): # noqa: ARG001 return f"${self.name}" @@ -206,6 +213,7 @@ def register_expressions(): Col.as_mql = col CombinedExpression.as_mql = combined_expression Exists.as_mql = exists + ExpressionList.as_mql = expression_list ExpressionWrapper.as_mql = expression_wrapper F.as_mql = f NegatedExpression.as_mql = negated_expression From 29623f6ca7d9d7fc7940b962fc56420d6519aaa4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 21 Feb 2025 20:01:42 -0500 Subject: [PATCH 6/6] fix model_fields.test_jsonfield.TestSaveLoad.test_bulk_update_custom_get_prep_value https://github.com/django/django/commit/9525135698bd4f97cf1431776ef52ae393dfb3c0 --- django_mongodb_backend/expressions.py | 6 ++++++ django_mongodb_backend/features.py | 2 ++ tests/expressions_/test_value.py | 19 +++++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/django_mongodb_backend/expressions.py b/django_mongodb_backend/expressions.py index e5ef3358..2bf7e0ee 100644 --- a/django_mongodb_backend/expressions.py +++ b/django_mongodb_backend/expressions.py @@ -186,6 +186,12 @@ def when(self, compiler, connection): def value(self, compiler, connection): # noqa: ARG001 value = self.value + output_field = self._output_field_or_none + if output_field is not None: + if self.for_save: + value = output_field.get_db_prep_save(value, connection=connection) + else: + value = output_field.get_db_prep_value(value, connection=connection) if isinstance(value, int): # Wrap numbers in $literal to prevent ambiguity when Value appears in # $project. diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 1317a729..bb709b1f 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -98,6 +98,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): "prefetch_related.tests.NestedPrefetchTests.test_nested_prefetch_is_not_overwritten_by_related_object", "prefetch_related.tests.NullableTest.test_prefetch_nullable", "prefetch_related.tests.Ticket19607Tests.test_bug", + # {'$project': {'name': Decimal128('1')} is broken? (gives None) + "expressions.tests.ValueTests.test_output_field_decimalfield", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { diff --git a/tests/expressions_/test_value.py b/tests/expressions_/test_value.py index c57c2f03..ad131a7f 100644 --- a/tests/expressions_/test_value.py +++ b/tests/expressions_/test_value.py @@ -3,6 +3,7 @@ from decimal import Decimal from bson import Decimal128 +from django.db import connection from django.db.models import Value from django.test import SimpleTestCase @@ -10,34 +11,36 @@ class ValueTests(SimpleTestCase): def test_date(self): self.assertEqual( - Value(datetime.date(2025, 1, 1)).as_mql(None, None), + Value(datetime.date(2025, 1, 1)).as_mql(None, connection), datetime.datetime(2025, 1, 1), ) def test_datetime(self): self.assertEqual( - Value(datetime.datetime(2025, 1, 1, 9, 8, 7)).as_mql(None, None), + Value(datetime.datetime(2025, 1, 1, 9, 8, 7)).as_mql(None, connection), datetime.datetime(2025, 1, 1, 9, 8, 7), ) def test_decimal(self): - self.assertEqual(Value(Decimal("1.0")).as_mql(None, None), Decimal128("1.0")) + self.assertEqual(Value(Decimal("1.0")).as_mql(None, connection), Decimal128("1.0")) def test_time(self): self.assertEqual( - Value(datetime.time(9, 8, 7)).as_mql(None, None), + Value(datetime.time(9, 8, 7)).as_mql(None, connection), datetime.datetime(1, 1, 1, 9, 8, 7), ) def test_timedelta(self): - self.assertEqual(Value(datetime.timedelta(3600)).as_mql(None, None), 311040000000.0) + self.assertEqual( + Value(datetime.timedelta(3600)).as_mql(None, connection), {"$literal": 311040000000} + ) def test_int(self): - self.assertEqual(Value(1).as_mql(None, None), {"$literal": 1}) + self.assertEqual(Value(1).as_mql(None, connection), {"$literal": 1}) def test_str(self): - self.assertEqual(Value("foo").as_mql(None, None), "foo") + self.assertEqual(Value("foo").as_mql(None, connection), "foo") def test_uuid(self): value = uuid.UUID(int=1) - self.assertEqual(Value(value).as_mql(None, None), "00000000000000000000000000000001") + self.assertEqual(Value(value).as_mql(None, connection), "00000000000000000000000000000001")