Skip to content

Commit

Permalink
Merge branch 'master' into __raw__and_mongoengine
Browse files Browse the repository at this point in the history
  • Loading branch information
idoshr authored Dec 26, 2023
2 parents c8607d5 + cfb4265 commit ed9d2f7
Show file tree
Hide file tree
Showing 17 changed files with 231 additions and 80 deletions.
10 changes: 10 additions & 0 deletions .github/workflows/github-actions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ env:
MONGODB_4_0: 4.0.28
MONGODB_4_4: 4.4
MONGODB_5_0: "5.0"
MONGODB_6_0: "6.0"
MONGODB_7_0: "7.0"

PYMONGO_3_4: 3.4
PYMONGO_3_6: 3.6
Expand All @@ -25,6 +27,8 @@ env:
PYMONGO_3_12: 3.12
PYMONGO_4_0: 4.0
PYMONGO_4_3: 4.3.2
PYMONGO_4_4: 4.4.1
PYMONGO_4_6: 4.6.0

MAIN_PYTHON_VERSION: 3.7

Expand Down Expand Up @@ -68,6 +72,12 @@ jobs:
- python-version: "3.11"
MONGODB: $MONGODB_5_0
PYMONGO: $PYMONGO_4_3
- python-version: "3.11"
MONGODB: $MONGODB_6_0
PYMONGO: $PYMONGO_4_4
- python-version: "3.11"
MONGODB: $MONGODB_7_0
PYMONGO: $PYMONGO_4_6
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/install_mongo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ elif [[ "$MONGODB" == *"4.4"* ]]; then
mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
elif [[ "$MONGODB" == *"5.0"* ]]; then
mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
elif [[ "$MONGODB" == *"6.0"* ]]; then
mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
elif [[ "$MONGODB" == *"7.0"* ]]; then
mongo_build=mongodb-linux-x86_64-ubuntu2004-v${MONGODB}-latest
fi

wget http://fastdl.mongodb.org/linux/$mongo_build.tgz
Expand Down
2 changes: 1 addition & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.

Supported MongoDB Versions
==========================
MongoEngine is currently tested against MongoDB v3.6, v4.0, v4.4 and v5.0. Future versions
MongoEngine is currently tested against MongoDB v3.6, v4.0, v4.4, v5.0, v6.0 and v7.0. Future versions
should be supported as well, but aren't actively tested at the moment. Make
sure to open an issue or submit a pull request if you experience any problems
with a more recent MongoDB versions.
Expand Down
8 changes: 8 additions & 0 deletions docs/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ Development
- (Fill this out as you fix issues and develop your features).
- Fix for uuidRepresentation not read when provided in URI #2741
- Fix combination of __raw__ and mongoengine syntax.
- Add tests against MongoDB 6.0 and MongoDB 7.0 in the pipeline
- Fix validate() not being called when inheritance is used in EmbeddedDocument and validate is overriden #2784
- Add support for readPreferenceTags in connection parameters #2644
- Use estimated_documents_count OR documents_count when count is called, based on the query #2529
- Fix no_dereference context manager which wasn't turning off auto-dereferencing correctly in some cases #2788
- BREAKING CHANGE: no_dereference context manager no longer returns the class in __enter__ #2788
as it was useless and making it look like it was returning a different class although it was the same.
Thus, it must be called like `with no_dereference(User):` and no longer `with no_dereference(User) as ...:`

Changes in 0.27.0
=================
Expand Down
2 changes: 1 addition & 1 deletion docs/guide/querying.rst
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ data. To turn off dereferencing of the results of a query use
You can also turn off all dereferencing for a fixed period by using the
:class:`~mongoengine.context_managers.no_dereference` context manager::

with no_dereference(Post) as Post:
with no_dereference(Post):
post = Post.objects.first()
assert(isinstance(post.author, DBRef))

Expand Down
6 changes: 3 additions & 3 deletions mongoengine/base/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ def __init__(
unique with (Creates an index).
:param primary_key: Mark this field as the primary key ((Creates an index)). Defaults to False.
:param validation: (optional) A callable to validate the value of the
field. The callable takes the value as parameter and should raise
field. The callable takes the value as parameter and should raise
a ValidationError if validation fails
:param choices: (optional) The valid choices
:param null: (optional) If the field value can be null when a default exist. If not set, the default value
will be used in case a field with a default value is set to None. Defaults to False.
:param null: (optional) If the field value can be null when a default exists. If not set, the default value
will be used in case a field with a default value is set to None. Defaults to False.
:param sparse: (optional) `sparse=True` combined with `unique=True` and `required=False`
means that uniqueness won't be enforced for `None` values (Creates an index). Defaults to False.
:param **kwargs: (optional) Arbitrary indirection-free metadata for
Expand Down
10 changes: 9 additions & 1 deletion mongoengine/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,16 @@ def _get_connection_settings(
preference.name.lower() == read_pf_mode
or preference.mode == read_pf_mode
):
conn_settings["read_preference"] = preference
ReadPrefClass = preference.__class__
break

if "readpreferencetags" in uri_options:
conn_settings["read_preference"] = ReadPrefClass(
tag_sets=uri_options["readpreferencetags"]
)
else:
conn_settings["read_preference"] = ReadPrefClass()

if "authmechanismproperties" in uri_options:
conn_settings["authmechanismproperties"] = uri_options[
"authmechanismproperties"
Expand Down
28 changes: 25 additions & 3 deletions mongoengine/context_managers.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import threading
from contextlib import contextmanager

from pymongo.read_concern import ReadConcern
Expand All @@ -18,6 +19,25 @@
)


thread_locals = threading.local()
thread_locals.no_dereferencing_class = {}


def no_dereferencing_active_for_class(cls):
return cls in thread_locals.no_dereferencing_class


def _register_no_dereferencing_for_class(cls):
thread_locals.no_dereferencing_class.setdefault(cls, 0)
thread_locals.no_dereferencing_class[cls] += 1


def _unregister_no_dereferencing_for_class(cls):
thread_locals.no_dereferencing_class[cls] -= 1
if thread_locals.no_dereferencing_class[cls] == 0:
thread_locals.no_dereferencing_class.pop(cls)


class switch_db:
"""switch_db alias context manager.
Expand Down Expand Up @@ -107,7 +127,7 @@ class no_dereference:
Turns off all dereferencing in Documents for the duration of the context
manager::
with no_dereference(Group) as Group:
with no_dereference(Group):
Group.objects.find()
"""

Expand All @@ -130,15 +150,17 @@ def __init__(self, cls):

def __enter__(self):
"""Change the objects default and _auto_dereference values."""
_register_no_dereferencing_for_class(self.cls)

for field in self.deref_fields:
self.cls._fields[field]._auto_dereference = False
return self.cls

def __exit__(self, t, value, traceback):
"""Reset the default and _auto_dereference values."""
_unregister_no_dereferencing_for_class(self.cls)

for field in self.deref_fields:
self.cls._fields[field]._auto_dereference = True
return self.cls


class no_sub_classes:
Expand Down
2 changes: 1 addition & 1 deletion mongoengine/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -761,7 +761,7 @@ def validate(self, value, clean=True):
"Invalid embedded document instance provided to an "
"EmbeddedDocumentField"
)
self.document_type.validate(value, clean)
value.validate(clean=clean)

def lookup_member(self, member_name):
doc_and_subclasses = [self.document_type] + self.document_type.__subclasses__()
Expand Down
3 changes: 3 additions & 0 deletions mongoengine/mongodb_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
MONGODB_36 = (3, 6)
MONGODB_42 = (4, 2)
MONGODB_44 = (4, 4)
MONGODB_50 = (5, 0)
MONGODB_60 = (6, 0)
MONGODB_70 = (7, 0)


def get_mongodb_version():
Expand Down
20 changes: 10 additions & 10 deletions mongoengine/pymongo_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,12 @@ def count_documents(
# count_documents appeared in pymongo 3.7
if PYMONGO_VERSION >= (3, 7):
try:
return collection.count_documents(filter=filter, **kwargs)
if not filter and set(kwargs) <= {"max_time_ms"}:
# when no filter is provided, estimated_document_count
# is a lot faster as it uses the collection metadata
return collection.estimated_document_count(**kwargs)
else:
return collection.count_documents(filter=filter, **kwargs)
except OperationFailure as err:
if PYMONGO_VERSION >= (4,):
raise
Expand All @@ -46,15 +51,10 @@ def count_documents(
# with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere)
# fallback to deprecated Cursor.count
# Keeping this should be reevaluated the day pymongo removes .count entirely
message = str(err)
if not (
"not allowed in this context" in message
and (
"$where" in message
or "$geoNear" in message
or "$near" in message
or "$nearSphere" in message
)
if (
"$geoNear, $near, and $nearSphere are not allowed in this context"
not in str(err)
and "$where is not allowed in this context" not in str(err)
):
raise

Expand Down
19 changes: 13 additions & 6 deletions mongoengine/queryset/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from mongoengine.common import _import_class
from mongoengine.connection import get_db
from mongoengine.context_managers import (
no_dereferencing_active_for_class,
set_read_write_concern,
set_write_concern,
switch_db,
Expand Down Expand Up @@ -51,9 +52,6 @@ class BaseQuerySet:
providing :class:`~mongoengine.Document` objects as the results.
"""

__dereference = False
_auto_dereference = True

def __init__(self, document, collection):
self._document = document
self._collection_obj = collection
Expand All @@ -74,6 +72,9 @@ def __init__(self, document, collection):
self._as_pymongo = False
self._search_text = None

self.__dereference = False
self.__auto_dereference = True

# If inheritance is allowed, only return instances and instances of
# subclasses of the class being used
if document._meta.get("allow_inheritance") is True:
Expand Down Expand Up @@ -795,7 +796,7 @@ def clone(self):
return self._clone_into(self.__class__(self._document, self._collection_obj))

def _clone_into(self, new_qs):
"""Copy all of the relevant properties of this queryset to
"""Copy all the relevant properties of this queryset to
a new queryset (which has to be an instance of
:class:`~mongoengine.queryset.base.BaseQuerySet`).
"""
Expand Down Expand Up @@ -825,7 +826,6 @@ def _clone_into(self, new_qs):
"_empty",
"_hint",
"_collation",
"_auto_dereference",
"_search_text",
"_max_time_ms",
"_comment",
Expand All @@ -836,6 +836,8 @@ def _clone_into(self, new_qs):
val = getattr(self, prop)
setattr(new_qs, prop, copy.copy(val))

new_qs.__auto_dereference = self._BaseQuerySet__auto_dereference

if self._cursor_obj:
new_qs._cursor_obj = self._cursor_obj.clone()

Expand Down Expand Up @@ -1741,10 +1743,15 @@ def _dereference(self):
self.__dereference = _import_class("DeReference")()
return self.__dereference

@property
def _auto_dereference(self):
should_deref = not no_dereferencing_active_for_class(self._document)
return should_deref and self.__auto_dereference

def no_dereference(self):
"""Turn off any dereferencing for the results of this queryset."""
queryset = self.clone()
queryset._auto_dereference = False
queryset.__auto_dereference = False
return queryset

# Helper Functions
Expand Down
41 changes: 14 additions & 27 deletions tests/document/test_indexes.py
Original file line number Diff line number Diff line change
Expand Up @@ -450,45 +450,28 @@ class Test(Document):
# the documents returned might have more keys in that here.
query_plan = Test.objects(id=obj.id).exclude("a").explain()
assert (
query_plan.get("queryPlanner")
.get("winningPlan")
.get("inputStage")
.get("stage")
== "IDHACK"
query_plan["queryPlanner"]["winningPlan"]["inputStage"]["stage"] == "IDHACK"
)

query_plan = Test.objects(id=obj.id).only("id").explain()
assert (
query_plan.get("queryPlanner")
.get("winningPlan")
.get("inputStage")
.get("stage")
== "IDHACK"
query_plan["queryPlanner"]["winningPlan"]["inputStage"]["stage"] == "IDHACK"
)

mongo_db = get_mongodb_version()
query_plan = Test.objects(a=1).only("a").exclude("id").explain()
assert (
query_plan.get("queryPlanner")
.get("winningPlan")
.get("inputStage")
.get("stage")
== "IXSCAN"
query_plan["queryPlanner"]["winningPlan"]["inputStage"]["stage"] == "IXSCAN"
)
mongo_db = get_mongodb_version()

PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED"
assert (
query_plan.get("queryPlanner").get("winningPlan").get("stage")
== PROJECTION_STR
)
assert query_plan["queryPlanner"]["winningPlan"]["stage"] == PROJECTION_STR

query_plan = Test.objects(a=1).explain()
assert (
query_plan.get("queryPlanner")
.get("winningPlan")
.get("inputStage")
.get("stage")
== "IXSCAN"
query_plan["queryPlanner"]["winningPlan"]["inputStage"]["stage"] == "IXSCAN"
)

assert query_plan.get("queryPlanner").get("winningPlan").get("stage") == "FETCH"

def test_index_on_id(self):
Expand Down Expand Up @@ -536,8 +519,12 @@ class BlogPost(Document):
BlogPost.objects.hint("Bad Name").count()

# Invalid shape argument (missing list brackets) should fail.
with pytest.raises(ValueError):
BlogPost.objects.hint(("tags", 1)).count()
if PYMONGO_VERSION <= (4, 3):
with pytest.raises(ValueError):
BlogPost.objects.hint(("tags", 1)).count()
else:
with pytest.raises(TypeError):
BlogPost.objects.hint(("tags", 1)).count()

def test_collation(self):
base = {"locale": "en", "strength": 2}
Expand Down
Loading

0 comments on commit ed9d2f7

Please sign in to comment.