fix GenericReference iterable query (i.e. __in)

.github/workflows/github-actions.yml

@@ -13,14 +13,11 @@ on:
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+*'
 env:
-  MONGODB_3_6: "3.6.23"
-  MONGODB_4_0: "4.0.28"
-  MONGODB_4_2: "4.2.25"
   MONGODB_4_4: "4.4.29"
   MONGODB_5_0: "5.0.31"
-  MONGODB_6_0: "6.0.20"
-  MONGODB_7_0: "7.0.17"
-  MONGODB_8_0: "8.0.5"
+  MONGODB_6_0: "6.0.22"
+  MONGODB_7_0: "7.0.19"
+  MONGODB_8_0: "8.0.9"
 
   PYMONGO_3_12: "3.12.3"
   PYMONGO_3_13: "3.13.0"
@@ -36,8 +33,6 @@ env:
 
   MAIN_PYTHON_VERSION: "3.9"
 
-  MONGOSH: "2.4.2" # Needed for MongoDB 6.0+
-
 jobs:
   linting:
     # Run pre-commit (https://pre-commit.com/)
@@ -55,17 +50,14 @@ jobs:
   test:
     # Test suite run against recent python versions
     # and against a few combination of MongoDB and pymongo
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
-        MONGODB: [$MONGODB_4_0]
+        MONGODB: [$MONGODB_4_4]
         PYMONGO: [$PYMONGO_3_12]
         include:
-          - python-version: "3.9"
-            MONGODB: $MONGODB_3_6
-            PYMONGO: $PYMONGO_3_12
           - python-version: "3.9"
             MONGODB: $MONGODB_4_4
             PYMONGO: $PYMONGO_3_13
@@ -115,7 +107,6 @@ jobs:
     - name: install mongo and ci dependencies
       run: |
         bash .github/workflows/install_mongo.sh ${{ matrix.MONGODB }}
-        bash .github/workflows/install_mongosh.sh ${{ matrix.MONGODB }} ${{ env.MONGOSH }}
         bash .github/workflows/install_ci_python_dep.sh
         bash .github/workflows/start_mongo.sh ${{ matrix.MONGODB }}
     - name: tox dry-run (to pre-install venv)

.github/workflows/install_mongo.sh

@@ -4,49 +4,79 @@ set -e  # Exit immediately if a command exits with a non-zero status
 set -u  # Treat unset variables as an error
 
 if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <mongodb-version>"
-  echo "Example: $0 8.0.5"
+  echo >&2 "Usage: $0 <mongodb-version>"
+  echo >&2 "Example: $0 8.0.5"
   exit 1
 fi
 
 MONGODB="$1"
+MONGOSH=2.5.1
 
 # Determine build name based on version
-if [[ "${MONGODB}" =~ ^(7.0|8.0) ]]; then
-  mongo_build="mongodb-linux-x86_64-ubuntu2004-${MONGODB}"
-elif [[ "${MONGODB}" =~ ^(4.0|4.2|4.4|5.0|6.0) ]]; then
-  mongo_build="mongodb-linux-x86_64-ubuntu1804-${MONGODB}"
-elif [[ "${MONGODB}" =~ ^3.6 ]]; then
-  mongo_build="mongodb-linux-x86_64-${MONGODB}"
+if [[ "$MONGODB" =~ ^(6.0|7.0|8.0) ]]; then
+  mongodb_build="mongodb-linux-x86_64-ubuntu2204-${MONGODB}"
+elif [[ "$MONGODB" =~ ^(4.4|5.0) ]]; then
+  mongodb_build="mongodb-linux-x86_64-ubuntu2004-${MONGODB}"
 else
-  echo "Error: Unsupported MongoDB version: ${MONGODB}"
+  echo >&2 "Error: Unsupported MongoDB version: ${MONGODB}"
   usage
 fi
 
-download_url="http://fastdl.mongodb.org/linux/${mongo_build}.tgz"
-tarball="${mongo_build}.tgz"
+mongodb_tarball="${mongodb_build}.tgz"
+mongodb_download_url="http://fastdl.mongodb.org/linux/${mongodb_tarball}"
 
-echo "Downloading MongoDB from ${download_url}..."
-if ! wget --quiet "${download_url}"; then
-  echo "Error: Failed to download MongoDB."
-  exit 1
-fi
+mongosh_build="mongosh-${MONGOSH}-linux-x64"
+mongosh_tarball="${mongosh_build}.tgz"
+mongosh_download_url="https://github.com/mongodb-js/mongosh/releases/download/v${MONGOSH}/${mongosh_tarball}"
+
+set -- \
+  MongoDB "$mongodb_tarball" "$mongodb_download_url" \
+  "MongoDB Shell" "$mongosh_tarball" "$mongosh_download_url"
+
+while (( $# > 0 )) ; do
+  name="$1"
+  tarball="$2"
+  download_url="$3"
+  shift 3
+
+  echo >&2 "Downloading ${name} from ${download_url}..."
+  if ! wget --quiet "$download_url"; then
+    echo >&2 "Error: Failed to download ${name}."
+    exit 1
+  fi
 
-echo "Extracting ${tarball}..."
-if ! tar xzf "${tarball}"; then
-  echo "Error: Failed to extract ${tarball}"
+  echo >&2 "Extracting ${tarball}..."
+  if ! tar xzf "${tarball}"; then
+    echo >&2 "Error: Failed to extract ${tarball}"
+    exit 1
+  fi
+done
+
+mongodb_dir=$(find "${PWD}/" -type d -name "mongodb-linux-x86_64*" -print -quit)
+if [ -z "$mongodb_dir" ]; then
+  echo >&2 "Error: Could not find MongoDB directory after extraction."
   exit 1
 fi
 
-mongodb_dir=$(find "${PWD}/" -type d -name "mongodb-linux-x86_64*" | head -n 1)
-if [ -z "${mongodb_dir}" ]; then
-  echo "Error: Could not find MongoDB directory after extraction."
-  exit 1
+mongosh_dir=$(find "${PWD}/" -type d -name "$mongosh_build" -print -quit)
+if [ ! -d "$mongosh_dir" ]; then
+    echo >&2 "Failed to find extracted mongosh directory."
+    rm -f "$TARBALL"
+    exit 1
 fi
 
-echo "MongoDB installed at: ${mongodb_dir}"
-"${mongodb_dir}/bin/mongod" --version
+echo >&2 "Creating mongo.path"
+echo "export PATH='${mongodb_dir}/bin:${mongosh_dir}/bin:'"'$PATH' \
+  | tee >&2 mongo.path
+
+. mongo.path
+
+echo >&2 "MongoDB is installed at: ${mongodb_dir}"
+mongod >&2 --version
+
+echo >&2 "MongoDB Shell is installed at: ${mongosh_dir}"
+mongosh >&2 --version
 
 # Cleanup
-echo "Cleaning up..."
-rm -f "${tarball}"
+echo >&2 "Cleaning up..."
+rm -f "$mongodb_tarball" "$mongosh_tarball"

.github/workflows/start_mongo.sh

@@ -1,8 +1,13 @@
 #!/bin/bash
 
+set -e  # Exit immediately if a command exits with a non-zero status
+set -u  # Treat unset variables as an error
+
+. mongo.path
+
 MONGODB=$1
 
-mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*" -print -quit)
 
 mkdir $mongodb_dir/data
 
@@ -15,12 +20,6 @@ if [ "$MAJOR" -gt 3 ] || ([ "$MAJOR" -eq 3 ] && [ "$MINOR" -ge 8 ]); then
     args+=(--setParameter maxTransactionLockRequestTimeoutMillis=1000)
 fi
 
-$mongodb_dir/bin/mongod "${args[@]}"
-
-if [ "$MAJOR" -lt 6 ]; then
-mongo --verbose --eval "rs.initiate()"
-mongo --quiet --eval "rs.status().ok"
-else
+mongod "${args[@]}"
 mongosh --verbose --eval "rs.initiate()"
 mongosh --quiet --eval "rs.status().ok"
-fi

README.rst

@@ -35,10 +35,10 @@ an `API reference <https://mongoengine-odm.readthedocs.io/apireference.html>`_.
 
 Supported MongoDB Versions
 ==========================
-MongoEngine is currently tested against MongoDB v3.6, v4.0, v4.4, v5.0, v6.0, v7.0 and v8.0. Future versions
-should be supported as well, but aren't actively tested at the moment. Make
-sure to open an issue or submit a pull request if you experience any problems
-with a more recent MongoDB versions.
+MongoEngine is currently tested against MongoDB v4.4, v5.0, v6.0, v7.0 and
+v8.0. Future versions should be supported as well, but aren't actively tested
+at the moment. Make sure to open an issue or submit a pull request if you
+experience any problems with a more recent MongoDB versions.
 
 Installation
 ============

mongoengine/queryset/transform.py

@@ -129,13 +129,15 @@ def query(_doc_cls=None, **kwargs):
 
             singular_ops = [None, "ne", "gt", "gte", "lt", "lte", "not"]
             singular_ops += STRING_OPERATORS
+            is_iterable = False
             if op in singular_ops:
                 value = field.prepare_query_value(op, value)
 
                 if isinstance(field, CachedReferenceField) and value:
                     value = value["_id"]
 
             elif op in ("in", "nin", "all", "near") and not isinstance(value, dict):
+                is_iterable = True
                 # Raise an error if the in/nin/all/near param is not iterable.
                 value = _prepare_query_for_iterable(field, op, value)
 
@@ -144,10 +146,24 @@ def query(_doc_cls=None, **kwargs):
             # * If the value is a DBRef, the key should be "field_name._ref".
             # * If the value is an ObjectId, the key should be "field_name._ref.$id".
             if isinstance(field, GenericReferenceField):
-                if isinstance(value, DBRef):
+                if isinstance(value, DBRef) or (
+                    is_iterable and all(isinstance(v, DBRef) for v in value)
+                ):
                     parts[-1] += "._ref"
-                elif isinstance(value, ObjectId):
+                elif isinstance(value, ObjectId) or (
+                    is_iterable and all(isinstance(v, ObjectId) for v in value)
+                ):
                     parts[-1] += "._ref.$id"
+                elif (
+                    is_iterable
+                    and any(isinstance(v, DBRef) for v in value)
+                    and any(isinstance(v, ObjectId) for v in value)
+                ):
+                    raise ValueError(
+                        "The `in`, `nin`, `all`, or `near`-operators cannot "
+                        "be applied to mixed queries of DBRef/ObjectId/%s"
+                        % _doc_cls.__name__
+                    )
 
         # if op and op not in COMPARISON_OPERATORS:
         if op: