From 1006f97f8f95a23e864f04e4db8bbeddec50049c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 17 Jun 2024 12:39:24 -0500 Subject: [PATCH] PYTHON-4502 Add integration test with pymongo (#836) --- .evergreen/config.yml | 62 +++++++++++++++++++ bindings/python/.evergreen/integ-setup.sh | 43 +++++++++++++ bindings/python/.evergreen/integ-teardown.sh | 6 ++ bindings/python/.evergreen/integ-test.sh | 50 +++++++++++++++ bindings/python/.evergreen/utils.sh | 4 +- bindings/python/pymongocrypt/binding.py | 10 +-- .../compact/success/encrypted-payload.json | 2 +- 7 files changed, 169 insertions(+), 8 deletions(-) create mode 100755 bindings/python/.evergreen/integ-setup.sh create mode 100755 bindings/python/.evergreen/integ-teardown.sh create mode 100755 bindings/python/.evergreen/integ-test.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 235faac28..9ed4fc909 100755 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -136,6 +136,56 @@ functions: export MONGOCRYPT_DIR="$(pwd)/all/${variant_name}" cd ./libmongocrypt/bindings/python && ${test_env|} ./.evergreen/test.sh + "test python integ": + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: + - project_directory + - variant_name + args: + - ./libmongocrypt/bindings/python/.evergreen/integ-setup.sh + - command: expansions.update + params: + file: expansion.yml + - command: subprocess.exec + params: + binary: bash + background: true + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh + - command: subprocess.exec + params: + binary: bash + env: + TOPOLOGY: replica_set + args: + - ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + - command: "subprocess.exec" + params: + binary: bash + include_expansions_in_env: + - DRIVERS_TOOLS + - PYMONGO_DIR + - MONGOCRYPT_DIR + args: + - ./libmongocrypt/bindings/python/.evergreen/integ-test.sh + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: + - DRIVERS_TOOLS + args: + - ./libmongocrypt/bindings/python/.evergreen/integ-teardown.sh + - command: attach.xunit_results + params: + file: ${PYMONGO_DIR}/xunit-results/TEST-*.xml + "build and test node": - command: "subprocess.exec" params: @@ -505,6 +555,16 @@ tasks: - func: "test python" vars: { variant_name: windows-test } +- name: test-python-integ + depends_on: + - build-and-test-and-upload + commands: + - func: "fetch source" + - func: "download tarball" + vars: { variant_name: "${build_variant}" } + - func: "test python integ" + vars: { variant_name: "${build_variant}" } + - name: "release-python-macos-1100" tags: ["release_python_tag"] run_on: macos-1100 @@ -1385,6 +1445,7 @@ buildvariants: - build-and-test-node - test-java - test-python + - test-python-integ - name: publish-packages distros: - rhel70-small @@ -1398,6 +1459,7 @@ buildvariants: tasks: - build-and-test-and-upload - test-python + - test-python-integ - name: publish-packages distros: - rhel70-small diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh new file mode 100755 index 000000000..60fa5e577 --- /dev/null +++ b/bindings/python/.evergreen/integ-setup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/bash + +set -eux + +DRIVERS_TOOLS="$(pwd)/drivers-tools" +PROJECT_DIRECTORY="${project_directory}" +PYMONGO_DIR="$(pwd)/mongo-python-driver" + +# Python has cygwin path problems on Windows. +if [ "Windows_NT" = "${OS:-}" ]; then + DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) +fi +export PROJECT_DIRECTORY +export DRIVERS_TOOLS + +export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" +export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" +export MONGOCRYPT_DIR=${PROJECT_DIRECTORY}/all/${variant_name} + +cat < expansion.yml +DRIVERS_TOOLS: "$DRIVERS_TOOLS" +MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" +MONGODB_BINARIES: "$MONGODB_BINARIES" +PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" +PYMONGO_DIR: "$PYMONGO_DIR" +MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" +EOT + +# Set up drivers-tools with a .env file. +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +cat < ${DRIVERS_TOOLS}/.env +DRIVERS_TOOLS="$DRIVERS_TOOLS" +MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" +MONGODB_BINARIES="$MONGODB_BINARIES" +PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +EOT + +# Get the secrets +bash $DRIVERS_TOOLS/.evergreen/csfle/setup-secrets.sh + +# Clone mongo-python-driver +git clone https://github.com/mongodb/mongo-python-driver.git ${PYMONGO_DIR} diff --git a/bindings/python/.evergreen/integ-teardown.sh b/bindings/python/.evergreen/integ-teardown.sh new file mode 100755 index 000000000..143e573e6 --- /dev/null +++ b/bindings/python/.evergreen/integ-teardown.sh @@ -0,0 +1,6 @@ +#! /bin/bash +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail + +bash ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh +bash ${DRIVERS_TOOLS}/.evergreen/teardown.sh diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh new file mode 100755 index 000000000..a6ac332e0 --- /dev/null +++ b/bindings/python/.evergreen/integ-test.sh @@ -0,0 +1,50 @@ +#! /bin/bash +set -eux + +pushd $(pwd)/libmongocrypt/bindings/python + +# For createvirtualenv and find_python3 +. .evergreen/utils.sh + +BASE_PYTHON=$(find_python3) + +# MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml +MONGOCRYPT_DIR="$MONGOCRYPT_DIR" + +MACHINE=$(uname -m) +if [ $MACHINE == "aarch64" ]; then + PYTHON="/opt/mongodbtoolchain/v4/bin/python3" + TARGET=rhel82 +else + TARGET=rhel80 + PYTHON="/opt/python/3.8/bin/python3" +fi + +CRYPT_SHARED_DIR="$(pwd)/crypt_shared" +/opt/mongodbtoolchain/v3/bin/python3 $DRIVERS_TOOLS/.evergreen/mongodl.py --component \ + crypt_shared --version latest --out $CRYPT_SHARED_DIR --target $TARGET + +if [ -e "${MONGOCRYPT_DIR}/lib64/" ]; then + export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib64/libmongocrypt.so + PYMONGOCRYPT_LIB_CRYPTO=${MONGOCRYPT_DIR}/lib64/libmongocrypt.so +else + export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib/libmongocrypt.so + PYMONGOCRYPT_LIB_CRYPTO=${MONGOCRYPT_DIR}/lib/libmongocrypt.so +fi + +createvirtualenv $PYTHON .venv +pip install -e . +pushd $PYMONGO_DIR +pip install -e ".[test,encryption]" +source ${DRIVERS_TOOLS}/.evergreen/csfle/secrets-export.sh +set -x +TEST_CRYPT_SHARED=1 DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR/lib/:${DYLD_FALLBACK_LIBRARY_PATH:-} \ + LD_LIBRARY_PATH=$CRYPT_SHARED_DIR/lib:${LD_LIBRARY_PATH-} \ + PATH=$CRYPT_SHARED_DIR/bin:$PATH \ + AUTH=auth SSL=ssl \ + .evergreen/run-tests.sh "test/test_encryption.py" + +popd +deactivate +rm -rf .venv +popd diff --git a/bindings/python/.evergreen/utils.sh b/bindings/python/.evergreen/utils.sh index 99e5362ca..57055fb1e 100755 --- a/bindings/python/.evergreen/utils.sh +++ b/bindings/python/.evergreen/utils.sh @@ -19,7 +19,7 @@ createvirtualenv () { # Workaround for bug in older versions of virtualenv. $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH fi - if [ "Windows_NT" = "$OS" ]; then + if [ "Windows_NT" = "${OS:-}" ]; then # Workaround https://bugs.python.org/issue32451: # mongovenv/Scripts/activate: line 3: $'\r': command not found dos2unix $VENVPATH/Scripts/activate || true @@ -49,7 +49,7 @@ find_python3() { elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.8" ]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.8/bin/python3" fi - elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin PYTHON="C:/python/Python38/python.exe" else # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.8+. diff --git a/bindings/python/pymongocrypt/binding.py b/bindings/python/pymongocrypt/binding.py index d709184f9..6d4234e8e 100644 --- a/bindings/python/pymongocrypt/binding.py +++ b/bindings/python/pymongocrypt/binding.py @@ -535,7 +535,7 @@ def _parse_version(version): * @brief Obtain a 64-bit constant encoding the version of the loaded * crypt_shared library, if available. * - * @param[in] crypt The mongocrypt_t object after a successul call to + * @param[in] crypt The mongocrypt_t object after a successful call to * mongocrypt_init. * * @return A 64-bit encoded version number, with the version encoded as four @@ -1220,7 +1220,7 @@ def _parse_version(version): * @param[out] status An optional status to pass error messages. See @ref * mongocrypt_status_set. * @returns A boolean indicating success. If returning false, set @p status - * with a message indiciating the error using @ref mongocrypt_status_set. + * with a message indicating the error using @ref mongocrypt_status_set. */ typedef bool (*mongocrypt_crypto_fn)(void *ctx, mongocrypt_binary_t *key, @@ -1245,7 +1245,7 @@ def _parse_version(version): * @param[out] status An optional status to pass error messages. See @ref * mongocrypt_status_set. * @returns A boolean indicating success. If returning false, set @p status - * with a message indiciating the error using @ref mongocrypt_status_set. + * with a message indicating the error using @ref mongocrypt_status_set. */ typedef bool (*mongocrypt_hmac_fn)(void *ctx, mongocrypt_binary_t *key, @@ -1264,7 +1264,7 @@ def _parse_version(version): * @param[out] status An optional status to pass error messages. See @ref * mongocrypt_status_set. * @returns A boolean indicating success. If returning false, set @p status - * with a message indiciating the error using @ref mongocrypt_status_set. + * with a message indicating the error using @ref mongocrypt_status_set. */ typedef bool (*mongocrypt_hash_fn)(void *ctx, mongocrypt_binary_t *in, @@ -1282,7 +1282,7 @@ def _parse_version(version): * @param[out] status An optional status to pass error messages. See @ref * mongocrypt_status_set. * @returns A boolean indicating success. If returning false, set @p status - * with a message indiciating the error using @ref mongocrypt_status_set. + * with a message indicating the error using @ref mongocrypt_status_set. */ typedef bool (*mongocrypt_random_fn)(void *ctx, mongocrypt_binary_t *out, uint32_t count, mongocrypt_status_t *status); diff --git a/bindings/python/test/data/compact/success/encrypted-payload.json b/bindings/python/test/data/compact/success/encrypted-payload.json index e82badfb0..4e837d28f 100644 --- a/bindings/python/test/data/compact/success/encrypted-payload.json +++ b/bindings/python/test/data/compact/success/encrypted-payload.json @@ -70,4 +70,4 @@ } } } -} \ No newline at end of file +}