bump to 0.4.1

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.4.0"
+current_version = "0.4.1"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 search = "{current_version}"
@@ -27,4 +27,4 @@ search = "make_version(GHEX {current_version})"
 replace = "make_version(GHEX {new_version})"
 
 [[tool.bumpversion.files]]
-filename = "bindings/python/pyproject.toml"
+filename = "pyproject.toml"

.github/workflows/pypi_deploy.yml

@@ -32,8 +32,9 @@ jobs:
       run: |
         python -m venv .venv
         . .venv/bin/activate
-        python -m pip install -r ./bindings/python/requirements-test.txt # before ghex installation to avoid mpi4py version conflict  
+        python -m pip install --upgrade pip
         python -m pip install dist/*.tar.gz
+        python -m pip install -r ./bindings/python/min-requirements-test.txt
         python -c "import ghex"
         python -m pytest -s ./test/bindings/python/
     - name: Upload artifact

.github/workflows/test_pip.yml

@@ -72,12 +72,12 @@ jobs:
         run: |
           . ~/env/bin/activate
           python3 -m pip install --upgrade pip
-          python3 -m pip install -r ./bindings/python/requirements-test.txt # before ghex installation to avoid mpi4py version conflict
           python3 -m pip install -v .
 
       - if: ${{ matrix.config.run == 'ON' }}
         name: Run Python tests
         run: |
           . ~/env/bin/activate
+          python3 -m pip install -r ./bindings/python/min-requirements-test.txt
           python3 -m pytest -s ./test/bindings/python/
           mpirun --oversubscribe -np 4 python3 -m pytest -s --with-mpi ./test/bindings/python/

CMakeLists.txt

@@ -14,7 +14,7 @@ include(ghex_version)
 # note: version should not be changed manually, use bump-my-version instead:
 # install: pip install bump-my-version
 # example: bump-my-version bump patch
-make_version(GHEX 0 4 0)
+make_version(GHEX 0 4 1)
 
 project(GHEX VERSION ${GHEX_VERSION} LANGUAGES C CXX)
 

bindings/python/min-requirements-test.txt

@@ -0,0 +1,7 @@
+iniconfig>=2.0.0
+mpi4py>=1.21
+numpy>=1.21
+packaging>=23.1
+pluggy>=1.3.0
+pytest>=7.4.3
+pytest-mpi>=0.6

bindings/python/src/_pyghex/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 set(pyghex_source
     config.cpp
     context_shim.cpp
@@ -33,13 +32,6 @@ target_link_libraries(pyghex_obj PUBLIC ghex_common)
 # Add GPU support
 compile_as_cuda(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} SOURCES ${pyghex_source})
 
-# Add support for mpi4py if available.
-find_python_module(mpi4py)
-if (HAVE_MPI4PY)
-    target_include_directories(pyghex_obj PRIVATE "${PY_MPI4PY}/include")
-    target_compile_definitions(pyghex_obj PRIVATE -DGHEX_ENABLE_MPI4PY)
-endif()
-
 # The Python module
 # ==================
 # Create the Python module in the build directory.

bindings/python/src/_pyghex/config.cpp

@@ -30,12 +30,6 @@ config()
 
     pybind11::dict dict;
 
-#ifdef GHEX_ENABLE_MPI4PY
-    dict[pybind11::str("mpi4py")] = pybind11::bool_(true);
-#else
-    dict[pybind11::str("mpi4py")] = pybind11::bool_(false);
-#endif
-
     dict[pybind11::str("transport")] = pybind11::str(mk_str(GHEX_TRANSPORT_BACKEND));
 
 #ifdef GHEX_USE_GPU

bindings/python/src/_pyghex/mpi_comm_shim.cpp

@@ -11,10 +11,6 @@
 #include <string>
 #include <sstream>
 
-#ifdef GHEX_ENABLE_MPI4PY
-#include <mpi4py/mpi4py.h>
-#endif
-
 #include <mpi_comm_shim.hpp>
 #include <util/to_string.hpp>
 
@@ -32,25 +28,13 @@ struct mpi_error : std::runtime_error
 namespace
 {
 
-// Test if a Python object can be converted to an mpi_comm_shim.
-inline bool
-can_convert_to_mpi_comm(pybind11::object o)
-{
-#ifdef GHEX_ENABLE_MPI4PY
-    import_mpi4py();
-    if (PyObject_TypeCheck(o.ptr(), &PyMPIComm_Type)) return true;
-#endif
-    return false;
-}
-
 // Convert a Python object to an MPI Communicator.
 inline MPI_Comm
 convert_to_mpi_comm(pybind11::object o)
 {
-#ifdef GHEX_ENABLE_MPI4PY
-    if (can_convert_to_mpi_comm(o)) return *PyMPIComm_Get(o.ptr());
-#endif
-    throw pybind11::type_error("Argument must be `mpi4py.MPI.Comm`");
+    if (!pybind11::hasattr(o, "py2f"))
+        throw pybind11::type_error("Argument must be `mpi4py.MPI.Comm`");
+    return MPI_Comm_f2c(o.attr("py2f")().cast<MPI_Fint>());
 }
 
 } // anonymous namespace
@@ -125,6 +109,7 @@ register_mpi(pybind11::module& m)
     m.def("mpi_finalize", &mpi_finalize, "Finalize MPI (calls MPI_Finalize)");
     m.def("mpi_is_initialized", &mpi_is_initialized, "Check if MPI is initialized.");
     m.def("mpi_is_finalized", &mpi_is_finalized, "Check if MPI is finalized.");
+
 }
 
 } // namespace pyghex

cmake/ghex_find_python_module.cmake

@@ -15,13 +15,19 @@ function(find_python_module module)
             RESULT_VARIABLE _${module}_status
             OUTPUT_VARIABLE _${module}_location
             ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+        execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+            "import re, ${module}; print(re.compile('/__init__.py.*').sub('',${module}.__version__))"
+            OUTPUT_VARIABLE _${module}_version
+            ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
         message(STATUS "python-exe = ${PYTHON_EXECUTABLE}")
         message(STATUS "status = ${_${module}_status}")
         message(STATUS "location = ${_${module}_location}")
+        message(STATUS "version = ${_${module}_version}")
 
         if(NOT _${module}_status)
             set(HAVE_${module_upper} ON CACHE INTERNAL "Python module available")
             set(PY_${module_upper} ${_${module}_location} CACHE STRING "Location of Python module ${module}")
+            set(PY_${module_upper}_VERSION ${_${module}_version} CACHE STRING "Version of Python module ${module}")
         else()
             set(HAVE_${module_upper} OFF CACHE INTERNAL "Python module available")
         endif()

cmake/ghex_python.cmake

@@ -4,35 +4,7 @@ set(PYBIND11_CPP_STANDARD -std=c++17)
 
 if (GHEX_BUILD_PYTHON_BINDINGS)
 
-    if (GHEX_WITH_TESTING)
-        # create a virtual environment
-        # according to https://discourse.cmake.org/t/possible-to-create-a-python-virtual-env-from-cmake-and-then-find-it-with-findpython3/1132
-        find_package (Python3 REQUIRED COMPONENTS Interpreter)
-        set(venv "${CMAKE_CURRENT_BINARY_DIR}/pyghex_venv")
-        set(venv_bin_dir "${venv}/bin")
-        set(reqs "${PROJECT_SOURCE_DIR}/bindings/python/requirements-test.txt")
-        message("Creating VENV from ${Python3_EXECUTABLE} to ${VENV}")
-        execute_process(COMMAND_ECHO STDOUT COMMAND ${Python3_EXECUTABLE} -m venv ${venv} )
-        execute_process(COMMAND_ECHO STDOUT COMMAND ${venv_bin_dir}/pip install -U pip setuptools wheel pybind11-stubgen)
-        execute_process(COMMAND_ECHO STDOUT COMMAND ${venv_bin_dir}/pip install -r ${reqs} )
-
-        #execute_process (COMMAND "${Python3_EXECUTABLE}" -m venv ${venv})
-        # Here is the trick
-        ## update the environment with VIRTUAL_ENV variable (mimic the activate script)
-        set (ENV{VIRTUAL_ENV} "${venv}")
-        ## change the context of the search
-        set (Python3_FIND_VIRTUALENV FIRST)
-        ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section)
-        unset (Python3_EXECUTABLE)
-        ## Launch a new search
-        find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module)
-    else()
-        #if(DEFINED PYTHON_EXECUTABLE)
-        #    set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
-        #endif()
-        #set(Python_FIND_STRATEGY LOCATION)
-        find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module)
-    endif()
+    find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module)
 
     if(${Python3_FOUND})
         set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
@@ -56,14 +28,4 @@ if (GHEX_BUILD_PYTHON_BINDINGS)
     set(GHEX_PYTHON_LIB_PATH ${GHEX_PYTHON_LIB_PATH_DEFAULT} CACHE PATH "path for installing Python bindings.")
     message(STATUS "Python bindings installation path: ${GHEX_PYTHON_LIB_PATH}")
 
-    if (GHEX_WITH_TESTING)
-        # GHEX_PYTHON_LIB_PATH_DEFAULT has the form /some/abs/install/path/<lib>/<python>/site-packages
-        # where <lib> is either lib or lib64, and <python> is python3.x
-        cmake_path(GET GHEX_PYTHON_LIB_PATH_DEFAULT PARENT_PATH GHEX_PYTHON_LIB_PATH_DEFAULT-1)
-        cmake_path(GET GHEX_PYTHON_LIB_PATH_DEFAULT-1 PARENT_PATH GHEX_PYTHON_LIB_PATH_DEFAULT-2)
-        cmake_path(GET GHEX_PYTHON_LIB_PATH_DEFAULT-2 PARENT_PATH GHEX_PYTHON_LIB_PATH_DEFAULT-3)
-        cmake_path(RELATIVE_PATH GHEX_PYTHON_LIB_PATH_DEFAULT BASE_DIRECTORY ${GHEX_PYTHON_LIB_PATH_DEFAULT-3} OUTPUT_VARIABLE result)
-        cmake_path(ABSOLUTE_PATH result BASE_DIRECTORY ${venv} NORMALIZE OUTPUT_VARIABLE venv_site_packages_dir)
-    endif()
-
 endif()

pyproject.toml

@@ -2,7 +2,6 @@
 build-backend = 'scikit_build_core.build'
 requires = [
     'cmake',
-    'mpi4py>=3.1',
     'pybind11>=2.6',
     'scikit-build-core',
     'wheel',
@@ -15,7 +14,7 @@ name = 'ghex'
 # note: version should not be changed manually, use bump-my-version instead:
 # install: pip install bump-my-version
 # example: bump-my-version bump patch
-version = '0.4.0'
+version = '0.4.1'
 license = {text = "BSD-3 License"}
 readme = {file = 'bindings/python/README.md', content-type = 'text/markdown'}
 

test/bindings/python/CMakeLists.txt

@@ -1,20 +1,72 @@
+# paths
+# =====
 
+# virtual environment
+set(venv_dir "${CMAKE_BINARY_DIR}/test_venv")
+# test requirements
+set(reqs "${CMAKE_SOURCE_DIR}/bindings/python/min-requirements-test.txt")
+# get the path to the pyghex module's parent directory
 get_target_property(python_mod_path pyghex LIBRARY_OUTPUT_DIRECTORY)
 get_filename_component(pyghex_test_workdir ${python_mod_path}/.. ABSOLUTE)
 
+
+# setup test virtual environment
+# ==============================
+
+# command to create a virtual environment
+add_custom_command(
+    OUTPUT ${venv_dir}
+    COMMAND ${Python3_EXECUTABLE} -m venv ${venv_dir}
+    COMMENT "Creating virtual environment for test dependencies"
+)
+
+# command to upgrade pip in the virtual environment
+add_custom_command(
+    OUTPUT ${venv_dir}/bin/pip-upgraded
+    COMMAND ${venv_dir}/bin/python -m pip install --upgrade pip
+    COMMAND ${CMAKE_COMMAND} -E touch ${venv_dir}/bin/pip-upgraded
+    DEPENDS ${venv_dir}
+    COMMENT "Upgrading pip in virtual environment"
+)
+
+# command to install test dependencies into the virtual environment
+add_custom_command(
+    OUTPUT ${venv_dir}/bin/pytest
+    COMMAND ${venv_dir}/bin/pip install -r ${reqs}
+    DEPENDS ${venv_dir}/bin/pip-upgraded
+    COMMENT "Installing test dependencies"
+)
+
+# add a custom target to trigger all the above commands
+add_custom_target(
+    setup_test_env ALL
+    DEPENDS ${venv_dir}/bin/pytest
+    COMMENT "Test environment setup complete"
+)
+
+
+# setup test target
+# =================
+
 add_custom_target(pyghex_tests)
 add_dependencies(pyghex pyghex_tests)
+add_dependencies(setup_test_env pyghex_tests)
+
 copy_files(TARGET pyghex_tests DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/fixtures FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/context.py)
 copy_files(TARGET pyghex_tests DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/conftest.py)
 
+
+# register unit tests
+# ===================
+
 function(ghex_reg_pytest t)
     copy_files(TARGET pyghex_tests DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILES
         ${CMAKE_CURRENT_SOURCE_DIR}/test_${t}.py)
     add_test(
         NAME py_${t}
-        COMMAND ${PYTHON_EXECUTABLE} -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py
+        COMMAND ${venv_dir}/bin/python -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py
         WORKING_DIRECTORY ${pyghex_test_workdir})
     set_tests_properties(py_${t} PROPERTIES RUN_SERIAL ON)
 endfunction()
@@ -25,7 +77,7 @@ function(ghex_reg_parallel_pytest t n)
     add_test(
         NAME py_${t}_parallel
         COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS}
-        ${PYTHON_EXECUTABLE} -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py
+        ${venv_dir}/bin/python -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py
         WORKING_DIRECTORY ${pyghex_test_workdir})
     set_tests_properties(py_${t}_parallel PROPERTIES RUN_SERIAL ON)
 endfunction()