diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index c9b5e4a5ee10..20b0f0845741 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 76df09cfd93b..a8aa852e6093 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 144c17e7820d..8aa708b4a52b 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 1df2a9fd23b3..5cc044589588 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index 393e42c03e54..f491c1cfee45 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 1b6ac8d26294..dfff068c39f5 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 9c4064825a0a..e1d2ba736c39 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" @@ -27,6 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 48a6887c1ba6..0f1f7a08848c 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" @@ -26,6 +26,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 4a10458aa535..2c77402bbfcb 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -14,7 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" @@ -26,6 +26,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index cd90dc0d0972..7d4f0c4b2979 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -91,6 +91,8 @@ function(add_hpx_test category name) set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) endif() + set(ENV_VAR "") + # cmake-format: off set(cmd "${Python_EXECUTABLE}" @@ -196,7 +198,7 @@ function(add_hpx_test category name) if(_add_test) set(_full_name "${category}.distributed.gasnet.${name}") add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" - "gasnet-smp" ${args} + "srun" ${args} ) set_tests_properties( "${_full_name}" diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 61690c08ddae..227d037fef83 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -119,7 +119,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-udp --disable-mpi --disable-smp --disable-ofi --disable-ucx && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -129,7 +129,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-smp --disable-udp --disable-ofi --disable-mpi --disable-ucx && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -139,7 +139,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --disable-smp --disable-mpi --disable-ucx --disable-udp --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -149,7 +149,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --disable-smp --disable-mpi --disable-ofi --disable-udp --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -157,40 +157,175 @@ macro(hpx_setup_gasnet) ) elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") if(NOT MPI_FOUND) - message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") + pkg_check_modules( + MPI + REQUIRED + GLOBAL + ompi + ompi-c + ompi_c + openmpi + mpi + ) + if(NOT MPI_FOUND) + message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") + endif() + endif() + + if(NOT MPI_CFLAGS) + set(MPI_CFLAGS_STR "-fPIC") + else() + list(LENGTH MPI_CFLAGS MPI_CFLAGS_COUNT) + if(MPI_CFLAGS_COUNT LESS "1") + set(MPI_CFLAGS_STR "-fPIC") + else() + set(MPI_CFLAGS_STR "") + list(JOIN MPI_CFLAGS " " MPI_CFLAGS_STR) + endif() + endif() + + if(NOT MPI_CFLAGS_OTHER) + set(MPI_CFLAGS_OTHER_STR "") + else() + list(LENGTH MPI_CFLAGS_OTHER MPI_CFLAGS_OTHER_COUNT) + if(MPI_CFLAGS_OTHER_COUNT LESS "1") + set(MPI_CFLAGS_OTHER_STR "") + else() + set(MPI_CFLAGS_OTHER_STR "") + list(JOIN MPI_CFLAGS_OTHER " " MPI_CFLAGS_OTHER_STR) + endif() + endif() + + if(NOT MPI_LDFLAGS) + set(MPI_LDFLAGS_STR "") + else() + list(LENGTH MPI_LDFLAGS MPI_LDFLAGS_COUNT) + if(MPI_LDFLAGS_COUNT LESS "1") + set(MPI_LDFLAGS_STR "") + else() + set(MPI_LDFLAGS_STR "") + list(JOIN MPI_LDFLAGS " " MPI_LDFLAGS_STR) + endif() + endif() + + if(NOT MPI_LIBRARIES) + set(MPI_LIBRARIES_STR "-lmpi") + else() + list(LENGTH MPI_LIBRARIES MPI_LIBRARIES_COUNT) + if(MPI_LIBRARIES_COUNT LESS "1") + set(MPI_LIBRARIES_STR "-lmpi") + else() + set(MPI_LIBRARIES_STR "") + list(JOIN MPI_LIBRARIES " -l" MPI_LIBRARIES_STR) + set(MPI_LIBRARIES_STR "-l${MPI_LIBRARIES_STR}") + endif() endif() - if(NOT TARGET Mpi::mpi) - message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") + if(NOT MPI_LIBRARY_DIRS) + set(MPI_LIBRARY_DIRS_STR "") + else() + list(LENGTH MPI_LIBRARY_DIRS MPI_LIBRARY_DIRS_COUNT) + if(MPI_LIBRARIES_DIRS_COUNT LESS "1") + set(MPI_LIBRARY_DIRS_STR "") + else() + set(MPI_LIBRARY_DIRS_STR "") + list(JOIN MPI_LIBRARY_DIRS " -l" MPI_LIBRARY_DIRS_STR) + set(MPI_LIBRARY_DIRS_STR "-l${MPI_LIBRARY_DIRS_STR}") + endif() endif() - if(${MPI_C_COMPILER}) - set(MPI_C_COMPILER ${CMAKE_C_COMPILER}) - set(MPI_CC ${CMAKE_C_COMPILER}) - set(ENV{MPI_CC} ${CMAKE_C_COMPILER}) - elseif(${MPI_CC}) - set(MPI_CC ${CMAKE_C_COMPILER}) - set(ENV{MPI_CC} ${CMAKE_C_COMPILER}) - elseif($ENV{MPI_CC}) - set(MPI_CC ${CMAKE_C_COMPILER}) - set(ENV{MPI_CC} ${CMAKE_C_COMPILER}) + if(NOT MPI_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS_STR "") else() - message(FATAL_ERROR "GASNet MPI Conduit selected; $MPI_CC not found!") + list(LENGTH MPI_INCLUDE_DIRS MPI_INCLUDE_DIRS_COUNT) + if(MPI_INCLUDE_DIRS_COUNT LESS "1") + set(MPI_INCLUDE_DIRS_STR "") + else() + set(MPI_INCLUDE_DIRS_STR "") + list(JOIN MPI_INCLUDE_DIRS " -I" MPI_INCLUDE_DIRS_STR) + set(MPI_INCLUDE_DIRS_STR "-I${MPI_INCLUDE_DIRS_STR}") + endif() endif() - if(NOT "$CMAKE_C_COMPILER" STREQUAL "${MPI_CC}") - message(FATAL_ERROR "GASNet MPI: $MPI_CC != $CMAKE_C_COMPILER!") + hpx_info("MPI version: " ${MPI_CXX_VERSION}) + + if(NOT MPI_FOUND) + message( + FATAL_ERROR + "GASNet MPI: MPI not found!" + ) endif() + message( + STATUS + "GASNet Build Command\nCC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + ) + execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS + RESULT_VARIABLE GASNET_BUILD_STATUS_A OUTPUT_FILE ${GASNET_BUILD_OUTPUT} ERROR_FILE ${GASNET_ERROR_FILE} ) + + if(GASNET_BUILD_STATUS_A) + execute_process( + COMMAND + bash -c + "pkg-config --cflags ompi" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE PKGCONFIG_CFLAGS_STATUS + OUTPUT_VARIABLE PKGCONFIG_CFLAGS_FLAGS + OUTPUT_FILE PKGCONFIG_CFLAGS_OUTPUT + ERROR_FILE PKGCONFIG_CFLAGS_ERROR_FILE + ) + + execute_process( + COMMAND + bash -c + "pkg-config --libs ompi" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE PKGCONFIG_LIBS_STATUS + OUTPUT_VARIABLE PKGCONFIG_LIBS_FLAGS + OUTPUT_FILE PKGCONFIG_LIBS_OUTPUT + ERROR_FILE PKGCONFIG_LIBS_ERROR_FILE + ) + + if(NOT (PKGCONFIG_LIBS_STATUS AND PKGCONFIG_CFLAGS_STATUS)) + message( + STATUS + "GASNet Build Command\nCC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" CCFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" CXXFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" --with-cxxflags=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + ) + + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" CCFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" CXXFLAGS=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" --with-cxxflags=\"-fPIC ${PKGCONFIG_CFLAGS_FLAGS} ${PKGCONFIG_LIBS_FLAGS}\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + else() + message( + STATUS + "GASNet Build Command\nCC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + ) + + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --disable-udp --disable-smp --disable-ucx --disable-ofi --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + endif() + endif() endif() if(GASNET_BUILD_STATUS) @@ -200,6 +335,51 @@ macro(hpx_setup_gasnet) ) else() + find_file(GASNET_TOOLS_PKGCONFIG_FILE_FOUND + gasnet_tools-par.pc + ${GASNET_DIR}/install/lib/pkgconfig + ) + + if(NOT GASNET_TOOLS_PKGCONFIG_FILE_FOUND) + message( + FATAL_ERROR + "PKG-CONFIG ERROR (${GASNET_TOOLS_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED GASNET_TOOLS: ${GASNET_DIR}/install/lib/pkgconfig" + ) + endif() + + install(CODE "set(GASNET_PATH \"${GASNET_DIR}\")") + + install( + CODE [[ + file( + READ + ${GASNET_PATH}/install/lib/pkgconfig/gasnet_tools-par.pc + GASNET_TOOLS_PKGCONFIG_FILE_CONTENT + ) + + if(NOT GASNET_TOOLS_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR INSTALLING GASNET_TOOLS") + endif() + + string(REPLACE "${GASNET_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_TOOLS_PKGCONFIG_FILE_CONTENT + ${GASNET_TOOLS_PKGCONFIG_FILE_CONTENT} + ) + + file( + WRITE + ${GASNET_PATH}/install/lib/pkgconfig/gasnet_tools-par.pc + ${GASNET_TOOLS_PKGCONFIG_FILE_CONTENT} + ) + + file(GLOB_RECURSE GASNET_FILES ${GASNET_PATH}/install/*) + + if(NOT GASNET_FILES) + message(STATUS "ERROR INSTALLING GASNET") + endif() + ]] + ) + find_file(GASNET_PKGCONFIG_FILE_FOUND gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc ${GASNET_DIR}/install/lib/pkgconfig diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 3bc50eacd078..711eadcc390b 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -195,20 +195,6 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -def run_gasnet(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_QUIET'] = 'yes' - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - run_mpi(cmd, localities, verbose) - -def run_gasnet_smp(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_BARRIER'] = 'PSHM' - os.environ['GASNET_QUIET'] = 'yes' - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - # No launcher needed - run_none(cmd, localities, nodes, verbose) - # Run with srun # This is executing srun with the '-n' option set to the number of localities def run_srun(cmd, localities, verbose): @@ -229,15 +215,14 @@ def run_jsrun(cmd, localities, verbose): def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'none': run_none(cmd, localities, nodes, verbose) + if runwrapper == 'gasnet': + assert nodes is None, "nodes option only valid with tcp parcelport." + os.environ['GASNET_SPAWNFN'] = 'C' + os.environ['GASNET_CSPAWN_CMD'] = '' + run_srun(cmd, localities, verbose) if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) - if runwrapper == 'gasnet': - assert nodes is None, "nodes option only valid with tcp parcelport." - run_gasnet(cmd, localities, verbose) - if runwrapper == 'gasnet-smp': - assert nodes is None, "nodes option only valid with smp parcelport." - run_gasnet_smp(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -303,7 +288,7 @@ def check_options(parser, options, args): sys.exit(1) check_valid_runwrapper = (lambda x: - x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'gasnet' or x == 'gasnet-smp'); + x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun'); if not check_valid_runwrapper(options.runwrapper): print('Error: Runwrapper option not valid\n', sys.stderr) parser.print_help() @@ -388,7 +373,7 @@ Used by the tcp parcelport only. parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, gasnet, gasnet-smp, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index a61a1ad999c9..8cb8ede2968b 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -251,7 +251,7 @@ namespace hpx::util { bool gasnet_environment::check_gasnet_environment( util::runtime_configuration const& cfg) { -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) // We disable the GASNET parcelport if any of these hold: // // - The parcelport is explicitly disabled @@ -278,7 +278,7 @@ namespace hpx::util { } } // namespace hpx::util -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE)) +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) namespace hpx::util {